diff options
Diffstat (limited to 'Fix-integer-overflows-in-calc-of-stride_in_bytes.patch')
| -rw-r--r-- | Fix-integer-overflows-in-calc-of-stride_in_bytes.patch | 127 |
1 files changed, 127 insertions, 0 deletions
diff --git a/Fix-integer-overflows-in-calc-of-stride_in_bytes.patch b/Fix-integer-overflows-in-calc-of-stride_in_bytes.patch new file mode 100644 index 0000000..7382e8b --- /dev/null +++ b/Fix-integer-overflows-in-calc-of-stride_in_bytes.patch @@ -0,0 +1,127 @@ +From 19d9966572a410804349e1a8ee2017fed49a6dab Mon Sep 17 00:00:00 2001 +From: Wan-Teh Chang <wtc@google.com> +Date: Wed, 3 Apr 2024 20:08:16 +0000 +Subject: [PATCH 1/2] Fix integer overflows in calc of stride_in_bytes + +Fix unsigned integer overflows in the calculation of stride_in_bytes in +img_alloc_helper() when d_w is huge. + +Change the type of stride_in_bytes from unsigned int to int because it +will be assigned to img->stride[AOM_PLANE_Y], which is of the int type. + +Test: +cmake ../aom -G Ninja -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Debug \ + -DSANITIZE=unsigned-integer-overflow +ninja +./test_libaom --gtest_filter=AomImageTest.AomImgAllocHugeWidth + +Bug: chromium:332382766 +Change-Id: Iaccb83bcd13ddc3ea5e6f01da91bb01215ddb461 +(cherry picked from commit 7aa2edc2b09f98c32820923d813fd73eb23b5861) +--- + aom/src/aom_image.c | 15 ++++++++------- + test/aom_image_test.cc | 36 ++++++++++++++++++++++++++++++++++++ + 2 files changed, 44 insertions(+), 7 deletions(-) + +diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c +index 8e94d5d..acd3694 100644 +--- a/aom/src/aom_image.c ++++ b/aom/src/aom_image.c +@@ -36,8 +36,7 @@ static aom_image_t *img_alloc_helper( + /* NOTE: In this function, bit_depth is either 8 or 16 (if + * AOM_IMG_FMT_HIGHBITDEPTH is set), never 10 or 12. + */ +- unsigned int h, w, s, xcs, ycs, bps, bit_depth; +- unsigned int stride_in_bytes; ++ unsigned int h, w, xcs, ycs, bps, bit_depth; + + if (img != NULL) memset(img, 0, sizeof(aom_image_t)); + +@@ -106,9 +105,11 @@ static aom_image_t *img_alloc_helper( + w = align_image_dimension(d_w, xcs, size_align); + h = align_image_dimension(d_h, ycs, size_align); + +- s = (fmt & AOM_IMG_FMT_PLANAR) ? w : bps * w / bit_depth; ++ uint64_t s = (fmt & AOM_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / bit_depth; + s = (s + 2 * border + stride_align - 1) & ~(stride_align - 1); +- stride_in_bytes = s * bit_depth / 8; ++ s = s * bit_depth / 8; ++ if (s > INT_MAX) goto fail; ++ const int stride_in_bytes = (int)s; + + /* Allocate the new image */ + if (!img) { +@@ -230,7 +231,7 @@ int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y, + + img->planes[AOM_PLANE_Y] = + data + x * bytes_per_sample + y * img->stride[AOM_PLANE_Y]; +- data += (img->h + 2 * border) * img->stride[AOM_PLANE_Y]; ++ data += ((size_t)img->h + 2 * border) * img->stride[AOM_PLANE_Y]; + + unsigned int uv_border_h = border >> img->y_chroma_shift; + unsigned int uv_x = x >> img->x_chroma_shift; +@@ -242,14 +243,14 @@ int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y, + } else if (!(img->fmt & AOM_IMG_FMT_UV_FLIP)) { + img->planes[AOM_PLANE_U] = + data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_U]; +- data += ((img->h >> img->y_chroma_shift) + 2 * uv_border_h) * ++ data += ((size_t)(img->h >> img->y_chroma_shift) + 2 * uv_border_h) * + img->stride[AOM_PLANE_U]; + img->planes[AOM_PLANE_V] = + data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_V]; + } else { + img->planes[AOM_PLANE_V] = + data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_V]; +- data += ((img->h >> img->y_chroma_shift) + 2 * uv_border_h) * ++ data += ((size_t)(img->h >> img->y_chroma_shift) + 2 * uv_border_h) * + img->stride[AOM_PLANE_V]; + img->planes[AOM_PLANE_U] = + data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_U]; +diff --git a/test/aom_image_test.cc b/test/aom_image_test.cc +index ad48e73..69b777b 100644 +--- a/test/aom_image_test.cc ++++ b/test/aom_image_test.cc +@@ -60,3 +60,39 @@ TEST(AomImageTest, AomImgAllocNv12) { + EXPECT_EQ(img.planes[AOM_PLANE_V], nullptr); + aom_img_free(&img); + } ++ ++TEST(AomImageTest, AomImgAllocHugeWidth) { ++ // The stride (0x80000000 * 2) would overflow unsigned int. ++ aom_image_t *image = ++ aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, 0x80000000, 1, 1); ++ ASSERT_EQ(image, nullptr); ++ ++ // The stride (0x80000000) would overflow int. ++ image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 0x80000000, 1, 1); ++ ASSERT_EQ(image, nullptr); ++ ++ image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 0x7ffffffe, 1, 1); ++ if (image) { ++ aom_img_free(image); ++ } ++ ++ image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 285245883, 64, 1); ++ if (image) { ++ aom_img_free(image); ++ } ++ ++ image = aom_img_alloc(nullptr, AOM_IMG_FMT_NV12, 285245883, 64, 1); ++ if (image) { ++ aom_img_free(image); ++ } ++ ++ image = aom_img_alloc(nullptr, AOM_IMG_FMT_YV12, 285245883, 64, 1); ++ if (image) { ++ aom_img_free(image); ++ } ++ ++ image = aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, 285245883, 2, 1); ++ if (image) { ++ aom_img_free(image); ++ } ++} +-- +2.41.0 + |