Adding test to test closer boundary for overflow

Bug: 137801859
Test: build, boot, GraphicBuffer_test
Change-Id: Ic5c74a52dcf325c9151f63bd9bbb11ea17222b0b

1 files changed, 17 insertions, 0 deletions

diff --git a/libs/ui/tests/GraphicBuffer_test.cpp b/libs/ui/tests/GraphicBuffer_test.cpp
index 127f7eedd6..5e0b094b7b 100644
--- a/libs/ui/tests/GraphicBuffer_test.cpp
+++ b/libs/ui/tests/GraphicBuffer_test.cpp
@@ -44,11 +44,28 @@ TEST_F(GraphicBufferTest, AllocateNoError) {
 
 TEST_F(GraphicBufferTest, AllocateBadDimensions) {
     PixelFormat format = PIXEL_FORMAT_RGBA_8888;
+    if (std::numeric_limits<size_t>::max() / std::numeric_limits<uint32_t>::max() /
+                bytesPerPixel(format) >=
+        std::numeric_limits<uint32_t>::max()) {
+        GTEST_SUCCEED() << "Cannot overflow with this format";
+    }
     uint32_t width, height;
     width = height = std::numeric_limits<uint32_t>::max();
     sp<GraphicBuffer> gb(new GraphicBuffer(width, height, format, kTestLayerCount, kTestUsage,
                                            std::string("test")));
     ASSERT_EQ(BAD_VALUE, gb->initCheck());
+
+    const size_t targetArea = std::numeric_limits<size_t>::max() / bytesPerPixel(format);
+    const size_t widthCandidate = targetArea / std::numeric_limits<uint32_t>::max();
+    if (widthCandidate == 0) {
+        width = 1;
+    } else {
+        width = std::numeric_limits<uint32_t>::max();
+    }
+    height = (targetArea / width) + 1;
+    sp<GraphicBuffer> gb2(new GraphicBuffer(width, height, format, kTestLayerCount, kTestUsage,
+                                            std::string("test")));
+    ASSERT_EQ(BAD_VALUE, gb2->initCheck());
 }
 
 TEST_F(GraphicBufferTest, CreateFromBufferHubBuffer) {