Fixing overflow bug

Client should not be requesting such large buffers.
Limit byte size to max(size_t)

Bug: 137801859
Test: build, boot
Change-Id: Idef0c1e926c180bfaf640b627046adba5d3043c3

2 files changed, 25 insertions, 1 deletions

diff --git a/libs/ui/GraphicBufferAllocator.cpp b/libs/ui/GraphicBufferAllocator.cpp
index 0861a1f9a3..9c7d1fda90 100644
--- a/libs/ui/GraphicBufferAllocator.cpp
+++ b/libs/ui/GraphicBufferAllocator.cpp
@@ -20,6 +20,7 @@
 
 #include <ui/GraphicBufferAllocator.h>
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <grallocusage/GrallocUsageConversion.h>
@@ -114,6 +115,14 @@ status_t GraphicBufferAllocator::allocate(uint32_t width, uint32_t height,
     if (!width || !height)
         width = height = 1;
 
+    const uint32_t bpp = bytesPerPixel(format);
+    if (std::numeric_limits<size_t>::max() / width / height < static_cast<size_t>(bpp)) {
+        ALOGE("Failed to allocate (%u x %u) layerCount %u format %d "
+              "usage %" PRIx64 ": Requesting too large a buffer size",
+              width, height, layerCount, format, usage);
+        return BAD_VALUE;
+    }
+
     // Ensure that layerCount is valid.
     if (layerCount < 1)
         layerCount = 1;
@@ -126,7 +135,6 @@ status_t GraphicBufferAllocator::allocate(uint32_t width, uint32_t height,
     if (error == NO_ERROR) {
         Mutex::Autolock _l(sLock);
         KeyedVector<buffer_handle_t, alloc_rec_t>& list(sAllocList);
-        uint32_t bpp = bytesPerPixel(format);
         alloc_rec_t rec;
         rec.width = width;
         rec.height = height;
diff --git a/libs/ui/tests/GraphicBuffer_test.cpp b/libs/ui/tests/GraphicBuffer_test.cpp
index a7c248c105..127f7eedd6 100644
--- a/libs/ui/tests/GraphicBuffer_test.cpp
+++ b/libs/ui/tests/GraphicBuffer_test.cpp
@@ -35,6 +35,22 @@ constexpr uint64_t kTestUsage = GraphicBuffer::USAGE_SW_WRITE_OFTEN;
 
 class GraphicBufferTest : public testing::Test {};
 
+TEST_F(GraphicBufferTest, AllocateNoError) {
+    PixelFormat format = PIXEL_FORMAT_RGBA_8888;
+    sp<GraphicBuffer> gb(new GraphicBuffer(kTestWidth, kTestHeight, format, kTestLayerCount,
+                                           kTestUsage, std::string("test")));
+    ASSERT_EQ(NO_ERROR, gb->initCheck());
+}
+
+TEST_F(GraphicBufferTest, AllocateBadDimensions) {
+    PixelFormat format = PIXEL_FORMAT_RGBA_8888;
+    uint32_t width, height;
+    width = height = std::numeric_limits<uint32_t>::max();
+    sp<GraphicBuffer> gb(new GraphicBuffer(width, height, format, kTestLayerCount, kTestUsage,
+                                           std::string("test")));
+    ASSERT_EQ(BAD_VALUE, gb->initCheck());
+}
+
 TEST_F(GraphicBufferTest, CreateFromBufferHubBuffer) {
     std::unique_ptr<BufferHubBuffer> b1 =
             BufferHubBuffer::create(kTestWidth, kTestHeight, kTestLayerCount, kTestFormat,