BufferQueue: Fix deadlock in setMaxAcquiredBufferCount

Uncovered this while testing. The deadlock happens when:

- ConsumerBase::setMaxAcquiredBufferCount locks itself
- Calls IGBC::setMaxAcquiredBufferCount, which can call
  ConsumerListener::onBuffersReleased
- Which, in ConsumerBase, will take the lock again

Instead of this, we add a callback to be called instead of the
IConsumerListener. This callback is called on the same stack, with the
lock held, so that we can resolve everything atomically.

Bug: b/393639203
Flag: EXEMPT small cleanup
Test: new test
Change-Id: Iddd8f902d1fd0aeed6aac095eaa6c0b870ffff70

7 files changed, 52 insertions, 12 deletions

diff --git a/libs/gui/BufferQueueConsumer.cpp b/libs/gui/BufferQueueConsumer.cpp
index 270bfbdc64..4681c9ecbe 100644
--- a/libs/gui/BufferQueueConsumer.cpp
+++ b/libs/gui/BufferQueueConsumer.cpp
@@ -14,10 +14,6 @@
  * limitations under the License.
  */
 
-#include <inttypes.h>
-#include <pwd.h>
-#include <sys/types.h>
-
 #define LOG_TAG "BufferQueueConsumer"
 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
 //#define LOG_NDEBUG 0
@@ -48,6 +44,11 @@
 
 #include <com_android_graphics_libgui_flags.h>
 
+#include <inttypes.h>
+#include <pwd.h>
+#include <sys/types.h>
+#include <optional>
+
 namespace android {
 
 // Macros for include BufferQueueCore information in log messages
@@ -767,11 +768,15 @@ status_t BufferQueueConsumer::setMaxBufferCount(int bufferCount) {
     return NO_ERROR;
 }
 
+status_t BufferQueueConsumer::setMaxAcquiredBufferCount(int maxAcquiredBuffers) {
+    return setMaxAcquiredBufferCount(maxAcquiredBuffers, std::nullopt);
+}
+
 status_t BufferQueueConsumer::setMaxAcquiredBufferCount(
-        int maxAcquiredBuffers) {
+        int maxAcquiredBuffers, std::optional<OnBufferReleasedCallback> onBuffersReleasedCallback) {
     ATRACE_FORMAT("%s(%d)", __func__, maxAcquiredBuffers);
 
-    sp<IConsumerListener> listener;
+    std::optional<OnBufferReleasedCallback> callback;
     { // Autolock scope
         std::unique_lock<std::mutex> lock(mCore->mMutex);
 
@@ -833,13 +838,20 @@ status_t BufferQueueConsumer::setMaxAcquiredBufferCount(
         BQ_LOGV("setMaxAcquiredBufferCount: %d", maxAcquiredBuffers);
         mCore->mMaxAcquiredBufferCount = maxAcquiredBuffers;
         VALIDATE_CONSISTENCY();
-        if (delta < 0 && mCore->mBufferReleasedCbEnabled) {
-            listener = mCore->mConsumerListener;
+        if (delta < 0) {
+            if (onBuffersReleasedCallback) {
+                callback = std::move(onBuffersReleasedCallback);
+            } else if (mCore->mBufferReleasedCbEnabled) {
+                callback = [listener = mCore->mConsumerListener]() {
+                    listener->onBuffersReleased();
+                };
+            }
         }
     }
+
     // Call back without lock held
-    if (listener != nullptr) {
-        listener->onBuffersReleased();
+    if (callback) {
+        (*callback)();
     }
 
     return NO_ERROR;
diff --git a/libs/gui/ConsumerBase.cpp b/libs/gui/ConsumerBase.cpp
index 117a362661..5b89c6e17e 100644
--- a/libs/gui/ConsumerBase.cpp
+++ b/libs/gui/ConsumerBase.cpp
@@ -264,7 +264,10 @@ void ConsumerBase::onFrameReplaced(const BufferItem &item) {
 
 void ConsumerBase::onBuffersReleased() {
     Mutex::Autolock lock(mMutex);
+    onBuffersReleasedLocked();
+}
 
+void ConsumerBase::onBuffersReleasedLocked() {
     CB_LOGV("onBuffersReleased");
 
     if (mAbandoned) {
@@ -481,7 +484,8 @@ status_t ConsumerBase::setMaxAcquiredBufferCount(int maxAcquiredBuffers) {
         CB_LOGE("setMaxAcquiredBufferCount: ConsumerBase is abandoned!");
         return NO_INIT;
     }
-    return mConsumer->setMaxAcquiredBufferCount(maxAcquiredBuffers);
+    return mConsumer->setMaxAcquiredBufferCount(maxAcquiredBuffers,
+                                                {[this]() { onBuffersReleasedLocked(); }});
 }
 
 status_t ConsumerBase::setConsumerIsProtected(bool isProtected) {
diff --git a/libs/gui/include/gui/BufferQueueConsumer.h b/libs/gui/include/gui/BufferQueueConsumer.h
index ab1231ad6b..ba6a6a73f4 100644
--- a/libs/gui/include/gui/BufferQueueConsumer.h
+++ b/libs/gui/include/gui/BufferQueueConsumer.h
@@ -122,7 +122,10 @@ public:
     // setMaxAcquiredBufferCount sets the maximum number of buffers that can
     // be acquired by the consumer at one time (default 1).  This call will
     // fail if a producer is connected to the BufferQueue.
-    virtual status_t setMaxAcquiredBufferCount(int maxAcquiredBuffers);
+    virtual status_t setMaxAcquiredBufferCount(int maxAcquiredBuffers) override;
+    virtual status_t setMaxAcquiredBufferCount(
+            int maxAcquiredBuffers,
+            std::optional<OnBufferReleasedCallback> onBuffersReleasedCallback) override;
 
     // setConsumerName sets the name used in logging
     status_t setConsumerName(const String8& name) override;
diff --git a/libs/gui/include/gui/ConsumerBase.h b/libs/gui/include/gui/ConsumerBase.h
index fd67f09a4a..d2215ef7e6 100644
--- a/libs/gui/include/gui/ConsumerBase.h
+++ b/libs/gui/include/gui/ConsumerBase.h
@@ -191,6 +191,8 @@ protected:
 #endif
     virtual int getSlotForBufferLocked(const sp<GraphicBuffer>& buffer);
 
+    virtual void onBuffersReleasedLocked();
+
     virtual status_t detachBufferLocked(int slotIndex);
 
     // freeBufferLocked frees up the given buffer slot.  If the slot has been
diff --git a/libs/gui/include/gui/IGraphicBufferConsumer.h b/libs/gui/include/gui/IGraphicBufferConsumer.h
index 8272a591da..8066b070fb 100644
--- a/libs/gui/include/gui/IGraphicBufferConsumer.h
+++ b/libs/gui/include/gui/IGraphicBufferConsumer.h
@@ -243,6 +243,9 @@ public:
     // maxAcquiredBuffers must be (inclusive) between 1 and MAX_MAX_ACQUIRED_BUFFERS. It also cannot
     // cause the maxBufferCount value to be exceeded.
     //
+    // If called with onBuffersReleasedCallback, that call back will be called in lieu of
+    // IConsumerListener::onBuffersReleased.
+    //
     // Return of a value other than NO_ERROR means an error has occurred:
     // * NO_INIT - the BufferQueue has been abandoned
     // * BAD_VALUE - one of the below conditions occurred:
@@ -253,6 +256,11 @@ public:
     // * INVALID_OPERATION - attempting to call this after a producer connected.
     virtual status_t setMaxAcquiredBufferCount(int maxAcquiredBuffers) = 0;
 
+    using OnBufferReleasedCallback = std::function<void(void)>;
+    virtual status_t setMaxAcquiredBufferCount(
+            int maxAcquiredBuffers,
+            std::optional<OnBufferReleasedCallback> onBuffersReleasedCallback) = 0;
+
     // setConsumerName sets the name used in logging
     virtual status_t setConsumerName(const String8& name) = 0;
 
diff --git a/libs/gui/include/gui/mock/GraphicBufferConsumer.h b/libs/gui/include/gui/mock/GraphicBufferConsumer.h
index 24d26b12a1..18a7e12fa7 100644
--- a/libs/gui/include/gui/mock/GraphicBufferConsumer.h
+++ b/libs/gui/include/gui/mock/GraphicBufferConsumer.h
@@ -47,6 +47,7 @@ public:
     MOCK_METHOD2(setDefaultBufferSize, status_t(uint32_t, uint32_t));
     MOCK_METHOD1(setMaxBufferCount, status_t(int));
     MOCK_METHOD1(setMaxAcquiredBufferCount, status_t(int));
+    MOCK_METHOD2(setMaxAcquiredBufferCount, status_t(int, std::optional<OnBufferReleasedCallback>));
     MOCK_METHOD1(setConsumerName, status_t(const String8&));
     MOCK_METHOD1(setDefaultBufferFormat, status_t(PixelFormat));
     MOCK_METHOD1(setDefaultBufferDataSpace, status_t(android_dataspace));
diff --git a/libs/gui/tests/BufferItemConsumer_test.cpp b/libs/gui/tests/BufferItemConsumer_test.cpp
index b980f882ff..6b887bfd68 100644
--- a/libs/gui/tests/BufferItemConsumer_test.cpp
+++ b/libs/gui/tests/BufferItemConsumer_test.cpp
@@ -24,6 +24,7 @@
 #include <gui/Surface.h>
 #include <ui/BufferQueueDefs.h>
 #include <ui/GraphicBuffer.h>
+#include <utils/Errors.h>
 
 #include <unordered_set>
 
@@ -235,6 +236,15 @@ TEST_F(BufferItemConsumerTest, TriggerBufferFreed_DeleteBufferItemConsumer) {
     ASSERT_EQ(1, GetFreedBufferCount());
 }
 
+TEST_F(BufferItemConsumerTest, ResizeAcquireCount) {
+    EXPECT_EQ(OK, mBIC->setMaxAcquiredBufferCount(kMaxLockedBuffers + 1));
+    EXPECT_EQ(OK, mBIC->setMaxAcquiredBufferCount(kMaxLockedBuffers + 2));
+    EXPECT_EQ(OK, mBIC->setMaxAcquiredBufferCount(kMaxLockedBuffers - 1));
+    EXPECT_EQ(OK, mBIC->setMaxAcquiredBufferCount(kMaxLockedBuffers - 2));
+    EXPECT_EQ(OK, mBIC->setMaxAcquiredBufferCount(kMaxLockedBuffers + 1));
+    EXPECT_EQ(OK, mBIC->setMaxAcquiredBufferCount(kMaxLockedBuffers - 1));
+}
+
 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS)
 // Test that delete BufferItemConsumer triggers onBufferFreed.
 TEST_F(BufferItemConsumerTest, DetachBufferWithBuffer) {