Allow apps to associate a change in picture profiles alongside a buffer

What picture processing a buffer looks best with is often dependent on
the buffer contents itself. It is often necessary for a change in
picture profile to be tightly coupled to a change in buffer.

Bug: 337330263
Test: build
Test: atest BufferQueueTest
Flag: com.android.graphics.libgui.flags.apply_picture_profiles
Change-Id: I8bd3468519fb28a234f6853531638e348b1c5274

8 files changed, 173 insertions, 35 deletions

diff --git a/libs/gui/BLASTBufferQueue.cpp b/libs/gui/BLASTBufferQueue.cpp
index 495418b921..7aee90393b 100644
--- a/libs/gui/BLASTBufferQueue.cpp
+++ b/libs/gui/BLASTBufferQueue.cpp
@@ -286,18 +286,23 @@ void BLASTBufferQueue::update(const sp<SurfaceControl>& surface, uint32_t width,
     if (surfaceControlChanged && mSurfaceControl != nullptr) {
         BQA_LOGD("Updating SurfaceControl without recreating BBQ");
     }
-    bool applyTransaction = false;
 
     // Always update the native object even though they might have the same layer handle, so we can
     // get the updated transform hint from WM.
     mSurfaceControl = surface;
     SurfaceComposerClient::Transaction t;
+    bool applyTransaction = false;
     if (surfaceControlChanged) {
 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL)
         updateBufferReleaseProducer();
 #endif
         t.setFlags(mSurfaceControl, layer_state_t::eEnableBackpressure,
                    layer_state_t::eEnableBackpressure);
+        // Migrate the picture profile handle to the new surface control.
+        if (com_android_graphics_libgui_flags_apply_picture_profiles() &&
+            mPictureProfileHandle.has_value()) {
+            t.setPictureProfileHandle(mSurfaceControl, *mPictureProfileHandle);
+        }
         applyTransaction = true;
     }
     mTransformHint = mSurfaceControl->getTransformHint();
@@ -679,6 +684,17 @@ status_t BLASTBufferQueue::acquireNextBufferLocked(
     if (!bufferItem.mIsAutoTimestamp) {
         t->setDesiredPresentTime(bufferItem.mTimestamp);
     }
+    if (com_android_graphics_libgui_flags_apply_picture_profiles() &&
+        bufferItem.mPictureProfileHandle.has_value()) {
+        t->setPictureProfileHandle(mSurfaceControl, *bufferItem.mPictureProfileHandle);
+        // The current picture profile must be maintained in case the BBQ gets its
+        // SurfaceControl switched out.
+        mPictureProfileHandle = bufferItem.mPictureProfileHandle;
+        // Clear out the picture profile if the requestor has asked for it to be cleared
+        if (mPictureProfileHandle == PictureProfileHandle::NONE) {
+            mPictureProfileHandle = std::nullopt;
+        }
+    }
 
     // Drop stale frame timeline infos
     while (!mPendingFrameTimelines.empty() &&
diff --git a/libs/gui/BufferItem.cpp b/libs/gui/BufferItem.cpp
index 5beba02e63..3b2d337a21 100644
--- a/libs/gui/BufferItem.cpp
+++ b/libs/gui/BufferItem.cpp
@@ -38,26 +38,25 @@ static inline constexpr T to64(const uint32_t lo, const uint32_t hi) {
     return static_cast<T>(static_cast<uint64_t>(hi)<<32 | lo);
 }
 
-BufferItem::BufferItem() :
-    mGraphicBuffer(nullptr),
-    mFence(nullptr),
-    mCrop(Rect::INVALID_RECT),
-    mTransform(0),
-    mScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE),
-    mTimestamp(0),
-    mIsAutoTimestamp(false),
-    mDataSpace(HAL_DATASPACE_UNKNOWN),
-    mFrameNumber(0),
-    mSlot(INVALID_BUFFER_SLOT),
-    mIsDroppable(false),
-    mAcquireCalled(false),
-    mTransformToDisplayInverse(false),
-    mSurfaceDamage(),
-    mAutoRefresh(false),
-    mQueuedBuffer(true),
-    mIsStale(false),
-    mApi(0) {
-}
+BufferItem::BufferItem()
+      : mGraphicBuffer(nullptr),
+        mFence(nullptr),
+        mCrop(Rect::INVALID_RECT),
+        mTransform(0),
+        mScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE),
+        mTimestamp(0),
+        mIsAutoTimestamp(false),
+        mDataSpace(HAL_DATASPACE_UNKNOWN),
+        mFrameNumber(0),
+        mSlot(INVALID_BUFFER_SLOT),
+        mIsDroppable(false),
+        mAcquireCalled(false),
+        mTransformToDisplayInverse(false),
+        mSurfaceDamage(),
+        mAutoRefresh(false),
+        mQueuedBuffer(true),
+        mIsStale(false),
+        mApi(0) {}
 
 BufferItem::~BufferItem() {}
 
@@ -76,6 +75,11 @@ size_t BufferItem::getPodSize() const {
     addAligned(size, high32(mTimestamp));
     addAligned(size, mIsAutoTimestamp);
     addAligned(size, mDataSpace);
+#if COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
+    addAligned(size, mPictureProfileHandle.has_value());
+    addAligned(size, low32(PictureProfileHandle::NONE.getId()));
+    addAligned(size, high32(PictureProfileHandle::NONE.getId()));
+#endif // COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
     addAligned(size, low32(mFrameNumber));
     addAligned(size, high32(mFrameNumber));
     addAligned(size, mSlot);
@@ -170,6 +174,16 @@ status_t BufferItem::flatten(
     writeAligned(buffer, size, high32(mTimestamp));
     writeAligned(buffer, size, mIsAutoTimestamp);
     writeAligned(buffer, size, mDataSpace);
+#if COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
+    writeAligned(buffer, size, mPictureProfileHandle.has_value());
+    if (mPictureProfileHandle.has_value()) {
+        writeAligned(buffer, size, low32(mPictureProfileHandle->getId()));
+        writeAligned(buffer, size, high32(mPictureProfileHandle->getId()));
+    } else {
+        writeAligned(buffer, size, low32(PictureProfileHandle::NONE.getId()));
+        writeAligned(buffer, size, high32(PictureProfileHandle::NONE.getId()));
+    }
+#endif // COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
     writeAligned(buffer, size, low32(mFrameNumber));
     writeAligned(buffer, size, high32(mFrameNumber));
     writeAligned(buffer, size, mSlot);
@@ -231,6 +245,7 @@ status_t BufferItem::unflatten(
 
     uint32_t timestampLo = 0, timestampHi = 0;
     uint32_t frameNumberLo = 0, frameNumberHi = 0;
+    int32_t pictureProfileIdLo = 0, pictureProfileIdHi = 0;
 
     readAligned(buffer, size, mCrop);
     readAligned(buffer, size, mTransform);
@@ -240,6 +255,16 @@ status_t BufferItem::unflatten(
     mTimestamp = to64<int64_t>(timestampLo, timestampHi);
     readAligned(buffer, size, mIsAutoTimestamp);
     readAligned(buffer, size, mDataSpace);
+#if COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
+    bool hasPictureProfileHandle;
+    readAligned(buffer, size, hasPictureProfileHandle);
+    readAligned(buffer, size, pictureProfileIdLo);
+    readAligned(buffer, size, pictureProfileIdHi);
+    mPictureProfileHandle = hasPictureProfileHandle
+            ? std::optional(PictureProfileHandle(
+                      to64<PictureProfileId>(pictureProfileIdLo, pictureProfileIdHi)))
+            : std::nullopt;
+#endif // COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
     readAligned(buffer, size, frameNumberLo);
     readAligned(buffer, size, frameNumberHi);
     mFrameNumber = to64<uint64_t>(frameNumberLo, frameNumberHi);
diff --git a/libs/gui/BufferQueueProducer.cpp b/libs/gui/BufferQueueProducer.cpp
index 473a374a59..39209f9745 100644
--- a/libs/gui/BufferQueueProducer.cpp
+++ b/libs/gui/BufferQueueProducer.cpp
@@ -938,6 +938,8 @@ status_t BufferQueueProducer::queueBuffer(int slot,
             &getFrameTimestamps);
     const Region& surfaceDamage = input.getSurfaceDamage();
     const HdrMetadata& hdrMetadata = input.getHdrMetadata();
+    const std::optional<PictureProfileHandle>& pictureProfileHandle =
+            input.getPictureProfileHandle();
 
     if (acquireFence == nullptr) {
         BQ_LOGE("queueBuffer: fence is NULL");
@@ -1044,6 +1046,7 @@ status_t BufferQueueProducer::queueBuffer(int slot,
         item.mIsAutoTimestamp = isAutoTimestamp;
         item.mDataSpace = dataSpace;
         item.mHdrMetadata = hdrMetadata;
+        item.mPictureProfileHandle = pictureProfileHandle;
         item.mFrameNumber = currentFrameNumber;
         item.mSlot = slot;
         item.mFence = acquireFence;
diff --git a/libs/gui/IGraphicBufferProducerFlattenables.cpp b/libs/gui/IGraphicBufferProducerFlattenables.cpp
index c8b9b6751d..4e92a39973 100644
--- a/libs/gui/IGraphicBufferProducerFlattenables.cpp
+++ b/libs/gui/IGraphicBufferProducerFlattenables.cpp
@@ -20,21 +20,19 @@
 namespace android {
 
 constexpr size_t IGraphicBufferProducer::QueueBufferInput::minFlattenedSize() {
-    return sizeof(timestamp) +
-            sizeof(isAutoTimestamp) +
-            sizeof(dataSpace) +
-            sizeof(crop) +
-            sizeof(scalingMode) +
-            sizeof(transform) +
-            sizeof(stickyTransform) +
-            sizeof(getFrameTimestamps) +
-            sizeof(slot);
+    return sizeof(timestamp) + sizeof(isAutoTimestamp) + sizeof(dataSpace) + sizeof(crop) +
+            sizeof(scalingMode) + sizeof(transform) + sizeof(stickyTransform) +
+            sizeof(getFrameTimestamps) + sizeof(slot) +
+#if COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
+            sizeof(decltype(pictureProfileHandle.has_value())) +
+            sizeof(decltype(pictureProfileHandle.getId()));
+#else
+            0;
+#endif // COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
 }
 
 size_t IGraphicBufferProducer::QueueBufferInput::getFlattenedSize() const {
-    return minFlattenedSize() +
-            fence->getFlattenedSize() +
-            surfaceDamage.getFlattenedSize() +
+    return minFlattenedSize() + fence->getFlattenedSize() + surfaceDamage.getFlattenedSize() +
             hdrMetadata.getFlattenedSize();
 }
 
@@ -57,6 +55,12 @@ status_t IGraphicBufferProducer::QueueBufferInput::flatten(
     FlattenableUtils::write(buffer, size, transform);
     FlattenableUtils::write(buffer, size, stickyTransform);
     FlattenableUtils::write(buffer, size, getFrameTimestamps);
+#if COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
+    FlattenableUtils::write(buffer, size, pictureProfileHandle.has_value());
+    FlattenableUtils::write(buffer, size,
+                            pictureProfileHandle.has_value() ? pictureProfileHandle->getId()
+                                                             : PictureProfileHandle::NONE.getId());
+#endif // COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
 
     status_t result = fence->flatten(buffer, size, fds, count);
     if (result != NO_ERROR) {
@@ -91,6 +95,15 @@ status_t IGraphicBufferProducer::QueueBufferInput::unflatten(
     FlattenableUtils::read(buffer, size, transform);
     FlattenableUtils::read(buffer, size, stickyTransform);
     FlattenableUtils::read(buffer, size, getFrameTimestamps);
+#if COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
+    bool hasPictureProfileHandle;
+    FlattenableUtils::read(buffer, size, hasPictureProfileHandle);
+    PictureProfileId pictureProfileId;
+    FlattenableUtils::read(buffer, size, pictureProfileId);
+    pictureProfileHandle = hasPictureProfileHandle
+            ? std::optional(PictureProfileHandle(pictureProfileId))
+            : std::nullopt;
+#endif // COM_ANDROID_GRAPHICS_LIBUI_FLAGS_APPLY_PICTURE_PROFILES
 
     fence = new Fence();
     status_t result = fence->unflatten(buffer, size, fds, count);
diff --git a/libs/gui/include/gui/BLASTBufferQueue.h b/libs/gui/include/gui/BLASTBufferQueue.h
index 8894b66c6d..07558aa49d 100644
--- a/libs/gui/include/gui/BLASTBufferQueue.h
+++ b/libs/gui/include/gui/BLASTBufferQueue.h
@@ -17,7 +17,9 @@
 #ifndef ANDROID_GUI_BLAST_BUFFER_QUEUE_H
 #define ANDROID_GUI_BLAST_BUFFER_QUEUE_H
 
-#include <com_android_graphics_libgui_flags.h>
+#include <optional>
+#include <queue>
+
 #include <gui/BufferItem.h>
 #include <gui/BufferItemConsumer.h>
 #include <gui/IGraphicBufferConsumer.h>
@@ -29,7 +31,6 @@
 #include <utils/RefBase.h>
 
 #include <system/window.h>
-#include <queue>
 
 #include <com_android_graphics_libgui_flags.h>
 
@@ -222,6 +223,10 @@ private:
     ui::Size mRequestedSize GUARDED_BY(mMutex);
     int32_t mFormat GUARDED_BY(mMutex);
 
+    // Keep a copy of the current picture profile handle, so it can be moved to a new
+    // SurfaceControl when BBQ migrates via ::update.
+    std::optional<PictureProfileHandle> mPictureProfileHandle;
+
     struct BufferInfo {
         bool hasBuffer = false;
         uint32_t width;
diff --git a/libs/gui/include/gui/BufferItem.h b/libs/gui/include/gui/BufferItem.h
index 218bb424fb..2f85c62a54 100644
--- a/libs/gui/include/gui/BufferItem.h
+++ b/libs/gui/include/gui/BufferItem.h
@@ -17,9 +17,12 @@
 #ifndef ANDROID_GUI_BUFFERITEM_H
 #define ANDROID_GUI_BUFFERITEM_H
 
+#include <optional>
+
 #include <gui/HdrMetadata.h>
 
 #include <ui/FenceTime.h>
+#include <ui/PictureProfileHandle.h>
 #include <ui/Rect.h>
 #include <ui/Region.h>
 
@@ -91,6 +94,10 @@ class BufferItem : public Flattenable<BufferItem> {
     // mHdrMetadata is the HDR metadata associated with this buffer slot.
     HdrMetadata mHdrMetadata;
 
+    // mPictureProfileHandle is a handle that points to a set of parameters that configure picture
+    // processing hardware to enhance the quality of buffer contents.
+    std::optional<PictureProfileHandle> mPictureProfileHandle;
+
     // mFrameNumber is the number of the queued frame for this slot.
     uint64_t mFrameNumber;
 
diff --git a/libs/gui/include/gui/IGraphicBufferProducer.h b/libs/gui/include/gui/IGraphicBufferProducer.h
index 3aac457a09..001e570982 100644
--- a/libs/gui/include/gui/IGraphicBufferProducer.h
+++ b/libs/gui/include/gui/IGraphicBufferProducer.h
@@ -19,6 +19,7 @@
 
 #include <stdint.h>
 #include <sys/types.h>
+#include <optional>
 
 #include <utils/Errors.h>
 #include <utils/RefBase.h>
@@ -28,6 +29,7 @@
 #include <ui/BufferQueueDefs.h>
 #include <ui/Fence.h>
 #include <ui/GraphicBuffer.h>
+#include <ui/PictureProfileHandle.h>
 #include <ui/Rect.h>
 #include <ui/Region.h>
 
@@ -365,6 +367,14 @@ public:
         const HdrMetadata& getHdrMetadata() const { return hdrMetadata; }
         void setHdrMetadata(const HdrMetadata& metadata) { hdrMetadata = metadata; }
 
+        const std::optional<PictureProfileHandle>& getPictureProfileHandle() const {
+            return pictureProfileHandle;
+        }
+        void setPictureProfileHandle(const PictureProfileHandle& profile) {
+            pictureProfileHandle = profile;
+        }
+        void clearPictureProfileHandle() { pictureProfileHandle = std::nullopt; }
+
         int64_t timestamp{0};
         int isAutoTimestamp{0};
         android_dataspace dataSpace{HAL_DATASPACE_UNKNOWN};
@@ -377,6 +387,7 @@ public:
         bool getFrameTimestamps{false};
         int slot{-1};
         HdrMetadata hdrMetadata;
+        std::optional<PictureProfileHandle> pictureProfileHandle;
     };
 
     struct QueueBufferOutput : public Flattenable<QueueBufferOutput> {
diff --git a/libs/gui/tests/BufferQueue_test.cpp b/libs/gui/tests/BufferQueue_test.cpp
index 2e6ffcb57f..b026e640aa 100644
--- a/libs/gui/tests/BufferQueue_test.cpp
+++ b/libs/gui/tests/BufferQueue_test.cpp
@@ -27,6 +27,7 @@
 #include <gui/Surface.h>
 
 #include <ui/GraphicBuffer.h>
+#include <ui/PictureProfileHandle.h>
 
 #include <android-base/properties.h>
 
@@ -1569,4 +1570,61 @@ TEST_F(BufferQueueTest, TestAdditionalOptions) {
     EXPECT_EQ(ADATASPACE_UNKNOWN, dataSpace);
 }
 
+TEST_F(BufferQueueTest, PassesThroughPictureProfileHandle) {
+    createBufferQueue();
+    sp<MockConsumer> mc(new MockConsumer);
+    mConsumer->consumerConnect(mc, false);
+
+    IGraphicBufferProducer::QueueBufferOutput qbo;
+    mProducer->connect(new StubProducerListener, NATIVE_WINDOW_API_CPU, false, &qbo);
+    mProducer->setMaxDequeuedBufferCount(2);
+    mConsumer->setMaxAcquiredBufferCount(2);
+
+    // First try to pass a valid picture profile handle
+    {
+        int slot;
+        sp<Fence> fence;
+        sp<GraphicBuffer> buf;
+        IGraphicBufferProducer::QueueBufferInput qbi(0, false, HAL_DATASPACE_UNKNOWN,
+                                                     Rect(0, 0, 1, 1),
+                                                     NATIVE_WINDOW_SCALING_MODE_FREEZE, 0,
+                                                     Fence::NO_FENCE);
+        qbi.setPictureProfileHandle(PictureProfileHandle(1));
+
+        EXPECT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION,
+                  mProducer->dequeueBuffer(&slot, &fence, 1, 1, 0, GRALLOC_USAGE_SW_READ_OFTEN,
+                                           nullptr, nullptr));
+        EXPECT_EQ(OK, mProducer->requestBuffer(slot, &buf));
+        EXPECT_EQ(OK, mProducer->queueBuffer(slot, qbi, &qbo));
+
+        BufferItem item;
+        EXPECT_EQ(OK, mConsumer->acquireBuffer(&item, 0));
+
+        ASSERT_TRUE(item.mPictureProfileHandle.has_value());
+        ASSERT_EQ(item.mPictureProfileHandle, PictureProfileHandle(1));
+    }
+
+    // Then validate that the picture profile handle isn't sticky and is reset for the next buffer
+    {
+        int slot;
+        sp<Fence> fence;
+        sp<GraphicBuffer> buf;
+        IGraphicBufferProducer::QueueBufferInput qbi(0, false, HAL_DATASPACE_UNKNOWN,
+                                                     Rect(0, 0, 1, 1),
+                                                     NATIVE_WINDOW_SCALING_MODE_FREEZE, 0,
+                                                     Fence::NO_FENCE);
+
+        EXPECT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION,
+                  mProducer->dequeueBuffer(&slot, &fence, 1, 1, 0, GRALLOC_USAGE_SW_READ_OFTEN,
+                                           nullptr, nullptr));
+        EXPECT_EQ(OK, mProducer->requestBuffer(slot, &buf));
+        EXPECT_EQ(OK, mProducer->queueBuffer(slot, qbi, &qbo));
+
+        BufferItem item;
+        EXPECT_EQ(OK, mConsumer->acquireBuffer(&item, 0));
+
+        ASSERT_FALSE(item.mPictureProfileHandle.has_value());
+    }
+}
+
 } // namespace android