Increase frame history size when SF buffer queue size changes

While the existing frame history size is sufficient for graphics, the
buffer queue sizes for video playback are much larger (more pipelined)
so in order for accurate frame tracking of video frames, increase the
frame event history size based on the buffer queue size (pipeline size),
so that buffer queue producers can always track all frames that have
been queued.

Bug: 234833109
Test: atest DecoderRenderTest
Change-Id: Ida587a239a03f74ebb099d8634ff722a500fcdda

8 files changed, 187 insertions, 27 deletions

diff --git a/libs/gui/BLASTBufferQueue.cpp b/libs/gui/BLASTBufferQueue.cpp
index 60603ba50a..948611218b 100644
--- a/libs/gui/BLASTBufferQueue.cpp
+++ b/libs/gui/BLASTBufferQueue.cpp
@@ -133,6 +133,11 @@ void BLASTBufferItemConsumer::onSidebandStreamChanged() {
     }
 }
 
+void BLASTBufferItemConsumer::resizeFrameEventHistory(size_t newSize) {
+    Mutex::Autolock lock(mMutex);
+    mFrameEventHistory.resize(newSize);
+}
+
 BLASTBufferQueue::BLASTBufferQueue(const std::string& name, bool updateDestinationFrame)
       : mSurfaceControl(nullptr),
         mSize(1, 1),
@@ -1036,8 +1041,9 @@ public:
 // can be non-blocking when the producer is in the client process.
 class BBQBufferQueueProducer : public BufferQueueProducer {
 public:
-    BBQBufferQueueProducer(const sp<BufferQueueCore>& core)
-          : BufferQueueProducer(core, false /* consumerIsSurfaceFlinger*/) {}
+    BBQBufferQueueProducer(const sp<BufferQueueCore>& core, wp<BLASTBufferQueue> bbq)
+          : BufferQueueProducer(core, false /* consumerIsSurfaceFlinger*/),
+            mBLASTBufferQueue(std::move(bbq)) {}
 
     status_t connect(const sp<IProducerListener>& listener, int api, bool producerControlledByApp,
                      QueueBufferOutput* output) override {
@@ -1049,6 +1055,26 @@ public:
                                             producerControlledByApp, output);
     }
 
+    // We want to resize the frame history when changing the size of the buffer queue
+    status_t setMaxDequeuedBufferCount(int maxDequeuedBufferCount) override {
+        int maxBufferCount;
+        status_t status = BufferQueueProducer::setMaxDequeuedBufferCount(maxDequeuedBufferCount,
+                                                                         &maxBufferCount);
+        // if we can't determine the max buffer count, then just skip growing the history size
+        if (status == OK) {
+            size_t newFrameHistorySize = maxBufferCount + 2; // +2 because triple buffer rendering
+            // optimize away resizing the frame history unless it will grow
+            if (newFrameHistorySize > FrameEventHistory::INITIAL_MAX_FRAME_HISTORY) {
+                sp<BLASTBufferQueue> bbq = mBLASTBufferQueue.promote();
+                if (bbq != nullptr) {
+                    ALOGV("increasing frame history size to %zu", newFrameHistorySize);
+                    bbq->resizeFrameEventHistory(newFrameHistorySize);
+                }
+            }
+        }
+        return status;
+    }
+
     int query(int what, int* value) override {
         if (what == NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER) {
             *value = 1;
@@ -1056,6 +1082,9 @@ public:
         }
         return BufferQueueProducer::query(what, value);
     }
+
+private:
+    const wp<BLASTBufferQueue> mBLASTBufferQueue;
 };
 
 // Similar to BufferQueue::createBufferQueue but creates an adapter specific bufferqueue producer.
@@ -1070,7 +1099,7 @@ void BLASTBufferQueue::createBufferQueue(sp<IGraphicBufferProducer>* outProducer
     sp<BufferQueueCore> core(new BufferQueueCore());
     LOG_ALWAYS_FATAL_IF(core == nullptr, "BLASTBufferQueue: failed to create BufferQueueCore");
 
-    sp<IGraphicBufferProducer> producer(new BBQBufferQueueProducer(core));
+    sp<IGraphicBufferProducer> producer(new BBQBufferQueueProducer(core, this));
     LOG_ALWAYS_FATAL_IF(producer == nullptr,
                         "BLASTBufferQueue: failed to create BBQBufferQueueProducer");
 
@@ -1083,6 +1112,16 @@ void BLASTBufferQueue::createBufferQueue(sp<IGraphicBufferProducer>* outProducer
     *outConsumer = consumer;
 }
 
+void BLASTBufferQueue::resizeFrameEventHistory(size_t newSize) {
+    // This can be null during creation of the buffer queue, but resizing won't do anything at that
+    // point in time, so just ignore. This can go away once the class relationships and lifetimes of
+    // objects are cleaned up with a major refactor of BufferQueue as a whole.
+    if (mBufferItemConsumer != nullptr) {
+        std::unique_lock _lock{mMutex};
+        mBufferItemConsumer->resizeFrameEventHistory(newSize);
+    }
+}
+
 PixelFormat BLASTBufferQueue::convertBufferFormat(PixelFormat& format) {
     PixelFormat convertedFormat = format;
     switch (format) {
diff --git a/libs/gui/BufferQueueProducer.cpp b/libs/gui/BufferQueueProducer.cpp
index 5fe5e71db9..9eb1a9f526 100644
--- a/libs/gui/BufferQueueProducer.cpp
+++ b/libs/gui/BufferQueueProducer.cpp
@@ -119,6 +119,12 @@ status_t BufferQueueProducer::requestBuffer(int slot, sp<GraphicBuffer>* buf) {
 
 status_t BufferQueueProducer::setMaxDequeuedBufferCount(
         int maxDequeuedBuffers) {
+    int maxBufferCount;
+    return setMaxDequeuedBufferCount(maxDequeuedBuffers, &maxBufferCount);
+}
+
+status_t BufferQueueProducer::setMaxDequeuedBufferCount(int maxDequeuedBuffers,
+                                                        int* maxBufferCount) {
     ATRACE_CALL();
     BQ_LOGV("setMaxDequeuedBufferCount: maxDequeuedBuffers = %d",
             maxDequeuedBuffers);
@@ -134,6 +140,8 @@ status_t BufferQueueProducer::setMaxDequeuedBufferCount(
             return NO_INIT;
         }
 
+        *maxBufferCount = mCore->getMaxBufferCountLocked();
+
         if (maxDequeuedBuffers == mCore->mMaxDequeuedBufferCount) {
             return NO_ERROR;
         }
@@ -183,6 +191,7 @@ status_t BufferQueueProducer::setMaxDequeuedBufferCount(
             return BAD_VALUE;
         }
         mCore->mMaxDequeuedBufferCount = maxDequeuedBuffers;
+        *maxBufferCount = mCore->getMaxBufferCountLocked();
         VALIDATE_CONSISTENCY();
         if (delta < 0) {
             listener = mCore->mConsumerListener;
diff --git a/libs/gui/FrameTimestamps.cpp b/libs/gui/FrameTimestamps.cpp
index e2ea3f9ab1..f3eb4e83aa 100644
--- a/libs/gui/FrameTimestamps.cpp
+++ b/libs/gui/FrameTimestamps.cpp
@@ -168,10 +168,11 @@ struct FrameNumberEqual {
 
 }  // namespace
 
-const size_t FrameEventHistory::MAX_FRAME_HISTORY =
+const size_t FrameEventHistory::INITIAL_MAX_FRAME_HISTORY =
         sysprop::LibGuiProperties::frame_event_history_size().value_or(8);
 
-FrameEventHistory::FrameEventHistory() : mFrames(std::vector<FrameEvents>(MAX_FRAME_HISTORY)) {}
+FrameEventHistory::FrameEventHistory()
+      : mFrames(std::vector<FrameEvents>(INITIAL_MAX_FRAME_HISTORY)) {}
 
 FrameEventHistory::~FrameEventHistory() = default;
 
@@ -227,7 +228,6 @@ void FrameEventHistory::dump(std::string& outString) const {
     }
 }
 
-
 // ============================================================================
 // ProducerFrameEventHistory
 // ============================================================================
@@ -273,6 +273,13 @@ void ProducerFrameEventHistory::applyDelta(
         const FrameEventHistoryDelta& delta) {
     mCompositorTiming = delta.mCompositorTiming;
 
+    // Deltas should have enough reserved capacity for the consumer-side, therefore if there's a
+    // different capacity, we re-sized on the consumer side and now need to resize on the producer
+    // side.
+    if (delta.mDeltas.capacity() > mFrames.capacity()) {
+        resize(delta.mDeltas.capacity());
+    }
+
     for (auto& d : delta.mDeltas) {
         // Avoid out-of-bounds access.
         if (CC_UNLIKELY(d.mIndex >= mFrames.size())) {
@@ -349,13 +356,48 @@ std::shared_ptr<FenceTime> ProducerFrameEventHistory::createFenceTime(
     return std::make_shared<FenceTime>(fence);
 }
 
+void ProducerFrameEventHistory::resize(size_t newSize) {
+    // we don't want to drop events by resizing too small, so don't resize in the negative direction
+    if (newSize <= mFrames.size()) {
+        return;
+    }
+
+    // This algorithm for resizing needs to be the same as ConsumerFrameEventHistory::resize,
+    // because the indexes need to match when communicating the FrameEventDeltas.
+
+    // We need to find the oldest frame, because that frame needs to move to index 0 in the new
+    // frame history.
+    size_t oldestFrameIndex = 0;
+    size_t oldestFrameNumber = INT32_MAX;
+    for (size_t i = 0; i < mFrames.size(); ++i) {
+        if (mFrames[i].frameNumber < oldestFrameNumber && mFrames[i].valid) {
+            oldestFrameNumber = mFrames[i].frameNumber;
+            oldestFrameIndex = i;
+        }
+    }
+
+    // move the existing frame information into a new vector, so that the oldest frames are at
+    // index 0, and the latest frames are at the end of the vector
+    std::vector<FrameEvents> newFrames(newSize);
+    size_t oldI = oldestFrameIndex;
+    size_t newI = 0;
+    do {
+        if (mFrames[oldI].valid) {
+            newFrames[newI++] = std::move(mFrames[oldI]);
+        }
+        oldI = (oldI + 1) % mFrames.size();
+    } while (oldI != oldestFrameIndex);
+
+    mFrames = std::move(newFrames);
+    mAcquireOffset = 0; // this is just a hint, so setting this to anything is fine
+}
 
 // ============================================================================
 // ConsumerFrameEventHistory
 // ============================================================================
 
 ConsumerFrameEventHistory::ConsumerFrameEventHistory()
-      : mFramesDirty(std::vector<FrameEventDirtyFields>(MAX_FRAME_HISTORY)) {}
+      : mFramesDirty(std::vector<FrameEventDirtyFields>(INITIAL_MAX_FRAME_HISTORY)) {}
 
 ConsumerFrameEventHistory::~ConsumerFrameEventHistory() = default;
 
@@ -489,6 +531,36 @@ void ConsumerFrameEventHistory::getAndResetDelta(
     }
 }
 
+void ConsumerFrameEventHistory::resize(size_t newSize) {
+    // we don't want to drop events by resizing too small, so don't resize in the negative direction
+    if (newSize <= mFrames.size()) {
+        return;
+    }
+
+    // This algorithm for resizing needs to be the same as ProducerFrameEventHistory::resize,
+    // because the indexes need to match when communicating the FrameEventDeltas.
+
+    // move the existing frame information into a new vector, so that the oldest frames are at
+    // index 0, and the latest frames are towards the end of the vector
+    std::vector<FrameEvents> newFrames(newSize);
+    std::vector<FrameEventDirtyFields> newFramesDirty(newSize);
+    size_t oldestFrameIndex = mQueueOffset;
+    size_t oldI = oldestFrameIndex;
+    size_t newI = 0;
+    do {
+        if (mFrames[oldI].valid) {
+            newFrames[newI] = std::move(mFrames[oldI]);
+            newFramesDirty[newI] = mFramesDirty[oldI];
+            newI += 1;
+        }
+        oldI = (oldI + 1) % mFrames.size();
+    } while (oldI != oldestFrameIndex);
+
+    mFrames = std::move(newFrames);
+    mFramesDirty = std::move(newFramesDirty);
+    mQueueOffset = newI;
+    mCompositionOffset = 0; // this is just a hint, so setting this to anything is fine
+}
 
 // ============================================================================
 // FrameEventsDelta
@@ -558,8 +630,7 @@ status_t FrameEventsDelta::flatten(void*& buffer, size_t& size, int*& fds,
         return NO_MEMORY;
     }
 
-    if (mIndex >= FrameEventHistory::MAX_FRAME_HISTORY ||
-            mIndex > std::numeric_limits<uint16_t>::max()) {
+    if (mIndex >= UINT8_MAX || mIndex < 0) {
         return BAD_VALUE;
     }
 
@@ -601,7 +672,7 @@ status_t FrameEventsDelta::unflatten(void const*& buffer, size_t& size,
     uint16_t temp16 = 0;
     FlattenableUtils::read(buffer, size, temp16);
     mIndex = temp16;
-    if (mIndex >= FrameEventHistory::MAX_FRAME_HISTORY) {
+    if (mIndex >= UINT8_MAX) {
         return BAD_VALUE;
     }
     uint8_t temp8 = 0;
@@ -627,6 +698,25 @@ status_t FrameEventsDelta::unflatten(void const*& buffer, size_t& size,
     return NO_ERROR;
 }
 
+uint64_t FrameEventsDelta::getFrameNumber() const {
+    return mFrameNumber;
+}
+
+bool FrameEventsDelta::getLatchTime(nsecs_t* latchTime) const {
+    if (mLatchTime == FrameEvents::TIMESTAMP_PENDING) {
+        return false;
+    }
+    *latchTime = mLatchTime;
+    return true;
+}
+
+bool FrameEventsDelta::getDisplayPresentFence(sp<Fence>* fence) const {
+    if (mDisplayPresentFence.fence == Fence::NO_FENCE) {
+        return false;
+    }
+    *fence = mDisplayPresentFence.fence;
+    return true;
+}
 
 // ============================================================================
 // FrameEventHistoryDelta
@@ -665,7 +755,7 @@ size_t FrameEventHistoryDelta::getFdCount() const {
 
 status_t FrameEventHistoryDelta::flatten(
             void*& buffer, size_t& size, int*& fds, size_t& count) const {
-    if (mDeltas.size() > FrameEventHistory::MAX_FRAME_HISTORY) {
+    if (mDeltas.size() > UINT8_MAX) {
         return BAD_VALUE;
     }
     if (size < getFlattenedSize()) {
@@ -695,7 +785,7 @@ status_t FrameEventHistoryDelta::unflatten(
 
     uint32_t deltaCount = 0;
     FlattenableUtils::read(buffer, size, deltaCount);
-    if (deltaCount > FrameEventHistory::MAX_FRAME_HISTORY) {
+    if (deltaCount > UINT8_MAX) {
         return BAD_VALUE;
     }
     mDeltas.resize(deltaCount);
@@ -708,5 +798,12 @@ status_t FrameEventHistoryDelta::unflatten(
     return NO_ERROR;
 }
 
+std::vector<FrameEventsDelta>::const_iterator FrameEventHistoryDelta::begin() const {
+    return mDeltas.begin();
+}
+
+std::vector<FrameEventsDelta>::const_iterator FrameEventHistoryDelta::end() const {
+    return mDeltas.end();
+}
 
 } // namespace android
diff --git a/libs/gui/bufferqueue/1.0/Conversion.cpp b/libs/gui/bufferqueue/1.0/Conversion.cpp
index 55462c3b1f..96679549b2 100644
--- a/libs/gui/bufferqueue/1.0/Conversion.cpp
+++ b/libs/gui/bufferqueue/1.0/Conversion.cpp
@@ -725,12 +725,7 @@ status_t unflatten(HGraphicBufferProducer::FrameEventsDelta* t,
     // These were written as uint8_t for alignment.
     uint8_t temp = 0;
     FlattenableUtils::read(buffer, size, temp);
-    size_t index = static_cast<size_t>(temp);
-    if (index >= ::android::FrameEventHistory::MAX_FRAME_HISTORY) {
-        return BAD_VALUE;
-    }
-    t->index = static_cast<uint32_t>(index);
-
+    t->index = static_cast<uint32_t>(temp);
     FlattenableUtils::read(buffer, size, temp);
     t->addPostCompositeCalled = static_cast<bool>(temp);
     FlattenableUtils::read(buffer, size, temp);
@@ -786,8 +781,7 @@ status_t unflatten(HGraphicBufferProducer::FrameEventsDelta* t,
 status_t flatten(HGraphicBufferProducer::FrameEventsDelta const& t,
         void*& buffer, size_t& size, int*& fds, size_t numFds) {
     // Check that t.index is within a valid range.
-    if (t.index >= static_cast<uint32_t>(FrameEventHistory::MAX_FRAME_HISTORY)
-            || t.index > std::numeric_limits<uint8_t>::max()) {
+    if (t.index > UINT8_MAX || t.index < 0) {
         return BAD_VALUE;
     }
 
@@ -887,8 +881,7 @@ status_t unflatten(
 
     uint32_t deltaCount = 0;
     FlattenableUtils::read(buffer, size, deltaCount);
-    if (static_cast<size_t>(deltaCount) >
-            ::android::FrameEventHistory::MAX_FRAME_HISTORY) {
+    if (deltaCount > UINT8_MAX) {
         return BAD_VALUE;
     }
     t->deltas.resize(deltaCount);
@@ -919,7 +912,7 @@ status_t unflatten(
 status_t flatten(
         HGraphicBufferProducer::FrameEventHistoryDelta const& t,
         void*& buffer, size_t& size, int*& fds, size_t& numFds) {
-    if (t.deltas.size() > ::android::FrameEventHistory::MAX_FRAME_HISTORY) {
+    if (t.deltas.size() > UINT8_MAX) {
         return BAD_VALUE;
     }
     if (size < getFlattenedSize(t)) {
diff --git a/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp b/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp
index cee1b811a4..f684874aec 100644
--- a/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp
+++ b/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp
@@ -725,8 +725,7 @@ inline status_t flatten(HGraphicBufferProducer::FrameEventsDelta const& t,
         std::vector<native_handle_t*>* nh,
         void*& buffer, size_t& size, int*& fds, size_t numFds) {
     // Check that t.index is within a valid range.
-    if (t.index >= static_cast<uint32_t>(FrameEventHistory::MAX_FRAME_HISTORY)
-            || t.index > std::numeric_limits<uint8_t>::max()) {
+    if (t.index > UINT8_MAX || t.index < 0) {
         return BAD_VALUE;
     }
 
@@ -829,7 +828,7 @@ inline status_t flatten(
         HGraphicBufferProducer::FrameEventHistoryDelta const& t,
         std::vector<std::vector<native_handle_t*> >* nh,
         void*& buffer, size_t& size, int*& fds, size_t& numFds) {
-    if (t.deltas.size() > ::android::FrameEventHistory::MAX_FRAME_HISTORY) {
+    if (t.deltas.size() > UINT8_MAX) {
         return BAD_VALUE;
     }
     if (size < getFlattenedSize(t)) {
diff --git a/libs/gui/include/gui/BLASTBufferQueue.h b/libs/gui/include/gui/BLASTBufferQueue.h
index c93ab86770..515363a4cd 100644
--- a/libs/gui/include/gui/BLASTBufferQueue.h
+++ b/libs/gui/include/gui/BLASTBufferQueue.h
@@ -54,6 +54,8 @@ public:
                                nsecs_t dequeueReadyTime) REQUIRES(mMutex);
     void getConnectionEvents(uint64_t frameNumber, bool* needsDisconnect);
 
+    void resizeFrameEventHistory(size_t newSize);
+
 protected:
     void onSidebandStreamChanged() override REQUIRES(mMutex);
 
@@ -123,6 +125,7 @@ public:
 
 private:
     friend class BLASTBufferQueueHelper;
+    friend class BBQBufferQueueProducer;
 
     // can't be copied
     BLASTBufferQueue& operator = (const BLASTBufferQueue& rhs);
@@ -130,6 +133,8 @@ private:
     void createBufferQueue(sp<IGraphicBufferProducer>* outProducer,
                            sp<IGraphicBufferConsumer>* outConsumer);
 
+    void resizeFrameEventHistory(size_t newSize);
+
     status_t acquireNextBufferLocked(
             const std::optional<SurfaceComposerClient::Transaction*> transaction) REQUIRES(mMutex);
     Rect computeCrop(const BufferItem& item) REQUIRES(mMutex);
diff --git a/libs/gui/include/gui/BufferQueueProducer.h b/libs/gui/include/gui/BufferQueueProducer.h
index 0ad3075a4d..1d13dab623 100644
--- a/libs/gui/include/gui/BufferQueueProducer.h
+++ b/libs/gui/include/gui/BufferQueueProducer.h
@@ -202,6 +202,11 @@ public:
     // See IGraphicBufferProducer::setAutoPrerotation
     virtual status_t setAutoPrerotation(bool autoPrerotation);
 
+protected:
+    // see IGraphicsBufferProducer::setMaxDequeuedBufferCount, but with the ability to retrieve the
+    // total maximum buffer count for the buffer queue (dequeued AND acquired)
+    status_t setMaxDequeuedBufferCount(int maxDequeuedBuffers, int* maxBufferCount);
+
 private:
     // This is required by the IBinder::DeathRecipient interface
     virtual void binderDied(const wp<IBinder>& who);
diff --git a/libs/gui/include/gui/FrameTimestamps.h b/libs/gui/include/gui/FrameTimestamps.h
index c08a9b1b6c..3d1be4d2eb 100644
--- a/libs/gui/include/gui/FrameTimestamps.h
+++ b/libs/gui/include/gui/FrameTimestamps.h
@@ -97,9 +97,11 @@ public:
     void checkFencesForCompletion();
     void dump(std::string& outString) const;
 
-    static const size_t MAX_FRAME_HISTORY;
+    static const size_t INITIAL_MAX_FRAME_HISTORY;
 
 protected:
+    void resize(size_t newSize);
+
     std::vector<FrameEvents> mFrames;
 
     CompositorTiming mCompositorTiming;
@@ -138,6 +140,8 @@ protected:
     virtual std::shared_ptr<FenceTime> createFenceTime(
             const sp<Fence>& fence) const;
 
+    void resize(size_t newSize);
+
     size_t mAcquireOffset{0};
 
     // The consumer updates it's timelines in Layer and SurfaceFlinger since
@@ -208,6 +212,8 @@ public:
 
     void getAndResetDelta(FrameEventHistoryDelta* delta);
 
+    void resize(size_t newSize);
+
 private:
     void getFrameDelta(FrameEventHistoryDelta* delta,
                        const std::vector<FrameEvents>::iterator& frame);
@@ -250,6 +256,10 @@ public:
     status_t unflatten(void const*& buffer, size_t& size, int const*& fds,
             size_t& count);
 
+    uint64_t getFrameNumber() const;
+    bool getLatchTime(nsecs_t* latchTime) const;
+    bool getDisplayPresentFence(sp<Fence>* fence) const;
+
 private:
     static constexpr size_t minFlattenedSize();
 
@@ -310,6 +320,9 @@ public:
     status_t unflatten(void const*& buffer, size_t& size, int const*& fds,
             size_t& count);
 
+    std::vector<FrameEventsDelta>::const_iterator begin() const;
+    std::vector<FrameEventsDelta>::const_iterator end() const;
+
 private:
     static constexpr size_t minFlattenedSize();