SF: Suppress frame rate when small area updating

The small area is defined as the area that user is hard to obvious the
content updating such as small icon animtion or progress bar.

When content detect frame rate feature is enabled, and no explict
refresh vote is set to the layer, it will heuristically calculate the
predict frame rate by counting the average frame time of recent frames.

This CL will enhance the contect detection by checking the small dirty
frames and supress the predict frame rate that could prevent the app
always running at max frame rate cause unexcpected battery consumption.

This will cover 2 use cases:
1. When a layer contains a textureview, it will enter the heuristic
   calcuation, and this would skip the small dirty frames and supress
   the calculated frame rate not always to be max.

2. When an app has the separated video surface and ui surface, and the
   video had set an explicit rate by the content, this will ignore the
   heristic calculation for ui layer when it detected the small dirty
   frames, so the final refresh rate selection could mainly rely on the
   weight of the video layer.

Bug: 281720315
Test: atest LayerHistoryTest
Merged-In: I9731de8adf8e68b72326b195fa720d51e02bae74
Change-Id: I9731de8adf8e68b72326b195fa720d51e02bae74

10 files changed, 212 insertions, 3 deletions

diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 5a010e8af6..68230efd20 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -3178,6 +3178,14 @@ bool Layer::setBuffer(std::shared_ptr<renderengine::ExternalTexture>& buffer,
     }
 
     mDrawingState.releaseBufferEndpoint = bufferData.releaseBufferEndpoint;
+
+    // If the layer had been updated a TextureView, this would make sure the present time could be
+    // same to TextureView update when it's a small dirty, and get the correct heuristic rate.
+    if (mFlinger->mScheduler->supportSmallDirtyDetection()) {
+        if (mDrawingState.useVsyncIdForRefreshRateSelection) {
+            mUsedVsyncIdForRefreshRateSelection = true;
+        }
+    }
     return true;
 }
 
@@ -3200,10 +3208,38 @@ void Layer::recordLayerHistoryBufferUpdate(const scheduler::LayerProps& layerPro
                             mDrawingState.latchedVsyncId);
             if (prediction.has_value()) {
                 ATRACE_FORMAT_INSTANT("predictedPresentTime");
+                mMaxTimeForUseVsyncId = prediction->presentTime +
+                        scheduler::LayerHistory::kMaxPeriodForHistory.count();
                 return prediction->presentTime;
             }
         }
 
+        if (!mFlinger->mScheduler->supportSmallDirtyDetection()) {
+            return static_cast<nsecs_t>(0);
+        }
+
+        // If the layer is not an application and didn't set an explicit rate or desiredPresentTime,
+        // return "0" to tell the layer history that it will use the max refresh rate without
+        // calculating the adaptive rate.
+        if (mWindowType != WindowInfo::Type::APPLICATION &&
+            mWindowType != WindowInfo::Type::BASE_APPLICATION) {
+            return static_cast<nsecs_t>(0);
+        }
+
+        // Return the valid present time only when the layer potentially updated a TextureView so
+        // LayerHistory could heuristically calculate the rate if the UI is continually updating.
+        if (mUsedVsyncIdForRefreshRateSelection) {
+            const auto prediction =
+                    mFlinger->mFrameTimeline->getTokenManager()->getPredictionsForToken(
+                            mDrawingState.latchedVsyncId);
+            if (prediction.has_value()) {
+                if (mMaxTimeForUseVsyncId >= prediction->presentTime) {
+                    return prediction->presentTime;
+                }
+                mUsedVsyncIdForRefreshRateSelection = false;
+            }
+        }
+
         return static_cast<nsecs_t>(0);
     }();
 
@@ -3263,6 +3299,7 @@ bool Layer::setSurfaceDamageRegion(const Region& surfaceDamage) {
     mDrawingState.surfaceDamageRegion = surfaceDamage;
     mDrawingState.modified = true;
     setTransactionFlags(eTransactionNeeded);
+    setIsSmallDirty();
     return true;
 }
 
@@ -4310,6 +4347,25 @@ void Layer::updateLastLatchTime(nsecs_t latchTime) {
     mLastLatchTime = latchTime;
 }
 
+void Layer::setIsSmallDirty() {
+    if (!mFlinger->mScheduler->supportSmallDirtyDetection()) {
+        return;
+    }
+
+    if (mWindowType != WindowInfo::Type::APPLICATION &&
+        mWindowType != WindowInfo::Type::BASE_APPLICATION) {
+        return;
+    }
+    Rect bounds = mDrawingState.surfaceDamageRegion.getBounds();
+    if (!bounds.isValid()) {
+        return;
+    }
+
+    // If the damage region is a small dirty, this could give the hint for the layer history that
+    // it could suppress the heuristic rate when calculating.
+    mSmallDirty = mFlinger->mScheduler->isSmallDirtyArea(bounds.getWidth() * bounds.getHeight());
+}
+
 // ---------------------------------------------------------------------------
 
 std::ostream& operator<<(std::ostream& stream, const Layer::FrameRate& rate) {
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index 2fbbbdcb5c..895d25aa03 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -842,6 +842,14 @@ public:
     mutable bool contentDirty{false};
     Region surfaceDamageRegion;
 
+    // True when the surfaceDamageRegion is recognized as a small area update.
+    bool mSmallDirty{false};
+    // Used to check if mUsedVsyncIdForRefreshRateSelection should be expired when it stop updating.
+    nsecs_t mMaxTimeForUseVsyncId = 0;
+    // True when DrawState.useVsyncIdForRefreshRateSelection previously set to true during updating
+    // buffer.
+    bool mUsedVsyncIdForRefreshRateSelection{false};
+
     // Layer serial number.  This gives layers an explicit ordering, so we
     // have a stable sort order when their layer stack and Z-order are
     // the same.
@@ -904,6 +912,7 @@ public:
                 .transform = getTransform(),
                 .setFrameRateVote = getFrameRateForLayerTree(),
                 .frameRateSelectionPriority = getFrameRateSelectionPriority(),
+                .isSmallDirty = mSmallDirty,
         };
     };
     bool hasBuffer() const { return mBufferInfo.mBuffer != nullptr; }
@@ -917,6 +926,9 @@ public:
     // Exposed so SurfaceFlinger can assert that it's held
     const sp<SurfaceFlinger> mFlinger;
 
+    // Check if the damage region is a small dirty.
+    void setIsSmallDirty();
+
 protected:
     // For unit tests
     friend class TestableSurfaceFlinger;
diff --git a/services/surfaceflinger/Scheduler/LayerHistory.cpp b/services/surfaceflinger/Scheduler/LayerHistory.cpp
index beaf9724a3..a812ab74a6 100644
--- a/services/surfaceflinger/Scheduler/LayerHistory.cpp
+++ b/services/surfaceflinger/Scheduler/LayerHistory.cpp
@@ -320,4 +320,11 @@ auto LayerHistory::findLayer(int32_t id) -> std::pair<LayerStatus, LayerPair*> {
     return {LayerStatus::NotFound, nullptr};
 }
 
+bool LayerHistory::isSmallDirtyArea(uint32_t dirtyArea) const {
+    const float ratio = (float)dirtyArea / mDisplayArea;
+    const bool isSmallDirty = ratio <= kSmallDirtyArea;
+    ATRACE_FORMAT_INSTANT("small dirty=%s, ratio=%.3f", isSmallDirty ? "true" : "false", ratio);
+    return isSmallDirty;
+}
+
 } // namespace android::scheduler
diff --git a/services/surfaceflinger/Scheduler/LayerHistory.h b/services/surfaceflinger/Scheduler/LayerHistory.h
index 69caf9ffd2..06364156bb 100644
--- a/services/surfaceflinger/Scheduler/LayerHistory.h
+++ b/services/surfaceflinger/Scheduler/LayerHistory.h
@@ -43,6 +43,7 @@ struct LayerProps;
 class LayerHistory {
 public:
     using LayerVoteType = RefreshRateSelector::LayerVoteType;
+    static constexpr std::chrono::nanoseconds kMaxPeriodForHistory = 1s;
 
     LayerHistory();
     ~LayerHistory();
@@ -87,10 +88,14 @@ public:
     void attachChoreographer(int32_t layerId,
                              const sp<EventThreadConnection>& choreographerConnection);
 
+    bool isSmallDirtyArea(uint32_t dirtyArea) const;
+
 private:
     friend class LayerHistoryTest;
     friend class TestableScheduler;
 
+    static constexpr float kSmallDirtyArea = 0.07f;
+
     using LayerPair = std::pair<Layer*, std::unique_ptr<LayerInfo>>;
     // keyed by id as returned from Layer::getSequence()
     using LayerInfos = std::unordered_map<int32_t, LayerPair>;
diff --git a/services/surfaceflinger/Scheduler/LayerInfo.cpp b/services/surfaceflinger/Scheduler/LayerInfo.cpp
index bae3739501..348e2b9c72 100644
--- a/services/surfaceflinger/Scheduler/LayerInfo.cpp
+++ b/services/surfaceflinger/Scheduler/LayerInfo.cpp
@@ -63,7 +63,8 @@ void LayerInfo::setLastPresentTime(nsecs_t lastPresentTime, nsecs_t now, LayerUp
         case LayerUpdateType::Buffer:
             FrameTimeData frameTime = {.presentTime = lastPresentTime,
                                        .queueTime = mLastUpdatedTime,
-                                       .pendingModeChange = pendingModeChange};
+                                       .pendingModeChange = pendingModeChange,
+                                       .isSmallDirty = props.isSmallDirty};
             mFrameTimes.push_back(frameTime);
             if (mFrameTimes.size() > HISTORY_SIZE) {
                 mFrameTimes.pop_front();
@@ -99,11 +100,15 @@ LayerInfo::Frequent LayerInfo::isFrequent(nsecs_t now) const {
     // classification.
     bool isFrequent = true;
     bool isInfrequent = true;
+    int32_t smallDirtyCount = 0;
     const auto n = mFrameTimes.size() - 1;
     for (size_t i = 0; i < kFrequentLayerWindowSize - 1; i++) {
         if (mFrameTimes[n - i].queueTime - mFrameTimes[n - i - 1].queueTime <
             kMaxPeriodForFrequentLayerNs.count()) {
             isInfrequent = false;
+            if (mFrameTimes[n - i].presentTime == 0 && mFrameTimes[n - i].isSmallDirty) {
+                smallDirtyCount++;
+            }
         } else {
             isFrequent = false;
         }
@@ -113,7 +118,8 @@ LayerInfo::Frequent LayerInfo::isFrequent(nsecs_t now) const {
         // If the layer was previously inconclusive, we clear
         // the history as indeterminate layers changed to frequent,
         // and we should not look at the stale data.
-        return {isFrequent, isFrequent && !mIsFrequencyConclusive, /* isConclusive */ true};
+        return {isFrequent, isFrequent && !mIsFrequencyConclusive, /* isConclusive */ true,
+                /* isSmallDirty */ smallDirtyCount >= kNumSmallDirtyThreshold};
     }
 
     // If we can't determine whether the layer is frequent or not, we return
@@ -202,6 +208,7 @@ std::optional<nsecs_t> LayerInfo::calculateAverageFrameTime() const {
 
     nsecs_t totalDeltas = 0;
     int numDeltas = 0;
+    int32_t smallDirtyCount = 0;
     auto prevFrame = mFrameTimes.begin();
     for (auto it = mFrameTimes.begin() + 1; it != mFrameTimes.end(); ++it) {
         const auto currDelta = getFrameTime(*it) - getFrameTime(*prevFrame);
@@ -210,6 +217,13 @@ std::optional<nsecs_t> LayerInfo::calculateAverageFrameTime() const {
             continue;
         }
 
+        // If this is a small area update, we don't want to consider it for calculating the average
+        // frame time. Instead, we let the bigger frame updates to drive the calculation.
+        if (it->isSmallDirty && currDelta < kMinPeriodBetweenSmallDirtyFrames) {
+            smallDirtyCount++;
+            continue;
+        }
+
         prevFrame = it;
 
         if (currDelta > kMaxPeriodBetweenFrames) {
@@ -221,6 +235,10 @@ std::optional<nsecs_t> LayerInfo::calculateAverageFrameTime() const {
         numDeltas++;
     }
 
+    if (smallDirtyCount > 0) {
+        ATRACE_FORMAT_INSTANT("small dirty = %" PRIu32, smallDirtyCount);
+    }
+
     if (numDeltas == 0) {
         return std::nullopt;
     }
@@ -295,6 +313,13 @@ LayerInfo::LayerVote LayerInfo::getRefreshRateVote(const RefreshRateSelector& se
         clearHistory(now);
     }
 
+    // Return no vote if the latest frames are small dirty.
+    if (frequent.isSmallDirty && !mLastRefreshRate.reported.isValid()) {
+        ATRACE_FORMAT_INSTANT("NoVote (small dirty)");
+        ALOGV("%s is small dirty", mName.c_str());
+        return {LayerHistory::LayerVoteType::NoVote, Fps()};
+    }
+
     auto refreshRate = calculateRefreshRateIfPossible(selector, now);
     if (refreshRate.has_value()) {
         ALOGV("%s calculated refresh rate: %s", mName.c_str(), to_string(*refreshRate).c_str());
diff --git a/services/surfaceflinger/Scheduler/LayerInfo.h b/services/surfaceflinger/Scheduler/LayerInfo.h
index c5a60573f5..6a8580630d 100644
--- a/services/surfaceflinger/Scheduler/LayerInfo.h
+++ b/services/surfaceflinger/Scheduler/LayerInfo.h
@@ -57,6 +57,7 @@ class LayerInfo {
     static constexpr Fps kMinFpsForFrequentLayer = 10_Hz;
     static constexpr auto kMaxPeriodForFrequentLayerNs =
             std::chrono::nanoseconds(kMinFpsForFrequentLayer.getPeriodNsecs()) + 1ms;
+    static constexpr size_t kNumSmallDirtyThreshold = 2;
 
     friend class LayerHistoryTest;
     friend class LayerInfoTest;
@@ -195,6 +196,7 @@ private:
         nsecs_t presentTime; // desiredPresentTime, if provided
         nsecs_t queueTime;  // buffer queue time
         bool pendingModeChange;
+        bool isSmallDirty;
     };
 
     // Holds information about the calculated and reported refresh rate
@@ -259,6 +261,8 @@ private:
         bool clearHistory;
         // Represents whether we were able to determine isFrequent conclusively
         bool isConclusive;
+        // Represents whether the latest frames are small dirty.
+        bool isSmallDirty = false;
     };
     Frequent isFrequent(nsecs_t now) const;
     bool isAnimating(nsecs_t now) const;
@@ -277,6 +281,11 @@ private:
     // this period apart from each other, the interval between them won't be
     // taken into account when calculating average frame rate.
     static constexpr nsecs_t kMaxPeriodBetweenFrames = kMinFpsForFrequentLayer.getPeriodNsecs();
+    // Used for sanitizing the heuristic data. If frames are small dirty updating and are less
+    // than this period apart from each other, the interval between them won't be
+    // taken into account when calculating average frame rate.
+    static constexpr nsecs_t kMinPeriodBetweenSmallDirtyFrames = (60_Hz).getPeriodNsecs();
+
     LayerHistory::LayerVoteType mDefaultVote;
 
     LayerVote mLayerVote;
@@ -291,7 +300,7 @@ private:
     std::chrono::time_point<std::chrono::steady_clock> mFrameTimeValidSince =
             std::chrono::steady_clock::now();
     static constexpr size_t HISTORY_SIZE = RefreshRateHistory::HISTORY_SIZE;
-    static constexpr std::chrono::nanoseconds HISTORY_DURATION = 1s;
+    static constexpr std::chrono::nanoseconds HISTORY_DURATION = LayerHistory::kMaxPeriodForHistory;
 
     std::unique_ptr<LayerProps> mLayerProps;
 
@@ -309,6 +318,7 @@ struct LayerProps {
     ui::Transform transform;
     LayerInfo::FrameRate setFrameRateVote;
     int32_t frameRateSelectionPriority = -1;
+    bool isSmallDirty = false;
 };
 
 } // namespace scheduler
diff --git a/services/surfaceflinger/Scheduler/Scheduler.h b/services/surfaceflinger/Scheduler/Scheduler.h
index b9137003c5..d3c8079c1d 100644
--- a/services/surfaceflinger/Scheduler/Scheduler.h
+++ b/services/surfaceflinger/Scheduler/Scheduler.h
@@ -305,6 +305,16 @@ public:
         return mLayerHistory.getLayerFramerate(now, id);
     }
 
+    // Returns true if the small dirty detection is enabled.
+    bool supportSmallDirtyDetection() const {
+        return mFeatures.test(Feature::kSmallDirtyContentDetection);
+    }
+
+    // Returns true if the dirty area is less than threshold.
+    bool isSmallDirtyArea(uint32_t dirtyArea) const {
+        return mLayerHistory.isSmallDirtyArea(dirtyArea);
+    }
+
 private:
     friend class TestableScheduler;
 
diff --git a/services/surfaceflinger/Scheduler/include/scheduler/Features.h b/services/surfaceflinger/Scheduler/include/scheduler/Features.h
index 200407d1a6..7c72ac6afc 100644
--- a/services/surfaceflinger/Scheduler/include/scheduler/Features.h
+++ b/services/surfaceflinger/Scheduler/include/scheduler/Features.h
@@ -28,6 +28,7 @@ enum class Feature : std::uint8_t {
     kContentDetection = 1 << 2,
     kTracePredictedVsync = 1 << 3,
     kBackpressureGpuComposition = 1 << 4,
+    kSmallDirtyContentDetection = 1 << 5,
 };
 
 using FeatureFlags = ftl::Flags<Feature>;
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index 39ea248ea6..628da0ba12 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -3924,6 +3924,9 @@ void SurfaceFlinger::initScheduler(const sp<const DisplayDevice>& display) {
 
     if (sysprop::use_content_detection_for_refresh_rate(false)) {
         features |= Feature::kContentDetection;
+        if (base::GetBoolProperty("debug.sf.enable_small_dirty_detection"s, false)) {
+            features |= Feature::kSmallDirtyContentDetection;
+        }
     }
     if (base::GetBoolProperty("debug.sf.show_predicted_vsync"s, false)) {
         features |= Feature::kTracePredictedVsync;
@@ -7959,6 +7962,15 @@ void SurfaceFlinger::sample() {
 void SurfaceFlinger::onActiveDisplaySizeChanged(const DisplayDevice& activeDisplay) {
     mScheduler->onActiveDisplayAreaChanged(activeDisplay.getWidth() * activeDisplay.getHeight());
     getRenderEngine().onActiveDisplaySizeChanged(activeDisplay.getSize());
+
+    // Notify layers to update small dirty flag.
+    if (mScheduler->supportSmallDirtyDetection()) {
+        mCurrentState.traverse([&](Layer* layer) {
+            if (layer->getLayerStack() == activeDisplay.getLayerStack()) {
+                layer->setIsSmallDirty();
+            }
+        });
+    }
 }
 
 sp<DisplayDevice> SurfaceFlinger::getActivatableDisplay() const {
diff --git a/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp b/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp
index 85d86a7acc..be4b026b35 100644
--- a/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp
+++ b/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp
@@ -959,6 +959,77 @@ TEST_F(LayerHistoryTest, heuristicLayerNotOscillating) {
     recordFramesAndExpect(layer, time, 27.1_Hz, 30_Hz, PRESENT_TIME_HISTORY_SIZE);
 }
 
+TEST_F(LayerHistoryTest, smallDirtyLayer) {
+    auto layer = createLayer();
+
+    EXPECT_CALL(*layer, isVisible()).WillRepeatedly(Return(true));
+    EXPECT_CALL(*layer, getFrameRateForLayerTree()).WillRepeatedly(Return(Layer::FrameRate()));
+
+    nsecs_t time = systemTime();
+
+    EXPECT_EQ(1, layerCount());
+    EXPECT_EQ(0, activeLayerCount());
+    EXPECT_EQ(0, frequentLayerCount(time));
+
+    LayerHistory::Summary summary;
+
+    // layer is active but infrequent.
+    for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) {
+        auto props = layer->getLayerProps();
+        if (i % 3 == 0) {
+            props.isSmallDirty = false;
+        } else {
+            props.isSmallDirty = true;
+        }
+
+        history().record(layer->getSequence(), props, time, time,
+                         LayerHistory::LayerUpdateType::Buffer);
+        time += HI_FPS_PERIOD;
+        summary = summarizeLayerHistory(time);
+    }
+
+    ASSERT_EQ(1, summary.size());
+    ASSERT_EQ(LayerHistory::LayerVoteType::Heuristic, summary[0].vote);
+    EXPECT_GE(HI_FPS, summary[0].desiredRefreshRate);
+}
+
+TEST_F(LayerHistoryTest, smallDirtyInMultiLayer) {
+    auto layer1 = createLayer("UI");
+    auto layer2 = createLayer("Video");
+
+    EXPECT_CALL(*layer1, isVisible()).WillRepeatedly(Return(true));
+    EXPECT_CALL(*layer1, getFrameRateForLayerTree()).WillRepeatedly(Return(Layer::FrameRate()));
+
+    EXPECT_CALL(*layer2, isVisible()).WillRepeatedly(Return(true));
+    EXPECT_CALL(*layer2, getFrameRateForLayerTree())
+            .WillRepeatedly(
+                    Return(Layer::FrameRate(30_Hz, Layer::FrameRateCompatibility::Default)));
+
+    nsecs_t time = systemTime();
+
+    EXPECT_EQ(2, layerCount());
+    EXPECT_EQ(0, activeLayerCount());
+    EXPECT_EQ(0, frequentLayerCount(time));
+
+    LayerHistory::Summary summary;
+
+    // layer1 is active but infrequent.
+    for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) {
+        auto props = layer1->getLayerProps();
+        props.isSmallDirty = true;
+        history().record(layer1->getSequence(), props, 0 /*presentTime*/, time,
+                         LayerHistory::LayerUpdateType::Buffer);
+        history().record(layer2->getSequence(), layer2->getLayerProps(), time, time,
+                         LayerHistory::LayerUpdateType::Buffer);
+        time += HI_FPS_PERIOD;
+        summary = summarizeLayerHistory(time);
+    }
+
+    ASSERT_EQ(1, summary.size());
+    ASSERT_EQ(LayerHistory::LayerVoteType::ExplicitDefault, summary[0].vote);
+    ASSERT_EQ(30_Hz, summary[0].desiredRefreshRate);
+}
+
 class LayerHistoryTestParameterized : public LayerHistoryTest,
                                       public testing::WithParamInterface<std::chrono::nanoseconds> {
 };