SF: Introduce LayerFE::prepareClientComposition

Renames Layer::prepareClientLayer, and alters it to not take a
RenderArea.

Test: atest libsurfaceflinger_unittest libcompositionengine_test
Bug: 121291683
Change-Id: I6da6005c2e34e6e44692043cdc7adf9c4aa57cf5

11 files changed, 141 insertions, 130 deletions

diff --git a/services/surfaceflinger/BufferLayer.cpp b/services/surfaceflinger/BufferLayer.cpp
index 59ea9afa97..87bec11609 100644
--- a/services/surfaceflinger/BufferLayer.cpp
+++ b/services/surfaceflinger/BufferLayer.cpp
@@ -132,13 +132,15 @@ static constexpr mat4 inverseOrientation(uint32_t transform) {
     return inverse(tr);
 }
 
-bool BufferLayer::prepareClientLayer(const RenderArea& renderArea, const Region& clip,
-                                     bool useIdentityTransform, Region& clearRegion,
-                                     const bool supportProtectedContent,
-                                     renderengine::LayerSettings& layer) {
+std::optional<renderengine::LayerSettings> BufferLayer::prepareClientComposition(
+        compositionengine::LayerFE::ClientCompositionTargetSettings& targetSettings) {
     ATRACE_CALL();
-    Layer::prepareClientLayer(renderArea, clip, useIdentityTransform, clearRegion,
-                              supportProtectedContent, layer);
+
+    auto result = Layer::prepareClientComposition(targetSettings);
+    if (!result) {
+        return result;
+    }
+
     if (CC_UNLIKELY(mActiveBuffer == 0)) {
         // the texture has not been created yet, this Layer has
         // in fact never been drawn into. This happens frequently with
@@ -159,15 +161,16 @@ bool BufferLayer::prepareClientLayer(const RenderArea& renderArea, const Region&
             under.orSelf(layer->visibleRegion);
         });
         // if not everything below us is covered, we plug the holes!
-        Region holes(clip.subtract(under));
+        Region holes(targetSettings.clip.subtract(under));
         if (!holes.isEmpty()) {
-            clearRegion.orSelf(holes);
+            targetSettings.clearRegion.orSelf(holes);
         }
-        return false;
+        return std::nullopt;
     }
-    bool blackOutLayer =
-            (isProtected() && !supportProtectedContent) || (isSecure() && !renderArea.isSecure());
+    bool blackOutLayer = (isProtected() && !targetSettings.supportProtectedContent) ||
+            (isSecure() && !targetSettings.isSecure);
     const State& s(getDrawingState());
+    auto& layer = *result;
     if (!blackOutLayer) {
         layer.source.buffer.buffer = mActiveBuffer;
         layer.source.buffer.isOpaque = isOpaque(s);
@@ -176,8 +179,7 @@ bool BufferLayer::prepareClientLayer(const RenderArea& renderArea, const Region&
         layer.source.buffer.usePremultipliedAlpha = getPremultipledAlpha();
         layer.source.buffer.isY410BT2020 = isHdrY410();
         // TODO: we could be more subtle with isFixedSize()
-        const bool useFiltering = needsFiltering(renderArea.getDisplayDevice()) ||
-                renderArea.needsFiltering() || isFixedSize();
+        const bool useFiltering = targetSettings.needsFiltering || mNeedsFiltering || isFixedSize();
 
         // Query the texture matrix given our current filtering mode.
         float textureMatrix[16];
@@ -244,7 +246,7 @@ bool BufferLayer::prepareClientLayer(const RenderArea& renderArea, const Region&
         layer.alpha = 1.0;
     }
 
-    return true;
+    return result;
 }
 
 bool BufferLayer::isHdrY410() const {
@@ -572,21 +574,23 @@ bool BufferLayer::getOpacityForFormat(uint32_t format) {
 }
 
 bool BufferLayer::needsFiltering(const sp<const DisplayDevice>& displayDevice) const {
-    // If we are not capturing based on the state of a known display device, we
-    // only return mNeedsFiltering
+    // If we are not capturing based on the state of a known display device,
+    // just return false.
     if (displayDevice == nullptr) {
-        return mNeedsFiltering;
+        return false;
     }
 
     const auto outputLayer = findOutputLayerForDisplay(displayDevice);
     if (outputLayer == nullptr) {
-        return mNeedsFiltering;
+        return false;
     }
 
+    // We need filtering if the sourceCrop rectangle size does not match the
+    // displayframe rectangle size (not a 1:1 render)
     const auto& compositionState = outputLayer->getState();
     const auto displayFrame = compositionState.displayFrame;
     const auto sourceCrop = compositionState.sourceCrop;
-    return mNeedsFiltering || sourceCrop.getHeight() != displayFrame.getHeight() ||
+    return sourceCrop.getHeight() != displayFrame.getHeight() ||
             sourceCrop.getWidth() != displayFrame.getWidth();
 }
 
diff --git a/services/surfaceflinger/BufferLayer.h b/services/surfaceflinger/BufferLayer.h
index bb0205de3e..c86acf08f8 100644
--- a/services/surfaceflinger/BufferLayer.h
+++ b/services/surfaceflinger/BufferLayer.h
@@ -82,7 +82,6 @@ public:
 
     bool isHdrY410() const override;
 
-    bool onPreComposition(nsecs_t refreshStartTime) override;
     bool onPostComposition(const std::optional<DisplayId>& displayId,
                            const std::shared_ptr<FenceTime>& glDoneFence,
                            const std::shared_ptr<FenceTime>& presentFence,
@@ -146,7 +145,13 @@ private:
     virtual status_t updateFrameNumber(nsecs_t latchTime) = 0;
 
 protected:
-    void latchPerFrameState(compositionengine::LayerFECompositionState& outState) const override;
+    /*
+     * compositionengine::LayerFE overrides
+     */
+    bool onPreComposition(nsecs_t) override;
+    void latchPerFrameState(compositionengine::LayerFECompositionState&) const override;
+    std::optional<renderengine::LayerSettings> prepareClientComposition(
+            compositionengine::LayerFE::ClientCompositionTargetSettings&) override;
 
     // Loads the corresponding system property once per process
     static bool latchUnsignaledBuffers();
@@ -163,15 +168,9 @@ protected:
 
     bool mRefreshPending{false};
 
-    // prepareClientLayer - constructs a RenderEngine layer for GPU composition.
-    bool prepareClientLayer(const RenderArea& renderArea, const Region& clip,
-                            bool useIdentityTransform, Region& clearRegion,
-                            const bool supportProtectedContent,
-                            renderengine::LayerSettings& layer) override;
-
 private:
     // Returns true if this layer requires filtering
-    bool needsFiltering(const sp<const DisplayDevice>& displayDevice) const;
+    bool needsFiltering(const sp<const DisplayDevice>& displayDevice) const override;
 
     uint64_t getHeadFrameNumber(nsecs_t expectedPresentTime) const;
 
diff --git a/services/surfaceflinger/ColorLayer.cpp b/services/surfaceflinger/ColorLayer.cpp
index f15957adf9..b65d351263 100644
--- a/services/surfaceflinger/ColorLayer.cpp
+++ b/services/surfaceflinger/ColorLayer.cpp
@@ -50,16 +50,14 @@ ColorLayer::ColorLayer(const LayerCreationArgs& args)
 
 ColorLayer::~ColorLayer() = default;
 
-bool ColorLayer::prepareClientLayer(const RenderArea& renderArea, const Region& clip,
-                                    bool useIdentityTransform, Region& clearRegion,
-                                    const bool supportProtectedContent,
-                                    renderengine::LayerSettings& layer) {
-    Layer::prepareClientLayer(renderArea, clip, useIdentityTransform, clearRegion,
-                              supportProtectedContent, layer);
-    half4 color(getColor());
-    half3 solidColor(color.r, color.g, color.b);
-    layer.source.solidColor = solidColor;
-    return true;
+std::optional<renderengine::LayerSettings> ColorLayer::prepareClientComposition(
+        compositionengine::LayerFE::ClientCompositionTargetSettings& targetSettings) {
+    auto result = Layer::prepareClientComposition(targetSettings);
+    if (!result) {
+        return result;
+    }
+    result->source.solidColor = getColor().rgb;
+    return result;
 }
 
 bool ColorLayer::isVisible() const {
diff --git a/services/surfaceflinger/ColorLayer.h b/services/surfaceflinger/ColorLayer.h
index 2483ff0bfb..015b939457 100644
--- a/services/surfaceflinger/ColorLayer.h
+++ b/services/surfaceflinger/ColorLayer.h
@@ -39,16 +39,13 @@ public:
 
     void commitTransaction(const State& stateToCommit) override;
 
-    bool onPreComposition(nsecs_t /*refreshStartTime*/) override { return false; }
-
 protected:
-    virtual bool prepareClientLayer(const RenderArea& renderArea, const Region& clip,
-                                    bool useIdentityTransform, Region& clearRegion,
-                                    const bool supportProtectedContent,
-                                    renderengine::LayerSettings& layer);
-
-private:
+    /*
+     * compositionengine::LayerFE overrides
+     */
     void latchPerFrameState(compositionengine::LayerFECompositionState&) const override;
+    std::optional<renderengine::LayerSettings> prepareClientComposition(
+            compositionengine::LayerFE::ClientCompositionTargetSettings&) override;
 
     std::shared_ptr<compositionengine::Layer> mCompositionLayer;
 };
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
index 1f2cae9c2b..94fab1fdf6 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
@@ -16,6 +16,9 @@
 
 #pragma once
 
+#include <optional>
+
+#include <renderengine/LayerSettings.h>
 #include <utils/RefBase.h>
 #include <utils/Timers.h>
 
@@ -40,6 +43,34 @@ public:
     // geometry state can be skipped.
     virtual void latchCompositionState(LayerFECompositionState&, bool includeGeometry) const = 0;
 
+    struct ClientCompositionTargetSettings {
+        // The clip region, or visible region that is being rendered to
+        const Region& clip;
+
+        // If true, the layer should use an identity transform for its position
+        // transform. Used only by the captureScreen API call.
+        const bool useIdentityTransform;
+
+        // If set to true, the layer should enable filtering when rendering.
+        const bool needsFiltering;
+
+        // If set to true, the buffer is being sent to a destination that is
+        // expected to treat the buffer contents as secure.
+        const bool isSecure;
+
+        // If set to true, the target buffer has protected content support.
+        const bool supportProtectedContent;
+
+        // Modified by each call to prepareClientComposition to indicate the
+        // region of the target buffer that should be cleared.
+        Region& clearRegion;
+    };
+
+    // Returns the LayerSettings to pass to RenderEngine::drawLayers, or
+    // nullopt_t if the layer does not render
+    virtual std::optional<renderengine::LayerSettings> prepareClientComposition(
+            ClientCompositionTargetSettings&) = 0;
+
     // Called after the layer is displayed to update the presentation fence
     virtual void onLayerDisplayed(const sp<Fence>&) = 0;
 
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
index 952f702e1c..48c2dbf3d6 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
@@ -33,6 +33,9 @@ public:
     MOCK_METHOD1(onPreComposition, bool(nsecs_t));
 
     MOCK_CONST_METHOD2(latchCompositionState, void(LayerFECompositionState&, bool));
+    MOCK_METHOD1(prepareClientComposition,
+                 std::optional<renderengine::LayerSettings>(
+                         compositionengine::LayerFE::ClientCompositionTargetSettings&));
     MOCK_METHOD1(onLayerDisplayed, void(const sp<Fence>&));
 
     MOCK_CONST_METHOD0(getDebugName, const char*());
diff --git a/services/surfaceflinger/ContainerLayer.cpp b/services/surfaceflinger/ContainerLayer.cpp
index 3a5f3faa20..d40a38c811 100644
--- a/services/surfaceflinger/ContainerLayer.cpp
+++ b/services/surfaceflinger/ContainerLayer.cpp
@@ -26,11 +26,6 @@ ContainerLayer::ContainerLayer(const LayerCreationArgs& args) : Layer(args) {}
 
 ContainerLayer::~ContainerLayer() = default;
 
-bool ContainerLayer::prepareClientLayer(const RenderArea&, const Region&, bool, Region&, const bool,
-                                        renderengine::LayerSettings&) {
-    return false;
-}
-
 bool ContainerLayer::isVisible() const {
     return false;
 }
diff --git a/services/surfaceflinger/ContainerLayer.h b/services/surfaceflinger/ContainerLayer.h
index 57267c70c8..a1607ffabb 100644
--- a/services/surfaceflinger/ContainerLayer.h
+++ b/services/surfaceflinger/ContainerLayer.h
@@ -34,14 +34,6 @@ public:
     bool canReceiveInput() const override;
 
     bool isCreatedFromMainThread() const override { return true; }
-
-    bool onPreComposition(nsecs_t /*refreshStartTime*/) override { return false; }
-
-protected:
-    bool prepareClientLayer(const RenderArea& renderArea, const Region& clip,
-                            bool useIdentityTransform, Region& clearRegion,
-                            const bool supportProtectedContent,
-                            renderengine::LayerSettings& layer) override;
 };
 
 } // namespace android
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 8150ca7dfc..a2eeea545b 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -458,6 +458,10 @@ void Layer::latchPerFrameState(compositionengine::LayerFECompositionState& compo
     }
 }
 
+bool Layer::onPreComposition(nsecs_t) {
+    return false;
+}
+
 void Layer::latchCompositionState(compositionengine::LayerFECompositionState& compositionState,
                                   bool includeGeometry) const {
     if (includeGeometry) {
@@ -507,54 +511,33 @@ void Layer::updateCursorPosition(const sp<const DisplayDevice>& display) {
 // drawing...
 // ---------------------------------------------------------------------------
 
-bool Layer::prepareClientLayer(const RenderArea& renderArea, const Region& clip,
-                               Region& clearRegion, const bool supportProtectedContent,
-                               renderengine::LayerSettings& layer) {
-    return prepareClientLayer(renderArea, clip, false, clearRegion, supportProtectedContent, layer);
-}
-
-bool Layer::prepareClientLayer(const RenderArea& renderArea, bool useIdentityTransform,
-                               Region& clearRegion, const bool supportProtectedContent,
-                               renderengine::LayerSettings& layer) {
-    return prepareClientLayer(renderArea, Region(renderArea.getBounds()), useIdentityTransform,
-                              clearRegion, supportProtectedContent, layer);
-}
+std::optional<renderengine::LayerSettings> Layer::prepareClientComposition(
+        compositionengine::LayerFE::ClientCompositionTargetSettings& targetSettings) {
+    if (!getCompositionLayer()) {
+        return {};
+    }
 
-bool Layer::prepareClientLayer(const RenderArea& /*renderArea*/, const Region& /*clip*/,
-                               bool useIdentityTransform, Region& /*clearRegion*/,
-                               const bool /*supportProtectedContent*/,
-                               renderengine::LayerSettings& layer) {
     FloatRect bounds = getBounds();
     half alpha = getAlpha();
-    layer.geometry.boundaries = bounds;
-    if (useIdentityTransform) {
-        layer.geometry.positionTransform = mat4();
+    renderengine::LayerSettings layerSettings;
+    layerSettings.geometry.boundaries = bounds;
+    if (targetSettings.useIdentityTransform) {
+        layerSettings.geometry.positionTransform = mat4();
     } else {
-        const ui::Transform transform = getTransform();
-        mat4 m;
-        m[0][0] = transform[0][0];
-        m[0][1] = transform[0][1];
-        m[0][3] = transform[0][2];
-        m[1][0] = transform[1][0];
-        m[1][1] = transform[1][1];
-        m[1][3] = transform[1][2];
-        m[3][0] = transform[2][0];
-        m[3][1] = transform[2][1];
-        m[3][3] = transform[2][2];
-        layer.geometry.positionTransform = m;
+        layerSettings.geometry.positionTransform = getTransform().asMatrix4();
     }
 
     if (hasColorTransform()) {
-        layer.colorTransform = getColorTransform();
+        layerSettings.colorTransform = getColorTransform();
     }
 
     const auto roundedCornerState = getRoundedCornerState();
-    layer.geometry.roundedCornersRadius = roundedCornerState.radius;
-    layer.geometry.roundedCornersCrop = roundedCornerState.cropRect;
+    layerSettings.geometry.roundedCornersRadius = roundedCornerState.radius;
+    layerSettings.geometry.roundedCornersCrop = roundedCornerState.cropRect;
 
-    layer.alpha = alpha;
-    layer.sourceDataspace = mCurrentDataSpace;
-    return true;
+    layerSettings.alpha = alpha;
+    layerSettings.sourceDataspace = mCurrentDataSpace;
+    return layerSettings;
 }
 
 Hwc2::IComposerClient::Composition Layer::getCompositionType(
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index aa7e2f1ec3..953f25ddf1 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -461,19 +461,17 @@ public:
         return s.activeTransparentRegion_legacy;
     }
     virtual Rect getCrop(const Layer::State& s) const { return s.crop_legacy; }
-
-protected:
-    virtual bool prepareClientLayer(const RenderArea& renderArea, const Region& clip,
-                                    bool useIdentityTransform, Region& clearRegion,
-                                    const bool supportProtectedContent,
-                                    renderengine::LayerSettings& layer);
+    virtual bool needsFiltering(const sp<const DisplayDevice>&) const { return false; }
 
 public:
     /*
      * compositionengine::LayerFE overrides
      */
+    bool onPreComposition(nsecs_t) override;
     void latchCompositionState(compositionengine::LayerFECompositionState&,
                                bool includeGeometry) const override;
+    std::optional<renderengine::LayerSettings> prepareClientComposition(
+            compositionengine::LayerFE::ClientCompositionTargetSettings&) override;
     void onLayerDisplayed(const sp<Fence>& releaseFence) override;
     const char* getDebugName() const override;
 
@@ -509,17 +507,6 @@ public:
     virtual void releasePendingBuffer(nsecs_t /*dequeueReadyTime*/) { }
 
     /*
-     * prepareClientLayer - populates a renderengine::LayerSettings to passed to
-     * RenderEngine::drawLayers. Returns true if the layer can be used, and
-     * false otherwise.
-     */
-    bool prepareClientLayer(const RenderArea& renderArea, const Region& clip, Region& clearRegion,
-                            const bool supportProtectedContent, renderengine::LayerSettings& layer);
-    bool prepareClientLayer(const RenderArea& renderArea, bool useIdentityTransform,
-                            Region& clearRegion, const bool supportProtectedContent,
-                            renderengine::LayerSettings& layer);
-
-    /*
      * doTransaction - process the transaction. This is a good place to figure
      * out which attributes of the surface have changed.
      */
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index 1aec0c5389..d877e77832 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -3319,6 +3319,7 @@ bool SurfaceFlinger::doComposeSurfaces(const sp<DisplayDevice>& displayDevice,
      */
 
     ALOGV("Rendering client layers");
+    const bool useIdentityTransform = false;
     bool firstLayer = true;
     Region clearRegion = Region::INVALID_REGION;
     for (auto& layer : displayDevice->getVisibleLayersSortedByZ()) {
@@ -3339,13 +3340,20 @@ bool SurfaceFlinger::doComposeSurfaces(const sp<DisplayDevice>& displayDevice,
                         layer->getRoundedCornerState().radius == 0.0f && hasClientComposition) {
                         // never clear the very first layer since we're
                         // guaranteed the FB is already cleared
-                        renderengine::LayerSettings layerSettings;
                         Region dummyRegion;
-                        bool prepared =
-                                layer->prepareClientLayer(renderArea, clip, dummyRegion,
-                                                          supportProtectedContent, layerSettings);
-
-                        if (prepared) {
+                        compositionengine::LayerFE::ClientCompositionTargetSettings targetSettings{
+                                clip,
+                                useIdentityTransform,
+                                layer->needsFiltering(renderArea.getDisplayDevice()) ||
+                                        renderArea.needsFiltering(),
+                                renderArea.isSecure(),
+                                supportProtectedContent,
+                                dummyRegion,
+                        };
+                        auto result = layer->prepareClientComposition(targetSettings);
+
+                        if (result) {
+                            auto& layerSettings = *result;
                             layerSettings.source.buffer.buffer = nullptr;
                             layerSettings.source.solidColor = half3(0.0, 0.0, 0.0);
                             layerSettings.alpha = half(0.0);
@@ -3356,12 +3364,18 @@ bool SurfaceFlinger::doComposeSurfaces(const sp<DisplayDevice>& displayDevice,
                     break;
                 }
                 case Hwc2::IComposerClient::Composition::CLIENT: {
-                    renderengine::LayerSettings layerSettings;
-                    bool prepared =
-                            layer->prepareClientLayer(renderArea, clip, clearRegion,
-                                                      supportProtectedContent, layerSettings);
-                    if (prepared) {
-                        clientCompositionLayers.push_back(layerSettings);
+                    compositionengine::LayerFE::ClientCompositionTargetSettings targetSettings{
+                            clip,
+                            useIdentityTransform,
+                            layer->needsFiltering(renderArea.getDisplayDevice()) ||
+                                    renderArea.needsFiltering(),
+                            renderArea.isSecure(),
+                            supportProtectedContent,
+                            clearRegion,
+                    };
+                    auto result = layer->prepareClientComposition(targetSettings);
+                    if (result) {
+                        clientCompositionLayers.push_back(*result);
                     }
                     break;
                 }
@@ -5902,11 +5916,19 @@ void SurfaceFlinger::renderScreenImplLocked(const RenderArea& renderArea,
 
     Region clearRegion = Region::INVALID_REGION;
     traverseLayers([&](Layer* layer) {
-        renderengine::LayerSettings layerSettings;
-        bool prepared = layer->prepareClientLayer(renderArea, useIdentityTransform, clearRegion,
-                                                  false, layerSettings);
-        if (prepared) {
-            clientCompositionLayers.push_back(layerSettings);
+        const bool supportProtectedContent = false;
+        Region clip(renderArea.getBounds());
+        compositionengine::LayerFE::ClientCompositionTargetSettings targetSettings{
+                clip,
+                useIdentityTransform,
+                layer->needsFiltering(renderArea.getDisplayDevice()) || renderArea.needsFiltering(),
+                renderArea.isSecure(),
+                supportProtectedContent,
+                clearRegion,
+        };
+        auto result = layer->prepareClientComposition(targetSettings);
+        if (result) {
+            clientCompositionLayers.push_back(*result);
         }
     });