Fix issue where surface corner rounding is ignored

It's not correct to only check the diagonal of the transform matrix
for scaling. The scale will be distributed across four components
when the layer is rotated.

Test: manual
Test: atest LayerTypeAndRenderTypeTransaction
Fixes: 147415720
Change-Id: I140b373efd7fad705d0cd54aa6e86b4142e190e5

2 files changed, 48 insertions, 1 deletions

diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 96552770c4..dcc213f943 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -2122,7 +2122,9 @@ Layer::RoundedCornerState Layer::getRoundedCornerState() const {
             // but a transform matrix can define horizontal and vertical scales.
             // Let's take the average between both of them and pass into the shader, practically we
             // never do this type of transformation on windows anyway.
-            parentState.radius *= (t[0][0] + t[1][1]) / 2.0f;
+            auto scaleX = sqrtf(t[0][0] * t[0][0] + t[0][1] * t[0][1]);
+            auto scaleY = sqrtf(t[1][0] * t[1][0] + t[1][1] * t[1][1]);
+            parentState.radius *= (scaleX + scaleY) / 2.0f;
             return parentState;
         }
     }
diff --git a/services/surfaceflinger/tests/LayerTypeAndRenderTypeTransaction_test.cpp b/services/surfaceflinger/tests/LayerTypeAndRenderTypeTransaction_test.cpp
index d666d7e01a..7d4314f4df 100644
--- a/services/surfaceflinger/tests/LayerTypeAndRenderTypeTransaction_test.cpp
+++ b/services/surfaceflinger/tests/LayerTypeAndRenderTypeTransaction_test.cpp
@@ -220,6 +220,51 @@ TEST_P(LayerTypeAndRenderTypeTransactionTest, SetCornerRadius) {
         shot->expectColor(Rect(size - testArea, 0, right, testArea), Color::BLACK);
         shot->expectColor(Rect(0, bottom - testArea, testArea, bottom), Color::BLACK);
         shot->expectColor(Rect(size - testArea, bottom - testArea, right, bottom), Color::BLACK);
+        // Solid center
+        shot->expectColor(Rect(size / 2 - testArea / 2, size / 2 - testArea / 2,
+                               size / 2 + testArea / 2, size / 2 + testArea / 2),
+                          Color::RED);
+    }
+}
+
+TEST_P(LayerTypeAndRenderTypeTransactionTest, SetCornerRadiusRotated) {
+    sp<SurfaceControl> parent;
+    sp<SurfaceControl> child;
+    const uint8_t size = 64;
+    const uint8_t testArea = 4;
+    const float cornerRadius = 20.0f;
+    ASSERT_NO_FATAL_FAILURE(parent = createLayer("parent", size, size));
+    ASSERT_NO_FATAL_FAILURE(fillLayerColor(parent, Color::RED, size, size));
+    ASSERT_NO_FATAL_FAILURE(child = createLayer("child", size, size));
+    ASSERT_NO_FATAL_FAILURE(fillLayerColor(child, Color::GREEN, size, size));
+
+    auto transaction = Transaction()
+                               .setCornerRadius(parent, cornerRadius)
+                               .setCrop_legacy(parent, Rect(0, 0, size, size))
+                               .reparent(child, parent->getHandle())
+                               .setPosition(child, 0, size)
+                               // Rotate by half PI
+                               .setMatrix(child, 0.0f, -1.0f, 1.0f, 0.0f);
+    if (mLayerType == ISurfaceComposerClient::eFXSurfaceBufferQueue) {
+        transaction.setCrop_legacy(parent, Rect(0, 0, size, size));
+    } else {
+        transaction.setFrame(parent, Rect(0, 0, size, size));
+    }
+    transaction.apply();
+
+    {
+        const uint8_t bottom = size - 1;
+        const uint8_t right = size - 1;
+        auto shot = getScreenCapture();
+        // Edges are transparent
+        shot->expectColor(Rect(0, 0, testArea, testArea), Color::BLACK);
+        shot->expectColor(Rect(size - testArea, 0, right, testArea), Color::BLACK);
+        shot->expectColor(Rect(0, bottom - testArea, testArea, bottom - testArea), Color::BLACK);
+        shot->expectColor(Rect(right - testArea, bottom - testArea, right, bottom), Color::BLACK);
+        // Solid center
+        shot->expectColor(Rect(size / 2 - testArea / 2, size / 2 - testArea / 2,
+                               size / 2 + testArea / 2, size / 2 + testArea / 2),
+                          Color::GREEN);
     }
 }