Ensure insets for input are transformed correctly.

This change does the following:

1. Takes the inverse transform and transforms the final layer bounds.
This is to find the real bounds, including insets after insets have been
added.

2. Use the top and left of the real bounds to get the translation value
for the transform. This is done by applying the original transform on
the new left and top.

3. Set the translation of the input transform as the calculated values
from above.

4. Send the inverse transform to input so the input coordinates can be
transformed to window space.

Also use the transform to correctly update the touchable regions in the
layer space.

Test: libgui_tests
Test: Rotated layer gets correct input
Fixes: 158802274
Change-Id: Ib0bfaa043509032ed702f442d806097bfd48d418

3 files changed, 112 insertions, 16 deletions

diff --git a/libs/gui/tests/EndToEndNativeInputTest.cpp b/libs/gui/tests/EndToEndNativeInputTest.cpp
index ac795830f3..ef4d870ec7 100644
--- a/libs/gui/tests/EndToEndNativeInputTest.cpp
+++ b/libs/gui/tests/EndToEndNativeInputTest.cpp
@@ -533,4 +533,73 @@ TEST_F(InputSurfacesTest, can_be_focused) {
 
     surface->assertFocusChange(true);
 }
+
+TEST_F(InputSurfacesTest, rotate_surface) {
+    std::unique_ptr<InputSurface> surface = makeSurface(100, 100);
+    surface->showAt(10, 10);
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, 0, 1, -1, 0); // 90 degrees
+    });
+    injectTap(8, 11);
+    surface->expectTap(1, 2);
+
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, -1, 0, 0, -1); // 180 degrees
+    });
+    injectTap(9, 8);
+    surface->expectTap(1, 2);
+
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, 0, -1, 1, 0); // 270 degrees
+    });
+    injectTap(12, 9);
+    surface->expectTap(1, 2);
+}
+
+TEST_F(InputSurfacesTest, rotate_surface_with_scale) {
+    std::unique_ptr<InputSurface> surface = makeSurface(100, 100);
+    surface->showAt(10, 10);
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, 0, 2, -4, 0); // 90 degrees
+    });
+    injectTap(2, 12);
+    surface->expectTap(1, 2);
+
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, -2, 0, 0, -4); // 180 degrees
+    });
+    injectTap(8, 2);
+    surface->expectTap(1, 2);
+
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, 0, -2, 4, 0); // 270 degrees
+    });
+    injectTap(18, 8);
+    surface->expectTap(1, 2);
+}
+
+TEST_F(InputSurfacesTest, rotate_surface_with_scale_and_insets) {
+    std::unique_ptr<InputSurface> surface = makeSurface(100, 100);
+    surface->mInputInfo.surfaceInset = 5;
+    surface->showAt(100, 100);
+
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, 0, 2, -4, 0); // 90 degrees
+    });
+    injectTap(40, 120);
+    surface->expectTap(5, 10);
+
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, -2, 0, 0, -4); // 180 degrees
+    });
+    injectTap(80, 40);
+    surface->expectTap(5, 10);
+
+    surface->doTransaction([](auto &t, auto &sc) {
+        t.setMatrix(sc, 0, -2, 4, 0); // 270 degrees
+    });
+    injectTap(160, 80);
+    surface->expectTap(5, 10);
+}
+
 } // namespace android::test
diff --git a/libs/ui/Transform.cpp b/libs/ui/Transform.cpp
index ec8a78afdc..cd68c1c0ec 100644
--- a/libs/ui/Transform.cpp
+++ b/libs/ui/Transform.cpp
@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 
+#undef LOG_TAG
+#define LOG_TAG "Transform"
+
 #include <math.h>
 
 #include <android-base/stringprintf.h>
@@ -132,7 +135,7 @@ float Transform::dsdy() const {
 }
 
 float Transform::getScaleX() const {
-    return sqrt(dsdx() * dsdx()) + (dtdx() * dtdx());
+    return sqrt((dsdx() * dsdx()) + (dtdx() * dtdx()));
 }
 
 float Transform::getScaleY() const {
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 09ec819d94..a324d56344 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -2416,40 +2416,64 @@ InputWindowInfo Layer::fillInputInfo() {
     const float xScale = t.getScaleX();
     const float yScale = t.getScaleY();
     if (xScale != 1.0f || yScale != 1.0f) {
-        info.touchableRegion.scaleSelf(xScale, yScale);
         xSurfaceInset = std::round(xSurfaceInset * xScale);
         ySurfaceInset = std::round(ySurfaceInset * yScale);
     }
 
-    layerBounds = t.transform(layerBounds);
+    Rect transformedLayerBounds = t.transform(layerBounds);
 
     // clamp inset to layer bounds
-    xSurfaceInset = (xSurfaceInset >= 0) ? std::min(xSurfaceInset, layerBounds.getWidth() / 2) : 0;
-    ySurfaceInset = (ySurfaceInset >= 0) ? std::min(ySurfaceInset, layerBounds.getHeight() / 2) : 0;
+    xSurfaceInset = (xSurfaceInset >= 0)
+            ? std::min(xSurfaceInset, transformedLayerBounds.getWidth() / 2)
+            : 0;
+    ySurfaceInset = (ySurfaceInset >= 0)
+            ? std::min(ySurfaceInset, transformedLayerBounds.getHeight() / 2)
+            : 0;
 
     // inset while protecting from overflow TODO(b/161235021): What is going wrong
     // in the overflow scenario?
     {
     int32_t tmp;
-    if (!__builtin_add_overflow(layerBounds.left, xSurfaceInset, &tmp)) layerBounds.left = tmp;
-    if (!__builtin_sub_overflow(layerBounds.right, xSurfaceInset, &tmp)) layerBounds.right = tmp;
-    if (!__builtin_add_overflow(layerBounds.top, ySurfaceInset, &tmp)) layerBounds.top = tmp;
-    if (!__builtin_sub_overflow(layerBounds.bottom, ySurfaceInset, &tmp)) layerBounds.bottom = tmp;
+    if (!__builtin_add_overflow(transformedLayerBounds.left, xSurfaceInset, &tmp))
+        transformedLayerBounds.left = tmp;
+    if (!__builtin_sub_overflow(transformedLayerBounds.right, xSurfaceInset, &tmp))
+        transformedLayerBounds.right = tmp;
+    if (!__builtin_add_overflow(transformedLayerBounds.top, ySurfaceInset, &tmp))
+        transformedLayerBounds.top = tmp;
+    if (!__builtin_sub_overflow(transformedLayerBounds.bottom, ySurfaceInset, &tmp))
+        transformedLayerBounds.bottom = tmp;
     }
 
     // Input coordinate should match the layer bounds.
-    info.frameLeft = layerBounds.left;
-    info.frameTop = layerBounds.top;
-    info.frameRight = layerBounds.right;
-    info.frameBottom = layerBounds.bottom;
-
+    info.frameLeft = transformedLayerBounds.left;
+    info.frameTop = transformedLayerBounds.top;
+    info.frameRight = transformedLayerBounds.right;
+    info.frameBottom = transformedLayerBounds.bottom;
+
+    // Compute the correct transform to send to input. This will allow it to transform the
+    // input coordinates from display space into window space. Therefore, it needs to use the
+    // final layer frame to create the inverse transform. Since surface insets are added later,
+    // along with the overflow, the best way to ensure we get the correct transform is to use
+    // the final frame calculated.
+    // 1. Take the original transform set on the window and get the inverse transform. This is
+    //    used to get the final bounds in display space (ignorning the transform). Apply the
+    //    inverse transform on the layerBounds to get the untransformed frame (in display space)
+    // 2. Take the top and left of the untransformed frame to get the real position on screen.
+    //    Apply the layer transform on top/left so it includes any scale or rotation. These will
+    //    be the new translation values for the transform.
+    // 3. Update the translation of the original transform to the new translation values.
+    // 4. Send the inverse transform to input so the coordinates can be transformed back into
+    //    window space.
+    ui::Transform inverseTransform = t.inverse();
+    Rect nonTransformedBounds = inverseTransform.transform(transformedLayerBounds);
+    vec2 translation = t.transform(nonTransformedBounds.left, nonTransformedBounds.top);
     ui::Transform inputTransform(t);
-    inputTransform.set(layerBounds.left, layerBounds.top);
+    inputTransform.set(translation.x, translation.y);
     info.transform = inputTransform.inverse();
 
     // Position the touchable region relative to frame screen location and restrict it to frame
     // bounds.
-    info.touchableRegion = info.touchableRegion.translate(info.frameLeft, info.frameTop);
+    info.touchableRegion = inputTransform.transform(info.touchableRegion);
     // For compatibility reasons we let layers which can receive input
     // receive input before they have actually submitted a buffer. Because
     // of this we use canReceiveInput instead of isVisible to check the