Use the cloned window's layer stack space for raw coordinates

Use the cloned window's layer stack (screen) space as the
raw coordinate space for input going to clones. This ensures the raw
coordinates are consistent between the clone and cloned windows while
mirroring.

Flag: com.android.input.flags.use_cloned_screen_coordinates_as_raw
Test: Presubmit
Bug: 377846505
Change-Id: I7370ac64585315d9c754eaa1eb0077c8555fc6a9

9 files changed, 162 insertions, 46 deletions

diff --git a/libs/gui/WindowInfo.cpp b/libs/gui/WindowInfo.cpp
index 82d2554340..6a32d55d25 100644
--- a/libs/gui/WindowInfo.cpp
+++ b/libs/gui/WindowInfo.cpp
@@ -59,6 +59,32 @@ std::ostream& operator<<(std::ostream& out, const Region& region) {
     return out;
 }
 
+status_t writeTransform(android::Parcel* parcel, const ui::Transform& transform) {
+    return parcel->writeFloat(transform.dsdx()) ?:
+            parcel->writeFloat(transform.dtdx()) ?:
+            parcel->writeFloat(transform.tx()) ?:
+            parcel->writeFloat(transform.dtdy()) ?:
+            parcel->writeFloat(transform.dsdy()) ?:
+            parcel->writeFloat(transform.ty());
+}
+
+status_t readTransform(const android::Parcel* parcel, ui::Transform& transform) {
+    float dsdx, dtdx, tx, dtdy, dsdy, ty;
+
+    const status_t status = parcel->readFloat(&dsdx) ?:
+            parcel->readFloat(&dtdx) ?:
+            parcel->readFloat(&tx) ?:
+            parcel->readFloat(&dtdy) ?:
+            parcel->readFloat(&dsdy) ?:
+            parcel->readFloat(&ty);
+    if (status != OK) {
+        return status;
+    }
+
+    transform.set({dsdx, dtdx, tx, dtdy, dsdy, ty, 0, 0, 1});
+    return OK;
+}
+
 } // namespace
 
 void WindowInfo::setInputConfig(ftl::Flags<InputConfig> config, bool value) {
@@ -135,12 +161,7 @@ status_t WindowInfo::writeToParcel(android::Parcel* parcel) const {
         parcel->writeInt32(surfaceInset) ?:
         parcel->writeFloat(globalScaleFactor) ?:
         parcel->writeFloat(alpha) ?:
-        parcel->writeFloat(transform.dsdx()) ?:
-        parcel->writeFloat(transform.dtdx()) ?:
-        parcel->writeFloat(transform.tx()) ?:
-        parcel->writeFloat(transform.dtdy()) ?:
-        parcel->writeFloat(transform.dsdy()) ?:
-        parcel->writeFloat(transform.ty()) ?:
+        writeTransform(parcel, transform) ?:
         parcel->writeInt32(static_cast<int32_t>(touchOcclusionMode)) ?:
         parcel->writeInt32(ownerPid.val()) ?:
         parcel->writeInt32(ownerUid.val()) ?:
@@ -153,8 +174,12 @@ status_t WindowInfo::writeToParcel(android::Parcel* parcel) const {
         parcel->writeStrongBinder(touchableRegionCropHandle.promote()) ?:
         parcel->writeStrongBinder(windowToken) ?:
         parcel->writeStrongBinder(focusTransferTarget) ?:
-        parcel->writeBool(canOccludePresentation);
+        parcel->writeBool(canOccludePresentation) ?:
+        parcel->writeBool(cloneLayerStackTransform.has_value());
     // clang-format on
+    if (cloneLayerStackTransform) {
+        status = status ?: writeTransform(parcel, *cloneLayerStackTransform);
+    }
     return status;
 }
 
@@ -174,10 +199,10 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
         return status;
     }
 
-    float dsdx, dtdx, tx, dtdy, dsdy, ty;
     int32_t lpFlags, lpType, touchOcclusionModeInt, inputConfigInt, ownerPidInt, ownerUidInt,
             displayIdInt;
     sp<IBinder> touchableRegionCropHandleSp;
+    bool hasCloneLayerStackTransform = false;
 
     // clang-format off
     status = parcel->readInt32(&lpFlags) ?:
@@ -188,12 +213,7 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
         parcel->readInt32(&surfaceInset) ?:
         parcel->readFloat(&globalScaleFactor) ?:
         parcel->readFloat(&alpha) ?:
-        parcel->readFloat(&dsdx) ?:
-        parcel->readFloat(&dtdx) ?:
-        parcel->readFloat(&tx) ?:
-        parcel->readFloat(&dtdy) ?:
-        parcel->readFloat(&dsdy) ?:
-        parcel->readFloat(&ty) ?:
+        readTransform(parcel, /*byRef*/ transform) ?:
         parcel->readInt32(&touchOcclusionModeInt) ?:
         parcel->readInt32(&ownerPidInt) ?:
         parcel->readInt32(&ownerUidInt) ?:
@@ -206,8 +226,8 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
         parcel->readNullableStrongBinder(&touchableRegionCropHandleSp) ?:
         parcel->readNullableStrongBinder(&windowToken) ?:
         parcel->readNullableStrongBinder(&focusTransferTarget) ?:
-        parcel->readBool(&canOccludePresentation);
-
+        parcel->readBool(&canOccludePresentation)?:
+        parcel->readBool(&hasCloneLayerStackTransform);
     // clang-format on
 
     if (status != OK) {
@@ -216,7 +236,6 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
 
     layoutParamsFlags = ftl::Flags<Flag>(lpFlags);
     layoutParamsType = static_cast<Type>(lpType);
-    transform.set({dsdx, dtdx, tx, dtdy, dsdy, ty, 0, 0, 1});
     touchOcclusionMode = static_cast<TouchOcclusionMode>(touchOcclusionModeInt);
     inputConfig = ftl::Flags<InputConfig>(inputConfigInt);
     ownerPid = Pid{ownerPidInt};
@@ -224,6 +243,15 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
     touchableRegionCropHandle = touchableRegionCropHandleSp;
     displayId = ui::LogicalDisplayId{displayIdInt};
 
+    cloneLayerStackTransform =
+            hasCloneLayerStackTransform ? std::make_optional<ui::Transform>() : std::nullopt;
+    if (cloneLayerStackTransform) {
+        status = readTransform(parcel, /*byRef*/ *cloneLayerStackTransform);
+        if (status != OK) {
+            return status;
+        }
+    }
+
     return OK;
 }
 
diff --git a/libs/gui/include/gui/WindowInfo.h b/libs/gui/include/gui/WindowInfo.h
index eb3be5588a..b48e696b4a 100644
--- a/libs/gui/include/gui/WindowInfo.h
+++ b/libs/gui/include/gui/WindowInfo.h
@@ -220,9 +220,14 @@ struct WindowInfo : public Parcelable {
     // An alpha of 1.0 means fully opaque and 0.0 means fully transparent.
     float alpha;
 
-    // Transform applied to individual windows.
+    // Transform applied to individual windows for input.
+    // Maps display coordinates to the window's input coordinate space.
     ui::Transform transform;
 
+    // Transform applied to get to the layer stack space of the cloned window for input.
+    // Maps display coordinates of the clone window to the layer stack space of the cloned window.
+    std::optional<ui::Transform> cloneLayerStackTransform;
+
     /*
      * This is filled in by the WM relative to the frame and then translated
      * to absolute coordinates by SurfaceFlinger once the frame is computed.
diff --git a/libs/gui/tests/WindowInfo_test.cpp b/libs/gui/tests/WindowInfo_test.cpp
index ce22082a9f..e3f9a07b34 100644
--- a/libs/gui/tests/WindowInfo_test.cpp
+++ b/libs/gui/tests/WindowInfo_test.cpp
@@ -40,7 +40,18 @@ TEST(WindowInfo, ParcellingWithoutToken) {
     ASSERT_EQ(OK, i.writeToParcel(&p));
     p.setDataPosition(0);
     i2.readFromParcel(&p);
-    ASSERT_TRUE(i2.token == nullptr);
+    ASSERT_EQ(i2.token, nullptr);
+}
+
+TEST(WindowInfo, ParcellingWithoutCloneTransform) {
+    WindowInfo i, i2;
+    i.cloneLayerStackTransform.reset();
+
+    Parcel p;
+    ASSERT_EQ(OK, i.writeToParcel(&p));
+    p.setDataPosition(0);
+    i2.readFromParcel(&p);
+    ASSERT_EQ(i2.cloneLayerStackTransform, std::nullopt);
 }
 
 TEST(WindowInfo, Parcelling) {
@@ -71,6 +82,8 @@ TEST(WindowInfo, Parcelling) {
     i.applicationInfo.token = new BBinder();
     i.applicationInfo.dispatchingTimeoutMillis = 0x12345678ABCD;
     i.focusTransferTarget = new BBinder();
+    i.cloneLayerStackTransform = ui::Transform();
+    i.cloneLayerStackTransform->set({5, -1, 100, 4, 0, 40, 0, 0, 1});
 
     Parcel p;
     i.writeToParcel(&p);
@@ -100,6 +113,7 @@ TEST(WindowInfo, Parcelling) {
     ASSERT_EQ(i.touchableRegionCropHandle, i2.touchableRegionCropHandle);
     ASSERT_EQ(i.applicationInfo, i2.applicationInfo);
     ASSERT_EQ(i.focusTransferTarget, i2.focusTransferTarget);
+    ASSERT_EQ(i.cloneLayerStackTransform, i2.cloneLayerStackTransform);
 }
 
 TEST(InputApplicationInfo, Parcelling) {
diff --git a/libs/input/input_flags.aconfig b/libs/input/input_flags.aconfig
index 6cdd249b9e..5da110ef02 100644
--- a/libs/input/input_flags.aconfig
+++ b/libs/input/input_flags.aconfig
@@ -224,3 +224,13 @@ flag {
   description: "Allow cursor to transition across multiple connected displays"
   bug: "362719483"
 }
+
+flag {
+  name: "use_cloned_screen_coordinates_as_raw"
+  namespace: "input"
+  description: "Use the cloned window's layer stack (screen) space as the raw coordinate space for input going to clones"
+  bug: "377846505"
+  metadata {
+    purpose: PURPOSE_BUGFIX
+  }
+}
diff --git a/services/inputflinger/dispatcher/InputDispatcher.cpp b/services/inputflinger/dispatcher/InputDispatcher.cpp
index bfab666278..28422d636a 100644
--- a/services/inputflinger/dispatcher/InputDispatcher.cpp
+++ b/services/inputflinger/dispatcher/InputDispatcher.cpp
@@ -3013,11 +3013,10 @@ void InputDispatcher::addWindowTargetLocked(const sp<WindowInfoHandle>& windowHa
                   windowHandle->getName().c_str());
             return;
         }
-        inputTargets.push_back(createInputTarget(connection, windowHandle, dispatchMode,
-                                                 targetFlags,
-                                                 mWindowInfos.getDisplayTransform(
-                                                         windowHandle->getInfo()->displayId),
-                                                 firstDownTimeInTarget));
+        inputTargets.push_back(
+                createInputTarget(connection, windowHandle, dispatchMode, targetFlags,
+                                  mWindowInfos.getRawTransform(*windowHandle->getInfo()),
+                                  firstDownTimeInTarget));
         it = inputTargets.end() - 1;
     }
 
@@ -3068,11 +3067,10 @@ void InputDispatcher::addPointerWindowTargetLocked(
                   windowHandle->getName().c_str());
             return;
         }
-        inputTargets.push_back(createInputTarget(connection, windowHandle, dispatchMode,
-                                                 targetFlags,
-                                                 mWindowInfos.getDisplayTransform(
-                                                         windowHandle->getInfo()->displayId),
-                                                 firstDownTimeInTarget));
+        inputTargets.push_back(
+                createInputTarget(connection, windowHandle, dispatchMode, targetFlags,
+                                  mWindowInfos.getRawTransform(*windowHandle->getInfo()),
+                                  firstDownTimeInTarget));
         it = inputTargets.end() - 1;
     }
 
@@ -3104,7 +3102,8 @@ void InputDispatcher::addGlobalMonitoringTargetsLocked(std::vector<InputTarget>&
     for (const Monitor& monitor : selectResponsiveMonitorsLocked(monitorsIt->second)) {
         InputTarget target{monitor.connection};
         // target.firstDownTimeInTarget is not set for global monitors. It is only required in split
-        // touch and global monitoring works as intended even without setting firstDownTimeInTarget
+        // touch and global monitoring works as intended even without setting firstDownTimeInTarget.
+        // Since global monitors don't have windows, use the display transform as the raw transform.
         target.rawTransform = mWindowInfos.getDisplayTransform(displayId);
         target.setDefaultPointerTransform(target.rawTransform);
         inputTargets.push_back(target);
@@ -4291,6 +4290,7 @@ void InputDispatcher::synthesizeCancelationEventsForConnectionLocked(
                                                  motionEntry.downTime, targets);
                 } else {
                     targets.emplace_back(fallbackTarget);
+                    // Since we don't have a window, use the display transform as the raw transform.
                     const ui::Transform displayTransform =
                             mWindowInfos.getDisplayTransform(motionEntry.displayId);
                     targets.back().rawTransform = displayTransform;
@@ -4376,6 +4376,7 @@ void InputDispatcher::synthesizePointerDownEventsForConnectionLocked(
                                                  targets);
                 } else {
                     targets.emplace_back(connection, targetFlags);
+                    // Since we don't have a window, use the display transform as the raw transform.
                     const ui::Transform displayTransform =
                             mWindowInfos.getDisplayTransform(motionEntry.displayId);
                     targets.back().rawTransform = displayTransform;
@@ -5289,6 +5290,16 @@ ui::Transform InputDispatcher::DispatcherWindowInfo::getDisplayTransform(
                                                 : kIdentityTransform;
 }
 
+ui::Transform InputDispatcher::DispatcherWindowInfo::getRawTransform(
+        const android::gui::WindowInfo& windowInfo) const {
+    // If the window has a cloneLayerStackTransform, always use it as the transform for the "getRaw"
+    // APIs. If not, fall back to using the DisplayInfo transform of the window's display.
+    return (input_flags::use_cloned_screen_coordinates_as_raw() &&
+            windowInfo.cloneLayerStackTransform)
+            ? *windowInfo.cloneLayerStackTransform
+            : getDisplayTransform(windowInfo.displayId);
+}
+
 std::string InputDispatcher::DispatcherWindowInfo::dumpDisplayAndWindowInfo() const {
     std::string dump;
     if (!mWindowHandlesByDisplay.empty()) {
@@ -6567,9 +6578,8 @@ void InputDispatcher::doDispatchCycleFinishedCommand(nsecs_t finishTime,
                                            createInputTarget(connection, windowHandle,
                                                              InputTarget::DispatchMode::AS_IS,
                                                              dispatchEntry->targetFlags,
-                                                             mWindowInfos.getDisplayTransform(
-                                                                     windowHandle->getInfo()
-                                                                             ->displayId),
+                                                             mWindowInfos.getRawTransform(
+                                                                     *windowHandle->getInfo()),
                                                              downTime));
             }
         }
diff --git a/services/inputflinger/dispatcher/InputDispatcher.h b/services/inputflinger/dispatcher/InputDispatcher.h
index fd550dd396..8485b44a03 100644
--- a/services/inputflinger/dispatcher/InputDispatcher.h
+++ b/services/inputflinger/dispatcher/InputDispatcher.h
@@ -384,6 +384,9 @@ private:
         // Get the transform for display, returns Identity-transform if display is missing.
         ui::Transform getDisplayTransform(ui::LogicalDisplayId displayId) const;
 
+        // Get the raw transform to use for motion events going to the given window.
+        ui::Transform getRawTransform(const android::gui::WindowInfo&) const;
+
         // Lookup for WindowInfoHandle from token and optionally a display-id. In cases where
         // display-id is not provided lookup is done for all displays.
         sp<android::gui::WindowInfoHandle> findWindowHandle(
diff --git a/services/inputflinger/tests/InputDispatcher_test.cpp b/services/inputflinger/tests/InputDispatcher_test.cpp
index 3413caa1f0..a5b8e4b0c4 100644
--- a/services/inputflinger/tests/InputDispatcher_test.cpp
+++ b/services/inputflinger/tests/InputDispatcher_test.cpp
@@ -6501,19 +6501,47 @@ TEST_F(InputDispatcherDisplayProjectionTest, WindowGetsEventsInCorrectCoordinate
                                ui::LogicalDisplayId::DEFAULT, {PointF{150, 220}}));
 
     firstWindow->assertNoEvents();
-    std::unique_ptr<MotionEvent> event = secondWindow->consumeMotionEvent();
-    ASSERT_NE(nullptr, event);
-    EXPECT_EQ(AMOTION_EVENT_ACTION_DOWN, event->getAction());
+    secondWindow->consumeMotionEvent(
+            AllOf(WithMotionAction(ACTION_DOWN),
+                  // Ensure that the events from the "getRaw" API are in logical display
+                  // coordinates, which has an x-scale of 2 and y-scale of 4.
+                  WithRawCoords(300, 880),
+                  // Ensure that the x and y values are in the window's coordinate space.
+                  // The left-top of the second window is at (100, 200) in display space, which is
+                  // (200, 800) in the logical display space. This will be the origin of the window
+                  // space.
+                  WithCoords(100, 80)));
+}
 
-    // Ensure that the events from the "getRaw" API are in logical display coordinates.
-    EXPECT_EQ(300, event->getRawX(0));
-    EXPECT_EQ(880, event->getRawY(0));
+TEST_F(InputDispatcherDisplayProjectionTest, UseCloneLayerStackTransformForRawCoordinates) {
+    SCOPED_FLAG_OVERRIDE(use_cloned_screen_coordinates_as_raw, true);
 
-    // Ensure that the x and y values are in the window's coordinate space.
-    // The left-top of the second window is at (100, 200) in display space, which is (200, 800) in
-    // the logical display space. This will be the origin of the window space.
-    EXPECT_EQ(100, event->getX(0));
-    EXPECT_EQ(80, event->getY(0));
+    auto [firstWindow, secondWindow] = setupScaledDisplayScenario();
+
+    const std::array<float, 9> matrix = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 0.0, 0.0, 1.0};
+    ui::Transform secondDisplayTransform;
+    secondDisplayTransform.set(matrix);
+    addDisplayInfo(SECOND_DISPLAY_ID, secondDisplayTransform);
+
+    // When a clone layer stack transform is provided for a window, we should use that as the
+    // "display transform" for input going to that window.
+    sp<FakeWindowHandle> secondWindowClone = secondWindow->clone(SECOND_DISPLAY_ID);
+    secondWindowClone->editInfo()->cloneLayerStackTransform = ui::Transform();
+    secondWindowClone->editInfo()->cloneLayerStackTransform->set(0.5, 0, 0, 0.25);
+    addWindow(secondWindowClone);
+
+    // Touch down on the clone window, and ensure its raw coordinates use
+    // the clone layer stack transform.
+    mDispatcher->notifyMotion(generateMotionArgs(ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN,
+                                                 SECOND_DISPLAY_ID, {PointF{150, 220}}));
+    secondWindowClone->consumeMotionEvent(
+            AllOf(WithMotionAction(ACTION_DOWN),
+                  // Ensure the x and y coordinates are in the window's coordinate space.
+                  // See previous test case for calculation.
+                  WithCoords(100, 80),
+                  // Ensure the "getRaw" API uses the clone layer stack transform when it is
+                  // provided for the window. It has an x-scale of 0.5 and y-scale of 0.25.
+                  WithRawCoords(75, 55)));
 }
 
 TEST_F(InputDispatcherDisplayProjectionTest, CancelMotionWithCorrectCoordinates) {
diff --git a/services/surfaceflinger/FrontEnd/LayerHierarchy.h b/services/surfaceflinger/FrontEnd/LayerHierarchy.h
index 47d0041a8b..4fdbae1831 100644
--- a/services/surfaceflinger/FrontEnd/LayerHierarchy.h
+++ b/services/surfaceflinger/FrontEnd/LayerHierarchy.h
@@ -102,6 +102,8 @@ public:
         // Returns true if the node is a clone.
         bool isClone() const { return !mirrorRootIds.empty(); }
 
+        TraversalPath getClonedFrom() const { return {.id = id, .variant = variant}; }
+
         bool operator==(const TraversalPath& other) const {
             return id == other.id && mirrorRootIds == other.mirrorRootIds;
         }
diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
index 022588deef..d0dc065353 100644
--- a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
+++ b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
@@ -16,6 +16,8 @@
 
 // #define LOG_NDEBUG 0
 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
+#include "FrontEnd/LayerSnapshot.h"
+#include "ui/Transform.h"
 #undef LOG_TAG
 #define LOG_TAG "SurfaceFlinger"
 
@@ -1205,13 +1207,27 @@ void LayerSnapshotBuilder::updateInput(LayerSnapshot& snapshot,
     snapshot.inputInfo.contentSize = {snapshot.croppedBufferSize.getHeight(),
                                       snapshot.croppedBufferSize.getWidth()};
 
-    // If the layer is a clone, we need to crop the input region to cloned root to prevent
-    // touches from going outside the cloned area.
+    snapshot.inputInfo.cloneLayerStackTransform.reset();
+
     if (path.isClone()) {
         snapshot.inputInfo.inputConfig |= InputConfig::CLONE;
         // Cloned layers shouldn't handle watch outside since their z order is not determined by
         // WM or the client.
         snapshot.inputInfo.inputConfig.clear(InputConfig::WATCH_OUTSIDE_TOUCH);
+
+        // Compute the transform that maps the clone's display to the layer stack space of the
+        // cloned window.
+        const LayerSnapshot* clonedSnapshot = getSnapshot(path.getClonedFrom());
+        if (clonedSnapshot != nullptr) {
+            const auto& [clonedInputBounds, s] =
+                    getInputBounds(*clonedSnapshot, /*fillParentBounds=*/false);
+            ui::Transform inputToLayer;
+            inputToLayer.set(clonedInputBounds.left, clonedInputBounds.top);
+            const ui::Transform& layerToLayerStack = getInputTransform(*clonedSnapshot);
+            const auto& displayToInput = snapshot.inputInfo.transform;
+            snapshot.inputInfo.cloneLayerStackTransform =
+                    layerToLayerStack * inputToLayer * displayToInput;
+        }
     }
 }