Merge "Re-implement full screen magnification continuous cursor following" into main

7 files changed, 325 insertions, 6 deletions

diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java
index 42834ce20783..c49151dd5e30 100644
--- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java
+++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java
@@ -287,7 +287,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
 
     public static final int INVALID_SERVICE_ID = -1;
 
-    // Each service has an ID. Also provide one for magnification gesture handling
+    // Each service has an ID. Also provide one for magnification gesture handling.
+    // This ID is also used for mouse event handling.
     public static final int MAGNIFICATION_GESTURE_HANDLER_ID = 0;
 
     private static int sIdCounter = MAGNIFICATION_GESTURE_HANDLER_ID + 1;
diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java b/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java
index 11b8ccb70dfb..004b3ffcb02b 100644
--- a/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java
+++ b/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java
@@ -121,6 +121,8 @@ public class FullScreenMagnificationController implements
     @NonNull private final Supplier<MagnificationThumbnail> mThumbnailSupplier;
     @NonNull private final Supplier<Boolean> mMagnificationConnectionStateSupplier;
 
+    private boolean mIsPointerMotionFilterInstalled = false;
+
     /**
      * This class implements {@link WindowManagerInternal.MagnificationCallbacks} and holds
      * magnification information per display.
@@ -830,9 +832,17 @@ public class FullScreenMagnificationController implements
                 return;
             }
 
-            final float nonNormOffsetX = mCurrentMagnificationSpec.offsetX - offsetX;
-            final float nonNormOffsetY = mCurrentMagnificationSpec.offsetY - offsetY;
-            if (updateCurrentSpecWithOffsetsLocked(nonNormOffsetX, nonNormOffsetY)) {
+            setOffset(mCurrentMagnificationSpec.offsetX - offsetX,
+                    mCurrentMagnificationSpec.offsetY - offsetY, id);
+        }
+
+        @GuardedBy("mLock")
+        void setOffset(float offsetX, float offsetY, int id) {
+            if (!mRegistered) {
+                return;
+            }
+
+            if (updateCurrentSpecWithOffsetsLocked(offsetX, offsetY)) {
                 onMagnificationChangedLocked(/* isScaleTransient= */ false);
             }
             if (id != INVALID_SERVICE_ID) {
@@ -1065,6 +1075,7 @@ public class FullScreenMagnificationController implements
             if (display.register()) {
                 mDisplays.put(displayId, display);
                 mScreenStateObserver.registerIfNecessary();
+                configurePointerMotionFilter(true);
             }
         }
     }
@@ -1613,6 +1624,28 @@ public class FullScreenMagnificationController implements
     }
 
     /**
+     * Sets the offset of the magnified region.
+     *
+     * @param displayId The logical display id.
+     * @param offsetX   the offset of the magnified region in the X coordinate, in current
+     *                  screen pixels.
+     * @param offsetY   the offset of the magnified region in the Y coordinate, in current
+     *                  screen pixels.
+     * @param id        the ID of the service requesting the change
+     */
+    @SuppressWarnings("GuardedBy")
+    // errorprone cannot recognize an inner class guarded by an outer class member.
+    public void setOffset(int displayId, float offsetX, float offsetY, int id) {
+        synchronized (mLock) {
+            final DisplayMagnification display = mDisplays.get(displayId);
+            if (display == null) {
+                return;
+            }
+            display.setOffset(offsetX, offsetY, id);
+        }
+    }
+
+    /**
      * Offsets the magnified region. Note that the offsetX and offsetY values actually move in the
      * opposite direction as the offsets passed in here.
      *
@@ -1885,6 +1918,7 @@ public class FullScreenMagnificationController implements
         }
         if (!hasRegister) {
             mScreenStateObserver.unregister();
+            configurePointerMotionFilter(false);
         }
     }
 
@@ -1900,6 +1934,22 @@ public class FullScreenMagnificationController implements
         }
     }
 
+    private void configurePointerMotionFilter(boolean enabled) {
+        if (!Flags.enableMagnificationFollowsMouseWithPointerMotionFilter()) {
+            return;
+        }
+        if (enabled == mIsPointerMotionFilterInstalled) {
+            return;
+        }
+        if (!enabled) {
+            mControllerCtx.getInputManager().registerAccessibilityPointerMotionFilter(null);
+        } else {
+            mControllerCtx.getInputManager().registerAccessibilityPointerMotionFilter(
+                    new FullScreenMagnificationPointerMotionEventFilter(this));
+        }
+        mIsPointerMotionFilterInstalled = enabled;
+    }
+
     private boolean traceEnabled() {
         return mControllerCtx.getTraceManager().isA11yTracingEnabledForTypes(
                 FLAGS_WINDOW_MANAGER_INTERNAL);
diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandler.java b/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandler.java
index e0dd8b601a3d..59b4a1613e08 100644
--- a/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandler.java
+++ b/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandler.java
@@ -182,6 +182,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
     private final int mMinimumVelocity;
     private final int mMaximumVelocity;
 
+    @Nullable
     private final MouseEventHandler mMouseEventHandler;
 
     public FullScreenMagnificationGestureHandler(
@@ -313,7 +314,9 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
         mOverscrollEdgeSlop = context.getResources().getDimensionPixelSize(
                 R.dimen.accessibility_fullscreen_magnification_gesture_edge_slop);
         mIsWatch = context.getPackageManager().hasSystemFeature(PackageManager.FEATURE_WATCH);
-        mMouseEventHandler = new MouseEventHandler(mFullScreenMagnificationController);
+        mMouseEventHandler =
+                Flags.enableMagnificationFollowsMouseWithPointerMotionFilter()
+                        ? null : new MouseEventHandler(mFullScreenMagnificationController);
 
         if (mDetectShortcutTrigger) {
             mScreenStateReceiver = new ScreenStateReceiver(context, this);
@@ -337,9 +340,11 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
 
     @Override
     void handleMouseOrStylusEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
-        if (!mFullScreenMagnificationController.isActivated(mDisplayId)) {
+        if (mMouseEventHandler == null
+                || !mFullScreenMagnificationController.isActivated(mDisplayId)) {
             return;
         }
+
         // TODO(b/354696546): Allow mouse/stylus to activate whichever display they are
         // over, rather than only interacting with the current display.
 
diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationPointerMotionEventFilter.java b/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationPointerMotionEventFilter.java
new file mode 100644
index 000000000000..f1ba83e80d54
--- /dev/null
+++ b/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationPointerMotionEventFilter.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.accessibility.magnification;
+
+import static com.android.server.accessibility.AccessibilityManagerService.MAGNIFICATION_GESTURE_HANDLER_ID;
+
+import android.annotation.NonNull;
+
+import com.android.server.input.InputManagerInternal;
+
+/**
+ * Handles pointer motion event for full screen magnification.
+ * Responsible for controlling magnification's cursor following feature.
+ */
+public class FullScreenMagnificationPointerMotionEventFilter implements
+        InputManagerInternal.AccessibilityPointerMotionFilter {
+
+    private final FullScreenMagnificationController mController;
+
+    public FullScreenMagnificationPointerMotionEventFilter(
+            FullScreenMagnificationController controller) {
+        mController = controller;
+    }
+
+    /**
+     * This call happens on the input hot path and it is extremely performance sensitive. It
+     * also must not call back into native code.
+     */
+    @Override
+    @NonNull
+    public float[] filterPointerMotionEvent(float dx, float dy, float currentX, float currentY,
+            int displayId) {
+        if (!mController.isActivated(displayId)) {
+            // unrelated display.
+            return new float[]{dx, dy};
+        }
+
+        // TODO(361817142): implement centered and edge following types.
+
+        // Continuous cursor following.
+        float scale = mController.getScale(displayId);
+        final float newCursorX = currentX + dx;
+        final float newCursorY = currentY + dy;
+        mController.setOffset(displayId,
+                newCursorX - newCursorX * scale, newCursorY - newCursorY * scale,
+                MAGNIFICATION_GESTURE_HANDLER_ID);
+
+        // In the continuous mode, the cursor speed in physical display is kept.
+        // Thus, we don't consume any motion delta.
+        return new float[]{dx, dy};
+    }
+}
diff --git a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java
index ac27a971102a..9ec99c651691 100644
--- a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java
+++ b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java
@@ -30,7 +30,10 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyFloat;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.clearInvocations;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
@@ -248,6 +251,40 @@ public class FullScreenMagnificationControllerTest {
     }
 
     @Test
+    @RequiresFlagsEnabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
+    public void testRegister_RegistersPointerMotionFilter() {
+        register(DISPLAY_0);
+
+        verify(mMockInputManager).registerAccessibilityPointerMotionFilter(
+                any(InputManagerInternal.AccessibilityPointerMotionFilter.class));
+
+        // If a filter is already registered, adding a display won't invoke another filter
+        // registration.
+        clearInvocations(mMockInputManager);
+        register(DISPLAY_1);
+        register(INVALID_DISPLAY);
+
+        verify(mMockInputManager, times(0)).registerAccessibilityPointerMotionFilter(
+                any(InputManagerInternal.AccessibilityPointerMotionFilter.class));
+    }
+
+    @Test
+    @RequiresFlagsEnabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
+    public void testUnregister_UnregistersPointerMotionFilter() {
+        register(DISPLAY_0);
+        register(DISPLAY_1);
+        clearInvocations(mMockInputManager);
+
+        mFullScreenMagnificationController.unregister(DISPLAY_1);
+        // There's still an active display. Don't unregister yet.
+        verify(mMockInputManager, times(0)).registerAccessibilityPointerMotionFilter(
+                nullable(InputManagerInternal.AccessibilityPointerMotionFilter.class));
+
+        mFullScreenMagnificationController.unregister(DISPLAY_0);
+        verify(mMockInputManager, times(1)).registerAccessibilityPointerMotionFilter(isNull());
+    }
+
+    @Test
     public void testInitialState_noMagnificationAndMagnificationRegionReadFromWindowManager() {
         for (int i = 0; i < DISPLAY_COUNT; i++) {
             initialState_noMagnificationAndMagnificationRegionReadFromWindowManager(i);
@@ -699,6 +736,63 @@ public class FullScreenMagnificationControllerTest {
     }
 
     @Test
+    public void testSetOffset_whileMagnifying_offsetsMove() {
+        for (int i = 0; i < DISPLAY_COUNT; i++) {
+            setOffset_whileMagnifying_offsetsMove(i);
+            resetMockWindowManager();
+        }
+    }
+
+    private void setOffset_whileMagnifying_offsetsMove(int displayId) {
+        register(displayId);
+        PointF startCenter = INITIAL_MAGNIFICATION_BOUNDS_CENTER;
+        for (final float scale : new float[]{2.0f, 2.5f, 3.0f}) {
+            assertTrue(mFullScreenMagnificationController
+                    .setScaleAndCenter(displayId, scale, startCenter.x, startCenter.y, true, false,
+                            SERVICE_ID_1));
+            mMessageCapturingHandler.sendAllMessages();
+
+            for (final PointF center : new PointF[]{
+                    INITIAL_BOUNDS_LOWER_RIGHT_2X_CENTER,
+                    INITIAL_BOUNDS_UPPER_LEFT_2X_CENTER}) {
+                Mockito.clearInvocations(mMockWindowManager);
+                PointF newOffsets = computeOffsets(INITIAL_MAGNIFICATION_BOUNDS, center, scale);
+                mFullScreenMagnificationController.setOffset(displayId, newOffsets.x, newOffsets.y,
+                        SERVICE_ID_1);
+                mMessageCapturingHandler.sendAllMessages();
+
+                MagnificationSpec expectedSpec = getMagnificationSpec(scale, newOffsets);
+                verify(mMockWindowManager)
+                        .setMagnificationSpec(eq(displayId), argThat(closeTo(expectedSpec)));
+                assertEquals(center.x, mFullScreenMagnificationController.getCenterX(displayId),
+                        0.0);
+                assertEquals(center.y, mFullScreenMagnificationController.getCenterY(displayId),
+                        0.0);
+                verify(mMockValueAnimator, times(0)).start();
+            }
+        }
+    }
+
+    @Test
+    public void testSetOffset_whileNotMagnifying_hasNoEffect() {
+        for (int i = 0; i < DISPLAY_COUNT; i++) {
+            setOffset_whileNotMagnifying_hasNoEffect(i);
+            resetMockWindowManager();
+        }
+    }
+
+    private void setOffset_whileNotMagnifying_hasNoEffect(int displayId) {
+        register(displayId);
+        Mockito.reset(mMockWindowManager);
+        MagnificationSpec startSpec = getCurrentMagnificationSpec(displayId);
+        mFullScreenMagnificationController.setOffset(displayId, 100, 100, SERVICE_ID_1);
+        assertThat(getCurrentMagnificationSpec(displayId), closeTo(startSpec));
+        mFullScreenMagnificationController.setOffset(displayId, 200, 200, SERVICE_ID_1);
+        assertThat(getCurrentMagnificationSpec(displayId), closeTo(startSpec));
+        verifyNoMoreInteractions(mMockWindowManager);
+    }
+
+    @Test
     @RequiresFlagsEnabled(Flags.FLAG_FULLSCREEN_FLING_GESTURE)
     public void testStartFling_whileMagnifying_flings() throws InterruptedException {
         for (int i = 0; i < DISPLAY_COUNT; i++) {
diff --git a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandlerTest.java b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandlerTest.java
index 5c126d1f5d3f..4ea5fcfd79c8 100644
--- a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandlerTest.java
+++ b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandlerTest.java
@@ -1419,6 +1419,12 @@ public class FullScreenMagnificationGestureHandlerTest {
     }
 
     @Test
+    @RequiresFlagsEnabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
+    public void testMouseMoveEventsDoNotMoveMagnifierViewport() {
+        runMoveEventsDoNotMoveMagnifierViewport(InputDevice.SOURCE_MOUSE);
+    }
+
+    @Test
     public void testStylusMoveEventsDoNotMoveMagnifierViewport() {
         runMoveEventsDoNotMoveMagnifierViewport(InputDevice.SOURCE_STYLUS);
     }
@@ -1467,11 +1473,28 @@ public class FullScreenMagnificationGestureHandlerTest {
     }
 
     @Test
+    @RequiresFlagsEnabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
+    public void testMouseHoverMoveEventsDoNotMoveMagnifierViewport() {
+        // Note that this means mouse hover shouldn't be handled here.
+        // FullScreenMagnificationPointerMotionEventFilter handles mouse input events.
+        runHoverMoveEventsDoNotMoveMagnifierViewport(InputDevice.SOURCE_MOUSE);
+    }
+
+    @Test
+    @RequiresFlagsEnabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
+    public void testStylusHoverMoveEventsDoNotMoveMagnifierViewport() {
+        // TODO(b/398984690): We will revisit the behavior.
+        runHoverMoveEventsDoNotMoveMagnifierViewport(InputDevice.SOURCE_STYLUS);
+    }
+
+    @Test
+    @RequiresFlagsDisabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
     public void testMouseHoverMoveEventsMoveMagnifierViewport() {
         runHoverMovesViewportTest(InputDevice.SOURCE_MOUSE);
     }
 
     @Test
+    @RequiresFlagsDisabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
     public void testStylusHoverMoveEventsMoveMagnifierViewport() {
         runHoverMovesViewportTest(InputDevice.SOURCE_STYLUS);
     }
@@ -1497,6 +1520,7 @@ public class FullScreenMagnificationGestureHandlerTest {
     }
 
     @Test
+    @RequiresFlagsDisabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
     public void testMouseMoveEventsMoveMagnifierViewport() {
         final EventCaptor eventCaptor = new EventCaptor();
         mMgh.setNext(eventCaptor);
diff --git a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationPointerMotionEventFilterTest.java b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationPointerMotionEventFilterTest.java
new file mode 100644
index 000000000000..a8315d4eb3a5
--- /dev/null
+++ b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationPointerMotionEventFilterTest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.accessibility.magnification;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import androidx.test.runner.AndroidJUnit4;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+@RunWith(AndroidJUnit4.class)
+public class FullScreenMagnificationPointerMotionEventFilterTest {
+    @Mock
+    private FullScreenMagnificationController mMockFullScreenMagnificationController;
+
+    private FullScreenMagnificationPointerMotionEventFilter mFilter;
+
+    @Before
+    public void setUp() throws Exception {
+        MockitoAnnotations.initMocks(this);
+
+        mFilter = new FullScreenMagnificationPointerMotionEventFilter(
+                mMockFullScreenMagnificationController);
+    }
+
+    @Test
+    public void inactiveDisplay_doNothing() {
+        when(mMockFullScreenMagnificationController.isActivated(anyInt())).thenReturn(false);
+
+        float[] delta = new float[]{1.f, 2.f};
+        float[] result = mFilter.filterPointerMotionEvent(delta[0], delta[1], 3.0f, 4.0f, 0);
+        assertThat(result).isEqualTo(delta);
+    }
+
+    @Test
+    public void testContinuousMove() {
+        when(mMockFullScreenMagnificationController.isActivated(anyInt())).thenReturn(true);
+        when(mMockFullScreenMagnificationController.getScale(anyInt())).thenReturn(3.f);
+
+        float[] delta = new float[]{5.f, 10.f};
+        float[] result = mFilter.filterPointerMotionEvent(delta[0], delta[1], 20.f, 30.f, 0);
+        assertThat(result).isEqualTo(delta);
+        // At the first cursor move, it goes to (20, 30) + (5, 10) = (25, 40). The scale is 3.0.
+        // The expected offset is (-25 * (3-1), -40 * (3-1)) = (-50, -80).
+        verify(mMockFullScreenMagnificationController)
+                .setOffset(eq(0), eq(-50.f), eq(-80.f), anyInt());
+
+        float[] delta2 = new float[]{10.f, 5.f};
+        float[] result2 = mFilter.filterPointerMotionEvent(delta2[0], delta2[1], 25.f, 40.f, 0);
+        assertThat(result2).isEqualTo(delta2);
+        // At the second cursor move, it goes to (25, 40) + (10, 5) = (35, 40). The scale is 3.0.
+        // The expected offset is (-35 * (3-1), -45 * (3-1)) = (-70, -90).
+        verify(mMockFullScreenMagnificationController)
+                .setOffset(eq(0), eq(-70.f), eq(-90.f), anyInt());
+    }
+}