diff options
9 files changed, 855 insertions, 0 deletions
diff --git a/core/java/android/hardware/input/input_framework.aconfig b/core/java/android/hardware/input/input_framework.aconfig index acd0d00f812d..16d9ef270f75 100644 --- a/core/java/android/hardware/input/input_framework.aconfig +++ b/core/java/android/hardware/input/input_framework.aconfig @@ -61,3 +61,10 @@ flag { description: "Allows system to provide keyboard specific key drawables and shortcuts via config files" bug: "345440920" } + +flag { + namespace: "input_native" + name: "keyboard_a11y_mouse_keys" + description: "Controls if the mouse keys accessibility feature for physical keyboard is available to the user" + bug: "341799888" +} diff --git a/core/java/android/provider/Settings.java b/core/java/android/provider/Settings.java index c954cdb270e8..2562c8e31095 100644 --- a/core/java/android/provider/Settings.java +++ b/core/java/android/provider/Settings.java @@ -12324,6 +12324,18 @@ public final class Settings { "accessibility_force_invert_color_enabled"; /** + * Whether to enable mouse keys for Physical Keyboard accessibility. + * + * If set to true, key presses (of the mouse keys) on + * physical keyboard will control mouse pointer on the display. + * + * @hide + */ + @Readable + public static final String ACCESSIBILITY_MOUSE_KEYS_ENABLED = + "accessibility_mouse_keys_enabled"; + + /** * Whether the Adaptive connectivity option is enabled. * * @hide diff --git a/packages/SettingsProvider/src/android/provider/settings/backup/SecureSettings.java b/packages/SettingsProvider/src/android/provider/settings/backup/SecureSettings.java index d4a4703d5caf..5f236516785d 100644 --- a/packages/SettingsProvider/src/android/provider/settings/backup/SecureSettings.java +++ b/packages/SettingsProvider/src/android/provider/settings/backup/SecureSettings.java @@ -228,6 +228,7 @@ public class SecureSettings { Settings.Secure.ACCESSIBILITY_MAGNIFICATION_ALWAYS_ON_ENABLED, Settings.Secure.ACCESSIBILITY_MAGNIFICATION_JOYSTICK_ENABLED, Settings.Secure.ACCESSIBILITY_MAGNIFICATION_TWO_FINGER_TRIPLE_TAP_ENABLED, + Settings.Secure.ACCESSIBILITY_MOUSE_KEYS_ENABLED, Settings.Secure.ACCESSIBILITY_PINCH_TO_ZOOM_ANYWHERE_ENABLED, Settings.Secure.ACCESSIBILITY_SINGLE_FINGER_PANNING_ENABLED, Settings.Secure.ODI_CAPTIONS_VOLUME_UI_ENABLED, diff --git a/packages/SettingsProvider/src/android/provider/settings/validators/SecureSettingsValidators.java b/packages/SettingsProvider/src/android/provider/settings/validators/SecureSettingsValidators.java index 6df1c45bd2ac..c8da8afc25c3 100644 --- a/packages/SettingsProvider/src/android/provider/settings/validators/SecureSettingsValidators.java +++ b/packages/SettingsProvider/src/android/provider/settings/validators/SecureSettingsValidators.java @@ -439,6 +439,7 @@ public class SecureSettingsValidators { VALIDATORS.put(Secure.ON_DEVICE_INFERENCE_UNBIND_TIMEOUT_MS, ANY_LONG_VALIDATOR); VALIDATORS.put(Secure.ON_DEVICE_INTELLIGENCE_UNBIND_TIMEOUT_MS, ANY_LONG_VALIDATOR); VALIDATORS.put(Secure.ON_DEVICE_INTELLIGENCE_IDLE_TIMEOUT_MS, NONE_NEGATIVE_LONG_VALIDATOR); + VALIDATORS.put(Secure.ACCESSIBILITY_MOUSE_KEYS_ENABLED, BOOLEAN_VALIDATOR); VALIDATORS.put(Secure.MANDATORY_BIOMETRICS, new InclusiveIntegerRangeValidator(0, 1)); } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java index d3efa21a2311..9fc64a965f4b 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java @@ -26,6 +26,8 @@ import android.annotation.MainThread; import android.annotation.NonNull; import android.content.Context; import android.graphics.Region; +import android.hardware.input.InputManager; +import android.os.Looper; import android.os.PowerManager; import android.os.SystemClock; import android.provider.Settings; @@ -54,6 +56,7 @@ import com.android.server.policy.WindowManagerPolicy; import java.io.FileDescriptor; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.Objects; import java.util.StringJoiner; /** @@ -158,6 +161,13 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo */ static final int FLAG_FEATURE_MAGNIFICATION_TWO_FINGER_TRIPLE_TAP = 0x00001000; + /** + * Flag for enabling the Accessibility mouse key events feature. + * + * @see #setUserAndEnabledFeatures(int, int) + */ + static final int FLAG_FEATURE_MOUSE_KEYS = 0x00002000; + static final int FEATURES_AFFECTING_MOTION_EVENTS = FLAG_FEATURE_INJECT_MOTION_EVENTS | FLAG_FEATURE_AUTOCLICK @@ -189,6 +199,8 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo private KeyboardInterceptor mKeyboardInterceptor; + private MouseKeysInterceptor mMouseKeysInterceptor; + private boolean mInstalled; private int mUserId; @@ -733,6 +745,15 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo // default display. addFirstEventHandler(Display.DEFAULT_DISPLAY, mKeyboardInterceptor); } + + if ((mEnabledFeatures & FLAG_FEATURE_MOUSE_KEYS) != 0) { + mMouseKeysInterceptor = new MouseKeysInterceptor(mAms, + Objects.requireNonNull(mContext.getSystemService( + InputManager.class)), + Looper.myLooper(), + Display.DEFAULT_DISPLAY); + addFirstEventHandler(Display.DEFAULT_DISPLAY, mMouseKeysInterceptor); + } } /** @@ -816,6 +837,11 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo mKeyboardInterceptor.onDestroy(); mKeyboardInterceptor = null; } + + if (mMouseKeysInterceptor != null) { + mMouseKeysInterceptor.onDestroy(); + mMouseKeysInterceptor = null; + } } private MagnificationGestureHandler createMagnificationGestureHandler( diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 32491b7eb0e0..b918d80fc63d 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -57,6 +57,7 @@ import static com.android.internal.accessibility.util.AccessibilityStatsLogUtils import static com.android.internal.util.FunctionalUtils.ignoreRemoteException; import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; import static com.android.server.accessibility.AccessibilityUserState.doesShortcutTargetsStringContain; +import static com.android.hardware.input.Flags.keyboardA11yMouseKeys; import static com.android.settingslib.RestrictedLockUtils.EnforcedAdmin; import android.accessibilityservice.AccessibilityGestureEvent; @@ -2936,6 +2937,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (combinedGenericMotionEventSources != 0) { flags |= AccessibilityInputFilter.FLAG_FEATURE_INTERCEPT_GENERIC_MOTION_EVENTS; } + if (userState.isMouseKeysEnabled()) { + flags |= AccessibilityInputFilter.FLAG_FEATURE_MOUSE_KEYS; + } if (flags != 0) { if (!mHasInputFilter) { mHasInputFilter = true; @@ -3216,6 +3220,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub somethingChanged |= readMagnificationCapabilitiesLocked(userState); somethingChanged |= readMagnificationFollowTypingLocked(userState); somethingChanged |= readAlwaysOnMagnificationLocked(userState); + somethingChanged |= readMouseKeysEnabledLocked(userState); return somethingChanged; } @@ -5476,6 +5481,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private final Uri mAlwaysOnMagnificationUri = Settings.Secure.getUriFor( Settings.Secure.ACCESSIBILITY_MAGNIFICATION_ALWAYS_ON_ENABLED); + private final Uri mMouseKeysUri = Settings.Secure.getUriFor( + Settings.Secure.ACCESSIBILITY_MOUSE_KEYS_ENABLED); + public AccessibilityContentObserver(Handler handler) { super(handler); } @@ -5524,6 +5532,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub mMagnificationFollowTypingUri, false, this, UserHandle.USER_ALL); contentResolver.registerContentObserver( mAlwaysOnMagnificationUri, false, this, UserHandle.USER_ALL); + contentResolver.registerContentObserver( + mMouseKeysUri, false, this, UserHandle.USER_ALL); } @Override @@ -5604,6 +5614,10 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub readMagnificationFollowTypingLocked(userState); } else if (mAlwaysOnMagnificationUri.equals(uri)) { readAlwaysOnMagnificationLocked(userState); + } else if (mMouseKeysUri.equals(uri)) { + if (readMouseKeysEnabledLocked(userState)) { + onUserStateChangedLocked(userState); + } } } } @@ -5742,6 +5756,20 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return false; } + boolean readMouseKeysEnabledLocked(AccessibilityUserState userState) { + if (!keyboardA11yMouseKeys()) { + return false; + } + final boolean isMouseKeysEnabled = + Settings.Secure.getIntForUser(mContext.getContentResolver(), + Settings.Secure.ACCESSIBILITY_MOUSE_KEYS_ENABLED, 0, userState.mUserId) == 1; + if (isMouseKeysEnabled != userState.isMouseKeysEnabled()) { + userState.setMouseKeysEnabled(isMouseKeysEnabled); + return true; + } + return false; + } + @Override public void setGestureDetectionPassthroughRegion(int displayId, Region region) { mMainHandler.sendMessage( diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityUserState.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityUserState.java index 7bcbc2768a16..b061065d44a5 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityUserState.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityUserState.java @@ -169,6 +169,8 @@ class AccessibilityUserState { private final int mFocusStrokeWidthDefaultValue; // The default value of the focus color. private final int mFocusColorDefaultValue; + /** Whether mouse keys feature is enabled. */ + private boolean mMouseKeysEnabled = false; private final Map<ComponentName, ComponentName> mA11yServiceToTileService = new ArrayMap<>(); private final Map<ComponentName, ComponentName> mA11yActivityToTileService = new ArrayMap<>(); @@ -674,6 +676,14 @@ class AccessibilityUserState { mIsFilterKeyEventsEnabled = enabled; } + public void setMouseKeysEnabled(boolean enabled) { + mMouseKeysEnabled = enabled; + } + + public boolean isMouseKeysEnabled() { + return mMouseKeysEnabled; + } + public int getInteractiveUiTimeoutLocked() { return mInteractiveUiTimeout; } diff --git a/services/accessibility/java/com/android/server/accessibility/MouseKeysInterceptor.java b/services/accessibility/java/com/android/server/accessibility/MouseKeysInterceptor.java new file mode 100644 index 000000000000..3f0f23f4a2f9 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/MouseKeysInterceptor.java @@ -0,0 +1,498 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility; + +import static android.accessibilityservice.AccessibilityTrace.FLAGS_INPUT_FILTER; +import static android.util.MathUtils.sqrt; + +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.companion.virtual.VirtualDeviceManager; +import android.companion.virtual.VirtualDeviceParams; +import android.hardware.input.InputManager; +import android.hardware.input.VirtualMouse; +import android.hardware.input.VirtualMouseButtonEvent; +import android.hardware.input.VirtualMouseConfig; +import android.hardware.input.VirtualMouseRelativeEvent; +import android.hardware.input.VirtualMouseScrollEvent; +import android.os.Handler; +import android.os.Looper; +import android.os.Message; +import android.util.Log; +import android.util.Slog; +import android.util.SparseArray; +import android.view.KeyEvent; + +import com.android.server.LocalServices; +import com.android.server.companion.virtual.VirtualDeviceManagerInternal; + +/** + * Implements the "mouse keys" accessibility feature for physical keyboards. + * + * If enabled, mouse keys will allow users to use a physical keyboard to + * control the mouse on the display. + * The following mouse functionality is supported by the mouse keys: + * <ul> + * <li> Move the mouse pointer in different directions (up, down, left, right and diagonally). + * <li> Click the mouse button (left, right and middle click). + * <li> Press and hold the mouse button. + * <li> Release the mouse button. + * <li> Scroll (up and down). + * </ul> + * + * The keys that are mapped to mouse keys are consumed by {@link AccessibilityInputFilter}. + * Non-mouse key {@link KeyEvent} will be passed to the parent handler to be handled as usual. + * A new {@link VirtualMouse} is created whenever the mouse keys feature is turned on in Settings. + * In case multiple physical keyboard are connected to a device, + * mouse keys of each physical keyboard will control a single (global) mouse pointer. + */ +public class MouseKeysInterceptor extends BaseEventStreamTransformation implements Handler.Callback, + InputManager.InputDeviceListener { + private static final String LOG_TAG = "MouseKeysInterceptor"; + + // To enable these logs, run: 'adb shell setprop log.tag.MouseKeysInterceptor DEBUG' + // (requires restart) + private static final boolean DEBUG = Log.isLoggable(LOG_TAG, Log.DEBUG); + + private static final int MESSAGE_MOVE_MOUSE_POINTER = 1; + private static final int MESSAGE_SCROLL_MOUSE_POINTER = 2; + private static final float MOUSE_POINTER_MOVEMENT_STEP = 1.8f; + private static final int KEY_NOT_SET = -1; + + /** Time interval after which mouse action will be repeated */ + private static final int INTERVAL_MILLIS = 10; + + private final AccessibilityManagerService mAms; + private final InputManager mInputManager; + private final Handler mHandler; + + private final int mDisplayId; + + VirtualDeviceManager.VirtualDevice mVirtualDevice = null; + + private VirtualMouse mVirtualMouse = null; + + /** + * State of the active directional mouse key. + * Multiple mouse keys will not be allowed to be used simultaneously i.e., + * once a mouse key is pressed, other mouse key presses will be disregarded + * (except for when the "HOLD" key is pressed). + */ + private int mActiveMoveKey = KEY_NOT_SET; + + /** State of the active scroll mouse key. */ + private int mActiveScrollKey = KEY_NOT_SET; + + /** Last time the key action was performed */ + private long mLastTimeKeyActionPerformed = 0; + + // TODO (b/346706749): This is currently using the numpad key bindings for mouse keys. + // Decide the final mouse key bindings with UX input. + public enum MouseKeyEvent { + DIAGONAL_DOWN_LEFT_MOVE(KeyEvent.KEYCODE_NUMPAD_1), + DOWN_MOVE(KeyEvent.KEYCODE_NUMPAD_2), + DIAGONAL_DOWN_RIGHT_MOVE(KeyEvent.KEYCODE_NUMPAD_3), + LEFT_MOVE(KeyEvent.KEYCODE_NUMPAD_4), + RIGHT_MOVE(KeyEvent.KEYCODE_NUMPAD_6), + DIAGONAL_UP_LEFT_MOVE(KeyEvent.KEYCODE_NUMPAD_7), + UP_MOVE(KeyEvent.KEYCODE_NUMPAD_8), + DIAGONAL_UP_RIGHT_MOVE(KeyEvent.KEYCODE_NUMPAD_9), + LEFT_CLICK(KeyEvent.KEYCODE_NUMPAD_5), + RIGHT_CLICK(KeyEvent.KEYCODE_NUMPAD_DOT), + HOLD(KeyEvent.KEYCODE_NUMPAD_MULTIPLY), + RELEASE(KeyEvent.KEYCODE_NUMPAD_SUBTRACT), + SCROLL_UP(KeyEvent.KEYCODE_A), + SCROLL_DOWN(KeyEvent.KEYCODE_S); + + private final int mKeyCode; + MouseKeyEvent(int enumValue) { + mKeyCode = enumValue; + } + + private static final SparseArray<MouseKeyEvent> VALUE_TO_ENUM_MAP = new SparseArray<>(); + + static { + for (MouseKeyEvent type : MouseKeyEvent.values()) { + VALUE_TO_ENUM_MAP.put(type.mKeyCode, type); + } + } + + public final int getKeyCodeValue() { + return mKeyCode; + } + + /** + * Convert int value of the key code to corresponding MouseEvent enum. If no matching + * value is found, this will return {@code null}. + */ + @Nullable + public static MouseKeyEvent from(int value) { + return VALUE_TO_ENUM_MAP.get(value); + } + } + + /** + * Construct a new MouseKeysInterceptor. + * + * @param service The service to notify of key events + * @param inputManager InputManager to track changes to connected input devices + * @param looper Looper to use for callbacks and messages + * @param displayId Display ID to send mouse events to + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + public MouseKeysInterceptor(AccessibilityManagerService service, InputManager inputManager, + Looper looper, int displayId) { + mAms = service; + mInputManager = inputManager; + mHandler = new Handler(looper, this); + mInputManager.registerInputDeviceListener(this, mHandler); + mDisplayId = displayId; + // Create the virtual mouse on a separate thread since virtual device creation + // should happen on an auxiliary thread, and not from the handler's thread. + new Thread(() -> { + mVirtualMouse = createVirtualMouse(); + }).start(); + + } + + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + private void sendVirtualMouseRelativeEvent(float x, float y) { + if (mVirtualMouse != null) { + mVirtualMouse.sendRelativeEvent(new VirtualMouseRelativeEvent.Builder() + .setRelativeX(x) + .setRelativeY(y) + .build() + ); + } + } + + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + private void sendVirtualMouseButtonEvent(int buttonCode, int actionCode) { + if (mVirtualMouse != null) { + mVirtualMouse.sendButtonEvent(new VirtualMouseButtonEvent.Builder() + .setAction(actionCode) + .setButtonCode(buttonCode) + .build() + ); + } + } + + /** + * Performs a mouse scroll action based on the provided key code. + * This method interprets the key code as a mouse scroll and sends + * the corresponding {@code VirtualMouseScrollEvent#mYAxisMovement}. + + * @param keyCode The key code representing the mouse scroll action. + * Supported keys are: + * <ul> + * <li>{@link MouseKeysInterceptor.MouseKeyEvent SCROLL_UP} + * <li>{@link MouseKeysInterceptor.MouseKeyEvent SCROLL_DOWN} + * </ul> + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + private void performMouseScrollAction(int keyCode) { + MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(keyCode); + float y = switch (mouseKeyEvent) { + case SCROLL_UP -> 1.0f; + case SCROLL_DOWN -> -1.0f; + default -> 0.0f; + }; + if (mVirtualMouse != null) { + mVirtualMouse.sendScrollEvent(new VirtualMouseScrollEvent.Builder() + .setYAxisMovement(y) + .build() + ); + } + if (DEBUG) { + Slog.d(LOG_TAG, "Performed mouse key event: " + mouseKeyEvent.name() + + " for scroll action with axis movement (y=" + y + ")"); + } + } + + /** + * Performs a mouse button action based on the provided key code. + * This method interprets the key code as a mouse button press and sends + * the corresponding press and release events to the virtual mouse. + + * @param keyCode The key code representing the mouse button action. + * Supported keys are: + * <ul> + * <li>{@link MouseKeysInterceptor.MouseKeyEvent LEFT_CLICK} (Primary Button) + * <li>{@link MouseKeysInterceptor.MouseKeyEvent RIGHT_CLICK} (Secondary + * Button) + * </ul> + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + private void performMouseButtonAction(int keyCode) { + MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(keyCode); + int buttonCode = switch (mouseKeyEvent) { + case LEFT_CLICK -> VirtualMouseButtonEvent.BUTTON_PRIMARY; + case RIGHT_CLICK -> VirtualMouseButtonEvent.BUTTON_SECONDARY; + default -> VirtualMouseButtonEvent.BUTTON_UNKNOWN; + }; + if (buttonCode != VirtualMouseButtonEvent.BUTTON_UNKNOWN) { + sendVirtualMouseButtonEvent(buttonCode, VirtualMouseButtonEvent.ACTION_BUTTON_PRESS); + sendVirtualMouseButtonEvent(buttonCode, VirtualMouseButtonEvent.ACTION_BUTTON_RELEASE); + } + if (DEBUG) { + if (buttonCode == VirtualMouseButtonEvent.BUTTON_UNKNOWN) { + Slog.d(LOG_TAG, "Button code is unknown for mouse key event: " + + mouseKeyEvent.name()); + } else { + Slog.d(LOG_TAG, "Performed mouse key event: " + mouseKeyEvent.name() + + " for button action"); + } + } + } + + /** + * Performs a mouse pointer action based on the provided key code. + * The method calculates the relative movement of the mouse pointer + * and sends the corresponding event to the virtual mouse. + * + * @param keyCode The key code representing the direction or button press. + * Supported keys are: + * <ul> + * <li>{@link MouseKeysInterceptor.MouseKeyEvent DIAGONAL_DOWN_LEFT} + * <li>{@link MouseKeysInterceptor.MouseKeyEvent DOWN} + * <li>{@link MouseKeysInterceptor.MouseKeyEvent DIAGONAL_DOWN_RIGHT} + * <li>{@link MouseKeysInterceptor.MouseKeyEvent LEFT} + * <li>{@link MouseKeysInterceptor.MouseKeyEvent RIGHT} + * <li>{@link MouseKeysInterceptor.MouseKeyEvent DIAGONAL_UP_LEFT} + * <li>{@link MouseKeysInterceptor.MouseKeyEvent UP} + * <li>{@link MouseKeysInterceptor.MouseKeyEvent DIAGONAL_UP_RIGHT} + * </ul> + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + private void performMousePointerAction(int keyCode) { + float x = 0f; + float y = 0f; + MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(keyCode); + switch (mouseKeyEvent) { + case DIAGONAL_DOWN_LEFT_MOVE -> { + x = -MOUSE_POINTER_MOVEMENT_STEP / sqrt(2); + y = MOUSE_POINTER_MOVEMENT_STEP / sqrt(2); + } + case DOWN_MOVE -> { + y = MOUSE_POINTER_MOVEMENT_STEP; + } + case DIAGONAL_DOWN_RIGHT_MOVE -> { + x = MOUSE_POINTER_MOVEMENT_STEP / sqrt(2); + y = MOUSE_POINTER_MOVEMENT_STEP / sqrt(2); + } + case LEFT_MOVE -> { + x = -MOUSE_POINTER_MOVEMENT_STEP; + } + case RIGHT_MOVE -> { + x = MOUSE_POINTER_MOVEMENT_STEP; + } + case DIAGONAL_UP_LEFT_MOVE -> { + x = -MOUSE_POINTER_MOVEMENT_STEP / sqrt(2); + y = -MOUSE_POINTER_MOVEMENT_STEP / sqrt(2); + } + case UP_MOVE -> { + y = -MOUSE_POINTER_MOVEMENT_STEP; + } + case DIAGONAL_UP_RIGHT_MOVE -> { + x = MOUSE_POINTER_MOVEMENT_STEP / sqrt(2); + y = -MOUSE_POINTER_MOVEMENT_STEP / sqrt(2); + } + default -> { + x = 0.0f; + y = 0.0f; + } + } + sendVirtualMouseRelativeEvent(x, y); + if (DEBUG) { + Slog.d(LOG_TAG, "Performed mouse key event: " + mouseKeyEvent.name() + + " for relative pointer movement (x=" + x + ", y=" + y + ")"); + } + } + + private boolean isMouseKey(int keyCode) { + return MouseKeyEvent.VALUE_TO_ENUM_MAP.contains(keyCode); + } + + private boolean isMouseButtonKey(int keyCode) { + return keyCode == MouseKeyEvent.LEFT_CLICK.getKeyCodeValue() + || keyCode == MouseKeyEvent.RIGHT_CLICK.getKeyCodeValue(); + } + + private boolean isMouseScrollKey(int keyCode) { + return keyCode == MouseKeyEvent.SCROLL_UP.getKeyCodeValue() + || keyCode == MouseKeyEvent.SCROLL_DOWN.getKeyCodeValue(); + } + + /** + * Create a virtual mouse using the VirtualDeviceManagerInternal. + * + * @return The created VirtualMouse. + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + private VirtualMouse createVirtualMouse() { + final VirtualDeviceManagerInternal localVdm = + LocalServices.getService(VirtualDeviceManagerInternal.class); + mVirtualDevice = localVdm.createVirtualDevice( + new VirtualDeviceParams.Builder().setName("Mouse Keys Virtual Device").build()); + VirtualMouse virtualMouse = mVirtualDevice.createVirtualMouse( + new VirtualMouseConfig.Builder() + .setInputDeviceName("Mouse Keys Virtual Mouse") + .setAssociatedDisplayId(mDisplayId) + .build()); + return virtualMouse; + } + + /** + * Handles key events and forwards mouse key events to the virtual mouse. + * + * @param event The key event to handle. + * @param policyFlags The policy flags associated with the key event. + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + @Override + public void onKeyEvent(KeyEvent event, int policyFlags) { + if (mAms.getTraceManager().isA11yTracingEnabledForTypes(FLAGS_INPUT_FILTER)) { + mAms.getTraceManager().logTrace(LOG_TAG + ".onKeyEvent", + FLAGS_INPUT_FILTER, "event=" + event + ";policyFlags=" + policyFlags); + } + boolean isDown = event.getAction() == KeyEvent.ACTION_DOWN; + int keyCode = event.getKeyCode(); + + if (!isMouseKey(keyCode)) { + // Pass non-mouse key events to the next handler + super.onKeyEvent(event, policyFlags); + } else if (keyCode == MouseKeyEvent.HOLD.getKeyCodeValue()) { + sendVirtualMouseButtonEvent(VirtualMouseButtonEvent.BUTTON_PRIMARY, + VirtualMouseButtonEvent.ACTION_BUTTON_PRESS); + } else if (keyCode == MouseKeyEvent.RELEASE.getKeyCodeValue()) { + sendVirtualMouseButtonEvent(VirtualMouseButtonEvent.BUTTON_PRIMARY, + VirtualMouseButtonEvent.ACTION_BUTTON_RELEASE); + } else if (isDown && isMouseButtonKey(keyCode)) { + performMouseButtonAction(keyCode); + } else if (isDown && isMouseScrollKey(keyCode)) { + // If the scroll key is pressed down and no other key is active, + // set it as the active key and send a message to scroll the pointer + if (mActiveScrollKey == KEY_NOT_SET) { + mActiveScrollKey = keyCode; + mLastTimeKeyActionPerformed = event.getDownTime(); + mHandler.sendEmptyMessage(MESSAGE_SCROLL_MOUSE_POINTER); + } + } else if (isDown) { + // This is a directional key. + // If the key is pressed down and no other key is active, + // set it as the active key and send a message to move the pointer + if (mActiveMoveKey == KEY_NOT_SET) { + mActiveMoveKey = keyCode; + mLastTimeKeyActionPerformed = event.getDownTime(); + mHandler.sendEmptyMessage(MESSAGE_MOVE_MOUSE_POINTER); + } + } else if (mActiveMoveKey == keyCode) { + // If the key is released, and it is the active key, stop moving the pointer + mActiveMoveKey = KEY_NOT_SET; + mHandler.removeMessages(MESSAGE_MOVE_MOUSE_POINTER); + } else if (mActiveScrollKey == keyCode) { + // If the key is released, and it is the active key, stop scrolling the pointer + mActiveScrollKey = KEY_NOT_SET; + mHandler.removeMessages(MESSAGE_SCROLL_MOUSE_POINTER); + } else { + Slog.i(LOG_TAG, "Dropping event with key code: '" + keyCode + + "', with no matching down event from deviceId = " + event.getDeviceId()); + } + } + + /** + * Handle messages for moving or scrolling the mouse pointer. + * + * @param msg The message to handle. + * @return True if the message was handled, false otherwise. + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + @Override + public boolean handleMessage(Message msg) { + switch (msg.what) { + case MESSAGE_MOVE_MOUSE_POINTER -> + handleMouseMessage(msg.getWhen(), mActiveMoveKey, MESSAGE_MOVE_MOUSE_POINTER); + case MESSAGE_SCROLL_MOUSE_POINTER -> + handleMouseMessage(msg.getWhen(), mActiveScrollKey, + MESSAGE_SCROLL_MOUSE_POINTER); + default -> { + Slog.e(LOG_TAG, "Unexpected message type"); + return false; + } + } + return true; + } + + /** + * Handles mouse-related messages for moving or scrolling the mouse pointer. + * This method checks if the specified time interval {@code INTERVAL_MILLIS} has passed since + * the last movement or scroll action and performs the corresponding action if necessary. + * If there is an active key, the message is rescheduled to be handled again + * after the specified {@code INTERVAL_MILLIS}. + * + * @param currentTime The current time when the message is being handled. + * @param activeKey The key code representing the active key. This determines + * the direction or type of action to be performed. + * @param messageType The type of message to be handled. It can be one of the + * following: + * <ul> + * <li>{@link #MESSAGE_MOVE_MOUSE_POINTER} - for moving the mouse pointer. + * <li>{@link #MESSAGE_SCROLL_MOUSE_POINTER} - for scrolling mouse pointer. + * </ul> + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + public void handleMouseMessage(long currentTime, int activeKey, int messageType) { + if (currentTime - mLastTimeKeyActionPerformed >= INTERVAL_MILLIS) { + if (messageType == MESSAGE_MOVE_MOUSE_POINTER) { + performMousePointerAction(activeKey); + } else if (messageType == MESSAGE_SCROLL_MOUSE_POINTER) { + performMouseScrollAction(activeKey); + } + mLastTimeKeyActionPerformed = currentTime; + } + if (activeKey != KEY_NOT_SET) { + // Reschedule the message if the key is still active + mHandler.sendEmptyMessageDelayed(messageType, INTERVAL_MILLIS); + } + } + + @Override + public void onInputDeviceAdded(int deviceId) { + } + + @Override + public void onInputDeviceRemoved(int deviceId) { + } + + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + @Override + public void onDestroy() { + // Clear mouse state + mActiveMoveKey = KEY_NOT_SET; + mActiveScrollKey = KEY_NOT_SET; + mLastTimeKeyActionPerformed = 0; + mHandler.removeCallbacksAndMessages(null); + + mVirtualDevice.close(); + mInputManager.unregisterInputDeviceListener(this); + } + + @Override + public void onInputDeviceChanged(int deviceId) { + } + +} diff --git a/services/tests/servicestests/src/com/android/server/accessibility/MouseKeysInterceptorTest.kt b/services/tests/servicestests/src/com/android/server/accessibility/MouseKeysInterceptorTest.kt new file mode 100644 index 000000000000..dc8d2390ef2d --- /dev/null +++ b/services/tests/servicestests/src/com/android/server/accessibility/MouseKeysInterceptorTest.kt @@ -0,0 +1,272 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility + +import android.companion.virtual.VirtualDeviceManager +import android.companion.virtual.VirtualDeviceParams +import android.content.Context +import android.hardware.input.IInputManager +import android.hardware.input.InputManager +import android.hardware.input.InputManagerGlobal +import android.hardware.input.VirtualMouse +import android.hardware.input.VirtualMouseButtonEvent +import android.hardware.input.VirtualMouseConfig +import android.hardware.input.VirtualMouseRelativeEvent +import android.hardware.input.VirtualMouseScrollEvent +import android.os.RemoteException +import android.os.test.TestLooper +import android.platform.test.annotations.Presubmit +import android.view.KeyEvent +import androidx.test.core.app.ApplicationProvider +import com.android.server.companion.virtual.VirtualDeviceManagerInternal +import com.android.server.LocalServices +import com.android.server.testutils.OffsettableClock +import junit.framework.Assert.assertEquals +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.mockito.ArgumentCaptor +import org.mockito.Mock +import org.mockito.Mockito +import org.mockito.MockitoAnnotations +import java.util.concurrent.TimeUnit +import java.util.LinkedList +import java.util.Queue +import android.util.ArraySet + +/** + * Tests for {@link MouseKeysInterceptor} + * + * Build/Install/Run: + * atest FrameworksServicesTests:MouseKeysInterceptorTest + */ +@Presubmit +class MouseKeysInterceptorTest { + companion object { + const val DISPLAY_ID = 1 + const val DEVICE_ID = 123 + // This delay is required for key events to be sent and handled correctly. + // The handler only performs a move/scroll event if it receives the key event + // at INTERVAL_MILLIS (which happens in practice). Hence, we need this delay in the tests. + const val KEYBOARD_POST_EVENT_DELAY_MILLIS = 20L + } + + private lateinit var mouseKeysInterceptor: MouseKeysInterceptor + private val clock = OffsettableClock() + private val testLooper = TestLooper { clock.now() } + private val nextInterceptor = TrackingInterceptor() + + @Mock + private lateinit var mockAms: AccessibilityManagerService + + @Mock + private lateinit var iInputManager: IInputManager + private lateinit var testSession: InputManagerGlobal.TestSession + private lateinit var mockInputManager: InputManager + + @Mock + private lateinit var mockVirtualDeviceManagerInternal: VirtualDeviceManagerInternal + @Mock + private lateinit var mockVirtualDevice: VirtualDeviceManager.VirtualDevice + @Mock + private lateinit var mockVirtualMouse: VirtualMouse + + @Mock + private lateinit var mockTraceManager: AccessibilityTraceManager + + @Before + @Throws(RemoteException::class) + fun setUp() { + MockitoAnnotations.initMocks(this) + val context = ApplicationProvider.getApplicationContext<Context>() + testSession = InputManagerGlobal.createTestSession(iInputManager) + mockInputManager = InputManager(context) + + Mockito.`when`(mockVirtualDeviceManagerInternal.getDeviceIdsForUid(Mockito.anyInt())) + .thenReturn(ArraySet(setOf(DEVICE_ID))) + LocalServices.removeServiceForTest(VirtualDeviceManagerInternal::class.java) + LocalServices.addService<VirtualDeviceManagerInternal>( + VirtualDeviceManagerInternal::class.java, mockVirtualDeviceManagerInternal + ) + + Mockito.`when`(mockVirtualDeviceManagerInternal.createVirtualDevice( + Mockito.any(VirtualDeviceParams::class.java) + )).thenReturn(mockVirtualDevice) + Mockito.`when`(mockVirtualDevice.createVirtualMouse( + Mockito.any(VirtualMouseConfig::class.java) + )).thenReturn(mockVirtualMouse) + + Mockito.`when`(iInputManager.inputDeviceIds).thenReturn(intArrayOf(DEVICE_ID)) + Mockito.`when`(mockAms.traceManager).thenReturn(mockTraceManager) + + mouseKeysInterceptor = MouseKeysInterceptor(mockAms, mockInputManager, + testLooper.looper, DISPLAY_ID) + // VirtualMouse is created on a separate thread. + // Wait for VirtualMouse to be created before running tests + TimeUnit.MILLISECONDS.sleep(20L) + mouseKeysInterceptor.next = nextInterceptor + } + + @After + fun tearDown() { + testLooper.dispatchAll() + if (this::testSession.isInitialized) { + testSession.close() + } + } + + @Test + fun whenNonMouseKeyEventArrives_eventIsPassedToNextInterceptor() { + val downTime = clock.now() + val downEvent = KeyEvent(downTime, downTime, KeyEvent.ACTION_DOWN, + KeyEvent.KEYCODE_Q, 0, 0, DEVICE_ID, 0) + mouseKeysInterceptor.onKeyEvent(downEvent, 0) + testLooper.dispatchAll() + + assertEquals(1, nextInterceptor.events.size) + assertEquals(downEvent, nextInterceptor.events.poll()) + } + + @Test + fun whenMouseDirectionalKeyIsPressed_relativeEventIsSent() { + // There should be some delay between the downTime of the key event and calling onKeyEvent + val downTime = clock.now() - KEYBOARD_POST_EVENT_DELAY_MILLIS + val keyCode = MouseKeysInterceptor.MouseKeyEvent.DOWN_MOVE.getKeyCodeValue() + val downEvent = KeyEvent(downTime, downTime, KeyEvent.ACTION_DOWN, + keyCode, 0, 0, DEVICE_ID, 0) + + mouseKeysInterceptor.onKeyEvent(downEvent, 0) + testLooper.dispatchAll() + + // Verify the sendRelativeEvent method is called once and capture the arguments + verifyRelativeEvents(arrayOf<Float>(0f), arrayOf<Float>(1.8f)) + } + + @Test + fun whenClickKeyIsPressed_buttonEventIsSent() { + // There should be some delay between the downTime of the key event and calling onKeyEvent + val downTime = clock.now() - KEYBOARD_POST_EVENT_DELAY_MILLIS + val keyCode = MouseKeysInterceptor.MouseKeyEvent.LEFT_CLICK.getKeyCodeValue() + val downEvent = KeyEvent(downTime, downTime, KeyEvent.ACTION_DOWN, + keyCode, 0, 0, DEVICE_ID, 0) + mouseKeysInterceptor.onKeyEvent(downEvent, 0) + testLooper.dispatchAll() + + val actions = arrayOf<Int>( + VirtualMouseButtonEvent.ACTION_BUTTON_PRESS, + VirtualMouseButtonEvent.ACTION_BUTTON_RELEASE) + val buttons = arrayOf<Int>( + VirtualMouseButtonEvent.BUTTON_PRIMARY, + VirtualMouseButtonEvent.BUTTON_PRIMARY) + // Verify the sendButtonEvent method is called twice and capture the arguments + verifyButtonEvents(actions, buttons) + } + + @Test + fun whenHoldKeyIsPressed_buttonEventIsSent() { + val downTime = clock.now() - KEYBOARD_POST_EVENT_DELAY_MILLIS + val keyCode = MouseKeysInterceptor.MouseKeyEvent.HOLD.getKeyCodeValue() + val downEvent = KeyEvent(downTime, downTime, KeyEvent.ACTION_DOWN, + keyCode, 0, 0, DEVICE_ID, 0) + mouseKeysInterceptor.onKeyEvent(downEvent, 0) + testLooper.dispatchAll() + + // Verify the sendButtonEvent method is called once and capture the arguments + verifyButtonEvents( + arrayOf<Int>( VirtualMouseButtonEvent.ACTION_BUTTON_PRESS), + arrayOf<Int>( VirtualMouseButtonEvent.BUTTON_PRIMARY) + ) + } + + @Test + fun whenReleaseKeyIsPressed_buttonEventIsSent() { + val downTime = clock.now() - KEYBOARD_POST_EVENT_DELAY_MILLIS + val keyCode = MouseKeysInterceptor.MouseKeyEvent.RELEASE.getKeyCodeValue() + val downEvent = KeyEvent(downTime, downTime, KeyEvent.ACTION_DOWN, + keyCode, 0, 0, DEVICE_ID, 0) + mouseKeysInterceptor.onKeyEvent(downEvent, 0) + testLooper.dispatchAll() + + // Verify the sendButtonEvent method is called once and capture the arguments + verifyButtonEvents( + arrayOf<Int>( VirtualMouseButtonEvent.ACTION_BUTTON_RELEASE), + arrayOf<Int>( VirtualMouseButtonEvent.BUTTON_PRIMARY) + ) + } + + @Test + fun whenScrollUpKeyIsPressed_scrollEventIsSent() { + // There should be some delay between the downTime of the key event and calling onKeyEvent + val downTime = clock.now() - KEYBOARD_POST_EVENT_DELAY_MILLIS + val keyCode = MouseKeysInterceptor.MouseKeyEvent.SCROLL_UP.getKeyCodeValue() + val downEvent = KeyEvent(downTime, downTime, KeyEvent.ACTION_DOWN, + keyCode, 0, 0, DEVICE_ID, 0) + + mouseKeysInterceptor.onKeyEvent(downEvent, 0) + testLooper.dispatchAll() + + // Verify the sendScrollEvent method is called once and capture the arguments + verifyScrollEvents(arrayOf<Float>(0f), arrayOf<Float>(1.0f)) + } + + private fun verifyRelativeEvents(expectedX: Array<Float>, expectedY: Array<Float>) { + assertEquals(expectedX.size, expectedY.size) + val captor = ArgumentCaptor.forClass(VirtualMouseRelativeEvent::class.java) + Mockito.verify(mockVirtualMouse, Mockito.times(expectedX.size)) + .sendRelativeEvent(captor.capture()) + + for (i in expectedX.indices) { + val captorEvent = captor.allValues[i] + assertEquals(expectedX[i], captorEvent.relativeX) + assertEquals(expectedY[i], captorEvent.relativeY) + } + } + + private fun verifyButtonEvents(actions: Array<Int>, buttons: Array<Int>) { + assertEquals(actions.size, buttons.size) + val captor = ArgumentCaptor.forClass(VirtualMouseButtonEvent::class.java) + Mockito.verify(mockVirtualMouse, Mockito.times(actions.size)) + .sendButtonEvent(captor.capture()) + + for (i in actions.indices) { + val captorEvent = captor.allValues[i] + assertEquals(actions[i], captorEvent.action) + assertEquals(buttons[i], captorEvent.buttonCode) + } + } + + private fun verifyScrollEvents(xAxisMovements: Array<Float>, yAxisMovements: Array<Float>) { + assertEquals(xAxisMovements.size, yAxisMovements.size) + val captor = ArgumentCaptor.forClass(VirtualMouseScrollEvent::class.java) + Mockito.verify(mockVirtualMouse, Mockito.times(xAxisMovements.size)) + .sendScrollEvent(captor.capture()) + + for (i in xAxisMovements.indices) { + val captorEvent = captor.allValues[i] + assertEquals(xAxisMovements[i], captorEvent.xAxisMovement) + assertEquals(yAxisMovements[i], captorEvent.yAxisMovement) + } + } + + private class TrackingInterceptor : BaseEventStreamTransformation() { + val events: Queue<KeyEvent> = LinkedList() + + override fun onKeyEvent(event: KeyEvent, policyFlags: Int) { + events.add(event) + } + } +} |