diff options
8 files changed, 372 insertions, 38 deletions
diff --git a/core/api/current.txt b/core/api/current.txt index fd4bb9532f2c..62091d483c02 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -20665,6 +20665,7 @@ package android.inputmethodservice { @UiContext public class InputMethodService extends android.inputmethodservice.AbstractInputMethodService { ctor public InputMethodService(); method @Deprecated public boolean enableHardwareAcceleration(); + method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public final void finishConnectionlessStylusHandwriting(@Nullable CharSequence); method public final void finishStylusHandwriting(); method public int getBackDisposition(); method public int getCandidatesHiddenVisibility(); @@ -20718,6 +20719,7 @@ package android.inputmethodservice { method public void onPrepareStylusHandwriting(); method public boolean onShowInputRequested(int, boolean); method public void onStartCandidatesView(android.view.inputmethod.EditorInfo, boolean); + method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public boolean onStartConnectionlessStylusHandwriting(int, @Nullable android.view.inputmethod.CursorAnchorInfo); method public void onStartInput(android.view.inputmethod.EditorInfo, boolean); method public void onStartInputView(android.view.inputmethod.EditorInfo, boolean); method public boolean onStartStylusHandwriting(); diff --git a/core/java/android/inputmethodservice/IInputMethodWrapper.java b/core/java/android/inputmethodservice/IInputMethodWrapper.java index b99996ff83c8..f5b58b920efb 100644 --- a/core/java/android/inputmethodservice/IInputMethodWrapper.java +++ b/core/java/android/inputmethodservice/IInputMethodWrapper.java @@ -32,6 +32,7 @@ import android.os.ResultReceiver; import android.util.Log; import android.view.InputChannel; import android.view.MotionEvent; +import android.view.inputmethod.CursorAnchorInfo; import android.view.inputmethod.ImeTracker; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputConnection; @@ -40,6 +41,7 @@ import android.view.inputmethod.InputMethodSession; import android.view.inputmethod.InputMethodSubtype; import com.android.internal.inputmethod.CancellationGroup; +import com.android.internal.inputmethod.IConnectionlessHandwritingCallback; import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback; import com.android.internal.inputmethod.IInputMethod; import com.android.internal.inputmethod.IInputMethodSession; @@ -85,6 +87,8 @@ class IInputMethodWrapper extends IInputMethod.Stub private static final int DO_UPDATE_TOOL_TYPE = 140; private static final int DO_REMOVE_STYLUS_HANDWRITING_WINDOW = 150; private static final int DO_SET_STYLUS_WINDOW_IDLE_TIMEOUT = 160; + private static final int DO_COMMIT_HANDWRITING_DELEGATION_TEXT_IF_AVAILABLE = 170; + private static final int DO_DISCARD_HANDWRITING_DELEGATION_TEXT = 180; final WeakReference<InputMethodServiceInternal> mTarget; final Context mContext; @@ -265,9 +269,13 @@ class IInputMethodWrapper extends IInputMethod.Stub return; } case DO_CAN_START_STYLUS_HANDWRITING: { + final SomeArgs args = (SomeArgs) msg.obj; if (isValid(inputMethod, target, "DO_CAN_START_STYLUS_HANDWRITING")) { - inputMethod.canStartStylusHandwriting(msg.arg1); + inputMethod.canStartStylusHandwriting(msg.arg1, + (IConnectionlessHandwritingCallback) args.arg1, + (CursorAnchorInfo) args.arg2, msg.arg2 != 0); } + args.recycle(); return; } case DO_UPDATE_TOOL_TYPE: { @@ -309,6 +317,19 @@ class IInputMethodWrapper extends IInputMethod.Stub } return; } + case DO_COMMIT_HANDWRITING_DELEGATION_TEXT_IF_AVAILABLE: { + if (isValid(inputMethod, target, + "DO_COMMIT_HANDWRITING_DELEGATION_TEXT_IF_AVAILABLE")) { + inputMethod.commitHandwritingDelegationTextIfAvailable(); + } + return; + } + case DO_DISCARD_HANDWRITING_DELEGATION_TEXT: { + if (isValid(inputMethod, target, "DO_DISCARD_HANDWRITING_DELEGATION_TEXT")) { + inputMethod.discardHandwritingDelegationText(); + } + return; + } } Log.w(TAG, "Unhandled message code: " + msg.what); } @@ -457,10 +478,13 @@ class IInputMethodWrapper extends IInputMethod.Stub @BinderThread @Override - public void canStartStylusHandwriting(int requestId) + public void canStartStylusHandwriting(int requestId, + IConnectionlessHandwritingCallback connectionlessCallback, + CursorAnchorInfo cursorAnchorInfo, boolean isConnectionlessForDelegation) throws RemoteException { - mCaller.executeOrSendMessage( - mCaller.obtainMessageI(DO_CAN_START_STYLUS_HANDWRITING, requestId)); + mCaller.executeOrSendMessage(mCaller.obtainMessageIIOO(DO_CAN_START_STYLUS_HANDWRITING, + requestId, isConnectionlessForDelegation ? 1 : 0, connectionlessCallback, + cursorAnchorInfo)); } @BinderThread @@ -483,6 +507,19 @@ class IInputMethodWrapper extends IInputMethod.Stub @BinderThread @Override + public void commitHandwritingDelegationTextIfAvailable() { + mCaller.executeOrSendMessage( + mCaller.obtainMessage(DO_COMMIT_HANDWRITING_DELEGATION_TEXT_IF_AVAILABLE)); + } + + @BinderThread + @Override + public void discardHandwritingDelegationText() { + mCaller.executeOrSendMessage(mCaller.obtainMessage(DO_DISCARD_HANDWRITING_DELEGATION_TEXT)); + } + + @BinderThread + @Override public void initInkWindow() { mCaller.executeOrSendMessage(mCaller.obtainMessage(DO_INIT_INK_WINDOW)); } diff --git a/core/java/android/inputmethodservice/InputMethodService.java b/core/java/android/inputmethodservice/InputMethodService.java index 281ee500f741..2c7ca27e6b07 100644 --- a/core/java/android/inputmethodservice/InputMethodService.java +++ b/core/java/android/inputmethodservice/InputMethodService.java @@ -51,6 +51,10 @@ import static android.view.ViewGroup.LayoutParams.MATCH_PARENT; import static android.view.ViewGroup.LayoutParams.WRAP_CONTENT; import static android.view.WindowInsets.Type.navigationBars; import static android.view.WindowInsets.Type.statusBars; +import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED; +import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_OTHER; +import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED; +import static android.view.inputmethod.Flags.FLAG_CONNECTIONLESS_HANDWRITING; import static java.lang.annotation.RetentionPolicy.SOURCE; @@ -58,6 +62,7 @@ import android.annotation.AnyThread; import android.annotation.CallSuper; import android.annotation.DrawableRes; import android.annotation.DurationMillisLong; +import android.annotation.FlaggedApi; import android.annotation.IntDef; import android.annotation.MainThread; import android.annotation.NonNull; @@ -89,6 +94,7 @@ import android.os.Handler; import android.os.IBinder; import android.os.Looper; import android.os.Process; +import android.os.RemoteException; import android.os.ResultReceiver; import android.os.SystemClock; import android.os.SystemProperties; @@ -97,6 +103,7 @@ import android.provider.Settings; import android.text.InputType; import android.text.Layout; import android.text.Spannable; +import android.text.TextUtils; import android.text.method.MovementMethod; import android.util.Log; import android.util.PrintWriterPrinter; @@ -123,6 +130,7 @@ import android.view.WindowInsets.Type; import android.view.WindowManager; import android.view.animation.AnimationUtils; import android.view.inputmethod.CompletionInfo; +import android.view.inputmethod.ConnectionlessHandwritingCallback; import android.view.inputmethod.CursorAnchorInfo; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.ExtractedText; @@ -151,6 +159,7 @@ import android.window.WindowMetricsHelper; import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; +import com.android.internal.inputmethod.IConnectionlessHandwritingCallback; import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback; import com.android.internal.inputmethod.IInputContentUriToken; import com.android.internal.inputmethod.IInputMethod; @@ -174,6 +183,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.OptionalInt; +import java.util.concurrent.Executor; /** * InputMethodService provides a standard implementation of an InputMethod, @@ -665,6 +675,12 @@ public class InputMethodService extends AbstractInputMethodService { /** Stylus handwriting Ink window. */ private InkWindow mInkWindow; + private IConnectionlessHandwritingCallback mConnectionlessHandwritingCallback; + private boolean mIsConnectionlessHandwritingForDelegation; + // Holds the recognized text from a connectionless handwriting session which can later be + // committed by commitHandwritingDelegationTextIfAvailable(). + private CharSequence mHandwritingDelegationText; + /** * An opaque {@link Binder} token of window requesting {@link InputMethodImpl#showSoftInput} * The original app window token is passed from client app window. @@ -1017,7 +1033,10 @@ public class InputMethodService extends AbstractInputMethodService { * @hide */ @Override - public void canStartStylusHandwriting(int requestId) { + public void canStartStylusHandwriting(int requestId, + @Nullable IConnectionlessHandwritingCallback connectionlessCallback, + @Nullable CursorAnchorInfo cursorAnchorInfo, + boolean isConnectionlessForDelegation) { if (DEBUG) Log.v(TAG, "canStartStylusHandwriting()"); if (mHandwritingRequestId.isPresent()) { Log.d(TAG, "There is an ongoing Handwriting session. ignoring."); @@ -1034,7 +1053,24 @@ public class InputMethodService extends AbstractInputMethodService { } // reset flag as it's not relevant after onStartStylusHandwriting(). mOnPreparedStylusHwCalled = false; - if (onStartStylusHandwriting()) { + if (connectionlessCallback != null) { + if (onStartConnectionlessStylusHandwriting( + InputType.TYPE_CLASS_TEXT, cursorAnchorInfo)) { + mConnectionlessHandwritingCallback = connectionlessCallback; + mIsConnectionlessHandwritingForDelegation = isConnectionlessForDelegation; + cancelStylusWindowIdleTimeout(); + mPrivOps.onStylusHandwritingReady(requestId, Process.myPid()); + } else { + Log.i(TAG, "IME is not ready " + + "or doesn't currently support connectionless handwriting"); + try { + connectionlessCallback.onError( + CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED); + } catch (RemoteException e) { + Log.e(TAG, "Couldn't send connectionless handwriting error result", e); + } + } + } else if (onStartStylusHandwriting()) { cancelStylusWindowIdleTimeout(); mPrivOps.onStylusHandwritingReady(requestId, Process.myPid()); } else { @@ -1093,6 +1129,24 @@ public class InputMethodService extends AbstractInputMethodService { * @hide */ @Override + public void commitHandwritingDelegationTextIfAvailable() { + InputMethodService.this.commitHandwritingDelegationTextIfAvailable(); + } + + /** + * {@inheritDoc} + * @hide + */ + @Override + public void discardHandwritingDelegationText() { + InputMethodService.this.discardHandwritingDelegationText(); + } + + /** + * {@inheritDoc} + * @hide + */ + @Override public void initInkWindow() { maybeCreateAndInitInkWindow(); onPrepareStylusHandwriting(); @@ -2581,6 +2635,32 @@ public class InputMethodService extends AbstractInputMethodService { } /** + * Called when an app requests to start a connectionless stylus handwriting session using one of + * {@link InputMethodManager#startConnectionlessStylusHandwriting(View, CursorAnchorInfo, + * Executor, ConnectionlessHandwritingCallback)}, {@link + * InputMethodManager#startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo, + * Executor, ConnectionlessHandwritingCallback)}, or {@link + * InputMethodManager#startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo, + * String, Executor, ConnectionlessHandwritingCallback)}. + * + * <p>A connectionless stylus handwriting session differs from a regular session in that an + * input connection is not used to communicate with a text editor. Instead, the recognised text + * is delivered when the IME finishes the connectionless session using {@link + * #finishConnectionlessStylusHandwriting(CharSequence)}. + * + * <p>If the IME can start the connectionless handwriting session, it should return {@code + * true}, ensure its inking views are attached to the {@link #getStylusHandwritingWindow()}, and + * handle stylus input received from {@link #onStylusHandwritingMotionEvent(MotionEvent)} on the + * {@link #getStylusHandwritingWindow()}. + */ + @FlaggedApi(FLAG_CONNECTIONLESS_HANDWRITING) + public boolean onStartConnectionlessStylusHandwriting( + int inputType, @Nullable CursorAnchorInfo cursorAnchorInfo) { + // Intentionally empty + return false; + } + + /** * Called after {@link #onStartStylusHandwriting()} returns {@code true} for every Stylus * {@link MotionEvent}. * By default, this method forwards all {@link MotionEvent}s to the @@ -2647,16 +2727,19 @@ public class InputMethodService extends AbstractInputMethodService { /** * Finish the current stylus handwriting session. * - * This dismisses the {@link #getStylusHandwritingWindow ink window} and stops intercepting + * <p>This dismisses the {@link #getStylusHandwritingWindow ink window} and stops intercepting * stylus {@code MotionEvent}s. * - * Note for IME developers: Call this method at any time to finish current handwriting session. - * Generally, this should be invoked after a short timeout, giving the user enough time + * <p>Note for IME developers: Call this method at any time to finish the current handwriting + * session. Generally, this should be invoked after a short timeout, giving the user enough time * to start the next stylus stroke, if any. By default, system will time-out after few seconds. * To override default timeout, use {@link #setStylusHandwritingSessionTimeout(Duration)}. * - * Handwriting session will be finished by framework on next {@link #onFinishInput()}. + * <p>Handwriting session will be finished by framework on next {@link #onFinishInput()}. */ + // TODO(b/300979854): Once connectionless APIs are finalised, update documentation to add: + // <p>Connectionless handwriting sessions should be finished using {@link + // #finishConnectionlessStylusHandwriting(CharSequence)}. public final void finishStylusHandwriting() { if (DEBUG) Log.v(TAG, "finishStylusHandwriting()"); if (mInkWindow == null) { @@ -2677,12 +2760,71 @@ public class InputMethodService extends AbstractInputMethodService { mHandwritingEventReceiver = null; mInkWindow.hide(false /* remove */); + if (mConnectionlessHandwritingCallback != null) { + Log.i(TAG, "Connectionless handwriting session did not complete successfully"); + try { + mConnectionlessHandwritingCallback.onError(CONNECTIONLESS_HANDWRITING_ERROR_OTHER); + } catch (RemoteException e) { + Log.e(TAG, "Couldn't send connectionless handwriting error result", e); + } + mConnectionlessHandwritingCallback = null; + } + mIsConnectionlessHandwritingForDelegation = false; + mPrivOps.resetStylusHandwriting(requestId); mOnPreparedStylusHwCalled = false; onFinishStylusHandwriting(); } /** + * Finishes the current connectionless stylus handwriting session and delivers the result. + * + * <p>This dismisses the {@link #getStylusHandwritingWindow ink window} and stops intercepting + * stylus {@code MotionEvent}s. + * + * <p>Note for IME developers: Call this method at any time to finish the current handwriting + * session. Generally, this should be invoked after a short timeout, giving the user enough time + * to start the next stylus stroke, if any. By default, system will time-out after few seconds. + * To override default timeout, use {@link #setStylusHandwritingSessionTimeout(Duration)}. + */ + @FlaggedApi(FLAG_CONNECTIONLESS_HANDWRITING) + public final void finishConnectionlessStylusHandwriting(@Nullable CharSequence text) { + if (DEBUG) Log.v(TAG, "finishConnectionlessStylusHandwriting()"); + if (mConnectionlessHandwritingCallback != null) { + try { + if (!TextUtils.isEmpty(text)) { + mConnectionlessHandwritingCallback.onResult(text); + if (mIsConnectionlessHandwritingForDelegation) { + mHandwritingDelegationText = text; + } + } else { + mConnectionlessHandwritingCallback.onError( + CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED); + } + } catch (RemoteException e) { + Log.e(TAG, "Couldn't send connectionless handwriting result", e); + } + mConnectionlessHandwritingCallback = null; + } + finishStylusHandwriting(); + } + + private void commitHandwritingDelegationTextIfAvailable() { + if (!TextUtils.isEmpty(mHandwritingDelegationText)) { + InputConnection ic = getCurrentInputConnection(); + if (ic != null) { + // Place cursor after inserted text. + ic.commitText(mHandwritingDelegationText, /* newCursorPosition= */ 1); + } + } + mHandwritingDelegationText = null; + } + + private void discardHandwritingDelegationText() { + mHandwritingDelegationText = null; + } + + /** * Remove Stylus handwriting window. * Typically, this is called when {@link InkWindow} should no longer be holding a surface in * memory. diff --git a/core/java/android/view/inputmethod/InputMethod.java b/core/java/android/view/inputmethod/InputMethod.java index 5b4efd8ced85..33f34c5697c4 100644 --- a/core/java/android/view/inputmethod/InputMethod.java +++ b/core/java/android/view/inputmethod/InputMethod.java @@ -32,6 +32,7 @@ import android.view.InputChannel; import android.view.MotionEvent; import android.view.View; +import com.android.internal.inputmethod.IConnectionlessHandwritingCallback; import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback; import com.android.internal.inputmethod.IInputMethod; import com.android.internal.inputmethod.InlineSuggestionsRequestInfo; @@ -387,10 +388,20 @@ public interface InputMethod { /** * Checks if IME is ready to start stylus handwriting session. * If yes, {@link #startStylusHandwriting(int, InputChannel, List)} is called. - * @param requestId + * + * @param requestId identifier for the session start request + * @param connectionlessCallback the callback to receive the session result if starting a + * connectionless handwriting session, or null if starting a regular session + * @param cursorAnchorInfo optional positional information about the view receiving stylus + * events for a connectionless handwriting session + * @param isConnectionlessForDelegation whether the connectionless handwriting session is for + * delegation. If true, the recognised text should be saved and can later be committed by + * {@link #commitHandwritingDelegationTextIfAvailable}. * @hide */ - default void canStartStylusHandwriting(int requestId) { + default void canStartStylusHandwriting(int requestId, + @Nullable IConnectionlessHandwritingCallback connectionlessCallback, + @Nullable CursorAnchorInfo cursorAnchorInfo, boolean isConnectionlessForDelegation) { // intentionally empty } @@ -413,6 +424,26 @@ public interface InputMethod { } /** + * Commits recognised text that was previously saved from a connectionless handwriting session + * for delegation. + * + * @hide + */ + default void commitHandwritingDelegationTextIfAvailable() { + // intentionally empty + } + + /** + * Discards recognised text that was previously saved from a connectionless handwriting session + * for delegation. + * + * @hide + */ + default void discardHandwritingDelegationText() { + // intentionally empty + } + + /** * Initialize Ink window early-on. * @hide */ diff --git a/core/java/com/android/internal/inputmethod/IInputMethod.aidl b/core/java/com/android/internal/inputmethod/IInputMethod.aidl index 8cb568d6e0e6..6abd9e8a8a17 100644 --- a/core/java/com/android/internal/inputmethod/IInputMethod.aidl +++ b/core/java/com/android/internal/inputmethod/IInputMethod.aidl @@ -20,11 +20,13 @@ import android.os.IBinder; import android.os.ResultReceiver; import android.view.InputChannel; import android.view.MotionEvent; +import android.view.inputmethod.CursorAnchorInfo; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.ImeTracker; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputMethodSubtype; import android.window.ImeOnBackInvokedDispatcher; +import com.android.internal.inputmethod.IConnectionlessHandwritingCallback; import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback; import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; import com.android.internal.inputmethod.IInputMethodSession; @@ -79,11 +81,17 @@ oneway interface IInputMethod { void changeInputMethodSubtype(in InputMethodSubtype subtype); - void canStartStylusHandwriting(int requestId); + void canStartStylusHandwriting(int requestId, + in IConnectionlessHandwritingCallback connectionlessCallback, + in CursorAnchorInfo cursorAnchorInfo, boolean isConnectionlessForDelegation); void startStylusHandwriting(int requestId, in InputChannel channel, in List<MotionEvent> events); + void commitHandwritingDelegationTextIfAvailable(); + + void discardHandwritingDelegationText(); + void initInkWindow(); void finishStylusHandwriting(); diff --git a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java index f96bb8fb6c6f..23fe5cca3d96 100644 --- a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java +++ b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java @@ -83,16 +83,19 @@ final class HandwritingModeController { private @Nullable String mDelegatePackageName; private @Nullable String mDelegatorPackageName; private boolean mDelegatorFromDefaultHomePackage; + private boolean mDelegationConnectionlessFlow; private Runnable mDelegationIdleTimeoutRunnable; private Handler mDelegationIdleTimeoutHandler; private IntConsumer mPointerToolTypeConsumer; + private final Runnable mDiscardDelegationTextRunnable; private HandwritingEventReceiverSurface mHandwritingSurface; private int mCurrentRequestId; @AnyThread HandwritingModeController(Context context, Looper uiThreadLooper, - Runnable inkWindowInitRunnable, IntConsumer toolTypeConsumer) { + Runnable inkWindowInitRunnable, IntConsumer toolTypeConsumer, + Runnable discardDelegationTextRunnable) { mContext = context; mLooper = uiThreadLooper; mCurrentDisplayId = Display.INVALID_DISPLAY; @@ -102,6 +105,7 @@ final class HandwritingModeController { mCurrentRequestId = 0; mInkWindowInitRunnable = inkWindowInitRunnable; mPointerToolTypeConsumer = toolTypeConsumer; + mDiscardDelegationTextRunnable = discardDelegationTextRunnable; } /** @@ -163,7 +167,8 @@ final class HandwritingModeController { * @see InputMethodManager#prepareStylusHandwritingDelegation(View, String) */ void prepareStylusHandwritingDelegation( - int userId, @NonNull String delegatePackageName, @NonNull String delegatorPackageName) { + int userId, @NonNull String delegatePackageName, @NonNull String delegatorPackageName, + boolean connectionless) { mDelegatePackageName = delegatePackageName; mDelegatorPackageName = delegatorPackageName; mDelegatorFromDefaultHomePackage = false; @@ -177,10 +182,13 @@ final class HandwritingModeController { delegatorPackageName.equals(defaultHomeActivity.getPackageName()); } } - if (mHandwritingBuffer == null) { - mHandwritingBuffer = new ArrayList<>(getHandwritingBufferSize()); - } else { - mHandwritingBuffer.ensureCapacity(getHandwritingBufferSize()); + mDelegationConnectionlessFlow = connectionless; + if (!connectionless) { + if (mHandwritingBuffer == null) { + mHandwritingBuffer = new ArrayList<>(getHandwritingBufferSize()); + } else { + mHandwritingBuffer.ensureCapacity(getHandwritingBufferSize()); + } } scheduleHandwritingDelegationTimeout(); } @@ -197,6 +205,10 @@ final class HandwritingModeController { return mDelegatorFromDefaultHomePackage; } + boolean isDelegationUsingConnectionlessFlow() { + return mDelegationConnectionlessFlow; + } + private void scheduleHandwritingDelegationTimeout() { if (mDelegationIdleTimeoutHandler == null) { mDelegationIdleTimeoutHandler = new Handler(mLooper); @@ -238,6 +250,10 @@ final class HandwritingModeController { mDelegatorPackageName = null; mDelegatePackageName = null; mDelegatorFromDefaultHomePackage = false; + if (mDelegationConnectionlessFlow) { + mDelegationConnectionlessFlow = false; + mDiscardDelegationTextRunnable.run(); + } } /** @@ -342,7 +358,9 @@ final class HandwritingModeController { } } - clearPendingHandwritingDelegation(); + if (!mDelegationConnectionlessFlow) { + clearPendingHandwritingDelegation(); + } mRecordingGesture = false; } @@ -390,7 +408,8 @@ final class HandwritingModeController { // If handwriting delegation is ongoing, don't clear the buffer so that multiple strokes // can be buffered across windows. - if (TextUtils.isEmpty(mDelegatePackageName) + // (This isn't needed for the connectionless delegation flow.) + if ((TextUtils.isEmpty(mDelegatePackageName) || mDelegationConnectionlessFlow) && (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL)) { mRecordingGesture = false; mHandwritingBuffer.clear(); diff --git a/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java b/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java index b12a816738da..776184fb098c 100644 --- a/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java +++ b/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java @@ -27,6 +27,7 @@ import android.os.ResultReceiver; import android.util.Slog; import android.view.InputChannel; import android.view.MotionEvent; +import android.view.inputmethod.CursorAnchorInfo; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.ImeTracker; import android.view.inputmethod.InputBinding; @@ -34,6 +35,7 @@ import android.view.inputmethod.InputMethod; import android.view.inputmethod.InputMethodSubtype; import android.window.ImeOnBackInvokedDispatcher; +import com.android.internal.inputmethod.IConnectionlessHandwritingCallback; import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback; import com.android.internal.inputmethod.IInputMethod; import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; @@ -242,9 +244,12 @@ final class IInputMethodInvoker { } @AnyThread - void canStartStylusHandwriting(int requestId) { + void canStartStylusHandwriting(int requestId, + IConnectionlessHandwritingCallback connectionlessCallback, + CursorAnchorInfo cursorAnchorInfo, boolean isConnectionlessForDelegation) { try { - mTarget.canStartStylusHandwriting(requestId); + mTarget.canStartStylusHandwriting(requestId, connectionlessCallback, cursorAnchorInfo, + isConnectionlessForDelegation); } catch (RemoteException e) { logRemoteException(e); } @@ -262,6 +267,24 @@ final class IInputMethodInvoker { } @AnyThread + void commitHandwritingDelegationTextIfAvailable() { + try { + mTarget.commitHandwritingDelegationTextIfAvailable(); + } catch (RemoteException e) { + logRemoteException(e); + } + } + + @AnyThread + void discardHandwritingDelegationText() { + try { + mTarget.discardHandwritingDelegationText(); + } catch (RemoteException e) { + logRemoteException(e); + } + } + + @AnyThread void initInkWindow() { try { mTarget.initInkWindow(); diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index 5db18ad7a07f..4bb8e1913199 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -47,6 +47,8 @@ import static android.view.Display.DEFAULT_DISPLAY; import static android.view.Display.INVALID_DISPLAY; import static android.view.WindowManager.DISPLAY_IME_POLICY_HIDE; import static android.view.WindowManager.DISPLAY_IME_POLICY_LOCAL; +import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_OTHER; +import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED; import static com.android.server.inputmethod.ImeVisibilityStateComputer.ImeTargetWindowState; import static com.android.server.inputmethod.ImeVisibilityStateComputer.ImeVisibilityResult; @@ -1688,8 +1690,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub IntConsumer toolTypeConsumer = Flags.useHandwritingListenerForTooltype() ? toolType -> onUpdateEditorToolType(toolType) : null; + Runnable discardDelegationTextRunnable = () -> discardHandwritingDelegationText(); mHwController = new HandwritingModeController(mContext, thread.getLooper(), - new InkWindowInitializer(), toolTypeConsumer); + new InkWindowInitializer(), toolTypeConsumer, discardDelegationTextRunnable); registerDeviceListenerAndCheckStylusSupport(); } @@ -1719,6 +1722,15 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + private void discardHandwritingDelegationText() { + synchronized (ImfLock.class) { + IInputMethodInvoker curMethod = getCurMethodLocked(); + if (curMethod != null) { + curMethod.discardHandwritingDelegationText(); + } + } + } + @GuardedBy("ImfLock.class") private void resetDefaultImeLocked(Context context) { // Do not reset the default (current) IME when it is a 3rd-party IME @@ -3388,58 +3400,106 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub public void startConnectionlessStylusHandwriting(IInputMethodClient client, int userId, @Nullable CursorAnchorInfo cursorAnchorInfo, @Nullable String delegatePackageName, @Nullable String delegatorPackageName, - @NonNull IConnectionlessHandwritingCallback callback) { - // TODO(b/300979854) + @NonNull IConnectionlessHandwritingCallback callback) throws RemoteException { + synchronized (ImfLock.class) { + if (!mBindingController.supportsConnectionlessStylusHandwriting()) { + Slog.w(TAG, "Connectionless stylus handwriting mode unsupported by IME."); + callback.onError(CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED); + return; + } + } + + IConnectionlessHandwritingCallback immsCallback = callback; + boolean isForDelegation = delegatePackageName != null && delegatorPackageName != null; + if (isForDelegation) { + synchronized (ImfLock.class) { + if (!mClientController.verifyClientAndPackageMatch(client, delegatorPackageName)) { + Slog.w(TAG, "startConnectionlessStylusHandwriting() fail"); + callback.onError(CONNECTIONLESS_HANDWRITING_ERROR_OTHER); + throw new IllegalArgumentException("Delegator doesn't match UID"); + } + } + immsCallback = new IConnectionlessHandwritingCallback.Stub() { + @Override + public void onResult(CharSequence text) throws RemoteException { + synchronized (ImfLock.class) { + mHwController.prepareStylusHandwritingDelegation( + userId, delegatePackageName, delegatorPackageName, + /* connectionless= */ true); + } + callback.onResult(text); + } + + @Override + public void onError(int errorCode) throws RemoteException { + callback.onError(errorCode); + } + }; + } + + if (!startStylusHandwriting( + client, false, immsCallback, cursorAnchorInfo, isForDelegation)) { + callback.onError(CONNECTIONLESS_HANDWRITING_ERROR_OTHER); + } + } + + private void startStylusHandwriting(IInputMethodClient client, boolean acceptingDelegation) { + startStylusHandwriting(client, acceptingDelegation, null, null, false); } - private void startStylusHandwriting(IInputMethodClient client, boolean usesDelegation) { + private boolean startStylusHandwriting(IInputMethodClient client, boolean acceptingDelegation, + IConnectionlessHandwritingCallback connectionlessCallback, + CursorAnchorInfo cursorAnchorInfo, boolean isConnectionlessForDelegation) { Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMMS.startStylusHandwriting"); try { ImeTracing.getInstance().triggerManagerServiceDump( "InputMethodManagerService#startStylusHandwriting"); int uid = Binder.getCallingUid(); synchronized (ImfLock.class) { - if (!usesDelegation) { + if (!acceptingDelegation) { mHwController.clearPendingHandwritingDelegation(); } if (!canInteractWithImeLocked(uid, client, "startStylusHandwriting", null /* statsToken */)) { - return; + return false; } if (!hasSupportedStylusLocked()) { Slog.w(TAG, "No supported Stylus hardware found on device. Ignoring" + " startStylusHandwriting()"); - return; + return false; } final long ident = Binder.clearCallingIdentity(); try { if (!mBindingController.supportsStylusHandwriting()) { Slog.w(TAG, "Stylus HW unsupported by IME. Ignoring startStylusHandwriting()"); - return; + return false; } final OptionalInt requestId = mHwController.getCurrentRequestId(); if (!requestId.isPresent()) { Slog.e(TAG, "Stylus handwriting was not initialized."); - return; + return false; } if (!mHwController.isStylusGestureOngoing()) { Slog.e(TAG, "There is no ongoing stylus gesture to start stylus handwriting."); - return; + return false; } if (mHwController.hasOngoingStylusHandwritingSession()) { // prevent duplicate calls to startStylusHandwriting(). Slog.e(TAG, "Stylus handwriting session is already ongoing." + " Ignoring startStylusHandwriting()."); - return; + return false; } if (DEBUG) Slog.v(TAG, "Client requesting Stylus Handwriting to be started"); final IInputMethodInvoker curMethod = getCurMethodLocked(); if (curMethod != null) { - curMethod.canStartStylusHandwriting(requestId.getAsInt()); + curMethod.canStartStylusHandwriting(requestId.getAsInt(), + connectionlessCallback, cursorAnchorInfo, + isConnectionlessForDelegation); + return true; } } finally { Binder.restoreCallingIdentity(ident); @@ -3448,6 +3508,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } finally { Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); } + return false; } @Override @@ -3487,8 +3548,18 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub if (!verifyDelegator(client, delegatePackageName, delegatorPackageName, flags)) { return false; } - - startStylusHandwriting(client, true /* usesDelegation */); + synchronized (ImfLock.class) { + if (mHwController.isDelegationUsingConnectionlessFlow()) { + final IInputMethodInvoker curMethod = getCurMethodLocked(); + if (curMethod == null) { + return false; + } + curMethod.commitHandwritingDelegationTextIfAvailable(); + mHwController.clearPendingHandwritingDelegation(); + } else { + startStylusHandwriting(client, true /* acceptingDelegation */); + } + } return true; } @@ -5002,7 +5073,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub int userId = msg.arg1; String delegate = (String) ((Pair) msg.obj).first; String delegator = (String) ((Pair) msg.obj).second; - mHwController.prepareStylusHandwritingDelegation(userId, delegate, delegator); + mHwController.prepareStylusHandwritingDelegation( + userId, delegate, delegator, /* connectionless= */ false); } return true; case MSG_START_HANDWRITING: |