diff options
27 files changed, 1544 insertions, 35 deletions
diff --git a/Android.bp b/Android.bp index f40aab15e26e..a80a5d3c6996 100644 --- a/Android.bp +++ b/Android.bp @@ -348,6 +348,7 @@ java_defaults { "core/java/android/view/accessibility/IAccessibilityManagerClient.aidl", "core/java/android/view/autofill/IAutoFillManager.aidl", "core/java/android/view/autofill/IAutoFillManagerClient.aidl", + "core/java/android/view/autofill/IAugmentedAutofillManagerClient.aidl", "core/java/android/view/autofill/IAutofillWindowPresenter.aidl", "core/java/android/view/intelligence/IIntelligenceManager.aidl", "core/java/android/view/IApplicationToken.aidl", diff --git a/api/system-current.txt b/api/system-current.txt index 5f0d20b5caec..ff85d2f30aad 100644 --- a/api/system-current.txt +++ b/api/system-current.txt @@ -4938,12 +4938,47 @@ package android.service.euicc { package android.service.intelligence { + public final class FillCallback { + method public void onSuccess(android.service.intelligence.FillResponse); + } + + public final class FillController { + method public void autofill(java.util.List<android.util.Pair<android.view.autofill.AutofillId, android.view.autofill.AutofillValue>>); + } + + public final class FillRequest { + method public android.view.autofill.AutofillId getFocusedId(); + method public android.service.intelligence.PresentationParams getPresentationParams(); + method public android.service.intelligence.InteractionSessionId getSessionId(); + } + + public final class FillResponse implements android.os.Parcelable { + method public int describeContents(); + method public void writeToParcel(android.os.Parcel, int); + field public static final android.os.Parcelable.Creator<android.service.intelligence.FillResponse> CREATOR; + } + + public static class FillResponse.Builder { + ctor public FillResponse.Builder(); + method public android.service.intelligence.FillResponse build(); + method public android.service.intelligence.FillResponse.Builder setFillWindow(android.service.intelligence.FillWindow); + method public android.service.intelligence.FillResponse.Builder setIgnoredIds(java.util.List<android.view.autofill.AutofillId>); + } + + public final class FillWindow { + ctor public FillWindow(); + method public void destroy(); + method public boolean update(android.service.intelligence.PresentationParams.Area, android.view.View, long); + field public static final long FLAG_METADATA_ADDRESS = 1L; // 0x1L + } + public abstract class IntelligenceService extends android.app.Service { ctor public IntelligenceService(); method public void onActivitySnapshot(android.service.intelligence.InteractionSessionId, android.service.intelligence.SnapshotData); method public abstract void onContentCaptureEvent(android.service.intelligence.InteractionSessionId, java.util.List<android.view.intelligence.ContentCaptureEvent>); method public void onCreateInteractionSession(android.service.intelligence.InteractionContext, android.service.intelligence.InteractionSessionId); method public void onDestroyInteractionSession(android.service.intelligence.InteractionSessionId); + method public void onFillRequest(android.service.intelligence.InteractionSessionId, android.service.intelligence.FillRequest, android.os.CancellationSignal, android.service.intelligence.FillController, android.service.intelligence.FillCallback); field public static final java.lang.String SERVICE_INTERFACE = "android.service.intelligence.IntelligenceService"; } @@ -4965,6 +5000,23 @@ package android.service.intelligence { field public static final android.os.Parcelable.Creator<android.service.intelligence.InteractionSessionId> CREATOR; } + public abstract class PresentationParams { + method public int getFlags(); + method public android.service.intelligence.PresentationParams.Area getFullArea(); + method public android.service.intelligence.PresentationParams.Area getSuggestionArea(); + field public static final int FLAG_HINT_GRAVITY_BOTTOM = 2; // 0x2 + field public static final int FLAG_HINT_GRAVITY_LEFT = 4; // 0x4 + field public static final int FLAG_HINT_GRAVITY_RIGHT = 8; // 0x8 + field public static final int FLAG_HINT_GRAVITY_TOP = 1; // 0x1 + field public static final int FLAG_HOST_IME = 16; // 0x10 + field public static final int FLAG_HOST_SYSTEM = 32; // 0x20 + } + + public static abstract class PresentationParams.Area { + method public android.graphics.Rect getBounds(); + method public android.service.intelligence.PresentationParams.Area getSubArea(android.graphics.Rect); + } + public final class SnapshotData implements android.os.Parcelable { method public int describeContents(); method public android.app.assist.AssistContent getAssistContent(); diff --git a/api/test-current.txt b/api/test-current.txt index 1c01cf1daf18..e3c165934ce8 100644 --- a/api/test-current.txt +++ b/api/test-current.txt @@ -985,6 +985,7 @@ package android.provider { public static final class Settings.Global extends android.provider.Settings.NameValueTable { field public static final java.lang.String AUTOFILL_COMPAT_MODE_ALLOWED_PACKAGES = "autofill_compat_mode_allowed_packages"; + field public static final java.lang.String AUTOFILL_SMART_SUGGESTION_EMULATION_FLAGS = "autofill_smart_suggestion_emulation_flags"; field public static final java.lang.String AUTOMATIC_POWER_SAVER_MODE = "automatic_power_saver_mode"; field public static final java.lang.String DYNAMIC_POWER_SAVINGS_DISABLE_THRESHOLD = "dynamic_power_savings_disable_threshold"; field public static final java.lang.String DYNAMIC_POWER_SAVINGS_ENABLED = "dynamic_power_savings_enabled"; diff --git a/core/java/android/provider/Settings.java b/core/java/android/provider/Settings.java index d579f0ff3681..fdb87cdc8e3a 100644 --- a/core/java/android/provider/Settings.java +++ b/core/java/android/provider/Settings.java @@ -12701,6 +12701,17 @@ public final class Settings { public static final String AUTOFILL_MAX_VISIBLE_DATASETS = "autofill_max_visible_datasets"; /** + * Used to emulate Smart Suggestion for Augmented Autofill during development + * + * <p>Valid values: {@code 0x1} for IME and/or {@code 0x2} for popup window. + * + * @hide + */ + @TestApi + public static final String AUTOFILL_SMART_SUGGESTION_EMULATION_FLAGS = + "autofill_smart_suggestion_emulation_flags"; + + /** * Exemptions to the hidden API blacklist. * * @hide diff --git a/core/java/android/service/intelligence/FillCallback.java b/core/java/android/service/intelligence/FillCallback.java new file mode 100644 index 000000000000..af2da79170ef --- /dev/null +++ b/core/java/android/service/intelligence/FillCallback.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.service.intelligence; + +import android.annotation.Nullable; +import android.annotation.SystemApi; + +/** + * Callback used to indicate at {@link FillRequest} has been fulfilled. + * + * @hide + */ +@SystemApi +public final class FillCallback { + + FillCallback() {} + + /** + * Sets the response associated with the request. + * + * @param response response associated with the request, or {@code null} if the service + * could not provide autofill for the request. + */ + public void onSuccess(@Nullable FillResponse response) { + final FillWindow fillWindow = response.getFillWindow(); + if (fillWindow != null) { + fillWindow.show(); + } + // TODO(b/111330312): properly implement on server-side by updating the Session state + // accordingly (and adding CTS tests) + } +} diff --git a/core/java/android/service/intelligence/FillController.java b/core/java/android/service/intelligence/FillController.java new file mode 100644 index 000000000000..c5e1242842bb --- /dev/null +++ b/core/java/android/service/intelligence/FillController.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.service.intelligence; + +import static android.service.intelligence.IntelligenceService.DEBUG; + +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.os.RemoteException; +import android.service.intelligence.IntelligenceService.AutofillProxy; +import android.util.Log; +import android.util.Pair; +import android.view.autofill.AutofillId; +import android.view.autofill.AutofillValue; + +import com.android.internal.util.Preconditions; + +import java.util.List; + +/** + * Object used to interact with the autofill system. + * + * @hide + */ +@SystemApi +public final class FillController { + private static final String TAG = "FillController"; + + private final AutofillProxy mProxy; + + FillController(@NonNull AutofillProxy proxy) { + mProxy = proxy; + } + + /** + * Fills the activity with the provided values. + * + * <p>As a side effect, the {@link FillWindow} associated with the {@link FillResponse} will be + * automatically {@link FillWindow#destroy() destroyed}. + */ + public void autofill(@NonNull List<Pair<AutofillId, AutofillValue>> values) { + Preconditions.checkNotNull(values); + + if (DEBUG) { + Log.d(TAG, "autofill() with " + values.size() + " values"); + } + + try { + mProxy.autofill(values); + final FillWindow fillWindow = mProxy.getFillWindow(); + if (fillWindow != null) { + fillWindow.destroy(); + } + } catch (RemoteException e) { + e.rethrowAsRuntimeException(); + } + } +} diff --git a/core/java/android/service/intelligence/FillRequest.java b/core/java/android/service/intelligence/FillRequest.java new file mode 100644 index 000000000000..95e922487906 --- /dev/null +++ b/core/java/android/service/intelligence/FillRequest.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.service.intelligence; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.service.intelligence.IntelligenceService.AutofillProxy; +import android.view.autofill.AutofillId; + +/** + * Represents a request to augment-fill an activity. + * @hide + */ +@SystemApi +public final class FillRequest { + + final AutofillProxy mProxy; + + /** @hide */ + FillRequest(@NonNull AutofillProxy proxy) { + mProxy = proxy; + } + + /** + * Gets the session associated with this request. + */ + @NonNull + public InteractionSessionId getSessionId() { + return mProxy.sessionId; + } + + /** + * Gets the id of the field that triggered the request. + */ + @NonNull + public AutofillId getFocusedId() { + return mProxy.focusedId; + } + + /** + * Gets the Smart Suggestions object used to embed the autofill UI. + * + * @return object used to embed the autofill UI, or {@code null} if not supported. + */ + @Nullable + public PresentationParams getPresentationParams() { + return mProxy.getSmartSuggestionParams(); + } + + @Override + public String toString() { + return "FillRequest[id=" + mProxy.focusedId + "]"; + } +} diff --git a/core/java/android/service/intelligence/FillResponse.java b/core/java/android/service/intelligence/FillResponse.java new file mode 100644 index 000000000000..860c0275732a --- /dev/null +++ b/core/java/android/service/intelligence/FillResponse.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.service.intelligence; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.os.Parcel; +import android.os.Parcelable; +import android.view.autofill.AutofillId; + +import java.util.List; + +/** + * Response to a {@link FillRequest}. + * + * @hide + */ +@SystemApi +public final class FillResponse implements Parcelable { + + private final FillWindow mFillWindow; + + private FillResponse(@NonNull Builder builder) { + mFillWindow = builder.mFillWindow; + } + + /** @hide */ + @Nullable + FillWindow getFillWindow() { + return mFillWindow; + } + + /** + * Builder for {@link FillResponse} objects. + * + * @hide + */ + @SystemApi + public static class Builder { + + private FillWindow mFillWindow; + + /** + * Sets the {@link FillWindow} used to display the Autofill UI. + * + * <p>Must be called when the service is handling the request so the Android System can + * properly synchronize the UI. + * + * @return this builder + */ + public Builder setFillWindow(@NonNull FillWindow fillWindow) { + // TODO(b/111330312): implement / check not null / unit test + // TODO(b/111330312): throw exception if FillWindow not updated yet + mFillWindow = fillWindow; + return this; + } + + /** + * Tells the Android System that the given {@code ids} should not trigger further + * {@link FillRequest requests} when focused. + * + * @param ids ids of the fields that should be ignored + * + * @return this builder + */ + public Builder setIgnoredIds(@NonNull List<AutofillId> ids) { + // TODO(b/111330312): implement / check not null / unit test + return this; + } + + /** + * Builds a new {@link FillResponse} instance. + * + * @throws IllegalStateException if any of the following conditions occur: + * <ol> + * <li>{@link #build()} was already called. + * <li>No call was made to {@link #setFillWindow(FillWindow)} or + * {@ling #setIgnoredIds(List<AutofillId>)}. + * </ol> + * + * @return A built response. + */ + public FillResponse build() { + // TODO(b/111330312): check conditions / add unit test + return new FillResponse(this); + } + + // TODO(b/111330312): add methods to disable app / activity, either here or on manager + } + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(Parcel parcel, int flags) { + // TODO(b/111330312): implement + } + + public static final Parcelable.Creator<FillResponse> CREATOR = + new Parcelable.Creator<FillResponse>() { + + @Override + public FillResponse createFromParcel(Parcel parcel) { + // TODO(b/111330312): implement + return null; + } + + @Override + public FillResponse[] newArray(int size) { + return new FillResponse[size]; + } + }; +} diff --git a/core/java/android/service/intelligence/FillWindow.java b/core/java/android/service/intelligence/FillWindow.java new file mode 100644 index 000000000000..4ea07bfc86cf --- /dev/null +++ b/core/java/android/service/intelligence/FillWindow.java @@ -0,0 +1,215 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.service.intelligence; + +import static android.service.intelligence.IntelligenceService.DEBUG; + +import android.annotation.LongDef; +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.app.Dialog; +import android.graphics.Rect; +import android.service.intelligence.PresentationParams.Area; +import android.util.Log; +import android.view.Gravity; +import android.view.View; +import android.view.ViewGroup; +import android.view.Window; +import android.view.WindowManager; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.util.Preconditions; + +import java.io.PrintWriter; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Handle to a window used to display the augmented autofill UI. + * + * <p>The steps to create an augmented autofill UI are: + * + * <ol> + * <li>Gets the {@link PresentationParams} from the {@link FillRequest}. + * <li>Gets the {@link Area} to display the UI (for example, through + * {@link PresentationParams#getSuggestionArea()}. + * <li>Creates a {@link View} that must fit in the {@link Area#getBounds() area boundaries}. + * <li>Set the proper listeners to the view (for example, a click listener that + * triggers {@link FillController#autofill(java.util.List)} + * <li>Call {@link #update(Area, View, long)} with these arguments. + * <li>Create a {@link FillResponse} with the {@link FillWindow}. + * <li>Pass such {@link FillResponse} to {@link FillCallback#onSuccess(FillResponse)}. + * </ol> + * + * @hide + */ +@SystemApi +public final class FillWindow { + private static final String TAG = "FillWindow"; + + /** Indicates the data being shown is a physical address */ + public static final long FLAG_METADATA_ADDRESS = 0x1; + + // TODO(b/111330312): add moar flags + + /** @hide */ + @LongDef(prefix = { "FLAG" }, value = { + FLAG_METADATA_ADDRESS, + }) + @Retention(RetentionPolicy.SOURCE) + @interface Flags{} + + private final Object mLock = new Object(); + + @GuardedBy("mLock") + private Dialog mDialog; + + @GuardedBy("mLock") + private boolean mDestroyed; + + /** + * Updates the content of the window. + * + * @param rootView new root view + * @param area coordinates to render the view. + * @param flags optional flags such as metadata of what will be rendered in the window. The + * Smart Suggestion host might decide whether or not to render the UI based on them. + * + * @return boolean whether the window was updated or not. + * + * @throws IllegalArgumentException if the area is not compatible with this window + */ + public boolean update(@NonNull Area area, @NonNull View rootView, @Flags long flags) { + if (DEBUG) { + Log.d(TAG, "Updating " + area + " + with " + rootView); + } + // TODO(b/111330312): add test case for null + Preconditions.checkNotNull(area); + Preconditions.checkNotNull(rootView); + // TODO(b/111330312): must check the area is a valid object returned by + // SmartSuggestionParams, throw IAE if not + + // TODO(b/111330312): must some how pass metadata to the SmartSuggestiongs provider + + + // TODO(b/111330312): use a SurfaceControl approach; for now, we're manually creating + // the window underneath the existing view. + + final PresentationParams smartSuggestion = area.proxy.getSmartSuggestionParams(); + if (smartSuggestion == null) { + Log.w(TAG, "No SmartSuggestionParams"); + return false; + } + + final Rect rect = area.getBounds(); + if (rect == null) { + Log.wtf(TAG, "No Rect on SmartSuggestionParams"); + return false; + } + + synchronized (mLock) { + checkNotDestroyedLocked(); + + // TODO(b/111330312): once we have the SurfaceControl approach, we should update the + // window instead of destroying. In fact, it might be better to allocate a full window + // initially, which is transparent (and let touches get through) everywhere but in the + // rect boundaries. + destroy(); + + // TODO(b/111330312): make sure all touch events are handled, window is always closed, + // etc. + + mDialog = new Dialog(rootView.getContext()); + final Window window = mDialog.getWindow(); + window.setType(WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY); + + final int height = rect.bottom - rect.top; + final int width = rect.right - rect.left; + final WindowManager.LayoutParams windowParams = window.getAttributes(); + windowParams.gravity = Gravity.TOP | Gravity.CENTER_HORIZONTAL; + windowParams.y = rect.top - height; + windowParams.height = height; + windowParams.x = rect.left; + windowParams.width = width; + + window.setAttributes(windowParams); + window.clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND); + + mDialog.requestWindowFeature(Window.FEATURE_NO_TITLE); + final ViewGroup.LayoutParams diagParams = new ViewGroup.LayoutParams(width, height); + mDialog.setContentView(rootView, diagParams); + + if (DEBUG) { + Log.d(TAG, "Created FillWindow: params= " + smartSuggestion + " view=" + rootView); + } + + area.proxy.setFillWindow(this); + return true; + } + } + + /** @hide */ + void show() { + // TODO(b/111330312): check if updated first / throw exception + if (DEBUG) Log.d(TAG, "show()"); + + synchronized (mLock) { + checkNotDestroyedLocked(); + if (mDialog == null) { + throw new IllegalStateException("update() not called yet, or already destroyed()"); + } + + mDialog.show(); + } + } + + /** + * Destroys the window. + * + * <p>Once destroyed, this window cannot be used anymore + */ + public void destroy() { + if (DEBUG) Log.d(TAG, "destroy(): mDestroyed = " + mDestroyed); + + synchronized (this) { + if (mDestroyed) return; + + if (mDialog != null) { + mDialog.dismiss(); + mDialog = null; + } + } + } + + private void checkNotDestroyedLocked() { + if (mDestroyed) { + throw new IllegalStateException("already destroyed()"); + } + } + + /** @hide */ + public void dump(@NonNull String prefix, @NonNull PrintWriter pw) { + synchronized (this) { + pw.print(prefix); pw.print("destroyed: "); pw.println(mDestroyed); + if (mDialog != null) { + pw.print(prefix); pw.print("dialog: "); + pw.println(mDialog.isShowing() ? "shown" : "hidden"); + pw.print(prefix); pw.print("window: "); + pw.println(mDialog.getWindow().getAttributes()); + } + } + } +} diff --git a/core/java/android/service/intelligence/IIntelligenceService.aidl b/core/java/android/service/intelligence/IIntelligenceService.aidl index 709c3b720579..e2260d7d3d76 100644 --- a/core/java/android/service/intelligence/IIntelligenceService.aidl +++ b/core/java/android/service/intelligence/IIntelligenceService.aidl @@ -16,10 +16,12 @@ package android.service.intelligence; +import android.os.IBinder; import android.service.intelligence.InteractionSessionId; import android.service.intelligence.InteractionContext; import android.service.intelligence.SnapshotData; +import android.view.autofill.AutofillId; import android.view.intelligence.ContentCaptureEvent; import java.util.List; @@ -40,4 +42,9 @@ oneway interface IIntelligenceService { void onActivitySnapshot(in InteractionSessionId sessionId, in SnapshotData snapshotData); + + void onAutofillRequest(in InteractionSessionId sessionId, in IBinder autofillManagerClient, + int autofilSessionId, in AutofillId focusedId); + + void onDestroyAutofillWindowsRequest(in InteractionSessionId sessionId); } diff --git a/core/java/android/service/intelligence/IntelligenceService.java b/core/java/android/service/intelligence/IntelligenceService.java index 27569b6003c5..040e25e977e4 100644 --- a/core/java/android/service/intelligence/IntelligenceService.java +++ b/core/java/android/service/intelligence/IntelligenceService.java @@ -22,13 +22,26 @@ import android.annotation.NonNull; import android.annotation.SystemApi; import android.app.Service; import android.content.Intent; +import android.graphics.Rect; +import android.os.CancellationSignal; import android.os.Handler; import android.os.IBinder; import android.os.Looper; import android.os.RemoteException; +import android.service.intelligence.PresentationParams.SystemPopupPresentationParams; +import android.util.ArrayMap; import android.util.Log; +import android.util.Pair; +import android.view.autofill.AutofillId; +import android.view.autofill.AutofillValue; +import android.view.autofill.IAugmentedAutofillManagerClient; import android.view.intelligence.ContentCaptureEvent; +import com.android.internal.annotations.GuardedBy; + +import java.io.FileDescriptor; +import java.io.PrintWriter; +import java.util.ArrayList; import java.util.List; /** @@ -44,6 +57,9 @@ public abstract class IntelligenceService extends Service { private static final String TAG = "IntelligenceService"; + // TODO(b/111330312): STOPSHIP use dynamic value, or change to false + static final boolean DEBUG = true; + /** * The {@link Intent} that must be declared as handled by the service. * To be supported, the service must also require the @@ -55,6 +71,8 @@ public abstract class IntelligenceService extends Service { private Handler mHandler; + private ArrayMap<InteractionSessionId, AutofillProxy> mAutofillProxies; + private final IIntelligenceService mInterface = new IIntelligenceService.Stub() { @Override @@ -87,6 +105,20 @@ public abstract class IntelligenceService extends Service { obtainMessage(IntelligenceService::onActivitySnapshot, IntelligenceService.this, sessionId, snapshotData)); } + + @Override + public void onAutofillRequest(InteractionSessionId sessionId, IBinder client, + int autofilSessionId, AutofillId focusedId) { + mHandler.sendMessage(obtainMessage(IntelligenceService::handleOnAutofillRequest, + IntelligenceService.this, sessionId, client, autofilSessionId, focusedId)); + } + + @Override + public void onDestroyAutofillWindowsRequest(InteractionSessionId sessionId) { + mHandler.sendMessage( + obtainMessage(IntelligenceService::handleOnDestroyAutofillWindowsRequest, + IntelligenceService.this, sessionId)); + } }; @CallSuper @@ -122,10 +154,93 @@ public abstract class IntelligenceService extends Service { * @param sessionId the session's Id * @param events the events */ - // TODO(b/111276913): rename to onContentCaptureEvents + // TODO(b/111276913): rename to onContentCaptureEvents or something like that; also, pass a + // Request object so it can be extended public abstract void onContentCaptureEvent(@NonNull InteractionSessionId sessionId, @NonNull List<ContentCaptureEvent> events); + private void handleOnAutofillRequest(@NonNull InteractionSessionId sessionId, + @NonNull IBinder client, int autofillSessionId, @NonNull AutofillId focusedId) { + if (mAutofillProxies == null) { + mAutofillProxies = new ArrayMap<>(); + } + AutofillProxy proxy = mAutofillProxies.get(sessionId); + if (proxy == null) { + proxy = new AutofillProxy(sessionId, client, autofillSessionId, focusedId); + mAutofillProxies.put(sessionId, proxy); + } else { + // TODO(b/111330312): figure out if it's ok to reuse the proxy; add logging + if (DEBUG) Log.d(TAG, "Reusing proxy for session " + sessionId); + } + // TODO(b/111330312): set cancellation signal + final CancellationSignal cancellationSignal = null; + onFillRequest(sessionId, new FillRequest(proxy), cancellationSignal, + new FillController(proxy), new FillCallback()); + } + + /** + * Asks the service to handle an "augmented" autofill request. + * + * <p>This method is called when the "stantard" autofill service cannot handle a request, which + * typically occurs when: + * <ul> + * <li>Service does not recognize what should be autofilled. + * <li>Service does not have data to fill the request. + * <li>Service blacklisted that app (or activity) for autofill. + * <li>App disabled itself for autofill. + * </ul> + * + * <p>Differently from the standard autofill workflow, on augmented autofill the service is + * responsible to generate the autofill UI and request the Android system to autofill the + * activity when the user taps an action in that UI (through the + * {@link FillController#autofill(List)} method). + * + * <p>The service <b>MUST</b> call {@link + * FillCallback#onSuccess(android.service.intelligence.FillResponse)} as soon as possible, + * passing {@code null} when it cannot fulfill the request. + * + * @param sessionId the session's id + * @param request the request to handle. + * @param cancellationSignal signal for observing cancellation requests. The system will use + * this to notify you that the fill result is no longer needed and you should stop + * handling this fill request in order to save resources. + * @param controller object used to interact with the autofill system. + * @param callback object used to notify the result of the request. Service <b>must</b> call + * {@link FillCallback#onSuccess(android.service.intelligence.FillResponse)}. + */ + public void onFillRequest(@NonNull InteractionSessionId sessionId, @NonNull FillRequest request, + @NonNull CancellationSignal cancellationSignal, @NonNull FillController controller, + @NonNull FillCallback callback) { + } + + private void handleOnDestroyAutofillWindowsRequest(@NonNull InteractionSessionId sessionId) { + AutofillProxy proxy = null; + if (mAutofillProxies != null) { + proxy = mAutofillProxies.get(sessionId); + } + if (proxy == null) { + // TODO(b/111330312): this might be fine, in which case we should logv it + Log.w(TAG, "No proxy for session " + sessionId); + return; + } + proxy.destroy(); + mAutofillProxies.remove(sessionId); + } + + @Override + protected void dump(FileDescriptor fd, PrintWriter pw, String[] args) { + if (mAutofillProxies != null) { + final int size = mAutofillProxies.size(); + pw.print("Number proxies: "); pw.println(size); + for (int i = 0; i < size; i++) { + final InteractionSessionId sessionId = mAutofillProxies.keyAt(i); + final AutofillProxy proxy = mAutofillProxies.valueAt(i); + pw.print(i); pw.print(") SessionId="); pw.print(sessionId); pw.println(":"); + proxy.dump(" ", pw); + } + } + } + /** * Notifies the service of {@link IntelligenceSnapshotData snapshot data} associated with a * session. @@ -142,4 +257,99 @@ public abstract class IntelligenceService extends Service { * @param sessionId the id of the session to destroy */ public void onDestroyInteractionSession(@NonNull InteractionSessionId sessionId) {} + + /** @hide */ + static final class AutofillProxy { + private final Object mLock = new Object(); + private final IAugmentedAutofillManagerClient mClient; + private final int mAutofillSessionId; + public final InteractionSessionId sessionId; + public final AutofillId focusedId; + + @GuardedBy("mLock") + private SystemPopupPresentationParams mSmartSuggestion; + + @GuardedBy("mLock") + private FillWindow mFillWindow; + + private AutofillProxy(@NonNull InteractionSessionId sessionId, @NonNull IBinder client, + int autofillSessionId, @NonNull AutofillId focusedId) { + this.sessionId = sessionId; + mClient = IAugmentedAutofillManagerClient.Stub.asInterface(client); + mAutofillSessionId = autofillSessionId; + this.focusedId = focusedId; + // TODO(b/111330312): linkToDeath + } + + @NonNull + public SystemPopupPresentationParams getSmartSuggestionParams() { + synchronized (mLock) { + if (mSmartSuggestion != null) { + return mSmartSuggestion; + } + Rect rect; + try { + rect = mClient.getViewCoordinates(focusedId); + } catch (RemoteException e) { + Log.w(TAG, "Could not get coordinates for " + focusedId); + return null; + } + if (rect == null) { + if (DEBUG) Log.d(TAG, "getViewCoordinates(" + focusedId + ") returned null"); + return null; + } + mSmartSuggestion = new SystemPopupPresentationParams(this, rect); + return mSmartSuggestion; + } + } + + public void autofill(@NonNull List<Pair<AutofillId, AutofillValue>> pairs) + throws RemoteException { + final int size = pairs.size(); + final List<AutofillId> ids = new ArrayList<>(size); + final List<AutofillValue> values = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + final Pair<AutofillId, AutofillValue> pair = pairs.get(i); + ids.add(pair.first); + values.add(pair.second); + } + mClient.autofill(mAutofillSessionId, ids, values); + } + + public void setFillWindow(@NonNull FillWindow fillWindow) { + synchronized (mLock) { + mFillWindow = fillWindow; + } + } + + public FillWindow getFillWindow() { + synchronized (mLock) { + return mFillWindow; + } + } + + public void dump(@NonNull String prefix, @NonNull PrintWriter pw) { + pw.print(prefix); pw.print("afSessionId: "); pw.println(mAutofillSessionId); + pw.print(prefix); pw.print("focusedId: "); pw.println(focusedId); + pw.print(prefix); pw.print("client: "); pw.println(mClient); + final String prefix2 = prefix + " "; + if (mFillWindow != null) { + pw.print(prefix); pw.println("window:"); + mFillWindow.dump(prefix2, pw); + } + if (mSmartSuggestion != null) { + pw.print(prefix); pw.println("smartSuggestion:"); + mSmartSuggestion.dump(prefix2, pw); + } + } + + private void destroy() { + synchronized (mLock) { + if (mFillWindow != null) { + if (DEBUG) Log.d(TAG, "destroying window"); + mFillWindow.destroy(); + } + } + } + } } diff --git a/core/java/android/service/intelligence/PresentationParams.java b/core/java/android/service/intelligence/PresentationParams.java new file mode 100644 index 000000000000..b92f8f1ada75 --- /dev/null +++ b/core/java/android/service/intelligence/PresentationParams.java @@ -0,0 +1,225 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.service.intelligence; + +import android.annotation.IntDef; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.graphics.Rect; +import android.service.intelligence.IntelligenceService.AutofillProxy; +import android.util.DebugUtils; +import android.view.View; + +import java.io.PrintWriter; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Abstraction of a "Smart Suggestion" component responsible to embed the autofill UI provided by + * the intelligence service. + * + * <p>The Smart Suggestion can embed the autofill UI in 3 distinct places: + * + * <ul> + * <li>A small area associated with suggestions (like a small strip in the top of the IME), + * returned by {@link #getSuggestionArea()} + * <li>The full area (like the full IME window), returned by {@link #getFullArea()} + * <li>A subset of the aforementioned areas, returned by {@link Area#getSubArea(Rect)} + * </ul> + * + * <p>The Smart Suggestion is represented by a {@link Area} object that contains the + * dimensions the smart suggestion window, so the service can use it to calculate the size of the + * view that will be passed to {@link FillWindow#update(Area, View, long)}. + * + * @hide + */ +@SystemApi +public abstract class PresentationParams { + + /** + * Flag indicating the Smart Suggestion is hosted in the top of its container. + */ + public static final int FLAG_HINT_GRAVITY_TOP = 0x1; + + /** + * Flag indicating the Smart Suggestion is hosted in the bottom of its container. + */ + public static final int FLAG_HINT_GRAVITY_BOTTOM = 0x2; + + /** + * Flag indicating the Smart Suggestion is hosted in the left of its container. + */ + public static final int FLAG_HINT_GRAVITY_LEFT = 0x4; + + /** + * Flag indicating the Smart Suggestion is hosted in the right of its container. + */ + public static final int FLAG_HINT_GRAVITY_RIGHT = 0x8; + + /** + * Flag indicating the Smart Suggestion is hosted by the IME. + */ + public static final int FLAG_HOST_IME = 0x10; + + /** + * Flag indicating the Smart Suggestion is hosted by the Android System as a floating popup + * window. + */ + public static final int FLAG_HOST_SYSTEM = 0x20; + + /** @hide */ + @IntDef(flag = true, prefix = { "FLAG_" }, value = { + FLAG_HINT_GRAVITY_TOP, + FLAG_HINT_GRAVITY_BOTTOM, + FLAG_HINT_GRAVITY_LEFT, + FLAG_HINT_GRAVITY_RIGHT, + FLAG_HOST_IME, + FLAG_HOST_SYSTEM + }) + @Retention(RetentionPolicy.SOURCE) + @interface Flags {} + + + // /** @hide */ + PresentationParams() {} + + /** + * Gets the area of the suggestion strip for the given {@code metadata} + * + * @return strip dimensions, or {@code null} if the Smart Suggestion provider does not support + * suggestions strip. + */ + @Nullable + public Area getSuggestionArea() { + return null; + } + + /** + * Gets the full area for the of the Smart Suggestion provider. + * + * @return full dimensions, or {@code null} if the Smart Suggestion provider does not support + * embeding the UI on its full area. + */ + @Nullable + public Area getFullArea() { + return null; + } + + /** + * Gets flags associated with the Smart Suggestion. + * + * @return any combination of {@link #FLAG_HINT_GRAVITY_TOP}, + * {@link #FLAG_HINT_GRAVITY_BOTTOM}, {@link #FLAG_HINT_GRAVITY_LEFT}, + * {@link #FLAG_HINT_GRAVITY_RIGHT}, {@link #FLAG_HOST_IME}, or + * {@link #FLAG_HOST_SYSTEM}, + */ + public @Flags int getFlags() { + return 0; + } + + /** @hide */ + void dump(@NonNull String prefix, @NonNull PrintWriter pw) { + final int flags = getFlags(); + if (flags > 0) { + pw.print(prefix); pw.print("flags: "); pw.println(flagsToString(flags)); + } + } + + private static String flagsToString(int flags) { + return DebugUtils.flagsToString(PresentationParams.class, "FLAG_", flags); + } + + /** + * Area associated with a {@link PresentationParams Smart Suggestions} provider. + * + * @hide + * */ + @SystemApi + public abstract static class Area { + + /** @hide */ + public final AutofillProxy proxy; + + private final Rect mBounds; + + private Area(@NonNull AutofillProxy proxy, @NonNull Rect bounds) { + this.proxy = proxy; + mBounds = bounds; + } + + /** + * Gets the area boundaries. + */ + @NonNull + public Rect getBounds() { + return mBounds; + } + + /** + * Gets a subarea limited by given boundaries. + * + * @param bounds boundaries relative to this Area. + * + * @throw {@link IllegalArgumentException} if the {@code bounds} is not fully-contained + * inside this full Area. + * + * @return new subarea, or {@code null} if the Smart Suggestion host does not support such + * subaarea. + */ + @Nullable + public Area getSubArea(@NonNull Rect bounds) { + // TODO(b/111330312): implement / check boundaries / throw IAE / add unit test + return null; + } + + @Override + public String toString() { + return mBounds.toString(); + } + } + + /** + * System-provided poup window anchored to a view. + * + * <p>Used just for debugging purposes. + * + * @hide + */ + public static final class SystemPopupPresentationParams extends PresentationParams { + private final Area mSuggestionArea; + + public SystemPopupPresentationParams(@NonNull AutofillProxy proxy, @NonNull Rect rect) { + mSuggestionArea = new Area(proxy, rect) {}; + } + + @Override + public Area getSuggestionArea() { + return mSuggestionArea; + } + + @Override + public int getFlags() { + return FLAG_HOST_SYSTEM | FLAG_HINT_GRAVITY_BOTTOM; + } + + @Override + void dump(@NonNull String prefix, @NonNull PrintWriter pw) { + super.dump(prefix, pw); + pw.print(prefix); pw.print("area: "); pw.println(mSuggestionArea); + } + } +} diff --git a/core/java/android/view/autofill/AutofillManager.java b/core/java/android/view/autofill/AutofillManager.java index d4c7069cdbf4..9227249fc6b1 100644 --- a/core/java/android/view/autofill/AutofillManager.java +++ b/core/java/android/view/autofill/AutofillManager.java @@ -202,6 +202,14 @@ public final class AutofillManager { public static final String EXTRA_RESTORE_SESSION_TOKEN = "android.view.autofill.extra.RESTORE_SESSION_TOKEN"; + /** + * Internal extra used to pass a binder to the {@link IAugmentedAutofillManagerClient}. + * + * @hide + */ + public static final String EXTRA_AUGMENTED_AUTOFILL_CLIENT = + "android.view.autofill.extra.AUGMENTED_AUTOFILL_CLIENT"; + private static final String SESSION_ID_TAG = "android:sessionId"; private static final String STATE_TAG = "android:state"; private static final String LAST_AUTOFILLED_DATA_TAG = "android:lastAutoFilledData"; @@ -370,6 +378,9 @@ public final class AutofillManager { private Cleaner mServiceClientCleaner; @GuardedBy("mLock") + private IAugmentedAutofillManagerClient mAugmentedAutofillServiceClient; + + @GuardedBy("mLock") private AutofillCallback mCallback; private final Context mContext; @@ -1664,6 +1675,8 @@ public final class AutofillManager { final IAutoFillManager service = mService; final IAutoFillManagerClient serviceClient = mServiceClient; mServiceClientCleaner = Cleaner.create(this, () -> { + // TODO(b/111330312): call service to also remove reference to + // mAugmentedAutofillServiceClient try { service.removeClient(serviceClient, userId); } catch (RemoteException e) { @@ -1808,6 +1821,7 @@ public final class AutofillManager { if ((flags & SET_STATE_FLAG_RESET_CLIENT) != 0) { // Reset connection to system mServiceClient = null; + mAugmentedAutofillServiceClient = null; if (mServiceClientCleaner != null) { mServiceClientCleaner.clean(); mServiceClientCleaner = null; @@ -2054,6 +2068,29 @@ public final class AutofillManager { } } + /** + * Gets a {@link AugmentedAutofillManagerClient} for this {@link AutofillManagerClient}. + * + * <p>These are 2 distinct objects because we need to restrict what the Augmented Autofill + * service can do (which is defined by {@code IAugmentedAutofillManagerClient.aidl}). + */ + private void getAugmentedAutofillClient(@NonNull IResultReceiver result) { + synchronized (mLock) { + if (mAugmentedAutofillServiceClient == null) { + mAugmentedAutofillServiceClient = new AugmentedAutofillManagerClient(this); + } + final Bundle resultData = new Bundle(); + resultData.putBinder(EXTRA_AUGMENTED_AUTOFILL_CLIENT, + mAugmentedAutofillServiceClient.asBinder()); + + try { + result.send(0, resultData); + } catch (RemoteException e) { + Log.w(TAG, "Could not send AugmentedAutofillClient back: " + e); + } + } + } + /** @hide */ public void requestHideFillUi() { requestHideFillUi(mIdShownFillUi, true); @@ -2801,7 +2838,7 @@ public final class AutofillManager { private static final class AutofillManagerClient extends IAutoFillManagerClient.Stub { private final WeakReference<AutofillManager> mAfm; - AutofillManagerClient(AutofillManager autofillManager) { + private AutofillManagerClient(AutofillManager autofillManager) { mAfm = new WeakReference<>(autofillManager); } @@ -2904,6 +2941,50 @@ public final class AutofillManager { afm.post(() -> afm.setSessionFinished(newState)); } } + + @Override + public void getAugmentedAutofillClient(IResultReceiver result) { + final AutofillManager afm = mAfm.get(); + if (afm != null) { + afm.post(() -> afm.getAugmentedAutofillClient(result)); + } + } + } + + private static final class AugmentedAutofillManagerClient + extends IAugmentedAutofillManagerClient.Stub { + private final WeakReference<AutofillManager> mAfm; + + private AugmentedAutofillManagerClient(AutofillManager autofillManager) { + mAfm = new WeakReference<>(autofillManager); + } + + @Override + public Rect getViewCoordinates(@NonNull AutofillId id) { + // TODO(b/111330312): use handler / callback? + final AutofillManager afm = mAfm.get(); + if (afm == null) return null; + + final View view = afm.getClient().autofillClientFindViewByAutofillIdTraversal(id); + // TODO(b/111330312): optimize (for example, use temp rect from attach info) and + // fix (for example, take system status bar height into account) logic below + final int[] location = new int[2]; + view.getLocationOnScreen(location); + final Rect rect = new Rect(location[0], location[1], location[0] + view.getWidth(), + location[1] + view.getHeight()); + if (sVerbose) { + Log.v(TAG, "Coordinates for " + id + ": " + rect); + } + return rect; + } + + @Override + public void autofill(int sessionId, List<AutofillId> ids, List<AutofillValue> values) { + final AutofillManager afm = mAfm.get(); + if (afm != null) { + afm.post(() -> afm.autofill(sessionId, ids, values)); + } + } } /** diff --git a/core/java/android/view/autofill/IAugmentedAutofillManagerClient.aidl b/core/java/android/view/autofill/IAugmentedAutofillManagerClient.aidl new file mode 100644 index 000000000000..67cd0bf87b99 --- /dev/null +++ b/core/java/android/view/autofill/IAugmentedAutofillManagerClient.aidl @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.autofill; + +import java.util.List; + +import android.graphics.Rect; +import android.view.autofill.AutofillId; +import android.view.autofill.AutofillValue; + +/** + * Object running in the application process and responsible to provide the functionalities + * required by an Augmented Autofill service. + * + * @hide + */ +interface IAugmentedAutofillManagerClient { + Rect getViewCoordinates(in AutofillId id); + void autofill(int sessionId, in List<AutofillId> ids, in List<AutofillValue> values); +} diff --git a/core/java/android/view/autofill/IAutoFillManagerClient.aidl b/core/java/android/view/autofill/IAutoFillManagerClient.aidl index 0ff7a0bdb963..63394b42eb5a 100644 --- a/core/java/android/view/autofill/IAutoFillManagerClient.aidl +++ b/core/java/android/view/autofill/IAutoFillManagerClient.aidl @@ -27,6 +27,8 @@ import android.view.autofill.AutofillValue; import android.view.autofill.IAutofillWindowPresenter; import android.view.KeyEvent; +import com.android.internal.os.IResultReceiver; + /** * Object running in the application process and responsible for autofilling it. * @@ -93,8 +95,18 @@ oneway interface IAutoFillManagerClient { /** * Marks the state of the session as finished. + * * @param newState STATE_FINISHED (because the autofill service returned a null * FillResponse) or STATE_UNKNOWN (because the session was removed). */ void setSessionFinished(int newState); + + /** + * Gets a reference to the binder object that can be used by the Augmented Autofill service. + * + * @param receiver, whose AutofillManager.EXTRA_AUGMENTED_AUTOFILL_CLIENT extra will contain + * the reference. + */ + void getAugmentedAutofillClient(in IResultReceiver result); + } diff --git a/core/java/android/view/intelligence/IntelligenceManager.java b/core/java/android/view/intelligence/IntelligenceManager.java index dfa52d94f4a1..755c54c5d25b 100644 --- a/core/java/android/view/intelligence/IntelligenceManager.java +++ b/core/java/android/view/intelligence/IntelligenceManager.java @@ -391,7 +391,7 @@ public final class IntelligenceManager { } /** - * Called by apps to explicitly enabled or disable content capture. + * Called by apps to explicitly enable or disable content capture. * * <p><b>Note: </b> this call is not persisted accross reboots, so apps should typically call * it on {@link android.app.Activity#onCreate(android.os.Bundle, android.os.PersistableBundle)}. diff --git a/core/tests/coretests/src/android/provider/SettingsBackupTest.java b/core/tests/coretests/src/android/provider/SettingsBackupTest.java index 4802ebea2511..baf7c1f47b75 100644 --- a/core/tests/coretests/src/android/provider/SettingsBackupTest.java +++ b/core/tests/coretests/src/android/provider/SettingsBackupTest.java @@ -124,6 +124,7 @@ public class SettingsBackupTest { Settings.Global.AUTOFILL_LOGGING_LEVEL, Settings.Global.AUTOFILL_MAX_PARTITIONS_SIZE, Settings.Global.AUTOFILL_MAX_VISIBLE_DATASETS, + Settings.Global.AUTOFILL_SMART_SUGGESTION_EMULATION_FLAGS, Settings.Global.AUTOMATIC_POWER_SAVER_MODE, Settings.Global.BATTERY_DISCHARGE_DURATION_THRESHOLD, Settings.Global.BATTERY_DISCHARGE_THRESHOLD, diff --git a/services/autofill/java/com/android/server/autofill/AutofillManagerService.java b/services/autofill/java/com/android/server/autofill/AutofillManagerService.java index 17d8ea7eb77c..c56f31efd953 100644 --- a/services/autofill/java/com/android/server/autofill/AutofillManagerService.java +++ b/services/autofill/java/com/android/server/autofill/AutofillManagerService.java @@ -18,11 +18,13 @@ package com.android.server.autofill; import static android.Manifest.permission.MANAGE_AUTO_FILL; import static android.content.Context.AUTOFILL_MANAGER_SERVICE; +import static android.util.DebugUtils.flagsToString; import static com.android.server.autofill.Helper.sDebug; import static com.android.server.autofill.Helper.sFullScreenMode; import static com.android.server.autofill.Helper.sVerbose; +import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.UserIdInt; @@ -72,9 +74,12 @@ import com.android.server.AbstractMasterSystemService; import com.android.server.FgThread; import com.android.server.LocalServices; import com.android.server.autofill.ui.AutoFillUI; +import com.android.server.intelligence.IntelligenceManagerInternal; import java.io.FileDescriptor; import java.io.PrintWriter; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -95,6 +100,27 @@ public final class AutofillManagerService private static final Object sLock = AutofillManagerService.class; + + /** + * IME supports Smart Suggestions. + */ + // NOTE: must be public because of flagsToString() + public static final int FLAG_SMART_SUGGESTION_IME = 0x1; + + /** + * System supports Smarts Suggestions (as a popup-window similar to standard Autofill). + */ + // NOTE: must be public because of flagsToString() + public static final int FLAG_SMART_SUGGESTION_SYSTEM = 0x2; + + /** @hide */ + @IntDef(flag = true, prefix = { "FLAG_SMART_SUGGESTION_" }, value = { + FLAG_SMART_SUGGESTION_IME, + FLAG_SMART_SUGGESTION_SYSTEM + }) + @Retention(RetentionPolicy.SOURCE) + @interface SmartSuggestionMode {} + static final String RECEIVER_BUNDLE_EXTRA_SESSIONS = "sessions"; private static final char COMPAT_PACKAGE_DELIMITER = ':'; @@ -102,7 +128,6 @@ public final class AutofillManagerService private static final char COMPAT_PACKAGE_URL_IDS_BLOCK_BEGIN = '['; private static final char COMPAT_PACKAGE_URL_IDS_BLOCK_END = ']'; - /** * Maximum number of partitions that can be allowed in a session. * @@ -130,6 +155,7 @@ public final class AutofillManagerService private final AutofillCompatState mAutofillCompatState = new AutofillCompatState(); private final LocalService mLocalService = new LocalService(); + final IntelligenceManagerInternal mIntelligenceManagerInternal; private final BroadcastReceiver mBroadcastReceiver = new BroadcastReceiver() { @Override @@ -153,13 +179,21 @@ public final class AutofillManagerService @GuardedBy("mLock") private boolean mAllowInstantService; + /** + * Supported modes for Augmented Autofill Smart Suggestions. + */ + @GuardedBy("mLock") + private int mSupportedSmartSuggestionModes; + public AutofillManagerService(Context context) { super(context, UserManager.DISALLOW_AUTOFILL); mUi = new AutoFillUI(ActivityThread.currentActivityThread().getSystemUiContext()); + mIntelligenceManagerInternal = LocalServices.getService(IntelligenceManagerInternal.class); setLogLevelFromSettings(); setMaxPartitionsFromSettings(); setMaxVisibleDatasetsFromSettings(); + setSmartSuggestionEmulationFromSettings(); final IntentFilter filter = new IntentFilter(); filter.addAction(Intent.ACTION_CLOSE_SYSTEM_DIALOGS); @@ -186,6 +220,9 @@ public final class AutofillManagerService resolver.registerContentObserver(Settings.Global.getUriFor( Settings.Global.AUTOFILL_MAX_VISIBLE_DATASETS), false, observer, UserHandle.USER_ALL); + resolver.registerContentObserver(Settings.Global.getUriFor( + Settings.Global.AUTOFILL_SMART_SUGGESTION_EMULATION_FLAGS), false, observer, + UserHandle.USER_ALL); } @Override // from AbstractMasterSystemService @@ -200,6 +237,9 @@ public final class AutofillManagerService case Settings.Global.AUTOFILL_MAX_VISIBLE_DATASETS: setMaxVisibleDatasetsFromSettings(); break; + case Settings.Global.AUTOFILL_SMART_SUGGESTION_EMULATION_FLAGS: + setSmartSuggestionEmulationFromSettings(); + break; default: Slog.w(TAG, "Unexpected property (" + property + "); updating cache instead"); // fall through @@ -243,6 +283,10 @@ public final class AutofillManagerService mUi.hideAll(null); } + @SmartSuggestionMode int getSupportedSmartSuggestionModesLocked() { + return mSupportedSmartSuggestionModes; + } + // Called by Shell command. void destroySessions(@UserIdInt int userId, IResultReceiver receiver) { Slog.i(TAG, "destroySessions() for userId " + userId); @@ -420,6 +464,19 @@ public final class AutofillManagerService } } + private void setSmartSuggestionEmulationFromSettings() { + final int flags = Settings.Global.getInt(getContext().getContentResolver(), + Settings.Global.AUTOFILL_SMART_SUGGESTION_EMULATION_FLAGS, 0); + if (sDebug) { + Slog.d(TAG, "setSmartSuggestionEmulationFromSettings(): " + + smartSuggestionFlagsToString(flags)); + } + + synchronized (mLock) { + mSupportedSmartSuggestionModes = flags; + } + } + // Called by Shell command. void getScore(@Nullable String algorithmName, @NonNull String value1, @NonNull String value2, @NonNull RemoteCallback callback) { @@ -610,6 +667,10 @@ public final class AutofillManagerService } } + static String smartSuggestionFlagsToString(int flags) { + return flagsToString(AutofillManagerService.class, "FLAG_SMART_SUGGESTION_", flags); + } + private final class LocalService extends AutofillManagerInternal { @Override public void onBackKeyPressed() { @@ -1158,6 +1219,10 @@ public final class AutofillManagerService pw.print("from settings: "); pw.println(getWhitelistedCompatModePackagesFromSettings()); pw.print("Allow instant service: "); pw.println(mAllowInstantService); + if (mSupportedSmartSuggestionModes != 0) { + pw.print("Smart Suggestion modes: "); + pw.println(smartSuggestionFlagsToString(mSupportedSmartSuggestionModes)); + } if (showHistory) { pw.println(); pw.println("Requests history:"); pw.println(); mRequestsHistory.reverseDump(fd, pw, args); diff --git a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java index 67ccc9b18543..0df99d4b6642 100644 --- a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java +++ b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java @@ -73,6 +73,7 @@ import com.android.internal.logging.nano.MetricsProto.MetricsEvent; import com.android.server.AbstractPerUserSystemService; import com.android.server.LocalServices; import com.android.server.autofill.AutofillManagerService.AutofillCompatState; +import com.android.server.autofill.AutofillManagerService.SmartSuggestionMode; import com.android.server.autofill.ui.AutoFillUI; import java.io.PrintWriter; @@ -268,8 +269,8 @@ final class AutofillManagerServiceImpl pruneAbandonedSessionsLocked(); final Session newSession = createSessionByTokenLocked(activityToken, taskId, uid, - appCallbackToken, hasCallback, componentName, compatMode, bindInstantServiceAllowed, - flags); + appCallbackToken, hasCallback, componentName, compatMode, + bindInstantServiceAllowed, flags); if (newSession == null) { return NO_SESSION; } @@ -823,6 +824,12 @@ final class AutofillManagerServiceImpl return true; } + @GuardedBy("mLock") + @SmartSuggestionMode int getSupportedSmartSuggestionModesLocked() { + // TODO(b/111330312): once we support IME, we need to set it per-user (OR'ed with master) + return mMaster.getSupportedSmartSuggestionModesLocked(); + } + @Override @GuardedBy("mLock") protected void dumpLocked(String prefix, PrintWriter pw) { @@ -962,6 +969,9 @@ final class AutofillManagerServiceImpl if (sDebug) Slog.d(TAG, "destroyFinishedSessionsLocked(): " + session.id); session.forceRemoveSelfLocked(); } + else { + session.destroyAugmentedAutofillWindowsLocked(); + } } } diff --git a/services/autofill/java/com/android/server/autofill/Session.java b/services/autofill/java/com/android/server/autofill/Session.java index 1ff1acdb3b57..8676f7f5bea0 100644 --- a/services/autofill/java/com/android/server/autofill/Session.java +++ b/services/autofill/java/com/android/server/autofill/Session.java @@ -25,6 +25,9 @@ import static android.view.autofill.AutofillManager.ACTION_VIEW_ENTERED; import static android.view.autofill.AutofillManager.ACTION_VIEW_EXITED; import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; +import static com.android.server.autofill.AutofillManagerService.FLAG_SMART_SUGGESTION_IME; +import static com.android.server.autofill.AutofillManagerService.FLAG_SMART_SUGGESTION_SYSTEM; +import static com.android.server.autofill.AutofillManagerService.smartSuggestionFlagsToString; import static com.android.server.autofill.Helper.getNumericValue; import static com.android.server.autofill.Helper.sDebug; import static com.android.server.autofill.Helper.sVerbose; @@ -93,8 +96,11 @@ import com.android.internal.logging.MetricsLogger; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; import com.android.internal.util.ArrayUtils; import com.android.server.AbstractRemoteService; +import com.android.server.autofill.AutofillManagerService.SmartSuggestionMode; import com.android.server.autofill.ui.AutoFillUI; import com.android.server.autofill.ui.PendingUi; +import com.android.server.intelligence.IntelligenceManagerInternal; +import com.android.server.intelligence.IntelligenceManagerInternal.AugmentedAutofillCallback; import java.io.PrintWriter; import java.util.ArrayList; @@ -242,6 +248,10 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState @GuardedBy("mLock") private final SparseArray<LogMaker> mRequestLogs = new SparseArray<>(1); + @GuardedBy("mLock") + @Nullable + private AugmentedAutofillCallback mAugmentedAutofillCallback; + /** * Receiver of assist data from the app's {@link Activity}. */ @@ -2497,15 +2507,83 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState processResponseLocked(newResponse, newClientState, 0); } + @GuardedBy("mLock") private void processNullResponseLocked(int flags) { - if (sVerbose) Slog.v(TAG, "canceling session " + id + " when server returned null"); if ((flags & FLAG_MANUAL_REQUEST) != 0) { getUiForShowing().showError(R.string.autofill_error_cannot_autofill, this); } mService.resetLastResponse(); - // Nothing to be done, but need to notify client. - notifyUnavailableToClient(AutofillManager.STATE_FINISHED); - removeSelf(); + + // The default autofill service cannot fullfill the request, let's check if the intelligence + // service can. + mAugmentedAutofillCallback = triggerAugmentedAutofillLocked(); + if (mAugmentedAutofillCallback == null) { + if (sVerbose) { + Slog.v(TAG, "canceling session " + id + " when server returned null and there is no" + + " AugmentedAutofill for user"); + } + // Nothing to be done, but need to notify client. + notifyUnavailableToClient(AutofillManager.STATE_FINISHED); + removeSelf(); + } else { + // TODO(b/111330312, b/119638958): must set internal state so when user focus other + // fields it does not generate a new call to the standard autofill service (right now + // it does). Must also add CTS tests to exercise this scenario. + if (sVerbose) { + Slog.v(TAG, "keeping session " + id + " when server returned null but " + + "there is an AugmentedAutofill for user"); + } + } + } + + /** + * Tries to trigger Augmented Autofill when the standard service could not fulfill a request. + * + * @return callback to the Augmented Autofill service, or {@code null} if not supported. + */ + // TODO(b/111330312): might need to call it in other places, like when the service returns a + // non-null response but without datasets (for example, just SaveInfo) + @GuardedBy("mLock") + private AugmentedAutofillCallback triggerAugmentedAutofillLocked() { + // Check if Smart Suggestions is supported... + final @SmartSuggestionMode int supportedModes = mService + .getSupportedSmartSuggestionModesLocked(); + if (supportedModes == 0) return null; + + // ...then if the service is set for the user + final IntelligenceManagerInternal intelligenceManagerInternal = mService + .getMaster().mIntelligenceManagerInternal; + if (intelligenceManagerInternal == null) return null; + + // Define which mode will be used + final int mode; + if ((supportedModes & FLAG_SMART_SUGGESTION_IME) != 0) { + // TODO(b/111330312): support it :-) + Slog.w(TAG, "Smart Suggestions on IME not supported yet"); + return null; + } else if ((supportedModes & FLAG_SMART_SUGGESTION_SYSTEM) != 0) { + mode = FLAG_SMART_SUGGESTION_SYSTEM; + } else { + Slog.w(TAG, "Unsupported Smart Suggestion Mode: " + supportedModes); + return null; + } + + if (mCurrentViewId == null) { + Slog.w(TAG, "triggerAugmentedAutofillLocked(): no view currently focused"); + return null; + } + + if (sVerbose) { + Slog.v(TAG, "calling IntelligenseService on view " + mCurrentViewId + + " using suggestion mode " + smartSuggestionFlagsToString(mode) + + " when server returned null for session " + this.id); + } + + // TODO(b/111330312): we might need to add a new state in the AutofillManager to optimize + // furgher AFM -> AFMS calls. + // TODO(b/119638958): add CTS tests + return intelligenceManagerInternal.requestAutofill(mService.getUserId(), mClient, + mActivityToken, this.id, mCurrentViewId); } @GuardedBy("mLock") @@ -2786,6 +2864,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState pw.print(prefix); pw.print("mSaveOnAllViewsInvisible: "); pw.println( mSaveOnAllViewsInvisible); pw.print(prefix); pw.print("mSelectedDatasetIds: "); pw.println(mSelectedDatasetIds); + if (mAugmentedAutofillCallback != null) { + pw.print(prefix); pw.println("has AugmentedAutofillCallback"); + } mRemoteFillService.dump(prefix, pw); } @@ -2957,6 +3038,14 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState Slog.e(TAG, "Error notifying client to finish session", e); } } + destroyAugmentedAutofillWindowsLocked(); + } + + @GuardedBy("mLock") + void destroyAugmentedAutofillWindowsLocked() { + if (mAugmentedAutofillCallback != null) { + mAugmentedAutofillCallback.destroy(); + } } /** diff --git a/services/core/java/com/android/server/AbstractPerUserSystemService.java b/services/core/java/com/android/server/AbstractPerUserSystemService.java index b37888f8ef2e..71d261c7f5b5 100644 --- a/services/core/java/com/android/server/AbstractPerUserSystemService.java +++ b/services/core/java/com/android/server/AbstractPerUserSystemService.java @@ -163,6 +163,20 @@ public abstract class AbstractPerUserSystemService<S extends AbstractPerUserSyst } /** + * Gets the user associated with this service. + */ + public final @UserIdInt int getUserId() { + return mUserId; + } + + /** + * Gets the master service. + */ + public final M getMaster() { + return mMaster; + } + + /** * Gets this UID of the remote service this service binds to, or {@code -1} if the service is * disabled. */ diff --git a/services/core/java/com/android/server/AbstractRemoteService.java b/services/core/java/com/android/server/AbstractRemoteService.java index 181d7fde1fb8..73a34d6e0f3c 100644 --- a/services/core/java/com/android/server/AbstractRemoteService.java +++ b/services/core/java/com/android/server/AbstractRemoteService.java @@ -205,6 +205,9 @@ public abstract class AbstractRemoteService implements DeathRecipient { protected void scheduleUnbind() { cancelScheduledUnbind(); + // TODO(b/111276913): implement "permanent binding" + // TODO(b/117779333): make sure it's unbound if the service settings changing (right now + // it's not) mHandler.sendMessageDelayed(obtainMessage(AbstractRemoteService::handleUnbind, this) .setWhat(MSG_UNBIND), getTimeoutIdleBindMillis()); } diff --git a/services/core/java/com/android/server/intelligence/IntelligenceManagerInternal.java b/services/core/java/com/android/server/intelligence/IntelligenceManagerInternal.java index aac83b62b0a5..6fe632459eaa 100644 --- a/services/core/java/com/android/server/intelligence/IntelligenceManagerInternal.java +++ b/services/core/java/com/android/server/intelligence/IntelligenceManagerInternal.java @@ -19,6 +19,8 @@ import android.annotation.NonNull; import android.annotation.UserIdInt; import android.os.Bundle; import android.os.IBinder; +import android.view.autofill.AutofillId; +import android.view.autofill.IAutoFillManagerClient; /** * Intelligence Manager local system service interface. @@ -41,4 +43,37 @@ public abstract class IntelligenceManagerInternal { */ public abstract boolean sendActivityAssistData(@UserIdInt int userId, @NonNull IBinder activityToken, @NonNull Bundle data); + + /** + * Asks the intelligence service to provide Augmented Autofill for a given activity. + * + * @param userId user handle + * @param client binder used to communicate with the activity that originated this request. + * @param activityToken activity that originated this request. + * @param autofillSessionId autofill session id (must be used on {@code client} calls. + * @param focusedId id of the the field that triggered this request. + * + * @return {@code false} if the service cannot handle this request, {@code true} otherwise. + * <b>NOTE: </b> it must return right away; typically it will return {@code false} if the + * service is disabled (or the activity blacklisted). + */ + public abstract AugmentedAutofillCallback requestAutofill(@UserIdInt int userId, + @NonNull IAutoFillManagerClient client, @NonNull IBinder activityToken, + int autofillSessionId, @NonNull AutofillId focusedId); + + /** + * Callback used by the Autofill Session to communicate with the Augmented Autofill service. + */ + public interface AugmentedAutofillCallback { + // TODO(b/111330312): this method is calling when the Autofill session is destroyed, the + // main reason being the cases where user tap HOME. + // Right now it's completely destroying the UI, but we need to decide whether / how to + // properly recover it later (for example, if the user switches back to the activity, + // should it be restored? Right not it kind of is, because Autofill's Session trigger a + // new FillRequest, which in turn triggers the Augmented Autofill request again) + /** + * Destroys the Autofill UI. + */ + void destroy(); + } } diff --git a/services/intelligence/java/com/android/server/intelligence/ContentCaptureSession.java b/services/intelligence/java/com/android/server/intelligence/ContentCaptureSession.java index 57e954f10fa7..08fbf5549a87 100644 --- a/services/intelligence/java/com/android/server/intelligence/ContentCaptureSession.java +++ b/services/intelligence/java/com/android/server/intelligence/ContentCaptureSession.java @@ -24,11 +24,14 @@ import android.service.intelligence.InteractionContext; import android.service.intelligence.InteractionSessionId; import android.service.intelligence.SnapshotData; import android.util.Slog; +import android.view.autofill.AutofillId; +import android.view.autofill.IAutoFillManagerClient; import android.view.intelligence.ContentCaptureEvent; import com.android.internal.annotations.GuardedBy; import com.android.internal.util.Preconditions; import com.android.server.AbstractRemoteService; +import com.android.server.intelligence.IntelligenceManagerInternal.AugmentedAutofillCallback; import com.android.server.intelligence.RemoteIntelligenceService.RemoteIntelligenceServiceCallbacks; import java.io.PrintWriter; @@ -39,12 +42,12 @@ final class ContentCaptureSession implements RemoteIntelligenceServiceCallbacks private static final String TAG = "ContentCaptureSession"; private final Object mLock; - private final IBinder mActivityToken; - + final IBinder mActivityToken; private final IntelligencePerUserService mService; private final RemoteIntelligenceService mRemoteService; private final InteractionContext mInterationContext; private final InteractionSessionId mId; + private AugmentedAutofillCallback mAutofillCallback; ContentCaptureSession(@NonNull Context context, int userId, @NonNull Object lock, @NonNull IBinder activityToken, @NonNull IntelligencePerUserService service, @@ -92,6 +95,18 @@ final class ContentCaptureSession implements RemoteIntelligenceServiceCallbacks } /** + * Requests the service to autofill the given field. + */ + public AugmentedAutofillCallback requestAutofillLocked(@NonNull IAutoFillManagerClient client, + int autofillSessionId, @NonNull AutofillId focusedId) { + mRemoteService.onRequestAutofillLocked(mId, client, autofillSessionId, focusedId); + if (mAutofillCallback == null) { + mAutofillCallback = () -> mRemoteService.onDestroyAutofillWindowsRequest(mId); + } + return mAutofillCallback; + } + + /** * Cleans up the session and removes it from the service. * * @param notifyRemoteService whether it should trigger a {@link @@ -119,6 +134,11 @@ final class ContentCaptureSession implements RemoteIntelligenceServiceCallbacks if (mService.isVerbose()) { Slog.v(TAG, "destroyLocked(notifyRemoteService=" + notifyRemoteService + ")"); } + if (mAutofillCallback != null) { + mAutofillCallback.destroy(); + mAutofillCallback = null; + } + // TODO(b/111276913): must call client to set session as FINISHED_BY_SERVER if (notifyRemoteService) { mRemoteService.onSessionLifecycleRequest(/* context= */ null, mId); @@ -152,6 +172,8 @@ final class ContentCaptureSession implements RemoteIntelligenceServiceCallbacks pw.print(prefix); pw.print("id: "); mId.dump(pw); pw.println(); pw.print(prefix); pw.print("context: "); mInterationContext.dump(pw); pw.println(); pw.print(prefix); pw.print("activity token: "); pw.println(mActivityToken); + pw.print(prefix); pw.print("has autofill callback: "); + pw.println(mAutofillCallback != null); } @Override diff --git a/services/intelligence/java/com/android/server/intelligence/IntelligenceManagerService.java b/services/intelligence/java/com/android/server/intelligence/IntelligenceManagerService.java index a7f45ee4c9bf..38810dd8d8d8 100644 --- a/services/intelligence/java/com/android/server/intelligence/IntelligenceManagerService.java +++ b/services/intelligence/java/com/android/server/intelligence/IntelligenceManagerService.java @@ -27,6 +27,8 @@ import android.os.Bundle; import android.os.IBinder; import android.os.UserManager; import android.service.intelligence.InteractionSessionId; +import android.view.autofill.AutofillId; +import android.view.autofill.IAutoFillManagerClient; import android.view.intelligence.ContentCaptureEvent; import android.view.intelligence.IIntelligenceManager; @@ -86,20 +88,6 @@ public final class IntelligenceManagerService extends service.destroyLocked(); } - /** - * Notifies the intelligence service of new assist data for the given activity. - * - * @return {@code false} if there was no service set for the given user - */ - private boolean sendActivityAssistDataLocked(@UserIdInt int userId, - @NonNull IBinder activityToken, @NonNull Bundle data) { - final IntelligencePerUserService service = peekServiceForUserLocked(userId); - if (service != null) { - return service.sendActivityAssistDataLocked(activityToken, data); - } - return false; - } - private ActivityManagerInternal getAmInternal() { synchronized (mLock) { if (mAm == null) { @@ -112,7 +100,7 @@ public final class IntelligenceManagerService extends final class IntelligenceManagerServiceStub extends IIntelligenceManager.Stub { @Override - public void startSession(int userId, @NonNull IBinder activityToken, + public void startSession(@UserIdInt int userId, @NonNull IBinder activityToken, @NonNull ComponentName componentName, @NonNull InteractionSessionId sessionId, int flags, @NonNull IResultReceiver result) { Preconditions.checkNotNull(activityToken); @@ -134,7 +122,7 @@ public final class IntelligenceManagerService extends } @Override - public void sendEvents(int userId, @NonNull InteractionSessionId sessionId, + public void sendEvents(@UserIdInt int userId, @NonNull InteractionSessionId sessionId, @NonNull List<ContentCaptureEvent> events) { Preconditions.checkNotNull(sessionId); Preconditions.checkNotNull(events); @@ -146,7 +134,7 @@ public final class IntelligenceManagerService extends } @Override - public void finishSession(int userId, @NonNull InteractionSessionId sessionId) { + public void finishSession(@UserIdInt int userId, @NonNull InteractionSessionId sessionId) { Preconditions.checkNotNull(sessionId); synchronized (mLock) { @@ -168,14 +156,13 @@ public final class IntelligenceManagerService extends private final class LocalService extends IntelligenceManagerInternal { @Override - public boolean isIntelligenceServiceForUser(int uid, int userId) { + public boolean isIntelligenceServiceForUser(int uid, @UserIdInt int userId) { synchronized (mLock) { final IntelligencePerUserService service = peekServiceForUserLocked(userId); if (service != null) { return service.isIntelligenceServiceForUserLocked(uid); } } - return false; } @@ -183,8 +170,26 @@ public final class IntelligenceManagerService extends public boolean sendActivityAssistData(@UserIdInt int userId, @NonNull IBinder activityToken, @NonNull Bundle data) { synchronized (mLock) { - return sendActivityAssistDataLocked(userId, activityToken, data); + final IntelligencePerUserService service = peekServiceForUserLocked(userId); + if (service != null) { + return service.sendActivityAssistDataLocked(activityToken, data); + } + } + return false; + } + + @Override + public AugmentedAutofillCallback requestAutofill(@UserIdInt int userId, + @NonNull IAutoFillManagerClient client, @NonNull IBinder activityToken, + int autofillSessionId, @NonNull AutofillId focusedId) { + synchronized (mLock) { + final IntelligencePerUserService service = peekServiceForUserLocked(userId); + if (service != null) { + return service.requestAutofill(client, activityToken, autofillSessionId, + focusedId); + } } + return null; } } } diff --git a/services/intelligence/java/com/android/server/intelligence/IntelligencePerUserService.java b/services/intelligence/java/com/android/server/intelligence/IntelligencePerUserService.java index 9694ab968e71..051f0d695fcf 100644 --- a/services/intelligence/java/com/android/server/intelligence/IntelligencePerUserService.java +++ b/services/intelligence/java/com/android/server/intelligence/IntelligencePerUserService.java @@ -22,6 +22,7 @@ import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_KEY_STRUC import android.Manifest; import android.annotation.NonNull; +import android.annotation.UserIdInt; import android.app.AppGlobals; import android.app.assist.AssistContent; import android.app.assist.AssistStructure; @@ -36,12 +37,15 @@ import android.service.intelligence.InteractionSessionId; import android.service.intelligence.SnapshotData; import android.util.ArrayMap; import android.util.Slog; +import android.view.autofill.AutofillId; +import android.view.autofill.IAutoFillManagerClient; import android.view.intelligence.ContentCaptureEvent; import android.view.intelligence.IntelligenceManager; import com.android.internal.annotations.GuardedBy; import com.android.internal.os.IResultReceiver; import com.android.server.AbstractPerUserSystemService; +import com.android.server.intelligence.IntelligenceManagerInternal.AugmentedAutofillCallback; import java.io.PrintWriter; import java.util.List; @@ -62,7 +66,7 @@ final class IntelligencePerUserService // TODO(b/111276913): add mechanism to prune stale sessions, similar to Autofill's protected IntelligencePerUserService( - IntelligenceManagerService master, Object lock, int userId) { + IntelligenceManagerService master, Object lock, @UserIdInt int userId) { super(master, lock, userId); } @@ -210,6 +214,17 @@ final class IntelligencePerUserService return uid == getServiceUidLocked(); } + @GuardedBy("mLock") + private ContentCaptureSession getSession(@NonNull IBinder activityToken) { + for (int i = 0; i < mSessions.size(); i++) { + final ContentCaptureSession session = mSessions.valueAt(i); + if (session.mActivityToken.equals(activityToken)) { + return session; + } + } + return null; + } + /** * Destroys the service and all state associated with it. * @@ -226,6 +241,22 @@ final class IntelligencePerUserService mSessions.clear(); } + public AugmentedAutofillCallback requestAutofill(@NonNull IAutoFillManagerClient client, + @NonNull IBinder activityToken, int autofillSessionId, @NonNull AutofillId focusedId) { + synchronized (mLock) { + final ContentCaptureSession session = getSession(activityToken); + if (session != null) { + // TODO(b/111330312): log metrics + if (mMaster.verbose) Slog.v(TAG, "requestAugmentedAutofill()"); + return session.requestAutofillLocked(client, autofillSessionId, focusedId); + } + if (mMaster.debug) { + Slog.d(TAG, "requestAutofill(): no session for " + activityToken); + } + return null; + } + } + @Override protected void dumpLocked(String prefix, PrintWriter pw) { super.dumpLocked(prefix, pw); diff --git a/services/intelligence/java/com/android/server/intelligence/RemoteIntelligenceService.java b/services/intelligence/java/com/android/server/intelligence/RemoteIntelligenceService.java index a27c1cf98a9a..00c5b6a1d67b 100644 --- a/services/intelligence/java/com/android/server/intelligence/RemoteIntelligenceService.java +++ b/services/intelligence/java/com/android/server/intelligence/RemoteIntelligenceService.java @@ -19,6 +19,7 @@ import android.annotation.NonNull; import android.annotation.Nullable; import android.content.ComponentName; import android.content.Context; +import android.os.Bundle; import android.os.IBinder; import android.os.IInterface; import android.os.RemoteException; @@ -28,8 +29,12 @@ import android.service.intelligence.InteractionSessionId; import android.service.intelligence.SnapshotData; import android.text.format.DateUtils; import android.util.Slog; +import android.view.autofill.AutofillId; +import android.view.autofill.AutofillManager; +import android.view.autofill.IAutoFillManagerClient; import android.view.intelligence.ContentCaptureEvent; +import com.android.internal.os.IResultReceiver; import com.android.server.AbstractRemoteService; import java.util.List; @@ -39,7 +44,7 @@ final class RemoteIntelligenceService extends AbstractRemoteService { private static final String TAG = "RemoteIntelligenceService"; private static final long TIMEOUT_IDLE_BIND_MILLIS = 2 * DateUtils.MINUTE_IN_MILLIS; - private static final long TIMEOUT_REMOTE_REQUEST_MILLIS = 2 * DateUtils.MINUTE_IN_MILLIS; + private static final long TIMEOUT_REMOTE_REQUEST_MILLIS = 2 * DateUtils.SECOND_IN_MILLIS; private final RemoteIntelligenceServiceCallbacks mCallbacks; private IIntelligenceService mService; @@ -101,6 +106,25 @@ final class RemoteIntelligenceService extends AbstractRemoteService { scheduleRequest(new PendingOnActivitySnapshotRequest(this, sessionId, snapshotData)); } + /** + * Called by {@link ContentCaptureSession} to request augmented autofill. + */ + public void onRequestAutofillLocked(@NonNull InteractionSessionId sessionId, + @NonNull IAutoFillManagerClient client, int autofillSessionId, + @NonNull AutofillId focusedId) { + cancelScheduledUnbind(); + scheduleRequest(new PendingAutofillRequest(this, sessionId, client, autofillSessionId, + focusedId)); + } + + /** + * Called by {@link ContentCaptureSession} when it's time to destroy all augmented autofill + * requests. + */ + public void onDestroyAutofillWindowsRequest(@NonNull InteractionSessionId sessionId) { + cancelScheduledUnbind(); + scheduleRequest(new PendingDestroyAutofillWindowsRequest(this, sessionId)); + } private abstract static class MyPendingRequest extends PendingRequest<RemoteIntelligenceService> { @@ -124,8 +148,9 @@ final class RemoteIntelligenceService extends AbstractRemoteService { final RemoteIntelligenceService remoteService = getService(); if (remoteService != null) { try { - myRun(remoteService); // We don't expect the service to call us back, so we finish right away. + myRun(remoteService); + // TODO(b/111330312): not true anymore!! finish(); } catch (RemoteException e) { Slog.w(TAG, "exception handling " + getClass().getSimpleName() + " for " @@ -191,6 +216,53 @@ final class RemoteIntelligenceService extends AbstractRemoteService { } } + private static final class PendingAutofillRequest extends MyPendingRequest { + private final @NonNull AutofillId mFocusedId; + private final @NonNull IAutoFillManagerClient mClient; + private final int mAutofillSessionId; + + protected PendingAutofillRequest(@NonNull RemoteIntelligenceService service, + @NonNull InteractionSessionId sessionId, @NonNull IAutoFillManagerClient client, + int autofillSessionId, @NonNull AutofillId focusedId) { + super(service, sessionId); + mClient = client; + mAutofillSessionId = autofillSessionId; + mFocusedId = focusedId; + } + + @Override // from MyPendingRequest + public void myRun(@NonNull RemoteIntelligenceService remoteService) throws RemoteException { + final IResultReceiver receiver = new IResultReceiver.Stub() { + + @Override + public void send(int resultCode, Bundle resultData) throws RemoteException { + final IBinder realClient = resultData + .getBinder(AutofillManager.EXTRA_AUGMENTED_AUTOFILL_CLIENT); + remoteService.mService.onAutofillRequest(mSessionId, realClient, + mAutofillSessionId, mFocusedId); + } + }; + + // TODO(b/111330312): set cancellation signal, timeout (from both mClient and service), + // cache IAugmentedAutofillManagerClient reference, etc... + mClient.getAugmentedAutofillClient(receiver); + } + } + + private static final class PendingDestroyAutofillWindowsRequest extends MyPendingRequest { + + protected PendingDestroyAutofillWindowsRequest(@NonNull RemoteIntelligenceService service, + @NonNull InteractionSessionId sessionId) { + super(service, sessionId); + } + + @Override + protected void myRun(@NonNull RemoteIntelligenceService service) throws RemoteException { + service.mService.onDestroyAutofillWindowsRequest(mSessionId); + // TODO(b/111330312): implement timeout + } + } + public interface RemoteIntelligenceServiceCallbacks extends VultureCallback { // To keep it simple, we use the same callback for all failures / timeouts. void onFailureOrTimeout(boolean timedOut); |