diff options
12 files changed, 404 insertions, 22 deletions
diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 2593ee2e2288..789b7ed24d31 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -13450,6 +13450,7 @@ package android.service.voice { @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public final class VisualQueryDetectedResult implements android.os.Parcelable { method public int describeContents(); + method @Nullable public byte[] getAccessibilityDetectionData(); method public static int getMaxSpeakerId(); method @NonNull public String getPartialQuery(); method public int getSpeakerId(); @@ -13460,6 +13461,7 @@ package android.service.voice { public static final class VisualQueryDetectedResult.Builder { ctor public VisualQueryDetectedResult.Builder(); method @NonNull public android.service.voice.VisualQueryDetectedResult build(); + method @NonNull public android.service.voice.VisualQueryDetectedResult.Builder setAccessibilityDetectionData(@NonNull byte...); method @NonNull public android.service.voice.VisualQueryDetectedResult.Builder setPartialQuery(@NonNull String); method @NonNull public android.service.voice.VisualQueryDetectedResult.Builder setSpeakerId(int); } @@ -13497,7 +13499,10 @@ package android.service.voice { } public class VisualQueryDetector { + method @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public void clearAccessibilityDetectionEnabledListener(); method public void destroy(); + method @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public boolean isAccessibilityDetectionEnabled(); + method @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public void setAccessibilityDetectionEnabledListener(@NonNull java.util.function.Consumer<java.lang.Boolean>); method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean startRecognition(); method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean stopRecognition(); method public void updateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory); diff --git a/core/java/android/service/voice/VisualQueryDetectedResult.java b/core/java/android/service/voice/VisualQueryDetectedResult.java index 322148acdca5..3b617948b41e 100644 --- a/core/java/android/service/voice/VisualQueryDetectedResult.java +++ b/core/java/android/service/voice/VisualQueryDetectedResult.java @@ -68,6 +68,22 @@ public final class VisualQueryDetectedResult implements Parcelable { return 15; } + /** + * Detected signal representing the arbitrary data that will make accessibility detections work. + * + * This field should only be set if the device setting for allowing accessibility data is on + * based on the result of {@link VisualQueryDetector#isAccessibilityDetectionEnabled()}. If the + * enable bit return by the method is {@code false}, it would suggest a failure to egress the + * {@link VisualQueryDetectedResult} object with this field set. The system server will prevent + * egress and invoke + * {@link VisualQueryDetector.Callback#onFailure(VisualQueryDetectionServiceFailure)}. + */ + @Nullable + private final byte[] mAccessibilityDetectionData; + private static byte[] defaultAccessibilityDetectionData() { + return null; + } + private void onConstructed() { Preconditions.checkArgumentInRange(mSpeakerId, 0, getMaxSpeakerId(), "speakerId"); } @@ -78,7 +94,10 @@ public final class VisualQueryDetectedResult implements Parcelable { * @hide */ public Builder buildUpon() { - return new Builder().setPartialQuery(mPartialQuery).setSpeakerId(mSpeakerId); + return new Builder() + .setPartialQuery(mPartialQuery) + .setSpeakerId(mSpeakerId) + .setAccessibilityDetectionData(mAccessibilityDetectionData); } @@ -98,11 +117,13 @@ public final class VisualQueryDetectedResult implements Parcelable { @DataClass.Generated.Member /* package-private */ VisualQueryDetectedResult( @NonNull String partialQuery, - int speakerId) { + int speakerId, + @Nullable byte[] accessibilityDetectionData) { this.mPartialQuery = partialQuery; com.android.internal.util.AnnotationValidations.validate( NonNull.class, null, mPartialQuery); this.mSpeakerId = speakerId; + this.mAccessibilityDetectionData = accessibilityDetectionData; onConstructed(); } @@ -125,6 +146,16 @@ public final class VisualQueryDetectedResult implements Parcelable { return mSpeakerId; } + /** + * Detected signal representing the data for allowing accessibility feature. This field can + * only be set when the secure device settings is set to true by either settings page UI or + * {@link VisualQueryDetector@setAccessibilityDetectionEnabled(boolean)} + */ + @DataClass.Generated.Member + public @Nullable byte[] getAccessibilityDetectionData() { + return mAccessibilityDetectionData; + } + @Override @DataClass.Generated.Member public String toString() { @@ -133,7 +164,8 @@ public final class VisualQueryDetectedResult implements Parcelable { return "VisualQueryDetectedResult { " + "partialQuery = " + mPartialQuery + ", " + - "speakerId = " + mSpeakerId + + "speakerId = " + mSpeakerId + ", " + + "accessibilityDetectionData = " + java.util.Arrays.toString(mAccessibilityDetectionData) + " }"; } @@ -151,7 +183,8 @@ public final class VisualQueryDetectedResult implements Parcelable { //noinspection PointlessBooleanExpression return true && Objects.equals(mPartialQuery, that.mPartialQuery) - && mSpeakerId == that.mSpeakerId; + && mSpeakerId == that.mSpeakerId + && java.util.Arrays.equals(mAccessibilityDetectionData, that.mAccessibilityDetectionData); } @Override @@ -163,6 +196,7 @@ public final class VisualQueryDetectedResult implements Parcelable { int _hash = 1; _hash = 31 * _hash + Objects.hashCode(mPartialQuery); _hash = 31 * _hash + mSpeakerId; + _hash = 31 * _hash + java.util.Arrays.hashCode(mAccessibilityDetectionData); return _hash; } @@ -174,6 +208,7 @@ public final class VisualQueryDetectedResult implements Parcelable { dest.writeString(mPartialQuery); dest.writeInt(mSpeakerId); + dest.writeByteArray(mAccessibilityDetectionData); } @Override @@ -189,11 +224,13 @@ public final class VisualQueryDetectedResult implements Parcelable { String partialQuery = in.readString(); int speakerId = in.readInt(); + byte[] accessibilityDetectionData = in.createByteArray(); this.mPartialQuery = partialQuery; com.android.internal.util.AnnotationValidations.validate( NonNull.class, null, mPartialQuery); this.mSpeakerId = speakerId; + this.mAccessibilityDetectionData = accessibilityDetectionData; onConstructed(); } @@ -221,6 +258,7 @@ public final class VisualQueryDetectedResult implements Parcelable { private @NonNull String mPartialQuery; private int mSpeakerId; + private @Nullable byte[] mAccessibilityDetectionData; private long mBuilderFieldsSet = 0L; @@ -251,10 +289,23 @@ public final class VisualQueryDetectedResult implements Parcelable { return this; } + /** + * Detected signal representing the data for allowing accessibility feature. This field can + * only be set when the secure device settings is set to true by either settings page UI or + * {@link VisualQueryDetector@setAccessibilityDetectionEnabled(boolean)} + */ + @DataClass.Generated.Member + public @NonNull Builder setAccessibilityDetectionData(@NonNull byte... value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x4; + mAccessibilityDetectionData = value; + return this; + } + /** Builds the instance. This builder should not be touched after calling this! */ public @NonNull VisualQueryDetectedResult build() { checkNotUsed(); - mBuilderFieldsSet |= 0x4; // Mark builder used + mBuilderFieldsSet |= 0x8; // Mark builder used if ((mBuilderFieldsSet & 0x1) == 0) { mPartialQuery = defaultPartialQuery(); @@ -262,14 +313,18 @@ public final class VisualQueryDetectedResult implements Parcelable { if ((mBuilderFieldsSet & 0x2) == 0) { mSpeakerId = defaultSpeakerId(); } + if ((mBuilderFieldsSet & 0x4) == 0) { + mAccessibilityDetectionData = defaultAccessibilityDetectionData(); + } VisualQueryDetectedResult o = new VisualQueryDetectedResult( mPartialQuery, - mSpeakerId); + mSpeakerId, + mAccessibilityDetectionData); return o; } private void checkNotUsed() { - if ((mBuilderFieldsSet & 0x4) != 0) { + if ((mBuilderFieldsSet & 0x8) != 0) { throw new IllegalStateException( "This Builder should not be reused. Use a new Builder instance instead"); } @@ -277,10 +332,10 @@ public final class VisualQueryDetectedResult implements Parcelable { } @DataClass.Generated( - time = 1704949386772L, + time = 1707429290528L, codegenVersion = "1.0.23", sourceFile = "frameworks/base/core/java/android/service/voice/VisualQueryDetectedResult.java", - inputSignatures = "private final @android.annotation.NonNull java.lang.String mPartialQuery\nprivate final int mSpeakerId\nprivate static java.lang.String defaultPartialQuery()\nprivate static int defaultSpeakerId()\npublic static int getMaxSpeakerId()\nprivate void onConstructed()\npublic android.service.voice.VisualQueryDetectedResult.Builder buildUpon()\nclass VisualQueryDetectedResult extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)") + inputSignatures = "private final @android.annotation.NonNull java.lang.String mPartialQuery\nprivate final int mSpeakerId\nprivate final @android.annotation.Nullable byte[] mAccessibilityDetectionData\nprivate static java.lang.String defaultPartialQuery()\nprivate static int defaultSpeakerId()\npublic static int getMaxSpeakerId()\nprivate static byte[] defaultAccessibilityDetectionData()\nprivate void onConstructed()\npublic android.service.voice.VisualQueryDetectedResult.Builder buildUpon()\nclass VisualQueryDetectedResult extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)") @Deprecated private void __metadata() {} diff --git a/core/java/android/service/voice/VisualQueryDetector.java b/core/java/android/service/voice/VisualQueryDetector.java index 1eb4d6730e1e..bf8de06fd244 100644 --- a/core/java/android/service/voice/VisualQueryDetector.java +++ b/core/java/android/service/voice/VisualQueryDetector.java @@ -39,6 +39,7 @@ import android.text.TextUtils; import android.util.Slog; import com.android.internal.app.IHotwordRecognitionStatusCallback; +import com.android.internal.app.IVoiceInteractionAccessibilitySettingsListener; import com.android.internal.app.IVoiceInteractionManagerService; import com.android.internal.infra.AndroidFuture; @@ -61,6 +62,8 @@ import java.util.function.Consumer; public class VisualQueryDetector { private static final String TAG = VisualQueryDetector.class.getSimpleName(); private static final boolean DEBUG = false; + private static final int SETTINGS_DISABLE_BIT = 0; + private static final int SETTINGS_ENABLE_BIT = 1; private final Callback mCallback; private final Executor mExecutor; @@ -68,6 +71,8 @@ public class VisualQueryDetector { private final IVoiceInteractionManagerService mManagerService; private final VisualQueryDetectorInitializationDelegate mInitializationDelegate; private final String mAttributionTag; + // Used to manage the internal mapping of exposed listener API and internal aidl impl + private AccessibilityDetectionEnabledListenerWrapper mActiveAccessibilityListenerWrapper = null; VisualQueryDetector( IVoiceInteractionManagerService managerService, @@ -174,6 +179,108 @@ public class VisualQueryDetector { } } + /** + * Gets the binary value that controls the egress of accessibility data from + * {@link VisualQueryDetectedResult#setAccessibilityDetectionData(byte[])} is enabled. + * + * @return boolean value denoting if the setting is on. Default is {@code false}. + * @hide + */ + @SystemApi + @FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) + public boolean isAccessibilityDetectionEnabled() { + Slog.d(TAG, "Fetching accessibility setting"); + synchronized (mInitializationDelegate.getLock()) { + try { + return mManagerService.getAccessibilityDetectionEnabled(); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + } + return false; + } + } + + /** + * Sets a listener subscribing to the value of the system setting that controls the egress of + * accessibility data from + * {@link VisualQueryDetectedResult#setAccessibilityDetectionData(byte[])} is enabled. + * + * Only one listener can be set at a time. The listener set must be unset with + * {@link clearAccessibilityDetectionEnabledListener(Consumer<Boolean>)} + * in order to set a new listener. Otherwise, this method will throw a + * {@link IllegalStateException}. + * + * @param listener Listener of type {@code Consumer<Boolean>} to subscribe to the value update. + * @hide + */ + @SystemApi + @FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) + public void setAccessibilityDetectionEnabledListener(@NonNull Consumer<Boolean> listener) { + Slog.d(TAG, "Registering Accessibility settings listener."); + synchronized (mInitializationDelegate.getLock()) { + try { + if (mActiveAccessibilityListenerWrapper != null) { + Slog.e(TAG, "Fail to register accessibility setting listener: " + + "already registered and not unregistered."); + throw new IllegalStateException( + "Cannot register listener with listeners already set."); + } + mActiveAccessibilityListenerWrapper = + new AccessibilityDetectionEnabledListenerWrapper(listener); + mManagerService.registerAccessibilityDetectionSettingsListener( + mActiveAccessibilityListenerWrapper); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + } + } + } + + /** + * Clear the listener that has been set with + * {@link setAccessibilityDetectionEnabledListener(Consumer<Boolean>)} such that when the value + * of the setting that controls the egress of accessibility data is changed the listener gets + * notified. + * + * If there is not listener that has been registered, the call to this method will lead to a + * {@link IllegalStateException}. + * + * @hide + */ + @SystemApi + @FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) + public void clearAccessibilityDetectionEnabledListener() { + Slog.d(TAG, "Unregistering Accessibility settings listener."); + synchronized (mInitializationDelegate.getLock()) { + try { + if (mActiveAccessibilityListenerWrapper == null) { + Slog.e(TAG, "Not able to remove the listener: listener does not exist."); + throw new IllegalStateException("Cannot clear listener since it is not set."); + } + mManagerService.unregisterAccessibilityDetectionSettingsListener( + mActiveAccessibilityListenerWrapper); + mActiveAccessibilityListenerWrapper = null; + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + } + } + } + + + private final class AccessibilityDetectionEnabledListenerWrapper + extends IVoiceInteractionAccessibilitySettingsListener.Stub { + + private Consumer<Boolean> mListener; + + AccessibilityDetectionEnabledListenerWrapper(Consumer<Boolean> listener) { + mListener = listener; + } + + @Override + public void onAccessibilityDetectionChanged(boolean enabled) { + mListener.accept(enabled); + } + } + /** @hide */ public void dump(String prefix, PrintWriter pw) { synchronized (mInitializationDelegate.getLock()) { diff --git a/core/java/com/android/internal/app/IVoiceInteractionAccessibilitySettingsListener.aidl b/core/java/com/android/internal/app/IVoiceInteractionAccessibilitySettingsListener.aidl new file mode 100644 index 000000000000..a9190353dc66 --- /dev/null +++ b/core/java/com/android/internal/app/IVoiceInteractionAccessibilitySettingsListener.aidl @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.app; + +oneway interface IVoiceInteractionAccessibilitySettingsListener { + /** + * Called when the value of secure setting has changed. + */ + void onAccessibilityDetectionChanged(boolean enable); +} diff --git a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl index 314ed69cb885..98d393977958 100644 --- a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl +++ b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl @@ -35,6 +35,7 @@ import android.service.voice.VisibleActivityInfo; import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVoiceActionCheckCallback; +import com.android.internal.app.IVoiceInteractionAccessibilitySettingsListener; import com.android.internal.app.IVoiceInteractionSessionListener; import com.android.internal.app.IVoiceInteractionSessionShowCallback; import com.android.internal.app.IVoiceInteractionSoundTriggerSession; @@ -382,4 +383,21 @@ interface IVoiceInteractionManagerService { oneway void notifyActivityEventChanged( in IBinder activityToken, int type); + + /** + * rely on the system server to get the secure settings + */ + boolean getAccessibilityDetectionEnabled(); + + /** + * register the listener + */ + oneway void registerAccessibilityDetectionSettingsListener( + in IVoiceInteractionAccessibilitySettingsListener listener); + + /** + * unregister the listener + */ + oneway void unregisterAccessibilityDetectionSettingsListener( + in IVoiceInteractionAccessibilitySettingsListener listener); } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java index 368a96b372fe..0a1f3c78e0e6 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java @@ -192,6 +192,7 @@ abstract class DetectorSession { final Object mLock; final int mVoiceInteractionServiceUid; final Context mContext; + final int mUserId; @Nullable AttentionManagerInternal mAttentionManagerInternal = null; @@ -224,12 +225,13 @@ abstract class DetectorSession { @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, - @NonNull DetectorRemoteExceptionListener listener) { + @NonNull DetectorRemoteExceptionListener listener, int userId) { mRemoteExceptionListener = listener; mRemoteDetectionService = remoteDetectionService; mLock = lock; mContext = context; mToken = token; + mUserId = userId; mCallback = callback; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java index 9a4fbdc4516a..8d08c6bb5e4b 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java @@ -87,10 +87,10 @@ final class DspTrustedHotwordDetectorSession extends DetectorSession { @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, - @NonNull DetectorRemoteExceptionListener listener) { + @NonNull DetectorRemoteExceptionListener listener, int userId) { super(remoteHotwordDetectionService, lock, context, token, callback, voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, - logging, listener); + logging, listener, userId); } @SuppressWarnings("GuardedBy") diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index f1f5458f161c..cfcc04b10107 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -72,6 +72,7 @@ import android.view.contentcapture.IContentCaptureManager; import com.android.internal.annotations.GuardedBy; import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVisualQueryDetectionAttentionListener; +import com.android.internal.app.IVoiceInteractionAccessibilitySettingsListener; import com.android.internal.infra.AndroidFuture; import com.android.internal.infra.ServiceConnector; import com.android.server.LocalServices; @@ -147,8 +148,9 @@ final class HotwordDetectionConnection { final int mVoiceInteractionServiceUid; final ComponentName mHotwordDetectionComponentName; final ComponentName mVisualQueryDetectionComponentName; - final int mUser; + final int mUserId; final Context mContext; + final AccessibilitySettingsListener mAccessibilitySettingsListener; volatile HotwordDetectionServiceIdentity mIdentity; //TODO: Consider rename this to SandboxedDetectionIdentity private Instant mLastRestartInstant; @@ -204,6 +206,27 @@ final class HotwordDetectionConnection { } }; + /** Listen to changes of {@link Settings.Secure.VISUAL_QUERY_ACCESSIBILITY_DETECTION_ENABLED}. + * + * This is registered to the {@link VoiceInteractionManagerServiceImpl} where all settings + * listeners are centralized and notified. + */ + private final class AccessibilitySettingsListener extends + IVoiceInteractionAccessibilitySettingsListener.Stub { + @Override + public void onAccessibilityDetectionChanged(boolean enable) { + synchronized (mLock) { + if (DEBUG) { + Slog.d(TAG, "Update settings change: " + enable); + } + VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked(); + if (session != null) { + session.updateAccessibilityEgressStateLocked(enable); + } + } + } + } + HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, @@ -216,11 +239,12 @@ final class HotwordDetectionConnection { mVoiceInteractorIdentity = voiceInteractorIdentity; mHotwordDetectionComponentName = hotwordDetectionServiceName; mVisualQueryDetectionComponentName = visualQueryDetectionServiceName; - mUser = userId; + mUserId = userId; mDetectorType = detectorType; mRemoteExceptionListener = listener; mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION, KEY_RESTART_PERIOD_IN_SECONDS, 0); + mAccessibilitySettingsListener = new AccessibilitySettingsListener(); final Intent hotwordDetectionServiceIntent = new Intent(HotwordDetectionService.SERVICE_INTERFACE); @@ -792,7 +816,7 @@ final class HotwordDetectionConnection { ServiceConnection createLocked() { ServiceConnection connection = - new ServiceConnection(mContext, mIntent, mBindingFlags, mUser, + new ServiceConnection(mContext, mIntent, mBindingFlags, mUserId, ISandboxedDetectionService.Stub::asInterface, mRestartCount % MAX_ISOLATED_PROCESS_NUMBER, mDetectionServiceType); connection.connect(); @@ -998,7 +1022,7 @@ final class HotwordDetectionConnection { session = new DspTrustedHotwordDetectorSession(mRemoteHotwordDetectionService, mLock, mContext, token, callback, mVoiceInteractionServiceUid, mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging, - mRemoteExceptionListener); + mRemoteExceptionListener, mUserId); } else if (detectorType == HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { if (mRemoteVisualQueryDetectionService == null) { mRemoteVisualQueryDetectionService = @@ -1007,7 +1031,8 @@ final class HotwordDetectionConnection { session = new VisualQueryDetectorSession( mRemoteVisualQueryDetectionService, mLock, mContext, token, callback, mVoiceInteractionServiceUid, mVoiceInteractorIdentity, - mScheduledExecutorService, mDebugHotwordLogging, mRemoteExceptionListener); + mScheduledExecutorService, mDebugHotwordLogging, mRemoteExceptionListener, + mUserId); } else { if (mRemoteHotwordDetectionService == null) { mRemoteHotwordDetectionService = @@ -1016,7 +1041,8 @@ final class HotwordDetectionConnection { session = new SoftwareTrustedHotwordDetectorSession( mRemoteHotwordDetectionService, mLock, mContext, token, callback, mVoiceInteractionServiceUid, mVoiceInteractorIdentity, - mScheduledExecutorService, mDebugHotwordLogging, mRemoteExceptionListener); + mScheduledExecutorService, mDebugHotwordLogging, + mRemoteExceptionListener, mUserId); } mHotwordRecognitionCallback = callback; mDetectorSessions.put(detectorType, session); diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java index f06c99729a19..120c161c1903 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java @@ -74,10 +74,10 @@ final class SoftwareTrustedHotwordDetectorSession extends DetectorSession { @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, - @NonNull DetectorRemoteExceptionListener listener) { + @NonNull DetectorRemoteExceptionListener listener, int userId) { super(remoteHotwordDetectionService, lock, context, token, callback, voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, - logging, listener); + logging, listener, userId); } @SuppressWarnings("GuardedBy") diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java index e4ac993f2d50..aef8e6fabc9b 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java @@ -29,6 +29,7 @@ import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; import android.os.RemoteException; import android.os.SharedMemory; +import android.provider.Settings; import android.service.voice.IDetectorSessionVisualQueryDetectionCallback; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.ISandboxedDetectionService; @@ -60,6 +61,7 @@ final class VisualQueryDetectorSession extends DetectorSession { private IVisualQueryDetectionAttentionListener mAttentionListener; private boolean mEgressingData; private boolean mQueryStreaming; + private boolean mEnableAccessibilityDataEgress; //TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc. VisualQueryDetectorSession( @@ -68,13 +70,17 @@ final class VisualQueryDetectorSession extends DetectorSession { @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, - @NonNull DetectorRemoteExceptionListener listener) { + @NonNull DetectorRemoteExceptionListener listener, int userId) { super(remoteService, lock, context, token, callback, voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, - logging, listener); + logging, listener, userId); mEgressingData = false; mQueryStreaming = false; mAttentionListener = null; + mEnableAccessibilityDataEgress = Settings.Secure.getIntForUser( + mContext.getContentResolver(), + Settings.Secure.VISUAL_QUERY_ACCESSIBILITY_DETECTION_ENABLED, 0, + mUserId) == 1; // TODO: handle notify RemoteException to client } @@ -186,6 +192,16 @@ final class VisualQueryDetectorSession extends DetectorSession { "Cannot stream results without attention signals.")); return; } + if (!checkDetectedResultDataLocked(partialResult)) { + Slog.v(TAG, "Accessibility data can be egressed only when the " + + "isAccessibilityDetectionEnabled() is true."); + callback.onVisualQueryDetectionServiceFailure( + new VisualQueryDetectionServiceFailure( + ERROR_CODE_ILLEGAL_STREAMING_STATE, + "Cannot stream accessibility data without " + + "enabling the setting.")); + return; + } mQueryStreaming = true; callback.onResultDetected(partialResult); Slog.i(TAG, "Egressed from visual query detection process."); @@ -227,6 +243,12 @@ final class VisualQueryDetectorSession extends DetectorSession { mQueryStreaming = false; } } + + @SuppressWarnings("GuardedBy") + private boolean checkDetectedResultDataLocked(VisualQueryDetectedResult result) { + return result.getAccessibilityDetectionData() == null + || mEnableAccessibilityDataEgress; + } }; return mRemoteDetectionService.run( service -> service.detectWithVisualSignals(internalCallback)); @@ -251,6 +273,12 @@ final class VisualQueryDetectorSession extends DetectorSession { + " should not be called from VisualQueryDetectorSession."); } + void updateAccessibilityEgressStateLocked(boolean enable) { + if (DEBUG) { + Slog.d(TAG, "updateAccessibilityEgressStateLocked"); + } + mEnableAccessibilityDataEgress = enable; + } @SuppressWarnings("GuardedBy") public void dumpLocked(String prefix, PrintWriter pw) { diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java index ecb0f9689ae7..889f8429077c 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java @@ -91,6 +91,7 @@ import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVisualQueryDetectionAttentionListener; import com.android.internal.app.IVisualQueryRecognitionStatusListener; import com.android.internal.app.IVoiceActionCheckCallback; +import com.android.internal.app.IVoiceInteractionAccessibilitySettingsListener; import com.android.internal.app.IVoiceInteractionManagerService; import com.android.internal.app.IVoiceInteractionSessionListener; import com.android.internal.app.IVoiceInteractionSessionShowCallback; @@ -2179,6 +2180,44 @@ public class VoiceInteractionManagerService extends SystemService { } } + public boolean getAccessibilityDetectionEnabled() { + synchronized (this) { + if (mImpl == null) { + Slog.w(TAG, "registerAccessibilityDetectionSettingsListener called without" + + " running voice interaction service"); + return false; + } + return mImpl.getAccessibilityDetectionEnabled(); + } + } + + @Override + public void registerAccessibilityDetectionSettingsListener( + IVoiceInteractionAccessibilitySettingsListener listener) { + synchronized (this) { + if (mImpl == null) { + Slog.w(TAG, "registerAccessibilityDetectionSettingsListener called without" + + " running voice interaction service"); + return; + } + mImpl.registerAccessibilityDetectionSettingsListenerLocked(listener); + } + } + + @Override + public void unregisterAccessibilityDetectionSettingsListener( + IVoiceInteractionAccessibilitySettingsListener listener) { + synchronized (this) { + if (mImpl == null) { + Slog.w(TAG, "unregisterAccessibilityDetectionSettingsListener called " + + "without running voice interaction service"); + return; + } + mImpl.unregisterAccessibilityDetectionSettingsListenerLocked(listener); + } + } + + @Override public void dump(FileDescriptor fd, PrintWriter pw, String[] args) { if (!DumpUtils.checkDumpPermission(mContext, TAG, pw)) return; diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java index 84b36d5948eb..e34e81908632 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java @@ -36,6 +36,7 @@ import android.app.IActivityManager; import android.app.IActivityTaskManager; import android.content.BroadcastReceiver; import android.content.ComponentName; +import android.content.ContentResolver; import android.content.Context; import android.content.Intent; import android.content.IntentFilter; @@ -45,10 +46,12 @@ import android.content.pm.PackageManager; import android.content.pm.PackageManagerInternal; import android.content.pm.ParceledListSlice; import android.content.pm.ServiceInfo; +import android.database.ContentObserver; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; import android.media.permission.Identity; +import android.net.Uri; import android.os.Bundle; import android.os.Handler; import android.os.IBinder; @@ -60,6 +63,7 @@ import android.os.ServiceManager; import android.os.SharedMemory; import android.os.SystemProperties; import android.os.UserHandle; +import android.provider.Settings; import android.service.voice.HotwordDetector; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback; @@ -77,6 +81,7 @@ import android.view.IWindowManager; import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVisualQueryDetectionAttentionListener; import com.android.internal.app.IVoiceActionCheckCallback; +import com.android.internal.app.IVoiceInteractionAccessibilitySettingsListener; import com.android.internal.app.IVoiceInteractionSessionShowCallback; import com.android.internal.app.IVoiceInteractor; import com.android.internal.util.function.pooled.PooledLambda; @@ -199,6 +204,10 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne } }; + final ArrayList< + IVoiceInteractionAccessibilitySettingsListener> mAccessibilitySettingsListeners = + new ArrayList<IVoiceInteractionAccessibilitySettingsListener>(); + VoiceInteractionManagerServiceImpl(Context context, Handler handler, VoiceInteractionManagerService.VoiceInteractionManagerServiceStub stub, int userHandle, ComponentName service) { @@ -250,6 +259,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne filter.addAction(Intent.ACTION_CLOSE_SYSTEM_DIALOGS); mContext.registerReceiver(mBroadcastReceiver, filter, null, handler, Context.RECEIVER_EXPORTED); + new AccessibilitySettingsContentObserver().register(mContext.getContentResolver()); } public void grantImplicitAccessLocked(int grantRecipientUid, @Nullable Intent intent) { @@ -745,6 +755,8 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne + "exception occurred."); } }); + registerAccessibilityDetectionSettingsListenerLocked( + mHotwordDetectionConnection.mAccessibilitySettingsListener); } else if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { // TODO: Logger events should be handled in session instead. Temporary adding the // checking to prevent confusion so VisualQueryDetection events won't be logged if the @@ -782,6 +794,8 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne return; } mHotwordDetectionConnection.cancelLocked(); + unregisterAccessibilityDetectionSettingsListenerLocked( + mHotwordDetectionConnection.mAccessibilitySettingsListener); mHotwordDetectionConnection = null; } @@ -974,6 +988,8 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne return; } mHotwordDetectionConnection.cancelLocked(); + unregisterAccessibilityDetectionSettingsListenerLocked( + mHotwordDetectionConnection.mAccessibilitySettingsListener); mHotwordDetectionConnection = null; } @@ -1015,6 +1031,29 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne } } + boolean getAccessibilityDetectionEnabled() { + return Settings.Secure.getIntForUser( + mContext.getContentResolver(), + Settings.Secure.VISUAL_QUERY_ACCESSIBILITY_DETECTION_ENABLED, 0, + mUser) == 1; + } + + void registerAccessibilityDetectionSettingsListenerLocked( + IVoiceInteractionAccessibilitySettingsListener listener) { + if (DEBUG) { + Slog.d(TAG, "registerAccessibilityDetectionSettingsListener"); + } + mAccessibilitySettingsListeners.add(listener); + } + + void unregisterAccessibilityDetectionSettingsListenerLocked( + IVoiceInteractionAccessibilitySettingsListener listener) { + if (DEBUG) { + Slog.d(TAG, "unregisterAccessibilityDetectionSettingsListener"); + } + mAccessibilitySettingsListeners.remove(listener); + } + void startLocked() { Intent intent = new Intent(VoiceInteractionService.SERVICE_INTERFACE); intent.setComponent(mComponent); @@ -1055,6 +1094,8 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne } if (mHotwordDetectionConnection != null) { mHotwordDetectionConnection.cancelLocked(); + unregisterAccessibilityDetectionSettingsListenerLocked( + mHotwordDetectionConnection.mAccessibilitySettingsListener); mHotwordDetectionConnection = null; } if (mBound) { @@ -1101,4 +1142,41 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne interface DetectorRemoteExceptionListener { void onDetectorRemoteException(@NonNull IBinder token, int detectorType); } + + private final class AccessibilitySettingsContentObserver extends ContentObserver { + private Uri mAccessibilitySettingsEnabledUri = Settings.Secure.getUriFor( + Settings.Secure.VISUAL_QUERY_ACCESSIBILITY_DETECTION_ENABLED); + + AccessibilitySettingsContentObserver() { + super(null); + } + + public void register(ContentResolver contentResolver) { + contentResolver.registerContentObserver( + mAccessibilitySettingsEnabledUri, false, this, UserHandle.USER_ALL); + } + + @Override + public void onChange(boolean selfChange, Uri uri) { + Slog.i(TAG, "OnChange called with uri:" + uri); + if (mAccessibilitySettingsEnabledUri.equals(uri)) { + boolean enable = Settings.Secure.getIntForUser( + mContext.getContentResolver(), + Settings.Secure.VISUAL_QUERY_ACCESSIBILITY_DETECTION_ENABLED, 0, + mUser) == 1; + Slog.i(TAG, "Notifying listeners with Accessibility setting set to " + + enable); + mAccessibilitySettingsListeners.forEach( + listener -> { + try { + listener.onAccessibilityDetectionChanged(enable); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + } + } + ); + + } + } + } } |