diff options
5 files changed, 445 insertions, 86 deletions
diff --git a/core/java/android/service/voice/VisualQueryDetector.java b/core/java/android/service/voice/VisualQueryDetector.java index 241f5bac55ff..d24e69da9f4d 100644 --- a/core/java/android/service/voice/VisualQueryDetector.java +++ b/core/java/android/service/voice/VisualQueryDetector.java @@ -25,12 +25,16 @@ import android.annotation.Nullable; import android.annotation.RequiresPermission; import android.annotation.SuppressLint; import android.annotation.SystemApi; +import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; +import android.os.Binder; import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; +import android.os.RemoteException; import android.os.SharedMemory; import android.util.Slog; +import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVoiceInteractionManagerService; import java.io.PrintWriter; @@ -208,8 +212,9 @@ public class VisualQueryDetector { @Override void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) { - //TODO(261783492): call initAndVerify to create VisualQueryDetectionService - // from the system server. + initAndVerifyDetector(options, sharedMemory, + new InitializationStateListener(mExecutor, mCallback), + DETECTOR_TYPE_VISUAL_QUERY_DETECTOR); } @Override @@ -238,4 +243,76 @@ public class VisualQueryDetector { return true; } } + + private static class InitializationStateListener + extends IHotwordRecognitionStatusCallback.Stub { + private final Executor mExecutor; + private final Callback mCallback; + + InitializationStateListener(Executor executor, Callback callback) { + this.mExecutor = executor; + this.mCallback = callback; + } + + @Override + public void onKeyphraseDetected( + SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + HotwordDetectedResult result) { + if (DEBUG) { + Slog.i(TAG, "Ignored #onKeyphraseDetected event"); + } + } + + @Override + public void onGenericSoundTriggerDetected( + SoundTrigger.GenericRecognitionEvent recognitionEvent) throws RemoteException { + if (DEBUG) { + Slog.i(TAG, "Ignored #onGenericSoundTriggerDetected event"); + } + } + + @Override + public void onRejected(HotwordRejectedResult result) throws RemoteException { + if (DEBUG) { + Slog.i(TAG, "Ignored #onRejected event"); + } + } + + @Override + public void onRecognitionPaused() throws RemoteException { + if (DEBUG) { + Slog.i(TAG, "Ignored #onRecognitionPaused event"); + } + } + + @Override + public void onRecognitionResumed() throws RemoteException { + if (DEBUG) { + Slog.i(TAG, "Ignored #onRecognitionResumed event"); + } + } + + @Override + public void onStatusReported(int status) { + Slog.v(TAG, "onStatusReported" + (DEBUG ? "(" + status + ")" : "")); + //TODO: rename the target callback with a more general term + Binder.withCleanCallingIdentity(() -> mExecutor.execute( + () -> mCallback.onVisualQueryDetectionServiceInitialized(status))); + + } + + @Override + public void onProcessRestarted() throws RemoteException { + Slog.v(TAG, "onProcessRestarted()"); + //TODO: rename the target callback with a more general term + Binder.withCleanCallingIdentity(() -> mExecutor.execute( + () -> mCallback.onVisualQueryDetectionServiceRestarted())); + } + + @Override + public void onError(int status) throws RemoteException { + Slog.v(TAG, "Initialization Error: (" + status + ")"); + // Do nothing + } + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java index 4d6d3205db40..b8536f904569 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java @@ -205,9 +205,15 @@ abstract class DetectorSession { mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; mAppOpsManager = mContext.getSystemService(AppOpsManager.class); - mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, getDetectorType(), - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, + getDetectorType(), + mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, + mVoiceInteractorIdentity.attributionTag); + } else { + mHotwordAudioStreamCopier = null; + } + mScheduledExecutorService = scheduledExecutorService; mDebugHotwordLogging = logging; @@ -236,9 +242,12 @@ abstract class DetectorSession { future.complete(null); if (mUpdateStateAfterStartFinished.getAndSet(true)) { Slog.w(TAG, "call callback after timeout"); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + if (getDetectorType() + != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, mVoiceInteractionServiceUid); + } return; } Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle); @@ -246,27 +255,37 @@ abstract class DetectorSession { int initResultMetricsResult = statusResultPair.second; try { mCallback.onStatusReported(status); - HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), - initResultMetricsResult, mVoiceInteractionServiceUid); + if (getDetectorType() + != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), + initResultMetricsResult, mVoiceInteractionServiceUid); + } } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status: " + e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() + != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, + mVoiceInteractionServiceUid); + } } } }; try { service.updateState(options, sharedMemory, statusCallback); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, + mVoiceInteractionServiceUid); + } } catch (RemoteException e) { // TODO: (b/181842909) Report an error to voice interactor Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, + mVoiceInteractionServiceUid); + } } return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); }).whenComplete((res, err) -> { @@ -277,13 +296,17 @@ abstract class DetectorSession { } try { mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); - HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), - METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), + METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid); + } } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, + mVoiceInteractionServiceUid); + } } } else if (err != null) { Slog.w(TAG, "Failed to update state: " + err); @@ -315,9 +338,11 @@ abstract class DetectorSession { @SuppressWarnings("GuardedBy") void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, Instant lastRestartInstant) { - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, + mVoiceInteractionServiceUid); + } // Prevent doing the init late, so restart is handled equally to a clean process start. // TODO(b/191742511): this logic needs a test if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus( @@ -399,9 +424,11 @@ abstract class DetectorSession { callback.onError(); } catch (RemoteException ex) { Slog.w(TAG, "Failed to report onError status: " + ex); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } } } finally { synchronized (mLock) { @@ -427,7 +454,8 @@ abstract class DetectorSession { throws RemoteException { synchronized (mLock) { mPerformingExternalSourceHotwordDetection = false; - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HotwordMetricsLogger.writeDetectorEvent( + getDetectorType(), METRICS_EXTERNAL_SOURCE_REJECTED, mVoiceInteractionServiceUid); mScheduledExecutorService.schedule( @@ -454,7 +482,8 @@ abstract class DetectorSession { throws RemoteException { synchronized (mLock) { mPerformingExternalSourceHotwordDetection = false; - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HotwordMetricsLogger.writeDetectorEvent( + getDetectorType(), METRICS_EXTERNAL_SOURCE_DETECTED, mVoiceInteractionServiceUid); mScheduledExecutorService.schedule( @@ -540,9 +569,11 @@ abstract class DetectorSession { mCallback.onError(status); } catch (RemoteException e) { Slog.w(TAG, "Failed to report onError status: " + e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } } } @@ -679,6 +710,8 @@ abstract class DetectorSession { return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP; } else if (this instanceof SoftwareTrustedHotwordDetectorSession) { return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE; + } else if (this instanceof VisualQueryDetectorSession) { + return HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR; } Slog.v(TAG, "Unexpected detector type"); return -1; diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index d501af7d83be..f1dd9091d5de 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -52,6 +52,7 @@ import android.service.voice.HotwordDetectionService; import android.service.voice.HotwordDetector; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.ISandboxedDetectionService; +import android.service.voice.VisualQueryDetectionService; import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity; import android.speech.IRecognitionServiceManager; import android.util.Slog; @@ -74,7 +75,8 @@ import java.util.function.Consumer; import java.util.function.Function; /** - * A class that provides the communication with the HotwordDetectionService. + * A class that provides the communication with the {@link HotwordDetectionService} and + * {@link VisualQueryDetectionService}. */ final class HotwordDetectionConnection { private static final String TAG = "HotwordDetectionConnection"; @@ -90,7 +92,8 @@ final class HotwordDetectionConnection { @Nullable private final ScheduledFuture<?> mCancellationTaskFuture; private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied; @NonNull private final ServiceConnectionFactory mHotwordDetectionServiceConnectionFactory; - private final int mDetectorType; + @NonNull private final ServiceConnectionFactory mVisualQueryDetectionServiceConnectionFactory; + private int mDetectorType; /** * Time after which each HotwordDetectionService process is stopped and replaced by a new one. * 0 indicates no restarts. @@ -100,9 +103,11 @@ final class HotwordDetectionConnection { final Object mLock; final int mVoiceInteractionServiceUid; final ComponentName mHotwordDetectionComponentName; + final ComponentName mVisualQueryDetectionComponentName; final int mUser; final Context mContext; volatile HotwordDetectionServiceIdentity mIdentity; + //TODO: add similar identity for visual query service for the use of content capturing private Instant mLastRestartInstant; private ScheduledFuture<?> mDebugHotwordLoggingTimeoutFuture = null; @@ -112,6 +117,7 @@ final class HotwordDetectionConnection { @Nullable private final Identity mVoiceInteractorIdentity; @NonNull private ServiceConnection mRemoteHotwordDetectionService; + @NonNull private ServiceConnection mRemoteVisualQueryDetectionService; private IBinder mAudioFlinger; @GuardedBy("mLock") private boolean mDebugHotwordLogging = false; @@ -126,26 +132,39 @@ final class HotwordDetectionConnection { new SparseArray<>(); HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid, - Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, int userId, + Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, + ComponentName visualQueryDetectionServiceName, int userId, boolean bindInstantServiceAllowed, int detectorType) { mLock = lock; mContext = context; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; mHotwordDetectionComponentName = hotwordDetectionServiceName; + mVisualQueryDetectionComponentName = visualQueryDetectionServiceName; mUser = userId; mDetectorType = detectorType; mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION, KEY_RESTART_PERIOD_IN_SECONDS, 0); + final Intent hotwordDetectionServiceIntent = new Intent(HotwordDetectionService.SERVICE_INTERFACE); hotwordDetectionServiceIntent.setComponent(mHotwordDetectionComponentName); + + final Intent visualQueryDetectionServiceIntent = + new Intent(VisualQueryDetectionService.SERVICE_INTERFACE); + visualQueryDetectionServiceIntent.setComponent(mVisualQueryDetectionComponentName); + initAudioFlingerLocked(); mHotwordDetectionServiceConnectionFactory = new ServiceConnectionFactory(hotwordDetectionServiceIntent, bindInstantServiceAllowed); - mRemoteHotwordDetectionService = mHotwordDetectionServiceConnectionFactory.createLocked(); + + mVisualQueryDetectionServiceConnectionFactory = + new ServiceConnectionFactory(visualQueryDetectionServiceIntent, + bindInstantServiceAllowed); + + mLastRestartInstant = Instant.now(); if (mReStartPeriodSeconds <= 0) { @@ -157,9 +176,11 @@ final class HotwordDetectionConnection { Slog.v(TAG, "Time to restart the process, TTL has passed"); synchronized (mLock) { restartProcessLocked(); - HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, - HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, + HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE, + mVoiceInteractionServiceUid); + } } }, mReStartPeriodSeconds, mReStartPeriodSeconds, TimeUnit.SECONDS); } @@ -193,9 +214,11 @@ final class HotwordDetectionConnection { // We restart the process instead of simply sending over the new binder, to avoid race // conditions with audio reading in the service. restartProcessLocked(); - HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, - HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, + HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED, + mVoiceInteractionServiceUid); + } } } @@ -208,9 +231,8 @@ final class HotwordDetectionConnection { }); mDetectorSessions.clear(); mDebugHotwordLogging = false; - mRemoteHotwordDetectionService.unbind(); - LocalServices.getService(PermissionManagerServiceInternal.class) - .setHotwordDetectionServiceProvider(null); + unbindVisualQueryDetectionService(); + unbindHotwordDetectionService(); if (mIdentity != null) { removeServiceUidForAudioPolicy(mIdentity.getIsolatedUid()); } @@ -223,6 +245,21 @@ final class HotwordDetectionConnection { } } + private void unbindVisualQueryDetectionService() { + if (mRemoteVisualQueryDetectionService != null) { + mRemoteVisualQueryDetectionService.unbind(); + //TODO: Set visual query detection service provider to null + } + } + + private void unbindHotwordDetectionService() { + if (mRemoteHotwordDetectionService != null) { + mRemoteHotwordDetectionService.unbind(); + LocalServices.getService(PermissionManagerServiceInternal.class) + .setHotwordDetectionServiceProvider(null); + } + } + @SuppressWarnings("GuardedBy") void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, @NonNull IBinder token) { @@ -342,6 +379,10 @@ final class HotwordDetectionConnection { } } + void setDetectorType(int detectorType) { + mDetectorType = detectorType; + } + private void clearDebugHotwordLoggingTimeoutLocked() { if (mDebugHotwordLoggingTimeoutFuture != null) { mDebugHotwordLoggingTimeoutFuture.cancel(/* mayInterruptIfRunning= */ true); @@ -353,24 +394,41 @@ final class HotwordDetectionConnection { private void restartProcessLocked() { // TODO(b/244598068): Check HotwordAudioStreamManager first Slog.v(TAG, "Restarting hotword detection process"); + ServiceConnection oldHotwordConnection = mRemoteHotwordDetectionService; + ServiceConnection oldVisualQueryDetectionConnection = mRemoteVisualQueryDetectionService; HotwordDetectionServiceIdentity previousIdentity = mIdentity; + //TODO: Add previousIdentity for visual query detection service mLastRestartInstant = Instant.now(); // Recreate connection to reset the cache. + mRemoteHotwordDetectionService = mHotwordDetectionServiceConnectionFactory.createLocked(); + mRemoteVisualQueryDetectionService = + mVisualQueryDetectionServiceConnectionFactory.createLocked(); Slog.v(TAG, "Started the new process, dispatching processRestarted to detector"); runForEachDetectorSessionLocked((session) -> { - session.updateRemoteSandboxedDetectionServiceLocked(mRemoteHotwordDetectionService); + HotwordDetectionConnection.ServiceConnection newRemoteService = + (session instanceof VisualQueryDetectorSession) + ? mRemoteVisualQueryDetectionService : mRemoteHotwordDetectionService; + session.updateRemoteSandboxedDetectionServiceLocked(newRemoteService); session.informRestartProcessLocked(); }); if (DEBUG) { Slog.i(TAG, "processRestarted is dispatched done, unbinding from the old process"); } - oldHotwordConnection.ignoreConnectionStatusEvents(); - oldHotwordConnection.unbind(); + if (oldHotwordConnection != null) { + oldHotwordConnection.ignoreConnectionStatusEvents(); + oldHotwordConnection.unbind(); + } + + if (oldVisualQueryDetectionConnection != null) { + oldVisualQueryDetectionConnection.ignoreConnectionStatusEvents(); + oldVisualQueryDetectionConnection.unbind(); + } + if (previousIdentity != null) { removeServiceUidForAudioPolicy(previousIdentity.getIsolatedUid()); } @@ -438,9 +496,14 @@ final class HotwordDetectionConnection { synchronized (mLock) { pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); pw.print(prefix); pw.print("mBound="); - pw.println(mRemoteHotwordDetectionService.isBound()); + pw.println(mRemoteHotwordDetectionService != null + && mRemoteHotwordDetectionService.isBound()); + pw.println(mRemoteVisualQueryDetectionService != null + && mRemoteHotwordDetectionService != null + && mRemoteHotwordDetectionService.isBound()); pw.print(prefix); pw.print("mRestartCount="); pw.println(mHotwordDetectionServiceConnectionFactory.mRestartCount); + pw.println(mVisualQueryDetectionServiceConnectionFactory.mRestartCount); pw.print(prefix); pw.print("mLastRestartInstant="); pw.println(mLastRestartInstant); pw.print(prefix); pw.print("mDetectorType="); pw.println(HotwordDetector.detectorTypeToString(mDetectorType)); @@ -489,11 +552,11 @@ final class HotwordDetectionConnection { private boolean mIsLoggedFirstConnect = false; ServiceConnection(@NonNull Context context, - @NonNull Intent intent, int bindingFlags, int userId, + @NonNull Intent serviceIntent, int bindingFlags, int userId, @Nullable Function<IBinder, ISandboxedDetectionService> binderAsInterface, int instanceNumber) { - super(context, intent, bindingFlags, userId, binderAsInterface); - this.mIntent = intent; + super(context, serviceIntent, bindingFlags, userId, binderAsInterface); + this.mIntent = serviceIntent; this.mBindingFlags = bindingFlags; this.mInstanceNumber = instanceNumber; } @@ -512,14 +575,18 @@ final class HotwordDetectionConnection { mIsBound = connected; if (!connected) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED, + mVoiceInteractionServiceUid); + } } else if (!mIsLoggedFirstConnect) { mIsLoggedFirstConnect = true; - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED, + mVoiceInteractionServiceUid); + } } } } @@ -546,35 +613,46 @@ final class HotwordDetectionConnection { }); } // Can improve to log exit reason if needed - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + mDetectorType, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH, + mVoiceInteractionServiceUid); + } } @Override protected boolean bindService( @NonNull android.content.ServiceConnection serviceConnection) { try { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE, + mVoiceInteractionServiceUid); + } + String instancePrefix = + mIntent.getAction().equals(HotwordDetectionService.SERVICE_INTERFACE) + ? "hotword_detector_" : "visual_query_detector_"; boolean bindResult = mContext.bindIsolatedService( mIntent, Context.BIND_AUTO_CREATE | Context.BIND_FOREGROUND_SERVICE | mBindingFlags, - "hotword_detector_" + mInstanceNumber, + instancePrefix + mInstanceNumber, mExecutor, serviceConnection); if (!bindResult) { + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, + mVoiceInteractionServiceUid); + } + } + return bindResult; + } catch (IllegalArgumentException e) { + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { HotwordMetricsLogger.writeDetectorEvent(mDetectorType, HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, mVoiceInteractionServiceUid); } - return bindResult; - } catch (IllegalArgumentException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, - mVoiceInteractionServiceUid); Slog.wtf(TAG, "Can't bind to the hotword detection service!", e); return false; } @@ -609,10 +687,27 @@ final class HotwordDetectionConnection { } final DetectorSession session; if (detectorType == HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP) { + if (mRemoteHotwordDetectionService == null) { + mRemoteHotwordDetectionService = + mHotwordDetectionServiceConnectionFactory.createLocked(); + } session = new DspTrustedHotwordDetectorSession(mRemoteHotwordDetectionService, mLock, mContext, token, callback, mVoiceInteractionServiceUid, mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging); + } else if (detectorType == HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + if (mRemoteVisualQueryDetectionService == null) { + mRemoteVisualQueryDetectionService = + mVisualQueryDetectionServiceConnectionFactory.createLocked(); + } + session = new VisualQueryDetectorSession( + mRemoteVisualQueryDetectionService, mLock, mContext, token, callback, + mVoiceInteractionServiceUid, mVoiceInteractorIdentity, + mScheduledExecutorService, mDebugHotwordLogging); } else { + if (mRemoteHotwordDetectionService == null) { + mRemoteHotwordDetectionService = + mHotwordDetectionServiceConnectionFactory.createLocked(); + } session = new SoftwareTrustedHotwordDetectorSession( mRemoteHotwordDetectionService, mLock, mContext, token, callback, mVoiceInteractionServiceUid, mVoiceInteractorIdentity, @@ -625,13 +720,23 @@ final class HotwordDetectionConnection { @SuppressWarnings("GuardedBy") void destroyDetectorLocked(@NonNull IBinder token) { final DetectorSession session = getDetectorSessionByTokenLocked(token); - if (session != null) { - session.destroyLocked(); - final int index = mDetectorSessions.indexOfValue(session); - if (index < 0 || index > mDetectorSessions.size() - 1) { - return; - } - mDetectorSessions.removeAt(index); + if (session == null) { + return; + } + session.destroyLocked(); + final int index = mDetectorSessions.indexOfValue(session); + if (index < 0 || index > mDetectorSessions.size() - 1) { + return; + } + mDetectorSessions.removeAt(index); + if (session instanceof VisualQueryDetectorSession) { + unbindVisualQueryDetectionService(); + } + // Handle case where all hotword detector sessions are destroyed with only the visual + // detector session left + if (mDetectorSessions.size() == 1 + && mDetectorSessions.get(0) instanceof VisualQueryDetectorSession) { + unbindHotwordDetectionService(); } } @@ -672,6 +777,15 @@ final class HotwordDetectionConnection { } @SuppressWarnings("GuardedBy") + private VisualQueryDetectorSession getVisualQueryDetectorSessionLocked() { + final DetectorSession session = mDetectorSessions.get( + HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR); + if (session == null || session.isDestroyed()) { + Slog.v(TAG, "Not found the look and talk perceiver"); + return null; + } + return (VisualQueryDetectorSession) session; + } private void runForEachDetectorSessionLocked( @NonNull Consumer<DetectorSession> action) { for (int i = 0; i < mDetectorSessions.size(); i++) { diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java new file mode 100644 index 000000000000..6e4bc05ea25b --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.Context; +import android.media.AudioFormat; +import android.media.permission.Identity; +import android.os.IBinder; +import android.os.ParcelFileDescriptor; +import android.os.PersistableBundle; +import android.os.SharedMemory; +import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; +import android.util.Slog; + +import com.android.internal.app.IHotwordRecognitionStatusCallback; + +import java.io.PrintWriter; +import java.util.concurrent.ScheduledExecutorService; + +/** + * A class that provides visual query detector to communicate with the {@link + * android.service.voice.VisualQueryDetectionService}. + * + * This class can handle the visual query detection whose detector is created by using + * {@link android.service.voice.VoiceInteractionService#createVisualQueryDetector(PersistableBundle + * ,SharedMemory, HotwordDetector.Callback)}. + */ +final class VisualQueryDetectorSession extends DetectorSession { + + private static final String TAG = "VisualQueryDetectorSession"; + + //TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc. + VisualQueryDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteService, + @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + super(remoteService, lock, context, token, callback, + voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, + logging); + } + + @Override + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + Slog.v(TAG, "informRestartProcessLocked"); + mUpdateStateAfterStartFinished.set(false); + //TODO(b/261783819): Starts detection in VisualQueryDetectionService. + } + + @Override + void startListeningFromExternalSourceLocked( + ParcelFileDescriptor audioStream, + AudioFormat audioFormat, + @Nullable PersistableBundle options, + IMicrophoneHotwordDetectionVoiceInteractionCallback callback) + throws UnsupportedOperationException { + throw new UnsupportedOperationException("HotwordDetectionService method" + + " should not be called from VisualQueryDetectorSession."); + } + + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + super.dumpLocked(prefix, pw); + pw.print(prefix); + } +} + + diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java index 13c0f17dbfbb..e320e69ebceb 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java @@ -58,6 +58,7 @@ import android.os.RemoteException; import android.os.ServiceManager; import android.os.SharedMemory; import android.os.UserHandle; +import android.service.voice.HotwordDetector; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.IVoiceInteractionService; import android.service.voice.IVoiceInteractionSession; @@ -109,6 +110,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne final ComponentName mSessionComponentName; final IWindowManager mIWindowManager; final ComponentName mHotwordDetectionComponentName; + final ComponentName mVisualQueryDetectionComponentName; boolean mBound = false; IVoiceInteractionService mService; volatile HotwordDetectionConnection mHotwordDetectionConnection; @@ -211,6 +213,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne mInfo = null; mSessionComponentName = null; mHotwordDetectionComponentName = null; + mVisualQueryDetectionComponentName = null; mIWindowManager = null; mValid = false; return; @@ -220,6 +223,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne Slog.w(TAG, "Bad voice interaction service: " + mInfo.getParseError()); mSessionComponentName = null; mHotwordDetectionComponentName = null; + mVisualQueryDetectionComponentName = null; mIWindowManager = null; mValid = false; return; @@ -230,6 +234,9 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne final String hotwordDetectionServiceName = mInfo.getHotwordDetectionService(); mHotwordDetectionComponentName = hotwordDetectionServiceName != null ? new ComponentName(service.getPackageName(), hotwordDetectionServiceName) : null; + final String visualQueryDetectionServiceName = mInfo.getVisualQueryDetectionService(); + mVisualQueryDetectionComponentName = visualQueryDetectionServiceName != null ? new + ComponentName(service.getPackageName(), visualQueryDetectionServiceName) : null; mIWindowManager = IWindowManager.Stub.asInterface( ServiceManager.getService(Context.WINDOW_SERVICE)); IntentFilter filter = new IntentFilter(); @@ -591,14 +598,11 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne } } - public void initAndVerifyDetectorLocked( - @NonNull Identity voiceInteractorIdentity, - @Nullable PersistableBundle options, + private void verifyDetectorForHotwordDetectionLocked( @Nullable SharedMemory sharedMemory, - @NonNull IBinder token, IHotwordRecognitionStatusCallback callback, int detectorType) { - Slog.v(TAG, "initAndVerifyDetectorLocked"); + Slog.v(TAG, "verifyDetectorForHotwordDetectionLocked"); int voiceInteractionServiceUid = mInfo.getServiceInfo().applicationInfo.uid; if (mHotwordDetectionComponentName == null) { Slog.w(TAG, "Hotword detection service name not found"); @@ -649,11 +653,55 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne logDetectorCreateEventIfNeeded(callback, detectorType, true, voiceInteractionServiceUid); + } + + private void verifyDetectorForVisualQueryDetectionLocked(@Nullable SharedMemory sharedMemory) { + Slog.v(TAG, "verifyDetectorForVisualQueryDetectionLocked"); + + if (mVisualQueryDetectionComponentName == null) { + Slog.w(TAG, "Visual query detection service name not found"); + throw new IllegalStateException("Visual query detection service name not found"); + } + ServiceInfo visualQueryDetectionServiceInfo = getServiceInfoLocked( + mVisualQueryDetectionComponentName, mUser); + if (visualQueryDetectionServiceInfo == null) { + Slog.w(TAG, "Visual query detection service info not found"); + throw new IllegalStateException("Visual query detection service name not found"); + } + if (!isIsolatedProcessLocked(visualQueryDetectionServiceInfo)) { + Slog.w(TAG, "Visual query detection service not in isolated process"); + throw new IllegalStateException("Visual query detection not in isolated process"); + } + if (sharedMemory != null && !sharedMemory.setProtect(OsConstants.PROT_READ)) { + Slog.w(TAG, "Can't set sharedMemory to be read-only"); + throw new IllegalStateException("Can't set sharedMemory to be read-only"); + } + } + + public void initAndVerifyDetectorLocked( + @NonNull Identity voiceInteractorIdentity, + @Nullable PersistableBundle options, + @Nullable SharedMemory sharedMemory, + @NonNull IBinder token, + IHotwordRecognitionStatusCallback callback, + int detectorType) { + + if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + verifyDetectorForHotwordDetectionLocked(sharedMemory, callback, detectorType); + } else { + verifyDetectorForVisualQueryDetectionLocked(sharedMemory); + } + if (mHotwordDetectionConnection == null) { mHotwordDetectionConnection = new HotwordDetectionConnection(mServiceStub, mContext, mInfo.getServiceInfo().applicationInfo.uid, voiceInteractorIdentity, - mHotwordDetectionComponentName, mUser, /* bindInstantServiceAllowed= */ false, - detectorType); + mHotwordDetectionComponentName, mVisualQueryDetectionComponentName, mUser, + /* bindInstantServiceAllowed= */ false, detectorType); + } else if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + // TODO: Logger events should be handled in session instead. Temporary adding the + // checking to prevent confusion so VisualQueryDetection events won't be logged if the + // connection is instantiated by the VisualQueryDetector. + mHotwordDetectionConnection.setDetectorType(detectorType); } mHotwordDetectionConnection.createDetectorLocked(options, sharedMemory, token, callback, detectorType); |