From fd9097a75811db008f32818d8acbc135ea620ebf Mon Sep 17 00:00:00 2001 From: Charles Chen Date: Mon, 19 Dec 2022 07:24:32 +0000 Subject: Integrate VQDS lifecycle into connection session The change focuses on the initiliazation and the life cycle of the VisualQueryDetectionService. It also adds the VisualQueryDetector to the service by adding necessary connection sessions to the existing HotwordDetectionConnection which will now contain two ServiceConnection instances, for HotwordDetectionService and VisualQueryDetectionService respectively. Bug: 261783492 Test: Manual & atest CtsVoiceInteractionTestCases Change-Id: Ia3c771f3304c3a8849aa7cc209a5fc4ad838db45 --- .../android/service/voice/VisualQueryDetector.java | 81 +++++++- .../server/voiceinteraction/DetectorSession.java | 95 ++++++---- .../HotwordDetectionConnection.java | 206 ++++++++++++++++----- .../VisualQueryDetectorSession.java | 87 +++++++++ .../VoiceInteractionManagerServiceImpl.java | 62 ++++++- 5 files changed, 445 insertions(+), 86 deletions(-) create mode 100644 services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java diff --git a/core/java/android/service/voice/VisualQueryDetector.java b/core/java/android/service/voice/VisualQueryDetector.java index 241f5bac55ff..d24e69da9f4d 100644 --- a/core/java/android/service/voice/VisualQueryDetector.java +++ b/core/java/android/service/voice/VisualQueryDetector.java @@ -25,12 +25,16 @@ import android.annotation.Nullable; import android.annotation.RequiresPermission; import android.annotation.SuppressLint; import android.annotation.SystemApi; +import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; +import android.os.Binder; import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; +import android.os.RemoteException; import android.os.SharedMemory; import android.util.Slog; +import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVoiceInteractionManagerService; import java.io.PrintWriter; @@ -208,8 +212,9 @@ public class VisualQueryDetector { @Override void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) { - //TODO(261783492): call initAndVerify to create VisualQueryDetectionService - // from the system server. + initAndVerifyDetector(options, sharedMemory, + new InitializationStateListener(mExecutor, mCallback), + DETECTOR_TYPE_VISUAL_QUERY_DETECTOR); } @Override @@ -238,4 +243,76 @@ public class VisualQueryDetector { return true; } } + + private static class InitializationStateListener + extends IHotwordRecognitionStatusCallback.Stub { + private final Executor mExecutor; + private final Callback mCallback; + + InitializationStateListener(Executor executor, Callback callback) { + this.mExecutor = executor; + this.mCallback = callback; + } + + @Override + public void onKeyphraseDetected( + SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + HotwordDetectedResult result) { + if (DEBUG) { + Slog.i(TAG, "Ignored #onKeyphraseDetected event"); + } + } + + @Override + public void onGenericSoundTriggerDetected( + SoundTrigger.GenericRecognitionEvent recognitionEvent) throws RemoteException { + if (DEBUG) { + Slog.i(TAG, "Ignored #onGenericSoundTriggerDetected event"); + } + } + + @Override + public void onRejected(HotwordRejectedResult result) throws RemoteException { + if (DEBUG) { + Slog.i(TAG, "Ignored #onRejected event"); + } + } + + @Override + public void onRecognitionPaused() throws RemoteException { + if (DEBUG) { + Slog.i(TAG, "Ignored #onRecognitionPaused event"); + } + } + + @Override + public void onRecognitionResumed() throws RemoteException { + if (DEBUG) { + Slog.i(TAG, "Ignored #onRecognitionResumed event"); + } + } + + @Override + public void onStatusReported(int status) { + Slog.v(TAG, "onStatusReported" + (DEBUG ? "(" + status + ")" : "")); + //TODO: rename the target callback with a more general term + Binder.withCleanCallingIdentity(() -> mExecutor.execute( + () -> mCallback.onVisualQueryDetectionServiceInitialized(status))); + + } + + @Override + public void onProcessRestarted() throws RemoteException { + Slog.v(TAG, "onProcessRestarted()"); + //TODO: rename the target callback with a more general term + Binder.withCleanCallingIdentity(() -> mExecutor.execute( + () -> mCallback.onVisualQueryDetectionServiceRestarted())); + } + + @Override + public void onError(int status) throws RemoteException { + Slog.v(TAG, "Initialization Error: (" + status + ")"); + // Do nothing + } + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java index 4d6d3205db40..b8536f904569 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java @@ -205,9 +205,15 @@ abstract class DetectorSession { mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; mAppOpsManager = mContext.getSystemService(AppOpsManager.class); - mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, getDetectorType(), - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, + getDetectorType(), + mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, + mVoiceInteractorIdentity.attributionTag); + } else { + mHotwordAudioStreamCopier = null; + } + mScheduledExecutorService = scheduledExecutorService; mDebugHotwordLogging = logging; @@ -236,9 +242,12 @@ abstract class DetectorSession { future.complete(null); if (mUpdateStateAfterStartFinished.getAndSet(true)) { Slog.w(TAG, "call callback after timeout"); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + if (getDetectorType() + != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, mVoiceInteractionServiceUid); + } return; } Pair statusResultPair = getInitStatusAndMetricsResult(bundle); @@ -246,27 +255,37 @@ abstract class DetectorSession { int initResultMetricsResult = statusResultPair.second; try { mCallback.onStatusReported(status); - HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), - initResultMetricsResult, mVoiceInteractionServiceUid); + if (getDetectorType() + != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), + initResultMetricsResult, mVoiceInteractionServiceUid); + } } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status: " + e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() + != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, + mVoiceInteractionServiceUid); + } } } }; try { service.updateState(options, sharedMemory, statusCallback); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, + mVoiceInteractionServiceUid); + } } catch (RemoteException e) { // TODO: (b/181842909) Report an error to voice interactor Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, + mVoiceInteractionServiceUid); + } } return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); }).whenComplete((res, err) -> { @@ -277,13 +296,17 @@ abstract class DetectorSession { } try { mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); - HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), - METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), + METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid); + } } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, + mVoiceInteractionServiceUid); + } } } else if (err != null) { Slog.w(TAG, "Failed to update state: " + err); @@ -315,9 +338,11 @@ abstract class DetectorSession { @SuppressWarnings("GuardedBy") void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, Instant lastRestartInstant) { - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, + mVoiceInteractionServiceUid); + } // Prevent doing the init late, so restart is handled equally to a clean process start. // TODO(b/191742511): this logic needs a test if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus( @@ -399,9 +424,11 @@ abstract class DetectorSession { callback.onError(); } catch (RemoteException ex) { Slog.w(TAG, "Failed to report onError status: " + ex); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } } } finally { synchronized (mLock) { @@ -427,7 +454,8 @@ abstract class DetectorSession { throws RemoteException { synchronized (mLock) { mPerformingExternalSourceHotwordDetection = false; - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HotwordMetricsLogger.writeDetectorEvent( + getDetectorType(), METRICS_EXTERNAL_SOURCE_REJECTED, mVoiceInteractionServiceUid); mScheduledExecutorService.schedule( @@ -454,7 +482,8 @@ abstract class DetectorSession { throws RemoteException { synchronized (mLock) { mPerformingExternalSourceHotwordDetection = false; - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HotwordMetricsLogger.writeDetectorEvent( + getDetectorType(), METRICS_EXTERNAL_SOURCE_DETECTED, mVoiceInteractionServiceUid); mScheduledExecutorService.schedule( @@ -540,9 +569,11 @@ abstract class DetectorSession { mCallback.onError(status); } catch (RemoteException e) { Slog.w(TAG, "Failed to report onError status: " + e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); + if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } } } @@ -679,6 +710,8 @@ abstract class DetectorSession { return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP; } else if (this instanceof SoftwareTrustedHotwordDetectorSession) { return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE; + } else if (this instanceof VisualQueryDetectorSession) { + return HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR; } Slog.v(TAG, "Unexpected detector type"); return -1; diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index d501af7d83be..f1dd9091d5de 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -52,6 +52,7 @@ import android.service.voice.HotwordDetectionService; import android.service.voice.HotwordDetector; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.ISandboxedDetectionService; +import android.service.voice.VisualQueryDetectionService; import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity; import android.speech.IRecognitionServiceManager; import android.util.Slog; @@ -74,7 +75,8 @@ import java.util.function.Consumer; import java.util.function.Function; /** - * A class that provides the communication with the HotwordDetectionService. + * A class that provides the communication with the {@link HotwordDetectionService} and + * {@link VisualQueryDetectionService}. */ final class HotwordDetectionConnection { private static final String TAG = "HotwordDetectionConnection"; @@ -90,7 +92,8 @@ final class HotwordDetectionConnection { @Nullable private final ScheduledFuture mCancellationTaskFuture; private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied; @NonNull private final ServiceConnectionFactory mHotwordDetectionServiceConnectionFactory; - private final int mDetectorType; + @NonNull private final ServiceConnectionFactory mVisualQueryDetectionServiceConnectionFactory; + private int mDetectorType; /** * Time after which each HotwordDetectionService process is stopped and replaced by a new one. * 0 indicates no restarts. @@ -100,9 +103,11 @@ final class HotwordDetectionConnection { final Object mLock; final int mVoiceInteractionServiceUid; final ComponentName mHotwordDetectionComponentName; + final ComponentName mVisualQueryDetectionComponentName; final int mUser; final Context mContext; volatile HotwordDetectionServiceIdentity mIdentity; + //TODO: add similar identity for visual query service for the use of content capturing private Instant mLastRestartInstant; private ScheduledFuture mDebugHotwordLoggingTimeoutFuture = null; @@ -112,6 +117,7 @@ final class HotwordDetectionConnection { @Nullable private final Identity mVoiceInteractorIdentity; @NonNull private ServiceConnection mRemoteHotwordDetectionService; + @NonNull private ServiceConnection mRemoteVisualQueryDetectionService; private IBinder mAudioFlinger; @GuardedBy("mLock") private boolean mDebugHotwordLogging = false; @@ -126,26 +132,39 @@ final class HotwordDetectionConnection { new SparseArray<>(); HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid, - Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, int userId, + Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, + ComponentName visualQueryDetectionServiceName, int userId, boolean bindInstantServiceAllowed, int detectorType) { mLock = lock; mContext = context; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; mHotwordDetectionComponentName = hotwordDetectionServiceName; + mVisualQueryDetectionComponentName = visualQueryDetectionServiceName; mUser = userId; mDetectorType = detectorType; mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION, KEY_RESTART_PERIOD_IN_SECONDS, 0); + final Intent hotwordDetectionServiceIntent = new Intent(HotwordDetectionService.SERVICE_INTERFACE); hotwordDetectionServiceIntent.setComponent(mHotwordDetectionComponentName); + + final Intent visualQueryDetectionServiceIntent = + new Intent(VisualQueryDetectionService.SERVICE_INTERFACE); + visualQueryDetectionServiceIntent.setComponent(mVisualQueryDetectionComponentName); + initAudioFlingerLocked(); mHotwordDetectionServiceConnectionFactory = new ServiceConnectionFactory(hotwordDetectionServiceIntent, bindInstantServiceAllowed); - mRemoteHotwordDetectionService = mHotwordDetectionServiceConnectionFactory.createLocked(); + + mVisualQueryDetectionServiceConnectionFactory = + new ServiceConnectionFactory(visualQueryDetectionServiceIntent, + bindInstantServiceAllowed); + + mLastRestartInstant = Instant.now(); if (mReStartPeriodSeconds <= 0) { @@ -157,9 +176,11 @@ final class HotwordDetectionConnection { Slog.v(TAG, "Time to restart the process, TTL has passed"); synchronized (mLock) { restartProcessLocked(); - HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, - HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, + HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE, + mVoiceInteractionServiceUid); + } } }, mReStartPeriodSeconds, mReStartPeriodSeconds, TimeUnit.SECONDS); } @@ -193,9 +214,11 @@ final class HotwordDetectionConnection { // We restart the process instead of simply sending over the new binder, to avoid race // conditions with audio reading in the service. restartProcessLocked(); - HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, - HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, + HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED, + mVoiceInteractionServiceUid); + } } } @@ -208,9 +231,8 @@ final class HotwordDetectionConnection { }); mDetectorSessions.clear(); mDebugHotwordLogging = false; - mRemoteHotwordDetectionService.unbind(); - LocalServices.getService(PermissionManagerServiceInternal.class) - .setHotwordDetectionServiceProvider(null); + unbindVisualQueryDetectionService(); + unbindHotwordDetectionService(); if (mIdentity != null) { removeServiceUidForAudioPolicy(mIdentity.getIsolatedUid()); } @@ -223,6 +245,21 @@ final class HotwordDetectionConnection { } } + private void unbindVisualQueryDetectionService() { + if (mRemoteVisualQueryDetectionService != null) { + mRemoteVisualQueryDetectionService.unbind(); + //TODO: Set visual query detection service provider to null + } + } + + private void unbindHotwordDetectionService() { + if (mRemoteHotwordDetectionService != null) { + mRemoteHotwordDetectionService.unbind(); + LocalServices.getService(PermissionManagerServiceInternal.class) + .setHotwordDetectionServiceProvider(null); + } + } + @SuppressWarnings("GuardedBy") void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, @NonNull IBinder token) { @@ -342,6 +379,10 @@ final class HotwordDetectionConnection { } } + void setDetectorType(int detectorType) { + mDetectorType = detectorType; + } + private void clearDebugHotwordLoggingTimeoutLocked() { if (mDebugHotwordLoggingTimeoutFuture != null) { mDebugHotwordLoggingTimeoutFuture.cancel(/* mayInterruptIfRunning= */ true); @@ -353,24 +394,41 @@ final class HotwordDetectionConnection { private void restartProcessLocked() { // TODO(b/244598068): Check HotwordAudioStreamManager first Slog.v(TAG, "Restarting hotword detection process"); + ServiceConnection oldHotwordConnection = mRemoteHotwordDetectionService; + ServiceConnection oldVisualQueryDetectionConnection = mRemoteVisualQueryDetectionService; HotwordDetectionServiceIdentity previousIdentity = mIdentity; + //TODO: Add previousIdentity for visual query detection service mLastRestartInstant = Instant.now(); // Recreate connection to reset the cache. + mRemoteHotwordDetectionService = mHotwordDetectionServiceConnectionFactory.createLocked(); + mRemoteVisualQueryDetectionService = + mVisualQueryDetectionServiceConnectionFactory.createLocked(); Slog.v(TAG, "Started the new process, dispatching processRestarted to detector"); runForEachDetectorSessionLocked((session) -> { - session.updateRemoteSandboxedDetectionServiceLocked(mRemoteHotwordDetectionService); + HotwordDetectionConnection.ServiceConnection newRemoteService = + (session instanceof VisualQueryDetectorSession) + ? mRemoteVisualQueryDetectionService : mRemoteHotwordDetectionService; + session.updateRemoteSandboxedDetectionServiceLocked(newRemoteService); session.informRestartProcessLocked(); }); if (DEBUG) { Slog.i(TAG, "processRestarted is dispatched done, unbinding from the old process"); } - oldHotwordConnection.ignoreConnectionStatusEvents(); - oldHotwordConnection.unbind(); + if (oldHotwordConnection != null) { + oldHotwordConnection.ignoreConnectionStatusEvents(); + oldHotwordConnection.unbind(); + } + + if (oldVisualQueryDetectionConnection != null) { + oldVisualQueryDetectionConnection.ignoreConnectionStatusEvents(); + oldVisualQueryDetectionConnection.unbind(); + } + if (previousIdentity != null) { removeServiceUidForAudioPolicy(previousIdentity.getIsolatedUid()); } @@ -438,9 +496,14 @@ final class HotwordDetectionConnection { synchronized (mLock) { pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); pw.print(prefix); pw.print("mBound="); - pw.println(mRemoteHotwordDetectionService.isBound()); + pw.println(mRemoteHotwordDetectionService != null + && mRemoteHotwordDetectionService.isBound()); + pw.println(mRemoteVisualQueryDetectionService != null + && mRemoteHotwordDetectionService != null + && mRemoteHotwordDetectionService.isBound()); pw.print(prefix); pw.print("mRestartCount="); pw.println(mHotwordDetectionServiceConnectionFactory.mRestartCount); + pw.println(mVisualQueryDetectionServiceConnectionFactory.mRestartCount); pw.print(prefix); pw.print("mLastRestartInstant="); pw.println(mLastRestartInstant); pw.print(prefix); pw.print("mDetectorType="); pw.println(HotwordDetector.detectorTypeToString(mDetectorType)); @@ -489,11 +552,11 @@ final class HotwordDetectionConnection { private boolean mIsLoggedFirstConnect = false; ServiceConnection(@NonNull Context context, - @NonNull Intent intent, int bindingFlags, int userId, + @NonNull Intent serviceIntent, int bindingFlags, int userId, @Nullable Function binderAsInterface, int instanceNumber) { - super(context, intent, bindingFlags, userId, binderAsInterface); - this.mIntent = intent; + super(context, serviceIntent, bindingFlags, userId, binderAsInterface); + this.mIntent = serviceIntent; this.mBindingFlags = bindingFlags; this.mInstanceNumber = instanceNumber; } @@ -512,14 +575,18 @@ final class HotwordDetectionConnection { mIsBound = connected; if (!connected) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED, + mVoiceInteractionServiceUid); + } } else if (!mIsLoggedFirstConnect) { mIsLoggedFirstConnect = true; - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED, + mVoiceInteractionServiceUid); + } } } } @@ -546,35 +613,46 @@ final class HotwordDetectionConnection { }); } // Can improve to log exit reason if needed - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + mDetectorType, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH, + mVoiceInteractionServiceUid); + } } @Override protected boolean bindService( @NonNull android.content.ServiceConnection serviceConnection) { try { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE, - mVoiceInteractionServiceUid); + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE, + mVoiceInteractionServiceUid); + } + String instancePrefix = + mIntent.getAction().equals(HotwordDetectionService.SERVICE_INTERFACE) + ? "hotword_detector_" : "visual_query_detector_"; boolean bindResult = mContext.bindIsolatedService( mIntent, Context.BIND_AUTO_CREATE | Context.BIND_FOREGROUND_SERVICE | mBindingFlags, - "hotword_detector_" + mInstanceNumber, + instancePrefix + mInstanceNumber, mExecutor, serviceConnection); if (!bindResult) { + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, + mVoiceInteractionServiceUid); + } + } + return bindResult; + } catch (IllegalArgumentException e) { + if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { HotwordMetricsLogger.writeDetectorEvent(mDetectorType, HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, mVoiceInteractionServiceUid); } - return bindResult; - } catch (IllegalArgumentException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, - mVoiceInteractionServiceUid); Slog.wtf(TAG, "Can't bind to the hotword detection service!", e); return false; } @@ -609,10 +687,27 @@ final class HotwordDetectionConnection { } final DetectorSession session; if (detectorType == HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP) { + if (mRemoteHotwordDetectionService == null) { + mRemoteHotwordDetectionService = + mHotwordDetectionServiceConnectionFactory.createLocked(); + } session = new DspTrustedHotwordDetectorSession(mRemoteHotwordDetectionService, mLock, mContext, token, callback, mVoiceInteractionServiceUid, mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging); + } else if (detectorType == HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + if (mRemoteVisualQueryDetectionService == null) { + mRemoteVisualQueryDetectionService = + mVisualQueryDetectionServiceConnectionFactory.createLocked(); + } + session = new VisualQueryDetectorSession( + mRemoteVisualQueryDetectionService, mLock, mContext, token, callback, + mVoiceInteractionServiceUid, mVoiceInteractorIdentity, + mScheduledExecutorService, mDebugHotwordLogging); } else { + if (mRemoteHotwordDetectionService == null) { + mRemoteHotwordDetectionService = + mHotwordDetectionServiceConnectionFactory.createLocked(); + } session = new SoftwareTrustedHotwordDetectorSession( mRemoteHotwordDetectionService, mLock, mContext, token, callback, mVoiceInteractionServiceUid, mVoiceInteractorIdentity, @@ -625,13 +720,23 @@ final class HotwordDetectionConnection { @SuppressWarnings("GuardedBy") void destroyDetectorLocked(@NonNull IBinder token) { final DetectorSession session = getDetectorSessionByTokenLocked(token); - if (session != null) { - session.destroyLocked(); - final int index = mDetectorSessions.indexOfValue(session); - if (index < 0 || index > mDetectorSessions.size() - 1) { - return; - } - mDetectorSessions.removeAt(index); + if (session == null) { + return; + } + session.destroyLocked(); + final int index = mDetectorSessions.indexOfValue(session); + if (index < 0 || index > mDetectorSessions.size() - 1) { + return; + } + mDetectorSessions.removeAt(index); + if (session instanceof VisualQueryDetectorSession) { + unbindVisualQueryDetectionService(); + } + // Handle case where all hotword detector sessions are destroyed with only the visual + // detector session left + if (mDetectorSessions.size() == 1 + && mDetectorSessions.get(0) instanceof VisualQueryDetectorSession) { + unbindHotwordDetectionService(); } } @@ -672,6 +777,15 @@ final class HotwordDetectionConnection { } @SuppressWarnings("GuardedBy") + private VisualQueryDetectorSession getVisualQueryDetectorSessionLocked() { + final DetectorSession session = mDetectorSessions.get( + HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR); + if (session == null || session.isDestroyed()) { + Slog.v(TAG, "Not found the look and talk perceiver"); + return null; + } + return (VisualQueryDetectorSession) session; + } private void runForEachDetectorSessionLocked( @NonNull Consumer action) { for (int i = 0; i < mDetectorSessions.size(); i++) { diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java new file mode 100644 index 000000000000..6e4bc05ea25b --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.Context; +import android.media.AudioFormat; +import android.media.permission.Identity; +import android.os.IBinder; +import android.os.ParcelFileDescriptor; +import android.os.PersistableBundle; +import android.os.SharedMemory; +import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; +import android.util.Slog; + +import com.android.internal.app.IHotwordRecognitionStatusCallback; + +import java.io.PrintWriter; +import java.util.concurrent.ScheduledExecutorService; + +/** + * A class that provides visual query detector to communicate with the {@link + * android.service.voice.VisualQueryDetectionService}. + * + * This class can handle the visual query detection whose detector is created by using + * {@link android.service.voice.VoiceInteractionService#createVisualQueryDetector(PersistableBundle + * ,SharedMemory, HotwordDetector.Callback)}. + */ +final class VisualQueryDetectorSession extends DetectorSession { + + private static final String TAG = "VisualQueryDetectorSession"; + + //TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc. + VisualQueryDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteService, + @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + super(remoteService, lock, context, token, callback, + voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, + logging); + } + + @Override + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + Slog.v(TAG, "informRestartProcessLocked"); + mUpdateStateAfterStartFinished.set(false); + //TODO(b/261783819): Starts detection in VisualQueryDetectionService. + } + + @Override + void startListeningFromExternalSourceLocked( + ParcelFileDescriptor audioStream, + AudioFormat audioFormat, + @Nullable PersistableBundle options, + IMicrophoneHotwordDetectionVoiceInteractionCallback callback) + throws UnsupportedOperationException { + throw new UnsupportedOperationException("HotwordDetectionService method" + + " should not be called from VisualQueryDetectorSession."); + } + + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + super.dumpLocked(prefix, pw); + pw.print(prefix); + } +} + + diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java index 964328271866..c68628087736 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java @@ -58,6 +58,7 @@ import android.os.RemoteException; import android.os.ServiceManager; import android.os.SharedMemory; import android.os.UserHandle; +import android.service.voice.HotwordDetector; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.IVoiceInteractionService; import android.service.voice.IVoiceInteractionSession; @@ -109,6 +110,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne final ComponentName mSessionComponentName; final IWindowManager mIWindowManager; final ComponentName mHotwordDetectionComponentName; + final ComponentName mVisualQueryDetectionComponentName; boolean mBound = false; IVoiceInteractionService mService; volatile HotwordDetectionConnection mHotwordDetectionConnection; @@ -211,6 +213,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne mInfo = null; mSessionComponentName = null; mHotwordDetectionComponentName = null; + mVisualQueryDetectionComponentName = null; mIWindowManager = null; mValid = false; return; @@ -220,6 +223,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne Slog.w(TAG, "Bad voice interaction service: " + mInfo.getParseError()); mSessionComponentName = null; mHotwordDetectionComponentName = null; + mVisualQueryDetectionComponentName = null; mIWindowManager = null; mValid = false; return; @@ -230,6 +234,9 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne final String hotwordDetectionServiceName = mInfo.getHotwordDetectionService(); mHotwordDetectionComponentName = hotwordDetectionServiceName != null ? new ComponentName(service.getPackageName(), hotwordDetectionServiceName) : null; + final String visualQueryDetectionServiceName = mInfo.getVisualQueryDetectionService(); + mVisualQueryDetectionComponentName = visualQueryDetectionServiceName != null ? new + ComponentName(service.getPackageName(), visualQueryDetectionServiceName) : null; mIWindowManager = IWindowManager.Stub.asInterface( ServiceManager.getService(Context.WINDOW_SERVICE)); IntentFilter filter = new IntentFilter(); @@ -573,14 +580,11 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne } } - public void initAndVerifyDetectorLocked( - @NonNull Identity voiceInteractorIdentity, - @Nullable PersistableBundle options, + private void verifyDetectorForHotwordDetectionLocked( @Nullable SharedMemory sharedMemory, - @NonNull IBinder token, IHotwordRecognitionStatusCallback callback, int detectorType) { - Slog.v(TAG, "initAndVerifyDetectorLocked"); + Slog.v(TAG, "verifyDetectorForHotwordDetectionLocked"); int voiceInteractionServiceUid = mInfo.getServiceInfo().applicationInfo.uid; if (mHotwordDetectionComponentName == null) { Slog.w(TAG, "Hotword detection service name not found"); @@ -631,11 +635,55 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne logDetectorCreateEventIfNeeded(callback, detectorType, true, voiceInteractionServiceUid); + } + + private void verifyDetectorForVisualQueryDetectionLocked(@Nullable SharedMemory sharedMemory) { + Slog.v(TAG, "verifyDetectorForVisualQueryDetectionLocked"); + + if (mVisualQueryDetectionComponentName == null) { + Slog.w(TAG, "Visual query detection service name not found"); + throw new IllegalStateException("Visual query detection service name not found"); + } + ServiceInfo visualQueryDetectionServiceInfo = getServiceInfoLocked( + mVisualQueryDetectionComponentName, mUser); + if (visualQueryDetectionServiceInfo == null) { + Slog.w(TAG, "Visual query detection service info not found"); + throw new IllegalStateException("Visual query detection service name not found"); + } + if (!isIsolatedProcessLocked(visualQueryDetectionServiceInfo)) { + Slog.w(TAG, "Visual query detection service not in isolated process"); + throw new IllegalStateException("Visual query detection not in isolated process"); + } + if (sharedMemory != null && !sharedMemory.setProtect(OsConstants.PROT_READ)) { + Slog.w(TAG, "Can't set sharedMemory to be read-only"); + throw new IllegalStateException("Can't set sharedMemory to be read-only"); + } + } + + public void initAndVerifyDetectorLocked( + @NonNull Identity voiceInteractorIdentity, + @Nullable PersistableBundle options, + @Nullable SharedMemory sharedMemory, + @NonNull IBinder token, + IHotwordRecognitionStatusCallback callback, + int detectorType) { + + if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + verifyDetectorForHotwordDetectionLocked(sharedMemory, callback, detectorType); + } else { + verifyDetectorForVisualQueryDetectionLocked(sharedMemory); + } + if (mHotwordDetectionConnection == null) { mHotwordDetectionConnection = new HotwordDetectionConnection(mServiceStub, mContext, mInfo.getServiceInfo().applicationInfo.uid, voiceInteractorIdentity, - mHotwordDetectionComponentName, mUser, /* bindInstantServiceAllowed= */ false, - detectorType); + mHotwordDetectionComponentName, mVisualQueryDetectionComponentName, mUser, + /* bindInstantServiceAllowed= */ false, detectorType); + } else if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { + // TODO: Logger events should be handled in session instead. Temporary adding the + // checking to prevent confusion so VisualQueryDetection events won't be logged if the + // connection is instantiated by the VisualQueryDetector. + mHotwordDetectionConnection.setDetectorType(detectorType); } mHotwordDetectionConnection.createDetectorLocked(options, sharedMemory, token, callback, detectorType); -- cgit v1.2.3-59-g8ed1b