diff options
| author | 2022-11-18 09:30:08 +0000 | |
|---|---|---|
| committer | 2022-12-08 20:45:06 +0000 | |
| commit | 96e34f053b2fa321f8aa687600e4d3f096676293 (patch) | |
| tree | 3278c5852433b48c3ab7f7d1e17d1905b9034145 | |
| parent | 1566016119cc3fa5365e380ca3cbc9d127786d86 (diff) | |
Move the methods into Dsp session and Software session
Move the methods of HotwordDetectorSession into
DspTrustedHotwordDetectorSession and
SoftwareTrustedHotwordDetectorSession
Test: atest CtsVoiceInteractionTestCases
Bug: 241041976
Change-Id: I53139507697ba86f04158896ef95095993f23f2c
4 files changed, 510 insertions, 399 deletions
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java index c092d9badcb2..2812264e989c 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java @@ -16,20 +16,61 @@ package com.android.server.voiceinteraction; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; + import android.annotation.NonNull; import android.content.Context; +import android.hardware.soundtrigger.SoundTrigger; import android.media.permission.Identity; +import android.os.PersistableBundle; +import android.os.RemoteException; +import android.os.SharedMemory; +import android.service.voice.AlwaysOnHotwordDetector; +import android.service.voice.HotwordDetectedResult; import android.service.voice.HotwordDetectionService; +import android.service.voice.HotwordDetector; +import android.service.voice.HotwordRejectedResult; +import android.service.voice.IDspHotwordDetectionCallback; +import android.util.Slog; +import com.android.internal.annotations.GuardedBy; import com.android.internal.app.IHotwordRecognitionStatusCallback; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Locale; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; /** * A class that provides Dsp trusted hotword detector to communicate with the {@link * HotwordDetectionService}. + * + * This class can handle the hotword detection which detector is created by using + * {@link android.service.voice.VoiceInteractionService#createAlwaysOnHotwordDetector(String, + * Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)}. */ final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession { + private static final String TAG = "DspTrustedHotwordDetectorSession"; + + // The validation timeout value is 3 seconds for onDetect of DSP trigger event. + private static final long VALIDATION_TIMEOUT_MILLIS = 3000; + // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. + private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; + + @GuardedBy("mLock") + private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture; + + @GuardedBy("mLock") + private boolean mValidatingDspTrigger = false; DspTrustedHotwordDetectorSession( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @@ -40,4 +81,174 @@ final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession { super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, logging); } + + @SuppressWarnings("GuardedBy") + void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + IHotwordRecognitionStatusCallback externalCallback) { + if (DEBUG) { + Slog.d(TAG, "detectFromDspSourceLocked"); + } + + AtomicBoolean timeoutDetected = new AtomicBoolean(false); + // TODO: consider making this a non-anonymous class. + IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { + @Override + public void onDetected(HotwordDetectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onDetected"); + } + synchronized (mLock) { + if (mCancellationKeyPhraseDetectionFuture != null) { + mCancellationKeyPhraseDetectionFuture.cancel(true); + } + if (timeoutDetected.get()) { + return; + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); + if (!mValidatingDspTrigger) { + Slog.i(TAG, "Ignoring #onDetected due to a process restart"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); + return; + } + mValidatingDspTrigger = false; + try { + enforcePermissionsForDataDelivery(); + enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); + } catch (SecurityException e) { + Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); + externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); + return; + } + saveProximityValueToBundle(result); + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); + } catch (IOException e) { + externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); + return; + } + externalCallback.onKeyphraseDetected(recognitionEvent, newResult); + Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + newResult); + } + } + } + + @Override + public void onRejected(HotwordRejectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onRejected"); + } + synchronized (mLock) { + if (mCancellationKeyPhraseDetectionFuture != null) { + mCancellationKeyPhraseDetectionFuture.cancel(true); + } + if (timeoutDetected.get()) { + return; + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); + if (!mValidatingDspTrigger) { + Slog.i(TAG, "Ignoring #onRejected due to a process restart"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); + return; + } + mValidatingDspTrigger = false; + externalCallback.onRejected(result); + if (mDebugHotwordLogging && result != null) { + Slog.i(TAG, "Egressed rejected result: " + result); + } + } + } + }; + + mValidatingDspTrigger = true; + mRemoteHotwordDetectionService.run(service -> { + // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke + // the callback before timeout value. In order to reduce the latency impact between + // server side and client side, we need to use another timeout value + // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. + mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( + () -> { + // TODO: avoid allocate every time + timeoutDetected.set(true); + Slog.w(TAG, "Timed out on #detectFromDspSource"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); + try { + externalCallback.onError(CALLBACK_DETECT_TIMEOUT); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report onError status: ", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } + }, + MAX_VALIDATION_TIMEOUT_MILLIS, + TimeUnit.MILLISECONDS); + service.detectFromDspSource( + recognitionEvent, + recognitionEvent.getCaptureFormat(), + VALIDATION_TIMEOUT_MILLIS, + internalCallback); + }); + } + + @Override + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + // TODO(b/244598068): Check HotwordAudioStreamManager first + Slog.v(TAG, "informRestartProcessLocked"); + if (mValidatingDspTrigger) { + // We're restarting the process while it's processing a DSP trigger, so report a + // rejection. This also allows the Interactor to startRecognition again + try { + mCallback.onRejected(new HotwordRejectedResult.Builder().build()); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to call #rejected"); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + mValidatingDspTrigger = false; + } + mUpdateStateAfterStartFinished.set(false); + + try { + mCallback.onProcessRestarted(); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("process restarted"); + } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + super.dumpLocked(prefix, pw); + pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index 7e105c3b18c5..2a4d4a15afdf 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -244,7 +244,8 @@ final class HotwordDetectionConnection { Slog.d(TAG, "It is not a software detector"); return; } - mHotwordDetectorSession.startListeningFromMicLocked(audioFormat, callback); + ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession) + .startListeningFromMicLocked(audioFormat, callback); } } @@ -274,7 +275,7 @@ final class HotwordDetectionConnection { Slog.d(TAG, "It is not a software detector"); return; } - mHotwordDetectorSession.stopListeningLocked(); + ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession).stopListeningLocked(); } } @@ -297,7 +298,8 @@ final class HotwordDetectionConnection { Slog.d(TAG, "It is not a Dsp detector"); return; } - mHotwordDetectorSession.detectFromDspSourceLocked(recognitionEvent, externalCallback); + ((DspTrustedHotwordDetectorSession) mHotwordDetectorSession).detectFromDspSourceLocked( + recognitionEvent, externalCallback); } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java index 9cd4f68222a9..f9f43c9ba49a 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java @@ -20,7 +20,6 @@ import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; import static android.Manifest.permission.RECORD_AUDIO; import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN; import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL; -import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE; import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT; import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS; import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN; @@ -33,8 +32,6 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERV import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION; @@ -43,13 +40,8 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENT import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight; @@ -75,7 +67,6 @@ import android.service.voice.HotwordDetectionService; import android.service.voice.HotwordDetector; import android.service.voice.HotwordRejectedResult; import android.service.voice.IDspHotwordDetectionCallback; -import android.service.voice.IHotwordDetectionService; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.text.TextUtils; import android.util.Pair; @@ -96,7 +87,6 @@ import java.time.Instant; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -104,8 +94,15 @@ import java.util.concurrent.atomic.AtomicBoolean; /** * A class that provides trusted hotword detector to communicate with the {@link * HotwordDetectionService}. + * + * This class provides the methods to do initialization with the {@link HotwordDetectionService} + * and handle external source detection. It also provides the methods to check if we can egress + * the data from the {@link HotwordDetectionService}. + * + * The subclass should override the {@link #informRestartProcessLocked()} to handle the trusted + * process restart. */ -class HotwordDetectorSession { +abstract class HotwordDetectorSession { private static final String TAG = "HotwordDetectorSession"; static final boolean DEBUG = false; @@ -119,10 +116,6 @@ class HotwordDetectorSession { static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; // TODO: These constants need to be refined. - // The validation timeout value is 3 seconds for onDetect of DSP trigger event. - private static final long VALIDATION_TIMEOUT_MILLIS = 3000; - // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. - private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000; private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; private static final Duration MAX_UPDATE_TIMEOUT_DURATION = @@ -140,29 +133,29 @@ class HotwordDetectorSession { private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS = HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = + static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = + static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; - private static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = + static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; private static final int METRICS_EXTERNAL_SOURCE_DETECTED = HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; private static final int METRICS_EXTERNAL_SOURCE_REJECTED = HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; - private static final int METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = + private static final int EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION = HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); // TODO: This may need to be a Handler(looper) - private final ScheduledExecutorService mScheduledExecutorService; + final ScheduledExecutorService mScheduledExecutorService; private final AppOpsManager mAppOpsManager; - private final HotwordAudioStreamCopier mHotwordAudioStreamCopier; - private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); - private final IHotwordRecognitionStatusCallback mCallback; + final HotwordAudioStreamCopier mHotwordAudioStreamCopier; + final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); + final IHotwordRecognitionStatusCallback mCallback; final Object mLock; final int mVoiceInteractionServiceUid; @@ -173,28 +166,22 @@ class HotwordDetectorSession { final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = this::setProximityValue; - private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; - - private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture; - /** Identity used for attributing app ops when delivering data to the Interactor. */ - @GuardedBy("mLock") @Nullable private final Identity mVoiceInteractorIdentity; @GuardedBy("mLock") - private ParcelFileDescriptor mCurrentAudioSink; - @GuardedBy("mLock") - private boolean mValidatingDspTrigger = false; + ParcelFileDescriptor mCurrentAudioSink; @GuardedBy("mLock") - private boolean mPerformingSoftwareHotwordDetection; - @NonNull private HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService; - private boolean mDebugHotwordLogging = false; + @NonNull HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService; + boolean mDebugHotwordLogging = false; @GuardedBy("mLock") private double mProximityMeters = PROXIMITY_UNKNOWN; @GuardedBy("mLock") private boolean mInitialized = false; @GuardedBy("mLock") private boolean mDestroyed = false; + @GuardedBy("mLock") + boolean mPerformingExternalSourceHotwordDetection; HotwordDetectorSession( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @@ -223,6 +210,7 @@ class HotwordDetectorSession { } } + @SuppressWarnings("GuardedBy") private void updateStateAfterProcessStartLocked(PersistableBundle options, SharedMemory sharedMemory) { if (DEBUG) { @@ -315,6 +303,7 @@ class HotwordDetectorSession { return new Pair<>(status, metricsResult); } + @SuppressWarnings("GuardedBy") void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, Instant lastRestartInstant) { HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), @@ -332,108 +321,13 @@ class HotwordDetectorSession { } } - /** - * This method is only used by SoftwareHotwordDetector. - */ - @SuppressWarnings("GuardedBy") - void startListeningFromMicLocked( - AudioFormat audioFormat, - IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { - if (DEBUG) { - Slog.d(TAG, "startListeningFromMic"); - } - mSoftwareCallback = callback; - - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword validation is already in progress, ignoring."); - return; - } - mPerformingSoftwareHotwordDetection = true; - - startListeningFromMicLocked(); - } - - /** - * This method is only used by SoftwareHotwordDetector. - */ - private void startListeningFromMicLocked() { - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection has already completed"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mPerformingSoftwareHotwordDetection = false; - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - mSoftwareCallback.onError(); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - mSoftwareCallback.onError(); - return; - } - mSoftwareCallback.onDetected(newResult, null, null); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.wtf(TAG, "onRejected"); - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - // onRejected isn't allowed here, and we are not expecting it. - } - }; - - mRemoteHotwordDetectionService.run( - service -> service.detectFromMicrophoneSource( - null, - AUDIO_SOURCE_MICROPHONE, - null, - null, - internalCallback)); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, - mVoiceInteractionServiceUid); - } - - public void startListeningFromExternalSourceLocked( + void startListeningFromExternalSourceLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { if (DEBUG) { - Slog.d(TAG, "startListeningFromExternalSource"); + Slog.d(TAG, "startListeningFromExternalSourceLocked"); } handleExternalSourceHotwordDetectionLocked( @@ -443,169 +337,6 @@ class HotwordDetectorSession { callback); } - /** - * This method is only used by SoftwareHotwordDetector. - */ - @SuppressWarnings("GuardedBy") - void stopListeningLocked() { - if (DEBUG) { - Slog.d(TAG, "stopListening"); - } - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection is not running"); - return; - } - mPerformingSoftwareHotwordDetection = false; - - mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested"); - bestEffortClose(mCurrentAudioSink); - } - mCurrentAudioSink = null; - } - - @SuppressWarnings("GuardedBy") - void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, - IHotwordRecognitionStatusCallback externalCallback) { - if (DEBUG) { - Slog.d(TAG, "detectFromDspSource"); - } - - AtomicBoolean timeoutDetected = new AtomicBoolean(false); - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onDetected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - try { - enforcePermissionsForDataDelivery(); - enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); - } catch (SecurityException e) { - Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); - return; - } - externalCallback.onKeyphraseDetected(recognitionEvent, newResult); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onRejected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onRejected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - externalCallback.onRejected(result); - if (mDebugHotwordLogging && result != null) { - Slog.i(TAG, "Egressed rejected result: " + result); - } - } - } - }; - - mValidatingDspTrigger = true; - mRemoteHotwordDetectionService.run(service -> { - // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke - // the callback before timeout value. In order to reduce the latency impact between - // server side and client side, we need to use another timeout value - // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. - mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( - () -> { - // TODO: avoid allocate every time - timeoutDetected.set(true); - Slog.w(TAG, "Timed out on #detectFromDspSource"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); - try { - externalCallback.onError(CALLBACK_DETECT_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: ", e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - }, - MAX_VALIDATION_TIMEOUT_MILLIS, - TimeUnit.MILLISECONDS); - service.detectFromDspSource( - recognitionEvent, - recognitionEvent.getCaptureFormat(), - VALIDATION_TIMEOUT_MILLIS, - internalCallback); - }); - } - - @SuppressWarnings("GuardedBy") - public void dumpLocked(String prefix, PrintWriter pw) { - pw.print(prefix); pw.print("mCallback="); pw.println(mCallback); - pw.print(prefix); pw.print("mUpdateStateAfterStartFinished="); - pw.println(mUpdateStateAfterStartFinished); - pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized); - pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed); - pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); - pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection="); - pw.println(mPerformingSoftwareHotwordDetection); - pw.print(prefix); pw.print("DetectorType="); - pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); - } - @SuppressWarnings("GuardedBy") private void handleExternalSourceHotwordDetectionLocked( ParcelFileDescriptor audioStream, @@ -615,6 +346,11 @@ class HotwordDetectorSession { if (DEBUG) { Slog.d(TAG, "#handleExternalSourceHotwordDetectionLocked"); } + if (mPerformingExternalSourceHotwordDetection) { + Slog.i(TAG, "Hotword validation is already in progress for external source."); + return; + } + InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe(); @@ -626,6 +362,7 @@ class HotwordDetectorSession { ParcelFileDescriptor serviceAudioSource = clientPipe.first; mCurrentAudioSink = serviceAudioSink; + mPerformingExternalSourceHotwordDetection = true; mAudioCopyExecutor.execute(() -> { try (InputStream source = audioSource; @@ -659,7 +396,8 @@ class HotwordDetectorSession { } } finally { synchronized (mLock) { - mCurrentAudioSink = null; + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("start external source"); } } }); @@ -678,23 +416,26 @@ class HotwordDetectorSession { @Override public void onRejected(HotwordRejectedResult result) throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_EXTERNAL_SOURCE_REJECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - callback.onRejected(result); - - if (result != null) { - Slog.i(TAG, "Egressed 'hotword rejected result' " - + "from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + result); + synchronized (mLock) { + mPerformingExternalSourceHotwordDetection = false; + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_EXTERNAL_SOURCE_REJECTED, + mVoiceInteractionServiceUid); + mScheduledExecutorService.schedule( + () -> { + bestEffortClose(serviceAudioSink, audioSource); + }, + EXTERNAL_HOTWORD_CLEANUP_MILLIS, + TimeUnit.MILLISECONDS); + + callback.onRejected(result); + + if (result != null) { + Slog.i(TAG, "Egressed 'hotword rejected result' " + + "from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + result); + } } } } @@ -702,43 +443,46 @@ class HotwordDetectorSession { @Override public void onDetected(HotwordDetectedResult triggerResult) throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_EXTERNAL_SOURCE_DETECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { + synchronized (mLock) { + mPerformingExternalSourceHotwordDetection = false; HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, + METRICS_EXTERNAL_SOURCE_DETECTED, mVoiceInteractionServiceUid); - callback.onError(); - return; - } - HotwordDetectedResult newResult; - try { - newResult = - mHotwordAudioStreamCopier.startCopyingAudioStreams( - triggerResult); - } catch (IOException e) { - // TODO: Write event - callback.onError(); - return; - } - callback.onDetected(newResult, null /* audioFormat */, - null /* audioStream */); - Slog.i(TAG, "Egressed " - + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, - "Egressed detected result: " + newResult); + mScheduledExecutorService.schedule( + () -> { + bestEffortClose(serviceAudioSink, audioSource); + }, + EXTERNAL_HOTWORD_CLEANUP_MILLIS, + TimeUnit.MILLISECONDS); + + try { + enforcePermissionsForDataDelivery(); + } catch (SecurityException e) { + HotwordMetricsLogger.writeDetectorEvent( + getDetectorType(), + EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, + mVoiceInteractionServiceUid); + callback.onError(); + return; + } + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier + .startCopyingAudioStreams(triggerResult); + } catch (IOException e) { + // TODO: Write event + callback.onError(); + return; + } + callback.onDetected(newResult, null /* audioFormat */, + null /* audioStream */); + Slog.i(TAG, "Egressed " + + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, + "Egressed detected result: " + newResult); + } } } }); @@ -772,60 +516,16 @@ class HotwordDetectorSession { } void setDebugHotwordLoggingLocked(boolean logging) { - Slog.v(TAG, "setDebugHotwordLogging: " + logging); + Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); mDebugHotwordLogging = logging; } + @SuppressWarnings("GuardedBy") void updateRemoteHotwordDetectionServiceLocked( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService) { mRemoteHotwordDetectionService = remoteHotwordDetectionService; } - @SuppressWarnings("GuardedBy") - void informRestartProcessLocked() { - // TODO(b/244598068): Check HotwordAudioStreamManager first - Slog.v(TAG, "informRestartProcess"); - if (mValidatingDspTrigger) { - // We're restarting the process while it's processing a DSP trigger, so report a - // rejection. This also allows the Interactor to startRecognition again - try { - mCallback.onRejected(new HotwordRejectedResult.Builder().build()); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to call #rejected"); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - mValidatingDspTrigger = false; - } - - mUpdateStateAfterStartFinished.set(false); - - try { - mCallback.onProcessRestarted(); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - - // Restart listening from microphone if the hotword process has been restarted. - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Process restarted: calling startRecognition() again"); - startListeningFromMicLocked(); - } - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted"); - bestEffortClose(mCurrentAudioSink); - mCurrentAudioSink = null; - } - } - void reportErrorLocked(int status) { try { mCallback.onError(status); @@ -837,6 +537,11 @@ class HotwordDetectorSession { } } + /** + * Called when the trusted process is restarted. + */ + abstract void informRestartProcessLocked(); + private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() { ParcelFileDescriptor[] fileDescriptors; try { @@ -849,7 +554,7 @@ class HotwordDetectorSession { return Pair.create(fileDescriptors[0], fileDescriptors[1]); } - private void saveProximityValueToBundle(HotwordDetectedResult result) { + void saveProximityValueToBundle(HotwordDetectedResult result) { synchronized (mLock) { if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { result.setProximity(mProximityMeters); @@ -863,6 +568,15 @@ class HotwordDetectorSession { } } + @SuppressWarnings("GuardedBy") + void closeExternalAudioStreamLocked(String reason) { + if (mCurrentAudioSink != null) { + Slog.i(TAG, "Closing external audio stream to hotword detector: " + reason); + bestEffortClose(mCurrentAudioSink); + mCurrentAudioSink = null; + } + } + private static void bestEffortClose(Closeable... closeables) { for (Closeable closeable : closeables) { bestEffortClose(closeable); @@ -880,7 +594,7 @@ class HotwordDetectorSession { } // TODO: Share this code with SoundTriggerMiddlewarePermission. - private void enforcePermissionsForDataDelivery() { + void enforcePermissionsForDataDelivery() { Binder.withCleanCallingIdentity(() -> { synchronized (mLock) { enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); @@ -914,7 +628,7 @@ class HotwordDetectorSession { } } - private static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, + static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { // verify the phrase ID in HotwordDetectedResult is not exposing extra phrases // the DSP did not detect @@ -936,4 +650,17 @@ class HotwordDetectorSession { Slog.v(TAG, "Unexpected detector type"); return -1; } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + pw.print(prefix); pw.print("mCallback="); pw.println(mCallback); + pw.print(prefix); pw.print("mUpdateStateAfterStartFinished="); + pw.println(mUpdateStateAfterStartFinished); + pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized); + pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed); + pw.print(prefix); pw.print("DetectorType="); + pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); + pw.print(prefix); pw.print("mPerformingExternalSourceHotwordDetection="); + pw.println(mPerformingExternalSourceHotwordDetection); + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java index b89930074707..693068958568 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java @@ -16,20 +16,50 @@ package com.android.server.voiceinteraction; +import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE; + +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; + import android.annotation.NonNull; import android.content.Context; +import android.media.AudioFormat; import android.media.permission.Identity; +import android.os.PersistableBundle; +import android.os.RemoteException; +import android.os.SharedMemory; +import android.service.voice.HotwordDetectedResult; import android.service.voice.HotwordDetectionService; +import android.service.voice.HotwordDetector; +import android.service.voice.HotwordRejectedResult; +import android.service.voice.IDspHotwordDetectionCallback; +import android.service.voice.IHotwordDetectionService; +import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; +import android.util.Slog; +import com.android.internal.annotations.GuardedBy; import com.android.internal.app.IHotwordRecognitionStatusCallback; +import java.io.IOException; +import java.io.PrintWriter; import java.util.concurrent.ScheduledExecutorService; /** * A class that provides software trusted hotword detector to communicate with the {@link * HotwordDetectionService}. + * + * This class can handle the hotword detection which detector is created by using + * {@link android.service.voice.VoiceInteractionService#createHotwordDetector(PersistableBundle, + * SharedMemory, HotwordDetector.Callback)}. */ final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession { + private static final String TAG = "SoftwareTrustedHotwordDetectorSession"; + + private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; + @GuardedBy("mLock") + private boolean mPerformingSoftwareHotwordDetection; SoftwareTrustedHotwordDetectorSession( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @@ -40,4 +70,145 @@ final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, logging); } + + @SuppressWarnings("GuardedBy") + void startListeningFromMicLocked( + AudioFormat audioFormat, + IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { + if (DEBUG) { + Slog.d(TAG, "startListeningFromMicLocked"); + } + mSoftwareCallback = callback; + + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword validation is already in progress, ignoring."); + return; + } + mPerformingSoftwareHotwordDetection = true; + + startListeningFromMicLocked(); + } + + @SuppressWarnings("GuardedBy") + private void startListeningFromMicLocked() { + // TODO: consider making this a non-anonymous class. + IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { + @Override + public void onDetected(HotwordDetectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onDetected"); + } + synchronized (mLock) { + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); + if (!mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword detection has already completed"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); + return; + } + mPerformingSoftwareHotwordDetection = false; + try { + enforcePermissionsForDataDelivery(); + } catch (SecurityException e) { + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); + mSoftwareCallback.onError(); + return; + } + saveProximityValueToBundle(result); + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); + } catch (IOException e) { + // TODO: Write event + mSoftwareCallback.onError(); + return; + } + mSoftwareCallback.onDetected(newResult, null, null); + Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + newResult); + } + } + } + + @Override + public void onRejected(HotwordRejectedResult result) throws RemoteException { + if (DEBUG) { + Slog.wtf(TAG, "onRejected"); + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); + // onRejected isn't allowed here, and we are not expecting it. + } + }; + + mRemoteHotwordDetectionService.run( + service -> service.detectFromMicrophoneSource( + null, + AUDIO_SOURCE_MICROPHONE, + null, + null, + internalCallback)); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, + mVoiceInteractionServiceUid); + } + + @SuppressWarnings("GuardedBy") + void stopListeningLocked() { + if (DEBUG) { + Slog.d(TAG, "stopListeningLocked"); + } + if (!mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword detection is not running"); + return; + } + mPerformingSoftwareHotwordDetection = false; + + mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); + + closeExternalAudioStreamLocked("stopping requested"); + } + + @Override + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + // TODO(b/244598068): Check HotwordAudioStreamManager first + Slog.v(TAG, "informRestartProcessLocked"); + mUpdateStateAfterStartFinished.set(false); + + try { + mCallback.onProcessRestarted(); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + + // Restart listening from microphone if the hotword process has been restarted. + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Process restarted: calling startRecognition() again"); + startListeningFromMicLocked(); + } + + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("process restarted"); + } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + super.dumpLocked(prefix, pw); + pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection="); + pw.println(mPerformingSoftwareHotwordDetection); + } } |