summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
author lpeter <lpeter@google.com> 2022-11-18 09:30:08 +0000
committer lpeter <lpeter@google.com> 2022-12-08 20:45:06 +0000
commit96e34f053b2fa321f8aa687600e4d3f096676293 (patch)
tree3278c5852433b48c3ab7f7d1e17d1905b9034145
parent1566016119cc3fa5365e380ca3cbc9d127786d86 (diff)
Move the methods into Dsp session and Software session
Move the methods of HotwordDetectorSession into DspTrustedHotwordDetectorSession and SoftwareTrustedHotwordDetectorSession Test: atest CtsVoiceInteractionTestCases Bug: 241041976 Change-Id: I53139507697ba86f04158896ef95095993f23f2c
-rw-r--r--services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java211
-rw-r--r--services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java8
-rw-r--r--services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java519
-rw-r--r--services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java171
4 files changed, 510 insertions, 399 deletions
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java
index c092d9badcb2..2812264e989c 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java
@@ -16,20 +16,61 @@
package com.android.server.voiceinteraction;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART;
+
import android.annotation.NonNull;
import android.content.Context;
+import android.hardware.soundtrigger.SoundTrigger;
import android.media.permission.Identity;
+import android.os.PersistableBundle;
+import android.os.RemoteException;
+import android.os.SharedMemory;
+import android.service.voice.AlwaysOnHotwordDetector;
+import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
+import android.service.voice.HotwordDetector;
+import android.service.voice.HotwordRejectedResult;
+import android.service.voice.IDspHotwordDetectionCallback;
+import android.util.Slog;
+import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Locale;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* A class that provides Dsp trusted hotword detector to communicate with the {@link
* HotwordDetectionService}.
+ *
+ * This class can handle the hotword detection which detector is created by using
+ * {@link android.service.voice.VoiceInteractionService#createAlwaysOnHotwordDetector(String,
+ * Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)}.
*/
final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession {
+ private static final String TAG = "DspTrustedHotwordDetectorSession";
+
+ // The validation timeout value is 3 seconds for onDetect of DSP trigger event.
+ private static final long VALIDATION_TIMEOUT_MILLIS = 3000;
+ // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS.
+ private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000;
+
+ @GuardedBy("mLock")
+ private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture;
+
+ @GuardedBy("mLock")
+ private boolean mValidatingDspTrigger = false;
DspTrustedHotwordDetectorSession(
@NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,
@@ -40,4 +81,174 @@ final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession {
super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid,
voiceInteractorIdentity, scheduledExecutorService, logging);
}
+
+ @SuppressWarnings("GuardedBy")
+ void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
+ IHotwordRecognitionStatusCallback externalCallback) {
+ if (DEBUG) {
+ Slog.d(TAG, "detectFromDspSourceLocked");
+ }
+
+ AtomicBoolean timeoutDetected = new AtomicBoolean(false);
+ // TODO: consider making this a non-anonymous class.
+ IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
+ @Override
+ public void onDetected(HotwordDetectedResult result) throws RemoteException {
+ if (DEBUG) {
+ Slog.d(TAG, "onDetected");
+ }
+ synchronized (mLock) {
+ if (mCancellationKeyPhraseDetectionFuture != null) {
+ mCancellationKeyPhraseDetectionFuture.cancel(true);
+ }
+ if (timeoutDetected.get()) {
+ return;
+ }
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);
+ if (!mValidatingDspTrigger) {
+ Slog.i(TAG, "Ignoring #onDetected due to a process restart");
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);
+ return;
+ }
+ mValidatingDspTrigger = false;
+ try {
+ enforcePermissionsForDataDelivery();
+ enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent);
+ } catch (SecurityException e) {
+ Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e);
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
+ externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION);
+ return;
+ }
+ saveProximityValueToBundle(result);
+ HotwordDetectedResult newResult;
+ try {
+ newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
+ } catch (IOException e) {
+ externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
+ return;
+ }
+ externalCallback.onKeyphraseDetected(recognitionEvent, newResult);
+ Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
+ + " bits from hotword trusted process");
+ if (mDebugHotwordLogging) {
+ Slog.i(TAG, "Egressed detected result: " + newResult);
+ }
+ }
+ }
+
+ @Override
+ public void onRejected(HotwordRejectedResult result) throws RemoteException {
+ if (DEBUG) {
+ Slog.d(TAG, "onRejected");
+ }
+ synchronized (mLock) {
+ if (mCancellationKeyPhraseDetectionFuture != null) {
+ mCancellationKeyPhraseDetectionFuture.cancel(true);
+ }
+ if (timeoutDetected.get()) {
+ return;
+ }
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);
+ if (!mValidatingDspTrigger) {
+ Slog.i(TAG, "Ignoring #onRejected due to a process restart");
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK);
+ return;
+ }
+ mValidatingDspTrigger = false;
+ externalCallback.onRejected(result);
+ if (mDebugHotwordLogging && result != null) {
+ Slog.i(TAG, "Egressed rejected result: " + result);
+ }
+ }
+ }
+ };
+
+ mValidatingDspTrigger = true;
+ mRemoteHotwordDetectionService.run(service -> {
+ // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke
+ // the callback before timeout value. In order to reduce the latency impact between
+ // server side and client side, we need to use another timeout value
+ // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it.
+ mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule(
+ () -> {
+ // TODO: avoid allocate every time
+ timeoutDetected.set(true);
+ Slog.w(TAG, "Timed out on #detectFromDspSource");
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT);
+ try {
+ externalCallback.onError(CALLBACK_DETECT_TIMEOUT);
+ } catch (RemoteException e) {
+ Slog.w(TAG, "Failed to report onError status: ", e);
+ HotwordMetricsLogger.writeDetectorEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
+ mVoiceInteractionServiceUid);
+ }
+ },
+ MAX_VALIDATION_TIMEOUT_MILLIS,
+ TimeUnit.MILLISECONDS);
+ service.detectFromDspSource(
+ recognitionEvent,
+ recognitionEvent.getCaptureFormat(),
+ VALIDATION_TIMEOUT_MILLIS,
+ internalCallback);
+ });
+ }
+
+ @Override
+ @SuppressWarnings("GuardedBy")
+ void informRestartProcessLocked() {
+ // TODO(b/244598068): Check HotwordAudioStreamManager first
+ Slog.v(TAG, "informRestartProcessLocked");
+ if (mValidatingDspTrigger) {
+ // We're restarting the process while it's processing a DSP trigger, so report a
+ // rejection. This also allows the Interactor to startRecognition again
+ try {
+ mCallback.onRejected(new HotwordRejectedResult.Builder().build());
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART);
+ } catch (RemoteException e) {
+ Slog.w(TAG, "Failed to call #rejected");
+ HotwordMetricsLogger.writeDetectorEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION,
+ mVoiceInteractionServiceUid);
+ }
+ mValidatingDspTrigger = false;
+ }
+ mUpdateStateAfterStartFinished.set(false);
+
+ try {
+ mCallback.onProcessRestarted();
+ } catch (RemoteException e) {
+ Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);
+ HotwordMetricsLogger.writeDetectorEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
+ HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,
+ mVoiceInteractionServiceUid);
+ }
+
+ mPerformingExternalSourceHotwordDetection = false;
+ closeExternalAudioStreamLocked("process restarted");
+ }
+
+ @SuppressWarnings("GuardedBy")
+ public void dumpLocked(String prefix, PrintWriter pw) {
+ super.dumpLocked(prefix, pw);
+ pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger);
+ }
}
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
index 7e105c3b18c5..2a4d4a15afdf 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
@@ -244,7 +244,8 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "It is not a software detector");
return;
}
- mHotwordDetectorSession.startListeningFromMicLocked(audioFormat, callback);
+ ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession)
+ .startListeningFromMicLocked(audioFormat, callback);
}
}
@@ -274,7 +275,7 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "It is not a software detector");
return;
}
- mHotwordDetectorSession.stopListeningLocked();
+ ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession).stopListeningLocked();
}
}
@@ -297,7 +298,8 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "It is not a Dsp detector");
return;
}
- mHotwordDetectorSession.detectFromDspSourceLocked(recognitionEvent, externalCallback);
+ ((DspTrustedHotwordDetectorSession) mHotwordDetectorSession).detectFromDspSourceLocked(
+ recognitionEvent, externalCallback);
}
}
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java
index 9cd4f68222a9..f9f43c9ba49a 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java
@@ -20,7 +20,6 @@ import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD;
import static android.Manifest.permission.RECORD_AUDIO;
import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN;
import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL;
-import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE;
import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT;
import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS;
import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN;
@@ -33,8 +32,6 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERV
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION;
-import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;
-import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION;
@@ -43,13 +40,8 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENT
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION;
-import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION;
-import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION;
-import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK;
-import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;
-import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK;
import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight;
@@ -75,7 +67,6 @@ import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.IDspHotwordDetectionCallback;
-import android.service.voice.IHotwordDetectionService;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.text.TextUtils;
import android.util.Pair;
@@ -96,7 +87,6 @@ import java.time.Instant;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -104,8 +94,15 @@ import java.util.concurrent.atomic.AtomicBoolean;
/**
* A class that provides trusted hotword detector to communicate with the {@link
* HotwordDetectionService}.
+ *
+ * This class provides the methods to do initialization with the {@link HotwordDetectionService}
+ * and handle external source detection. It also provides the methods to check if we can egress
+ * the data from the {@link HotwordDetectionService}.
+ *
+ * The subclass should override the {@link #informRestartProcessLocked()} to handle the trusted
+ * process restart.
*/
-class HotwordDetectorSession {
+abstract class HotwordDetectorSession {
private static final String TAG = "HotwordDetectorSession";
static final boolean DEBUG = false;
@@ -119,10 +116,6 @@ class HotwordDetectorSession {
static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4;
// TODO: These constants need to be refined.
- // The validation timeout value is 3 seconds for onDetect of DSP trigger event.
- private static final long VALIDATION_TIMEOUT_MILLIS = 3000;
- // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS.
- private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000;
private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000;
private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000;
private static final Duration MAX_UPDATE_TIMEOUT_DURATION =
@@ -140,29 +133,29 @@ class HotwordDetectorSession {
private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS =
HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS;
- private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION =
+ static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION =
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION;
- private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK =
+ static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK =
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK;
- private static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK =
+ static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK =
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK;
private static final int METRICS_EXTERNAL_SOURCE_DETECTED =
HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED;
private static final int METRICS_EXTERNAL_SOURCE_REJECTED =
HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED;
- private static final int METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION =
+ private static final int EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION =
HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION;
private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION =
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION;
private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool();
// TODO: This may need to be a Handler(looper)
- private final ScheduledExecutorService mScheduledExecutorService;
+ final ScheduledExecutorService mScheduledExecutorService;
private final AppOpsManager mAppOpsManager;
- private final HotwordAudioStreamCopier mHotwordAudioStreamCopier;
- private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);
- private final IHotwordRecognitionStatusCallback mCallback;
+ final HotwordAudioStreamCopier mHotwordAudioStreamCopier;
+ final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);
+ final IHotwordRecognitionStatusCallback mCallback;
final Object mLock;
final int mVoiceInteractionServiceUid;
@@ -173,28 +166,22 @@ class HotwordDetectorSession {
final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal =
this::setProximityValue;
- private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback;
-
- private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture;
-
/** Identity used for attributing app ops when delivering data to the Interactor. */
- @GuardedBy("mLock")
@Nullable
private final Identity mVoiceInteractorIdentity;
@GuardedBy("mLock")
- private ParcelFileDescriptor mCurrentAudioSink;
- @GuardedBy("mLock")
- private boolean mValidatingDspTrigger = false;
+ ParcelFileDescriptor mCurrentAudioSink;
@GuardedBy("mLock")
- private boolean mPerformingSoftwareHotwordDetection;
- @NonNull private HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService;
- private boolean mDebugHotwordLogging = false;
+ @NonNull HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService;
+ boolean mDebugHotwordLogging = false;
@GuardedBy("mLock")
private double mProximityMeters = PROXIMITY_UNKNOWN;
@GuardedBy("mLock")
private boolean mInitialized = false;
@GuardedBy("mLock")
private boolean mDestroyed = false;
+ @GuardedBy("mLock")
+ boolean mPerformingExternalSourceHotwordDetection;
HotwordDetectorSession(
@NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,
@@ -223,6 +210,7 @@ class HotwordDetectorSession {
}
}
+ @SuppressWarnings("GuardedBy")
private void updateStateAfterProcessStartLocked(PersistableBundle options,
SharedMemory sharedMemory) {
if (DEBUG) {
@@ -315,6 +303,7 @@ class HotwordDetectorSession {
return new Pair<>(status, metricsResult);
}
+ @SuppressWarnings("GuardedBy")
void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory,
Instant lastRestartInstant) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
@@ -332,108 +321,13 @@ class HotwordDetectorSession {
}
}
- /**
- * This method is only used by SoftwareHotwordDetector.
- */
- @SuppressWarnings("GuardedBy")
- void startListeningFromMicLocked(
- AudioFormat audioFormat,
- IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
- if (DEBUG) {
- Slog.d(TAG, "startListeningFromMic");
- }
- mSoftwareCallback = callback;
-
- if (mPerformingSoftwareHotwordDetection) {
- Slog.i(TAG, "Hotword validation is already in progress, ignoring.");
- return;
- }
- mPerformingSoftwareHotwordDetection = true;
-
- startListeningFromMicLocked();
- }
-
- /**
- * This method is only used by SoftwareHotwordDetector.
- */
- private void startListeningFromMicLocked() {
- // TODO: consider making this a non-anonymous class.
- IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
- @Override
- public void onDetected(HotwordDetectedResult result) throws RemoteException {
- if (DEBUG) {
- Slog.d(TAG, "onDetected");
- }
- synchronized (mLock) {
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);
- if (!mPerformingSoftwareHotwordDetection) {
- Slog.i(TAG, "Hotword detection has already completed");
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);
- return;
- }
- mPerformingSoftwareHotwordDetection = false;
- try {
- enforcePermissionsForDataDelivery();
- } catch (SecurityException e) {
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
- mSoftwareCallback.onError();
- return;
- }
- saveProximityValueToBundle(result);
- HotwordDetectedResult newResult;
- try {
- newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
- } catch (IOException e) {
- // TODO: Write event
- mSoftwareCallback.onError();
- return;
- }
- mSoftwareCallback.onDetected(newResult, null, null);
- Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
- + " bits from hotword trusted process");
- if (mDebugHotwordLogging) {
- Slog.i(TAG, "Egressed detected result: " + newResult);
- }
- }
- }
-
- @Override
- public void onRejected(HotwordRejectedResult result) throws RemoteException {
- if (DEBUG) {
- Slog.wtf(TAG, "onRejected");
- }
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);
- // onRejected isn't allowed here, and we are not expecting it.
- }
- };
-
- mRemoteHotwordDetectionService.run(
- service -> service.detectFromMicrophoneSource(
- null,
- AUDIO_SOURCE_MICROPHONE,
- null,
- null,
- internalCallback));
- HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
- HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION,
- mVoiceInteractionServiceUid);
- }
-
- public void startListeningFromExternalSourceLocked(
+ void startListeningFromExternalSourceLocked(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,
@Nullable PersistableBundle options,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
- Slog.d(TAG, "startListeningFromExternalSource");
+ Slog.d(TAG, "startListeningFromExternalSourceLocked");
}
handleExternalSourceHotwordDetectionLocked(
@@ -443,169 +337,6 @@ class HotwordDetectorSession {
callback);
}
- /**
- * This method is only used by SoftwareHotwordDetector.
- */
- @SuppressWarnings("GuardedBy")
- void stopListeningLocked() {
- if (DEBUG) {
- Slog.d(TAG, "stopListening");
- }
- if (!mPerformingSoftwareHotwordDetection) {
- Slog.i(TAG, "Hotword detection is not running");
- return;
- }
- mPerformingSoftwareHotwordDetection = false;
-
- mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection);
-
- if (mCurrentAudioSink != null) {
- Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested");
- bestEffortClose(mCurrentAudioSink);
- }
- mCurrentAudioSink = null;
- }
-
- @SuppressWarnings("GuardedBy")
- void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
- IHotwordRecognitionStatusCallback externalCallback) {
- if (DEBUG) {
- Slog.d(TAG, "detectFromDspSource");
- }
-
- AtomicBoolean timeoutDetected = new AtomicBoolean(false);
- // TODO: consider making this a non-anonymous class.
- IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
- @Override
- public void onDetected(HotwordDetectedResult result) throws RemoteException {
- if (DEBUG) {
- Slog.d(TAG, "onDetected");
- }
- synchronized (mLock) {
- if (mCancellationKeyPhraseDetectionFuture != null) {
- mCancellationKeyPhraseDetectionFuture.cancel(true);
- }
- if (timeoutDetected.get()) {
- return;
- }
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);
- if (!mValidatingDspTrigger) {
- Slog.i(TAG, "Ignoring #onDetected due to a process restart");
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);
- return;
- }
- mValidatingDspTrigger = false;
- try {
- enforcePermissionsForDataDelivery();
- enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent);
- } catch (SecurityException e) {
- Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e);
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
- externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION);
- return;
- }
- saveProximityValueToBundle(result);
- HotwordDetectedResult newResult;
- try {
- newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
- } catch (IOException e) {
- // TODO: Write event
- externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
- return;
- }
- externalCallback.onKeyphraseDetected(recognitionEvent, newResult);
- Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
- + " bits from hotword trusted process");
- if (mDebugHotwordLogging) {
- Slog.i(TAG, "Egressed detected result: " + newResult);
- }
- }
- }
-
- @Override
- public void onRejected(HotwordRejectedResult result) throws RemoteException {
- if (DEBUG) {
- Slog.d(TAG, "onRejected");
- }
- synchronized (mLock) {
- if (mCancellationKeyPhraseDetectionFuture != null) {
- mCancellationKeyPhraseDetectionFuture.cancel(true);
- }
- if (timeoutDetected.get()) {
- return;
- }
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);
- if (!mValidatingDspTrigger) {
- Slog.i(TAG, "Ignoring #onRejected due to a process restart");
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK);
- return;
- }
- mValidatingDspTrigger = false;
- externalCallback.onRejected(result);
- if (mDebugHotwordLogging && result != null) {
- Slog.i(TAG, "Egressed rejected result: " + result);
- }
- }
- }
- };
-
- mValidatingDspTrigger = true;
- mRemoteHotwordDetectionService.run(service -> {
- // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke
- // the callback before timeout value. In order to reduce the latency impact between
- // server side and client side, we need to use another timeout value
- // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it.
- mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule(
- () -> {
- // TODO: avoid allocate every time
- timeoutDetected.set(true);
- Slog.w(TAG, "Timed out on #detectFromDspSource");
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT);
- try {
- externalCallback.onError(CALLBACK_DETECT_TIMEOUT);
- } catch (RemoteException e) {
- Slog.w(TAG, "Failed to report onError status: ", e);
- HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
- HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
- mVoiceInteractionServiceUid);
- }
- },
- MAX_VALIDATION_TIMEOUT_MILLIS,
- TimeUnit.MILLISECONDS);
- service.detectFromDspSource(
- recognitionEvent,
- recognitionEvent.getCaptureFormat(),
- VALIDATION_TIMEOUT_MILLIS,
- internalCallback);
- });
- }
-
- @SuppressWarnings("GuardedBy")
- public void dumpLocked(String prefix, PrintWriter pw) {
- pw.print(prefix); pw.print("mCallback="); pw.println(mCallback);
- pw.print(prefix); pw.print("mUpdateStateAfterStartFinished=");
- pw.println(mUpdateStateAfterStartFinished);
- pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized);
- pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed);
- pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger);
- pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection=");
- pw.println(mPerformingSoftwareHotwordDetection);
- pw.print(prefix); pw.print("DetectorType=");
- pw.println(HotwordDetector.detectorTypeToString(getDetectorType()));
- }
-
@SuppressWarnings("GuardedBy")
private void handleExternalSourceHotwordDetectionLocked(
ParcelFileDescriptor audioStream,
@@ -615,6 +346,11 @@ class HotwordDetectorSession {
if (DEBUG) {
Slog.d(TAG, "#handleExternalSourceHotwordDetectionLocked");
}
+ if (mPerformingExternalSourceHotwordDetection) {
+ Slog.i(TAG, "Hotword validation is already in progress for external source.");
+ return;
+ }
+
InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream);
Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe();
@@ -626,6 +362,7 @@ class HotwordDetectorSession {
ParcelFileDescriptor serviceAudioSource = clientPipe.first;
mCurrentAudioSink = serviceAudioSink;
+ mPerformingExternalSourceHotwordDetection = true;
mAudioCopyExecutor.execute(() -> {
try (InputStream source = audioSource;
@@ -659,7 +396,8 @@ class HotwordDetectorSession {
}
} finally {
synchronized (mLock) {
- mCurrentAudioSink = null;
+ mPerformingExternalSourceHotwordDetection = false;
+ closeExternalAudioStreamLocked("start external source");
}
}
});
@@ -678,23 +416,26 @@ class HotwordDetectorSession {
@Override
public void onRejected(HotwordRejectedResult result)
throws RemoteException {
- HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
- METRICS_EXTERNAL_SOURCE_REJECTED,
- mVoiceInteractionServiceUid);
- mScheduledExecutorService.schedule(
- () -> {
- bestEffortClose(serviceAudioSink, audioSource);
- },
- EXTERNAL_HOTWORD_CLEANUP_MILLIS,
- TimeUnit.MILLISECONDS);
-
- callback.onRejected(result);
-
- if (result != null) {
- Slog.i(TAG, "Egressed 'hotword rejected result' "
- + "from hotword trusted process");
- if (mDebugHotwordLogging) {
- Slog.i(TAG, "Egressed detected result: " + result);
+ synchronized (mLock) {
+ mPerformingExternalSourceHotwordDetection = false;
+ HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
+ METRICS_EXTERNAL_SOURCE_REJECTED,
+ mVoiceInteractionServiceUid);
+ mScheduledExecutorService.schedule(
+ () -> {
+ bestEffortClose(serviceAudioSink, audioSource);
+ },
+ EXTERNAL_HOTWORD_CLEANUP_MILLIS,
+ TimeUnit.MILLISECONDS);
+
+ callback.onRejected(result);
+
+ if (result != null) {
+ Slog.i(TAG, "Egressed 'hotword rejected result' "
+ + "from hotword trusted process");
+ if (mDebugHotwordLogging) {
+ Slog.i(TAG, "Egressed detected result: " + result);
+ }
}
}
}
@@ -702,43 +443,46 @@ class HotwordDetectorSession {
@Override
public void onDetected(HotwordDetectedResult triggerResult)
throws RemoteException {
- HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
- METRICS_EXTERNAL_SOURCE_DETECTED,
- mVoiceInteractionServiceUid);
- mScheduledExecutorService.schedule(
- () -> {
- bestEffortClose(serviceAudioSink, audioSource);
- },
- EXTERNAL_HOTWORD_CLEANUP_MILLIS,
- TimeUnit.MILLISECONDS);
-
- try {
- enforcePermissionsForDataDelivery();
- } catch (SecurityException e) {
+ synchronized (mLock) {
+ mPerformingExternalSourceHotwordDetection = false;
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
- METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION,
+ METRICS_EXTERNAL_SOURCE_DETECTED,
mVoiceInteractionServiceUid);
- callback.onError();
- return;
- }
- HotwordDetectedResult newResult;
- try {
- newResult =
- mHotwordAudioStreamCopier.startCopyingAudioStreams(
- triggerResult);
- } catch (IOException e) {
- // TODO: Write event
- callback.onError();
- return;
- }
- callback.onDetected(newResult, null /* audioFormat */,
- null /* audioStream */);
- Slog.i(TAG, "Egressed "
- + HotwordDetectedResult.getUsageSize(newResult)
- + " bits from hotword trusted process");
- if (mDebugHotwordLogging) {
- Slog.i(TAG,
- "Egressed detected result: " + newResult);
+ mScheduledExecutorService.schedule(
+ () -> {
+ bestEffortClose(serviceAudioSink, audioSource);
+ },
+ EXTERNAL_HOTWORD_CLEANUP_MILLIS,
+ TimeUnit.MILLISECONDS);
+
+ try {
+ enforcePermissionsForDataDelivery();
+ } catch (SecurityException e) {
+ HotwordMetricsLogger.writeDetectorEvent(
+ getDetectorType(),
+ EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION,
+ mVoiceInteractionServiceUid);
+ callback.onError();
+ return;
+ }
+ HotwordDetectedResult newResult;
+ try {
+ newResult = mHotwordAudioStreamCopier
+ .startCopyingAudioStreams(triggerResult);
+ } catch (IOException e) {
+ // TODO: Write event
+ callback.onError();
+ return;
+ }
+ callback.onDetected(newResult, null /* audioFormat */,
+ null /* audioStream */);
+ Slog.i(TAG, "Egressed "
+ + HotwordDetectedResult.getUsageSize(newResult)
+ + " bits from hotword trusted process");
+ if (mDebugHotwordLogging) {
+ Slog.i(TAG,
+ "Egressed detected result: " + newResult);
+ }
}
}
});
@@ -772,60 +516,16 @@ class HotwordDetectorSession {
}
void setDebugHotwordLoggingLocked(boolean logging) {
- Slog.v(TAG, "setDebugHotwordLogging: " + logging);
+ Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging);
mDebugHotwordLogging = logging;
}
+ @SuppressWarnings("GuardedBy")
void updateRemoteHotwordDetectionServiceLocked(
@NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService) {
mRemoteHotwordDetectionService = remoteHotwordDetectionService;
}
- @SuppressWarnings("GuardedBy")
- void informRestartProcessLocked() {
- // TODO(b/244598068): Check HotwordAudioStreamManager first
- Slog.v(TAG, "informRestartProcess");
- if (mValidatingDspTrigger) {
- // We're restarting the process while it's processing a DSP trigger, so report a
- // rejection. This also allows the Interactor to startRecognition again
- try {
- mCallback.onRejected(new HotwordRejectedResult.Builder().build());
- HotwordMetricsLogger.writeKeyphraseTriggerEvent(
- getDetectorType(),
- HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART);
- } catch (RemoteException e) {
- Slog.w(TAG, "Failed to call #rejected");
- HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
- HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION,
- mVoiceInteractionServiceUid);
- }
- mValidatingDspTrigger = false;
- }
-
- mUpdateStateAfterStartFinished.set(false);
-
- try {
- mCallback.onProcessRestarted();
- } catch (RemoteException e) {
- Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);
- HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
- HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,
- mVoiceInteractionServiceUid);
- }
-
- // Restart listening from microphone if the hotword process has been restarted.
- if (mPerformingSoftwareHotwordDetection) {
- Slog.i(TAG, "Process restarted: calling startRecognition() again");
- startListeningFromMicLocked();
- }
-
- if (mCurrentAudioSink != null) {
- Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted");
- bestEffortClose(mCurrentAudioSink);
- mCurrentAudioSink = null;
- }
- }
-
void reportErrorLocked(int status) {
try {
mCallback.onError(status);
@@ -837,6 +537,11 @@ class HotwordDetectorSession {
}
}
+ /**
+ * Called when the trusted process is restarted.
+ */
+ abstract void informRestartProcessLocked();
+
private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() {
ParcelFileDescriptor[] fileDescriptors;
try {
@@ -849,7 +554,7 @@ class HotwordDetectorSession {
return Pair.create(fileDescriptors[0], fileDescriptors[1]);
}
- private void saveProximityValueToBundle(HotwordDetectedResult result) {
+ void saveProximityValueToBundle(HotwordDetectedResult result) {
synchronized (mLock) {
if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) {
result.setProximity(mProximityMeters);
@@ -863,6 +568,15 @@ class HotwordDetectorSession {
}
}
+ @SuppressWarnings("GuardedBy")
+ void closeExternalAudioStreamLocked(String reason) {
+ if (mCurrentAudioSink != null) {
+ Slog.i(TAG, "Closing external audio stream to hotword detector: " + reason);
+ bestEffortClose(mCurrentAudioSink);
+ mCurrentAudioSink = null;
+ }
+ }
+
private static void bestEffortClose(Closeable... closeables) {
for (Closeable closeable : closeables) {
bestEffortClose(closeable);
@@ -880,7 +594,7 @@ class HotwordDetectorSession {
}
// TODO: Share this code with SoundTriggerMiddlewarePermission.
- private void enforcePermissionsForDataDelivery() {
+ void enforcePermissionsForDataDelivery() {
Binder.withCleanCallingIdentity(() -> {
synchronized (mLock) {
enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO);
@@ -914,7 +628,7 @@ class HotwordDetectorSession {
}
}
- private static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result,
+ static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result,
SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) {
// verify the phrase ID in HotwordDetectedResult is not exposing extra phrases
// the DSP did not detect
@@ -936,4 +650,17 @@ class HotwordDetectorSession {
Slog.v(TAG, "Unexpected detector type");
return -1;
}
+
+ @SuppressWarnings("GuardedBy")
+ public void dumpLocked(String prefix, PrintWriter pw) {
+ pw.print(prefix); pw.print("mCallback="); pw.println(mCallback);
+ pw.print(prefix); pw.print("mUpdateStateAfterStartFinished=");
+ pw.println(mUpdateStateAfterStartFinished);
+ pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized);
+ pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed);
+ pw.print(prefix); pw.print("DetectorType=");
+ pw.println(HotwordDetector.detectorTypeToString(getDetectorType()));
+ pw.print(prefix); pw.print("mPerformingExternalSourceHotwordDetection=");
+ pw.println(mPerformingExternalSourceHotwordDetection);
+ }
}
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java
index b89930074707..693068958568 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java
@@ -16,20 +16,50 @@
package com.android.server.voiceinteraction;
+import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE;
+
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;
+import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;
+
import android.annotation.NonNull;
import android.content.Context;
+import android.media.AudioFormat;
import android.media.permission.Identity;
+import android.os.PersistableBundle;
+import android.os.RemoteException;
+import android.os.SharedMemory;
+import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
+import android.service.voice.HotwordDetector;
+import android.service.voice.HotwordRejectedResult;
+import android.service.voice.IDspHotwordDetectionCallback;
+import android.service.voice.IHotwordDetectionService;
+import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
+import android.util.Slog;
+import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
+import java.io.IOException;
+import java.io.PrintWriter;
import java.util.concurrent.ScheduledExecutorService;
/**
* A class that provides software trusted hotword detector to communicate with the {@link
* HotwordDetectionService}.
+ *
+ * This class can handle the hotword detection which detector is created by using
+ * {@link android.service.voice.VoiceInteractionService#createHotwordDetector(PersistableBundle,
+ * SharedMemory, HotwordDetector.Callback)}.
*/
final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession {
+ private static final String TAG = "SoftwareTrustedHotwordDetectorSession";
+
+ private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback;
+ @GuardedBy("mLock")
+ private boolean mPerformingSoftwareHotwordDetection;
SoftwareTrustedHotwordDetectorSession(
@NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,
@@ -40,4 +70,145 @@ final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession
super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid,
voiceInteractorIdentity, scheduledExecutorService, logging);
}
+
+ @SuppressWarnings("GuardedBy")
+ void startListeningFromMicLocked(
+ AudioFormat audioFormat,
+ IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
+ if (DEBUG) {
+ Slog.d(TAG, "startListeningFromMicLocked");
+ }
+ mSoftwareCallback = callback;
+
+ if (mPerformingSoftwareHotwordDetection) {
+ Slog.i(TAG, "Hotword validation is already in progress, ignoring.");
+ return;
+ }
+ mPerformingSoftwareHotwordDetection = true;
+
+ startListeningFromMicLocked();
+ }
+
+ @SuppressWarnings("GuardedBy")
+ private void startListeningFromMicLocked() {
+ // TODO: consider making this a non-anonymous class.
+ IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
+ @Override
+ public void onDetected(HotwordDetectedResult result) throws RemoteException {
+ if (DEBUG) {
+ Slog.d(TAG, "onDetected");
+ }
+ synchronized (mLock) {
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
+ HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);
+ if (!mPerformingSoftwareHotwordDetection) {
+ Slog.i(TAG, "Hotword detection has already completed");
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
+ METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);
+ return;
+ }
+ mPerformingSoftwareHotwordDetection = false;
+ try {
+ enforcePermissionsForDataDelivery();
+ } catch (SecurityException e) {
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
+ METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
+ mSoftwareCallback.onError();
+ return;
+ }
+ saveProximityValueToBundle(result);
+ HotwordDetectedResult newResult;
+ try {
+ newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
+ } catch (IOException e) {
+ // TODO: Write event
+ mSoftwareCallback.onError();
+ return;
+ }
+ mSoftwareCallback.onDetected(newResult, null, null);
+ Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
+ + " bits from hotword trusted process");
+ if (mDebugHotwordLogging) {
+ Slog.i(TAG, "Egressed detected result: " + newResult);
+ }
+ }
+ }
+
+ @Override
+ public void onRejected(HotwordRejectedResult result) throws RemoteException {
+ if (DEBUG) {
+ Slog.wtf(TAG, "onRejected");
+ }
+ HotwordMetricsLogger.writeKeyphraseTriggerEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
+ HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);
+ // onRejected isn't allowed here, and we are not expecting it.
+ }
+ };
+
+ mRemoteHotwordDetectionService.run(
+ service -> service.detectFromMicrophoneSource(
+ null,
+ AUDIO_SOURCE_MICROPHONE,
+ null,
+ null,
+ internalCallback));
+ HotwordMetricsLogger.writeDetectorEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
+ HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION,
+ mVoiceInteractionServiceUid);
+ }
+
+ @SuppressWarnings("GuardedBy")
+ void stopListeningLocked() {
+ if (DEBUG) {
+ Slog.d(TAG, "stopListeningLocked");
+ }
+ if (!mPerformingSoftwareHotwordDetection) {
+ Slog.i(TAG, "Hotword detection is not running");
+ return;
+ }
+ mPerformingSoftwareHotwordDetection = false;
+
+ mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection);
+
+ closeExternalAudioStreamLocked("stopping requested");
+ }
+
+ @Override
+ @SuppressWarnings("GuardedBy")
+ void informRestartProcessLocked() {
+ // TODO(b/244598068): Check HotwordAudioStreamManager first
+ Slog.v(TAG, "informRestartProcessLocked");
+ mUpdateStateAfterStartFinished.set(false);
+
+ try {
+ mCallback.onProcessRestarted();
+ } catch (RemoteException e) {
+ Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);
+ HotwordMetricsLogger.writeDetectorEvent(
+ HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
+ HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,
+ mVoiceInteractionServiceUid);
+ }
+
+ // Restart listening from microphone if the hotword process has been restarted.
+ if (mPerformingSoftwareHotwordDetection) {
+ Slog.i(TAG, "Process restarted: calling startRecognition() again");
+ startListeningFromMicLocked();
+ }
+
+ mPerformingExternalSourceHotwordDetection = false;
+ closeExternalAudioStreamLocked("process restarted");
+ }
+
+ @SuppressWarnings("GuardedBy")
+ public void dumpLocked(String prefix, PrintWriter pw) {
+ super.dumpLocked(prefix, pw);
+ pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection=");
+ pw.println(mPerformingSoftwareHotwordDetection);
+ }
}