diff options
5 files changed, 1207 insertions, 2094 deletions
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java new file mode 100644 index 000000000000..2812264e989c --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java @@ -0,0 +1,254 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; + +import android.annotation.NonNull; +import android.content.Context; +import android.hardware.soundtrigger.SoundTrigger; +import android.media.permission.Identity; +import android.os.PersistableBundle; +import android.os.RemoteException; +import android.os.SharedMemory; +import android.service.voice.AlwaysOnHotwordDetector; +import android.service.voice.HotwordDetectedResult; +import android.service.voice.HotwordDetectionService; +import android.service.voice.HotwordDetector; +import android.service.voice.HotwordRejectedResult; +import android.service.voice.IDspHotwordDetectionCallback; +import android.util.Slog; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.app.IHotwordRecognitionStatusCallback; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Locale; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A class that provides Dsp trusted hotword detector to communicate with the {@link + * HotwordDetectionService}. + * + * This class can handle the hotword detection which detector is created by using + * {@link android.service.voice.VoiceInteractionService#createAlwaysOnHotwordDetector(String, + * Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)}. + */ +final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession { + private static final String TAG = "DspTrustedHotwordDetectorSession"; + + // The validation timeout value is 3 seconds for onDetect of DSP trigger event. + private static final long VALIDATION_TIMEOUT_MILLIS = 3000; + // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. + private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; + + @GuardedBy("mLock") + private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture; + + @GuardedBy("mLock") + private boolean mValidatingDspTrigger = false; + + DspTrustedHotwordDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, + @NonNull Object lock, @NonNull Context context, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, + voiceInteractorIdentity, scheduledExecutorService, logging); + } + + @SuppressWarnings("GuardedBy") + void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + IHotwordRecognitionStatusCallback externalCallback) { + if (DEBUG) { + Slog.d(TAG, "detectFromDspSourceLocked"); + } + + AtomicBoolean timeoutDetected = new AtomicBoolean(false); + // TODO: consider making this a non-anonymous class. + IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { + @Override + public void onDetected(HotwordDetectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onDetected"); + } + synchronized (mLock) { + if (mCancellationKeyPhraseDetectionFuture != null) { + mCancellationKeyPhraseDetectionFuture.cancel(true); + } + if (timeoutDetected.get()) { + return; + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); + if (!mValidatingDspTrigger) { + Slog.i(TAG, "Ignoring #onDetected due to a process restart"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); + return; + } + mValidatingDspTrigger = false; + try { + enforcePermissionsForDataDelivery(); + enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); + } catch (SecurityException e) { + Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); + externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); + return; + } + saveProximityValueToBundle(result); + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); + } catch (IOException e) { + externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); + return; + } + externalCallback.onKeyphraseDetected(recognitionEvent, newResult); + Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + newResult); + } + } + } + + @Override + public void onRejected(HotwordRejectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onRejected"); + } + synchronized (mLock) { + if (mCancellationKeyPhraseDetectionFuture != null) { + mCancellationKeyPhraseDetectionFuture.cancel(true); + } + if (timeoutDetected.get()) { + return; + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); + if (!mValidatingDspTrigger) { + Slog.i(TAG, "Ignoring #onRejected due to a process restart"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); + return; + } + mValidatingDspTrigger = false; + externalCallback.onRejected(result); + if (mDebugHotwordLogging && result != null) { + Slog.i(TAG, "Egressed rejected result: " + result); + } + } + } + }; + + mValidatingDspTrigger = true; + mRemoteHotwordDetectionService.run(service -> { + // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke + // the callback before timeout value. In order to reduce the latency impact between + // server side and client side, we need to use another timeout value + // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. + mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( + () -> { + // TODO: avoid allocate every time + timeoutDetected.set(true); + Slog.w(TAG, "Timed out on #detectFromDspSource"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); + try { + externalCallback.onError(CALLBACK_DETECT_TIMEOUT); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report onError status: ", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } + }, + MAX_VALIDATION_TIMEOUT_MILLIS, + TimeUnit.MILLISECONDS); + service.detectFromDspSource( + recognitionEvent, + recognitionEvent.getCaptureFormat(), + VALIDATION_TIMEOUT_MILLIS, + internalCallback); + }); + } + + @Override + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + // TODO(b/244598068): Check HotwordAudioStreamManager first + Slog.v(TAG, "informRestartProcessLocked"); + if (mValidatingDspTrigger) { + // We're restarting the process while it's processing a DSP trigger, so report a + // rejection. This also allows the Interactor to startRecognition again + try { + mCallback.onRejected(new HotwordRejectedResult.Builder().build()); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to call #rejected"); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + mValidatingDspTrigger = false; + } + mUpdateStateAfterStartFinished.set(false); + + try { + mCallback.onProcessRestarted(); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("process restarted"); + } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + super.dumpLocked(prefix, pw); + pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); + } +} diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index 2ac25b65a082..2a4d4a15afdf 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -16,67 +16,27 @@ package com.android.server.voiceinteraction; -import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; -import static android.Manifest.permission.RECORD_AUDIO; -import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN; -import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL; -import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE; -import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT; -import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS; -import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN; -import static android.service.voice.HotwordDetectionService.KEY_INITIALIZATION_STATUS; - -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__NORMAL_DETECTOR; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; -import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight; import android.annotation.NonNull; import android.annotation.Nullable; -import android.app.AppOpsManager; -import android.attention.AttentionManagerInternal; import android.content.ComponentName; import android.content.ContentCaptureOptions; import android.content.Context; import android.content.Intent; -import android.content.PermissionChecker; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; import android.media.AudioManagerInternal; import android.media.permission.Identity; -import android.media.permission.PermissionUtil; import android.os.Binder; import android.os.Bundle; import android.os.IBinder; @@ -87,41 +47,27 @@ import android.os.RemoteException; import android.os.ServiceManager; import android.os.SharedMemory; import android.provider.DeviceConfig; -import android.service.voice.HotwordDetectedResult; import android.service.voice.HotwordDetectionService; import android.service.voice.HotwordDetector; -import android.service.voice.HotwordRejectedResult; -import android.service.voice.IDspHotwordDetectionCallback; import android.service.voice.IHotwordDetectionService; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity; import android.speech.IRecognitionServiceManager; -import android.text.TextUtils; -import android.util.Pair; import android.util.Slog; import android.view.contentcapture.IContentCaptureManager; import com.android.internal.annotations.GuardedBy; import com.android.internal.app.IHotwordRecognitionStatusCallback; -import com.android.internal.infra.AndroidFuture; import com.android.internal.infra.ServiceConnector; import com.android.server.LocalServices; import com.android.server.pm.permission.PermissionManagerServiceInternal; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.io.PrintWriter; -import java.time.Duration; import java.time.Instant; -import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; /** @@ -132,63 +78,15 @@ final class HotwordDetectionConnection { static final boolean DEBUG = false; private static final String KEY_RESTART_PERIOD_IN_SECONDS = "restart_period_in_seconds"; - // TODO: These constants need to be refined. - // The validation timeout value is 3 seconds for onDetect of DSP trigger event. - private static final long VALIDATION_TIMEOUT_MILLIS = 3000; - // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. - private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; - private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000; - private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; - private static final Duration MAX_UPDATE_TIMEOUT_DURATION = - Duration.ofMillis(MAX_UPDATE_TIMEOUT_MILLIS); private static final long RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS = 60 * 60 * 1000; // 1 hour private static final int MAX_ISOLATED_PROCESS_NUMBER = 10; - // The error codes are used for onError callback - private static final int HOTWORD_DETECTION_SERVICE_DIED = -1; - private static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2; - private static final int CALLBACK_DETECT_TIMEOUT = -3; - private static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; - - // Hotword metrics - private static final int METRICS_INIT_UNKNOWN_TIMEOUT = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; - private static final int METRICS_INIT_UNKNOWN_NO_VALUE = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; - private static final int METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; - private static final int METRICS_INIT_CALLBACK_STATE_ERROR = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; - private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; - - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; - private static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; - - private static final int METRICS_EXTERNAL_SOURCE_DETECTED = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; - private static final int METRICS_EXTERNAL_SOURCE_REJECTED = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; - private static final int METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; - private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION = - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; - - private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); // TODO: This may need to be a Handler(looper) private final ScheduledExecutorService mScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - private final AppOpsManager mAppOpsManager; - private final HotwordAudioStreamCopier mHotwordAudioStreamCopier; @Nullable private final ScheduledFuture<?> mCancellationTaskFuture; - private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied; - private final @NonNull ServiceConnectionFactory mServiceConnectionFactory; - private final IHotwordRecognitionStatusCallback mCallback; + @NonNull private final ServiceConnectionFactory mServiceConnectionFactory; private final int mDetectorType; /** * Time after which each HotwordDetectionService process is stopped and replaced by a new one. @@ -201,35 +99,22 @@ final class HotwordDetectionConnection { final ComponentName mDetectionComponentName; final int mUser; final Context mContext; - - @Nullable AttentionManagerInternal mAttentionManagerInternal = null; - - final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = - this::setProximityValue; - - volatile HotwordDetectionServiceIdentity mIdentity; - private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; private Instant mLastRestartInstant; - private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture; private ScheduledFuture<?> mDebugHotwordLoggingTimeoutFuture = null; /** Identity used for attributing app ops when delivering data to the Interactor. */ @GuardedBy("mLock") @Nullable private final Identity mVoiceInteractorIdentity; - @GuardedBy("mLock") - private ParcelFileDescriptor mCurrentAudioSink; - @GuardedBy("mLock") - private boolean mValidatingDspTrigger = false; - @GuardedBy("mLock") - private boolean mPerformingSoftwareHotwordDetection; - private @NonNull ServiceConnection mRemoteHotwordDetectionService; + @NonNull private ServiceConnection mRemoteHotwordDetectionService; private IBinder mAudioFlinger; + @GuardedBy("mLock") private boolean mDebugHotwordLogging = false; + @GuardedBy("mLock") - private double mProximityMeters = PROXIMITY_UNKNOWN; + private final HotwordDetectorSession mHotwordDetectorSession; HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, ComponentName serviceName, int userId, @@ -244,13 +129,8 @@ final class HotwordDetectionConnection { mContext = context; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; - mAppOpsManager = mContext.getSystemService(AppOpsManager.class); - mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, detectorType, - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag); mDetectionComponentName = serviceName; mUser = userId; - mCallback = callback; mDetectorType = detectorType; mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION, KEY_RESTART_PERIOD_IN_SECONDS, 0); @@ -259,17 +139,21 @@ final class HotwordDetectionConnection { initAudioFlingerLocked(); mServiceConnectionFactory = new ServiceConnectionFactory(intent, bindInstantServiceAllowed); - mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); - if (ENABLE_PROXIMITY_RESULT) { - mAttentionManagerInternal = LocalServices.getService(AttentionManagerInternal.class); - if (mAttentionManagerInternal != null) { - mAttentionManagerInternal.onStartProximityUpdates(mProximityCallbackInternal); - } - } - mLastRestartInstant = Instant.now(); - updateStateAfterProcessStart(options, sharedMemory); + + if (detectorType == HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP) { + mHotwordDetectorSession = new DspTrustedHotwordDetectorSession( + mRemoteHotwordDetectionService, mLock, mContext, callback, + mVoiceInteractionServiceUid, mVoiceInteractorIdentity, + mScheduledExecutorService, mDebugHotwordLogging); + } else { + mHotwordDetectorSession = new SoftwareTrustedHotwordDetectorSession( + mRemoteHotwordDetectionService, mLock, mContext, callback, + mVoiceInteractionServiceUid, mVoiceInteractorIdentity, + mScheduledExecutorService, mDebugHotwordLogging); + } + mHotwordDetectorSession.initialize(options, sharedMemory); if (mReStartPeriodSeconds <= 0) { mCancellationTaskFuture = null; @@ -320,106 +204,11 @@ final class HotwordDetectionConnection { } } - private void updateStateAfterProcessStart( - PersistableBundle options, SharedMemory sharedMemory) { - if (DEBUG) { - Slog.d(TAG, "updateStateAfterProcessStart"); - } - mRemoteHotwordDetectionService.postAsync(service -> { - AndroidFuture<Void> future = new AndroidFuture<>(); - IRemoteCallback statusCallback = new IRemoteCallback.Stub() { - @Override - public void sendResult(Bundle bundle) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "updateState finish"); - } - future.complete(null); - if (mUpdateStateAfterStartFinished.getAndSet(true)) { - Slog.w(TAG, "call callback after timeout"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, - mVoiceInteractionServiceUid); - return; - } - Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle); - int status = statusResultPair.first; - int initResultMetricsResult = statusResultPair.second; - try { - mCallback.onStatusReported(status); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, - initResultMetricsResult); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report initialization status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - } - }; - try { - service.updateState(options, sharedMemory, statusCallback); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); - } catch (RemoteException e) { - // TODO: (b/181842909) Report an error to voice interactor - Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, - mVoiceInteractionServiceUid); - } - return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); - }).whenComplete((res, err) -> { - if (err instanceof TimeoutException) { - Slog.w(TAG, "updateState timed out"); - if (mUpdateStateAfterStartFinished.getAndSet(true)) { - return; - } - try { - mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, - METRICS_INIT_UNKNOWN_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - } else if (err != null) { - Slog.w(TAG, "Failed to update state: " + err); - } else { - // NOTE: so far we don't need to take any action. - } - }); - } - - private static Pair<Integer, Integer> getInitStatusAndMetricsResult(Bundle bundle) { - if (bundle == null) { - return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, METRICS_INIT_UNKNOWN_NO_VALUE); - } - int status = bundle.getInt(KEY_INITIALIZATION_STATUS, INITIALIZATION_STATUS_UNKNOWN); - if (status > HotwordDetectionService.getMaxCustomInitializationStatus()) { - return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, - status == INITIALIZATION_STATUS_UNKNOWN - ? METRICS_INIT_UNKNOWN_NO_VALUE - : METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE); - } - // TODO: should guard against negative here - int metricsResult = status == INITIALIZATION_STATUS_SUCCESS - ? METRICS_INIT_CALLBACK_STATE_SUCCESS - : METRICS_INIT_CALLBACK_STATE_ERROR; - return new Pair<>(status, metricsResult); - } - - private boolean isBound() { - synchronized (mLock) { - return mRemoteHotwordDetectionService.isBound(); - } - } - + @SuppressWarnings("GuardedBy") void cancelLocked() { Slog.v(TAG, "cancelLocked"); clearDebugHotwordLoggingTimeoutLocked(); + mHotwordDetectorSession.destroyLocked(); mDebugHotwordLogging = false; mRemoteHotwordDetectionService.unbind(); LocalServices.getService(PermissionManagerServiceInternal.class) @@ -434,118 +223,32 @@ final class HotwordDetectionConnection { if (mAudioFlinger != null) { mAudioFlinger.unlinkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); } - if (mAttentionManagerInternal != null) { - mAttentionManagerInternal.onStopProximityUpdates(mProximityCallbackInternal); - } } + @SuppressWarnings("GuardedBy") void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); - - // Prevent doing the init late, so restart is handled equally to a clean process start. - // TODO(b/191742511): this logic needs a test - if (!mUpdateStateAfterStartFinished.get() - && Instant.now().minus(MAX_UPDATE_TIMEOUT_DURATION).isBefore(mLastRestartInstant)) { - Slog.v(TAG, "call updateStateAfterProcessStart"); - updateStateAfterProcessStart(options, sharedMemory); - } else { - mRemoteHotwordDetectionService.run( - service -> service.updateState(options, sharedMemory, null /* callback */)); - } + mHotwordDetectorSession.updateStateLocked(options, sharedMemory, mLastRestartInstant); } + /** + * This method is only used by SoftwareHotwordDetector. + */ void startListeningFromMic( AudioFormat audioFormat, IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { if (DEBUG) { Slog.d(TAG, "startListeningFromMic"); } - mSoftwareCallback = callback; - synchronized (mLock) { - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword validation is already in progress, ignoring."); + if (!(mHotwordDetectorSession instanceof SoftwareTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a software detector"); return; } - mPerformingSoftwareHotwordDetection = true; - - startListeningFromMicLocked(); + ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession) + .startListeningFromMicLocked(audioFormat, callback); } } - private void startListeningFromMicLocked() { - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection has already completed"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mPerformingSoftwareHotwordDetection = false; - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - mSoftwareCallback.onError(); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - mSoftwareCallback.onError(); - return; - } - mSoftwareCallback.onDetected(newResult, null, null); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.wtf(TAG, "onRejected"); - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - // onRejected isn't allowed here, and we are not expecting it. - } - }; - - mRemoteHotwordDetectionService.run( - service -> service.detectFromMicrophoneSource( - null, - AUDIO_SOURCE_MICROPHONE, - null, - null, - internalCallback)); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, - mVoiceInteractionServiceUid); - } - public void startListeningFromExternalSource( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @@ -554,37 +257,26 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "startListeningFromExternalSource"); } - - handleExternalSourceHotwordDetection( - audioStream, - audioFormat, - options, - callback); + synchronized (mLock) { + mHotwordDetectorSession.startListeningFromExternalSourceLocked(audioStream, audioFormat, + options, callback); + } } + /** + * This method is only used by SoftwareHotwordDetector. + */ void stopListening() { if (DEBUG) { Slog.d(TAG, "stopListening"); } synchronized (mLock) { - stopListeningLocked(); - } - } - - private void stopListeningLocked() { - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection is not running"); - return; - } - mPerformingSoftwareHotwordDetection = false; - - mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested"); - bestEffortClose(mCurrentAudioSink); + if (!(mHotwordDetectorSession instanceof SoftwareTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a software detector"); + return; + } + ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession).stopListeningLocked(); } - mCurrentAudioSink = null; } void triggerHardwareRecognitionEventForTestLocked( @@ -596,130 +288,18 @@ final class HotwordDetectionConnection { detectFromDspSource(event, callback); } - void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, IHotwordRecognitionStatusCallback externalCallback) { if (DEBUG) { Slog.d(TAG, "detectFromDspSource"); } - - AtomicBoolean timeoutDetected = new AtomicBoolean(false); - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onDetected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - try { - enforcePermissionsForDataDelivery(); - enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); - } catch (SecurityException e) { - Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); - return; - } - externalCallback.onKeyphraseDetected(recognitionEvent, newResult); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onRejected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onRejected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - externalCallback.onRejected(result); - if (mDebugHotwordLogging && result != null) { - Slog.i(TAG, "Egressed rejected result: " + result); - } - } - } - }; - synchronized (mLock) { - mValidatingDspTrigger = true; - mRemoteHotwordDetectionService.run(service -> { - // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke - // the callback before timeout value. In order to reduce the latency impact between - // server side and client side, we need to use another timeout value - // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. - mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( - () -> { - // TODO: avoid allocate every time - timeoutDetected.set(true); - Slog.w(TAG, "Timed out on #detectFromDspSource"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); - try { - externalCallback.onError(CALLBACK_DETECT_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: ", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - }, - MAX_VALIDATION_TIMEOUT_MILLIS, - TimeUnit.MILLISECONDS); - service.detectFromDspSource( - recognitionEvent, - recognitionEvent.getCaptureFormat(), - VALIDATION_TIMEOUT_MILLIS, - internalCallback); - }); + if (!(mHotwordDetectorSession instanceof DspTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a Dsp detector"); + return; + } + ((DspTrustedHotwordDetectorSession) mHotwordDetectorSession).detectFromDspSourceLocked( + recognitionEvent, externalCallback); } } @@ -730,10 +310,12 @@ final class HotwordDetectionConnection { } } + @SuppressWarnings("GuardedBy") void setDebugHotwordLoggingLocked(boolean logging) { Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); clearDebugHotwordLoggingTimeoutLocked(); mDebugHotwordLogging = logging; + mHotwordDetectorSession.setDebugHotwordLoggingLocked(logging); if (logging) { // Reset mDebugHotwordLogging to false after one hour @@ -741,6 +323,7 @@ final class HotwordDetectionConnection { Slog.v(TAG, "Timeout to reset mDebugHotwordLogging to false"); synchronized (mLock) { mDebugHotwordLogging = false; + mHotwordDetectorSession.setDebugHotwordLoggingLocked(false); } }, RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); } @@ -753,60 +336,23 @@ final class HotwordDetectionConnection { } } + @SuppressWarnings("GuardedBy") private void restartProcessLocked() { // TODO(b/244598068): Check HotwordAudioStreamManager first Slog.v(TAG, "Restarting hotword detection process"); ServiceConnection oldConnection = mRemoteHotwordDetectionService; HotwordDetectionServiceIdentity previousIdentity = mIdentity; - // TODO(volnov): this can be done after connect() has been successful. - if (mValidatingDspTrigger) { - // We're restarting the process while it's processing a DSP trigger, so report a - // rejection. This also allows the Interactor to startReco again - try { - mCallback.onRejected(new HotwordRejectedResult.Builder().build()); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to call #rejected"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - mValidatingDspTrigger = false; - } - - mUpdateStateAfterStartFinished.set(false); mLastRestartInstant = Instant.now(); - // Recreate connection to reset the cache. mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); - Slog.v(TAG, "Started the new process, issuing #onProcessRestarted"); - try { - mCallback.onProcessRestarted(); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - - // Restart listening from microphone if the hotword process has been restarted. - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Process restarted: calling startRecognition() again"); - startListeningFromMicLocked(); - } - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted"); - bestEffortClose(mCurrentAudioSink); - mCurrentAudioSink = null; - } - + Slog.v(TAG, "Started the new process, dispatching processRestarted to detector"); + mHotwordDetectorSession.updateRemoteHotwordDetectionServiceLocked( + mRemoteHotwordDetectionService); + mHotwordDetectorSession.informRestartProcessLocked(); if (DEBUG) { - Slog.i(TAG, "#onProcessRestarted called, unbinding from the old process"); + Slog.i(TAG, "processRestarted is dispatched done, unbinding from the old process"); } oldConnection.ignoreConnectionStatusEvents(); oldConnection.unbind(); @@ -816,7 +362,6 @@ final class HotwordDetectionConnection { } static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub { - private SoundTrigger.KeyphraseRecognitionEvent mRecognitionEvent; private final HotwordDetectionConnection mHotwordDetectionConnection; private final IHotwordRecognitionStatusCallback mExternalCallback; @@ -837,7 +382,6 @@ final class HotwordDetectionConnection { HotwordMetricsLogger.writeKeyphraseTriggerEvent( HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP, HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER); - mRecognitionEvent = recognitionEvent; mHotwordDetectionConnection.detectFromDspSource( recognitionEvent, mExternalCallback); } else { @@ -872,160 +416,18 @@ final class HotwordDetectionConnection { } public void dump(String prefix, PrintWriter pw) { - pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); - pw.print(prefix); - pw.print("mBound=" + mRemoteHotwordDetectionService.isBound()); - pw.print(", mValidatingDspTrigger=" + mValidatingDspTrigger); - pw.print(", mPerformingSoftwareHotwordDetection=" + mPerformingSoftwareHotwordDetection); - pw.print(", mRestartCount=" + mServiceConnectionFactory.mRestartCount); - pw.print(", mLastRestartInstant=" + mLastRestartInstant); - pw.println(", mDetectorType=" + HotwordDetector.detectorTypeToString(mDetectorType)); - } - - private void handleExternalSourceHotwordDetection( - ParcelFileDescriptor audioStream, - AudioFormat audioFormat, - @Nullable PersistableBundle options, - IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { - if (DEBUG) { - Slog.d(TAG, "#handleExternalSourceHotwordDetection"); - } - InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); - - Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe(); - if (clientPipe == null) { - // TODO: Need to propagate as unknown error or something? - return; - } - ParcelFileDescriptor serviceAudioSink = clientPipe.second; - ParcelFileDescriptor serviceAudioSource = clientPipe.first; - synchronized (mLock) { - mCurrentAudioSink = serviceAudioSink; + pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); + pw.print(prefix); pw.print("mBound="); + pw.println(mRemoteHotwordDetectionService.isBound()); + pw.print(prefix); pw.print("mRestartCount="); + pw.println(mServiceConnectionFactory.mRestartCount); + pw.print(prefix); pw.print("mLastRestartInstant="); pw.println(mLastRestartInstant); + pw.print(prefix); pw.print("mDetectorType="); + pw.println(HotwordDetector.detectorTypeToString(mDetectorType)); + pw.print(prefix); pw.println("HotwordDetectorSession"); + mHotwordDetectorSession.dumpLocked(prefix, pw); } - - mAudioCopyExecutor.execute(() -> { - try (InputStream source = audioSource; - OutputStream fos = - new ParcelFileDescriptor.AutoCloseOutputStream(serviceAudioSink)) { - - byte[] buffer = new byte[1024]; - while (true) { - int bytesRead = source.read(buffer, 0, 1024); - - if (bytesRead < 0) { - Slog.i(TAG, "Reached end of stream for external hotword"); - break; - } - - // TODO: First write to ring buffer to make sure we don't lose data if the next - // statement fails. - // ringBuffer.append(buffer, bytesRead); - fos.write(buffer, 0, bytesRead); - } - } catch (IOException e) { - Slog.w(TAG, "Failed supplying audio data to validator", e); - - try { - callback.onError(); - } catch (RemoteException ex) { - Slog.w(TAG, "Failed to report onError status: " + ex); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - } finally { - synchronized (mLock) { - mCurrentAudioSink = null; - } - } - }); - - // TODO: handle cancellations well - // TODO: what if we cancelled and started a new one? - mRemoteHotwordDetectionService.run( - service -> { - service.detectFromMicrophoneSource( - serviceAudioSource, - // TODO: consider making a proxy callback + copy of audio format - AUDIO_SOURCE_EXTERNAL, - audioFormat, - options, - new IDspHotwordDetectionCallback.Stub() { - @Override - public void onRejected(HotwordRejectedResult result) - throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_REJECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - callback.onRejected(result); - - if (result != null) { - Slog.i(TAG, "Egressed 'hotword rejected result' " - + "from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + result); - } - } - } - - @Override - public void onDetected(HotwordDetectedResult triggerResult) - throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_DETECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, - mVoiceInteractionServiceUid); - callback.onError(); - return; - } - HotwordDetectedResult newResult; - try { - newResult = - mHotwordAudioStreamCopier.startCopyingAudioStreams( - triggerResult); - } catch (IOException e) { - // TODO: Write event - callback.onError(); - return; - } - callback.onDetected(newResult, null /* audioFormat */, - null /* audioStream */); - Slog.i(TAG, "Egressed " - + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, - "Egressed detected result: " + newResult); - } - } - }); - - // A copy of this has been created and passed to the hotword validator - bestEffortClose(serviceAudioSource); - }); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION, - mVoiceInteractionServiceUid); } private class ServiceConnectionFactory { @@ -1054,7 +456,7 @@ final class HotwordDetectionConnection { } } - private class ServiceConnection extends ServiceConnector.Impl<IHotwordDetectionService> { + class ServiceConnection extends ServiceConnector.Impl<IHotwordDetectionService> { private final Object mLock = new Object(); private final Intent mIntent; @@ -1109,21 +511,16 @@ final class HotwordDetectionConnection { @Override public void binderDied() { super.binderDied(); + Slog.w(TAG, "binderDied"); synchronized (mLock) { if (!mRespectServiceConnectionStatusChanged) { Slog.v(TAG, "Ignored #binderDied event"); return; } - - Slog.w(TAG, "binderDied"); - try { - mCallback.onError(HOTWORD_DETECTION_SERVICE_DIED); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } + } + synchronized (HotwordDetectionConnection.this.mLock) { + mHotwordDetectorSession.reportErrorLocked( + HotwordDetectorSession.HOTWORD_DETECTION_SERVICE_DIED); } } @@ -1168,18 +565,6 @@ final class HotwordDetectionConnection { } } - private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() { - ParcelFileDescriptor[] fileDescriptors; - try { - fileDescriptors = ParcelFileDescriptor.createPipe(); - } catch (IOException e) { - Slog.e(TAG, "Failed to create audio stream pipe", e); - return null; - } - - return Pair.create(fileDescriptors[0], fileDescriptors[1]); - } - private static void updateAudioFlinger(ServiceConnection connection, IBinder audioFlinger) { // TODO: Consider using a proxy that limits the exposed API surface. connection.run(service -> service.updateAudioFlinger(audioFlinger)); @@ -1241,85 +626,4 @@ final class HotwordDetectionConnection { } }); } - - private void saveProximityValueToBundle(HotwordDetectedResult result) { - synchronized (mLock) { - if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { - result.setProximity(mProximityMeters); - } - } - } - - private void setProximityValue(double proximityMeters) { - synchronized (mLock) { - mProximityMeters = proximityMeters; - } - } - - private static void bestEffortClose(Closeable... closeables) { - for (Closeable closeable : closeables) { - bestEffortClose(closeable); - } - } - - private static void bestEffortClose(Closeable closeable) { - try { - closeable.close(); - } catch (IOException e) { - if (DEBUG) { - Slog.w(TAG, "Failed closing", e); - } - } - } - - // TODO: Share this code with SoundTriggerMiddlewarePermission. - private void enforcePermissionsForDataDelivery() { - Binder.withCleanCallingIdentity(() -> { - enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); - int hotwordOp = AppOpsManager.strOpToOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD); - mAppOpsManager.noteOpNoThrow(hotwordOp, - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag, OP_MESSAGE); - enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, - CAPTURE_AUDIO_HOTWORD, OP_MESSAGE); - }); - } - - /** - * Throws a {@link SecurityException} iff the given identity has given permission to receive - * data. - * - * @param context A {@link Context}, used for permission checks. - * @param identity The identity to check. - * @param permission The identifier of the permission we want to check. - * @param reason The reason why we're requesting the permission, for auditing purposes. - */ - private static void enforcePermissionForDataDelivery(@NonNull Context context, - @NonNull Identity identity, - @NonNull String permission, @NonNull String reason) { - final int status = PermissionUtil.checkPermissionForDataDelivery(context, identity, - permission, reason); - if (status != PermissionChecker.PERMISSION_GRANTED) { - throw new SecurityException( - TextUtils.formatSimple("Failed to obtain permission %s for identity %s", - permission, - SoundTriggerSessionPermissionsDecorator.toString(identity))); - } - } - - private static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, - SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { - // verify the phrase ID in HotwordDetectedResult is not exposing extra phrases - // the DSP did not detect - for (SoundTrigger.KeyphraseRecognitionExtra keyphrase : recognitionEvent.keyphraseExtras) { - if (keyphrase.getKeyphraseId() == result.getHotwordPhraseId()) { - return; - } - } - throw new SecurityException("Ignoring #onDetected due to trusted service " - + "sharing a keyphrase ID which the DSP did not detect"); - } - - private static final String OP_MESSAGE = - "Providing hotword detection result to VoiceInteractionService"; } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java new file mode 100644 index 000000000000..f9f43c9ba49a --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java @@ -0,0 +1,666 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; +import static android.Manifest.permission.RECORD_AUDIO; +import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN; +import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL; +import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT; +import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS; +import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN; +import static android.service.voice.HotwordDetectionService.KEY_INITIALIZATION_STATUS; + +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; +import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.app.AppOpsManager; +import android.attention.AttentionManagerInternal; +import android.content.Context; +import android.content.PermissionChecker; +import android.hardware.soundtrigger.SoundTrigger; +import android.media.AudioFormat; +import android.media.permission.Identity; +import android.media.permission.PermissionUtil; +import android.os.Binder; +import android.os.Bundle; +import android.os.IRemoteCallback; +import android.os.ParcelFileDescriptor; +import android.os.PersistableBundle; +import android.os.RemoteException; +import android.os.SharedMemory; +import android.service.voice.HotwordDetectedResult; +import android.service.voice.HotwordDetectionService; +import android.service.voice.HotwordDetector; +import android.service.voice.HotwordRejectedResult; +import android.service.voice.IDspHotwordDetectionCallback; +import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; +import android.text.TextUtils; +import android.util.Pair; +import android.util.Slog; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.app.IHotwordRecognitionStatusCallback; +import com.android.internal.infra.AndroidFuture; +import com.android.server.LocalServices; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A class that provides trusted hotword detector to communicate with the {@link + * HotwordDetectionService}. + * + * This class provides the methods to do initialization with the {@link HotwordDetectionService} + * and handle external source detection. It also provides the methods to check if we can egress + * the data from the {@link HotwordDetectionService}. + * + * The subclass should override the {@link #informRestartProcessLocked()} to handle the trusted + * process restart. + */ +abstract class HotwordDetectorSession { + private static final String TAG = "HotwordDetectorSession"; + static final boolean DEBUG = false; + + private static final String OP_MESSAGE = + "Providing hotword detection result to VoiceInteractionService"; + + // The error codes are used for onError callback + static final int HOTWORD_DETECTION_SERVICE_DIED = -1; + static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2; + static final int CALLBACK_DETECT_TIMEOUT = -3; + static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; + + // TODO: These constants need to be refined. + private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000; + private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; + private static final Duration MAX_UPDATE_TIMEOUT_DURATION = + Duration.ofMillis(MAX_UPDATE_TIMEOUT_MILLIS); + + // Hotword metrics + private static final int METRICS_INIT_UNKNOWN_TIMEOUT = + HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; + private static final int METRICS_INIT_UNKNOWN_NO_VALUE = + HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; + private static final int METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE = + HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; + private static final int METRICS_INIT_CALLBACK_STATE_ERROR = + HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; + private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS = + HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; + + static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; + static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; + static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; + + private static final int METRICS_EXTERNAL_SOURCE_DETECTED = + HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; + private static final int METRICS_EXTERNAL_SOURCE_REJECTED = + HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; + private static final int EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = + HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; + private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION = + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; + + private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); + // TODO: This may need to be a Handler(looper) + final ScheduledExecutorService mScheduledExecutorService; + private final AppOpsManager mAppOpsManager; + final HotwordAudioStreamCopier mHotwordAudioStreamCopier; + final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); + final IHotwordRecognitionStatusCallback mCallback; + + final Object mLock; + final int mVoiceInteractionServiceUid; + final Context mContext; + + @Nullable AttentionManagerInternal mAttentionManagerInternal = null; + + final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = + this::setProximityValue; + + /** Identity used for attributing app ops when delivering data to the Interactor. */ + @Nullable + private final Identity mVoiceInteractorIdentity; + @GuardedBy("mLock") + ParcelFileDescriptor mCurrentAudioSink; + @GuardedBy("mLock") + @NonNull HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService; + boolean mDebugHotwordLogging = false; + @GuardedBy("mLock") + private double mProximityMeters = PROXIMITY_UNKNOWN; + @GuardedBy("mLock") + private boolean mInitialized = false; + @GuardedBy("mLock") + private boolean mDestroyed = false; + @GuardedBy("mLock") + boolean mPerformingExternalSourceHotwordDetection; + + HotwordDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, + @NonNull Object lock, @NonNull Context context, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + mRemoteHotwordDetectionService = remoteHotwordDetectionService; + mLock = lock; + mContext = context; + mCallback = callback; + mVoiceInteractionServiceUid = voiceInteractionServiceUid; + mVoiceInteractorIdentity = voiceInteractorIdentity; + mAppOpsManager = mContext.getSystemService(AppOpsManager.class); + mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, getDetectorType(), + mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, + mVoiceInteractorIdentity.attributionTag); + mScheduledExecutorService = scheduledExecutorService; + mDebugHotwordLogging = logging; + + if (ENABLE_PROXIMITY_RESULT) { + mAttentionManagerInternal = LocalServices.getService(AttentionManagerInternal.class); + if (mAttentionManagerInternal != null) { + mAttentionManagerInternal.onStartProximityUpdates(mProximityCallbackInternal); + } + } + } + + @SuppressWarnings("GuardedBy") + private void updateStateAfterProcessStartLocked(PersistableBundle options, + SharedMemory sharedMemory) { + if (DEBUG) { + Slog.d(TAG, "updateStateAfterProcessStartLocked"); + } + AndroidFuture<Void> voidFuture = mRemoteHotwordDetectionService.postAsync(service -> { + AndroidFuture<Void> future = new AndroidFuture<>(); + IRemoteCallback statusCallback = new IRemoteCallback.Stub() { + @Override + public void sendResult(Bundle bundle) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "updateState finish"); + } + future.complete(null); + if (mUpdateStateAfterStartFinished.getAndSet(true)) { + Slog.w(TAG, "call callback after timeout"); + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, + mVoiceInteractionServiceUid); + return; + } + Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle); + int status = statusResultPair.first; + int initResultMetricsResult = statusResultPair.second; + try { + mCallback.onStatusReported(status); + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), + initResultMetricsResult); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report initialization status: " + e); + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + } + }; + try { + service.updateState(options, sharedMemory, statusCallback); + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, + mVoiceInteractionServiceUid); + } catch (RemoteException e) { + // TODO: (b/181842909) Report an error to voice interactor + Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, + mVoiceInteractionServiceUid); + } + return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + }).whenComplete((res, err) -> { + if (err instanceof TimeoutException) { + Slog.w(TAG, "updateState timed out"); + if (mUpdateStateAfterStartFinished.getAndSet(true)) { + return; + } + try { + mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), + METRICS_INIT_UNKNOWN_TIMEOUT); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + } else if (err != null) { + Slog.w(TAG, "Failed to update state: " + err); + } + }); + if (voidFuture == null) { + Slog.w(TAG, "Failed to create AndroidFuture"); + } + } + + private static Pair<Integer, Integer> getInitStatusAndMetricsResult(Bundle bundle) { + if (bundle == null) { + return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, METRICS_INIT_UNKNOWN_NO_VALUE); + } + int status = bundle.getInt(KEY_INITIALIZATION_STATUS, INITIALIZATION_STATUS_UNKNOWN); + if (status > HotwordDetectionService.getMaxCustomInitializationStatus()) { + return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, + status == INITIALIZATION_STATUS_UNKNOWN + ? METRICS_INIT_UNKNOWN_NO_VALUE + : METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE); + } + // TODO: should guard against negative here + int metricsResult = status == INITIALIZATION_STATUS_SUCCESS + ? METRICS_INIT_CALLBACK_STATE_SUCCESS + : METRICS_INIT_CALLBACK_STATE_ERROR; + return new Pair<>(status, metricsResult); + } + + @SuppressWarnings("GuardedBy") + void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, + Instant lastRestartInstant) { + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, + mVoiceInteractionServiceUid); + // Prevent doing the init late, so restart is handled equally to a clean process start. + // TODO(b/191742511): this logic needs a test + if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus( + MAX_UPDATE_TIMEOUT_DURATION).isBefore(lastRestartInstant)) { + Slog.v(TAG, "call updateStateAfterProcessStartLocked"); + updateStateAfterProcessStartLocked(options, sharedMemory); + } else { + mRemoteHotwordDetectionService.run( + service -> service.updateState(options, sharedMemory, /* callback= */ null)); + } + } + + void startListeningFromExternalSourceLocked( + ParcelFileDescriptor audioStream, + AudioFormat audioFormat, + @Nullable PersistableBundle options, + IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { + if (DEBUG) { + Slog.d(TAG, "startListeningFromExternalSourceLocked"); + } + + handleExternalSourceHotwordDetectionLocked( + audioStream, + audioFormat, + options, + callback); + } + + @SuppressWarnings("GuardedBy") + private void handleExternalSourceHotwordDetectionLocked( + ParcelFileDescriptor audioStream, + AudioFormat audioFormat, + @Nullable PersistableBundle options, + IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { + if (DEBUG) { + Slog.d(TAG, "#handleExternalSourceHotwordDetectionLocked"); + } + if (mPerformingExternalSourceHotwordDetection) { + Slog.i(TAG, "Hotword validation is already in progress for external source."); + return; + } + + InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); + + Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe(); + if (clientPipe == null) { + // TODO: Need to propagate as unknown error or something? + return; + } + ParcelFileDescriptor serviceAudioSink = clientPipe.second; + ParcelFileDescriptor serviceAudioSource = clientPipe.first; + + mCurrentAudioSink = serviceAudioSink; + mPerformingExternalSourceHotwordDetection = true; + + mAudioCopyExecutor.execute(() -> { + try (InputStream source = audioSource; + OutputStream fos = + new ParcelFileDescriptor.AutoCloseOutputStream(serviceAudioSink)) { + + byte[] buffer = new byte[1024]; + while (true) { + int bytesRead = source.read(buffer, 0, 1024); + + if (bytesRead < 0) { + Slog.i(TAG, "Reached end of stream for external hotword"); + break; + } + + // TODO: First write to ring buffer to make sure we don't lose data if the next + // statement fails. + // ringBuffer.append(buffer, bytesRead); + fos.write(buffer, 0, bytesRead); + } + } catch (IOException e) { + Slog.w(TAG, "Failed supplying audio data to validator", e); + + try { + callback.onError(); + } catch (RemoteException ex) { + Slog.w(TAG, "Failed to report onError status: " + ex); + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } + } finally { + synchronized (mLock) { + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("start external source"); + } + } + }); + + // TODO: handle cancellations well + // TODO: what if we cancelled and started a new one? + mRemoteHotwordDetectionService.run( + service -> { + service.detectFromMicrophoneSource( + serviceAudioSource, + // TODO: consider making a proxy callback + copy of audio format + AUDIO_SOURCE_EXTERNAL, + audioFormat, + options, + new IDspHotwordDetectionCallback.Stub() { + @Override + public void onRejected(HotwordRejectedResult result) + throws RemoteException { + synchronized (mLock) { + mPerformingExternalSourceHotwordDetection = false; + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_EXTERNAL_SOURCE_REJECTED, + mVoiceInteractionServiceUid); + mScheduledExecutorService.schedule( + () -> { + bestEffortClose(serviceAudioSink, audioSource); + }, + EXTERNAL_HOTWORD_CLEANUP_MILLIS, + TimeUnit.MILLISECONDS); + + callback.onRejected(result); + + if (result != null) { + Slog.i(TAG, "Egressed 'hotword rejected result' " + + "from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + result); + } + } + } + } + + @Override + public void onDetected(HotwordDetectedResult triggerResult) + throws RemoteException { + synchronized (mLock) { + mPerformingExternalSourceHotwordDetection = false; + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_EXTERNAL_SOURCE_DETECTED, + mVoiceInteractionServiceUid); + mScheduledExecutorService.schedule( + () -> { + bestEffortClose(serviceAudioSink, audioSource); + }, + EXTERNAL_HOTWORD_CLEANUP_MILLIS, + TimeUnit.MILLISECONDS); + + try { + enforcePermissionsForDataDelivery(); + } catch (SecurityException e) { + HotwordMetricsLogger.writeDetectorEvent( + getDetectorType(), + EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, + mVoiceInteractionServiceUid); + callback.onError(); + return; + } + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier + .startCopyingAudioStreams(triggerResult); + } catch (IOException e) { + // TODO: Write event + callback.onError(); + return; + } + callback.onDetected(newResult, null /* audioFormat */, + null /* audioStream */); + Slog.i(TAG, "Egressed " + + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, + "Egressed detected result: " + newResult); + } + } + } + }); + + // A copy of this has been created and passed to the hotword validator + bestEffortClose(serviceAudioSource); + }); + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION, + mVoiceInteractionServiceUid); + } + + void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) { + synchronized (mLock) { + if (mInitialized || mDestroyed) { + return; + } + updateStateAfterProcessStartLocked(options, sharedMemory); + mInitialized = true; + } + } + + @SuppressWarnings("GuardedBy") + void destroyLocked() { + mDestroyed = true; + mDebugHotwordLogging = false; + mRemoteHotwordDetectionService = null; + if (mAttentionManagerInternal != null) { + mAttentionManagerInternal.onStopProximityUpdates(mProximityCallbackInternal); + } + } + + void setDebugHotwordLoggingLocked(boolean logging) { + Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); + mDebugHotwordLogging = logging; + } + + @SuppressWarnings("GuardedBy") + void updateRemoteHotwordDetectionServiceLocked( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService) { + mRemoteHotwordDetectionService = remoteHotwordDetectionService; + } + + void reportErrorLocked(int status) { + try { + mCallback.onError(status); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report onError status: " + e); + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } + } + + /** + * Called when the trusted process is restarted. + */ + abstract void informRestartProcessLocked(); + + private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() { + ParcelFileDescriptor[] fileDescriptors; + try { + fileDescriptors = ParcelFileDescriptor.createPipe(); + } catch (IOException e) { + Slog.e(TAG, "Failed to create audio stream pipe", e); + return null; + } + + return Pair.create(fileDescriptors[0], fileDescriptors[1]); + } + + void saveProximityValueToBundle(HotwordDetectedResult result) { + synchronized (mLock) { + if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { + result.setProximity(mProximityMeters); + } + } + } + + private void setProximityValue(double proximityMeters) { + synchronized (mLock) { + mProximityMeters = proximityMeters; + } + } + + @SuppressWarnings("GuardedBy") + void closeExternalAudioStreamLocked(String reason) { + if (mCurrentAudioSink != null) { + Slog.i(TAG, "Closing external audio stream to hotword detector: " + reason); + bestEffortClose(mCurrentAudioSink); + mCurrentAudioSink = null; + } + } + + private static void bestEffortClose(Closeable... closeables) { + for (Closeable closeable : closeables) { + bestEffortClose(closeable); + } + } + + private static void bestEffortClose(Closeable closeable) { + try { + closeable.close(); + } catch (IOException e) { + if (DEBUG) { + Slog.w(TAG, "Failed closing", e); + } + } + } + + // TODO: Share this code with SoundTriggerMiddlewarePermission. + void enforcePermissionsForDataDelivery() { + Binder.withCleanCallingIdentity(() -> { + synchronized (mLock) { + enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); + int hotwordOp = AppOpsManager.strOpToOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD); + mAppOpsManager.noteOpNoThrow(hotwordOp, + mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, + mVoiceInteractorIdentity.attributionTag, OP_MESSAGE); + enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, + CAPTURE_AUDIO_HOTWORD, OP_MESSAGE); + } + }); + } + + /** + * Throws a {@link SecurityException} if the given identity has no permission to receive data. + * + * @param context A {@link Context}, used for permission checks. + * @param identity The identity to check. + * @param permission The identifier of the permission we want to check. + * @param reason The reason why we're requesting the permission, for auditing purposes. + */ + private static void enforcePermissionForDataDelivery(@NonNull Context context, + @NonNull Identity identity, @NonNull String permission, @NonNull String reason) { + final int status = PermissionUtil.checkPermissionForDataDelivery(context, identity, + permission, reason); + if (status != PermissionChecker.PERMISSION_GRANTED) { + throw new SecurityException( + TextUtils.formatSimple("Failed to obtain permission %s for identity %s", + permission, + SoundTriggerSessionPermissionsDecorator.toString(identity))); + } + } + + static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, + SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { + // verify the phrase ID in HotwordDetectedResult is not exposing extra phrases + // the DSP did not detect + for (SoundTrigger.KeyphraseRecognitionExtra keyphrase : recognitionEvent.keyphraseExtras) { + if (keyphrase.getKeyphraseId() == result.getHotwordPhraseId()) { + return; + } + } + throw new SecurityException("Ignoring #onDetected due to trusted service " + + "sharing a keyphrase ID which the DSP did not detect"); + } + + private int getDetectorType() { + if (this instanceof DspTrustedHotwordDetectorSession) { + return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP; + } else if (this instanceof SoftwareTrustedHotwordDetectorSession) { + return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE; + } + Slog.v(TAG, "Unexpected detector type"); + return -1; + } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + pw.print(prefix); pw.print("mCallback="); pw.println(mCallback); + pw.print(prefix); pw.print("mUpdateStateAfterStartFinished="); + pw.println(mUpdateStateAfterStartFinished); + pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized); + pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed); + pw.print(prefix); pw.print("DetectorType="); + pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); + pw.print(prefix); pw.print("mPerformingExternalSourceHotwordDetection="); + pw.println(mPerformingExternalSourceHotwordDetection); + } +} diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java new file mode 100644 index 000000000000..693068958568 --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE; + +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; + +import android.annotation.NonNull; +import android.content.Context; +import android.media.AudioFormat; +import android.media.permission.Identity; +import android.os.PersistableBundle; +import android.os.RemoteException; +import android.os.SharedMemory; +import android.service.voice.HotwordDetectedResult; +import android.service.voice.HotwordDetectionService; +import android.service.voice.HotwordDetector; +import android.service.voice.HotwordRejectedResult; +import android.service.voice.IDspHotwordDetectionCallback; +import android.service.voice.IHotwordDetectionService; +import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; +import android.util.Slog; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.app.IHotwordRecognitionStatusCallback; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.concurrent.ScheduledExecutorService; + +/** + * A class that provides software trusted hotword detector to communicate with the {@link + * HotwordDetectionService}. + * + * This class can handle the hotword detection which detector is created by using + * {@link android.service.voice.VoiceInteractionService#createHotwordDetector(PersistableBundle, + * SharedMemory, HotwordDetector.Callback)}. + */ +final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession { + private static final String TAG = "SoftwareTrustedHotwordDetectorSession"; + + private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; + @GuardedBy("mLock") + private boolean mPerformingSoftwareHotwordDetection; + + SoftwareTrustedHotwordDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, + @NonNull Object lock, @NonNull Context context, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, + voiceInteractorIdentity, scheduledExecutorService, logging); + } + + @SuppressWarnings("GuardedBy") + void startListeningFromMicLocked( + AudioFormat audioFormat, + IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { + if (DEBUG) { + Slog.d(TAG, "startListeningFromMicLocked"); + } + mSoftwareCallback = callback; + + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword validation is already in progress, ignoring."); + return; + } + mPerformingSoftwareHotwordDetection = true; + + startListeningFromMicLocked(); + } + + @SuppressWarnings("GuardedBy") + private void startListeningFromMicLocked() { + // TODO: consider making this a non-anonymous class. + IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { + @Override + public void onDetected(HotwordDetectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onDetected"); + } + synchronized (mLock) { + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); + if (!mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword detection has already completed"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); + return; + } + mPerformingSoftwareHotwordDetection = false; + try { + enforcePermissionsForDataDelivery(); + } catch (SecurityException e) { + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); + mSoftwareCallback.onError(); + return; + } + saveProximityValueToBundle(result); + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); + } catch (IOException e) { + // TODO: Write event + mSoftwareCallback.onError(); + return; + } + mSoftwareCallback.onDetected(newResult, null, null); + Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + newResult); + } + } + } + + @Override + public void onRejected(HotwordRejectedResult result) throws RemoteException { + if (DEBUG) { + Slog.wtf(TAG, "onRejected"); + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); + // onRejected isn't allowed here, and we are not expecting it. + } + }; + + mRemoteHotwordDetectionService.run( + service -> service.detectFromMicrophoneSource( + null, + AUDIO_SOURCE_MICROPHONE, + null, + null, + internalCallback)); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, + mVoiceInteractionServiceUid); + } + + @SuppressWarnings("GuardedBy") + void stopListeningLocked() { + if (DEBUG) { + Slog.d(TAG, "stopListeningLocked"); + } + if (!mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword detection is not running"); + return; + } + mPerformingSoftwareHotwordDetection = false; + + mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); + + closeExternalAudioStreamLocked("stopping requested"); + } + + @Override + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + // TODO(b/244598068): Check HotwordAudioStreamManager first + Slog.v(TAG, "informRestartProcessLocked"); + mUpdateStateAfterStartFinished.set(false); + + try { + mCallback.onProcessRestarted(); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + + // Restart listening from microphone if the hotword process has been restarted. + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Process restarted: calling startRecognition() again"); + startListeningFromMicLocked(); + } + + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("process restarted"); + } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + super.dumpLocked(prefix, pw); + pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection="); + pw.println(mPerformingSoftwareHotwordDetection); + } +} diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java deleted file mode 100644 index 02f5889cc520..000000000000 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java +++ /dev/null @@ -1,1325 +0,0 @@ -/* - * Copyright (C) 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.server.voiceinteraction; - -import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; -import static android.Manifest.permission.RECORD_AUDIO; -import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN; -import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL; -import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE; -import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT; -import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS; -import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN; -import static android.service.voice.HotwordDetectionService.KEY_INITIALIZATION_STATUS; - -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__NORMAL_DETECTOR; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; -import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight; - -import android.annotation.NonNull; -import android.annotation.Nullable; -import android.app.AppOpsManager; -import android.attention.AttentionManagerInternal; -import android.content.ComponentName; -import android.content.ContentCaptureOptions; -import android.content.Context; -import android.content.Intent; -import android.content.PermissionChecker; -import android.hardware.soundtrigger.IRecognitionStatusCallback; -import android.hardware.soundtrigger.SoundTrigger; -import android.media.AudioFormat; -import android.media.AudioManagerInternal; -import android.media.permission.Identity; -import android.media.permission.PermissionUtil; -import android.os.Binder; -import android.os.Bundle; -import android.os.IBinder; -import android.os.IRemoteCallback; -import android.os.ParcelFileDescriptor; -import android.os.PersistableBundle; -import android.os.RemoteException; -import android.os.ServiceManager; -import android.os.SharedMemory; -import android.provider.DeviceConfig; -import android.service.voice.HotwordDetectedResult; -import android.service.voice.HotwordDetectionService; -import android.service.voice.HotwordDetector; -import android.service.voice.HotwordRejectedResult; -import android.service.voice.IDspHotwordDetectionCallback; -import android.service.voice.IHotwordDetectionService; -import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; -import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity; -import android.speech.IRecognitionServiceManager; -import android.text.TextUtils; -import android.util.Pair; -import android.util.Slog; -import android.view.contentcapture.IContentCaptureManager; - -import com.android.internal.annotations.GuardedBy; -import com.android.internal.app.IHotwordRecognitionStatusCallback; -import com.android.internal.infra.AndroidFuture; -import com.android.internal.infra.ServiceConnector; -import com.android.server.LocalServices; -import com.android.server.pm.permission.PermissionManagerServiceInternal; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; - -/** - * A class that provides the communication with the HotwordDetectionService. - */ -final class TrustedHotwordDetectorSession { - private static final String TAG = "TrustedHotwordDetectorSession"; - static final boolean DEBUG = false; - - private static final String KEY_RESTART_PERIOD_IN_SECONDS = "restart_period_in_seconds"; - // TODO: These constants need to be refined. - // The validation timeout value is 3 seconds for onDetect of DSP trigger event. - private static final long VALIDATION_TIMEOUT_MILLIS = 3000; - // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. - private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; - private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000; - private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; - private static final Duration MAX_UPDATE_TIMEOUT_DURATION = - Duration.ofMillis(MAX_UPDATE_TIMEOUT_MILLIS); - private static final long RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS = 60 * 60 * 1000; // 1 hour - private static final int MAX_ISOLATED_PROCESS_NUMBER = 10; - - // The error codes are used for onError callback - private static final int HOTWORD_DETECTION_SERVICE_DIED = -1; - private static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2; - private static final int CALLBACK_DETECT_TIMEOUT = -3; - private static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; - - // Hotword metrics - private static final int METRICS_INIT_UNKNOWN_TIMEOUT = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; - private static final int METRICS_INIT_UNKNOWN_NO_VALUE = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; - private static final int METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; - private static final int METRICS_INIT_CALLBACK_STATE_ERROR = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; - private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; - - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; - private static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; - - private static final int METRICS_EXTERNAL_SOURCE_DETECTED = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; - private static final int METRICS_EXTERNAL_SOURCE_REJECTED = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; - private static final int METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; - private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION = - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; - - private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); - // TODO: This may need to be a Handler(looper) - private final ScheduledExecutorService mScheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(); - private final AppOpsManager mAppOpsManager; - private final HotwordAudioStreamCopier mHotwordAudioStreamCopier; - @Nullable private final ScheduledFuture<?> mCancellationTaskFuture; - private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); - private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied; - private final @NonNull ServiceConnectionFactory mServiceConnectionFactory; - private final IHotwordRecognitionStatusCallback mCallback; - private final int mDetectorType; - /** - * Time after which each HotwordDetectionService process is stopped and replaced by a new one. - * 0 indicates no restarts. - */ - private final int mReStartPeriodSeconds; - - final Object mLock; - final int mVoiceInteractionServiceUid; - final ComponentName mDetectionComponentName; - final int mUser; - final Context mContext; - - @Nullable AttentionManagerInternal mAttentionManagerInternal = null; - - final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = - this::setProximityValue; - - - volatile HotwordDetectionServiceIdentity mIdentity; - private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; - private Instant mLastRestartInstant; - - private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture; - private ScheduledFuture<?> mDebugHotwordLoggingTimeoutFuture = null; - - /** Identity used for attributing app ops when delivering data to the Interactor. */ - @GuardedBy("mLock") - @Nullable - private final Identity mVoiceInteractorIdentity; - @GuardedBy("mLock") - private ParcelFileDescriptor mCurrentAudioSink; - @GuardedBy("mLock") - private boolean mValidatingDspTrigger = false; - @GuardedBy("mLock") - private boolean mPerformingSoftwareHotwordDetection; - private @NonNull ServiceConnection mRemoteHotwordDetectionService; - private IBinder mAudioFlinger; - private boolean mDebugHotwordLogging = false; - @GuardedBy("mLock") - private double mProximityMeters = PROXIMITY_UNKNOWN; - - TrustedHotwordDetectorSession(Object lock, Context context, int voiceInteractionServiceUid, - Identity voiceInteractorIdentity, ComponentName serviceName, int userId, - boolean bindInstantServiceAllowed, @Nullable PersistableBundle options, - @Nullable SharedMemory sharedMemory, - @NonNull IHotwordRecognitionStatusCallback callback, int detectorType) { - if (callback == null) { - Slog.w(TAG, "Callback is null while creating connection"); - throw new IllegalArgumentException("Callback is null while creating connection"); - } - mLock = lock; - mContext = context; - mVoiceInteractionServiceUid = voiceInteractionServiceUid; - mVoiceInteractorIdentity = voiceInteractorIdentity; - mAppOpsManager = mContext.getSystemService(AppOpsManager.class); - mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, detectorType, - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag); - mDetectionComponentName = serviceName; - mUser = userId; - mCallback = callback; - mDetectorType = detectorType; - mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION, - KEY_RESTART_PERIOD_IN_SECONDS, 0); - final Intent intent = new Intent(HotwordDetectionService.SERVICE_INTERFACE); - intent.setComponent(mDetectionComponentName); - initAudioFlingerLocked(); - - mServiceConnectionFactory = new ServiceConnectionFactory(intent, bindInstantServiceAllowed); - - mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); - if (ENABLE_PROXIMITY_RESULT) { - mAttentionManagerInternal = LocalServices.getService(AttentionManagerInternal.class); - if (mAttentionManagerInternal != null) { - mAttentionManagerInternal.onStartProximityUpdates(mProximityCallbackInternal); - } - } - - mLastRestartInstant = Instant.now(); - updateStateAfterProcessStart(options, sharedMemory); - - if (mReStartPeriodSeconds <= 0) { - mCancellationTaskFuture = null; - } else { - // TODO: we need to be smarter here, e.g. schedule it a bit more often, - // but wait until the current session is closed. - mCancellationTaskFuture = mScheduledExecutorService.scheduleAtFixedRate(() -> { - Slog.v(TAG, "Time to restart the process, TTL has passed"); - synchronized (mLock) { - restartProcessLocked(); - HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, - HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE); - } - }, mReStartPeriodSeconds, mReStartPeriodSeconds, TimeUnit.SECONDS); - } - } - - private void initAudioFlingerLocked() { - if (DEBUG) { - Slog.d(TAG, "initAudioFlingerLocked"); - } - mAudioFlinger = ServiceManager.waitForService("media.audio_flinger"); - if (mAudioFlinger == null) { - throw new IllegalStateException("Service media.audio_flinger wasn't found."); - } - if (DEBUG) { - Slog.d(TAG, "Obtained audio_flinger binder."); - } - try { - mAudioFlinger.linkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); - } catch (RemoteException e) { - Slog.w(TAG, "Audio server died before we registered a DeathRecipient; retrying init.", - e); - initAudioFlingerLocked(); - } - } - - private void audioServerDied() { - Slog.w(TAG, "Audio server died; restarting the HotwordDetectionService."); - synchronized (mLock) { - // TODO: Check if this needs to be scheduled on a different thread. - initAudioFlingerLocked(); - // We restart the process instead of simply sending over the new binder, to avoid race - // conditions with audio reading in the service. - restartProcessLocked(); - HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, - HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED); - } - } - - private void updateStateAfterProcessStart( - PersistableBundle options, SharedMemory sharedMemory) { - if (DEBUG) { - Slog.d(TAG, "updateStateAfterProcessStart"); - } - mRemoteHotwordDetectionService.postAsync(service -> { - AndroidFuture<Void> future = new AndroidFuture<>(); - IRemoteCallback statusCallback = new IRemoteCallback.Stub() { - @Override - public void sendResult(Bundle bundle) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "updateState finish"); - } - future.complete(null); - if (mUpdateStateAfterStartFinished.getAndSet(true)) { - Slog.w(TAG, "call callback after timeout"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, - mVoiceInteractionServiceUid); - return; - } - Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle); - int status = statusResultPair.first; - int initResultMetricsResult = statusResultPair.second; - try { - mCallback.onStatusReported(status); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, - initResultMetricsResult); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report initialization status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - } - }; - try { - service.updateState(options, sharedMemory, statusCallback); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); - } catch (RemoteException e) { - // TODO: (b/181842909) Report an error to voice interactor - Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, - mVoiceInteractionServiceUid); - } - return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); - }).whenComplete((res, err) -> { - if (err instanceof TimeoutException) { - Slog.w(TAG, "updateState timed out"); - if (mUpdateStateAfterStartFinished.getAndSet(true)) { - return; - } - try { - mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, - METRICS_INIT_UNKNOWN_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - } else if (err != null) { - Slog.w(TAG, "Failed to update state: " + err); - } else { - // NOTE: so far we don't need to take any action. - } - }); - } - - private static Pair<Integer, Integer> getInitStatusAndMetricsResult(Bundle bundle) { - if (bundle == null) { - return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, METRICS_INIT_UNKNOWN_NO_VALUE); - } - int status = bundle.getInt(KEY_INITIALIZATION_STATUS, INITIALIZATION_STATUS_UNKNOWN); - if (status > HotwordDetectionService.getMaxCustomInitializationStatus()) { - return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, - status == INITIALIZATION_STATUS_UNKNOWN - ? METRICS_INIT_UNKNOWN_NO_VALUE - : METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE); - } - // TODO: should guard against negative here - int metricsResult = status == INITIALIZATION_STATUS_SUCCESS - ? METRICS_INIT_CALLBACK_STATE_SUCCESS - : METRICS_INIT_CALLBACK_STATE_ERROR; - return new Pair<>(status, metricsResult); - } - - private boolean isBound() { - synchronized (mLock) { - return mRemoteHotwordDetectionService.isBound(); - } - } - - void cancelLocked() { - Slog.v(TAG, "cancelLocked"); - clearDebugHotwordLoggingTimeoutLocked(); - mDebugHotwordLogging = false; - mRemoteHotwordDetectionService.unbind(); - LocalServices.getService(PermissionManagerServiceInternal.class) - .setHotwordDetectionServiceProvider(null); - if (mIdentity != null) { - removeServiceUidForAudioPolicy(mIdentity.getIsolatedUid()); - } - mIdentity = null; - if (mCancellationTaskFuture != null) { - mCancellationTaskFuture.cancel(/* may interrupt */ true); - } - if (mAudioFlinger != null) { - mAudioFlinger.unlinkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); - } - if (mAttentionManagerInternal != null) { - mAttentionManagerInternal.onStopProximityUpdates(mProximityCallbackInternal); - } - } - - void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); - - // Prevent doing the init late, so restart is handled equally to a clean process start. - // TODO(b/191742511): this logic needs a test - if (!mUpdateStateAfterStartFinished.get() - && Instant.now().minus(MAX_UPDATE_TIMEOUT_DURATION).isBefore(mLastRestartInstant)) { - Slog.v(TAG, "call updateStateAfterProcessStart"); - updateStateAfterProcessStart(options, sharedMemory); - } else { - mRemoteHotwordDetectionService.run( - service -> service.updateState(options, sharedMemory, null /* callback */)); - } - } - - void startListeningFromMic( - AudioFormat audioFormat, - IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { - if (DEBUG) { - Slog.d(TAG, "startListeningFromMic"); - } - mSoftwareCallback = callback; - - synchronized (mLock) { - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword validation is already in progress, ignoring."); - return; - } - mPerformingSoftwareHotwordDetection = true; - - startListeningFromMicLocked(); - } - } - - private void startListeningFromMicLocked() { - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection has already completed"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mPerformingSoftwareHotwordDetection = false; - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - mSoftwareCallback.onError(); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - mSoftwareCallback.onError(); - return; - } - mSoftwareCallback.onDetected(newResult, null, null); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.wtf(TAG, "onRejected"); - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - // onRejected isn't allowed here, and we are not expecting it. - } - }; - - mRemoteHotwordDetectionService.run( - service -> service.detectFromMicrophoneSource( - null, - AUDIO_SOURCE_MICROPHONE, - null, - null, - internalCallback)); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, - mVoiceInteractionServiceUid); - } - - public void startListeningFromExternalSource( - ParcelFileDescriptor audioStream, - AudioFormat audioFormat, - @Nullable PersistableBundle options, - IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { - if (DEBUG) { - Slog.d(TAG, "startListeningFromExternalSource"); - } - - handleExternalSourceHotwordDetection( - audioStream, - audioFormat, - options, - callback); - } - - void stopListening() { - if (DEBUG) { - Slog.d(TAG, "stopListening"); - } - synchronized (mLock) { - stopListeningLocked(); - } - } - - private void stopListeningLocked() { - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection is not running"); - return; - } - mPerformingSoftwareHotwordDetection = false; - - mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested"); - bestEffortClose(mCurrentAudioSink); - } - mCurrentAudioSink = null; - } - - void triggerHardwareRecognitionEventForTestLocked( - SoundTrigger.KeyphraseRecognitionEvent event, - IHotwordRecognitionStatusCallback callback) { - if (DEBUG) { - Slog.d(TAG, "triggerHardwareRecognitionEventForTestLocked"); - } - detectFromDspSource(event, callback); - } - - private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, - IHotwordRecognitionStatusCallback externalCallback) { - if (DEBUG) { - Slog.d(TAG, "detectFromDspSource"); - } - - AtomicBoolean timeoutDetected = new AtomicBoolean(false); - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onDetected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - try { - enforcePermissionsForDataDelivery(); - enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); - } catch (SecurityException e) { - Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); - return; - } - externalCallback.onKeyphraseDetected(recognitionEvent, newResult); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onRejected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onRejected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - externalCallback.onRejected(result); - if (mDebugHotwordLogging && result != null) { - Slog.i(TAG, "Egressed rejected result: " + result); - } - } - } - }; - - synchronized (mLock) { - mValidatingDspTrigger = true; - mRemoteHotwordDetectionService.run(service -> { - // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke - // the callback before timeout value. In order to reduce the latency impact between - // server side and client side, we need to use another timeout value - // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. - mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( - () -> { - // TODO: avoid allocate every time - timeoutDetected.set(true); - Slog.w(TAG, "Timed out on #detectFromDspSource"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); - try { - externalCallback.onError(CALLBACK_DETECT_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: ", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - }, - MAX_VALIDATION_TIMEOUT_MILLIS, - TimeUnit.MILLISECONDS); - service.detectFromDspSource( - recognitionEvent, - recognitionEvent.getCaptureFormat(), - VALIDATION_TIMEOUT_MILLIS, - internalCallback); - }); - } - } - - void forceRestart() { - Slog.v(TAG, "Requested to restart the service internally. Performing the restart"); - synchronized (mLock) { - restartProcessLocked(); - } - } - - void setDebugHotwordLoggingLocked(boolean logging) { - Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); - clearDebugHotwordLoggingTimeoutLocked(); - mDebugHotwordLogging = logging; - - if (logging) { - // Reset mDebugHotwordLogging to false after one hour - mDebugHotwordLoggingTimeoutFuture = mScheduledExecutorService.schedule(() -> { - Slog.v(TAG, "Timeout to reset mDebugHotwordLogging to false"); - synchronized (mLock) { - mDebugHotwordLogging = false; - } - }, RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); - } - } - - private void clearDebugHotwordLoggingTimeoutLocked() { - if (mDebugHotwordLoggingTimeoutFuture != null) { - mDebugHotwordLoggingTimeoutFuture.cancel(/* mayInterruptIfRunning= */true); - mDebugHotwordLoggingTimeoutFuture = null; - } - } - - private void restartProcessLocked() { - // TODO(b/244598068): Check HotwordAudioStreamManager first - Slog.v(TAG, "Restarting hotword detection process"); - ServiceConnection oldConnection = mRemoteHotwordDetectionService; - HotwordDetectionServiceIdentity previousIdentity = mIdentity; - - // TODO(volnov): this can be done after connect() has been successful. - if (mValidatingDspTrigger) { - // We're restarting the process while it's processing a DSP trigger, so report a - // rejection. This also allows the Interactor to startReco again - try { - mCallback.onRejected(new HotwordRejectedResult.Builder().build()); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to call #rejected"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - mValidatingDspTrigger = false; - } - - mUpdateStateAfterStartFinished.set(false); - mLastRestartInstant = Instant.now(); - - // Recreate connection to reset the cache. - mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); - - Slog.v(TAG, "Started the new process, issuing #onProcessRestarted"); - try { - mCallback.onProcessRestarted(); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - - // Restart listening from microphone if the hotword process has been restarted. - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Process restarted: calling startRecognition() again"); - startListeningFromMicLocked(); - } - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted"); - bestEffortClose(mCurrentAudioSink); - mCurrentAudioSink = null; - } - - if (DEBUG) { - Slog.i(TAG, "#onProcessRestarted called, unbinding from the old process"); - } - oldConnection.ignoreConnectionStatusEvents(); - oldConnection.unbind(); - if (previousIdentity != null) { - removeServiceUidForAudioPolicy(previousIdentity.getIsolatedUid()); - } - } - - static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub { - private SoundTrigger.KeyphraseRecognitionEvent mRecognitionEvent; - private final HotwordDetectionConnection mHotwordDetectionConnection; - private final IHotwordRecognitionStatusCallback mExternalCallback; - - SoundTriggerCallback(IHotwordRecognitionStatusCallback callback, - HotwordDetectionConnection connection) { - mHotwordDetectionConnection = connection; - mExternalCallback = callback; - } - - @Override - public void onKeyphraseDetected(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) - throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onKeyphraseDetected recognitionEvent : " + recognitionEvent); - } - final boolean useHotwordDetectionService = mHotwordDetectionConnection != null; - if (useHotwordDetectionService) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER); - mRecognitionEvent = recognitionEvent; - mHotwordDetectionConnection.detectFromDspSource( - recognitionEvent, mExternalCallback); - } else { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__NORMAL_DETECTOR, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER); - mExternalCallback.onKeyphraseDetected(recognitionEvent, null); - } - } - - @Override - public void onGenericSoundTriggerDetected( - SoundTrigger.GenericRecognitionEvent recognitionEvent) - throws RemoteException { - mExternalCallback.onGenericSoundTriggerDetected(recognitionEvent); - } - - @Override - public void onError(int status) throws RemoteException { - mExternalCallback.onError(status); - } - - @Override - public void onRecognitionPaused() throws RemoteException { - mExternalCallback.onRecognitionPaused(); - } - - @Override - public void onRecognitionResumed() throws RemoteException { - mExternalCallback.onRecognitionResumed(); - } - } - - public void dump(String prefix, PrintWriter pw) { - pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); - pw.print(prefix); - pw.print("mBound=" + mRemoteHotwordDetectionService.isBound()); - pw.print(", mValidatingDspTrigger=" + mValidatingDspTrigger); - pw.print(", mPerformingSoftwareHotwordDetection=" + mPerformingSoftwareHotwordDetection); - pw.print(", mRestartCount=" + mServiceConnectionFactory.mRestartCount); - pw.print(", mLastRestartInstant=" + mLastRestartInstant); - pw.println(", mDetectorType=" + HotwordDetector.detectorTypeToString(mDetectorType)); - } - - private void handleExternalSourceHotwordDetection( - ParcelFileDescriptor audioStream, - AudioFormat audioFormat, - @Nullable PersistableBundle options, - IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { - if (DEBUG) { - Slog.d(TAG, "#handleExternalSourceHotwordDetection"); - } - InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); - - Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe(); - if (clientPipe == null) { - // TODO: Need to propagate as unknown error or something? - return; - } - ParcelFileDescriptor serviceAudioSink = clientPipe.second; - ParcelFileDescriptor serviceAudioSource = clientPipe.first; - - synchronized (mLock) { - mCurrentAudioSink = serviceAudioSink; - } - - mAudioCopyExecutor.execute(() -> { - try (InputStream source = audioSource; - OutputStream fos = - new ParcelFileDescriptor.AutoCloseOutputStream(serviceAudioSink)) { - - byte[] buffer = new byte[1024]; - while (true) { - int bytesRead = source.read(buffer, 0, 1024); - - if (bytesRead < 0) { - Slog.i(TAG, "Reached end of stream for external hotword"); - break; - } - - // TODO: First write to ring buffer to make sure we don't lose data if the next - // statement fails. - // ringBuffer.append(buffer, bytesRead); - fos.write(buffer, 0, bytesRead); - } - } catch (IOException e) { - Slog.w(TAG, "Failed supplying audio data to validator", e); - - try { - callback.onError(); - } catch (RemoteException ex) { - Slog.w(TAG, "Failed to report onError status: " + ex); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - } finally { - synchronized (mLock) { - mCurrentAudioSink = null; - } - } - }); - - // TODO: handle cancellations well - // TODO: what if we cancelled and started a new one? - mRemoteHotwordDetectionService.run( - service -> { - service.detectFromMicrophoneSource( - serviceAudioSource, - // TODO: consider making a proxy callback + copy of audio format - AUDIO_SOURCE_EXTERNAL, - audioFormat, - options, - new IDspHotwordDetectionCallback.Stub() { - @Override - public void onRejected(HotwordRejectedResult result) - throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_REJECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - callback.onRejected(result); - - if (result != null) { - Slog.i(TAG, "Egressed 'hotword rejected result' " - + "from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + result); - } - } - } - - @Override - public void onDetected(HotwordDetectedResult triggerResult) - throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_DETECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, - mVoiceInteractionServiceUid); - callback.onError(); - return; - } - HotwordDetectedResult newResult; - try { - newResult = - mHotwordAudioStreamCopier.startCopyingAudioStreams( - triggerResult); - } catch (IOException e) { - // TODO: Write event - callback.onError(); - return; - } - callback.onDetected(newResult, null /* audioFormat */, - null /* audioStream */); - Slog.i(TAG, "Egressed " - + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, - "Egressed detected result: " + newResult); - } - } - }); - - // A copy of this has been created and passed to the hotword validator - bestEffortClose(serviceAudioSource); - }); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION, - mVoiceInteractionServiceUid); - } - - private class ServiceConnectionFactory { - private final Intent mIntent; - private final int mBindingFlags; - - private int mRestartCount = 0; - - ServiceConnectionFactory(@NonNull Intent intent, boolean bindInstantServiceAllowed) { - mIntent = intent; - mBindingFlags = bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0; - } - - ServiceConnection createLocked() { - ServiceConnection connection = - new ServiceConnection(mContext, mIntent, mBindingFlags, mUser, - IHotwordDetectionService.Stub::asInterface, - mRestartCount++ % MAX_ISOLATED_PROCESS_NUMBER); - connection.connect(); - - updateAudioFlinger(connection, mAudioFlinger); - updateContentCaptureManager(connection); - updateSpeechService(connection); - updateServiceIdentity(connection); - return connection; - } - } - - private class ServiceConnection extends ServiceConnector.Impl<IHotwordDetectionService> { - private final Object mLock = new Object(); - - private final Intent mIntent; - private final int mBindingFlags; - private final int mInstanceNumber; - - private boolean mRespectServiceConnectionStatusChanged = true; - private boolean mIsBound = false; - private boolean mIsLoggedFirstConnect = false; - - ServiceConnection(@NonNull Context context, - @NonNull Intent intent, int bindingFlags, int userId, - @Nullable Function<IBinder, IHotwordDetectionService> binderAsInterface, - int instanceNumber) { - super(context, intent, bindingFlags, userId, binderAsInterface); - this.mIntent = intent; - this.mBindingFlags = bindingFlags; - this.mInstanceNumber = instanceNumber; - } - - @Override // from ServiceConnector.Impl - protected void onServiceConnectionStatusChanged(IHotwordDetectionService service, - boolean connected) { - if (DEBUG) { - Slog.d(TAG, "onServiceConnectionStatusChanged connected = " + connected); - } - synchronized (mLock) { - if (!mRespectServiceConnectionStatusChanged) { - Slog.v(TAG, "Ignored onServiceConnectionStatusChanged event"); - return; - } - mIsBound = connected; - - if (!connected) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED, - mVoiceInteractionServiceUid); - } else if (!mIsLoggedFirstConnect) { - mIsLoggedFirstConnect = true; - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED, - mVoiceInteractionServiceUid); - } - } - } - - @Override - protected long getAutoDisconnectTimeoutMs() { - return -1; - } - - @Override - public void binderDied() { - super.binderDied(); - synchronized (mLock) { - if (!mRespectServiceConnectionStatusChanged) { - Slog.v(TAG, "Ignored #binderDied event"); - return; - } - - Slog.w(TAG, "binderDied"); - try { - mCallback.onError(HOTWORD_DETECTION_SERVICE_DIED); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - } - } - - @Override - protected boolean bindService( - @NonNull android.content.ServiceConnection serviceConnection) { - try { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE, - mVoiceInteractionServiceUid); - boolean bindResult = mContext.bindIsolatedService( - mIntent, - Context.BIND_AUTO_CREATE | Context.BIND_FOREGROUND_SERVICE | mBindingFlags, - "hotword_detector_" + mInstanceNumber, - mExecutor, - serviceConnection); - if (!bindResult) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, - mVoiceInteractionServiceUid); - } - return bindResult; - } catch (IllegalArgumentException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, - mVoiceInteractionServiceUid); - Slog.wtf(TAG, "Can't bind to the hotword detection service!", e); - return false; - } - } - - boolean isBound() { - synchronized (mLock) { - return mIsBound; - } - } - - void ignoreConnectionStatusEvents() { - synchronized (mLock) { - mRespectServiceConnectionStatusChanged = false; - } - } - } - - private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() { - ParcelFileDescriptor[] fileDescriptors; - try { - fileDescriptors = ParcelFileDescriptor.createPipe(); - } catch (IOException e) { - Slog.e(TAG, "Failed to create audio stream pipe", e); - return null; - } - - return Pair.create(fileDescriptors[0], fileDescriptors[1]); - } - - private static void updateAudioFlinger(ServiceConnection connection, IBinder audioFlinger) { - // TODO: Consider using a proxy that limits the exposed API surface. - connection.run(service -> service.updateAudioFlinger(audioFlinger)); - } - - private static void updateContentCaptureManager(ServiceConnection connection) { - IBinder b = ServiceManager - .getService(Context.CONTENT_CAPTURE_MANAGER_SERVICE); - IContentCaptureManager binderService = IContentCaptureManager.Stub.asInterface(b); - connection.run( - service -> service.updateContentCaptureManager(binderService, - new ContentCaptureOptions(null))); - } - - private static void updateSpeechService(ServiceConnection connection) { - IBinder b = ServiceManager.getService(Context.SPEECH_RECOGNITION_SERVICE); - IRecognitionServiceManager binderService = IRecognitionServiceManager.Stub.asInterface(b); - connection.run(service -> { - service.updateRecognitionServiceManager(binderService); - }); - } - - private void updateServiceIdentity(ServiceConnection connection) { - connection.run(service -> service.ping(new IRemoteCallback.Stub() { - @Override - public void sendResult(Bundle bundle) throws RemoteException { - // TODO: Exit if the service has been unbound already (though there's a very low - // chance this happens). - if (DEBUG) { - Slog.d(TAG, "updating hotword UID " + Binder.getCallingUid()); - } - // TODO: Have the provider point to the current state stored in - // VoiceInteractionManagerServiceImpl. - final int uid = Binder.getCallingUid(); - LocalServices.getService(PermissionManagerServiceInternal.class) - .setHotwordDetectionServiceProvider(() -> uid); - mIdentity = new HotwordDetectionServiceIdentity(uid, mVoiceInteractionServiceUid); - addServiceUidForAudioPolicy(uid); - } - })); - } - - private void addServiceUidForAudioPolicy(int uid) { - mScheduledExecutorService.execute(() -> { - AudioManagerInternal audioManager = - LocalServices.getService(AudioManagerInternal.class); - if (audioManager != null) { - audioManager.addAssistantServiceUid(uid); - } - }); - } - - private void removeServiceUidForAudioPolicy(int uid) { - mScheduledExecutorService.execute(() -> { - AudioManagerInternal audioManager = - LocalServices.getService(AudioManagerInternal.class); - if (audioManager != null) { - audioManager.removeAssistantServiceUid(uid); - } - }); - } - - private void saveProximityValueToBundle(HotwordDetectedResult result) { - synchronized (mLock) { - if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { - result.setProximity(mProximityMeters); - } - } - } - - private void setProximityValue(double proximityMeters) { - synchronized (mLock) { - mProximityMeters = proximityMeters; - } - } - - private static void bestEffortClose(Closeable... closeables) { - for (Closeable closeable : closeables) { - bestEffortClose(closeable); - } - } - - private static void bestEffortClose(Closeable closeable) { - try { - closeable.close(); - } catch (IOException e) { - if (DEBUG) { - Slog.w(TAG, "Failed closing", e); - } - } - } - - // TODO: Share this code with SoundTriggerMiddlewarePermission. - private void enforcePermissionsForDataDelivery() { - Binder.withCleanCallingIdentity(() -> { - enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); - int hotwordOp = AppOpsManager.strOpToOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD); - mAppOpsManager.noteOpNoThrow(hotwordOp, - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag, OP_MESSAGE); - enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, - CAPTURE_AUDIO_HOTWORD, OP_MESSAGE); - }); - } - - /** - * Throws a {@link SecurityException} iff the given identity has given permission to receive - * data. - * - * @param context A {@link Context}, used for permission checks. - * @param identity The identity to check. - * @param permission The identifier of the permission we want to check. - * @param reason The reason why we're requesting the permission, for auditing purposes. - */ - private static void enforcePermissionForDataDelivery(@NonNull Context context, - @NonNull Identity identity, - @NonNull String permission, @NonNull String reason) { - final int status = PermissionUtil.checkPermissionForDataDelivery(context, identity, - permission, reason); - if (status != PermissionChecker.PERMISSION_GRANTED) { - throw new SecurityException( - TextUtils.formatSimple("Failed to obtain permission %s for identity %s", - permission, - SoundTriggerSessionPermissionsDecorator.toString(identity))); - } - } - - private static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, - SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { - // verify the phrase ID in HotwordDetectedResult is not exposing extra phrases - // the DSP did not detect - for (SoundTrigger.KeyphraseRecognitionExtra keyphrase : recognitionEvent.keyphraseExtras) { - if (keyphrase.getKeyphraseId() == result.getHotwordPhraseId()) { - return; - } - } - throw new SecurityException("Ignoring #onDetected due to trusted service " - + "sharing a keyphrase ID which the DSP did not detect"); - } - - private static final String OP_MESSAGE = - "Providing hotword detection result to VoiceInteractionService"; -} |