summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
author Atneya Nair <atneya@google.com> 2023-03-01 15:08:26 -0800
committer Atneya Nair <atneya@google.com> 2023-04-06 21:54:37 -0700
commit31b112febfca30acad744e0a3b364b12eaac48d9 (patch)
treeb762e7302057ad6fa55820478b415a2147033296
parent9407076fb95551732421e7bc79cc4967402aef8c (diff)
ST HAL Instrumentation client side
- Add model params to TestApi - Add RecognitionConfig to TestApi - Add client-side STInstrumentation attach method to STManager - STInstrumentation implements ISoundTriggerInjection, dispatches appropriate callbacks and handles session staleness. Test: atest CtsVoiceInteractionTestCases Fixes: 271198164 Change-Id: Iaaaf3c0199980bb5f03b70f2b38f4f44d795666c
-rw-r--r--core/api/test-current.txt69
-rw-r--r--core/java/android/hardware/soundtrigger/SoundTrigger.java36
-rw-r--r--core/java/com/android/internal/app/ISoundTriggerService.aidl7
-rw-r--r--media/java/android/media/soundtrigger/SoundTriggerInstrumentation.java630
-rw-r--r--media/java/android/media/soundtrigger/SoundTriggerManager.java25
-rw-r--r--services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java25
6 files changed, 782 insertions, 10 deletions
diff --git a/core/api/test-current.txt b/core/api/test-current.txt
index 87835974b5d6..e57493009ea0 100644
--- a/core/api/test-current.txt
+++ b/core/api/test-current.txt
@@ -1651,6 +1651,11 @@ package android.hardware.soundtrigger {
field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.KeyphraseMetadata> CREATOR;
}
+ public class SoundTrigger {
+ field public static final int MODEL_PARAM_INVALID = -1; // 0xffffffff
+ field public static final int MODEL_PARAM_THRESHOLD_FACTOR = 0; // 0x0
+ }
+
public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable {
ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int);
}
@@ -1663,6 +1668,19 @@ package android.hardware.soundtrigger {
ctor public SoundTrigger.ModuleProperties(int, @NonNull String, @NonNull String, @NonNull String, int, @NonNull String, int, int, int, int, boolean, int, boolean, int, boolean, int);
}
+ public static final class SoundTrigger.RecognitionConfig implements android.os.Parcelable {
+ ctor public SoundTrigger.RecognitionConfig(boolean, boolean, @Nullable android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[], @Nullable byte[], int);
+ ctor public SoundTrigger.RecognitionConfig(boolean, boolean, @Nullable android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[], @Nullable byte[]);
+ method public int describeContents();
+ method public void writeToParcel(@NonNull android.os.Parcel, int);
+ field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.SoundTrigger.RecognitionConfig> CREATOR;
+ field public final boolean allowMultipleTriggers;
+ field public final int audioCapabilities;
+ field public final boolean captureRequested;
+ field @NonNull public final byte[] data;
+ field @NonNull public final android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[] keyphrases;
+ }
+
public static class SoundTrigger.RecognitionEvent {
ctor public SoundTrigger.RecognitionEvent(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[], long);
}
@@ -2000,6 +2018,57 @@ package android.media.metrics {
}
+package android.media.soundtrigger {
+
+ public final class SoundTriggerInstrumentation {
+ method public void setResourceContention(boolean);
+ method public void triggerOnResourcesAvailable();
+ method public void triggerRestart();
+ }
+
+ public static interface SoundTriggerInstrumentation.GlobalCallback {
+ method public default void onClientAttached();
+ method public default void onClientDetached();
+ method public default void onFrameworkDetached();
+ method public void onModelLoaded(@NonNull android.media.soundtrigger.SoundTriggerInstrumentation.ModelSession);
+ method public default void onPreempted();
+ method public default void onRestarted();
+ }
+
+ public static interface SoundTriggerInstrumentation.ModelCallback {
+ method public default void onModelUnloaded();
+ method public default void onParamSet(int, int);
+ method public void onRecognitionStarted(@NonNull android.media.soundtrigger.SoundTriggerInstrumentation.RecognitionSession);
+ }
+
+ public class SoundTriggerInstrumentation.ModelSession {
+ method public void clearModelCallback();
+ method @NonNull public java.util.List<android.hardware.soundtrigger.SoundTrigger.Keyphrase> getPhrases();
+ method @NonNull public android.media.soundtrigger.SoundTriggerManager.Model getSoundModel();
+ method public boolean isKeyphrase();
+ method public void setModelCallback(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.ModelCallback);
+ method public void triggerUnloadModel();
+ }
+
+ public static interface SoundTriggerInstrumentation.RecognitionCallback {
+ method public void onRecognitionStopped();
+ }
+
+ public class SoundTriggerInstrumentation.RecognitionSession {
+ method public void clearRecognitionCallback();
+ method public int getAudioSession();
+ method @NonNull public android.hardware.soundtrigger.SoundTrigger.RecognitionConfig getRecognitionConfig();
+ method public void setRecognitionCallback(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.RecognitionCallback);
+ method public void triggerAbortRecognition();
+ method public void triggerRecognitionEvent(@NonNull byte[], @Nullable java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra>);
+ }
+
+ public final class SoundTriggerManager {
+ method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) public static android.media.soundtrigger.SoundTriggerInstrumentation attachInstrumentation(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.GlobalCallback);
+ }
+
+}
+
package android.media.tv {
public final class TvInputManager {
diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java
index fa16e167f7d1..6d43ddf7fe94 100644
--- a/core/java/android/hardware/soundtrigger/SoundTrigger.java
+++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java
@@ -1051,6 +1051,29 @@ public class SoundTrigger {
return "ModelParamRange [start=" + mStart + ", end=" + mEnd + "]";
}
}
+ /**
+ * SoundTrigger model parameter types.
+ * @hide
+ */
+ @Retention(RetentionPolicy.SOURCE)
+ @IntDef(flag = true, prefix = { "MODEL_PARAM" }, value = {
+ MODEL_PARAM_INVALID,
+ MODEL_PARAM_THRESHOLD_FACTOR
+ })
+ public @interface ModelParamTypes {}
+
+ /**
+ * See {@link ModelParams.INVALID}
+ * @hide
+ */
+ @TestApi
+ public static final int MODEL_PARAM_INVALID = ModelParams.INVALID;
+ /**
+ * See {@link ModelParams.THRESHOLD_FACTOR}
+ * @hide
+ */
+ @TestApi
+ public static final int MODEL_PARAM_THRESHOLD_FACTOR = ModelParams.THRESHOLD_FACTOR;
/**
* Modes for key phrase recognition
@@ -1450,7 +1473,8 @@ public class SoundTrigger {
*
* @hide
*/
- public static class RecognitionConfig implements Parcelable {
+ @TestApi
+ public static final class RecognitionConfig implements Parcelable {
/** True if the DSP should capture the trigger sound and make it available for further
* capture. */
@UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
@@ -1464,6 +1488,7 @@ public class SoundTrigger {
* options for each keyphrase. */
@UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
@NonNull
+ @SuppressLint("ArrayReturn")
public final KeyphraseRecognitionExtra keyphrases[];
/** Opaque data for use by system applications who know about voice engine internals,
* typically during enrollment. */
@@ -1479,8 +1504,8 @@ public class SoundTrigger {
public final int audioCapabilities;
public RecognitionConfig(boolean captureRequested, boolean allowMultipleTriggers,
- @Nullable KeyphraseRecognitionExtra[] keyphrases, @Nullable byte[] data,
- int audioCapabilities) {
+ @SuppressLint("ArrayReturn") @Nullable KeyphraseRecognitionExtra[] keyphrases,
+ @Nullable byte[] data, int audioCapabilities) {
this.captureRequested = captureRequested;
this.allowMultipleTriggers = allowMultipleTriggers;
this.keyphrases = keyphrases != null ? keyphrases : new KeyphraseRecognitionExtra[0];
@@ -1490,7 +1515,8 @@ public class SoundTrigger {
@UnsupportedAppUsage
public RecognitionConfig(boolean captureRequested, boolean allowMultipleTriggers,
- @Nullable KeyphraseRecognitionExtra[] keyphrases, @Nullable byte[] data) {
+ @SuppressLint("ArrayReturn") @Nullable KeyphraseRecognitionExtra[] keyphrases,
+ @Nullable byte[] data) {
this(captureRequested, allowMultipleTriggers, keyphrases, data, 0);
}
@@ -1517,7 +1543,7 @@ public class SoundTrigger {
}
@Override
- public void writeToParcel(Parcel dest, int flags) {
+ public void writeToParcel(@NonNull Parcel dest, int flags) {
dest.writeByte((byte) (captureRequested ? 1 : 0));
dest.writeByte((byte) (allowMultipleTriggers ? 1 : 0));
dest.writeTypedArray(keyphrases, flags);
diff --git a/core/java/com/android/internal/app/ISoundTriggerService.aidl b/core/java/com/android/internal/app/ISoundTriggerService.aidl
index ab7f602e2dfc..ed751cb481c5 100644
--- a/core/java/com/android/internal/app/ISoundTriggerService.aidl
+++ b/core/java/com/android/internal/app/ISoundTriggerService.aidl
@@ -16,8 +16,9 @@
package com.android.internal.app;
-import android.media.permission.Identity;
import android.hardware.soundtrigger.SoundTrigger;
+import android.media.permission.Identity;
+import android.media.soundtrigger_middleware.ISoundTriggerInjection;
import com.android.internal.app.ISoundTriggerSession;
/**
@@ -74,4 +75,8 @@ interface ISoundTriggerService {
*/
List<SoundTrigger.ModuleProperties> listModuleProperties(in Identity originatorIdentity);
+ /**
+ * Attach an HAL injection interface.
+ */
+ void attachInjection(ISoundTriggerInjection injection);
}
diff --git a/media/java/android/media/soundtrigger/SoundTriggerInstrumentation.java b/media/java/android/media/soundtrigger/SoundTriggerInstrumentation.java
new file mode 100644
index 000000000000..80bc5c07dd66
--- /dev/null
+++ b/media/java/android/media/soundtrigger/SoundTriggerInstrumentation.java
@@ -0,0 +1,630 @@
+/**
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media.soundtrigger;
+
+import android.annotation.CallbackExecutor;
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.annotation.RequiresPermission;
+import android.annotation.TestApi;
+import android.hardware.soundtrigger.ConversionUtil;
+import android.hardware.soundtrigger.SoundTrigger;
+import android.media.soundtrigger_middleware.IAcknowledgeEvent;
+import android.media.soundtrigger_middleware.IInjectGlobalEvent;
+import android.media.soundtrigger_middleware.IInjectModelEvent;
+import android.media.soundtrigger_middleware.IInjectRecognitionEvent;
+import android.media.soundtrigger_middleware.ISoundTriggerInjection;
+import android.os.IBinder;
+import android.os.RemoteException;
+
+import com.android.internal.annotations.GuardedBy;
+import com.android.internal.app.ISoundTriggerService;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+
+/**
+ * Used to inject/observe events when using a fake SoundTrigger HAL for test purposes.
+ * Created by {@link SoundTriggerManager#getInjection(Executor, GlobalCallback)}.
+ * Only one instance of this class is valid at any given time, old instances will be delivered
+ * {@link GlobalCallback#onPreempted()}.
+ * @hide
+ */
+@TestApi
+public final class SoundTriggerInstrumentation {
+
+ private final Object mLock = new Object();
+ @GuardedBy("mLock")
+ private IInjectGlobalEvent mInjectGlobalEvent = null;
+
+ @GuardedBy("mLock")
+ private Map<IBinder, ModelSession> mModelSessionMap = new HashMap<>();
+ @GuardedBy("mLock")
+ private Map<IBinder, RecognitionSession> mRecognitionSessionMap = new HashMap<>();
+ @GuardedBy("mLock")
+ private IBinder mClientToken = null;
+
+ private final GlobalCallback mClientCallback;
+ private final Executor mGlobalCallbackExecutor;
+
+ /**
+ * Callback interface for un-sessioned events observed from the fake STHAL.
+ * Registered upon construction of {@link SoundTriggerInstrumentation}
+ * @hide
+ */
+ @TestApi
+ public interface GlobalCallback {
+ /**
+ * Called when the created {@link SoundTriggerInstrumentation} object is invalidated
+ * by another client creating an {@link SoundTriggerInstrumentation} to instrument the
+ * fake STHAL. Only one client may inject at a time.
+ * All sessions are invalidated, no further events will be received, and no
+ * injected events will be delivered.
+ */
+ default void onPreempted() {}
+ /**
+ * Called when the STHAL has been restarted by the framework, due to unexpected
+ * error conditions.
+ * Not called when {@link SoundTriggerInstrumentation#triggerRestart()} is injected.
+ */
+ default void onRestarted() {}
+ /**
+ * Called when the framework detaches from the fake HAL.
+ * This is not transmitted to real HALs, but it indicates that the
+ * framework has flushed its global state.
+ */
+ default void onFrameworkDetached() {}
+ /**
+ * Called when a client application attaches to the framework.
+ * This is not transmitted to real HALs, but it represents the state of
+ * the framework.
+ */
+ default void onClientAttached() {}
+ /**
+ * Called when a client application detaches from the framework.
+ * This is not transmitted to real HALs, but it represents the state of
+ * the framework.
+ */
+ default void onClientDetached() {}
+ /**
+ * Called when the fake HAL receives a model load from the framework.
+ * @param modelSession - A session which exposes additional injection
+ * functionality associated with the newly loaded
+ * model. See {@link ModelSession}.
+ */
+ void onModelLoaded(@NonNull ModelSession modelSession);
+ }
+
+ /**
+ * Callback for HAL events related to a loaded model. Register with
+ * {@link ModelSession#setModelCallback(Executor, ModelCallback)}
+ * Note, callbacks will not be delivered for events triggered by the injection.
+ * @hide
+ */
+ @TestApi
+ public interface ModelCallback {
+ /**
+ * Called when the model associated with the {@link ModelSession} this callback
+ * was registered for was unloaded by the framework.
+ */
+ default void onModelUnloaded() {}
+ /**
+ * Called when the model associated with the {@link ModelSession} this callback
+ * was registered for receives a set parameter call from the framework.
+ * @param param - Parameter being set.
+ * See {@link SoundTrigger.ModelParamTypes}
+ * @param value - Value the model parameter was set to.
+ */
+ default void onParamSet(@SoundTrigger.ModelParamTypes int param, int value) {}
+ /**
+ * Called when the model associated with the {@link ModelSession} this callback
+ * was registered for receives a recognition start request.
+ * @param recognitionSession - A session which exposes additional injection
+ * functionality associated with the newly started
+ * recognition. See {@link RecognitionSession}
+ */
+ void onRecognitionStarted(@NonNull RecognitionSession recognitionSession);
+ }
+
+ /**
+ * Callback for HAL events related to a started recognition. Register with
+ * {@link RecognitionSession#setRecognitionCallback(Executor, RecognitionCallback)}
+ * Note, callbacks will not be delivered for events triggered by the injection.
+ * @hide
+ */
+ @TestApi
+ public interface RecognitionCallback {
+ /**
+ * Called when the recognition associated with the {@link RecognitionSession} this
+ * callback was registered for was stopped by the framework.
+ */
+ void onRecognitionStopped();
+ }
+
+ /**
+ * Session associated with a loaded model in the fake STHAL.
+ * Can be used to query details about the loaded model, register a callback for future
+ * model events, or trigger HAL events associated with a loaded model.
+ * This session is invalid once the model is unloaded, caused by a
+ * {@link ModelSession#triggerUnloadModel()},
+ * the client unloading recognition, or if a {@link GlobalCallback#onRestarted()} is
+ * received.
+ * Further injections on an invalidated session will not be respected, and no future
+ * callbacks will be delivered.
+ * @hide
+ */
+ @TestApi
+ public class ModelSession {
+
+ /**
+ * Trigger the HAL to preemptively unload the model associated with this session.
+ * Typically occurs when a higher priority model is loaded which utilizes the same
+ * resources.
+ */
+ public void triggerUnloadModel() {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ try {
+ mInjectModelEvent.triggerUnloadModel();
+ } catch (RemoteException e) {
+ throw e.rethrowFromSystemServer();
+ }
+ mModelSessionMap.remove(mInjectModelEvent.asBinder());
+ }
+ }
+
+ /**
+ * Get the {@link SoundTriggerManager.Model} associated with this session.
+ * @return - The model associated with this session.
+ */
+ public @NonNull SoundTriggerManager.Model getSoundModel() {
+ return mModel;
+ }
+
+ /**
+ * Get the list of {@link SoundTrigger.Keyphrase} associated with this session.
+ * @return - The keyphrases associated with this session.
+ */
+ public @NonNull List<SoundTrigger.Keyphrase> getPhrases() {
+ if (mPhrases == null) {
+ return new ArrayList<>();
+ } else {
+ return new ArrayList<>(Arrays.asList(mPhrases));
+ }
+ }
+
+ /**
+ * Get whether this model is of keyphrase type.
+ * @return - true if the model is a keyphrase model, false otherwise
+ */
+ public boolean isKeyphrase() {
+ return (mPhrases != null);
+ }
+
+ /**
+ * Registers the model callback associated with this session. Events associated
+ * with this model session will be reported via this callback.
+ * See {@link ModelCallback}
+ * @param executor - Executor which the callback is dispatched on
+ * @param callback - Model callback for reporting model session events.
+ */
+ public void setModelCallback(@NonNull @CallbackExecutor Executor executor, @NonNull
+ ModelCallback callback) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ mModelCallback = Objects.requireNonNull(callback);
+ mModelExecutor = Objects.requireNonNull(executor);
+ }
+ }
+
+ /**
+ * Clear the model callback associated with this session, if any has been
+ * set by {@link #setModelCallback(Executor, ModelCallback)}.
+ */
+ public void clearModelCallback() {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ mModelCallback = null;
+ mModelExecutor = null;
+ }
+ }
+
+ private ModelSession(SoundModel model, Phrase[] phrases,
+ IInjectModelEvent injection) {
+ mModel = SoundTriggerManager.Model.create(UUID.fromString(model.uuid),
+ UUID.fromString(model.vendorUuid),
+ ConversionUtil.sharedMemoryToByteArray(model.data, model.dataSize));
+ if (phrases != null) {
+ mPhrases = new SoundTrigger.Keyphrase[phrases.length];
+ int i = 0;
+ for (var phrase : phrases) {
+ mPhrases[i++] = ConversionUtil.aidl2apiPhrase(phrase);
+ }
+ } else {
+ mPhrases = null;
+ }
+ mInjectModelEvent = injection;
+ }
+
+ private void wrap(Consumer<ModelCallback> consumer) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ if (mModelCallback != null && mModelExecutor != null) {
+ final ModelCallback callback = mModelCallback;
+ mModelExecutor.execute(() -> consumer.accept(callback));
+ }
+ }
+ }
+
+ private final SoundTriggerManager.Model mModel;
+ private final SoundTrigger.Keyphrase[] mPhrases;
+ private final IInjectModelEvent mInjectModelEvent;
+
+ @GuardedBy("SoundTriggerInstrumentation.this.mLock")
+ private ModelCallback mModelCallback = null;
+ @GuardedBy("SoundTriggerInstrumentation.this.mLock")
+ private Executor mModelExecutor = null;
+ }
+
+ /**
+ * Session associated with a recognition start in the fake STHAL.
+ * Can be used to get information about the started recognition, register a callback
+ * for future events associated with this recognition, and triggering
+ * recognition events or aborts.
+ * This session is invalid once the recognition is stopped, caused by a
+ * {@link RecognitionSession#triggerAbortRecognition()},
+ * {@link RecognitionSession#triggerRecognitionEvent(byte[], List)},
+ * the client stopping recognition, or any operation which invalidates the
+ * {@link ModelSession} which the session was created from.
+ * Further injections on an invalidated session will not be respected, and no future
+ * callbacks will be delivered.
+ * @hide
+ */
+ @TestApi
+ public class RecognitionSession {
+
+ /**
+ * Get an integer token representing the audio session associated with this
+ * recognition in the STHAL.
+ * @return - The session token.
+ */
+ public int getAudioSession() {
+ return mAudioSession;
+ }
+
+ /**
+ * Get the recognition config used to start this recognition.
+ * @return - The config passed to the HAL for startRecognition.
+ */
+ public @NonNull SoundTrigger.RecognitionConfig getRecognitionConfig() {
+ return mRecognitionConfig;
+ }
+
+ /**
+ * Trigger a recognition in the fake STHAL.
+ * @param data - The opaque data buffer included in the recognition event.
+ * @param phraseExtras - Keyphrase metadata included in the event. The
+ * event must include metadata for the keyphrase id
+ * associated with this model to be received by the
+ * client application.
+ */
+ public void triggerRecognitionEvent(@NonNull byte[] data, @Nullable
+ List<SoundTrigger.KeyphraseRecognitionExtra> phraseExtras) {
+ PhraseRecognitionExtra[] converted = null;
+ if (phraseExtras != null) {
+ converted = new PhraseRecognitionExtra[phraseExtras.size()];
+ int i = 0;
+ for (var phraseExtra : phraseExtras) {
+ converted[i++] = ConversionUtil.api2aidlPhraseRecognitionExtra(phraseExtra);
+ }
+ }
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ mRecognitionSessionMap.remove(mInjectRecognitionEvent.asBinder());
+ try {
+ mInjectRecognitionEvent.triggerRecognitionEvent(data, converted);
+ } catch (RemoteException e) {
+ throw e.rethrowFromSystemServer();
+ }
+ }
+ }
+
+ /**
+ * Trigger an abort recognition event in the fake HAL. This represents a
+ * preemptive ending of the recognition session by the HAL, despite no
+ * recognition detection. Typically occurs during contention for microphone
+ * usage, or if model limits are hit.
+ * See {@link SoundTriggerInstrumentation#setResourceContention(boolean)} to block
+ * subsequent downward calls for contention reasons.
+ */
+ public void triggerAbortRecognition() {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ mRecognitionSessionMap.remove(mInjectRecognitionEvent.asBinder());
+ try {
+ mInjectRecognitionEvent.triggerAbortRecognition();
+ } catch (RemoteException e) {
+ throw e.rethrowFromSystemServer();
+ }
+ }
+ }
+
+ /**
+ * Registers the recognition callback associated with this session. Events associated
+ * with this recognition session will be reported via this callback.
+ * See {@link RecognitionCallback}
+ * @param executor - Executor which the callback is dispatched on
+ * @param callback - Recognition callback for reporting recognition session events.
+ */
+ public void setRecognitionCallback(@NonNull @CallbackExecutor Executor executor,
+ @NonNull RecognitionCallback callback) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ mRecognitionCallback = callback;
+ mRecognitionExecutor = executor;
+ }
+ }
+
+ /**
+ * Clear the recognition callback associated with this session, if any has been
+ * set by {@link #setRecognitionCallback(Executor, RecognitionCallback)}.
+ */
+ public void clearRecognitionCallback() {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ mRecognitionCallback = null;
+ mRecognitionExecutor = null;
+ }
+ }
+
+ private RecognitionSession(int audioSession,
+ RecognitionConfig recognitionConfig,
+ IInjectRecognitionEvent injectRecognitionEvent) {
+ mAudioSession = audioSession;
+ mRecognitionConfig = ConversionUtil.aidl2apiRecognitionConfig(recognitionConfig);
+ mInjectRecognitionEvent = injectRecognitionEvent;
+ }
+
+ private void wrap(Consumer<RecognitionCallback> consumer) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ if (mRecognitionCallback != null && mRecognitionExecutor != null) {
+ final RecognitionCallback callback = mRecognitionCallback;
+ mRecognitionExecutor.execute(() -> consumer.accept(callback));
+ }
+ }
+ }
+
+ private final int mAudioSession;
+ private final SoundTrigger.RecognitionConfig mRecognitionConfig;
+ private final IInjectRecognitionEvent mInjectRecognitionEvent;
+
+ @GuardedBy("SoundTriggerInstrumentation.this.mLock")
+ private Executor mRecognitionExecutor = null;
+ @GuardedBy("SoundTriggerInstrumentation.this.mLock")
+ private RecognitionCallback mRecognitionCallback = null;
+ }
+
+ // Implementation of injection interface passed to the HAL.
+ // This class will re-associate events received on this callback interface
+ // with sessions, to avoid staleness issues.
+ private class Injection extends ISoundTriggerInjection.Stub {
+ @Override
+ public void registerGlobalEventInjection(IInjectGlobalEvent globalInjection) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ mInjectGlobalEvent = globalInjection;
+ }
+ }
+
+ @Override
+ public void onSoundModelLoaded(SoundModel model, @Nullable Phrase[] phrases,
+ IInjectModelEvent modelInjection, IInjectGlobalEvent globalSession) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return;
+ ModelSession modelSession = new ModelSession(model, phrases, modelInjection);
+ mModelSessionMap.put(modelInjection.asBinder(), modelSession);
+ mGlobalCallbackExecutor.execute(() -> mClientCallback.onModelLoaded(modelSession));
+ }
+ }
+
+ @Override
+ public void onSoundModelUnloaded(IInjectModelEvent modelSession) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ ModelSession clientModelSession = mModelSessionMap.remove(modelSession.asBinder());
+ if (clientModelSession == null) return;
+ clientModelSession.wrap((ModelCallback cb) -> cb.onModelUnloaded());
+ }
+ }
+
+ @Override
+ public void onRecognitionStarted(int audioSessionHandle, RecognitionConfig config,
+ IInjectRecognitionEvent recognitionInjection, IInjectModelEvent modelSession) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ ModelSession clientModelSession = mModelSessionMap.get(modelSession.asBinder());
+ if (clientModelSession == null) return;
+ RecognitionSession recogSession = new RecognitionSession(
+ audioSessionHandle, config, recognitionInjection);
+ mRecognitionSessionMap.put(recognitionInjection.asBinder(), recogSession);
+ clientModelSession.wrap((ModelCallback cb) ->
+ cb.onRecognitionStarted(recogSession));
+ }
+ }
+
+ @Override
+ public void onRecognitionStopped(IInjectRecognitionEvent recognitionSession) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ RecognitionSession clientRecognitionSession =
+ mRecognitionSessionMap.remove(recognitionSession.asBinder());
+ if (clientRecognitionSession == null) return;
+ clientRecognitionSession.wrap((RecognitionCallback cb)
+ -> cb.onRecognitionStopped());
+ }
+ }
+
+ @Override
+ public void onParamSet(int modelParam, int value, IInjectModelEvent modelSession) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ ModelSession clientModelSession = mModelSessionMap.get(modelSession.asBinder());
+ if (clientModelSession == null) return;
+ clientModelSession.wrap((ModelCallback cb) -> cb.onParamSet(modelParam, value));
+ }
+ }
+
+
+ @Override
+ public void onRestarted(IInjectGlobalEvent globalSession) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return;
+ mRecognitionSessionMap.clear();
+ mModelSessionMap.clear();
+ mGlobalCallbackExecutor.execute(() -> mClientCallback.onRestarted());
+ }
+ }
+
+ @Override
+ public void onFrameworkDetached(IInjectGlobalEvent globalSession) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return;
+ mGlobalCallbackExecutor.execute(() -> mClientCallback.onFrameworkDetached());
+ }
+ }
+
+ @Override
+ public void onClientAttached(IBinder token, IInjectGlobalEvent globalSession) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return;
+ mClientToken = token;
+ mGlobalCallbackExecutor.execute(() -> mClientCallback.onClientAttached());
+ }
+ }
+
+ @Override
+ public void onClientDetached(IBinder token) {
+ synchronized (SoundTriggerInstrumentation.this.mLock) {
+ if (token != mClientToken) return;
+ mClientToken = null;
+ mGlobalCallbackExecutor.execute(() -> mClientCallback.onClientDetached());
+ }
+ }
+
+ @Override
+ public void onPreempted() {
+ // This is always valid, independent of session
+ mGlobalCallbackExecutor.execute(() -> mClientCallback.onPreempted());
+ // Callbacks will no longer be delivered, and injection will be silently dropped.
+ }
+ }
+
+ /**
+ * @hide
+ */
+ @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER)
+ public SoundTriggerInstrumentation(ISoundTriggerService service,
+ @CallbackExecutor @NonNull Executor executor,
+ @NonNull GlobalCallback callback) {
+ mClientCallback = Objects.requireNonNull(callback);
+ mGlobalCallbackExecutor = Objects.requireNonNull(executor);
+ try {
+ service.attachInjection(new Injection());
+ } catch (RemoteException e) {
+ e.rethrowFromSystemServer();
+ }
+ }
+
+ /**
+ * Simulate a HAL restart, typically caused by the framework on an unexpected error,
+ * or a restart of the core audio HAL.
+ * Application sessions will be detached, and all state will be cleared. The framework
+ * will re-attach to the HAL following restart.
+ * @hide
+ */
+ @TestApi
+ public void triggerRestart() {
+ synchronized (mLock) {
+ if (mInjectGlobalEvent == null) {
+ throw new IllegalStateException(
+ "Attempted to trigger HAL restart before registration");
+ }
+ try {
+ mInjectGlobalEvent.triggerRestart();
+ } catch (RemoteException e) {
+ throw e.rethrowFromSystemServer();
+ }
+ }
+ }
+
+ /**
+ * Trigger a resource available callback from the fake SoundTrigger HAL to the framework.
+ * This callback notifies the framework that methods which previously failed due to
+ * resource contention may now succeed.
+ * @hide
+ */
+ @TestApi
+ public void triggerOnResourcesAvailable() {
+ synchronized (mLock) {
+ if (mInjectGlobalEvent == null) {
+ throw new IllegalStateException(
+ "Attempted to trigger HAL resources available before registration");
+ }
+ try {
+ mInjectGlobalEvent.triggerOnResourcesAvailable();
+ } catch (RemoteException e) {
+ throw e.rethrowFromSystemServer();
+ }
+ }
+ }
+
+ /**
+ * Simulate resource contention, similar to when HAL which does not
+ * support concurrent capture opens a capture stream, or when a HAL
+ * has reached its maximum number of models.
+ * Subsequent model loads and recognition starts will gracefully error.
+ * Since this call does not trigger a callback through the framework, the
+ * call will block until the fake HAL has acknowledged the state change.
+ * @param isResourceContended - true to enable contention, false to return
+ * to normal functioning.
+ * @hide
+ */
+ @TestApi
+ public void setResourceContention(boolean isResourceContended) {
+ synchronized (mLock) {
+ if (mInjectGlobalEvent == null) {
+ throw new IllegalStateException("Injection interface not set up");
+ }
+ IInjectGlobalEvent current = mInjectGlobalEvent;
+ final CountDownLatch signal = new CountDownLatch(1);
+ try {
+ current.setResourceContention(isResourceContended, new IAcknowledgeEvent.Stub() {
+ @Override
+ public void eventReceived() {
+ signal.countDown();
+ }
+ });
+ } catch (RemoteException e) {
+ throw e.rethrowFromSystemServer();
+ }
+
+ // Block until we get a callback from the service that our request was serviced.
+ try {
+ // Rely on test timeout if we don't get a response.
+ signal.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
+
diff --git a/media/java/android/media/soundtrigger/SoundTriggerManager.java b/media/java/android/media/soundtrigger/SoundTriggerManager.java
index ae8121a59abf..c41bd1bc3094 100644
--- a/media/java/android/media/soundtrigger/SoundTriggerManager.java
+++ b/media/java/android/media/soundtrigger/SoundTriggerManager.java
@@ -18,11 +18,13 @@ package android.media.soundtrigger;
import static android.hardware.soundtrigger.SoundTrigger.STATUS_ERROR;
+import android.annotation.CallbackExecutor;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SystemApi;
import android.annotation.SystemService;
+import android.annotation.TestApi;
import android.app.ActivityThread;
import android.compat.annotation.UnsupportedAppUsage;
import android.content.ComponentName;
@@ -45,6 +47,7 @@ import android.os.Handler;
import android.os.IBinder;
import android.os.ParcelUuid;
import android.os.RemoteException;
+import android.os.ServiceManager;
import android.provider.Settings;
import android.util.Slog;
@@ -53,9 +56,9 @@ import com.android.internal.app.ISoundTriggerSession;
import com.android.internal.util.Preconditions;
import java.util.HashMap;
-import java.util.List;
import java.util.Objects;
import java.util.UUID;
+import java.util.concurrent.Executor;
/**
* This class provides management of non-voice (general sound trigger) based sound recognition
@@ -609,4 +612,24 @@ public final class SoundTriggerManager {
throw e.rethrowFromSystemServer();
}
}
+
+ /**
+ * Create a {@link SoundTriggerInstrumentation} for test purposes, which instruments a fake
+ * STHAL. Clients must attach to the appropriate underlying ST module.
+ * @param executor - Executor to dispatch global callbacks on
+ * @param callback - Callback for unsessioned events received by the fake STHAL
+ * @return - A {@link SoundTriggerInstrumentation} for observation/injection.
+ * @hide
+ */
+ @TestApi
+ @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER)
+ @NonNull
+ public static SoundTriggerInstrumentation attachInstrumentation(
+ @CallbackExecutor @NonNull Executor executor,
+ @NonNull SoundTriggerInstrumentation.GlobalCallback callback) {
+ ISoundTriggerService service = ISoundTriggerService.Stub.asInterface(
+ ServiceManager.getService(Context.SOUND_TRIGGER_SERVICE));
+ return new SoundTriggerInstrumentation(service, executor, callback);
+ }
+
}
diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java
index 04c1c0451e63..a54e3560e5a3 100644
--- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java
+++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java
@@ -39,12 +39,13 @@ import android.app.ActivityThread;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
+import android.content.PermissionChecker;
import android.content.ServiceConnection;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
+import android.hardware.soundtrigger.ConversionUtil;
import android.hardware.soundtrigger.IRecognitionStatusCallback;
import android.hardware.soundtrigger.ModelParams;
-import android.hardware.soundtrigger.ConversionUtil;
import android.hardware.soundtrigger.SoundTrigger;
import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;
@@ -64,6 +65,7 @@ import android.media.permission.SafeCloseable;
import android.media.soundtrigger.ISoundTriggerDetectionService;
import android.media.soundtrigger.ISoundTriggerDetectionServiceClient;
import android.media.soundtrigger.SoundTriggerDetectionService;
+import android.media.soundtrigger_middleware.ISoundTriggerInjection;
import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService;
import android.os.Binder;
import android.os.Bundle;
@@ -74,8 +76,8 @@ import android.os.Parcel;
import android.os.ParcelUuid;
import android.os.PowerManager;
import android.os.RemoteException;
-import android.os.ServiceSpecificException;
import android.os.ServiceManager;
+import android.os.ServiceSpecificException;
import android.os.SystemClock;
import android.os.UserHandle;
import android.provider.Settings;
@@ -98,8 +100,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.UUID;
-import java.util.stream.Collectors;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
/**
* A single SystemService to manage all sound/voice-based sound models on the DSP.
@@ -296,6 +298,23 @@ public class SoundTriggerService extends SystemService {
return listUnderlyingModuleProperties(originatorIdentity);
}
}
+
+ @Override
+ public void attachInjection(@NonNull ISoundTriggerInjection injection) {
+ if (PermissionChecker.checkCallingPermissionForPreflight(mContext,
+ android.Manifest.permission.MANAGE_SOUND_TRIGGER, null)
+ != PermissionChecker.PERMISSION_GRANTED) {
+ throw new SecurityException();
+ }
+ try {
+ ISoundTriggerMiddlewareService.Stub
+ .asInterface(ServiceManager
+ .waitForService(Context.SOUND_TRIGGER_MIDDLEWARE_SERVICE))
+ .attachFakeHalInjection(injection);
+ } catch (RemoteException e) {
+ throw e.rethrowFromSystemServer();
+ }
+ }
}
class SoundTriggerSessionStub extends ISoundTriggerSession.Stub {