diff options
| author | 2023-03-01 15:08:26 -0800 | |
|---|---|---|
| committer | 2023-04-06 21:54:37 -0700 | |
| commit | 31b112febfca30acad744e0a3b364b12eaac48d9 (patch) | |
| tree | b762e7302057ad6fa55820478b415a2147033296 | |
| parent | 9407076fb95551732421e7bc79cc4967402aef8c (diff) | |
ST HAL Instrumentation client side
- Add model params to TestApi
- Add RecognitionConfig to TestApi
- Add client-side STInstrumentation attach method to STManager
- STInstrumentation implements ISoundTriggerInjection, dispatches appropriate
callbacks and handles session staleness.
Test: atest CtsVoiceInteractionTestCases
Fixes: 271198164
Change-Id: Iaaaf3c0199980bb5f03b70f2b38f4f44d795666c
6 files changed, 782 insertions, 10 deletions
diff --git a/core/api/test-current.txt b/core/api/test-current.txt index 87835974b5d6..e57493009ea0 100644 --- a/core/api/test-current.txt +++ b/core/api/test-current.txt @@ -1651,6 +1651,11 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.KeyphraseMetadata> CREATOR; } + public class SoundTrigger { + field public static final int MODEL_PARAM_INVALID = -1; // 0xffffffff + field public static final int MODEL_PARAM_THRESHOLD_FACTOR = 0; // 0x0 + } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int); } @@ -1663,6 +1668,19 @@ package android.hardware.soundtrigger { ctor public SoundTrigger.ModuleProperties(int, @NonNull String, @NonNull String, @NonNull String, int, @NonNull String, int, int, int, int, boolean, int, boolean, int, boolean, int); } + public static final class SoundTrigger.RecognitionConfig implements android.os.Parcelable { + ctor public SoundTrigger.RecognitionConfig(boolean, boolean, @Nullable android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[], @Nullable byte[], int); + ctor public SoundTrigger.RecognitionConfig(boolean, boolean, @Nullable android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[], @Nullable byte[]); + method public int describeContents(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.SoundTrigger.RecognitionConfig> CREATOR; + field public final boolean allowMultipleTriggers; + field public final int audioCapabilities; + field public final boolean captureRequested; + field @NonNull public final byte[] data; + field @NonNull public final android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[] keyphrases; + } + public static class SoundTrigger.RecognitionEvent { ctor public SoundTrigger.RecognitionEvent(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[], long); } @@ -2000,6 +2018,57 @@ package android.media.metrics { } +package android.media.soundtrigger { + + public final class SoundTriggerInstrumentation { + method public void setResourceContention(boolean); + method public void triggerOnResourcesAvailable(); + method public void triggerRestart(); + } + + public static interface SoundTriggerInstrumentation.GlobalCallback { + method public default void onClientAttached(); + method public default void onClientDetached(); + method public default void onFrameworkDetached(); + method public void onModelLoaded(@NonNull android.media.soundtrigger.SoundTriggerInstrumentation.ModelSession); + method public default void onPreempted(); + method public default void onRestarted(); + } + + public static interface SoundTriggerInstrumentation.ModelCallback { + method public default void onModelUnloaded(); + method public default void onParamSet(int, int); + method public void onRecognitionStarted(@NonNull android.media.soundtrigger.SoundTriggerInstrumentation.RecognitionSession); + } + + public class SoundTriggerInstrumentation.ModelSession { + method public void clearModelCallback(); + method @NonNull public java.util.List<android.hardware.soundtrigger.SoundTrigger.Keyphrase> getPhrases(); + method @NonNull public android.media.soundtrigger.SoundTriggerManager.Model getSoundModel(); + method public boolean isKeyphrase(); + method public void setModelCallback(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.ModelCallback); + method public void triggerUnloadModel(); + } + + public static interface SoundTriggerInstrumentation.RecognitionCallback { + method public void onRecognitionStopped(); + } + + public class SoundTriggerInstrumentation.RecognitionSession { + method public void clearRecognitionCallback(); + method public int getAudioSession(); + method @NonNull public android.hardware.soundtrigger.SoundTrigger.RecognitionConfig getRecognitionConfig(); + method public void setRecognitionCallback(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.RecognitionCallback); + method public void triggerAbortRecognition(); + method public void triggerRecognitionEvent(@NonNull byte[], @Nullable java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra>); + } + + public final class SoundTriggerManager { + method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) public static android.media.soundtrigger.SoundTriggerInstrumentation attachInstrumentation(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.GlobalCallback); + } + +} + package android.media.tv { public final class TvInputManager { diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java index fa16e167f7d1..6d43ddf7fe94 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.java +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java @@ -1051,6 +1051,29 @@ public class SoundTrigger { return "ModelParamRange [start=" + mStart + ", end=" + mEnd + "]"; } } + /** + * SoundTrigger model parameter types. + * @hide + */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(flag = true, prefix = { "MODEL_PARAM" }, value = { + MODEL_PARAM_INVALID, + MODEL_PARAM_THRESHOLD_FACTOR + }) + public @interface ModelParamTypes {} + + /** + * See {@link ModelParams.INVALID} + * @hide + */ + @TestApi + public static final int MODEL_PARAM_INVALID = ModelParams.INVALID; + /** + * See {@link ModelParams.THRESHOLD_FACTOR} + * @hide + */ + @TestApi + public static final int MODEL_PARAM_THRESHOLD_FACTOR = ModelParams.THRESHOLD_FACTOR; /** * Modes for key phrase recognition @@ -1450,7 +1473,8 @@ public class SoundTrigger { * * @hide */ - public static class RecognitionConfig implements Parcelable { + @TestApi + public static final class RecognitionConfig implements Parcelable { /** True if the DSP should capture the trigger sound and make it available for further * capture. */ @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) @@ -1464,6 +1488,7 @@ public class SoundTrigger { * options for each keyphrase. */ @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) @NonNull + @SuppressLint("ArrayReturn") public final KeyphraseRecognitionExtra keyphrases[]; /** Opaque data for use by system applications who know about voice engine internals, * typically during enrollment. */ @@ -1479,8 +1504,8 @@ public class SoundTrigger { public final int audioCapabilities; public RecognitionConfig(boolean captureRequested, boolean allowMultipleTriggers, - @Nullable KeyphraseRecognitionExtra[] keyphrases, @Nullable byte[] data, - int audioCapabilities) { + @SuppressLint("ArrayReturn") @Nullable KeyphraseRecognitionExtra[] keyphrases, + @Nullable byte[] data, int audioCapabilities) { this.captureRequested = captureRequested; this.allowMultipleTriggers = allowMultipleTriggers; this.keyphrases = keyphrases != null ? keyphrases : new KeyphraseRecognitionExtra[0]; @@ -1490,7 +1515,8 @@ public class SoundTrigger { @UnsupportedAppUsage public RecognitionConfig(boolean captureRequested, boolean allowMultipleTriggers, - @Nullable KeyphraseRecognitionExtra[] keyphrases, @Nullable byte[] data) { + @SuppressLint("ArrayReturn") @Nullable KeyphraseRecognitionExtra[] keyphrases, + @Nullable byte[] data) { this(captureRequested, allowMultipleTriggers, keyphrases, data, 0); } @@ -1517,7 +1543,7 @@ public class SoundTrigger { } @Override - public void writeToParcel(Parcel dest, int flags) { + public void writeToParcel(@NonNull Parcel dest, int flags) { dest.writeByte((byte) (captureRequested ? 1 : 0)); dest.writeByte((byte) (allowMultipleTriggers ? 1 : 0)); dest.writeTypedArray(keyphrases, flags); diff --git a/core/java/com/android/internal/app/ISoundTriggerService.aidl b/core/java/com/android/internal/app/ISoundTriggerService.aidl index ab7f602e2dfc..ed751cb481c5 100644 --- a/core/java/com/android/internal/app/ISoundTriggerService.aidl +++ b/core/java/com/android/internal/app/ISoundTriggerService.aidl @@ -16,8 +16,9 @@ package com.android.internal.app; -import android.media.permission.Identity; import android.hardware.soundtrigger.SoundTrigger; +import android.media.permission.Identity; +import android.media.soundtrigger_middleware.ISoundTriggerInjection; import com.android.internal.app.ISoundTriggerSession; /** @@ -74,4 +75,8 @@ interface ISoundTriggerService { */ List<SoundTrigger.ModuleProperties> listModuleProperties(in Identity originatorIdentity); + /** + * Attach an HAL injection interface. + */ + void attachInjection(ISoundTriggerInjection injection); } diff --git a/media/java/android/media/soundtrigger/SoundTriggerInstrumentation.java b/media/java/android/media/soundtrigger/SoundTriggerInstrumentation.java new file mode 100644 index 000000000000..80bc5c07dd66 --- /dev/null +++ b/media/java/android/media/soundtrigger/SoundTriggerInstrumentation.java @@ -0,0 +1,630 @@ +/** + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.soundtrigger; + +import android.annotation.CallbackExecutor; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.annotation.TestApi; +import android.hardware.soundtrigger.ConversionUtil; +import android.hardware.soundtrigger.SoundTrigger; +import android.media.soundtrigger_middleware.IAcknowledgeEvent; +import android.media.soundtrigger_middleware.IInjectGlobalEvent; +import android.media.soundtrigger_middleware.IInjectModelEvent; +import android.media.soundtrigger_middleware.IInjectRecognitionEvent; +import android.media.soundtrigger_middleware.ISoundTriggerInjection; +import android.os.IBinder; +import android.os.RemoteException; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.app.ISoundTriggerService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +/** + * Used to inject/observe events when using a fake SoundTrigger HAL for test purposes. + * Created by {@link SoundTriggerManager#getInjection(Executor, GlobalCallback)}. + * Only one instance of this class is valid at any given time, old instances will be delivered + * {@link GlobalCallback#onPreempted()}. + * @hide + */ +@TestApi +public final class SoundTriggerInstrumentation { + + private final Object mLock = new Object(); + @GuardedBy("mLock") + private IInjectGlobalEvent mInjectGlobalEvent = null; + + @GuardedBy("mLock") + private Map<IBinder, ModelSession> mModelSessionMap = new HashMap<>(); + @GuardedBy("mLock") + private Map<IBinder, RecognitionSession> mRecognitionSessionMap = new HashMap<>(); + @GuardedBy("mLock") + private IBinder mClientToken = null; + + private final GlobalCallback mClientCallback; + private final Executor mGlobalCallbackExecutor; + + /** + * Callback interface for un-sessioned events observed from the fake STHAL. + * Registered upon construction of {@link SoundTriggerInstrumentation} + * @hide + */ + @TestApi + public interface GlobalCallback { + /** + * Called when the created {@link SoundTriggerInstrumentation} object is invalidated + * by another client creating an {@link SoundTriggerInstrumentation} to instrument the + * fake STHAL. Only one client may inject at a time. + * All sessions are invalidated, no further events will be received, and no + * injected events will be delivered. + */ + default void onPreempted() {} + /** + * Called when the STHAL has been restarted by the framework, due to unexpected + * error conditions. + * Not called when {@link SoundTriggerInstrumentation#triggerRestart()} is injected. + */ + default void onRestarted() {} + /** + * Called when the framework detaches from the fake HAL. + * This is not transmitted to real HALs, but it indicates that the + * framework has flushed its global state. + */ + default void onFrameworkDetached() {} + /** + * Called when a client application attaches to the framework. + * This is not transmitted to real HALs, but it represents the state of + * the framework. + */ + default void onClientAttached() {} + /** + * Called when a client application detaches from the framework. + * This is not transmitted to real HALs, but it represents the state of + * the framework. + */ + default void onClientDetached() {} + /** + * Called when the fake HAL receives a model load from the framework. + * @param modelSession - A session which exposes additional injection + * functionality associated with the newly loaded + * model. See {@link ModelSession}. + */ + void onModelLoaded(@NonNull ModelSession modelSession); + } + + /** + * Callback for HAL events related to a loaded model. Register with + * {@link ModelSession#setModelCallback(Executor, ModelCallback)} + * Note, callbacks will not be delivered for events triggered by the injection. + * @hide + */ + @TestApi + public interface ModelCallback { + /** + * Called when the model associated with the {@link ModelSession} this callback + * was registered for was unloaded by the framework. + */ + default void onModelUnloaded() {} + /** + * Called when the model associated with the {@link ModelSession} this callback + * was registered for receives a set parameter call from the framework. + * @param param - Parameter being set. + * See {@link SoundTrigger.ModelParamTypes} + * @param value - Value the model parameter was set to. + */ + default void onParamSet(@SoundTrigger.ModelParamTypes int param, int value) {} + /** + * Called when the model associated with the {@link ModelSession} this callback + * was registered for receives a recognition start request. + * @param recognitionSession - A session which exposes additional injection + * functionality associated with the newly started + * recognition. See {@link RecognitionSession} + */ + void onRecognitionStarted(@NonNull RecognitionSession recognitionSession); + } + + /** + * Callback for HAL events related to a started recognition. Register with + * {@link RecognitionSession#setRecognitionCallback(Executor, RecognitionCallback)} + * Note, callbacks will not be delivered for events triggered by the injection. + * @hide + */ + @TestApi + public interface RecognitionCallback { + /** + * Called when the recognition associated with the {@link RecognitionSession} this + * callback was registered for was stopped by the framework. + */ + void onRecognitionStopped(); + } + + /** + * Session associated with a loaded model in the fake STHAL. + * Can be used to query details about the loaded model, register a callback for future + * model events, or trigger HAL events associated with a loaded model. + * This session is invalid once the model is unloaded, caused by a + * {@link ModelSession#triggerUnloadModel()}, + * the client unloading recognition, or if a {@link GlobalCallback#onRestarted()} is + * received. + * Further injections on an invalidated session will not be respected, and no future + * callbacks will be delivered. + * @hide + */ + @TestApi + public class ModelSession { + + /** + * Trigger the HAL to preemptively unload the model associated with this session. + * Typically occurs when a higher priority model is loaded which utilizes the same + * resources. + */ + public void triggerUnloadModel() { + synchronized (SoundTriggerInstrumentation.this.mLock) { + try { + mInjectModelEvent.triggerUnloadModel(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + mModelSessionMap.remove(mInjectModelEvent.asBinder()); + } + } + + /** + * Get the {@link SoundTriggerManager.Model} associated with this session. + * @return - The model associated with this session. + */ + public @NonNull SoundTriggerManager.Model getSoundModel() { + return mModel; + } + + /** + * Get the list of {@link SoundTrigger.Keyphrase} associated with this session. + * @return - The keyphrases associated with this session. + */ + public @NonNull List<SoundTrigger.Keyphrase> getPhrases() { + if (mPhrases == null) { + return new ArrayList<>(); + } else { + return new ArrayList<>(Arrays.asList(mPhrases)); + } + } + + /** + * Get whether this model is of keyphrase type. + * @return - true if the model is a keyphrase model, false otherwise + */ + public boolean isKeyphrase() { + return (mPhrases != null); + } + + /** + * Registers the model callback associated with this session. Events associated + * with this model session will be reported via this callback. + * See {@link ModelCallback} + * @param executor - Executor which the callback is dispatched on + * @param callback - Model callback for reporting model session events. + */ + public void setModelCallback(@NonNull @CallbackExecutor Executor executor, @NonNull + ModelCallback callback) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + mModelCallback = Objects.requireNonNull(callback); + mModelExecutor = Objects.requireNonNull(executor); + } + } + + /** + * Clear the model callback associated with this session, if any has been + * set by {@link #setModelCallback(Executor, ModelCallback)}. + */ + public void clearModelCallback() { + synchronized (SoundTriggerInstrumentation.this.mLock) { + mModelCallback = null; + mModelExecutor = null; + } + } + + private ModelSession(SoundModel model, Phrase[] phrases, + IInjectModelEvent injection) { + mModel = SoundTriggerManager.Model.create(UUID.fromString(model.uuid), + UUID.fromString(model.vendorUuid), + ConversionUtil.sharedMemoryToByteArray(model.data, model.dataSize)); + if (phrases != null) { + mPhrases = new SoundTrigger.Keyphrase[phrases.length]; + int i = 0; + for (var phrase : phrases) { + mPhrases[i++] = ConversionUtil.aidl2apiPhrase(phrase); + } + } else { + mPhrases = null; + } + mInjectModelEvent = injection; + } + + private void wrap(Consumer<ModelCallback> consumer) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + if (mModelCallback != null && mModelExecutor != null) { + final ModelCallback callback = mModelCallback; + mModelExecutor.execute(() -> consumer.accept(callback)); + } + } + } + + private final SoundTriggerManager.Model mModel; + private final SoundTrigger.Keyphrase[] mPhrases; + private final IInjectModelEvent mInjectModelEvent; + + @GuardedBy("SoundTriggerInstrumentation.this.mLock") + private ModelCallback mModelCallback = null; + @GuardedBy("SoundTriggerInstrumentation.this.mLock") + private Executor mModelExecutor = null; + } + + /** + * Session associated with a recognition start in the fake STHAL. + * Can be used to get information about the started recognition, register a callback + * for future events associated with this recognition, and triggering + * recognition events or aborts. + * This session is invalid once the recognition is stopped, caused by a + * {@link RecognitionSession#triggerAbortRecognition()}, + * {@link RecognitionSession#triggerRecognitionEvent(byte[], List)}, + * the client stopping recognition, or any operation which invalidates the + * {@link ModelSession} which the session was created from. + * Further injections on an invalidated session will not be respected, and no future + * callbacks will be delivered. + * @hide + */ + @TestApi + public class RecognitionSession { + + /** + * Get an integer token representing the audio session associated with this + * recognition in the STHAL. + * @return - The session token. + */ + public int getAudioSession() { + return mAudioSession; + } + + /** + * Get the recognition config used to start this recognition. + * @return - The config passed to the HAL for startRecognition. + */ + public @NonNull SoundTrigger.RecognitionConfig getRecognitionConfig() { + return mRecognitionConfig; + } + + /** + * Trigger a recognition in the fake STHAL. + * @param data - The opaque data buffer included in the recognition event. + * @param phraseExtras - Keyphrase metadata included in the event. The + * event must include metadata for the keyphrase id + * associated with this model to be received by the + * client application. + */ + public void triggerRecognitionEvent(@NonNull byte[] data, @Nullable + List<SoundTrigger.KeyphraseRecognitionExtra> phraseExtras) { + PhraseRecognitionExtra[] converted = null; + if (phraseExtras != null) { + converted = new PhraseRecognitionExtra[phraseExtras.size()]; + int i = 0; + for (var phraseExtra : phraseExtras) { + converted[i++] = ConversionUtil.api2aidlPhraseRecognitionExtra(phraseExtra); + } + } + synchronized (SoundTriggerInstrumentation.this.mLock) { + mRecognitionSessionMap.remove(mInjectRecognitionEvent.asBinder()); + try { + mInjectRecognitionEvent.triggerRecognitionEvent(data, converted); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + } + + /** + * Trigger an abort recognition event in the fake HAL. This represents a + * preemptive ending of the recognition session by the HAL, despite no + * recognition detection. Typically occurs during contention for microphone + * usage, or if model limits are hit. + * See {@link SoundTriggerInstrumentation#setResourceContention(boolean)} to block + * subsequent downward calls for contention reasons. + */ + public void triggerAbortRecognition() { + synchronized (SoundTriggerInstrumentation.this.mLock) { + mRecognitionSessionMap.remove(mInjectRecognitionEvent.asBinder()); + try { + mInjectRecognitionEvent.triggerAbortRecognition(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + } + + /** + * Registers the recognition callback associated with this session. Events associated + * with this recognition session will be reported via this callback. + * See {@link RecognitionCallback} + * @param executor - Executor which the callback is dispatched on + * @param callback - Recognition callback for reporting recognition session events. + */ + public void setRecognitionCallback(@NonNull @CallbackExecutor Executor executor, + @NonNull RecognitionCallback callback) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + mRecognitionCallback = callback; + mRecognitionExecutor = executor; + } + } + + /** + * Clear the recognition callback associated with this session, if any has been + * set by {@link #setRecognitionCallback(Executor, RecognitionCallback)}. + */ + public void clearRecognitionCallback() { + synchronized (SoundTriggerInstrumentation.this.mLock) { + mRecognitionCallback = null; + mRecognitionExecutor = null; + } + } + + private RecognitionSession(int audioSession, + RecognitionConfig recognitionConfig, + IInjectRecognitionEvent injectRecognitionEvent) { + mAudioSession = audioSession; + mRecognitionConfig = ConversionUtil.aidl2apiRecognitionConfig(recognitionConfig); + mInjectRecognitionEvent = injectRecognitionEvent; + } + + private void wrap(Consumer<RecognitionCallback> consumer) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + if (mRecognitionCallback != null && mRecognitionExecutor != null) { + final RecognitionCallback callback = mRecognitionCallback; + mRecognitionExecutor.execute(() -> consumer.accept(callback)); + } + } + } + + private final int mAudioSession; + private final SoundTrigger.RecognitionConfig mRecognitionConfig; + private final IInjectRecognitionEvent mInjectRecognitionEvent; + + @GuardedBy("SoundTriggerInstrumentation.this.mLock") + private Executor mRecognitionExecutor = null; + @GuardedBy("SoundTriggerInstrumentation.this.mLock") + private RecognitionCallback mRecognitionCallback = null; + } + + // Implementation of injection interface passed to the HAL. + // This class will re-associate events received on this callback interface + // with sessions, to avoid staleness issues. + private class Injection extends ISoundTriggerInjection.Stub { + @Override + public void registerGlobalEventInjection(IInjectGlobalEvent globalInjection) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + mInjectGlobalEvent = globalInjection; + } + } + + @Override + public void onSoundModelLoaded(SoundModel model, @Nullable Phrase[] phrases, + IInjectModelEvent modelInjection, IInjectGlobalEvent globalSession) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return; + ModelSession modelSession = new ModelSession(model, phrases, modelInjection); + mModelSessionMap.put(modelInjection.asBinder(), modelSession); + mGlobalCallbackExecutor.execute(() -> mClientCallback.onModelLoaded(modelSession)); + } + } + + @Override + public void onSoundModelUnloaded(IInjectModelEvent modelSession) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + ModelSession clientModelSession = mModelSessionMap.remove(modelSession.asBinder()); + if (clientModelSession == null) return; + clientModelSession.wrap((ModelCallback cb) -> cb.onModelUnloaded()); + } + } + + @Override + public void onRecognitionStarted(int audioSessionHandle, RecognitionConfig config, + IInjectRecognitionEvent recognitionInjection, IInjectModelEvent modelSession) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + ModelSession clientModelSession = mModelSessionMap.get(modelSession.asBinder()); + if (clientModelSession == null) return; + RecognitionSession recogSession = new RecognitionSession( + audioSessionHandle, config, recognitionInjection); + mRecognitionSessionMap.put(recognitionInjection.asBinder(), recogSession); + clientModelSession.wrap((ModelCallback cb) -> + cb.onRecognitionStarted(recogSession)); + } + } + + @Override + public void onRecognitionStopped(IInjectRecognitionEvent recognitionSession) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + RecognitionSession clientRecognitionSession = + mRecognitionSessionMap.remove(recognitionSession.asBinder()); + if (clientRecognitionSession == null) return; + clientRecognitionSession.wrap((RecognitionCallback cb) + -> cb.onRecognitionStopped()); + } + } + + @Override + public void onParamSet(int modelParam, int value, IInjectModelEvent modelSession) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + ModelSession clientModelSession = mModelSessionMap.get(modelSession.asBinder()); + if (clientModelSession == null) return; + clientModelSession.wrap((ModelCallback cb) -> cb.onParamSet(modelParam, value)); + } + } + + + @Override + public void onRestarted(IInjectGlobalEvent globalSession) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return; + mRecognitionSessionMap.clear(); + mModelSessionMap.clear(); + mGlobalCallbackExecutor.execute(() -> mClientCallback.onRestarted()); + } + } + + @Override + public void onFrameworkDetached(IInjectGlobalEvent globalSession) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return; + mGlobalCallbackExecutor.execute(() -> mClientCallback.onFrameworkDetached()); + } + } + + @Override + public void onClientAttached(IBinder token, IInjectGlobalEvent globalSession) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return; + mClientToken = token; + mGlobalCallbackExecutor.execute(() -> mClientCallback.onClientAttached()); + } + } + + @Override + public void onClientDetached(IBinder token) { + synchronized (SoundTriggerInstrumentation.this.mLock) { + if (token != mClientToken) return; + mClientToken = null; + mGlobalCallbackExecutor.execute(() -> mClientCallback.onClientDetached()); + } + } + + @Override + public void onPreempted() { + // This is always valid, independent of session + mGlobalCallbackExecutor.execute(() -> mClientCallback.onPreempted()); + // Callbacks will no longer be delivered, and injection will be silently dropped. + } + } + + /** + * @hide + */ + @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) + public SoundTriggerInstrumentation(ISoundTriggerService service, + @CallbackExecutor @NonNull Executor executor, + @NonNull GlobalCallback callback) { + mClientCallback = Objects.requireNonNull(callback); + mGlobalCallbackExecutor = Objects.requireNonNull(executor); + try { + service.attachInjection(new Injection()); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + } + } + + /** + * Simulate a HAL restart, typically caused by the framework on an unexpected error, + * or a restart of the core audio HAL. + * Application sessions will be detached, and all state will be cleared. The framework + * will re-attach to the HAL following restart. + * @hide + */ + @TestApi + public void triggerRestart() { + synchronized (mLock) { + if (mInjectGlobalEvent == null) { + throw new IllegalStateException( + "Attempted to trigger HAL restart before registration"); + } + try { + mInjectGlobalEvent.triggerRestart(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + } + + /** + * Trigger a resource available callback from the fake SoundTrigger HAL to the framework. + * This callback notifies the framework that methods which previously failed due to + * resource contention may now succeed. + * @hide + */ + @TestApi + public void triggerOnResourcesAvailable() { + synchronized (mLock) { + if (mInjectGlobalEvent == null) { + throw new IllegalStateException( + "Attempted to trigger HAL resources available before registration"); + } + try { + mInjectGlobalEvent.triggerOnResourcesAvailable(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + } + + /** + * Simulate resource contention, similar to when HAL which does not + * support concurrent capture opens a capture stream, or when a HAL + * has reached its maximum number of models. + * Subsequent model loads and recognition starts will gracefully error. + * Since this call does not trigger a callback through the framework, the + * call will block until the fake HAL has acknowledged the state change. + * @param isResourceContended - true to enable contention, false to return + * to normal functioning. + * @hide + */ + @TestApi + public void setResourceContention(boolean isResourceContended) { + synchronized (mLock) { + if (mInjectGlobalEvent == null) { + throw new IllegalStateException("Injection interface not set up"); + } + IInjectGlobalEvent current = mInjectGlobalEvent; + final CountDownLatch signal = new CountDownLatch(1); + try { + current.setResourceContention(isResourceContended, new IAcknowledgeEvent.Stub() { + @Override + public void eventReceived() { + signal.countDown(); + } + }); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + + // Block until we get a callback from the service that our request was serviced. + try { + // Rely on test timeout if we don't get a response. + signal.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } +} + diff --git a/media/java/android/media/soundtrigger/SoundTriggerManager.java b/media/java/android/media/soundtrigger/SoundTriggerManager.java index ae8121a59abf..c41bd1bc3094 100644 --- a/media/java/android/media/soundtrigger/SoundTriggerManager.java +++ b/media/java/android/media/soundtrigger/SoundTriggerManager.java @@ -18,11 +18,13 @@ package android.media.soundtrigger; import static android.hardware.soundtrigger.SoundTrigger.STATUS_ERROR; +import android.annotation.CallbackExecutor; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; import android.annotation.SystemApi; import android.annotation.SystemService; +import android.annotation.TestApi; import android.app.ActivityThread; import android.compat.annotation.UnsupportedAppUsage; import android.content.ComponentName; @@ -45,6 +47,7 @@ import android.os.Handler; import android.os.IBinder; import android.os.ParcelUuid; import android.os.RemoteException; +import android.os.ServiceManager; import android.provider.Settings; import android.util.Slog; @@ -53,9 +56,9 @@ import com.android.internal.app.ISoundTriggerSession; import com.android.internal.util.Preconditions; import java.util.HashMap; -import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.Executor; /** * This class provides management of non-voice (general sound trigger) based sound recognition @@ -609,4 +612,24 @@ public final class SoundTriggerManager { throw e.rethrowFromSystemServer(); } } + + /** + * Create a {@link SoundTriggerInstrumentation} for test purposes, which instruments a fake + * STHAL. Clients must attach to the appropriate underlying ST module. + * @param executor - Executor to dispatch global callbacks on + * @param callback - Callback for unsessioned events received by the fake STHAL + * @return - A {@link SoundTriggerInstrumentation} for observation/injection. + * @hide + */ + @TestApi + @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) + @NonNull + public static SoundTriggerInstrumentation attachInstrumentation( + @CallbackExecutor @NonNull Executor executor, + @NonNull SoundTriggerInstrumentation.GlobalCallback callback) { + ISoundTriggerService service = ISoundTriggerService.Stub.asInterface( + ServiceManager.getService(Context.SOUND_TRIGGER_SERVICE)); + return new SoundTriggerInstrumentation(service, executor, callback); + } + } diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java index 04c1c0451e63..a54e3560e5a3 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java @@ -39,12 +39,13 @@ import android.app.ActivityThread; import android.content.ComponentName; import android.content.Context; import android.content.Intent; +import android.content.PermissionChecker; import android.content.ServiceConnection; import android.content.pm.PackageManager; import android.content.pm.ResolveInfo; +import android.hardware.soundtrigger.ConversionUtil; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.ModelParams; -import android.hardware.soundtrigger.ConversionUtil; import android.hardware.soundtrigger.SoundTrigger; import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; @@ -64,6 +65,7 @@ import android.media.permission.SafeCloseable; import android.media.soundtrigger.ISoundTriggerDetectionService; import android.media.soundtrigger.ISoundTriggerDetectionServiceClient; import android.media.soundtrigger.SoundTriggerDetectionService; +import android.media.soundtrigger_middleware.ISoundTriggerInjection; import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService; import android.os.Binder; import android.os.Bundle; @@ -74,8 +76,8 @@ import android.os.Parcel; import android.os.ParcelUuid; import android.os.PowerManager; import android.os.RemoteException; -import android.os.ServiceSpecificException; import android.os.ServiceManager; +import android.os.ServiceSpecificException; import android.os.SystemClock; import android.os.UserHandle; import android.provider.Settings; @@ -98,8 +100,8 @@ import java.util.Map; import java.util.Objects; import java.util.TreeMap; import java.util.UUID; -import java.util.stream.Collectors; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * A single SystemService to manage all sound/voice-based sound models on the DSP. @@ -296,6 +298,23 @@ public class SoundTriggerService extends SystemService { return listUnderlyingModuleProperties(originatorIdentity); } } + + @Override + public void attachInjection(@NonNull ISoundTriggerInjection injection) { + if (PermissionChecker.checkCallingPermissionForPreflight(mContext, + android.Manifest.permission.MANAGE_SOUND_TRIGGER, null) + != PermissionChecker.PERMISSION_GRANTED) { + throw new SecurityException(); + } + try { + ISoundTriggerMiddlewareService.Stub + .asInterface(ServiceManager + .waitForService(Context.SOUND_TRIGGER_MIDDLEWARE_SERVICE)) + .attachFakeHalInjection(injection); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } } class SoundTriggerSessionStub extends ISoundTriggerSession.Stub { |