diff options
12 files changed, 1055 insertions, 148 deletions
diff --git a/api/system-current.txt b/api/system-current.txt index e4437aaac77c..6509ac811294 100644 --- a/api/system-current.txt +++ b/api/system-current.txt @@ -24281,19 +24281,26 @@ package android.media.session { package android.media.soundtrigger { public final class SoundTriggerDetector { - method public boolean startRecognition(); + method public boolean startRecognition(int); method public boolean stopRecognition(); + field public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 2; // 0x2 + field public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 1; // 0x1 } - public abstract class SoundTriggerDetector.Callback { + public static abstract class SoundTriggerDetector.Callback { ctor public SoundTriggerDetector.Callback(); method public abstract void onAvailabilityChanged(int); - method public abstract void onDetected(); + method public abstract void onDetected(android.media.soundtrigger.SoundTriggerDetector.EventPayload); method public abstract void onError(); method public abstract void onRecognitionPaused(); method public abstract void onRecognitionResumed(); } + public static class SoundTriggerDetector.EventPayload { + method public android.media.AudioFormat getCaptureAudioFormat(); + method public byte[] getTriggerAudio(); + } + public final class SoundTriggerManager { method public android.media.soundtrigger.SoundTriggerDetector createSoundTriggerDetector(java.util.UUID, android.media.soundtrigger.SoundTriggerDetector.Callback, android.os.Handler); method public void deleteModel(java.util.UUID); diff --git a/core/java/com/android/internal/app/ISoundTriggerService.aidl b/core/java/com/android/internal/app/ISoundTriggerService.aidl index 9de4a6c62a49..f4c18c300513 100644 --- a/core/java/com/android/internal/app/ISoundTriggerService.aidl +++ b/core/java/com/android/internal/app/ISoundTriggerService.aidl @@ -33,10 +33,11 @@ interface ISoundTriggerService { void deleteSoundModel(in ParcelUuid soundModelId); - void startRecognition(in ParcelUuid soundModelId, in IRecognitionStatusCallback callback); + int startRecognition(in ParcelUuid soundModelId, in IRecognitionStatusCallback callback, + in SoundTrigger.RecognitionConfig config); /** * Stops recognition. */ - void stopRecognition(in ParcelUuid soundModelId, in IRecognitionStatusCallback callback); + int stopRecognition(in ParcelUuid soundModelId, in IRecognitionStatusCallback callback); } diff --git a/media/java/android/media/soundtrigger/SoundTriggerDetector.java b/media/java/android/media/soundtrigger/SoundTriggerDetector.java index 707db06fdd54..8f022db5b02e 100644 --- a/media/java/android/media/soundtrigger/SoundTriggerDetector.java +++ b/media/java/android/media/soundtrigger/SoundTriggerDetector.java @@ -16,12 +16,17 @@ package android.media.soundtrigger; +import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemApi; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; +import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; +import android.media.AudioFormat; import android.os.Handler; +import android.os.Looper; +import android.os.Message; import android.os.ParcelUuid; import android.os.RemoteException; import android.util.Slog; @@ -29,6 +34,8 @@ import android.util.Slog; import com.android.internal.app.ISoundTriggerService; import java.io.PrintWriter; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.util.UUID; /** @@ -45,6 +52,12 @@ public final class SoundTriggerDetector { private static final boolean DBG = false; private static final String TAG = "SoundTriggerDetector"; + private static final int MSG_AVAILABILITY_CHANGED = 1; + private static final int MSG_SOUND_TRIGGER_DETECTED = 2; + private static final int MSG_DETECTION_ERROR = 3; + private static final int MSG_DETECTION_PAUSE = 4; + private static final int MSG_DETECTION_RESUME = 5; + private final Object mLock = new Object(); private final ISoundTriggerService mSoundTriggerService; @@ -53,7 +66,121 @@ public final class SoundTriggerDetector { private final Handler mHandler; private final RecognitionCallback mRecognitionCallback; - public abstract class Callback { + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(flag = true, + value = { + RECOGNITION_FLAG_NONE, + RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO, + RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS + }) + public @interface RecognitionFlags {} + + /** + * Empty flag for {@link #startRecognition(int)}. + * + * @hide + */ + public static final int RECOGNITION_FLAG_NONE = 0; + + /** + * Recognition flag for {@link #startRecognition(int)} that indicates + * whether the trigger audio for hotword needs to be captured. + */ + public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; + + /** + * Recognition flag for {@link #startRecognition(int)} that indicates + * whether the recognition should keep going on even after the + * model triggers. + * If this flag is specified, it's possible to get multiple + * triggers after a call to {@link #startRecognition(int)}, if the model + * triggers multiple times. + * When this isn't specified, the default behavior is to stop recognition once the + * trigger happenss, till the caller starts recognition again. + */ + public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2; + + /** + * Additional payload for {@link Callback#onDetected}. + */ + public static class EventPayload { + private final boolean mTriggerAvailable; + + // Indicates if {@code captureSession} can be used to continue capturing more audio + // from the DSP hardware. + private final boolean mCaptureAvailable; + // The session to use when attempting to capture more audio from the DSP hardware. + private final int mCaptureSession; + private final AudioFormat mAudioFormat; + // Raw data associated with the event. + // This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true. + private final byte[] mData; + + private EventPayload(boolean triggerAvailable, boolean captureAvailable, + AudioFormat audioFormat, int captureSession, byte[] data) { + mTriggerAvailable = triggerAvailable; + mCaptureAvailable = captureAvailable; + mCaptureSession = captureSession; + mAudioFormat = audioFormat; + mData = data; + } + + /** + * Gets the format of the audio obtained using {@link #getTriggerAudio()}. + * May be null if there's no audio present. + */ + @Nullable + public AudioFormat getCaptureAudioFormat() { + return mAudioFormat; + } + + /** + * Gets the raw audio that triggered the keyphrase. + * This may be null if the trigger audio isn't available. + * If non-null, the format of the audio can be obtained by calling + * {@link #getCaptureAudioFormat()}. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + @Nullable + public byte[] getTriggerAudio() { + if (mTriggerAvailable) { + return mData; + } else { + return null; + } + } + + /** + * Gets the session ID to start a capture from the DSP. + * This may be null if streaming capture isn't possible. + * If non-null, the format of the audio that can be captured can be + * obtained using {@link #getCaptureAudioFormat()}. + * + * TODO: Candidate for Public API when the API to start capture with a session ID + * is made public. + * + * TODO: Add this to {@link #getCaptureAudioFormat()}: + * "Gets the format of the audio obtained using {@link #getTriggerAudio()} + * or {@link #getCaptureSession()}. May be null if no audio can be obtained + * for either the trigger or a streaming session." + * + * TODO: Should this return a known invalid value instead? + * + * @hide + */ + @Nullable + public Integer getCaptureSession() { + if (mCaptureAvailable) { + return mCaptureSession; + } else { + return null; + } + } + } + + public static abstract class Callback { /** * Called when the availability of the sound model changes. */ @@ -63,7 +190,7 @@ public final class SoundTriggerDetector { * Called when the sound model has triggered (such as when it matched a * given sound pattern). */ - public abstract void onDetected(); + public abstract void onDetected(@NonNull EventPayload eventPayload); /** * Called when the detection fails due to an error. @@ -95,9 +222,9 @@ public final class SoundTriggerDetector { mSoundModelId = soundModelId; mCallback = callback; if (handler == null) { - mHandler = new Handler(); + mHandler = new MyHandler(); } else { - mHandler = handler; + mHandler = new MyHandler(handler.getLooper()); } mRecognitionCallback = new RecognitionCallback(); } @@ -107,13 +234,19 @@ public final class SoundTriggerDetector { * {@link Callback}. * @return Indicates whether the call succeeded or not. */ - public boolean startRecognition() { + public boolean startRecognition(@RecognitionFlags int recognitionFlags) { if (DBG) { Slog.d(TAG, "startRecognition()"); } + boolean captureTriggerAudio = + (recognitionFlags & RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; + + boolean allowMultipleTriggers = + (recognitionFlags & RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0; try { mSoundTriggerService.startRecognition(new ParcelUuid(mSoundModelId), - mRecognitionCallback); + mRecognitionCallback, new RecognitionConfig(captureTriggerAudio, + allowMultipleTriggers, null, null)); } catch (RemoteException e) { return false; } @@ -144,17 +277,25 @@ public final class SoundTriggerDetector { /** * Callback that handles events from the lower sound trigger layer. + * + * Note that these callbacks will be called synchronously from the SoundTriggerService + * layer and thus should do minimal work (such as sending a message on a handler to do + * the real work). * @hide */ - private static class RecognitionCallback extends - IRecognitionStatusCallback.Stub { + private class RecognitionCallback extends IRecognitionStatusCallback.Stub { /** * @hide */ @Override public void onDetected(SoundTrigger.RecognitionEvent event) { - Slog.e(TAG, "onDetected()" + event); + Slog.d(TAG, "onDetected()" + event); + Message.obtain(mHandler, + MSG_SOUND_TRIGGER_DETECTED, + new EventPayload(event.triggerInData, event.captureAvailable, + event.captureFormat, event.captureSession, event.data)) + .sendToTarget(); } /** @@ -162,7 +303,8 @@ public final class SoundTriggerDetector { */ @Override public void onError(int status) { - Slog.e(TAG, "onError()" + status); + Slog.d(TAG, "onError()" + status); + mHandler.sendEmptyMessage(MSG_DETECTION_ERROR); } /** @@ -170,7 +312,8 @@ public final class SoundTriggerDetector { */ @Override public void onRecognitionPaused() { - Slog.e(TAG, "onRecognitionPaused()"); + Slog.d(TAG, "onRecognitionPaused()"); + mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE); } /** @@ -178,7 +321,44 @@ public final class SoundTriggerDetector { */ @Override public void onRecognitionResumed() { - Slog.e(TAG, "onRecognitionResumed()"); + Slog.d(TAG, "onRecognitionResumed()"); + mHandler.sendEmptyMessage(MSG_DETECTION_RESUME); + } + } + + private class MyHandler extends Handler { + + MyHandler() { + super(); + } + + MyHandler(Looper looper) { + super(looper); + } + + @Override + public void handleMessage(Message msg) { + if (mCallback == null) { + Slog.w(TAG, "Received message: " + msg.what + " for NULL callback."); + return; + } + switch (msg.what) { + case MSG_SOUND_TRIGGER_DETECTED: + mCallback.onDetected((EventPayload) msg.obj); + break; + case MSG_DETECTION_ERROR: + mCallback.onError(); + break; + case MSG_DETECTION_PAUSE: + mCallback.onRecognitionPaused(); + break; + case MSG_DETECTION_RESUME: + mCallback.onRecognitionResumed(); + break; + default: + super.handleMessage(msg); + + } } } } diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerDbHelper.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerDbHelper.java index 18a5d59543e4..f7cd6a3d2245 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerDbHelper.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerDbHelper.java @@ -54,6 +54,7 @@ public class SoundTriggerDbHelper extends SQLiteOpenHelper { private static final String CREATE_TABLE_ST_SOUND_MODEL = "CREATE TABLE " + GenericSoundModelContract.TABLE + "(" + GenericSoundModelContract.KEY_MODEL_UUID + " TEXT PRIMARY KEY," + + GenericSoundModelContract.KEY_VENDOR_UUID + " TEXT," + GenericSoundModelContract.KEY_DATA + " BLOB" + " )"; diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java index 354075ea1762..cde47bd0666a 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java @@ -16,12 +16,16 @@ package com.android.server.soundtrigger; +import static android.hardware.soundtrigger.SoundTrigger.STATUS_ERROR; + import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; import android.content.IntentFilter; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; +import android.hardware.soundtrigger.SoundTrigger.GenericRecognitionEvent; +import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; import android.hardware.soundtrigger.SoundTrigger.Keyphrase; import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; @@ -29,6 +33,7 @@ import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; +import android.hardware.soundtrigger.SoundTrigger.SoundModel; import android.hardware.soundtrigger.SoundTrigger.SoundModelEvent; import android.hardware.soundtrigger.SoundTriggerModule; import android.os.PowerManager; @@ -40,9 +45,16 @@ import android.util.Slog; import java.io.FileDescriptor; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.HashMap; +import java.util.UUID; /** - * Helper for {@link SoundTrigger} APIs. + * Helper for {@link SoundTrigger} APIs. Supports two types of models: + * (i) A voice model which is exported via the {@link VoiceInteractionService}. There can only be + * a single voice model running on the DSP at any given time. + * + * (ii) Generic sound-trigger models: Supports multiple of these. + * * Currently this just acts as an abstraction over all SoundTrigger API calls. * * @hide @@ -62,7 +74,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { private static final int INVALID_VALUE = Integer.MIN_VALUE; /** The {@link ModuleProperties} for the system, or null if none exists. */ - final ModuleProperties moduleProperties; + final ModuleProperties mModuleProperties; /** The properties for the DSP module */ private SoundTriggerModule mModule; @@ -72,21 +84,36 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { private final PhoneStateListener mPhoneStateListener; private final PowerManager mPowerManager; - // TODO: Since many layers currently only deal with one recognition + // TODO: Since the voice layer currently only handles one recognition // we simplify things by assuming one listener here too. - private IRecognitionStatusCallback mActiveListener; + private IRecognitionStatusCallback mKeyphraseListener; + + // The SoundTriggerManager layer handles multiple generic recognition models. We store the + // ModelData here in a hashmap. + private final HashMap<UUID, ModelData> mGenericModelDataMap; + + // Note: KeyphraseId is not really used. private int mKeyphraseId = INVALID_VALUE; - private int mCurrentSoundModelHandle = INVALID_VALUE; + + // Current voice sound model handle. We only allow one voice model to run at any given time. + private int mCurrentKeyphraseModelHandle = INVALID_VALUE; private KeyphraseSoundModel mCurrentSoundModel = null; // FIXME: Ideally this should not be stored if allowMultipleTriggers happens at a lower layer. private RecognitionConfig mRecognitionConfig = null; + + // Whether we are requesting recognition to start. private boolean mRequested = false; private boolean mCallActive = false; private boolean mIsPowerSaveMode = false; // Indicates if the native sound trigger service is disabled or not. // This is an indirect indication of the microphone being open in some other application. private boolean mServiceDisabled = false; - private boolean mStarted = false; + + // Whether we have ANY recognition (keyphrase or generic) running. + private boolean mRecognitionRunning = false; + + // Keeps track of whether the keyphrase recognition is running. + private boolean mKeyphraseStarted = false; private boolean mRecognitionAborted = false; private PowerSaveModeListener mPowerSaveModeListener; @@ -96,14 +123,87 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { mContext = context; mTelephonyManager = (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE); mPowerManager = (PowerManager) context.getSystemService(Context.POWER_SERVICE); + mGenericModelDataMap = new HashMap<UUID, ModelData>(); mPhoneStateListener = new MyCallStateListener(); if (status != SoundTrigger.STATUS_OK || modules.size() == 0) { Slog.w(TAG, "listModules status=" + status + ", # of modules=" + modules.size()); - moduleProperties = null; + mModuleProperties = null; mModule = null; } else { // TODO: Figure out how to determine which module corresponds to the DSP hardware. - moduleProperties = modules.get(0); + mModuleProperties = modules.get(0); + } + } + + /** + * Starts recognition for the given generic sound model ID. + * + * @param soundModel The sound model to use for recognition. + * @param listener The listener for the recognition events related to the given keyphrase. + * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. + */ + int startGenericRecognition(UUID modelId, GenericSoundModel soundModel, + IRecognitionStatusCallback callback, RecognitionConfig recognitionConfig) { + if (soundModel == null || callback == null || recognitionConfig == null) { + Slog.w(TAG, "Passed in bad data to startGenericRecognition()."); + return STATUS_ERROR; + } + + synchronized (mLock) { + + if (mModuleProperties == null) { + Slog.w(TAG, "Attempting startRecognition without the capability"); + return STATUS_ERROR; + } + + if (mModule == null) { + mModule = SoundTrigger.attachModule(mModuleProperties.id, this, null); + if (mModule == null) { + Slog.w(TAG, "startRecognition cannot attach to sound trigger module"); + return STATUS_ERROR; + } + } + + // Initialize power save, call active state monitoring logic. + if (!mRecognitionRunning) { + initializeTelephonyAndPowerStateListeners(); + } + + // Fetch a ModelData instance from the hash map. Creates a new one if none + // exists. + ModelData modelData = getOrCreateGenericModelData(modelId); + + IRecognitionStatusCallback oldCallback = modelData.getCallback(); + if (oldCallback != null) { + Slog.w(TAG, "Canceling previous recognition for model id: " + modelId); + try { + oldCallback.onError(STATUS_ERROR); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException in onDetectionStopped", e); + } + modelData.clearCallback(); + } + + // Load the model if its not loaded. + if (!modelData.isModelLoaded()) { + // Load the model + int[] handle = new int[] { INVALID_VALUE }; + int status = mModule.loadSoundModel(soundModel, handle); + if (status != SoundTrigger.STATUS_OK) { + Slog.w(TAG, "loadSoundModel call failed with " + status); + return status; + } + if (handle[0] == INVALID_VALUE) { + Slog.w(TAG, "loadSoundModel call returned invalid sound model handle"); + return STATUS_ERROR; + } + modelData.setHandle(handle[0]); + } + modelData.setCallback(callback); + modelData.setRecognitionConfig(recognitionConfig); + + // Don't notify for synchronous calls. + return startGenericRecognitionLocked(modelData, false); } } @@ -116,7 +216,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { * @param listener The listener for the recognition events related to the given keyphrase. * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. */ - int startRecognition(int keyphraseId, + int startKeyphraseRecognition(int keyphraseId, KeyphraseSoundModel soundModel, IRecognitionStatusCallback listener, RecognitionConfig recognitionConfig) { @@ -129,36 +229,24 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { Slog.d(TAG, "startRecognition for keyphraseId=" + keyphraseId + " soundModel=" + soundModel + ", listener=" + listener.asBinder() + ", recognitionConfig=" + recognitionConfig); - Slog.d(TAG, "moduleProperties=" + moduleProperties); + Slog.d(TAG, "moduleProperties=" + mModuleProperties); Slog.d(TAG, "current listener=" - + (mActiveListener == null ? "null" : mActiveListener.asBinder())); - Slog.d(TAG, "current SoundModel handle=" + mCurrentSoundModelHandle); + + (mKeyphraseListener == null ? "null" : mKeyphraseListener.asBinder())); + Slog.d(TAG, "current SoundModel handle=" + mCurrentKeyphraseModelHandle); Slog.d(TAG, "current SoundModel UUID=" + (mCurrentSoundModel == null ? null : mCurrentSoundModel.uuid)); } - if (!mStarted) { - // Get the current call state synchronously for the first recognition. - mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE; - // Register for call state changes when the first call to start recognition occurs. - mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE); - - // Register for power saver mode changes when the first call to start recognition - // occurs. - if (mPowerSaveModeListener == null) { - mPowerSaveModeListener = new PowerSaveModeListener(); - mContext.registerReceiver(mPowerSaveModeListener, - new IntentFilter(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED)); - } - mIsPowerSaveMode = mPowerManager.isPowerSaveMode(); + if (!mRecognitionRunning) { + initializeTelephonyAndPowerStateListeners(); } - if (moduleProperties == null) { + if (mModuleProperties == null) { Slog.w(TAG, "Attempting startRecognition without the capability"); return STATUS_ERROR; } if (mModule == null) { - mModule = SoundTrigger.attachModule(moduleProperties.id, this, null); + mModule = SoundTrigger.attachModule(mModuleProperties.id, this, null); if (mModule == null) { Slog.w(TAG, "startRecognition cannot attach to sound trigger module"); return STATUS_ERROR; @@ -168,32 +256,32 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { // Unload the previous model if the current one isn't invalid // and, it's not the same as the new one. // This helps use cache and reuse the model and just start/stop it when necessary. - if (mCurrentSoundModelHandle != INVALID_VALUE + if (mCurrentKeyphraseModelHandle != INVALID_VALUE && !soundModel.equals(mCurrentSoundModel)) { Slog.w(TAG, "Unloading previous sound model"); - int status = mModule.unloadSoundModel(mCurrentSoundModelHandle); + int status = mModule.unloadSoundModel(mCurrentKeyphraseModelHandle); if (status != SoundTrigger.STATUS_OK) { Slog.w(TAG, "unloadSoundModel call failed with " + status); } - internalClearSoundModelLocked(); - mStarted = false; + internalClearKeyphraseSoundModelLocked(); + mKeyphraseStarted = false; } // If the previous recognition was by a different listener, // Notify them that it was stopped. - if (mActiveListener != null && mActiveListener.asBinder() != listener.asBinder()) { + if (mKeyphraseListener != null && mKeyphraseListener.asBinder() != listener.asBinder()) { Slog.w(TAG, "Canceling previous recognition"); try { - mActiveListener.onError(STATUS_ERROR); + mKeyphraseListener.onError(STATUS_ERROR); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onDetectionStopped", e); } - mActiveListener = null; + mKeyphraseListener = null; } // Load the sound model if the current one is null. - int soundModelHandle = mCurrentSoundModelHandle; - if (mCurrentSoundModelHandle == INVALID_VALUE + int soundModelHandle = mCurrentKeyphraseModelHandle; + if (mCurrentKeyphraseModelHandle == INVALID_VALUE || mCurrentSoundModel == null) { int[] handle = new int[] { INVALID_VALUE }; int status = mModule.loadSoundModel(soundModel, handle); @@ -213,18 +301,81 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { // Start the recognition. mRequested = true; mKeyphraseId = keyphraseId; - mCurrentSoundModelHandle = soundModelHandle; + mCurrentKeyphraseModelHandle = soundModelHandle; mCurrentSoundModel = soundModel; mRecognitionConfig = recognitionConfig; // Register the new listener. This replaces the old one. // There can only be a maximum of one active listener at any given time. - mActiveListener = listener; + mKeyphraseListener = listener; return updateRecognitionLocked(false /* don't notify for synchronous calls */); } } /** + * Stops recognition for the given generic sound model. + * + * @param modelId The identifier of the generic sound model for which + * the recognition is to be stopped. + * @param listener The listener for the recognition events related to the given sound model. + * + * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. + */ + int stopGenericRecognition(UUID modelId, IRecognitionStatusCallback listener) { + if (listener == null) { + return STATUS_ERROR; + } + + synchronized (mLock) { + ModelData modelData = mGenericModelDataMap.get(modelId); + if (modelData == null) { + Slog.w(TAG, "Attempting stopRecognition on invalid model with id:" + modelId); + return STATUS_ERROR; + } + + IRecognitionStatusCallback currentCallback = modelData.getCallback(); + if (DBG) { + Slog.d(TAG, "stopRecognition for modelId=" + modelId + + ", listener=" + listener.asBinder()); + Slog.d(TAG, "current callback =" + + (currentCallback == null ? "null" : currentCallback.asBinder())); + } + + if (mModuleProperties == null || mModule == null) { + Slog.w(TAG, "Attempting stopRecognition without the capability"); + return STATUS_ERROR; + } + + if (currentCallback == null || !modelData.modelStarted()) { + // startRecognition hasn't been called or it failed. + Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); + return STATUS_ERROR; + } + if (currentCallback.asBinder() != listener.asBinder()) { + // We don't allow a different listener to stop the recognition than the one + // that started it. + Slog.w(TAG, "Attempting stopRecognition for another recognition"); + return STATUS_ERROR; + } + + int status = stopGenericRecognitionLocked(modelData, false /* don't notify for synchronous calls */); + if (status != SoundTrigger.STATUS_OK) { + return status; + } + + // We leave the sound model loaded but not started, this helps us when we start + // back. + // Also clear the internal state once the recognition has been stopped. + modelData.clearState(); + modelData.clearCallback(); + if (!computeRecognitionRunning()) { + internalClearGlobalStateLocked(); + } + return status; + } + } + + /** * Stops recognition for the given {@link Keyphrase} if a recognition is * currently active. * @@ -234,7 +385,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { * * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. */ - int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { + int stopKeyphraseRecognition(int keyphraseId, IRecognitionStatusCallback listener) { if (listener == null) { return STATUS_ERROR; } @@ -244,20 +395,20 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { Slog.d(TAG, "stopRecognition for keyphraseId=" + keyphraseId + ", listener=" + listener.asBinder()); Slog.d(TAG, "current listener=" - + (mActiveListener == null ? "null" : mActiveListener.asBinder())); + + (mKeyphraseListener == null ? "null" : mKeyphraseListener.asBinder())); } - if (moduleProperties == null || mModule == null) { + if (mModuleProperties == null || mModule == null) { Slog.w(TAG, "Attempting stopRecognition without the capability"); return STATUS_ERROR; } - if (mActiveListener == null) { + if (mKeyphraseListener == null) { // startRecognition hasn't been called or it failed. Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); return STATUS_ERROR; } - if (mActiveListener.asBinder() != listener.asBinder()) { + if (mKeyphraseListener.asBinder() != listener.asBinder()) { // We don't allow a different listener to stop the recognition than the one // that started it. Slog.w(TAG, "Attempting stopRecognition for another recognition"); @@ -274,7 +425,8 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { // We leave the sound model loaded but not started, this helps us when we start // back. // Also clear the internal state once the recognition has been stopped. - internalClearStateLocked(); + internalClearKeyphraseStateLocked(); + internalClearGlobalStateLocked(); return status; } } @@ -284,38 +436,56 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { */ void stopAllRecognitions() { synchronized (mLock) { - if (moduleProperties == null || mModule == null) { + if (mModuleProperties == null || mModule == null) { return; } - if (mCurrentSoundModelHandle == INVALID_VALUE) { - return; + // Stop Keyphrase recognition if one exists. + if (mCurrentKeyphraseModelHandle != INVALID_VALUE) { + + mRequested = false; + int status = updateRecognitionLocked( + false /* don't notify for synchronous calls */); + internalClearKeyphraseStateLocked(); } - mRequested = false; - int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); - internalClearStateLocked(); + // Stop all generic recognition models. + for (ModelData model : mGenericModelDataMap.values()) { + if (model.modelStarted()) { + int status = stopGenericRecognitionLocked(model, + false /* do not notify for synchronous calls */); + if (status != STATUS_OK) { + // What else can we do if there is an error here. + Slog.w(TAG, "Error stopping generic model: " + model.getHandle()); + } + model.clearState(); + model.clearCallback(); + } + } + internalClearGlobalStateLocked(); } } public ModuleProperties getModuleProperties() { - return moduleProperties; + return mModuleProperties; } //---- SoundTrigger.StatusListener methods @Override public void onRecognition(RecognitionEvent event) { - if (event == null || !(event instanceof KeyphraseRecognitionEvent)) { - Slog.w(TAG, "Invalid recognition event!"); + if (event == null) { + Slog.w(TAG, "Null recognition event!"); + return; + } + + if (!(event instanceof KeyphraseRecognitionEvent) && + !(event instanceof GenericRecognitionEvent)) { + Slog.w(TAG, "Invalid recognition event type (not one of generic or keyphrase) !"); return; } if (DBG) Slog.d(TAG, "onRecognition: " + event); synchronized (mLock) { - if (mActiveListener == null) { - Slog.w(TAG, "received onRecognition event without any listener for it"); - return; - } switch (event.status) { // Fire aborts/failures to all listeners since it's not tied to a keyphrase. case SoundTrigger.RECOGNITION_STATUS_ABORT: @@ -325,12 +495,60 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { onRecognitionFailureLocked(); break; case SoundTrigger.RECOGNITION_STATUS_SUCCESS: - onRecognitionSuccessLocked((KeyphraseRecognitionEvent) event); + + if (isKeyphraseRecognitionEvent(event)) { + onKeyphraseRecognitionSuccessLocked((KeyphraseRecognitionEvent) event); + } else { + onGenericRecognitionSuccessLocked((GenericRecognitionEvent) event); + } + break; } } } + private boolean isKeyphraseRecognitionEvent(RecognitionEvent event) { + return mCurrentKeyphraseModelHandle == event.soundModelHandle; + } + + private void onGenericRecognitionSuccessLocked(GenericRecognitionEvent event) { + if (event.status != SoundTrigger.RECOGNITION_STATUS_SUCCESS) { + return; + } + ModelData model = getModelDataFor(event.soundModelHandle); + if (model == null) { + Slog.w(TAG, "Generic recognition event: Model does not exist for handle: " + + event.soundModelHandle); + return; + } + + IRecognitionStatusCallback callback = model.getCallback(); + if (callback == null) { + Slog.w(TAG, "Generic recognition event: Null callback for model handle: " + + event.soundModelHandle); + return; + } + + try { + callback.onDetected((GenericRecognitionEvent) event); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException in onDetected", e); + } + + model.setStopped(); + RecognitionConfig config = model.getRecognitionConfig(); + if (config == null) { + Slog.w(TAG, "Generic recognition event: Null RecognitionConfig for model handle: " + + event.soundModelHandle); + return; + } + + // TODO: Remove this block if the lower layer supports multiple triggers. + if (config.allowMultipleTriggers) { + startGenericRecognitionLocked(model, true /* notify */); + } + } + @Override public void onSoundModelUpdate(SoundModelEvent event) { if (event == null) { @@ -399,18 +617,25 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { private void onRecognitionFailureLocked() { Slog.w(TAG, "Recognition failure"); try { - if (mActiveListener != null) { - mActiveListener.onError(STATUS_ERROR); + if (mKeyphraseListener != null) { + mKeyphraseListener.onError(STATUS_ERROR); } } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onError", e); } finally { - internalClearStateLocked(); + internalClearKeyphraseStateLocked(); + internalClearGlobalStateLocked(); } } - private void onRecognitionSuccessLocked(KeyphraseRecognitionEvent event) { + private void onKeyphraseRecognitionSuccessLocked(KeyphraseRecognitionEvent event) { Slog.i(TAG, "Recognition success"); + + if (mKeyphraseListener == null) { + Slog.w(TAG, "received onRecognition event without any listener for it"); + return; + } + KeyphraseRecognitionExtra[] keyphraseExtras = ((KeyphraseRecognitionEvent) event).keyphraseExtras; if (keyphraseExtras == null || keyphraseExtras.length == 0) { @@ -424,14 +649,14 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { } try { - if (mActiveListener != null) { - mActiveListener.onDetected((KeyphraseRecognitionEvent) event); + if (mKeyphraseListener != null) { + mKeyphraseListener.onDetected((KeyphraseRecognitionEvent) event); } } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onDetected", e); } - mStarted = false; + mKeyphraseStarted = false; mRequested = mRecognitionConfig.allowMultipleTriggers; // TODO: Remove this block if the lower layer supports multiple triggers. if (mRequested) { @@ -441,14 +666,16 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { private void onServiceDiedLocked() { try { - if (mActiveListener != null) { - mActiveListener.onError(SoundTrigger.STATUS_DEAD_OBJECT); + if (mKeyphraseListener != null) { + mKeyphraseListener.onError(SoundTrigger.STATUS_DEAD_OBJECT); } } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onError", e); } finally { - internalClearSoundModelLocked(); - internalClearStateLocked(); + internalClearKeyphraseSoundModelLocked(); + internalClearKeyphraseStateLocked(); + internalClearGenericModelStateLocked(); + internalClearGlobalStateLocked(); if (mModule != null) { mModule.detach(); mModule = null; @@ -457,14 +684,14 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { } private int updateRecognitionLocked(boolean notify) { - if (mModule == null || moduleProperties == null - || mCurrentSoundModelHandle == INVALID_VALUE || mActiveListener == null) { + if (mModule == null || mModuleProperties == null + || mCurrentKeyphraseModelHandle == INVALID_VALUE || mKeyphraseListener == null) { // Nothing to do here. return STATUS_OK; } boolean start = mRequested && !mCallActive && !mServiceDisabled && !mIsPowerSaveMode; - if (start == mStarted) { + if (start == mKeyphraseStarted) { // No-op. return STATUS_OK; } @@ -472,23 +699,24 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { // See if the recognition needs to be started. if (start) { // Start recognition. - int status = mModule.startRecognition(mCurrentSoundModelHandle, mRecognitionConfig); + int status = mModule.startRecognition(mCurrentKeyphraseModelHandle, + mRecognitionConfig); if (status != SoundTrigger.STATUS_OK) { Slog.w(TAG, "startRecognition failed with " + status); // Notify of error if needed. if (notify) { try { - mActiveListener.onError(status); + mKeyphraseListener.onError(status); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onError", e); } } } else { - mStarted = true; + mKeyphraseStarted = true; // Notify of resume if needed. if (notify) { try { - mActiveListener.onRecognitionResumed(); + mKeyphraseListener.onRecognitionResumed(); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onRecognitionResumed", e); } @@ -499,7 +727,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { // Stop recognition (only if we haven't been aborted). int status = STATUS_OK; if (!mRecognitionAborted) { - status = mModule.stopRecognition(mCurrentSoundModelHandle); + status = mModule.stopRecognition(mCurrentKeyphraseModelHandle); } else { mRecognitionAborted = false; } @@ -507,17 +735,17 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { Slog.w(TAG, "stopRecognition call failed with " + status); if (notify) { try { - mActiveListener.onError(status); + mKeyphraseListener.onError(status); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onError", e); } } } else { - mStarted = false; + mKeyphraseStarted = false; // Notify of pause if needed. if (notify) { try { - mActiveListener.onRecognitionPaused(); + mKeyphraseListener.onRecognitionPaused(); } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onRecognitionPaused", e); } @@ -527,14 +755,11 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { } } - private void internalClearStateLocked() { - mStarted = false; - mRequested = false; - - mKeyphraseId = INVALID_VALUE; - mRecognitionConfig = null; - mActiveListener = null; - + // internalClearGlobalStateLocked() gets split into two routines. Cleanup that is + // specific to keyphrase sound models named as internalClearKeyphraseStateLocked() and + // internalClearGlobalStateLocked() for global state. The global cleanup routine will be used + // by the cleanup happening with the generic sound models. + private void internalClearGlobalStateLocked() { // Unregister from call state changes. mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_NONE); @@ -545,8 +770,27 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { } } - private void internalClearSoundModelLocked() { - mCurrentSoundModelHandle = INVALID_VALUE; + private void internalClearKeyphraseStateLocked() { + mKeyphraseStarted = false; + mRequested = false; + + mKeyphraseId = INVALID_VALUE; + mRecognitionConfig = null; + mKeyphraseListener = null; + } + + private void internalClearGenericModelStateLocked() { + for (UUID modelId : mGenericModelDataMap.keySet()) { + ModelData modelData = mGenericModelDataMap.get(modelId); + modelData.clearState(); + modelData.clearCallback(); + } + } + + // This routine is a replacement for internalClearSoundModelLocked(). However, we + // should see why this should be different from internalClearKeyphraseStateLocked(). + private void internalClearKeyphraseSoundModelLocked() { + mCurrentKeyphraseModelHandle = INVALID_VALUE; mCurrentSoundModel = null; } @@ -577,19 +821,251 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { void dump(FileDescriptor fd, PrintWriter pw, String[] args) { synchronized (mLock) { pw.print(" module properties="); - pw.println(moduleProperties == null ? "null" : moduleProperties); + pw.println(mModuleProperties == null ? "null" : mModuleProperties); pw.print(" keyphrase ID="); pw.println(mKeyphraseId); - pw.print(" sound model handle="); pw.println(mCurrentSoundModelHandle); + pw.print(" sound model handle="); pw.println(mCurrentKeyphraseModelHandle); pw.print(" sound model UUID="); pw.println(mCurrentSoundModel == null ? "null" : mCurrentSoundModel.uuid); pw.print(" current listener="); - pw.println(mActiveListener == null ? "null" : mActiveListener.asBinder()); + pw.println(mKeyphraseListener == null ? "null" : mKeyphraseListener.asBinder()); pw.print(" requested="); pw.println(mRequested); - pw.print(" started="); pw.println(mStarted); + pw.print(" started="); pw.println(mKeyphraseStarted); pw.print(" call active="); pw.println(mCallActive); pw.print(" power save mode active="); pw.println(mIsPowerSaveMode); pw.print(" service disabled="); pw.println(mServiceDisabled); } } + + private void initializeTelephonyAndPowerStateListeners() { + // Get the current call state synchronously for the first recognition. + mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE; + + // Register for call state changes when the first call to start recognition occurs. + mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE); + + // Register for power saver mode changes when the first call to start recognition + // occurs. + if (mPowerSaveModeListener == null) { + mPowerSaveModeListener = new PowerSaveModeListener(); + mContext.registerReceiver(mPowerSaveModeListener, + new IntentFilter(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED)); + } + mIsPowerSaveMode = mPowerManager.isPowerSaveMode(); + } + + private ModelData getOrCreateGenericModelData(UUID modelId) { + ModelData modelData = mGenericModelDataMap.get(modelId); + if (modelData == null) { + modelData = new ModelData(modelId); + modelData.setTypeGeneric(); + mGenericModelDataMap.put(modelId, modelData); + } + return modelData; + } + + // Instead of maintaining a second hashmap of modelHandle -> ModelData, we just + // iterate through to find the right object (since we don't expect 100s of models + // to be stored). + private ModelData getModelDataFor(int modelHandle) { + // Fetch ModelData object corresponding to the model handle. + for (ModelData model : mGenericModelDataMap.values()) { + if (model.getHandle() == modelHandle) { + return model; + } + } + return null; + } + + // Whether we are allowed to run any recognition at all. The conditions that let us run + // a recognition include: no active phone call or not being in a power save mode. Also, + // the native service should be enabled. + private boolean isRecognitionAllowed() { + return !mCallActive && !mServiceDisabled && !mIsPowerSaveMode; + } + + private int startGenericRecognitionLocked(ModelData modelData, boolean notify) { + IRecognitionStatusCallback callback = modelData.getCallback(); + int handle = modelData.getHandle(); + RecognitionConfig config = modelData.getRecognitionConfig(); + if (callback == null || handle == INVALID_VALUE || config == null) { + // Nothing to do here. + Slog.w(TAG, "startGenericRecognition: Bad data passed in."); + return STATUS_ERROR; + } + + if (!isRecognitionAllowed()) { + // Nothing to do here. + Slog.w(TAG, "startGenericRecognition requested but not allowed."); + return STATUS_OK; + } + + int status = mModule.startRecognition(handle, config); + if (status != SoundTrigger.STATUS_OK) { + Slog.w(TAG, "startRecognition failed with " + status); + // Notify of error if needed. + if (notify) { + try { + callback.onError(status); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException in onError", e); + } + } + } else { + modelData.setStarted(); + // Notify of resume if needed. + if (notify) { + try { + callback.onRecognitionResumed(); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException in onRecognitionResumed", e); + } + } + } + return status; + } + + private int stopGenericRecognitionLocked(ModelData modelData, boolean notify) { + IRecognitionStatusCallback callback = modelData.getCallback(); + + // Stop recognition (only if we haven't been aborted). + int status = mModule.stopRecognition(modelData.getHandle()); + if (status != SoundTrigger.STATUS_OK) { + Slog.w(TAG, "stopRecognition call failed with " + status); + if (notify) { + try { + callback.onError(status); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException in onError", e); + } + } + } else { + modelData.setStopped(); + // Notify of pause if needed. + if (notify) { + try { + callback.onRecognitionPaused(); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException in onRecognitionPaused", e); + } + } + } + return status; + } + + // Computes whether we have any recognition running at all (voice or generic). Sets + // the mRecognitionRunning variable with the result. + private boolean computeRecognitionRunning() { + synchronized (mLock) { + if (mModuleProperties == null || mModule == null) { + mRecognitionRunning = false; + return mRecognitionRunning; + } + if (mKeyphraseListener != null && + mKeyphraseStarted && + mCurrentKeyphraseModelHandle != INVALID_VALUE && + mCurrentSoundModel != null) { + mRecognitionRunning = true; + return mRecognitionRunning; + } + for (UUID modelId : mGenericModelDataMap.keySet()) { + ModelData modelData = mGenericModelDataMap.get(modelId); + if (modelData.modelStarted()) { + mRecognitionRunning = true; + return mRecognitionRunning; + } + } + mRecognitionRunning = false; + } + return mRecognitionRunning; + } + + // This class encapsulates the callbacks, state, handles and any other information that + // represents a model. + private static class ModelData { + // Model not loaded (and hence not started). + static final int MODEL_NOTLOADED = 0; + + // Loaded implies model was successfully loaded. Model not started yet. + static final int MODEL_LOADED = 1; + + // Started implies model was successfully loaded and start was called. + static final int MODEL_STARTED = 2; + + // One of MODEL_NOTLOADED, MODEL_LOADED, MODEL_STARTED (which implies loaded). + private int mModelState; + + private UUID mModelId; + + // One of SoundModel.TYPE_GENERIC or SoundModel.TYPE_KEYPHRASE. Initially set + // to SoundModel.TYPE_UNKNOWN; + private int mModelType = SoundModel.TYPE_UNKNOWN; + private IRecognitionStatusCallback mCallback = null; + private SoundModel mSoundModel = null; + private RecognitionConfig mRecognitionConfig = null; + + + // Model handle is an integer used by the HAL as an identifier for sound + // models. + private int mModelHandle = INVALID_VALUE; + + ModelData(UUID modelId) { + mModelId = modelId; + } + + synchronized void setTypeGeneric() { + mModelType = SoundModel.TYPE_GENERIC_SOUND; + } + + synchronized void setCallback(IRecognitionStatusCallback callback) { + mCallback = callback; + } + + synchronized IRecognitionStatusCallback getCallback() { + return mCallback; + } + + synchronized boolean isModelLoaded() { + return (mModelState == MODEL_LOADED || mModelState == MODEL_STARTED) && + mSoundModel != null; + } + + synchronized void setStarted() { + mModelState = MODEL_STARTED; + } + + synchronized void setStopped() { + mModelState = MODEL_LOADED; + } + + synchronized boolean modelStarted() { + return mModelState == MODEL_STARTED; + } + + synchronized void clearState() { + mModelState = MODEL_NOTLOADED; + mSoundModel = null; + mModelHandle = INVALID_VALUE; + } + + synchronized void clearCallback() { + mCallback = null; + } + + synchronized void setHandle(int handle) { + mModelHandle = handle; + } + + synchronized void setRecognitionConfig(RecognitionConfig config) { + mRecognitionConfig = config; + } + + synchronized int getHandle() { + return mModelHandle; + } + + synchronized RecognitionConfig getRecognitionConfig() { + return mRecognitionConfig; + } + } } diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java index 682f4a412f0e..251f3146f8cc 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java @@ -15,6 +15,7 @@ */ package com.android.server.soundtrigger; +import static android.hardware.soundtrigger.SoundTrigger.STATUS_ERROR; import android.content.Context; import android.content.pm.PackageManager; @@ -47,13 +48,14 @@ import java.util.UUID; * @hide */ public class SoundTriggerService extends SystemService { - static final String TAG = "SoundTriggerService"; - static final boolean DEBUG = false; + private static final String TAG = "SoundTriggerService"; + private static final boolean DEBUG = true; final Context mContext; private final SoundTriggerServiceStub mServiceStub; private final LocalSoundTriggerService mLocalSoundTriggerService; private SoundTriggerDbHelper mDbHelper; + private SoundTriggerHelper mSoundTriggerHelper; public SoundTriggerService(Context context) { super(context); @@ -71,7 +73,8 @@ public class SoundTriggerService extends SystemService { @Override public void onBootPhase(int phase) { if (PHASE_SYSTEM_SERVICES_READY == phase) { - mLocalSoundTriggerService.initSoundTriggerHelper(); + initSoundTriggerHelper(); + mLocalSoundTriggerService.setSoundTriggerHelper(mSoundTriggerHelper); } else if (PHASE_THIRD_PARTY_APPS_CAN_START == phase) { mDbHelper = new SoundTriggerDbHelper(mContext); } @@ -85,6 +88,20 @@ public class SoundTriggerService extends SystemService { public void onSwitchUser(int userHandle) { } + private synchronized void initSoundTriggerHelper() { + if (mSoundTriggerHelper == null) { + mSoundTriggerHelper = new SoundTriggerHelper(mContext); + } + } + + private synchronized boolean isInitialized() { + if (mSoundTriggerHelper == null ) { + Slog.e(TAG, "SoundTriggerHelper not initialized."); + return false; + } + return true; + } + class SoundTriggerServiceStub extends ISoundTriggerService.Stub { @Override public boolean onTransact(int code, Parcel data, Parcel reply, int flags) @@ -102,19 +119,32 @@ public class SoundTriggerService extends SystemService { } @Override - public void startRecognition(ParcelUuid parcelUuid, IRecognitionStatusCallback callback) { + public int startRecognition(ParcelUuid parcelUuid, IRecognitionStatusCallback callback, + RecognitionConfig config) { enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); if (DEBUG) { Slog.i(TAG, "startRecognition(): Uuid : " + parcelUuid); } + if (!isInitialized()) return STATUS_ERROR; + + GenericSoundModel model = getSoundModel(parcelUuid); + if (model == null) { + Slog.e(TAG, "Null model in database for id: " + parcelUuid); + return STATUS_ERROR; + } + + return mSoundTriggerHelper.startGenericRecognition(parcelUuid.getUuid(), model, + callback, config); } @Override - public void stopRecognition(ParcelUuid parcelUuid, IRecognitionStatusCallback callback) { + public int stopRecognition(ParcelUuid parcelUuid, IRecognitionStatusCallback callback) { enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); if (DEBUG) { Slog.i(TAG, "stopRecognition(): Uuid : " + parcelUuid); } + if (!isInitialized()) return STATUS_ERROR; + return mSoundTriggerHelper.stopGenericRecognition(parcelUuid.getUuid(), callback); } @Override @@ -123,10 +153,8 @@ public class SoundTriggerService extends SystemService { if (DEBUG) { Slog.i(TAG, "getSoundModel(): id = " + soundModelId); } - SoundTrigger.GenericSoundModel model = mDbHelper.getGenericSoundModel(soundModelId.getUuid()); - if (model == null) { - Slog.e(TAG, "Null model in database."); - } + SoundTrigger.GenericSoundModel model = mDbHelper.getGenericSoundModel( + soundModelId.getUuid()); return model; } @@ -157,38 +185,49 @@ public class SoundTriggerService extends SystemService { mContext = context; } - void initSoundTriggerHelper() { - if (mSoundTriggerHelper == null) { - mSoundTriggerHelper = new SoundTriggerHelper(mContext); - } + synchronized void setSoundTriggerHelper(SoundTriggerHelper helper) { + mSoundTriggerHelper = helper; } @Override public int startRecognition(int keyphraseId, KeyphraseSoundModel soundModel, IRecognitionStatusCallback listener, RecognitionConfig recognitionConfig) { - return mSoundTriggerHelper.startRecognition(keyphraseId, soundModel, listener, + if (!isInitialized()) return STATUS_ERROR; + return mSoundTriggerHelper.startKeyphraseRecognition(keyphraseId, soundModel, listener, recognitionConfig); } @Override - public int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { - return mSoundTriggerHelper.stopRecognition(keyphraseId, listener); + public synchronized int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { + if (!isInitialized()) return STATUS_ERROR; + return mSoundTriggerHelper.stopKeyphraseRecognition(keyphraseId, listener); } @Override public void stopAllRecognitions() { + if (!isInitialized()) return; mSoundTriggerHelper.stopAllRecognitions(); } @Override public ModuleProperties getModuleProperties() { + if (!isInitialized()) return null; return mSoundTriggerHelper.getModuleProperties(); } @Override public void dump(FileDescriptor fd, PrintWriter pw, String[] args) { + if (!isInitialized()) return; mSoundTriggerHelper.dump(fd, pw, args); } + + private synchronized boolean isInitialized() { + if (mSoundTriggerHelper == null ) { + Slog.e(TAG, "SoundTriggerHelper not initialized."); + return false; + } + return true; + } } private void enforceCallingPermission(String permission) { diff --git a/tests/SoundTriggerTestApp/Android.mk b/tests/SoundTriggerTestApp/Android.mk index 7bcab5e53772..c327b0956811 100644 --- a/tests/SoundTriggerTestApp/Android.mk +++ b/tests/SoundTriggerTestApp/Android.mk @@ -8,5 +8,6 @@ LOCAL_PACKAGE_NAME := SoundTriggerTestApp LOCAL_MODULE_TAGS := optional LOCAL_PRIVILEGED_MODULE := true +LOCAL_CERTIFICATE := platform include $(BUILD_PACKAGE) diff --git a/tests/SoundTriggerTestApp/AndroidManifest.xml b/tests/SoundTriggerTestApp/AndroidManifest.xml index 40619da156ee..a72b3ddb7c0c 100644 --- a/tests/SoundTriggerTestApp/AndroidManifest.xml +++ b/tests/SoundTriggerTestApp/AndroidManifest.xml @@ -2,16 +2,22 @@ package="com.android.test.soundtrigger"> <uses-permission android:name="android.permission.MANAGE_SOUND_TRIGGER" /> - <application - android:permission="android.permission.MANAGE_SOUND_TRIGGER"> + <application> <activity android:name="TestSoundTriggerActivity" android:label="SoundTrigger Test Application" - android:theme="@android:style/Theme.Material.Light.Voice"> + android:theme="@android:style/Theme.Material"> + <!-- <intent-filter> <action android:name="com.android.intent.action.MANAGE_SOUND_TRIGGER" /> <category android:name="android.intent.category.DEFAULT" /> </intent-filter> + --> + <intent-filter> + <action android:name="android.intent.action.MAIN" /> + <category android:name="android.intent.category.DEFAULT" /> + <category android:name="android.intent.category.LAUNCHER" /> + </intent-filter> </activity> </application> </manifest> diff --git a/tests/SoundTriggerTestApp/res/layout/main.xml b/tests/SoundTriggerTestApp/res/layout/main.xml index 9d2b9d92c016..5ecc7705cd75 100644 --- a/tests/SoundTriggerTestApp/res/layout/main.xml +++ b/tests/SoundTriggerTestApp/res/layout/main.xml @@ -18,6 +18,11 @@ <LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" android:layout_width="match_parent" android:layout_height="match_parent" + android:orientation="vertical" + > +<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" + android:layout_width="match_parent" + android:layout_height="wrap_content" > <Button @@ -37,7 +42,57 @@ <Button android:layout_width="wrap_content" android:layout_height="wrap_content" + android:text="@string/start_recog" + android:onClick="onStartRecognitionButtonClicked" + android:padding="20dp" /> + + <Button + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="@string/stop_recog" + android:onClick="onStopRecognitionButtonClicked" + android:padding="20dp" /> + + <Button + android:layout_width="wrap_content" + android:layout_height="wrap_content" android:text="@string/unenroll" android:onClick="onUnEnrollButtonClicked" android:padding="20dp" /> -</LinearLayout>
\ No newline at end of file + +</LinearLayout> + +<RadioGroup xmlns:android="http://schemas.android.com/apk/res/android" + android:layout_width="fill_parent" + android:layout_height="wrap_content" + android:padding="20dp" + android:orientation="vertical"> + <RadioButton android:id="@+id/model_one" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="@string/model_one" + android:onClick="onRadioButtonClicked"/> + <RadioButton android:id="@+id/model_two" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="@string/model_two" + android:onClick="onRadioButtonClicked"/> + <RadioButton android:id="@+id/model_three" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="@string/model_three" + android:onClick="onRadioButtonClicked"/> +</RadioGroup> + + <TextView + android:id="@+id/console" + android:gravity="left" + android:paddingTop="20pt" + android:layout_height="fill_parent" + android:layout_width="match_parent" + android:maxLines="40" + android:textSize="14dp" + android:scrollbars = "vertical" + android:text="@string/none"> + </TextView> +</LinearLayout> diff --git a/tests/SoundTriggerTestApp/res/values/strings.xml b/tests/SoundTriggerTestApp/res/values/strings.xml index 07bac2a263ef..5f0fb1daf3e9 100644 --- a/tests/SoundTriggerTestApp/res/values/strings.xml +++ b/tests/SoundTriggerTestApp/res/values/strings.xml @@ -16,7 +16,13 @@ --> <resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> - <string name="enroll">Enroll</string> - <string name="reenroll">Re-enroll</string> - <string name="unenroll">Un-enroll</string> -</resources>
\ No newline at end of file + <string name="enroll">Load</string> + <string name="reenroll">Re-load</string> + <string name="unenroll">Un-load</string> + <string name="start_recog">Start</string> + <string name="stop_recog">Stop</string> + <string name="model_one">Model One</string> + <string name="model_two">Model Two</string> + <string name="model_three">Model Three</string> + <string name="none">Debug messages appear here:</string> +</resources> diff --git a/tests/SoundTriggerTestApp/src/com/android/test/soundtrigger/SoundTriggerUtil.java b/tests/SoundTriggerTestApp/src/com/android/test/soundtrigger/SoundTriggerUtil.java index 4702835eae43..1c95c25370d2 100644 --- a/tests/SoundTriggerTestApp/src/com/android/test/soundtrigger/SoundTriggerUtil.java +++ b/tests/SoundTriggerTestApp/src/com/android/test/soundtrigger/SoundTriggerUtil.java @@ -20,6 +20,7 @@ import android.annotation.Nullable; import android.content.Context; import android.hardware.soundtrigger.SoundTrigger; import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; +import android.media.soundtrigger.SoundTriggerDetector; import android.media.soundtrigger.SoundTriggerManager; import android.os.RemoteException; import android.os.ServiceManager; @@ -28,6 +29,7 @@ import android.util.Log; import com.android.internal.app.ISoundTriggerService; +import java.lang.RuntimeException; import java.util.UUID; /** @@ -56,6 +58,9 @@ public class SoundTriggerUtil { */ public boolean addOrUpdateSoundModel(GenericSoundModel soundModel) { try { + if (soundModel == null) { + throw new RuntimeException("Bad sound model"); + } mSoundTriggerService.updateSoundModel(soundModel); } catch (RemoteException e) { Log.e(TAG, "RemoteException in updateSoundModel", e); @@ -112,4 +117,10 @@ public class SoundTriggerUtil { public void deleteSoundModelUsingManager(UUID modelId) { mSoundTriggerManager.deleteModel(modelId); } + + public SoundTriggerDetector createSoundTriggerDetector(UUID modelId, + SoundTriggerDetector.Callback callback) { + return mSoundTriggerManager.createSoundTriggerDetector(modelId, callback, null); + } + } diff --git a/tests/SoundTriggerTestApp/src/com/android/test/soundtrigger/TestSoundTriggerActivity.java b/tests/SoundTriggerTestApp/src/com/android/test/soundtrigger/TestSoundTriggerActivity.java index 966179b8dbd5..96a69661a7aa 100644 --- a/tests/SoundTriggerTestApp/src/com/android/test/soundtrigger/TestSoundTriggerActivity.java +++ b/tests/SoundTriggerTestApp/src/com/android/test/soundtrigger/TestSoundTriggerActivity.java @@ -22,11 +22,17 @@ import java.util.UUID; import android.app.Activity; import android.hardware.soundtrigger.SoundTrigger; import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; +import android.media.AudioFormat; +import android.media.soundtrigger.SoundTriggerDetector; import android.media.soundtrigger.SoundTriggerManager; +import android.text.Editable; +import android.text.method.ScrollingMovementMethod; import android.os.Bundle; import android.os.UserManager; import android.util.Log; import android.view.View; +import android.widget.RadioButton; +import android.widget.TextView; import android.widget.Toast; public class TestSoundTriggerActivity extends Activity { @@ -35,42 +41,75 @@ public class TestSoundTriggerActivity extends Activity { private SoundTriggerUtil mSoundTriggerUtil; private Random mRandom; - private UUID mModelUuid = UUID.randomUUID(); + private UUID mModelUuid1 = UUID.randomUUID(); private UUID mModelUuid2 = UUID.randomUUID(); + private UUID mModelUuid3 = UUID.randomUUID(); private UUID mVendorUuid = UUID.randomUUID(); + private SoundTriggerDetector mDetector1 = null; + private SoundTriggerDetector mDetector2 = null; + private SoundTriggerDetector mDetector3 = null; + + private TextView mDebugView = null; + private int mSelectedModelId = 1; + @Override protected void onCreate(Bundle savedInstanceState) { if (DBG) Log.d(TAG, "onCreate"); super.onCreate(savedInstanceState); setContentView(R.layout.main); + mDebugView = (TextView) findViewById(R.id.console); + mDebugView.setText(mDebugView.getText(), TextView.BufferType.EDITABLE); + mDebugView.setMovementMethod(new ScrollingMovementMethod()); mSoundTriggerUtil = new SoundTriggerUtil(this); mRandom = new Random(); } + private void postMessage(String msg) { + Log.i(TAG, "Posted: " + msg); + ((Editable) mDebugView.getText()).append(msg + "\n"); + } + + private UUID getSelectedUuid() { + if (mSelectedModelId == 2) return mModelUuid2; + if (mSelectedModelId == 3) return mModelUuid3; + return mModelUuid1; // Default. + } + + private void setDetector(SoundTriggerDetector detector) { + if (mSelectedModelId == 2) mDetector2 = detector; + if (mSelectedModelId == 3) mDetector3 = detector; + mDetector1 = detector; + } + + private SoundTriggerDetector getDetector() { + if (mSelectedModelId == 2) return mDetector2; + if (mSelectedModelId == 3) return mDetector3; + return mDetector1; + } + /** * Called when the user clicks the enroll button. * Performs a fresh enrollment. */ public void onEnrollButtonClicked(View v) { + postMessage("Loading model: " + mSelectedModelId); // Generate a fake model to push. byte[] data = new byte[1024]; mRandom.nextBytes(data); - GenericSoundModel model = new GenericSoundModel(mModelUuid, mVendorUuid, data); + UUID modelUuid = getSelectedUuid(); + GenericSoundModel model = new GenericSoundModel(modelUuid, mVendorUuid, data); boolean status = mSoundTriggerUtil.addOrUpdateSoundModel(model); if (status) { Toast.makeText( - this, "Successfully created sound trigger model UUID=" + mModelUuid, Toast.LENGTH_SHORT) - .show(); + this, "Successfully created sound trigger model UUID=" + modelUuid, + Toast.LENGTH_SHORT).show(); } else { - Toast.makeText(this, "Failed to enroll!!!" + mModelUuid, Toast.LENGTH_SHORT).show(); + Toast.makeText(this, "Failed to enroll!!!" + modelUuid, Toast.LENGTH_SHORT).show(); } // Test the SoundManager API. - SoundTriggerManager.Model tmpModel = SoundTriggerManager.Model.create(mModelUuid2, - mVendorUuid, data); - mSoundTriggerUtil.addOrUpdateSoundModel(tmpModel); } /** @@ -78,12 +117,14 @@ public class TestSoundTriggerActivity extends Activity { * Clears the enrollment information for the user. */ public void onUnEnrollButtonClicked(View v) { - GenericSoundModel soundModel = mSoundTriggerUtil.getSoundModel(mModelUuid); + postMessage("Unloading model: " + mSelectedModelId); + UUID modelUuid = getSelectedUuid(); + GenericSoundModel soundModel = mSoundTriggerUtil.getSoundModel(modelUuid); if (soundModel == null) { Toast.makeText(this, "Sound model not found!!!", Toast.LENGTH_SHORT).show(); return; } - boolean status = mSoundTriggerUtil.deleteSoundModel(mModelUuid); + boolean status = mSoundTriggerUtil.deleteSoundModel(mModelUuid1); if (status) { Toast.makeText(this, "Successfully deleted model UUID=" + soundModel.uuid, Toast.LENGTH_SHORT) @@ -91,7 +132,6 @@ public class TestSoundTriggerActivity extends Activity { } else { Toast.makeText(this, "Failed to delete sound model!!!", Toast.LENGTH_SHORT).show(); } - mSoundTriggerUtil.deleteSoundModelUsingManager(mModelUuid2); } /** @@ -99,7 +139,9 @@ public class TestSoundTriggerActivity extends Activity { * Uses the previously enrolled sound model and makes changes to it before pushing it back. */ public void onReEnrollButtonClicked(View v) { - GenericSoundModel soundModel = mSoundTriggerUtil.getSoundModel(mModelUuid); + postMessage("Re-loading model: " + mSelectedModelId); + UUID modelUuid = getSelectedUuid(); + GenericSoundModel soundModel = mSoundTriggerUtil.getSoundModel(modelUuid); if (soundModel == null) { Toast.makeText(this, "Sound model not found!!!", Toast.LENGTH_SHORT).show(); return; @@ -118,4 +160,86 @@ public class TestSoundTriggerActivity extends Activity { Toast.makeText(this, "Failed to re-enroll!!!", Toast.LENGTH_SHORT).show(); } } + + public void onStartRecognitionButtonClicked(View v) { + UUID modelUuid = getSelectedUuid(); + SoundTriggerDetector detector = getDetector(); + if (detector == null) { + Log.i(TAG, "Created an instance of the SoundTriggerDetector."); + detector = mSoundTriggerUtil.createSoundTriggerDetector(modelUuid, + new DetectorCallback()); + setDetector(detector); + } + postMessage("Triggering start recognition for model: " + mSelectedModelId); + if (!detector.startRecognition( + SoundTriggerDetector.RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS)) { + Log.e(TAG, "Fast failure attempting to start recognition."); + } + } + + public void onStopRecognitionButtonClicked(View v) { + SoundTriggerDetector detector = getDetector(); + if (detector == null) { + Log.e(TAG, "Stop called on null detector."); + return; + } + postMessage("Triggering stop recognition for model: " + mSelectedModelId); + if (!detector.stopRecognition()) { + Log.e(TAG, "Fast failure attempting to stop recognition."); + } + } + + public void onRadioButtonClicked(View view) { + // Is the button now checked? + boolean checked = ((RadioButton) view).isChecked(); + // Check which radio button was clicked + switch(view.getId()) { + case R.id.model_one: + if (checked) mSelectedModelId = 1; + postMessage("Selected model one."); + break; + case R.id.model_two: + if (checked) mSelectedModelId = 2; + postMessage("Selected model two."); + break; + case R.id.model_three: + if (checked) mSelectedModelId = 3; + postMessage("Selected model three."); + break; + } + } + + // Implementation of SoundTriggerDetector.Callback. + public class DetectorCallback extends SoundTriggerDetector.Callback { + public void onAvailabilityChanged(int status) { + postMessage("Availability changed to: " + status); + } + + public void onDetected(SoundTriggerDetector.EventPayload event) { + postMessage("onDetected(): " + eventPayloadToString(event)); + } + + public void onError() { + postMessage("onError()"); + } + + public void onRecognitionPaused() { + postMessage("onRecognitionPaused()"); + } + + public void onRecognitionResumed() { + postMessage("onRecognitionResumed()"); + } + } + + private String eventPayloadToString(SoundTriggerDetector.EventPayload event) { + String result = "EventPayload("; + AudioFormat format = event.getCaptureAudioFormat(); + result = result + "AudioFormat: " + ((format == null) ? "null" : format.toString()); + byte[] triggerAudio = event.getTriggerAudio(); + result = result + "TriggerAudio: " + (triggerAudio == null ? "null" : triggerAudio.length); + result = result + "CaptureSession: " + event.getCaptureSession(); + result += " )"; + return result; + } } |