diff options
4 files changed, 316 insertions, 25 deletions
diff --git a/core/java/android/service/voice/HotwordAudioStream.java b/core/java/android/service/voice/HotwordAudioStream.java index c44542ceec6b..5442860df007 100644 --- a/core/java/android/service/voice/HotwordAudioStream.java +++ b/core/java/android/service/voice/HotwordAudioStream.java @@ -123,6 +123,16 @@ public final class HotwordAudioStream implements Parcelable { } } + /** + * Provides an instance of {@link Builder} with state corresponding to this instance. + * @hide + */ + public Builder buildUpon() { + return new Builder(mAudioFormat, mAudioStreamParcelFileDescriptor) + .setTimestamp(mTimestamp) + .setMetadata(mMetadata); + } + /* package-private */ HotwordAudioStream( @NonNull AudioFormat audioFormat, diff --git a/core/java/android/service/voice/HotwordDetectedResult.java b/core/java/android/service/voice/HotwordDetectedResult.java index 0468a619b798..990e136197d9 100644 --- a/core/java/android/service/voice/HotwordDetectedResult.java +++ b/core/java/android/service/voice/HotwordDetectedResult.java @@ -397,6 +397,25 @@ public final class HotwordDetectedResult implements Parcelable { } } + /** + * Provides an instance of {@link Builder} with state corresponding to this instance. + * @hide + */ + public Builder buildUpon() { + return new Builder() + .setConfidenceLevel(mConfidenceLevel) + .setMediaSyncEvent(mMediaSyncEvent) + .setHotwordOffsetMillis(mHotwordOffsetMillis) + .setHotwordDurationMillis(mHotwordDurationMillis) + .setAudioChannel(mAudioChannel) + .setHotwordDetectionPersonalized(mHotwordDetectionPersonalized) + .setScore(mScore) + .setPersonalizedScore(mPersonalizedScore) + .setHotwordPhraseId(mHotwordPhraseId) + .setAudioStreams(mAudioStreams) + .setExtras(mExtras); + } + // Code below generated by codegen v1.0.23. @@ -993,10 +1012,10 @@ public final class HotwordDetectedResult implements Parcelable { } @DataClass.Generated( - time = 1668405106028L, + time = 1668466781144L, codegenVersion = "1.0.23", sourceFile = "frameworks/base/core/java/android/service/voice/HotwordDetectedResult.java", - inputSignatures = "public static final int CONFIDENCE_LEVEL_NONE\npublic static final int CONFIDENCE_LEVEL_LOW\npublic static final int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final int CONFIDENCE_LEVEL_HIGH\npublic static final int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final int HOTWORD_OFFSET_UNSET\npublic static final int AUDIO_CHANNEL_UNSET\nprivate static final int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final int LIMIT_AUDIO_CHANNEL_MAX_VALUE\npublic static final java.lang.String EXTRA_PROXIMITY_METERS\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate int mHotwordOffsetMillis\nprivate int mHotwordDurationMillis\nprivate int mAudioChannel\nprivate boolean mHotwordDetectionPersonalized\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int sMaxBundleSize\nprivate static int defaultConfidenceLevel()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\npublic static int getParcelableSize(android.os.Parcelable)\npublic static int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static int bitCount(long)\nprivate void onConstructed()\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []") + inputSignatures = "public static final int CONFIDENCE_LEVEL_NONE\npublic static final int CONFIDENCE_LEVEL_LOW\npublic static final int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final int CONFIDENCE_LEVEL_HIGH\npublic static final int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final int HOTWORD_OFFSET_UNSET\npublic static final int AUDIO_CHANNEL_UNSET\nprivate static final int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final int LIMIT_AUDIO_CHANNEL_MAX_VALUE\npublic static final java.lang.String EXTRA_PROXIMITY_METERS\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate int mHotwordOffsetMillis\nprivate int mHotwordDurationMillis\nprivate int mAudioChannel\nprivate boolean mHotwordDetectionPersonalized\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int sMaxBundleSize\nprivate static int defaultConfidenceLevel()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\npublic static int getParcelableSize(android.os.Parcelable)\npublic static int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static int bitCount(long)\nprivate void onConstructed()\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\npublic android.service.voice.HotwordDetectedResult.Builder buildUpon()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []") @Deprecated private void __metadata() {} diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java new file mode 100644 index 000000000000..f9211181c924 --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java @@ -0,0 +1,233 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import static android.app.AppOpsManager.MODE_ALLOWED; + +import static com.android.server.voiceinteraction.HotwordDetectionConnection.DEBUG; + +import android.annotation.NonNull; +import android.app.AppOpsManager; +import android.media.permission.Identity; +import android.os.ParcelFileDescriptor; +import android.service.voice.HotwordAudioStream; +import android.service.voice.HotwordDetectedResult; +import android.util.Pair; +import android.util.Slog; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +final class HotwordAudioStreamManager { + + private static final String TAG = "HotwordAudioStreamManager"; + private static final String OP_MESSAGE = "Streaming hotword audio to VoiceInteractionService"; + private static final String TASK_ID_PREFIX = "HotwordDetectedResult@"; + private static final String THREAD_NAME_PREFIX = "Copy-"; + + private final AppOpsManager mAppOpsManager; + private final Identity mVoiceInteractorIdentity; + private final ExecutorService mExecutorService = Executors.newCachedThreadPool(); + + HotwordAudioStreamManager(@NonNull AppOpsManager appOpsManager, + @NonNull Identity voiceInteractorIdentity) { + mAppOpsManager = appOpsManager; + mVoiceInteractorIdentity = voiceInteractorIdentity; + } + + /** + * Starts copying the audio streams in the given {@link HotwordDetectedResult}. + * <p> + * The returned {@link HotwordDetectedResult} is identical the one that was passed in, except + * that the {@link ParcelFileDescriptor}s within {@link HotwordDetectedResult#getAudioStreams()} + * are replaced with descriptors from pipes managed by {@link HotwordAudioStreamManager}. The + * returned value should be passed on to the client (i.e., the voice interactor). + * </p> + * + * @throws IOException If there was an error creating the managed pipe. + */ + @NonNull + public HotwordDetectedResult startCopyingAudioStreams(@NonNull HotwordDetectedResult result) + throws IOException { + List<HotwordAudioStream> audioStreams = result.getAudioStreams(); + if (audioStreams.isEmpty()) { + return result; + } + + List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size()); + List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks = new ArrayList<>( + audioStreams.size()); + for (HotwordAudioStream audioStream : audioStreams) { + ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe(); + ParcelFileDescriptor clientAudioSource = clientPipe[0]; + ParcelFileDescriptor clientAudioSink = clientPipe[1]; + HotwordAudioStream newAudioStream = + audioStream.buildUpon().setAudioStreamParcelFileDescriptor( + clientAudioSource).build(); + newAudioStreams.add(newAudioStream); + + ParcelFileDescriptor serviceAudioSource = + audioStream.getAudioStreamParcelFileDescriptor(); + sourcesAndSinks.add(new Pair<>(serviceAudioSource, clientAudioSink)); + } + + String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result); + mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, sourcesAndSinks)); + + return result.buildUpon().setAudioStreams(newAudioStreams).build(); + } + + private class HotwordDetectedResultCopyTask implements Runnable { + private final String mResultTaskId; + private final List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> mSourcesAndSinks; + private final ExecutorService mExecutorService = Executors.newCachedThreadPool(); + + HotwordDetectedResultCopyTask(String resultTaskId, + List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks) { + mResultTaskId = resultTaskId; + mSourcesAndSinks = sourcesAndSinks; + } + + @Override + public void run() { + Thread.currentThread().setName(THREAD_NAME_PREFIX + mResultTaskId); + int size = mSourcesAndSinks.size(); + List<SingleAudioStreamCopyTask> tasks = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink = + mSourcesAndSinks.get(i); + ParcelFileDescriptor serviceAudioSource = sourceAndSink.first; + ParcelFileDescriptor clientAudioSink = sourceAndSink.second; + String streamTaskId = mResultTaskId + "@" + i; + tasks.add(new SingleAudioStreamCopyTask(streamTaskId, serviceAudioSource, + clientAudioSink)); + } + + if (mAppOpsManager.startOpNoThrow(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD, + mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, + mVoiceInteractorIdentity.attributionTag, OP_MESSAGE) == MODE_ALLOWED) { + try { + // TODO(b/244599891): Set timeout, close after inactivity + mExecutorService.invokeAll(tasks); + } catch (InterruptedException e) { + Slog.e(TAG, mResultTaskId + ": Task was interrupted", e); + bestEffortPropagateError(e.getMessage()); + } finally { + mAppOpsManager.finishOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD, + mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, + mVoiceInteractorIdentity.attributionTag); + } + } else { + bestEffortPropagateError( + "Failed to obtain RECORD_AUDIO_HOTWORD permission for " + + SoundTriggerSessionPermissionsDecorator.toString( + mVoiceInteractorIdentity)); + } + } + + private void bestEffortPropagateError(@NonNull String errorMessage) { + try { + for (Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink : + mSourcesAndSinks) { + ParcelFileDescriptor serviceAudioSource = sourceAndSink.first; + ParcelFileDescriptor clientAudioSink = sourceAndSink.second; + serviceAudioSource.closeWithError(errorMessage); + clientAudioSink.closeWithError(errorMessage); + } + } catch (IOException e) { + Slog.e(TAG, mResultTaskId + ": Failed to propagate error", e); + } + } + } + + private static class SingleAudioStreamCopyTask implements Callable<Void> { + // TODO: Make this buffer size customizable from updateState() + private static final int COPY_BUFFER_LENGTH = 1_024; + + private final String mStreamTaskId; + private final ParcelFileDescriptor mAudioSource; + private final ParcelFileDescriptor mAudioSink; + + SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource, + ParcelFileDescriptor audioSink) { + mStreamTaskId = streamTaskId; + mAudioSource = audioSource; + mAudioSink = audioSink; + } + + @Override + public Void call() throws Exception { + Thread.currentThread().setName(THREAD_NAME_PREFIX + mStreamTaskId); + + // Note: We are intentionally NOT using try-with-resources here. If we did, + // the ParcelFileDescriptors will be automatically closed WITHOUT errors before we go + // into the IOException-catch block. We want to propagate the error while closing the + // PFDs. + InputStream fis = null; + OutputStream fos = null; + try { + fis = new ParcelFileDescriptor.AutoCloseInputStream(mAudioSource); + fos = new ParcelFileDescriptor.AutoCloseOutputStream(mAudioSink); + byte[] buffer = new byte[COPY_BUFFER_LENGTH]; + while (true) { + if (Thread.interrupted()) { + Slog.e(TAG, + mStreamTaskId + ": SingleAudioStreamCopyTask task was interrupted"); + break; + } + + int bytesRead = fis.read(buffer); + if (bytesRead < 0) { + Slog.i(TAG, mStreamTaskId + ": Reached end of audio stream"); + break; + } + if (bytesRead > 0) { + if (DEBUG) { + // TODO(b/244599440): Add proper logging + Slog.d(TAG, mStreamTaskId + ": Copied " + bytesRead + + " bytes from audio stream. First 20 bytes=" + Arrays.toString( + Arrays.copyOfRange(buffer, 0, 20))); + } + fos.write(buffer, 0, bytesRead); + } + // TODO(b/244599891): Close PFDs after inactivity + } + } catch (IOException e) { + mAudioSource.closeWithError(e.getMessage()); + mAudioSink.closeWithError(e.getMessage()); + Slog.e(TAG, mStreamTaskId + ": Failed to copy audio stream", e); + } finally { + if (fis != null) { + fis.close(); + } + if (fos != null) { + fos.close(); + } + } + + return null; + } + } + +} diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index a6e1a3256cb6..ee8070888725 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -59,6 +59,7 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPH import android.annotation.NonNull; import android.annotation.Nullable; +import android.app.AppOpsManager; import android.attention.AttentionManagerInternal; import android.content.ComponentName; import android.content.ContentCaptureOptions; @@ -137,6 +138,7 @@ final class HotwordDetectionConnection { // The error codes are used for onError callback private static final int HOTWORD_DETECTION_SERVICE_DIED = -1; private static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2; + private static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; // Hotword metrics private static final int METRICS_INIT_UNKNOWN_TIMEOUT = @@ -168,6 +170,8 @@ final class HotwordDetectionConnection { // TODO: This may need to be a Handler(looper) private final ScheduledExecutorService mScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + private final AppOpsManager mAppOpsManager; + private final HotwordAudioStreamManager mHotwordAudioStreamManager; @Nullable private final ScheduledFuture<?> mCancellationTaskFuture; private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied; @@ -228,6 +232,9 @@ final class HotwordDetectionConnection { mContext = context; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; + mAppOpsManager = mContext.getSystemService(AppOpsManager.class); + mHotwordAudioStreamManager = new HotwordAudioStreamManager(mAppOpsManager, + mVoiceInteractorIdentity); mDetectionComponentName = serviceName; mUser = userId; mCallback = callback; @@ -482,13 +489,19 @@ final class HotwordDetectionConnection { return; } saveProximityMetersToBundle(result); - mSoftwareCallback.onDetected(result, null, null); - if (result != null) { - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(result) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + result); - } + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result); + } catch (IOException e) { + // TODO: Write event + mSoftwareCallback.onError(); + return; + } + mSoftwareCallback.onDetected(newResult, null, null); + Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + newResult); } } } @@ -660,6 +673,7 @@ final class HotwordDetectionConnection { try { enforcePermissionsForDataDelivery(); } catch (SecurityException e) { + Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); HotwordMetricsLogger.writeKeyphraseTriggerEvent( mDetectorType, METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); @@ -667,13 +681,19 @@ final class HotwordDetectionConnection { return; } saveProximityMetersToBundle(result); - externalCallback.onKeyphraseDetected(recognitionEvent, result); - if (result != null) { - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(result) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + result); - } + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result); + } catch (IOException e) { + // TODO: Write event + externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); + return; + } + externalCallback.onKeyphraseDetected(recognitionEvent, newResult); + Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + newResult); } } } @@ -757,6 +777,7 @@ final class HotwordDetectionConnection { } private void restartProcessLocked() { + // TODO(b/244598068): Check HotwordAudioStreamManager first Slog.v(TAG, "Restarting hotword detection process"); ServiceConnection oldConnection = mRemoteHotwordDetectionService; HotwordDetectionServiceIdentity previousIdentity = mIdentity; @@ -991,16 +1012,24 @@ final class HotwordDetectionConnection { callback.onError(); return; } - callback.onDetected(triggerResult, null /* audioFormat */, + HotwordDetectedResult newResult; + try { + newResult = + mHotwordAudioStreamManager.startCopyingAudioStreams( + triggerResult); + } catch (IOException e) { + // TODO: Write event + callback.onError(); + return; + } + callback.onDetected(newResult, null /* audioFormat */, null /* audioStream */); - if (triggerResult != null) { - Slog.i(TAG, "Egressed " - + HotwordDetectedResult.getUsageSize(triggerResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, - "Egressed detected result: " + triggerResult); - } + Slog.i(TAG, "Egressed " + + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, + "Egressed detected result: " + newResult); } } }); |