diff options
10 files changed, 526 insertions, 34 deletions
diff --git a/core/api/system-current.txt b/core/api/system-current.txt index f30c8cf15dc1..4aae6e07e838 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -13176,10 +13176,12 @@ package android.service.voice { method public static int getMaxBundleSize(); method public static int getMaxHotwordPhraseId(); method public static int getMaxScore(); + method @FlaggedApi("android.service.voice.flags.allow_speaker_id_egress") public static int getMaxSpeakerId(); method @Nullable public android.media.MediaSyncEvent getMediaSyncEvent(); method public int getPersonalizedScore(); method public int getProximity(); method public int getScore(); + method @FlaggedApi("android.service.voice.flags.allow_speaker_id_egress") public int getSpeakerId(); method public boolean isHotwordDetectionPersonalized(); method public void writeToParcel(@NonNull android.os.Parcel, int); field public static final int AUDIO_CHANNEL_UNSET = -1; // 0xffffffff @@ -13213,6 +13215,7 @@ package android.service.voice { method @NonNull public android.service.voice.HotwordDetectedResult.Builder setMediaSyncEvent(@NonNull android.media.MediaSyncEvent); method @NonNull public android.service.voice.HotwordDetectedResult.Builder setPersonalizedScore(int); method @NonNull public android.service.voice.HotwordDetectedResult.Builder setScore(int); + method @FlaggedApi("android.service.voice.flags.allow_speaker_id_egress") @NonNull public android.service.voice.HotwordDetectedResult.Builder setSpeakerId(int); } public abstract class HotwordDetectionService extends android.app.Service implements android.service.voice.SandboxedDetectionInitializer { @@ -13351,6 +13354,22 @@ package android.service.voice { field public static final int ERROR_CODE_UNKNOWN = 0; // 0x0 } + @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public final class VisualQueryDetectedResult implements android.os.Parcelable { + method public int describeContents(); + method public static int getMaxSpeakerId(); + method @NonNull public String getPartialQuery(); + method public int getSpeakerId(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator<android.service.voice.VisualQueryDetectedResult> CREATOR; + } + + public static final class VisualQueryDetectedResult.Builder { + ctor public VisualQueryDetectedResult.Builder(); + method @NonNull public android.service.voice.VisualQueryDetectedResult build(); + method @NonNull public android.service.voice.VisualQueryDetectedResult.Builder setPartialQuery(@NonNull String); + method @NonNull public android.service.voice.VisualQueryDetectedResult.Builder setSpeakerId(int); + } + public abstract class VisualQueryDetectionService extends android.app.Service implements android.service.voice.SandboxedDetectionInitializer { ctor public VisualQueryDetectionService(); method public final void finishQuery() throws java.lang.IllegalStateException; @@ -13362,6 +13381,7 @@ package android.service.voice { method public void onUpdateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, long, @Nullable java.util.function.IntConsumer); method public final void rejectQuery() throws java.lang.IllegalStateException; method public final void streamQuery(@NonNull String) throws java.lang.IllegalStateException; + method @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public final void streamQuery(@NonNull android.service.voice.VisualQueryDetectedResult); field public static final String SERVICE_INTERFACE = "android.service.voice.VisualQueryDetectionService"; } @@ -13390,6 +13410,7 @@ package android.service.voice { public static interface VisualQueryDetector.Callback { method public void onFailure(@NonNull android.service.voice.VisualQueryDetectionServiceFailure); method public void onQueryDetected(@NonNull String); + method @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public default void onQueryDetected(@NonNull android.service.voice.VisualQueryDetectedResult); method public void onQueryFinished(); method public void onQueryRejected(); method public void onUnknownFailure(@NonNull String); diff --git a/core/java/android/service/voice/HotwordDetectedResult.java b/core/java/android/service/voice/HotwordDetectedResult.java index 1e08fd8061e0..61408127893f 100644 --- a/core/java/android/service/voice/HotwordDetectedResult.java +++ b/core/java/android/service/voice/HotwordDetectedResult.java @@ -16,6 +16,7 @@ package android.service.voice; +import android.annotation.FlaggedApi; import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; @@ -26,6 +27,7 @@ import android.media.MediaSyncEvent; import android.os.Parcel; import android.os.Parcelable; import android.os.PersistableBundle; +import android.service.voice.flags.Flags; import com.android.internal.R; import com.android.internal.util.DataClass; @@ -130,6 +132,21 @@ public final class HotwordDetectedResult implements Parcelable { @Retention(RetentionPolicy.SOURCE) public @interface ProximityValue {} + /** Id of the current speaker + * + * <p>Only values between 0 and {@link #getMaxSpeakerId} (inclusive) are accepted. + */ + private final int mSpeakerId; + private static int defaultSpeakerId() { + return 0; + } + + /** Maximum number of active speaker ids. **/ + @FlaggedApi(Flags.FLAG_ALLOW_SPEAKER_ID_EGRESS) + public static int getMaxSpeakerId() { + return 15; + } + /** Confidence level in the trigger outcome. */ @HotwordConfidenceLevelValue private final int mConfidenceLevel; @@ -378,6 +395,7 @@ public final class HotwordDetectedResult implements Parcelable { } private void onConstructed() { + Preconditions.checkArgumentInRange(mSpeakerId, 0, getMaxSpeakerId(), "speakerId"); Preconditions.checkArgumentInRange(mScore, 0, getMaxScore(), "score"); Preconditions.checkArgumentInRange(mPersonalizedScore, 0, getMaxScore(), "personalizedScore"); @@ -480,8 +498,8 @@ public final class HotwordDetectedResult implements Parcelable { public @NonNull Builder setAudioStreams(@NonNull List<HotwordAudioStream> value) { Objects.requireNonNull(value, "value should not be null"); final Builder builder = (Builder) this; - // If the code gen flag in build() is changed, we must update the flag e.g. 0x200 here. - builder.mBuilderFieldsSet |= 0x200; + // If the code gen flag in build() is changed, we must update the flag e.g. 0x400 here. + builder.mBuilderFieldsSet |= 0x400; builder.mAudioStreams = List.copyOf(value); return builder; } @@ -504,7 +522,8 @@ public final class HotwordDetectedResult implements Parcelable { .setHotwordPhraseId(mHotwordPhraseId) .setAudioStreams(mAudioStreams) .setExtras(mExtras) - .setBackgroundAudioPower(mBackgroundAudioPower); + .setBackgroundAudioPower(mBackgroundAudioPower) + .setSpeakerId(mSpeakerId); } @@ -605,6 +624,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member /* package-private */ HotwordDetectedResult( + int speakerId, @HotwordConfidenceLevelValue int confidenceLevel, @Nullable MediaSyncEvent mediaSyncEvent, int hotwordOffsetMillis, @@ -617,6 +637,7 @@ public final class HotwordDetectedResult implements Parcelable { @NonNull List<HotwordAudioStream> audioStreams, @NonNull PersistableBundle extras, int backgroundAudioPower) { + this.mSpeakerId = speakerId; this.mConfidenceLevel = confidenceLevel; com.android.internal.util.AnnotationValidations.validate( HotwordConfidenceLevelValue.class, null, mConfidenceLevel); @@ -640,6 +661,15 @@ public final class HotwordDetectedResult implements Parcelable { } /** + * Id of the current speaker + */ + @DataClass.Generated.Member + @FlaggedApi(Flags.FLAG_ALLOW_SPEAKER_ID_EGRESS) + public int getSpeakerId() { + return mSpeakerId; + } + + /** * Confidence level in the trigger outcome. */ @DataClass.Generated.Member @@ -769,6 +799,7 @@ public final class HotwordDetectedResult implements Parcelable { // String fieldNameToString() { ... } return "HotwordDetectedResult { " + + "speakerId = " + mSpeakerId + ", " + "confidenceLevel = " + mConfidenceLevel + ", " + "mediaSyncEvent = " + mMediaSyncEvent + ", " + "hotwordOffsetMillis = " + mHotwordOffsetMillis + ", " + @@ -797,6 +828,7 @@ public final class HotwordDetectedResult implements Parcelable { HotwordDetectedResult that = (HotwordDetectedResult) o; //noinspection PointlessBooleanExpression return true + && mSpeakerId == that.mSpeakerId && mConfidenceLevel == that.mConfidenceLevel && Objects.equals(mMediaSyncEvent, that.mMediaSyncEvent) && mHotwordOffsetMillis == that.mHotwordOffsetMillis @@ -818,6 +850,7 @@ public final class HotwordDetectedResult implements Parcelable { // int fieldNameHashCode() { ... } int _hash = 1; + _hash = 31 * _hash + mSpeakerId; _hash = 31 * _hash + mConfidenceLevel; _hash = 31 * _hash + Objects.hashCode(mMediaSyncEvent); _hash = 31 * _hash + mHotwordOffsetMillis; @@ -840,9 +873,10 @@ public final class HotwordDetectedResult implements Parcelable { // void parcelFieldName(Parcel dest, int flags) { ... } int flg = 0; - if (mHotwordDetectionPersonalized) flg |= 0x20; - if (mMediaSyncEvent != null) flg |= 0x2; + if (mHotwordDetectionPersonalized) flg |= 0x40; + if (mMediaSyncEvent != null) flg |= 0x4; dest.writeInt(flg); + dest.writeInt(mSpeakerId); dest.writeInt(mConfidenceLevel); if (mMediaSyncEvent != null) dest.writeTypedObject(mMediaSyncEvent, flags); dest.writeInt(mHotwordOffsetMillis); @@ -868,9 +902,10 @@ public final class HotwordDetectedResult implements Parcelable { // static FieldType unparcelFieldName(Parcel in) { ... } int flg = in.readInt(); - boolean hotwordDetectionPersonalized = (flg & 0x20) != 0; + boolean hotwordDetectionPersonalized = (flg & 0x40) != 0; + int speakerId = in.readInt(); int confidenceLevel = in.readInt(); - MediaSyncEvent mediaSyncEvent = (flg & 0x2) == 0 ? null : (MediaSyncEvent) in.readTypedObject(MediaSyncEvent.CREATOR); + MediaSyncEvent mediaSyncEvent = (flg & 0x4) == 0 ? null : (MediaSyncEvent) in.readTypedObject(MediaSyncEvent.CREATOR); int hotwordOffsetMillis = in.readInt(); int hotwordDurationMillis = in.readInt(); int audioChannel = in.readInt(); @@ -882,6 +917,7 @@ public final class HotwordDetectedResult implements Parcelable { PersistableBundle extras = (PersistableBundle) in.readTypedObject(PersistableBundle.CREATOR); int backgroundAudioPower = in.readInt(); + this.mSpeakerId = speakerId; this.mConfidenceLevel = confidenceLevel; com.android.internal.util.AnnotationValidations.validate( HotwordConfidenceLevelValue.class, null, mConfidenceLevel); @@ -925,6 +961,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public static final class Builder extends BaseBuilder { + private int mSpeakerId; private @HotwordConfidenceLevelValue int mConfidenceLevel; private @Nullable MediaSyncEvent mMediaSyncEvent; private int mHotwordOffsetMillis; @@ -944,12 +981,24 @@ public final class HotwordDetectedResult implements Parcelable { } /** + * Id of the current speaker + */ + @DataClass.Generated.Member + @FlaggedApi(Flags.FLAG_ALLOW_SPEAKER_ID_EGRESS) + public @NonNull Builder setSpeakerId(int value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x1; + mSpeakerId = value; + return this; + } + + /** * Confidence level in the trigger outcome. */ @DataClass.Generated.Member public @NonNull Builder setConfidenceLevel(@HotwordConfidenceLevelValue int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x1; + mBuilderFieldsSet |= 0x2; mConfidenceLevel = value; return this; } @@ -962,7 +1011,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setMediaSyncEvent(@NonNull MediaSyncEvent value) { checkNotUsed(); - mBuilderFieldsSet |= 0x2; + mBuilderFieldsSet |= 0x4; mMediaSyncEvent = value; return this; } @@ -976,7 +1025,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setHotwordOffsetMillis(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x4; + mBuilderFieldsSet |= 0x8; mHotwordOffsetMillis = value; return this; } @@ -990,7 +1039,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setHotwordDurationMillis(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x8; + mBuilderFieldsSet |= 0x10; mHotwordDurationMillis = value; return this; } @@ -1003,7 +1052,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setAudioChannel(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x10; + mBuilderFieldsSet |= 0x20; mAudioChannel = value; return this; } @@ -1015,7 +1064,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setHotwordDetectionPersonalized(boolean value) { checkNotUsed(); - mBuilderFieldsSet |= 0x20; + mBuilderFieldsSet |= 0x40; mHotwordDetectionPersonalized = value; return this; } @@ -1028,7 +1077,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setScore(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x40; + mBuilderFieldsSet |= 0x80; mScore = value; return this; } @@ -1041,7 +1090,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setPersonalizedScore(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x80; + mBuilderFieldsSet |= 0x100; mPersonalizedScore = value; return this; } @@ -1054,7 +1103,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setHotwordPhraseId(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x100; + mBuilderFieldsSet |= 0x200; mHotwordPhraseId = value; return this; } @@ -1087,7 +1136,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setExtras(@NonNull PersistableBundle value) { checkNotUsed(); - mBuilderFieldsSet |= 0x400; + mBuilderFieldsSet |= 0x800; mExtras = value; return this; } @@ -1104,7 +1153,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setBackgroundAudioPower(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x800; + mBuilderFieldsSet |= 0x1000; mBackgroundAudioPower = value; return this; } @@ -1112,45 +1161,49 @@ public final class HotwordDetectedResult implements Parcelable { /** Builds the instance. This builder should not be touched after calling this! */ public @NonNull HotwordDetectedResult build() { checkNotUsed(); - mBuilderFieldsSet |= 0x1000; // Mark builder used + mBuilderFieldsSet |= 0x2000; // Mark builder used if ((mBuilderFieldsSet & 0x1) == 0) { - mConfidenceLevel = defaultConfidenceLevel(); + mSpeakerId = defaultSpeakerId(); } if ((mBuilderFieldsSet & 0x2) == 0) { - mMediaSyncEvent = null; + mConfidenceLevel = defaultConfidenceLevel(); } if ((mBuilderFieldsSet & 0x4) == 0) { - mHotwordOffsetMillis = HOTWORD_OFFSET_UNSET; + mMediaSyncEvent = null; } if ((mBuilderFieldsSet & 0x8) == 0) { - mHotwordDurationMillis = 0; + mHotwordOffsetMillis = HOTWORD_OFFSET_UNSET; } if ((mBuilderFieldsSet & 0x10) == 0) { - mAudioChannel = AUDIO_CHANNEL_UNSET; + mHotwordDurationMillis = 0; } if ((mBuilderFieldsSet & 0x20) == 0) { - mHotwordDetectionPersonalized = false; + mAudioChannel = AUDIO_CHANNEL_UNSET; } if ((mBuilderFieldsSet & 0x40) == 0) { - mScore = defaultScore(); + mHotwordDetectionPersonalized = false; } if ((mBuilderFieldsSet & 0x80) == 0) { - mPersonalizedScore = defaultPersonalizedScore(); + mScore = defaultScore(); } if ((mBuilderFieldsSet & 0x100) == 0) { - mHotwordPhraseId = defaultHotwordPhraseId(); + mPersonalizedScore = defaultPersonalizedScore(); } if ((mBuilderFieldsSet & 0x200) == 0) { - mAudioStreams = defaultAudioStreams(); + mHotwordPhraseId = defaultHotwordPhraseId(); } if ((mBuilderFieldsSet & 0x400) == 0) { - mExtras = defaultExtras(); + mAudioStreams = defaultAudioStreams(); } if ((mBuilderFieldsSet & 0x800) == 0) { + mExtras = defaultExtras(); + } + if ((mBuilderFieldsSet & 0x1000) == 0) { mBackgroundAudioPower = defaultBackgroundAudioPower(); } HotwordDetectedResult o = new HotwordDetectedResult( + mSpeakerId, mConfidenceLevel, mMediaSyncEvent, mHotwordOffsetMillis, @@ -1167,7 +1220,7 @@ public final class HotwordDetectedResult implements Parcelable { } private void checkNotUsed() { - if ((mBuilderFieldsSet & 0x1000) != 0) { + if ((mBuilderFieldsSet & 0x2000) != 0) { throw new IllegalStateException( "This Builder should not be reused. Use a new Builder instance instead"); } @@ -1175,10 +1228,10 @@ public final class HotwordDetectedResult implements Parcelable { } @DataClass.Generated( - time = 1679517179528L, + time = 1704944087827L, codegenVersion = "1.0.23", sourceFile = "frameworks/base/core/java/android/service/voice/HotwordDetectedResult.java", - inputSignatures = "public static final int CONFIDENCE_LEVEL_NONE\npublic static final int CONFIDENCE_LEVEL_LOW\npublic static final int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final int CONFIDENCE_LEVEL_HIGH\npublic static final int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final int HOTWORD_OFFSET_UNSET\npublic static final int AUDIO_CHANNEL_UNSET\npublic static final int BACKGROUND_AUDIO_POWER_UNSET\nprivate static final int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final int LIMIT_AUDIO_CHANNEL_MAX_VALUE\nprivate static final java.lang.String EXTRA_PROXIMITY\npublic static final int PROXIMITY_UNKNOWN\npublic static final int PROXIMITY_NEAR\npublic static final int PROXIMITY_FAR\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate int mHotwordOffsetMillis\nprivate int mHotwordDurationMillis\nprivate int mAudioChannel\nprivate boolean mHotwordDetectionPersonalized\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int sMaxBundleSize\nprivate final int mBackgroundAudioPower\nprivate static int defaultConfidenceLevel()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\nprivate static int defaultBackgroundAudioPower()\npublic static int getMaxBackgroundAudioPower()\npublic static int getParcelableSize(android.os.Parcelable)\npublic static int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static int bitCount(long)\nprivate void onConstructed()\npublic @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\npublic void setProximity(double)\npublic @android.service.voice.HotwordDetectedResult.ProximityValue int getProximity()\nprivate @android.service.voice.HotwordDetectedResult.ProximityValue int convertToProximityLevel(double)\npublic android.service.voice.HotwordDetectedResult.Builder buildUpon()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []") + inputSignatures = "public static final int CONFIDENCE_LEVEL_NONE\npublic static final int CONFIDENCE_LEVEL_LOW\npublic static final int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final int CONFIDENCE_LEVEL_HIGH\npublic static final int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final int HOTWORD_OFFSET_UNSET\npublic static final int AUDIO_CHANNEL_UNSET\npublic static final int BACKGROUND_AUDIO_POWER_UNSET\nprivate static final int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final int LIMIT_AUDIO_CHANNEL_MAX_VALUE\nprivate static final java.lang.String EXTRA_PROXIMITY\npublic static final int PROXIMITY_UNKNOWN\npublic static final int PROXIMITY_NEAR\npublic static final int PROXIMITY_FAR\nprivate final int mSpeakerId\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate int mHotwordOffsetMillis\nprivate int mHotwordDurationMillis\nprivate int mAudioChannel\nprivate boolean mHotwordDetectionPersonalized\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int sMaxBundleSize\nprivate final int mBackgroundAudioPower\nprivate static int defaultSpeakerId()\npublic static @android.annotation.FlaggedApi int getMaxSpeakerId()\nprivate static int defaultConfidenceLevel()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\nprivate static int defaultBackgroundAudioPower()\npublic static int getMaxBackgroundAudioPower()\npublic static int getParcelableSize(android.os.Parcelable)\npublic static int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static int bitCount(long)\nprivate void onConstructed()\npublic @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\npublic void setProximity(double)\npublic @android.service.voice.HotwordDetectedResult.ProximityValue int getProximity()\nprivate @android.service.voice.HotwordDetectedResult.ProximityValue int convertToProximityLevel(double)\npublic android.service.voice.HotwordDetectedResult.Builder buildUpon()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []") @Deprecated private void __metadata() {} diff --git a/core/java/android/service/voice/IDetectorSessionVisualQueryDetectionCallback.aidl b/core/java/android/service/voice/IDetectorSessionVisualQueryDetectionCallback.aidl index 22172ed36f2f..4f6eb88e34e9 100644 --- a/core/java/android/service/voice/IDetectorSessionVisualQueryDetectionCallback.aidl +++ b/core/java/android/service/voice/IDetectorSessionVisualQueryDetectionCallback.aidl @@ -16,6 +16,8 @@ package android.service.voice; +import android.service.voice.VisualQueryDetectedResult; + /** * Callback for returning the detected result from the {@link VisualQueryDetectionService}. * @@ -42,6 +44,11 @@ oneway interface IDetectorSessionVisualQueryDetectionCallback { void onQueryDetected(in String partialQuery); /** + * Called when the detected result is streamed. + */ + void onResultDetected(in VisualQueryDetectedResult partialResult); + + /** * Called when the detected result is valid. */ void onQueryFinished(); diff --git a/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl b/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl index cedb7ff62497..fa6f1d17308c 100644 --- a/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl +++ b/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl @@ -17,6 +17,7 @@ package android.service.voice; import android.service.voice.VisualQueryDetectionServiceFailure; +import android.service.voice.VisualQueryDetectedResult; /** * Callback for returning the detected result from the VisualQueryDetectionService. @@ -31,6 +32,11 @@ oneway interface IVisualQueryDetectionVoiceInteractionCallback { void onQueryDetected(in String partialQuery); /** + * Called when the detected result is streamed. + */ + void onResultDetected(in VisualQueryDetectedResult partialResult); + + /** * Called when the detected result is valid. */ void onQueryFinished(); diff --git a/core/java/android/service/voice/VisualQueryDetectedResult.aidl b/core/java/android/service/voice/VisualQueryDetectedResult.aidl new file mode 100644 index 000000000000..d5b9a665f7b8 --- /dev/null +++ b/core/java/android/service/voice/VisualQueryDetectedResult.aidl @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.voice; + +parcelable VisualQueryDetectedResult; diff --git a/core/java/android/service/voice/VisualQueryDetectedResult.java b/core/java/android/service/voice/VisualQueryDetectedResult.java new file mode 100644 index 000000000000..13cdfdebc86a --- /dev/null +++ b/core/java/android/service/voice/VisualQueryDetectedResult.java @@ -0,0 +1,293 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.voice; + +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.os.Parcel; +import android.os.Parcelable; +import android.service.voice.flags.Flags; + +import com.android.internal.util.DataClass; +import com.android.internal.util.Preconditions; + +import java.util.Objects; + +/** + * Represents a result supporting the visual query detection. + * + * @hide + */ +@DataClass( + genConstructor = false, + genBuilder = true, + genEqualsHashCode = true, + genHiddenConstDefs = true, + genParcelable = true, + genToString = true +) +@SystemApi +@FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) +public final class VisualQueryDetectedResult implements Parcelable { + + /** + * Text query being associated with the detection result. + **/ + @NonNull + private final String mPartialQuery; + private static String defaultPartialQuery() { + return ""; + } + + /** Id of the current speaker + * + * <p>Only values between 0 and {@link #getMaxSpeakerId} (inclusive) are accepted. + */ + private final int mSpeakerId; + private static int defaultSpeakerId() { + return 0; + } + /** Maximum number of active speaker ids. **/ + public static int getMaxSpeakerId() { + return 15; + } + + private void onConstructed() { + Preconditions.checkArgumentInRange(mSpeakerId, 0, getMaxSpeakerId(), "speakerId"); + } + + /** + * Provides an instance of {@link Builder} with state corresponding to this instance. + * + * @hide + */ + public Builder buildUpon() { + return new Builder().setPartialQuery(mPartialQuery).setSpeakerId(mSpeakerId); + } + + + + + // Code below generated by codegen v1.0.23. + // + // DO NOT MODIFY! + // CHECKSTYLE:OFF Generated code + // + // To regenerate run: + // $ codegen $ANDROID_BUILD_TOP/frameworks/base/core/java/android/service/voice/VisualQueryDetectedResult.java + // + // To exclude the generated code from IntelliJ auto-formatting enable (one-time): + // Settings > Editor > Code Style > Formatter Control + //@formatter:off + + + @DataClass.Generated.Member + /* package-private */ VisualQueryDetectedResult( + @NonNull String partialQuery, + int speakerId) { + this.mPartialQuery = partialQuery; + com.android.internal.util.AnnotationValidations.validate( + NonNull.class, null, mPartialQuery); + this.mSpeakerId = speakerId; + + onConstructed(); + } + + /** + * Text query being associated with the detection result. + */ + @DataClass.Generated.Member + public @NonNull String getPartialQuery() { + return mPartialQuery; + } + + /** + * Id of the current speaker + * + * <p>Only values between 0 and {@link #getMaxSpeakerId} (inclusive) are accepted. + */ + @DataClass.Generated.Member + public int getSpeakerId() { + return mSpeakerId; + } + + @Override + @DataClass.Generated.Member + public String toString() { + // You can override field toString logic by defining methods like: + // String fieldNameToString() { ... } + + return "VisualQueryDetectedResult { " + + "partialQuery = " + mPartialQuery + ", " + + "speakerId = " + mSpeakerId + + " }"; + } + + @Override + @DataClass.Generated.Member + public boolean equals(@Nullable Object o) { + // You can override field equality logic by defining either of the methods like: + // boolean fieldNameEquals(VisualQueryDetectedResult other) { ... } + // boolean fieldNameEquals(FieldType otherValue) { ... } + + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + @SuppressWarnings("unchecked") + VisualQueryDetectedResult that = (VisualQueryDetectedResult) o; + //noinspection PointlessBooleanExpression + return true + && Objects.equals(mPartialQuery, that.mPartialQuery) + && mSpeakerId == that.mSpeakerId; + } + + @Override + @DataClass.Generated.Member + public int hashCode() { + // You can override field hashCode logic by defining methods like: + // int fieldNameHashCode() { ... } + + int _hash = 1; + _hash = 31 * _hash + Objects.hashCode(mPartialQuery); + _hash = 31 * _hash + mSpeakerId; + return _hash; + } + + @Override + @DataClass.Generated.Member + public void writeToParcel(@NonNull Parcel dest, int flags) { + // You can override field parcelling by defining methods like: + // void parcelFieldName(Parcel dest, int flags) { ... } + + dest.writeString(mPartialQuery); + dest.writeInt(mSpeakerId); + } + + @Override + @DataClass.Generated.Member + public int describeContents() { return 0; } + + /** @hide */ + @SuppressWarnings({"unchecked", "RedundantCast"}) + @DataClass.Generated.Member + /* package-private */ VisualQueryDetectedResult(@NonNull Parcel in) { + // You can override field unparcelling by defining methods like: + // static FieldType unparcelFieldName(Parcel in) { ... } + + String partialQuery = in.readString(); + int speakerId = in.readInt(); + + this.mPartialQuery = partialQuery; + com.android.internal.util.AnnotationValidations.validate( + NonNull.class, null, mPartialQuery); + this.mSpeakerId = speakerId; + + onConstructed(); + } + + @DataClass.Generated.Member + public static final @NonNull Parcelable.Creator<VisualQueryDetectedResult> CREATOR + = new Parcelable.Creator<VisualQueryDetectedResult>() { + @Override + public VisualQueryDetectedResult[] newArray(int size) { + return new VisualQueryDetectedResult[size]; + } + + @Override + public VisualQueryDetectedResult createFromParcel(@NonNull Parcel in) { + return new VisualQueryDetectedResult(in); + } + }; + + /** + * A builder for {@link VisualQueryDetectedResult} + */ + @SuppressWarnings("WeakerAccess") + @DataClass.Generated.Member + public static final class Builder { + + private @NonNull String mPartialQuery; + private int mSpeakerId; + + private long mBuilderFieldsSet = 0L; + + public Builder() { + } + + /** + * Text query being associated with the detection result. + */ + @DataClass.Generated.Member + public @NonNull Builder setPartialQuery(@NonNull String value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x1; + mPartialQuery = value; + return this; + } + + /** + * Id of the current speaker + * + * <p>Only values between 0 and {@link #getMaxSpeakerId} (inclusive) are accepted. + */ + @DataClass.Generated.Member + public @NonNull Builder setSpeakerId(int value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x2; + mSpeakerId = value; + return this; + } + + /** Builds the instance. This builder should not be touched after calling this! */ + public @NonNull VisualQueryDetectedResult build() { + checkNotUsed(); + mBuilderFieldsSet |= 0x4; // Mark builder used + + if ((mBuilderFieldsSet & 0x1) == 0) { + mPartialQuery = defaultPartialQuery(); + } + if ((mBuilderFieldsSet & 0x2) == 0) { + mSpeakerId = defaultSpeakerId(); + } + VisualQueryDetectedResult o = new VisualQueryDetectedResult( + mPartialQuery, + mSpeakerId); + return o; + } + + private void checkNotUsed() { + if ((mBuilderFieldsSet & 0x4) != 0) { + throw new IllegalStateException( + "This Builder should not be reused. Use a new Builder instance instead"); + } + } + } + + @DataClass.Generated( + time = 1704949386772L, + codegenVersion = "1.0.23", + sourceFile = "frameworks/base/core/java/android/service/voice/VisualQueryDetectedResult.java", + inputSignatures = "private final @android.annotation.NonNull java.lang.String mPartialQuery\nprivate final int mSpeakerId\nprivate static java.lang.String defaultPartialQuery()\nprivate static int defaultSpeakerId()\npublic static int getMaxSpeakerId()\nprivate void onConstructed()\npublic android.service.voice.VisualQueryDetectedResult.Builder buildUpon()\nclass VisualQueryDetectedResult extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)") + @Deprecated + private void __metadata() {} + + + //@formatter:on + // End of generated code + +} diff --git a/core/java/android/service/voice/VisualQueryDetectionService.java b/core/java/android/service/voice/VisualQueryDetectionService.java index 76b076be8fab..b60c77502247 100644 --- a/core/java/android/service/voice/VisualQueryDetectionService.java +++ b/core/java/android/service/voice/VisualQueryDetectionService.java @@ -17,6 +17,7 @@ package android.service.voice; import android.annotation.DurationMillisLong; +import android.annotation.FlaggedApi; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SdkConstant; @@ -35,6 +36,7 @@ import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; import android.os.RemoteException; import android.os.SharedMemory; +import android.service.voice.flags.Flags; import android.speech.IRecognitionServiceManager; import android.util.Log; import android.view.contentcapture.ContentCaptureManager; @@ -282,6 +284,7 @@ public abstract class VisualQueryDetectionService extends Service } } + // TODO(b/324341724): Properly deprecate this API. /** * Informs the {@link VisualQueryDetector} with the text content being captured about the * query from the audio source. {@code partialQuery} is provided to the @@ -289,6 +292,9 @@ public abstract class VisualQueryDetectionService extends Service * {@link VisualQueryDetectionService#gainedAttention()} is called to put the service into the * attention gained state. * + * Usage of this method is not recommended, please use + * {@link VisualQueryDetectionService#streamQuery(VisualQueryDetectedResult)} instead. + * * @param partialQuery Partially detected query in string. * @throws IllegalStateException if method called without attention gained. */ @@ -303,6 +309,27 @@ public abstract class VisualQueryDetectionService extends Service } /** + * Informs the {@link VisualQueryDetector} with the text content being captured about the + * query from the audio source. {@code partialResult} is provided to the + * {@link VisualQueryDetector}. This method is expected to be only triggered if + * {@link VisualQueryDetectionService#gainedAttention()} is called to put the service into + * the attention gained state. + * + * @param partialResult Partially detected result in the format of + * {@link VisualQueryDetectedResult}. + */ + @FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) + public final void streamQuery(@NonNull VisualQueryDetectedResult partialResult) { + Objects.requireNonNull(partialResult); + try { + mRemoteCallback.onResultDetected(partialResult); + } catch (RemoteException e) { + throw new IllegalStateException("#streamQuery must be only be triggered after " + + "calling #gainedAttention to be in the attention gained state."); + } + } + + /** * Informs the {@link VisualQueryDetector} to abandon the streamed partial query that has * been sent to {@link VisualQueryDetector}.This method is expected to be only triggered if * {@link VisualQueryDetectionService#streamQuery(String)} is called to put the service into diff --git a/core/java/android/service/voice/VisualQueryDetector.java b/core/java/android/service/voice/VisualQueryDetector.java index f2bdbf67e76e..aee25f5e1f00 100644 --- a/core/java/android/service/voice/VisualQueryDetector.java +++ b/core/java/android/service/voice/VisualQueryDetector.java @@ -20,6 +20,7 @@ import static android.Manifest.permission.CAMERA; import static android.Manifest.permission.RECORD_AUDIO; import android.annotation.CallbackExecutor; +import android.annotation.FlaggedApi; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; @@ -33,6 +34,7 @@ import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; import android.os.RemoteException; import android.os.SharedMemory; +import android.service.voice.flags.Flags; import android.text.TextUtils; import android.util.Slog; @@ -206,6 +208,17 @@ public class VisualQueryDetector { void onQueryDetected(@NonNull String partialQuery); /** + * Called when the {@link VisualQueryDetectionService} starts to stream partial results + * with {@link VisualQueryDetectionService#streamQuery(VisualQueryDetectedResult)}. + * + * @param partialResult The partial query in a text form being streamed. + */ + @FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) + default void onQueryDetected(@NonNull VisualQueryDetectedResult partialResult) { + throw new UnsupportedOperationException("This emthod must be implemented for use."); + } + + /** * Called when the {@link VisualQueryDetectionService} decides to abandon the streamed * partial queries with {@link VisualQueryDetectionService#rejectQuery()}. */ @@ -319,7 +332,7 @@ public class VisualQueryDetector { this.mLock = lock; } - /** Called when the detected result is valid. */ + /** Called when the detected query is valid. */ @Override public void onQueryDetected(@NonNull String partialQuery) { Slog.v(TAG, "BinderCallback#onQueryDetected"); @@ -330,6 +343,17 @@ public class VisualQueryDetector { }); } + /** Called when the detected result is valid. */ + @Override + public void onResultDetected(@NonNull VisualQueryDetectedResult partialResult) { + Slog.v(TAG, "BinderCallback#onResultDetected"); + Binder.withCleanCallingIdentity(() -> { + synchronized (mLock) { + mCallback.onQueryDetected(partialResult); + } + }); + } + @Override public void onQueryFinished() { Slog.v(TAG, "BinderCallback#onQueryFinished"); diff --git a/core/java/android/service/voice/flags/flags.aconfig b/core/java/android/service/voice/flags/flags.aconfig index 1c8752b234c1..f870db5f95f0 100644 --- a/core/java/android/service/voice/flags/flags.aconfig +++ b/core/java/android/service/voice/flags/flags.aconfig @@ -20,3 +20,24 @@ flag { description: "This flag allows providing foreground app component along with onShow args." bug: "319409708" } + +flag { + name: "allow_various_attention_types" + namespace: "visual_query" + description: "This flag allows visual query detection service to set different attention types." + bug: "318617199" +} + +flag { + name: "allow_complex_results_egress_from_vqds" + namespace: "visual_query" + description: "This flag allows visual query detection service egress detailed results. " + bug: "318617199" +} + +flag { + name: "allow_speaker_id_egress" + namespace: "machine_learning" + description: "This flag allows hotword detection service and visual query detection service to egress current speaker profile id." + bug: "318617199" +}
\ No newline at end of file diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java index f9fa9b7a9524..d7b860f6a857 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java @@ -33,6 +33,7 @@ import android.service.voice.IDetectorSessionVisualQueryDetectionCallback; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.ISandboxedDetectionService; import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback; +import android.service.voice.VisualQueryDetectedResult; import android.service.voice.VisualQueryDetectionServiceFailure; import android.util.Slog; @@ -171,6 +172,26 @@ final class VisualQueryDetectorSession extends DetectorSession { } @Override + public void onResultDetected(@NonNull VisualQueryDetectedResult partialResult) + throws RemoteException { + Slog.v(TAG, "BinderCallback#onResultDetected"); + synchronized (mLock) { + Objects.requireNonNull(partialResult); + if (!mEgressingData) { + Slog.v(TAG, "Result should not be egressed within the unattention state."); + callback.onVisualQueryDetectionServiceFailure( + new VisualQueryDetectionServiceFailure( + ERROR_CODE_ILLEGAL_STREAMING_STATE, + "Cannot stream results without attention signals.")); + return; + } + mQueryStreaming = true; + callback.onResultDetected(partialResult); + Slog.i(TAG, "Egressed from visual query detection process."); + } + } + + @Override public void onQueryFinished() throws RemoteException { Slog.v(TAG, "BinderCallback#onQueryFinished"); synchronized (mLock) { |