diff options
9 files changed, 372 insertions, 1 deletions
diff --git a/core/api/system-current.txt b/core/api/system-current.txt index f30c8cf15dc1..c6ddaaef97bb 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -13351,6 +13351,19 @@ package android.service.voice { field public static final int ERROR_CODE_UNKNOWN = 0; // 0x0 } + @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public final class VisualQueryDetectedResult implements android.os.Parcelable { + method public int describeContents(); + method @NonNull public String getPartialQuery(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator<android.service.voice.VisualQueryDetectedResult> CREATOR; + } + + public static final class VisualQueryDetectedResult.Builder { + ctor public VisualQueryDetectedResult.Builder(); + method @NonNull public android.service.voice.VisualQueryDetectedResult build(); + method @NonNull public android.service.voice.VisualQueryDetectedResult.Builder setPartialQuery(@NonNull String); + } + public abstract class VisualQueryDetectionService extends android.app.Service implements android.service.voice.SandboxedDetectionInitializer { ctor public VisualQueryDetectionService(); method public final void finishQuery() throws java.lang.IllegalStateException; @@ -13362,6 +13375,7 @@ package android.service.voice { method public void onUpdateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, long, @Nullable java.util.function.IntConsumer); method public final void rejectQuery() throws java.lang.IllegalStateException; method public final void streamQuery(@NonNull String) throws java.lang.IllegalStateException; + method @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public final void streamQuery(@NonNull android.service.voice.VisualQueryDetectedResult); field public static final String SERVICE_INTERFACE = "android.service.voice.VisualQueryDetectionService"; } @@ -13390,6 +13404,7 @@ package android.service.voice { public static interface VisualQueryDetector.Callback { method public void onFailure(@NonNull android.service.voice.VisualQueryDetectionServiceFailure); method public void onQueryDetected(@NonNull String); + method @FlaggedApi("android.service.voice.flags.allow_complex_results_egress_from_vqds") public default void onQueryDetected(@NonNull android.service.voice.VisualQueryDetectedResult); method public void onQueryFinished(); method public void onQueryRejected(); method public void onUnknownFailure(@NonNull String); diff --git a/core/java/android/service/voice/IDetectorSessionVisualQueryDetectionCallback.aidl b/core/java/android/service/voice/IDetectorSessionVisualQueryDetectionCallback.aidl index 22172ed36f2f..4f6eb88e34e9 100644 --- a/core/java/android/service/voice/IDetectorSessionVisualQueryDetectionCallback.aidl +++ b/core/java/android/service/voice/IDetectorSessionVisualQueryDetectionCallback.aidl @@ -16,6 +16,8 @@ package android.service.voice; +import android.service.voice.VisualQueryDetectedResult; + /** * Callback for returning the detected result from the {@link VisualQueryDetectionService}. * @@ -42,6 +44,11 @@ oneway interface IDetectorSessionVisualQueryDetectionCallback { void onQueryDetected(in String partialQuery); /** + * Called when the detected result is streamed. + */ + void onResultDetected(in VisualQueryDetectedResult partialResult); + + /** * Called when the detected result is valid. */ void onQueryFinished(); diff --git a/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl b/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl index cedb7ff62497..fa6f1d17308c 100644 --- a/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl +++ b/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl @@ -17,6 +17,7 @@ package android.service.voice; import android.service.voice.VisualQueryDetectionServiceFailure; +import android.service.voice.VisualQueryDetectedResult; /** * Callback for returning the detected result from the VisualQueryDetectionService. @@ -31,6 +32,11 @@ oneway interface IVisualQueryDetectionVoiceInteractionCallback { void onQueryDetected(in String partialQuery); /** + * Called when the detected result is streamed. + */ + void onResultDetected(in VisualQueryDetectedResult partialResult); + + /** * Called when the detected result is valid. */ void onQueryFinished(); diff --git a/core/java/android/service/voice/VisualQueryDetectedResult.aidl b/core/java/android/service/voice/VisualQueryDetectedResult.aidl new file mode 100644 index 000000000000..d5b9a665f7b8 --- /dev/null +++ b/core/java/android/service/voice/VisualQueryDetectedResult.aidl @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.voice; + +parcelable VisualQueryDetectedResult; diff --git a/core/java/android/service/voice/VisualQueryDetectedResult.java b/core/java/android/service/voice/VisualQueryDetectedResult.java new file mode 100644 index 000000000000..246c28ab95c9 --- /dev/null +++ b/core/java/android/service/voice/VisualQueryDetectedResult.java @@ -0,0 +1,238 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.voice; + +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.os.Parcel; +import android.os.Parcelable; +import android.service.voice.flags.Flags; + +import com.android.internal.util.DataClass; + +import java.util.Objects; + +/** + * Represents a result supporting the visual query detection. + * + * @hide + */ +@DataClass( + genConstructor = false, + genBuilder = true, + genEqualsHashCode = true, + genHiddenConstDefs = true, + genParcelable = true, + genToString = true +) +@SystemApi +@FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) +public final class VisualQueryDetectedResult implements Parcelable { + + /** + * Text query being associated with the detection result. + **/ + @NonNull + private final String mPartialQuery; + + private static String defaultPartialQuery() { + return ""; + } + + /** + * Provides an instance of {@link Builder} with state corresponding to this instance. + * + * @hide + */ + public Builder buildUpon() { + return new Builder().setPartialQuery(mPartialQuery); + } + + + // Code below generated by codegen v1.0.23. + // + // DO NOT MODIFY! + // CHECKSTYLE:OFF Generated code + // + // To regenerate run: + // $ codegen $ANDROID_BUILD_TOP/frameworks/base/core/java/android/service/voice/VisualQueryDetectedResult.java + // + // To exclude the generated code from IntelliJ auto-formatting enable (one-time): + // Settings > Editor > Code Style > Formatter Control + //@formatter:off + + + @DataClass.Generated.Member + /* package-private */ VisualQueryDetectedResult( + @NonNull String partialQuery) { + this.mPartialQuery = partialQuery; + com.android.internal.util.AnnotationValidations.validate( + NonNull.class, null, mPartialQuery); + + // onConstructed(); // You can define this method to get a callback + } + + /** + * Text query being associated with the detection result. * + */ + @DataClass.Generated.Member + public @NonNull String getPartialQuery() { + return mPartialQuery; + } + + @Override + @DataClass.Generated.Member + public String toString() { + // You can override field toString logic by defining methods like: + // String fieldNameToString() { ... } + + return "VisualQueryDetectedResult { " + + "partialQuery = " + mPartialQuery + + " }"; + } + + @Override + @DataClass.Generated.Member + public boolean equals(@Nullable Object o) { + // You can override field equality logic by defining either of the methods like: + // boolean fieldNameEquals(VisualQueryDetectedResult other) { ... } + // boolean fieldNameEquals(FieldType otherValue) { ... } + + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + @SuppressWarnings("unchecked") + VisualQueryDetectedResult that = (VisualQueryDetectedResult) o; + //noinspection PointlessBooleanExpression + return true + && Objects.equals(mPartialQuery, that.mPartialQuery); + } + + @Override + @DataClass.Generated.Member + public int hashCode() { + // You can override field hashCode logic by defining methods like: + // int fieldNameHashCode() { ... } + + int _hash = 1; + _hash = 31 * _hash + Objects.hashCode(mPartialQuery); + return _hash; + } + + @Override + @DataClass.Generated.Member + public void writeToParcel(@NonNull Parcel dest, int flags) { + // You can override field parcelling by defining methods like: + // void parcelFieldName(Parcel dest, int flags) { ... } + + dest.writeString(mPartialQuery); + } + + @Override + @DataClass.Generated.Member + public int describeContents() { return 0; } + + /** @hide */ + @SuppressWarnings({"unchecked", "RedundantCast"}) + @DataClass.Generated.Member + /* package-private */ VisualQueryDetectedResult(@NonNull Parcel in) { + // You can override field unparcelling by defining methods like: + // static FieldType unparcelFieldName(Parcel in) { ... } + + String partialQuery = in.readString(); + + this.mPartialQuery = partialQuery; + com.android.internal.util.AnnotationValidations.validate( + NonNull.class, null, mPartialQuery); + + // onConstructed(); // You can define this method to get a callback + } + + @DataClass.Generated.Member + public static final @NonNull Parcelable.Creator<VisualQueryDetectedResult> CREATOR + = new Parcelable.Creator<VisualQueryDetectedResult>() { + @Override + public VisualQueryDetectedResult[] newArray(int size) { + return new VisualQueryDetectedResult[size]; + } + + @Override + public VisualQueryDetectedResult createFromParcel(@NonNull Parcel in) { + return new VisualQueryDetectedResult(in); + } + }; + + /** + * A builder for {@link VisualQueryDetectedResult} + */ + @SuppressWarnings("WeakerAccess") + @DataClass.Generated.Member + public static final class Builder { + + private @NonNull String mPartialQuery; + + private long mBuilderFieldsSet = 0L; + + public Builder() { + } + + /** + * Text query being associated with the detection result. * + */ + @DataClass.Generated.Member + public @NonNull Builder setPartialQuery(@NonNull String value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x1; + mPartialQuery = value; + return this; + } + + /** Builds the instance. This builder should not be touched after calling this! */ + public @NonNull VisualQueryDetectedResult build() { + checkNotUsed(); + mBuilderFieldsSet |= 0x2; // Mark builder used + + if ((mBuilderFieldsSet & 0x1) == 0) { + mPartialQuery = defaultPartialQuery(); + } + VisualQueryDetectedResult o = new VisualQueryDetectedResult( + mPartialQuery); + return o; + } + + private void checkNotUsed() { + if ((mBuilderFieldsSet & 0x2) != 0) { + throw new IllegalStateException( + "This Builder should not be reused. Use a new Builder instance instead"); + } + } + } + + @DataClass.Generated( + time = 1704944313642L, + codegenVersion = "1.0.23", + sourceFile = "frameworks/base/core/java/android/service/voice/VisualQueryDetectedResult.java", + inputSignatures = "private final @android.annotation.NonNull java.lang.String mPartialQuery\nprivate static java.lang.String defaultPartialQuery()\npublic android.service.voice.VisualQueryDetectedResult.Builder buildUpon()\nclass VisualQueryDetectedResult extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)") + @Deprecated + private void __metadata() {} + + + //@formatter:on + // End of generated code + +} diff --git a/core/java/android/service/voice/VisualQueryDetectionService.java b/core/java/android/service/voice/VisualQueryDetectionService.java index 76b076be8fab..b60c77502247 100644 --- a/core/java/android/service/voice/VisualQueryDetectionService.java +++ b/core/java/android/service/voice/VisualQueryDetectionService.java @@ -17,6 +17,7 @@ package android.service.voice; import android.annotation.DurationMillisLong; +import android.annotation.FlaggedApi; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SdkConstant; @@ -35,6 +36,7 @@ import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; import android.os.RemoteException; import android.os.SharedMemory; +import android.service.voice.flags.Flags; import android.speech.IRecognitionServiceManager; import android.util.Log; import android.view.contentcapture.ContentCaptureManager; @@ -282,6 +284,7 @@ public abstract class VisualQueryDetectionService extends Service } } + // TODO(b/324341724): Properly deprecate this API. /** * Informs the {@link VisualQueryDetector} with the text content being captured about the * query from the audio source. {@code partialQuery} is provided to the @@ -289,6 +292,9 @@ public abstract class VisualQueryDetectionService extends Service * {@link VisualQueryDetectionService#gainedAttention()} is called to put the service into the * attention gained state. * + * Usage of this method is not recommended, please use + * {@link VisualQueryDetectionService#streamQuery(VisualQueryDetectedResult)} instead. + * * @param partialQuery Partially detected query in string. * @throws IllegalStateException if method called without attention gained. */ @@ -303,6 +309,27 @@ public abstract class VisualQueryDetectionService extends Service } /** + * Informs the {@link VisualQueryDetector} with the text content being captured about the + * query from the audio source. {@code partialResult} is provided to the + * {@link VisualQueryDetector}. This method is expected to be only triggered if + * {@link VisualQueryDetectionService#gainedAttention()} is called to put the service into + * the attention gained state. + * + * @param partialResult Partially detected result in the format of + * {@link VisualQueryDetectedResult}. + */ + @FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) + public final void streamQuery(@NonNull VisualQueryDetectedResult partialResult) { + Objects.requireNonNull(partialResult); + try { + mRemoteCallback.onResultDetected(partialResult); + } catch (RemoteException e) { + throw new IllegalStateException("#streamQuery must be only be triggered after " + + "calling #gainedAttention to be in the attention gained state."); + } + } + + /** * Informs the {@link VisualQueryDetector} to abandon the streamed partial query that has * been sent to {@link VisualQueryDetector}.This method is expected to be only triggered if * {@link VisualQueryDetectionService#streamQuery(String)} is called to put the service into diff --git a/core/java/android/service/voice/VisualQueryDetector.java b/core/java/android/service/voice/VisualQueryDetector.java index f2bdbf67e76e..aee25f5e1f00 100644 --- a/core/java/android/service/voice/VisualQueryDetector.java +++ b/core/java/android/service/voice/VisualQueryDetector.java @@ -20,6 +20,7 @@ import static android.Manifest.permission.CAMERA; import static android.Manifest.permission.RECORD_AUDIO; import android.annotation.CallbackExecutor; +import android.annotation.FlaggedApi; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; @@ -33,6 +34,7 @@ import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; import android.os.RemoteException; import android.os.SharedMemory; +import android.service.voice.flags.Flags; import android.text.TextUtils; import android.util.Slog; @@ -206,6 +208,17 @@ public class VisualQueryDetector { void onQueryDetected(@NonNull String partialQuery); /** + * Called when the {@link VisualQueryDetectionService} starts to stream partial results + * with {@link VisualQueryDetectionService#streamQuery(VisualQueryDetectedResult)}. + * + * @param partialResult The partial query in a text form being streamed. + */ + @FlaggedApi(Flags.FLAG_ALLOW_COMPLEX_RESULTS_EGRESS_FROM_VQDS) + default void onQueryDetected(@NonNull VisualQueryDetectedResult partialResult) { + throw new UnsupportedOperationException("This emthod must be implemented for use."); + } + + /** * Called when the {@link VisualQueryDetectionService} decides to abandon the streamed * partial queries with {@link VisualQueryDetectionService#rejectQuery()}. */ @@ -319,7 +332,7 @@ public class VisualQueryDetector { this.mLock = lock; } - /** Called when the detected result is valid. */ + /** Called when the detected query is valid. */ @Override public void onQueryDetected(@NonNull String partialQuery) { Slog.v(TAG, "BinderCallback#onQueryDetected"); @@ -330,6 +343,17 @@ public class VisualQueryDetector { }); } + /** Called when the detected result is valid. */ + @Override + public void onResultDetected(@NonNull VisualQueryDetectedResult partialResult) { + Slog.v(TAG, "BinderCallback#onResultDetected"); + Binder.withCleanCallingIdentity(() -> { + synchronized (mLock) { + mCallback.onQueryDetected(partialResult); + } + }); + } + @Override public void onQueryFinished() { Slog.v(TAG, "BinderCallback#onQueryFinished"); diff --git a/core/java/android/service/voice/flags/flags.aconfig b/core/java/android/service/voice/flags/flags.aconfig index 1c8752b234c1..a15812e5e70e 100644 --- a/core/java/android/service/voice/flags/flags.aconfig +++ b/core/java/android/service/voice/flags/flags.aconfig @@ -20,3 +20,17 @@ flag { description: "This flag allows providing foreground app component along with onShow args." bug: "319409708" } + +flag { + name: "allow_various_attention_types" + namespace: "visual_query" + description: "This flag allows visual query detection service to set different attention types." + bug: "318617199" +} + +flag { + name: "allow_complex_results_egress_from_vqds" + namespace: "visual_query" + description: "This flag allows visual query detection service egress detailed results. " + bug: "318617199" +}
\ No newline at end of file diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java index f9fa9b7a9524..d7b860f6a857 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java @@ -33,6 +33,7 @@ import android.service.voice.IDetectorSessionVisualQueryDetectionCallback; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.ISandboxedDetectionService; import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback; +import android.service.voice.VisualQueryDetectedResult; import android.service.voice.VisualQueryDetectionServiceFailure; import android.util.Slog; @@ -171,6 +172,26 @@ final class VisualQueryDetectorSession extends DetectorSession { } @Override + public void onResultDetected(@NonNull VisualQueryDetectedResult partialResult) + throws RemoteException { + Slog.v(TAG, "BinderCallback#onResultDetected"); + synchronized (mLock) { + Objects.requireNonNull(partialResult); + if (!mEgressingData) { + Slog.v(TAG, "Result should not be egressed within the unattention state."); + callback.onVisualQueryDetectionServiceFailure( + new VisualQueryDetectionServiceFailure( + ERROR_CODE_ILLEGAL_STREAMING_STATE, + "Cannot stream results without attention signals.")); + return; + } + mQueryStreaming = true; + callback.onResultDetected(partialResult); + Slog.i(TAG, "Egressed from visual query detection process."); + } + } + + @Override public void onQueryFinished() throws RemoteException { Slog.v(TAG, "BinderCallback#onQueryFinished"); synchronized (mLock) { |