Add VisaulQueryDetector API

Added VisualQueryDetector to manage VisualQueryDetectionService
lifecycle and provide corresponding methods to start service
functionalities.

Bug: 261783492
Test: atest CtsVoiceInteractionTestCases
Change-Id: I2b1d37ab246b28e27f75bdaf8d9ad8f2d36d6dc7
API-Coverage-Bug: 264039061

5 files changed, 389 insertions, 10 deletions

diff --git a/core/api/system-current.txt b/core/api/system-current.txt
index a9924d8832e5..bb754327a58a 100644
--- a/core/api/system-current.txt
+++ b/core/api/system-current.txt
@@ -12564,11 +12564,28 @@ package android.service.voice {
     method public void onQueryRejected() throws java.lang.IllegalStateException;
   }
 
+  public class VisualQueryDetector {
+    method public void destroy();
+    method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean startRecognition() throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
+    method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean stopRecognition() throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
+    method public void updateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory) throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
+  }
+
+  public static interface VisualQueryDetector.Callback {
+    method public void onError();
+    method public void onQueryDetected(@NonNull String);
+    method public void onQueryFinished();
+    method public void onQueryRejected();
+    method public void onVisualQueryDetectionServiceInitialized(int);
+    method public void onVisualQueryDetectionServiceRestarted();
+  }
+
   public class VoiceInteractionService extends android.app.Service {
     method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback);
     method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, android.service.voice.AlwaysOnHotwordDetector.Callback);
     method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
     method @NonNull @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public final android.media.voice.KeyphraseModelManager createKeyphraseModelManager();
+    method @NonNull public final android.service.voice.VisualQueryDetector createVisualQueryDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull java.util.concurrent.Executor, @NonNull android.service.voice.VisualQueryDetector.Callback);
   }
 
 }
diff --git a/core/java/android/service/voice/AbstractDetector.java b/core/java/android/service/voice/AbstractDetector.java
index db0ede5a9e70..a70f7837d767 100644
--- a/core/java/android/service/voice/AbstractDetector.java
+++ b/core/java/android/service/voice/AbstractDetector.java
@@ -84,7 +84,7 @@ abstract class AbstractDetector implements HotwordDetector {
             @Nullable SharedMemory sharedMemory);
 
     /**
-     * Detect hotword from an externally supplied stream of data.
+     * Detect from an externally supplied stream of data.
      *
      * @return {@code true} if the request to start recognition succeeded
      */
@@ -114,7 +114,25 @@ abstract class AbstractDetector implements HotwordDetector {
         return true;
     }
 
-    /** {@inheritDoc} */
+    /**
+     * Set configuration and pass read-only data to trusted detection service.
+     *
+     * @param options Application configuration data to provide to the
+     *         {@link VisualQueryDetectionService} and {@link HotwordDetectionService}.
+     *         PersistableBundle does not allow any remotable objects or other contents that can be
+     *         used to communicate with other processes.
+     * @param sharedMemory The unrestricted data blob to provide to the
+     *        {@link VisualQueryDetectionService} and {@link HotwordDetectionService}. Use this to
+     *         provide the hotword models data or other such data to the trusted process.
+     * @throws IllegalDetectorStateException Thrown when a caller has a target SDK of
+     *         Android Tiramisu or above and attempts to start a recognition when the detector is
+     *         not able based on the state. Because the caller receives updates via an asynchronous
+     *         callback and the state of the detector can change without caller's knowledge, a
+     *         checked exception is thrown.
+     * @throws IllegalStateException if this {@link HotwordDetector} wasn't specified to use a
+     *         {@link HotwordDetectionService} or {@link VisualQueryDetectionService} when it was
+     *         created.
+     */
     @Override
     public void updateState(@Nullable PersistableBundle options,
             @Nullable SharedMemory sharedMemory) throws IllegalDetectorStateException {
diff --git a/core/java/android/service/voice/HotwordDetector.java b/core/java/android/service/voice/HotwordDetector.java
index 669c22b73edc..562277e9d097 100644
--- a/core/java/android/service/voice/HotwordDetector.java
+++ b/core/java/android/service/voice/HotwordDetector.java
@@ -35,7 +35,8 @@ import android.util.AndroidException;
 import java.io.PrintWriter;
 
 /**
- * Basic functionality for sandboxed detectors.
+ * Basic functionality for sandboxed detectors. This interface will be used by detectors that
+ * manages their service lifecycle.
  *
  * @hide
  */
@@ -81,9 +82,20 @@ public interface HotwordDetector {
     int DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE = 2;
 
     /**
+     * Indicates that it is a visual query detector.
+     *
+     * @hide
+     */
+    int DETECTOR_TYPE_VISUAL_QUERY_DETECTOR = 3;
+
+    /**
      * Starts sandboxed detection recognition.
      * <p>
-     * On calling this, the system streams audio from the device microphone to this application's
+     * If a {@link VisualQueryDetector} calls this method, {@link VisualQueryDetectionService
+     * #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start detection.
+     * <p>
+     * Otherwise if a {@link AlwaysOnHotwordDetector} or {@link SoftwareHotwordDetector} calls this,
+     * the system streams audio from the device microphone to this application's
      * {@link HotwordDetectionService}. Audio is streamed until {@link #stopRecognition()} is
      * called.
      * <p>
@@ -192,6 +204,8 @@ public interface HotwordDetector {
                 return "trusted_hotword_dsp";
             case DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE:
                 return "trusted_hotword_software";
+            case DETECTOR_TYPE_VISUAL_QUERY_DETECTOR:
+                return "visual_query_detector";
             default:
                 return Integer.toString(detectorType);
         }
@@ -244,18 +258,21 @@ public interface HotwordDetector {
         void onRejected(@NonNull HotwordRejectedResult result);
 
         /**
-         * Called when the {@link HotwordDetectionService} is created by the system and given a
-         * short amount of time to report its initialization state.
+         * Called when the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
+         * created by the system and given a short amount of time to report their initialization
+         * state.
          *
-         * @param status Info about initialization state of {@link HotwordDetectionService}; the
-         * allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
+         * @param status Info about initialization state of {@link HotwordDetectionService} or
+         * {@link VisualQueryDetectionService}; allowed values are
+         * {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
          * 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
          * {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
          */
         void onHotwordDetectionServiceInitialized(int status);
 
         /**
-         * Called with the {@link HotwordDetectionService} is restarted.
+         * Called with the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
+         * restarted.
          *
          * Clients are expected to call {@link HotwordDetector#updateState} to share the state with
          * the newly created service.
diff --git a/core/java/android/service/voice/VisualQueryDetector.java b/core/java/android/service/voice/VisualQueryDetector.java
new file mode 100644
index 000000000000..241f5bac55ff
--- /dev/null
+++ b/core/java/android/service/voice/VisualQueryDetector.java
@@ -0,0 +1,241 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.service.voice;
+
+import static android.Manifest.permission.CAMERA;
+import static android.Manifest.permission.RECORD_AUDIO;
+
+import android.annotation.CallbackExecutor;
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.annotation.RequiresPermission;
+import android.annotation.SuppressLint;
+import android.annotation.SystemApi;
+import android.media.AudioFormat;
+import android.os.ParcelFileDescriptor;
+import android.os.PersistableBundle;
+import android.os.SharedMemory;
+import android.util.Slog;
+
+import com.android.internal.app.IVoiceInteractionManagerService;
+
+import java.io.PrintWriter;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+
+/**
+ * Manages VisualQueryDetectionService.
+ *
+ * This detector provides necessary functionalities to initialize, start, update and destroy a
+ * {@link VisualQueryDetectionService}.
+ *
+ * @hide
+ **/
+@SystemApi
+@SuppressLint("NotCloseable")
+public class VisualQueryDetector {
+    private static final String TAG = VisualQueryDetector.class.getSimpleName();
+    private static final boolean DEBUG = false;
+
+    private final Callback mCallback;
+    private final Executor mExecutor;
+    private final IVoiceInteractionManagerService mManagerService;
+    private final VisualQueryDetectorInitializationDelegate mInitializationDelegate;
+
+    VisualQueryDetector(
+            IVoiceInteractionManagerService managerService,
+            @NonNull @CallbackExecutor Executor executor,
+            Callback callback) {
+
+        mManagerService = managerService;
+        mCallback = callback;
+        mExecutor = executor;
+        mInitializationDelegate = new VisualQueryDetectorInitializationDelegate();
+    }
+
+    /**
+     * Initialize the {@link VisualQueryDetectionService} by passing configurations and read-only
+     * data.
+     */
+    void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
+        mInitializationDelegate.initialize(options, sharedMemory);
+    }
+
+    /**
+     * Set configuration and pass read-only data to {@link VisualQueryDetectionService}.
+     *
+     * @see HotwordDetector#updateState(PersistableBundle, SharedMemory)
+     */
+    public void updateState(@Nullable PersistableBundle options,
+            @Nullable SharedMemory sharedMemory) throws
+            HotwordDetector.IllegalDetectorStateException {
+        mInitializationDelegate.updateState(options, sharedMemory);
+    }
+
+
+    /**
+     * On calling this method, {@link VisualQueryDetectionService
+     * #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start using
+     * visual signals such as camera frames and microphone audio to perform detection. When user
+     * attention is captured and the {@link VisualQueryDetectionService} streams queries,
+     * {@link VisualQueryDetector.Callback#onQueryDetected(String)} is called to control the
+     * behavior of handling {@code transcribedText}. When the query streaming is finished,
+     * {@link VisualQueryDetector.Callback#onQueryFinished()} is called. If the current streamed
+     * query is invalid, {@link VisualQueryDetector.Callback#onQueryRejected()} is called to abandon
+     * the streamed query.
+     *
+     * @see HotwordDetector#startRecognition()
+     */
+    @RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
+    public boolean startRecognition() throws HotwordDetector.IllegalDetectorStateException {
+        if (DEBUG) {
+            Slog.i(TAG, "#startRecognition");
+        }
+        // TODO(b/261783819): Call StartDetection on VisualQueryDetectionService with the system.
+        return false;
+    }
+
+    /**
+     * Stops visual query detection recognition.
+     *
+     * @see HotwordDetector#stopRecognition()
+     */
+    @RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
+    public boolean stopRecognition() throws HotwordDetector.IllegalDetectorStateException {
+        if (DEBUG) {
+            Slog.i(TAG, "#stopRecognition");
+        }
+        // TODO(b/261783819): Call StopDetection on VisualQueryDetectionService with the system.
+        return false;
+    }
+
+    /**
+     * Destroy the current detector.
+     *
+     * @see HotwordDetector#destroy()
+     */
+    public void destroy() {
+        if (DEBUG) {
+            Slog.i(TAG, "#destroy");
+        }
+        mInitializationDelegate.destroy();
+    }
+
+    /** @hide */
+    public void dump(String prefix, PrintWriter pw) {
+        // TODO: implement this
+    }
+
+    /** @hide */
+    public HotwordDetector getInitializationDelegate() {
+        return mInitializationDelegate;
+    }
+
+    /** @hide */
+    void registerOnDestroyListener(Consumer<AbstractDetector> onDestroyListener) {
+        mInitializationDelegate.registerOnDestroyListener(onDestroyListener);
+    }
+
+    /**
+     * A class that lets a VoiceInteractionService implementation interact with
+     * visual query detection APIs.
+     */
+    public interface Callback {
+
+        /**
+         * Called when the {@link VisualQueryDetectionService} starts to stream partial queries.
+         *
+         * @param partialQuery The partial query in a text form being streamed.
+         */
+        void onQueryDetected(@NonNull String partialQuery);
+
+        /**
+         * Called when the {@link VisualQueryDetectionService} decides to abandon the streamed
+         * partial queries.
+         */
+        void onQueryRejected();
+
+        /**
+         *  Called when the {@link VisualQueryDetectionService} finishes streaming partial queries.
+         */
+        void onQueryFinished();
+
+        /**
+         * Called when the {@link VisualQueryDetectionService} is created by the system and given a
+         * short amount of time to report its initialization state.
+         *
+         * @param status Info about initialization state of {@link VisualQueryDetectionService}; the
+         * allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
+         * 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
+         * {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
+         */
+        void onVisualQueryDetectionServiceInitialized(int status);
+
+         /**
+         * Called with the {@link VisualQueryDetectionService} is restarted.
+         *
+         * Clients are expected to call {@link HotwordDetector#updateState} to share the state with
+         * the newly created service.
+         */
+        void onVisualQueryDetectionServiceRestarted();
+
+        /**
+         * Called when the detection fails due to an error.
+         */
+        //TODO(b/265390855): Replace this callback with the new onError(DetectorError) design.
+        void onError();
+    }
+
+    private class VisualQueryDetectorInitializationDelegate extends AbstractDetector {
+
+        VisualQueryDetectorInitializationDelegate() {
+            super(mManagerService, null);
+        }
+
+        @Override
+        void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
+            //TODO(261783492): call initAndVerify to create VisualQueryDetectionService
+            // from the system server.
+        }
+
+        @Override
+        public boolean stopRecognition() throws IllegalDetectorStateException {
+            //No-op, we only reuse the initialization methods.
+            return false;
+        }
+
+        @Override
+        public boolean startRecognition() throws IllegalDetectorStateException {
+            //No-op, we only reuse the initialization methods.
+            return false;
+        }
+
+        @Override
+        public final boolean startRecognition(
+                @NonNull ParcelFileDescriptor audioStream,
+                @NonNull AudioFormat audioFormat,
+                @Nullable PersistableBundle options) throws IllegalDetectorStateException {
+            //No-op, not supported by VisualQueryDetector as it should be trusted.
+            return false;
+        }
+
+        @Override
+        public boolean isUsingSandboxedDetectionService() {
+            return true;
+        }
+    }
+}
diff --git a/core/java/android/service/voice/VoiceInteractionService.java b/core/java/android/service/voice/VoiceInteractionService.java
index 9e1518d899e0..1a69f28b338b 100644
--- a/core/java/android/service/voice/VoiceInteractionService.java
+++ b/core/java/android/service/voice/VoiceInteractionService.java
@@ -17,6 +17,7 @@
 package android.service.voice;
 
 import android.Manifest;
+import android.annotation.CallbackExecutor;
 import android.annotation.NonNull;
 import android.annotation.Nullable;
 import android.annotation.RequiresPermission;
@@ -58,7 +59,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
 import java.util.Set;
-
+import java.util.concurrent.Executor;
 /**
  * Top-level service of the current global voice interactor, which is providing
  * support for hotwording, the back-end of a {@link android.app.VoiceInteractor}, etc.
@@ -164,6 +165,8 @@ public class VoiceInteractionService extends Service {
 
     IVoiceInteractionManagerService mSystemService;
 
+    private VisualQueryDetector mActiveVisualQueryDetector;
+
     private final Object mLock = new Object();
 
     private KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
@@ -544,6 +547,85 @@ public class VoiceInteractionService extends Service {
     }
 
     /**
+     * Creates a {@link VisualQueryDetector} and initializes the application's
+     * {@link VisualQueryDetectionService} using {@code options} and {@code sharedMemory}.
+     *
+     * <p>To be able to call this, you need to set android:visualQueryDetectionService in the
+     * android.voice_interaction metadata file to a valid visual query detection service, and set
+     * android:isolatedProcess="true" in the service's declaration. Otherwise, this throws an
+     * {@link IllegalStateException}.
+     *
+     * <p>Using this has a noticeable impact on battery, since the microphone is kept open
+     * for the lifetime of the recognition {@link VisualQueryDetector#startRecognition() session}.
+     *
+     * @param options Application configuration data to be provided to the
+     * {@link VisualQueryDetectionService}. PersistableBundle does not allow any remotable objects
+     * or other contents that can be used to communicate with other processes.
+     * @param sharedMemory The unrestricted data blob to be provided to the
+     * {@link VisualQueryDetectionService}. Use this to provide models or other such data to the
+     * sandboxed process.
+     * @param callback The callback to notify of detection events.
+     * @return An instanece of {@link VisualQueryDetector}.
+     * @throws UnsupportedOperationException if only single detector is supported. Multiple detector
+     * is only available for apps targeting {@link Build.VERSION_CODES#TIRAMISU} and above.
+     * @throws IllegalStateException when there is an existing {@link VisualQueryDetector}, or when
+     * there is a non-trusted hotword detector running.
+     *
+     * @hide
+     */
+    // TODO: add MANAGE_HOTWORD_DETECTION permission to protect this API and update java doc.
+    @SystemApi
+    @NonNull
+    public final VisualQueryDetector createVisualQueryDetector(
+            @Nullable PersistableBundle options,
+            @Nullable SharedMemory sharedMemory,
+            @NonNull @CallbackExecutor Executor executor,
+            @NonNull VisualQueryDetector.Callback callback) {
+        Objects.requireNonNull(executor);
+        Objects.requireNonNull(callback);
+
+        if (mSystemService == null) {
+            throw new IllegalStateException("Not available until onReady() is called");
+        }
+        synchronized (mLock) {
+            if (!CompatChanges.isChangeEnabled(MULTIPLE_ACTIVE_HOTWORD_DETECTORS)) {
+                throw new UnsupportedOperationException("VisualQueryDetector is only available if "
+                        + "multiple detectors are allowed");
+            } else {
+                if (mActiveVisualQueryDetector != null) {
+                    throw new IllegalStateException(
+                                "There is already an active VisualQueryDetector. "
+                                        + "It must be destroyed to create a new one.");
+                }
+                for (HotwordDetector detector : mActiveDetectors) {
+                    if (!detector.isUsingSandboxedDetectionService()) {
+                        throw new IllegalStateException(
+                                "It disallows to create trusted and non-trusted detectors "
+                                        + "at the same time.");
+                    }
+                }
+            }
+
+            VisualQueryDetector visualQueryDetector =
+                    new VisualQueryDetector(mSystemService, executor, callback);
+            HotwordDetector visualQueryDetectorInitializationDelegate =
+                    visualQueryDetector.getInitializationDelegate();
+            mActiveDetectors.add(visualQueryDetectorInitializationDelegate);
+
+            try {
+                visualQueryDetector.registerOnDestroyListener(this::onHotwordDetectorDestroyed);
+                visualQueryDetector.initialize(options, sharedMemory);
+            } catch (Exception e) {
+                mActiveDetectors.remove(visualQueryDetectorInitializationDelegate);
+                visualQueryDetector.destroy();
+                throw e;
+            }
+            mActiveVisualQueryDetector = visualQueryDetector;
+            return visualQueryDetector;
+        }
+    }
+
+    /**
      * Creates an {@link KeyphraseModelManager} to use for enrolling voice models outside of the
      * pre-bundled system voice models.
      * @hide
@@ -598,6 +680,10 @@ public class VoiceInteractionService extends Service {
 
     private void onHotwordDetectorDestroyed(@NonNull HotwordDetector detector) {
         synchronized (mLock) {
+            if (mActiveVisualQueryDetector!= null &&
+                    detector == mActiveVisualQueryDetector.getInitializationDelegate()) {
+                mActiveVisualQueryDetector = null;
+            }
             mActiveDetectors.remove(detector);
             shutdownHotwordDetectionServiceIfRequiredLocked();
         }