Merge "Add system change to enable stop/start recognition"

6 files changed, 230 insertions, 0 deletions

diff --git a/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl b/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl
new file mode 100644
index 000000000000..2eb24706da30
--- /dev/null
+++ b/core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.service.voice;
+
+import android.media.AudioFormat;
+
+/**
+ * Callback for returning the detected result from the VisualQueryDetectionService.
+ *
+ * @hide
+ */
+oneway interface IVisualQueryDetectionVoiceInteractionCallback {
+
+    /**
+     * Called when the detected query is streamed
+     */
+    void onQueryDetected(in String partialQuery);
+
+    /**
+     * Called when the detected result is valid.
+     */
+    void onQueryFinished();
+
+    /**
+     * Called when the detected result is invalid.
+     */
+    void onQueryRejected();
+
+    /**
+     * Called when the detection fails due to an error.
+     */
+    void onError();
+
+}
diff --git a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
index b9ca557da4bd..88d642542235 100644
--- a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
+++ b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
@@ -27,6 +27,7 @@ import android.os.ParcelFileDescriptor;
 import android.os.PersistableBundle;
 import android.os.RemoteCallback;
 import android.os.SharedMemory;
+import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
 import android.service.voice.IVoiceInteractionService;
 import android.service.voice.IVoiceInteractionSession;
@@ -299,6 +300,10 @@ interface IVoiceInteractionManagerService {
      */
     void shutdownHotwordDetectionService();
 
+    void startPerceiving(in IVisualQueryDetectionVoiceInteractionCallback callback);
+
+    void stopPerceiving();
+
     void startListeningFromMic(
         in AudioFormat audioFormat,
         in IMicrophoneHotwordDetectionVoiceInteractionCallback callback);
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
index f1dd9091d5de..c37330586c9e 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
@@ -52,6 +52,7 @@ import android.service.voice.HotwordDetectionService;
 import android.service.voice.HotwordDetector;
 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
 import android.service.voice.ISandboxedDetectionService;
+import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
 import android.service.voice.VisualQueryDetectionService;
 import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity;
 import android.speech.IRecognitionServiceManager;
@@ -290,6 +291,34 @@ final class HotwordDetectionConnection {
         session.startListeningFromMicLocked(audioFormat, callback);
     }
 
+    /**
+     * This method is only used by VisualQueryDetector.
+     */
+    void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
+        if (DEBUG) {
+            Slog.d(TAG, "startPerceivingLocked");
+        }
+        final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked();
+        if (session == null) {
+            return;
+        }
+        session.startPerceivingLocked(callback);
+    }
+
+    /**
+     * This method is only used by VisaulQueryDetector.
+     */
+    void stopPerceivingLocked() {
+        if (DEBUG) {
+            Slog.d(TAG, "stopPerceivingLocked");
+        }
+        final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked();
+        if (session == null) {
+            return;
+        }
+        session.stopPerceivingLocked();
+    }
+
     public void startListeningFromExternalSourceLocked(
             ParcelFileDescriptor audioStream,
             AudioFormat audioFormat,
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java
index 6e4bc05ea25b..621c3de3d9a9 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java
@@ -24,13 +24,18 @@ import android.media.permission.Identity;
 import android.os.IBinder;
 import android.os.ParcelFileDescriptor;
 import android.os.PersistableBundle;
+import android.os.RemoteException;
 import android.os.SharedMemory;
+import android.service.voice.IDetectorSessionVisualQueryDetectionCallback;
 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
+import android.service.voice.ISandboxedDetectionService;
+import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
 import android.util.Slog;
 
 import com.android.internal.app.IHotwordRecognitionStatusCallback;
 
 import java.io.PrintWriter;
+import java.util.Objects;
 import java.util.concurrent.ScheduledExecutorService;
 
 /**
@@ -44,6 +49,8 @@ import java.util.concurrent.ScheduledExecutorService;
 final class VisualQueryDetectorSession extends DetectorSession {
 
     private static final String TAG = "VisualQueryDetectorSession";
+    private boolean mEgressingData;
+    private boolean mQueryStreaming;
 
     //TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.
     VisualQueryDetectorSession(
@@ -55,6 +62,8 @@ final class VisualQueryDetectorSession extends DetectorSession {
         super(remoteService, lock, context, token, callback,
                 voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,
                 logging);
+        mEgressingData = false;
+        mQueryStreaming = false;
     }
 
     @Override
@@ -65,6 +74,77 @@ final class VisualQueryDetectorSession extends DetectorSession {
         //TODO(b/261783819): Starts detection in VisualQueryDetectionService.
     }
 
+    @SuppressWarnings("GuardedBy")
+    void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
+        if (DEBUG) {
+            Slog.d(TAG, "startPerceivingLocked");
+        }
+
+        IDetectorSessionVisualQueryDetectionCallback internalCallback =
+                new IDetectorSessionVisualQueryDetectionCallback.Stub(){
+
+            @Override
+            public void onAttentionGained() {
+                Slog.v(TAG, "BinderCallback#onAttentionGained");
+                //TODO check to see if there is an active SysUI listener registered
+                mEgressingData = true;
+            }
+
+            @Override
+            public void onAttentionLost() {
+                Slog.v(TAG, "BinderCallback#onAttentionLost");
+                //TODO check to see if there is an active SysUI listener registered
+                mEgressingData = false;
+            }
+
+            @Override
+            public void onQueryDetected(@NonNull String partialQuery) throws RemoteException {
+                Objects.requireNonNull(partialQuery);
+                Slog.v(TAG, "BinderCallback#onQueryDetected");
+                if (!mEgressingData) {
+                    Slog.v(TAG, "Query should not be egressed within the unattention state.");
+                    return;
+                }
+                mQueryStreaming = true;
+                callback.onQueryDetected(partialQuery);
+                Slog.i(TAG, "Egressed from visual query detection process.");
+            }
+
+            @Override
+            public void onQueryFinished() throws RemoteException {
+                Slog.v(TAG, "BinderCallback#onQueryFinished");
+                if (!mQueryStreaming) {
+                    Slog.v(TAG, "Query streaming state signal FINISHED is block since there is"
+                            + " no active query being streamed.");
+                    return;
+                }
+                callback.onQueryFinished();
+                mQueryStreaming = false;
+            }
+
+            @Override
+            public void onQueryRejected() throws RemoteException {
+                Slog.v(TAG, "BinderCallback#onQueryRejected");
+                if (!mQueryStreaming) {
+                    Slog.v(TAG, "Query streaming state signal REJECTED is block since there is"
+                            + " no active query being streamed.");
+                    return;
+                }
+                callback.onQueryRejected();
+                mQueryStreaming = false;
+            }
+        };
+        mRemoteDetectionService.run(service -> service.detectWithVisualSignals(internalCallback));
+    }
+
+    @SuppressWarnings("GuardedBy")
+    void stopPerceivingLocked() {
+        if (DEBUG) {
+            Slog.d(TAG, "stopPerceivingLocked");
+        }
+        mRemoteDetectionService.run(ISandboxedDetectionService::stopDetection);
+    }
+
     @Override
      void startListeningFromExternalSourceLocked(
             ParcelFileDescriptor audioStream,
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
index 9a0218845038..19aa374f96a1 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
@@ -72,6 +72,7 @@ import android.os.Trace;
 import android.os.UserHandle;
 import android.provider.Settings;
 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
+import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
 import android.service.voice.IVoiceInteractionSession;
 import android.service.voice.VoiceInteractionManagerInternal;
 import android.service.voice.VoiceInteractionService;
@@ -1314,6 +1315,46 @@ public class VoiceInteractionManagerService extends SystemService {
         }
 
         @Override
+        public void startPerceiving(
+                IVisualQueryDetectionVoiceInteractionCallback callback)
+                throws RemoteException {
+            enforceCallingPermission(Manifest.permission.RECORD_AUDIO);
+            enforceCallingPermission(Manifest.permission.CAMERA);
+            synchronized (this) {
+                enforceIsCurrentVoiceInteractionService();
+
+                if (mImpl == null) {
+                    Slog.w(TAG, "startPerceiving without running voice interaction service");
+                    return;
+                }
+                final long caller = Binder.clearCallingIdentity();
+                try {
+                    mImpl.startPerceivingLocked(callback);
+                } finally {
+                    Binder.restoreCallingIdentity(caller);
+                }
+            }
+        }
+
+        @Override
+        public void stopPerceiving() throws RemoteException {
+            synchronized (this) {
+                enforceIsCurrentVoiceInteractionService();
+
+                if (mImpl == null) {
+                    Slog.w(TAG, "stopPerceiving without running voice interaction service");
+                    return;
+                }
+                final long caller = Binder.clearCallingIdentity();
+                try {
+                    mImpl.stopPerceivingLocked();
+                } finally {
+                    Binder.restoreCallingIdentity(caller);
+                }
+            }
+        }
+
+        @Override
         public void startListeningFromMic(
                 AudioFormat audioFormat,
                 IMicrophoneHotwordDetectionVoiceInteractionCallback callback)
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
index e320e69ebceb..4ee33067cc12 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
@@ -60,6 +60,7 @@ import android.os.SharedMemory;
 import android.os.UserHandle;
 import android.service.voice.HotwordDetector;
 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
+import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
 import android.service.voice.IVoiceInteractionService;
 import android.service.voice.IVoiceInteractionSession;
 import android.service.voice.VoiceInteractionService;
@@ -737,6 +738,32 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
         mHotwordDetectionConnection = null;
     }
 
+    public void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
+        if (DEBUG) {
+            Slog.d(TAG, "startPerceivingLocked");
+        }
+
+        if (mHotwordDetectionConnection == null) {
+            // TODO: callback.onError();
+            return;
+        }
+
+        mHotwordDetectionConnection.startPerceivingLocked(callback);
+    }
+
+    public void stopPerceivingLocked() {
+        if (DEBUG) {
+            Slog.d(TAG, "stopPerceivingLocked");
+        }
+
+        if (mHotwordDetectionConnection == null) {
+            Slog.w(TAG, "stopPerceivingLocked() called but connection isn't established");
+            return;
+        }
+
+        mHotwordDetectionConnection.stopPerceivingLocked();
+    }
+
     public void startListeningFromMicLocked(
             AudioFormat audioFormat,
             IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {