Camera2: Stop repeating request for abandoned output

Stop repeating request if any of its output stream is abandoned
and notify that repeating request has been stopped.

Update binder tests for binder interface changes.

Update CameraDeviceImpl to expect an exception when canceling
a repeating request that is already stopped.

Bug: 21270879
Change-Id: I9fa72ae7218948aac88cb1a8e57839bd022c4a5e

9 files changed, 173 insertions, 11 deletions

diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
index b5423392b96e..5743b4d42a37 100644
--- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
+++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
@@ -921,7 +921,16 @@ public class CameraDeviceImpl extends CameraDevice
                 int requestId = mRepeatingRequestId;
                 mRepeatingRequestId = REQUEST_ID_NONE;
 
-                long lastFrameNumber = mRemoteDevice.cancelRequest(requestId);
+                long lastFrameNumber;
+                try {
+                    lastFrameNumber = mRemoteDevice.cancelRequest(requestId);
+                } catch (IllegalArgumentException e) {
+                    if (DEBUG) {
+                        Log.v(TAG, "Repeating request was already stopped for request " + requestId);
+                    }
+                    // Repeating request was already stopped. Nothing more to do.
+                    return;
+                }
 
                 checkEarlyTriggerSequenceComplete(requestId, lastFrameNumber);
             }
@@ -1686,6 +1695,24 @@ public class CameraDeviceImpl extends CameraDevice
         }
 
         @Override
+        public void onRepeatingRequestError(long lastFrameNumber) {
+            if (DEBUG) {
+                Log.d(TAG, "Repeating request error received. Last frame number is " +
+                        lastFrameNumber);
+            }
+
+            synchronized(mInterfaceLock) {
+                // Camera is already closed or no repeating request is present.
+                if (mRemoteDevice == null || mRepeatingRequestId == REQUEST_ID_NONE) {
+                    return; // Camera already closed
+                }
+
+                checkEarlyTriggerSequenceComplete(mRepeatingRequestId, lastFrameNumber);
+                mRepeatingRequestId = REQUEST_ID_NONE;
+            }
+        }
+
+        @Override
         public void onDeviceIdle() {
             if (DEBUG) {
                 Log.d(TAG, "Camera now idle");
diff --git a/core/java/android/hardware/camera2/legacy/CameraDeviceState.java b/core/java/android/hardware/camera2/legacy/CameraDeviceState.java
index b0b94e3ac009..e48bce193b52 100644
--- a/core/java/android/hardware/camera2/legacy/CameraDeviceState.java
+++ b/core/java/android/hardware/camera2/legacy/CameraDeviceState.java
@@ -76,6 +76,7 @@ public class CameraDeviceState {
         void onBusy();
         void onCaptureStarted(RequestHolder holder, long timestamp);
         void onCaptureResult(CameraMetadataNative result, RequestHolder holder);
+        void onRepeatingRequestError(long lastFrameNumber);
     }
 
     /**
@@ -201,6 +202,22 @@ public class CameraDeviceState {
     }
 
     /**
+     * Set repeating request error.
+     *
+     * <p>Repeating request has been stopped due to an error such as abandoned output surfaces.</p>
+     *
+     * @param lastFrameNumber Frame number of the last repeating request before it is stopped.
+     */
+    public synchronized void setRepeatingRequestError(final long lastFrameNumber) {
+        mCurrentHandler.post(new Runnable() {
+            @Override
+            public void run() {
+                mCurrentListener.onRepeatingRequestError(lastFrameNumber);
+            }
+        });
+    }
+
+    /**
      * Set the listener for state transition callbacks.
      *
      * @param handler handler on which to call the callbacks.
diff --git a/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java b/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
index f99928a884ee..acbf214942c5 100644
--- a/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
+++ b/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
@@ -205,6 +205,7 @@ public class CameraDeviceUserShim implements ICameraDeviceUser {
         private static final int CAPTURE_STARTED = 2;
         private static final int RESULT_RECEIVED = 3;
         private static final int PREPARED = 4;
+        private static final int REPEATING_REQUEST_ERROR = 5;
 
         private final HandlerThread mHandlerThread;
         private Handler mHandler;
@@ -261,6 +262,15 @@ public class CameraDeviceUserShim implements ICameraDeviceUser {
             getHandler().sendMessage(msg);
         }
 
+
+        @Override
+        public void onRepeatingRequestError(long lastFrameNumber) {
+            Message msg = getHandler().obtainMessage(REPEATING_REQUEST_ERROR,
+                    /*arg1*/ (int) (lastFrameNumber & 0xFFFFFFFFL),
+                    /*arg2*/ (int) ( (lastFrameNumber >> 32) & 0xFFFFFFFFL));
+            getHandler().sendMessage(msg);
+        }
+
         @Override
         public IBinder asBinder() {
             // This is solely intended to be used for in-process binding.
@@ -311,6 +321,12 @@ public class CameraDeviceUserShim implements ICameraDeviceUser {
                             mCallbacks.onPrepared(streamId);
                             break;
                         }
+                        case REPEATING_REQUEST_ERROR: {
+                            long lastFrameNumber = msg.arg2 & 0xFFFFFFFFL;
+                            lastFrameNumber = (lastFrameNumber << 32) | (msg.arg1 & 0xFFFFFFFFL);
+                            mCallbacks.onRepeatingRequestError(lastFrameNumber);
+                            break;
+                        }
                         default:
                             throw new IllegalArgumentException(
                                 "Unknown callback message " + msg.what);
diff --git a/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java b/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
index 6c9586986b97..3e791182734d 100644
--- a/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
+++ b/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
@@ -242,6 +242,25 @@ public class LegacyCameraDevice implements AutoCloseable {
                 }
             });
         }
+
+        @Override
+        public void onRepeatingRequestError(final long lastFrameNumber) {
+            mResultHandler.post(new Runnable() {
+                @Override
+                public void run() {
+                    if (DEBUG) {
+                        Log.d(TAG, "doing onRepeatingRequestError callback.");
+                    }
+                    try {
+                        mDeviceCallbacks.onRepeatingRequestError(lastFrameNumber);
+                    } catch (RemoteException e) {
+                        throw new IllegalStateException(
+                                "Received remote exception during onRepeatingRequestError " +
+                                "callback: ", e);
+                    }
+                }
+            });
+        }
     };
 
     private final RequestThreadManager mRequestThreadManager;
@@ -397,8 +416,15 @@ public class LegacyCameraDevice implements AutoCloseable {
                     "submitRequestList - Empty/null requests are not allowed");
         }
 
-        List<Long> surfaceIds = (mConfiguredSurfaces == null) ? new ArrayList<Long>() :
-                getSurfaceIds(mConfiguredSurfaces);
+        List<Long> surfaceIds;
+
+        try {
+            surfaceIds = (mConfiguredSurfaces == null) ? new ArrayList<Long>() :
+                    getSurfaceIds(mConfiguredSurfaces);
+        } catch (BufferQueueAbandonedException e) {
+            throw new ServiceSpecificException(BAD_VALUE,
+                    "submitRequestList - configured surface is abandoned.");
+        }
 
         // Make sure that there all requests have at least 1 surface; all surfaces are non-null
         for (CaptureRequest request : requestList) {
@@ -674,12 +700,17 @@ public class LegacyCameraDevice implements AutoCloseable {
         LegacyExceptionUtils.throwOnError(nativeSetSurfaceDimens(surface, width, height));
     }
 
-    static long getSurfaceId(Surface surface) {
+    static long getSurfaceId(Surface surface) throws BufferQueueAbandonedException {
         checkNotNull(surface);
-        return nativeGetSurfaceId(surface);
+        try {
+            return nativeGetSurfaceId(surface);
+        } catch (IllegalArgumentException e) {
+            throw new BufferQueueAbandonedException();
+        }
     }
 
-    static List<Long> getSurfaceIds(SparseArray<Surface> surfaces) {
+    static List<Long> getSurfaceIds(SparseArray<Surface> surfaces)
+            throws BufferQueueAbandonedException {
         if (surfaces == null) {
             throw new NullPointerException("Null argument surfaces");
         }
@@ -696,7 +727,8 @@ public class LegacyCameraDevice implements AutoCloseable {
         return surfaceIds;
     }
 
-    static List<Long> getSurfaceIds(Collection<Surface> surfaces) {
+    static List<Long> getSurfaceIds(Collection<Surface> surfaces)
+            throws BufferQueueAbandonedException {
         if (surfaces == null) {
             throw new NullPointerException("Null argument surfaces");
         }
@@ -713,7 +745,13 @@ public class LegacyCameraDevice implements AutoCloseable {
     }
 
     static boolean containsSurfaceId(Surface s, Collection<Long> ids) {
-        long id = getSurfaceId(s);
+        long id = 0;
+        try {
+            id = getSurfaceId(s);
+        } catch (BufferQueueAbandonedException e) {
+            // If surface is abandoned, return false.
+            return false;
+        }
         return ids.contains(id);
     }
 
diff --git a/core/java/android/hardware/camera2/legacy/RequestHolder.java b/core/java/android/hardware/camera2/legacy/RequestHolder.java
index 476c3debc571..98b761b8a04f 100644
--- a/core/java/android/hardware/camera2/legacy/RequestHolder.java
+++ b/core/java/android/hardware/camera2/legacy/RequestHolder.java
@@ -40,6 +40,7 @@ public class RequestHolder {
     private final int mNumJpegTargets;
     private final int mNumPreviewTargets;
     private volatile boolean mFailed = false;
+    private boolean mOutputAbandoned = false;
 
     private final Collection<Long> mJpegSurfaceIds;
 
@@ -266,4 +267,17 @@ public class RequestHolder {
         return mFailed;
     }
 
+    /**
+     * Mark at least one of this request's output surfaces is abandoned.
+     */
+    public void setOutputAbandoned() {
+        mOutputAbandoned = true;
+    }
+
+    /**
+     * Return if any of this request's output surface is abandoned.
+     */
+    public boolean isOutputAbandoned() {
+        return mOutputAbandoned;
+    }
 }
diff --git a/core/java/android/hardware/camera2/legacy/RequestThreadManager.java b/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
index a3fdd56cc65c..da62f5445daa 100644
--- a/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
+++ b/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
@@ -710,6 +710,7 @@ public class RequestThreadManager {
                     break;
                 case MSG_SUBMIT_CAPTURE_REQUEST:
                     Handler handler = RequestThreadManager.this.mRequestThread.getHandler();
+                    boolean anyRequestOutputAbandoned = false;
 
                     // Get the next burst from the request queue.
                     Pair<BurstHolder, Long> nextBurst = mRequestQueue.getNext();
@@ -910,7 +911,22 @@ public class RequestThreadManager {
                         if (!holder.requestFailed()) {
                             mDeviceState.setCaptureResult(holder, result);
                         }
+
+                        if (holder.isOutputAbandoned()) {
+                            anyRequestOutputAbandoned = true;
+                        }
+                    }
+
+                    // Stop the repeating request if any of its output surfaces is abandoned.
+                    if (anyRequestOutputAbandoned && nextBurst.first.isRepeating()) {
+                        long lastFrameNumber = cancelRepeating(nextBurst.first.getRequestId());
+                        if (DEBUG) {
+                            Log.d(TAG, "Stopped repeating request. Last frame number is " +
+                                    lastFrameNumber);
+                        }
+                        mDeviceState.setRepeatingRequestError(lastFrameNumber);
                     }
+
                     if (DEBUG) {
                         long totalTime = SystemClock.elapsedRealtimeNanos() - startTime;
                         Log.d(TAG, "Capture request took " + totalTime + " ns");
diff --git a/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java b/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
index 70bc2fd19648..e0d3905ea942 100644
--- a/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
+++ b/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
@@ -525,9 +525,16 @@ public class SurfaceTextureRenderer {
         checkEglError("makeCurrent");
     }
 
-    private boolean swapBuffers(EGLSurface surface) {
+    private boolean swapBuffers(EGLSurface surface)
+            throws LegacyExceptionUtils.BufferQueueAbandonedException {
         boolean result = EGL14.eglSwapBuffers(mEGLDisplay, surface);
-        checkEglError("swapBuffers");
+        int error = EGL14.eglGetError();
+        if (error == EGL14.EGL_BAD_SURFACE) {
+            throw new LegacyExceptionUtils.BufferQueueAbandonedException();
+        } else if (error != EGL14.EGL_SUCCESS) {
+            throw new IllegalStateException("swapBuffers: EGL error: 0x" +
+                    Integer.toHexString(error));
+        }
         return result;
     }
 
@@ -722,7 +729,14 @@ public class SurfaceTextureRenderer {
             addGlTimestamp(timestamp);
         }
 
-        List<Long> targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces);
+        List<Long> targetSurfaceIds = new ArrayList();
+        try {
+            targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces);
+        } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
+            Log.w(TAG, "Surface abandoned, dropping frame. ", e);
+            request.setOutputAbandoned();
+        }
+
         for (EGLSurfaceHolder holder : mSurfaces) {
             if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
                 try{
@@ -737,6 +751,7 @@ public class SurfaceTextureRenderer {
                     swapBuffers(holder.eglSurface);
                 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
                     Log.w(TAG, "Surface abandoned, dropping frame. ", e);
+                    request.setOutputAbandoned();
                 }
             }
         }
@@ -761,6 +776,7 @@ public class SurfaceTextureRenderer {
                             holder.width, holder.height, format);
                 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
                     Log.w(TAG, "Surface abandoned, dropping frame. ", e);
+                    request.setOutputAbandoned();
                 }
             }
         }
diff --git a/media/tests/MediaFrameworkTest/src/com/android/mediaframeworktest/integration/CameraBinderTest.java b/media/tests/MediaFrameworkTest/src/com/android/mediaframeworktest/integration/CameraBinderTest.java
index 9a0946ea981b..bbc249f0ebe9 100644
--- a/media/tests/MediaFrameworkTest/src/com/android/mediaframeworktest/integration/CameraBinderTest.java
+++ b/media/tests/MediaFrameworkTest/src/com/android/mediaframeworktest/integration/CameraBinderTest.java
@@ -254,6 +254,15 @@ public class CameraBinderTest extends AndroidTestCase {
             // TODO Auto-generated method stub
 
         }
+
+        /*
+         * (non-Javadoc)
+         * @see android.hardware.camera2.ICameraDeviceCallbacks#onRepeatingRequestError()
+         */
+        @Override
+        public void onRepeatingRequestError(long lastFrameNumber) {
+            // TODO Auto-generated method stub
+        }
     }
 
     @SmallTest
diff --git a/media/tests/MediaFrameworkTest/src/com/android/mediaframeworktest/integration/CameraDeviceBinderTest.java b/media/tests/MediaFrameworkTest/src/com/android/mediaframeworktest/integration/CameraDeviceBinderTest.java
index 5c1d8a7e143b..6c879b99dd32 100644
--- a/media/tests/MediaFrameworkTest/src/com/android/mediaframeworktest/integration/CameraDeviceBinderTest.java
+++ b/media/tests/MediaFrameworkTest/src/com/android/mediaframeworktest/integration/CameraDeviceBinderTest.java
@@ -145,6 +145,15 @@ public class CameraDeviceBinderTest extends AndroidTestCase {
             // TODO Auto-generated method stub
 
         }
+
+        /*
+         * (non-Javadoc)
+         * @see android.hardware.camera2.ICameraDeviceCallbacks#onRepeatingRequestError()
+         */
+        @Override
+        public void onRepeatingRequestError(long lastFrameNumber) {
+            // TODO Auto-generated method stub
+        }
     }
 
     class IsMetadataNotEmpty extends ArgumentMatcher<CameraMetadataNative> {