Camera2: add deferred output config support
Allow surfaces to be deferred during session creation. Once the surfaces are ready,
the application can finish the deferred output configuration to be able to submit
requests with these surface targets.
Bug: 28323863
Change-Id: Id6634c3ef2ecc84422a88f63de0a19a0cb496e96
diff --git a/core/java/android/hardware/camera2/CameraCaptureSession.java b/core/java/android/hardware/camera2/CameraCaptureSession.java
index 38279a4..4b21187 100644
--- a/core/java/android/hardware/camera2/CameraCaptureSession.java
+++ b/core/java/android/hardware/camera2/CameraCaptureSession.java
@@ -18,6 +18,7 @@
import android.annotation.NonNull;
import android.annotation.Nullable;
+import android.hardware.camera2.params.OutputConfiguration;
import android.os.Handler;
import android.view.Surface;
@@ -220,6 +221,53 @@
public abstract void tearDown(@NonNull Surface surface) throws CameraAccessException;
/**
+ * <p>
+ * Finish the deferred output configurations where the output Surface was not configured before.
+ * </p>
+ * <p>
+ * For camera use cases where a preview and other output configurations need to be configured,
+ * it can take some time for the preview Surface to be ready (e.g., if the preview Surface is
+ * obtained from {@link android.view.SurfaceView}, the SurfaceView is ready after the UI layout
+ * is done, then it takes some time to get the preview Surface).
+ * </p>
+ * <p>
+ * To speed up camera startup time, the application can configure the
+ * {@link CameraCaptureSession} with the desired preview size, and defer the preview output
+ * configuration until the Surface is ready. After the {@link CameraCaptureSession} is created
+ * successfully with this deferred configuration and other normal configurations, the
+ * application can submit requests that don't include deferred output Surfaces. Once the
+ * deferred Surface is ready, the application can set the Surface to the same deferred output
+ * configuration with the {@link OutputConfiguration#setDeferredSurface} method, and then finish
+ * the deferred output configuration via this method, before it can submit requests with this
+ * output target.
+ * </p>
+ * <p>
+ * The output Surfaces included by this list of deferred {@link OutputConfiguration
+ * OutputConfigurations} can be used as {@link CaptureRequest} targets as soon as this call
+ * returns;
+ * </p>
+ * <p>
+ * This method is not supported by Legacy devices.
+ * </p>
+ *
+ * @param deferredOutputConfigs a list of {@link OutputConfiguration OutputConfigurations} that
+ * have had {@link OutputConfiguration#setDeferredSurface setDeferredSurface} invoked
+ * with a valid output Surface.
+ * @throws CameraAccessException if the camera device is no longer connected or has encountered
+ * a fatal error.
+ * @throws IllegalStateException if this session is no longer active, either because the session
+ * was explicitly closed, a new session has been created or the camera device has
+ * been closed. Or if this output configuration was already finished with the
+ * included surface before.
+ * @throws IllegalArgumentException for invalid output configurations, including ones where the
+ * source of the Surface is no longer valid or the Surface is from a unsupported
+ * source.
+ * @hide
+ */
+ public abstract void finishDeferredConfiguration(
+ List<OutputConfiguration> deferredOutputConfigs) throws CameraAccessException;
+
+ /**
* <p>Submit a request for an image to be captured by the camera device.</p>
*
* <p>The request defines all the parameters for capturing the single image,
diff --git a/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java
index 6736d34..c23bd5b 100644
--- a/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java
+++ b/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java
@@ -24,6 +24,7 @@
import android.hardware.camera2.dispatch.DuckTypingDispatcher;
import android.hardware.camera2.dispatch.HandlerDispatcher;
import android.hardware.camera2.dispatch.InvokeDispatcher;
+import android.hardware.camera2.params.OutputConfiguration;
import android.hardware.camera2.utils.TaskDrainer;
import android.hardware.camera2.utils.TaskSingleDrainer;
import android.os.Handler;
@@ -156,6 +157,12 @@
}
@Override
+ public void finishDeferredConfiguration(
+ List<OutputConfiguration> deferredOutputConfigs) throws CameraAccessException {
+ mDeviceImpl.finishDeferredConfig(deferredOutputConfigs);
+ }
+
+ @Override
public synchronized int capture(CaptureRequest request, CaptureCallback callback,
Handler handler) throws CameraAccessException {
if (request == null) {
diff --git a/core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java
index 8cd1da5..1c8e124 100644
--- a/core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java
+++ b/core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java
@@ -21,6 +21,7 @@
import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession;
import android.hardware.camera2.CameraDevice;
import android.hardware.camera2.CaptureRequest;
+import android.hardware.camera2.params.OutputConfiguration;
import android.hardware.camera2.params.StreamConfigurationMap;
import android.hardware.camera2.utils.SurfaceUtils;
import android.os.Handler;
@@ -256,6 +257,12 @@
return mSessionImpl.isAborting();
}
+ @Override
+ public void finishDeferredConfiguration(List<OutputConfiguration> deferredOutputConfigs)
+ throws CameraAccessException {
+ mSessionImpl.finishDeferredConfiguration(deferredOutputConfigs);
+ }
+
private class WrapperCallback extends StateCallback {
private final StateCallback mCallback;
@@ -263,26 +270,32 @@
mCallback = callback;
}
+ @Override
public void onConfigured(CameraCaptureSession session) {
mCallback.onConfigured(CameraConstrainedHighSpeedCaptureSessionImpl.this);
}
+ @Override
public void onConfigureFailed(CameraCaptureSession session) {
mCallback.onConfigureFailed(CameraConstrainedHighSpeedCaptureSessionImpl.this);
}
+ @Override
public void onReady(CameraCaptureSession session) {
mCallback.onReady(CameraConstrainedHighSpeedCaptureSessionImpl.this);
}
+ @Override
public void onActive(CameraCaptureSession session) {
mCallback.onActive(CameraConstrainedHighSpeedCaptureSessionImpl.this);
}
+ @Override
public void onClosed(CameraCaptureSession session) {
mCallback.onClosed(CameraConstrainedHighSpeedCaptureSessionImpl.this);
}
+ @Override
public void onSurfacePrepared(CameraCaptureSession session, Surface surface) {
mCallback.onSurfacePrepared(CameraConstrainedHighSpeedCaptureSessionImpl.this,
surface);
diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
index 5743b4d..e184d6d 100644
--- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
+++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
@@ -407,7 +407,10 @@
int streamId = mConfiguredOutputs.keyAt(i);
OutputConfiguration outConfig = mConfiguredOutputs.valueAt(i);
- if (!outputs.contains(outConfig)) {
+ if (!outputs.contains(outConfig) || outConfig.isDeferredConfiguration()) {
+ // Always delete the deferred output configuration when the session
+ // is created, as the deferred output configuration doesn't have unique surface
+ // related identifies.
deleteList.add(streamId);
} else {
addSet.remove(outConfig); // Don't create a stream previously created
@@ -744,6 +747,37 @@
}
}
+ public void finishDeferredConfig(List<OutputConfiguration> deferredConfigs)
+ throws CameraAccessException {
+ if (deferredConfigs == null || deferredConfigs.size() == 0) {
+ throw new IllegalArgumentException("deferred config is null or empty");
+ }
+
+ synchronized(mInterfaceLock) {
+ for (OutputConfiguration config : deferredConfigs) {
+ int streamId = -1;
+ for (int i = 0; i < mConfiguredOutputs.size(); i++) {
+ // Have to use equal here, as createCaptureSessionByOutputConfigurations() and
+ // createReprocessableCaptureSessionByConfigurations() do a copy of the configs.
+ if (config.equals(mConfiguredOutputs.valueAt(i))) {
+ streamId = mConfiguredOutputs.keyAt(i);
+ break;
+ }
+ }
+ if (streamId == -1) {
+ throw new IllegalArgumentException("Deferred config is not part of this "
+ + "session");
+ }
+
+ if (config.getSurface() == null) {
+ throw new IllegalArgumentException("The deferred config for stream " + streamId
+ + " must have a non-null surface");
+ }
+ mRemoteDevice.setDeferredConfiguration(streamId, config);
+ }
+ }
+ }
+
public int capture(CaptureRequest request, CaptureCallback callback, Handler handler)
throws CameraAccessException {
if (DEBUG) {
@@ -2037,6 +2071,7 @@
*
* <p> Handle binder death for ICameraDeviceUser. Trigger onError.</p>
*/
+ @Override
public void binderDied() {
Log.w(TAG, "CameraDevice " + mCameraId + " died unexpectedly");
diff --git a/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java b/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java
index ef5f6d7..d77f60b 100644
--- a/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java
+++ b/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java
@@ -215,5 +215,14 @@
}
}
+ public void setDeferredConfiguration(int streamId, OutputConfiguration deferredConfig)
+ throws CameraAccessException {
+ try {
+ mRemoteDevice.setDeferredConfiguration(streamId, deferredConfig);
+ } catch (Throwable t) {
+ CameraManager.throwAsPublicException(t);
+ throw new UnsupportedOperationException("Unexpected exception", t);
+ }
+ }
}
diff --git a/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java b/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
index acbf214..b9e75ee 100644
--- a/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
+++ b/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
@@ -567,6 +567,13 @@
}
@Override
+ public void setDeferredConfiguration(int steamId, OutputConfiguration config) {
+ String err = "Set deferred configuration is not supported on legacy devices";
+ Log.e(TAG, err);
+ throw new ServiceSpecificException(ICameraService.ERROR_INVALID_OPERATION, err);
+ }
+
+ @Override
public int createInputStream(int width, int height, int format) {
String err = "Creating input stream is not supported on legacy devices";
Log.e(TAG, err);
diff --git a/core/java/android/hardware/camera2/params/OutputConfiguration.java b/core/java/android/hardware/camera2/params/OutputConfiguration.java
index 61b534b..69c00e9 100644
--- a/core/java/android/hardware/camera2/params/OutputConfiguration.java
+++ b/core/java/android/hardware/camera2/params/OutputConfiguration.java
@@ -20,6 +20,8 @@
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
+import android.graphics.ImageFormat;
+import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CameraDevice;
import android.hardware.camera2.utils.HashCodeHelpers;
import android.hardware.camera2.utils.SurfaceUtils;
@@ -95,6 +97,21 @@
}
/**
+ * Unknown surface source type.
+ */
+ private final int SURFACE_TYPE_UNKNOWN = -1;
+
+ /**
+ * The surface is obtained from {@link android.view.SurfaceView}.
+ */
+ private final int SURFACE_TYPE_SURFACE_VIEW = 0;
+
+ /**
+ * The surface is obtained from {@link android.graphics.SurfaceTexture}.
+ */
+ private final int SURFACE_TYPE_SURFACE_TEXTURE = 1;
+
+ /**
* Create a new {@link OutputConfiguration} instance with a {@link Surface},
* with a surface group ID.
*
@@ -179,12 +196,110 @@
checkNotNull(surface, "Surface must not be null");
checkArgumentInRange(rotation, ROTATION_0, ROTATION_270, "Rotation constant");
mSurfaceGroupId = surfaceGroupId;
+ mSurfaceType = SURFACE_TYPE_UNKNOWN;
mSurface = surface;
mRotation = rotation;
mConfiguredSize = SurfaceUtils.getSurfaceSize(surface);
mConfiguredFormat = SurfaceUtils.getSurfaceFormat(surface);
mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(surface);
mConfiguredGenerationId = surface.getGenerationId();
+ mIsDeferredConfig = false;
+ }
+
+ /**
+ * Create a new {@link OutputConfiguration} instance, with desired Surface size and Surface
+ * source class.
+ * <p>
+ * This constructor takes an argument for desired Surface size and the Surface source class
+ * without providing the actual output Surface. This is used to setup a output configuration
+ * with a deferred Surface. The application can use this output configuration to create a
+ * session.
+ * </p>
+ * <p>
+ * However, the actual output Surface must be set via {@link #setDeferredSurface} and finish the
+ * deferred Surface configuration via {@link CameraCaptureSession#finishDeferredConfiguration}
+ * before submitting a request with this Surface target. The deferred Surface can only be
+ * obtained from either from {@link android.view.SurfaceView} by calling
+ * {@link android.view.SurfaceHolder#getSurface}, or from
+ * {@link android.graphics.SurfaceTexture} via
+ * {@link android.view.Surface#Surface(android.graphics.SurfaceTexture)}).
+ * </p>
+ *
+ * @param surfaceSize Size for the deferred surface.
+ * @param klass a non-{@code null} {@link Class} object reference that indicates the source of
+ * this surface. Only {@link android.view.SurfaceHolder SurfaceHolder.class} and
+ * {@link android.graphics.SurfaceTexture SurfaceTexture.class} are supported.
+ * @hide
+ */
+ public <T> OutputConfiguration(@NonNull Size surfaceSize, @NonNull Class<T> klass) {
+ checkNotNull(klass, "surfaceSize must not be null");
+ checkNotNull(klass, "klass must not be null");
+ if (klass == android.view.SurfaceHolder.class) {
+ mSurfaceType = SURFACE_TYPE_SURFACE_VIEW;
+ } else if (klass == android.graphics.SurfaceTexture.class) {
+ mSurfaceType = SURFACE_TYPE_SURFACE_TEXTURE;
+ } else {
+ mSurfaceType = SURFACE_TYPE_UNKNOWN;
+ throw new IllegalArgumentException("Unknow surface source class type");
+ }
+
+ mSurfaceGroupId = SURFACE_GROUP_ID_NONE;
+ mSurface = null;
+ mRotation = ROTATION_0;
+ mConfiguredSize = surfaceSize;
+ mConfiguredFormat = StreamConfigurationMap.imageFormatToInternal(ImageFormat.PRIVATE);
+ mConfiguredDataspace = StreamConfigurationMap.imageFormatToDataspace(ImageFormat.PRIVATE);
+ mConfiguredGenerationId = 0;
+ mIsDeferredConfig = true;
+ }
+
+ /**
+ * Check if this configuration has deferred configuration.
+ *
+ * <p>This will return true if the output configuration was constructed with surface deferred.
+ * It will return true even after the deferred surface is set later.</p>
+ *
+ * @return true if this configuration has deferred surface.
+ * @hide
+ */
+ public boolean isDeferredConfiguration() {
+ return mIsDeferredConfig;
+ }
+
+ /**
+ * Set the deferred surface to this OutputConfiguration.
+ *
+ * <p>
+ * The deferred surface must be obtained from either from {@link android.view.SurfaceView} by
+ * calling {@link android.view.SurfaceHolder#getSurface}, or from
+ * {@link android.graphics.SurfaceTexture} via
+ * {@link android.view.Surface#Surface(android.graphics.SurfaceTexture)}). After the deferred
+ * surface is set, the application must finish the deferred surface configuration via
+ * {@link CameraCaptureSession#finishDeferredConfiguration} before submitting a request with
+ * this surface target.
+ * </p>
+ *
+ * @param surface The deferred surface to be set.
+ * @throws IllegalArgumentException if the Surface is invalid.
+ * @throws IllegalStateException if a Surface was already set to this deferred
+ * OutputConfiguration.
+ * @hide
+ */
+ public void setDeferredSurface(@NonNull Surface surface) {
+ checkNotNull(surface, "Surface must not be null");
+ if (mSurface != null) {
+ throw new IllegalStateException("Deferred surface is already set!");
+ }
+
+ // This will throw IAE is the surface was abandoned.
+ Size surfaceSize = SurfaceUtils.getSurfaceSize(surface);
+ if (!surfaceSize.equals(mConfiguredSize)) {
+ Log.w(TAG, "Deferred surface size " + surfaceSize +
+ " is different with pre-configured size " + mConfiguredSize +
+ ", the pre-configured size will be used.");
+ }
+
+ mSurface = surface;
}
/**
@@ -203,10 +318,12 @@
this.mSurface = other.mSurface;
this.mRotation = other.mRotation;
this.mSurfaceGroupId = other.mSurfaceGroupId;
+ this.mSurfaceType = other.mSurfaceType;
this.mConfiguredDataspace = other.mConfiguredDataspace;
this.mConfiguredFormat = other.mConfiguredFormat;
this.mConfiguredSize = other.mConfiguredSize;
this.mConfiguredGenerationId = other.mConfiguredGenerationId;
+ this.mIsDeferredConfig = other.mIsDeferredConfig;
}
/**
@@ -215,16 +332,30 @@
private OutputConfiguration(@NonNull Parcel source) {
int rotation = source.readInt();
int surfaceSetId = source.readInt();
+ int surfaceType = source.readInt();
+ int width = source.readInt();
+ int height = source.readInt();
Surface surface = Surface.CREATOR.createFromParcel(source);
- checkNotNull(surface, "Surface must not be null");
checkArgumentInRange(rotation, ROTATION_0, ROTATION_270, "Rotation constant");
mSurfaceGroupId = surfaceSetId;
mSurface = surface;
mRotation = rotation;
- mConfiguredSize = SurfaceUtils.getSurfaceSize(mSurface);
- mConfiguredFormat = SurfaceUtils.getSurfaceFormat(mSurface);
- mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(mSurface);
- mConfiguredGenerationId = mSurface.getGenerationId();
+ if (surface != null) {
+ mSurfaceType = SURFACE_TYPE_UNKNOWN;
+ mConfiguredSize = SurfaceUtils.getSurfaceSize(mSurface);
+ mConfiguredFormat = SurfaceUtils.getSurfaceFormat(mSurface);
+ mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(mSurface);
+ mConfiguredGenerationId = mSurface.getGenerationId();
+ mIsDeferredConfig = true;
+ } else {
+ mSurfaceType = surfaceType;
+ mConfiguredSize = new Size(width, height);
+ mConfiguredFormat = StreamConfigurationMap.imageFormatToInternal(ImageFormat.PRIVATE);
+ mConfiguredGenerationId = 0;
+ mConfiguredDataspace =
+ StreamConfigurationMap.imageFormatToDataspace(ImageFormat.PRIVATE);
+ mIsDeferredConfig = false;
+ }
}
/**
@@ -291,7 +422,12 @@
}
dest.writeInt(mRotation);
dest.writeInt(mSurfaceGroupId);
- mSurface.writeToParcel(dest, flags);
+ dest.writeInt(mSurfaceType);
+ dest.writeInt(mConfiguredSize.getWidth());
+ dest.writeInt(mConfiguredSize.getHeight());
+ if (mSurface != null) {
+ mSurface.writeToParcel(dest, flags);
+ }
}
/**
@@ -311,13 +447,20 @@
return true;
} else if (obj instanceof OutputConfiguration) {
final OutputConfiguration other = (OutputConfiguration) obj;
+ boolean iSSurfaceEqual = mSurface == other.mSurface &&
+ mConfiguredGenerationId == other.mConfiguredGenerationId ;
+ if (mIsDeferredConfig) {
+ Log.i(TAG, "deferred config has the same surface");
+ iSSurfaceEqual = true;
+ }
return mRotation == other.mRotation &&
- mSurface == other.mSurface &&
- mConfiguredGenerationId == other.mConfiguredGenerationId &&
+ iSSurfaceEqual&&
mConfiguredSize.equals(other.mConfiguredSize) &&
mConfiguredFormat == other.mConfiguredFormat &&
mConfiguredDataspace == other.mConfiguredDataspace &&
- mSurfaceGroupId == other.mSurfaceGroupId;
+ mSurfaceGroupId == other.mSurfaceGroupId &&
+ mSurfaceType == other.mSurfaceType &&
+ mIsDeferredConfig == other.mIsDeferredConfig;
}
return false;
}
@@ -327,15 +470,26 @@
*/
@Override
public int hashCode() {
+ // Need ensure that the hashcode remains unchanged after set a deferred surface. Otherwise
+ // The deferred output configuration will be lost in the camera streammap after the deferred
+ // surface is set.
+ if (mIsDeferredConfig) {
+ return HashCodeHelpers.hashCode(
+ mRotation, mConfiguredSize.hashCode(), mConfiguredFormat, mConfiguredDataspace,
+ mSurfaceGroupId, mSurfaceType);
+ }
+
return HashCodeHelpers.hashCode(
mRotation, mSurface.hashCode(), mConfiguredGenerationId,
mConfiguredSize.hashCode(), mConfiguredFormat, mConfiguredDataspace, mSurfaceGroupId);
}
private static final String TAG = "OutputConfiguration";
- private final Surface mSurface;
+ private Surface mSurface;
private final int mRotation;
- private int mSurfaceGroupId;
+ private final int mSurfaceGroupId;
+ // Surface source type, this is only used by the deferred surface configuration objects.
+ private final int mSurfaceType;
// The size, format, and dataspace of the surface when OutputConfiguration is created.
private final Size mConfiguredSize;
@@ -343,4 +497,6 @@
private final int mConfiguredDataspace;
// Surface generation ID to distinguish changes to Surface native internals
private final int mConfiguredGenerationId;
+ // Flag indicating if this config has deferred surface.
+ private final boolean mIsDeferredConfig;
}