diff options
| -rw-r--r-- | api/current.txt | 28 | ||||
| -rw-r--r-- | media/java/android/media/Image.java | 44 | ||||
| -rw-r--r-- | media/java/android/media/ImageReader.java | 309 | ||||
| -rw-r--r-- | media/jni/Android.mk | 1 | ||||
| -rw-r--r-- | media/jni/android_media_ImageReader.cpp | 747 | ||||
| -rw-r--r-- | media/jni/android_media_MediaPlayer.cpp | 6 |
6 files changed, 1079 insertions, 56 deletions
diff --git a/api/current.txt b/api/current.txt index 7604f957d36c..facaabe3d221 100644 --- a/api/current.txt +++ b/api/current.txt @@ -11817,26 +11817,24 @@ package android.media { field public static final int EULER_Z = 2; // 0x2 } - public abstract class Image implements java.lang.AutoCloseable { - ctor public Image(); + public abstract interface Image implements java.lang.AutoCloseable { method public abstract void close(); - method protected final void finalize(); - method public int getFormat(); - method public int getHeight(); - method public android.media.Image.Plane[] getPlanes(); - method public long getTimestamp(); - method public int getWidth(); + method public abstract int getFormat(); + method public abstract int getHeight(); + method public abstract android.media.Image.Plane[] getPlanes(); + method public abstract long getTimestamp(); + method public abstract int getWidth(); } - public static final class Image.Plane { - ctor public Image.Plane(); - method public java.nio.ByteBuffer getBuffer(); - method public int getPixelStride(); - method public int getRowStride(); + public static abstract interface Image.Plane { + method public abstract java.nio.ByteBuffer getBuffer(); + method public abstract int getPixelStride(); + method public abstract int getRowStride(); } - public final class ImageReader { + public final class ImageReader implements java.lang.AutoCloseable { ctor public ImageReader(int, int, int, int); + method public void close(); method public int getHeight(); method public int getImageFormat(); method public int getMaxImages(); @@ -11844,7 +11842,7 @@ package android.media { method public android.view.Surface getSurface(); method public int getWidth(); method public void releaseImage(android.media.Image); - method public void setOnImageAvailableListener(android.media.ImageReader.OnImageAvailableListener); + method public void setImageAvailableListener(android.media.ImageReader.OnImageAvailableListener, android.os.Handler); } public static abstract interface ImageReader.OnImageAvailableListener { diff --git a/media/java/android/media/Image.java b/media/java/android/media/Image.java index eb943463d8a5..f55756c62a32 100644 --- a/media/java/android/media/Image.java +++ b/media/java/android/media/Image.java @@ -44,7 +44,7 @@ import java.lang.AutoCloseable; * * @see ImageReader */ -public abstract class Image implements AutoCloseable { +public interface Image extends AutoCloseable { /** * Get the format for this image. This format determines the number of * ByteBuffers needed to represent the image, and the general layout of the @@ -87,25 +87,19 @@ public abstract class Image implements AutoCloseable { * * @see android.graphics.ImageFormat */ - public int getFormat() { - return ImageFormat.UNKNOWN; - } + public int getFormat(); /** * The width of the image in pixels. For formats where some color channels * are subsampled, this is the width of the largest-resolution plane. */ - public int getWidth() { - return 0; - } + public int getWidth(); /** * The height of the image in pixels. For formats where some color channels * are subsampled, this is the height of the largest-resolution plane. */ - public int getHeight() { - return 0; - } + public int getHeight(); /** * Get the timestamp associated with this frame. The timestamp is measured @@ -113,17 +107,13 @@ public abstract class Image implements AutoCloseable { * and whether the timestamp can be compared against other sources of time * or images depend on the source of this image. */ - public long getTimestamp() { - return 0; - } + public long getTimestamp(); /** * Get the array of pixel planes for this Image. The number of planes is * determined by the format of the Image. */ - public Plane[] getPlanes() { - return null; - } + public Plane[] getPlanes(); /** * Free up this frame for reuse. After calling this method, calling any @@ -131,11 +121,7 @@ public abstract class Image implements AutoCloseable { * attempting to read from ByteBuffers returned by an earlier * {@code Plane#getBuffer} call will have undefined behavior. */ - public abstract void close(); - - protected final void finalize() { - close(); - } + public void close(); /** * <p>A single color plane of image data.</p> @@ -148,17 +134,14 @@ public abstract class Image implements AutoCloseable { * * @see #getFormat */ - public static final class Plane { + public interface Plane { /** * <p>The row stride for this color plane, in bytes. * * <p>This is the distance between the start of two consecutive rows of * pixels in the image.</p> */ - public int getRowStride() { - return 0; - } - + public int getRowStride(); /** * <p>The distance between adjacent pixel samples, in bytes.</p> * @@ -166,19 +149,14 @@ public abstract class Image implements AutoCloseable { * of pixels. It may be larger than the size of a single pixel to * account for interleaved image data or padded formats.</p> */ - public int getPixelStride() { - return 0; - } - + public int getPixelStride(); /** * <p>Get a set of direct {@link java.nio.ByteBuffer byte buffers} * containing the frame data.</p> * * @return the byte buffer containing the image data for this plane. */ - public ByteBuffer getBuffer() { - return null; - } + public ByteBuffer getBuffer(); } } diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java index 9384c14c6006..8f09b54bdad0 100644 --- a/media/java/android/media/ImageReader.java +++ b/media/java/android/media/ImageReader.java @@ -16,8 +16,15 @@ package android.media; +import android.graphics.ImageFormat; +import android.os.Handler; +import android.os.Looper; +import android.os.Message; import android.view.Surface; -import java.lang.AutoCloseable; + +import java.lang.ref.WeakReference; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; /** * <p>The ImageReader class allows direct application access to image data @@ -39,7 +46,7 @@ import java.lang.AutoCloseable; * ImageReader does not obtain and release Images at a rate equal to the * production rate.</p> */ -public final class ImageReader { +public final class ImageReader implements AutoCloseable { /** * <p>Create a new reader for images of the desired size and format.</p> @@ -62,7 +69,7 @@ public final class ImageReader { * access simultaneously. This should be as small as possible to limit * memory use. Once maxImages Images are obtained by the user, one of them * has to be released before a new Image will become available for access - * through getImage(). Must be greater than 0. + * through getNextImage(). Must be greater than 0. * * @see Image */ @@ -80,6 +87,12 @@ public final class ImageReader { throw new IllegalArgumentException( "Maximum outstanding image count must be at least 1"); } + + mNumPlanes = getNumPlanesFromFormat(); + + nativeInit(new WeakReference<ImageReader>(this), width, height, format, maxImages); + + mSurface = nativeGetSurface(); } public int getWidth() { @@ -111,7 +124,7 @@ public final class ImageReader { * @return A Surface to use for a drawing target for various APIs. */ public Surface getSurface() { - return null; + return mSurface; } /** @@ -122,6 +135,13 @@ public final class ImageReader { * available. */ public Image getNextImage() { + SurfaceImage si = new SurfaceImage(); + if (nativeImageSetup(si)) { + // create SurfacePlane objects + si.createSurfacePlanes(); + si.setImageValid(true); + return si; + } return null; } @@ -138,34 +158,307 @@ public final class ImageReader { throw new IllegalArgumentException( "This image was not produced by this ImageReader"); } + + si.clearSurfacePlanes(); + nativeReleaseImage(i); + si.setImageValid(false); } - public void setOnImageAvailableListener(OnImageAvailableListener l) { - mImageListener = l; + /** + * Register a listener to be invoked when a new image becomes available + * from the ImageReader. + * @param listener the listener that will be run + * @param handler The handler on which the listener should be invoked, or null + * if the listener should be invoked on the calling thread's looper. + */ + public void setImageAvailableListener(OnImageAvailableListener listener, Handler handler) { + mImageListener = listener; + + Looper looper; + mHandler = handler; + if (mHandler == null) { + if ((looper = Looper.myLooper()) != null) { + mHandler = new Handler(); + } else { + throw new IllegalArgumentException( + "Looper doesn't exist in the calling thread"); + } + } } + /** + * Callback interface for being notified that a new image is available. + * The onImageAvailable is called per image basis, that is, callback fires for every new frame + * available from ImageReader. + */ public interface OnImageAvailableListener { + /** + * Callback that is called when a new image is available from ImageReader. + * @param reader the ImageReader the callback is associated with. + * @see ImageReader + * @see Image + */ void onImageAvailable(ImageReader reader); } + /** + * Free up all the resources associated with this ImageReader. After + * Calling this method, this ImageReader can not be used. calling + * any methods on this ImageReader and Images previously provided by {@link #getNextImage} + * will result in an IllegalStateException, and attempting to read from + * ByteBuffers returned by an earlier {@code Plane#getBuffer} call will + * have undefined behavior. + */ + @Override + public void close() { + nativeClose(); + } + + @Override + protected void finalize() throws Throwable { + try { + close(); + } finally { + super.finalize(); + } + } + + private int getNumPlanesFromFormat() { + switch (mFormat) { + case ImageFormat.YV12: + case ImageFormat.YUV_420_888: + case ImageFormat.NV21: + return 3; + case ImageFormat.NV16: + return 2; + case ImageFormat.RGB_565: + case ImageFormat.JPEG: + case ImageFormat.YUY2: + case ImageFormat.Y8: + case ImageFormat.Y16: + case ImageFormat.RAW_SENSOR: + return 1; + default: + throw new UnsupportedOperationException( + String.format("Invalid format specified %d", mFormat)); + } + } + + /** + * Called from Native code when an Event happens. + */ + private static void postEventFromNative(Object selfRef) { + WeakReference weakSelf = (WeakReference)selfRef; + final ImageReader ir = (ImageReader)weakSelf.get(); + if (ir == null) { + return; + } + + if (ir.mHandler != null) { + ir.mHandler.post(new Runnable() { + @Override + public void run() { + ir.mImageListener.onImageAvailable(ir); + } + }); + } + } + private final int mWidth; private final int mHeight; private final int mFormat; private final int mMaxImages; + private final int mNumPlanes; + private final Surface mSurface; + private Handler mHandler; private OnImageAvailableListener mImageListener; - private class SurfaceImage extends android.media.Image { + /** + * This field is used by native code, do not access or modify. + */ + private long mNativeContext; + + private class SurfaceImage implements android.media.Image { public SurfaceImage() { + mIsImageValid = false; } @Override public void close() { - ImageReader.this.releaseImage(this); + if (mIsImageValid) { + ImageReader.this.releaseImage(this); + } } public ImageReader getReader() { return ImageReader.this; } + + @Override + public int getFormat() { + if (mIsImageValid) { + return ImageReader.this.mFormat; + } else { + throw new IllegalStateException("Image is already released"); + } + } + + @Override + public int getWidth() { + if (mIsImageValid) { + return ImageReader.this.mWidth; + } else { + throw new IllegalStateException("Image is already released"); + } + } + + @Override + public int getHeight() { + if (mIsImageValid) { + return ImageReader.this.mHeight; + } else { + throw new IllegalStateException("Image is already released"); + } + } + + @Override + public long getTimestamp() { + if (mIsImageValid) { + return mTimestamp; + } else { + throw new IllegalStateException("Image is already released"); + } + } + + @Override + public Plane[] getPlanes() { + if (mIsImageValid) { + // Shallow copy is fine. + return mPlanes.clone(); + } else { + throw new IllegalStateException("Image is already released"); + } + } + + @Override + protected final void finalize() throws Throwable { + try { + close(); + } finally { + super.finalize(); + } + } + + private void setImageValid(boolean isValid) { + mIsImageValid = isValid; + } + + private boolean isImageValid() { + return mIsImageValid; + } + + private void clearSurfacePlanes() { + if (mIsImageValid) { + for (int i = 0; i < mPlanes.length; i++) { + if (mPlanes[i] != null) { + mPlanes[i].clearBuffer(); + mPlanes[i] = null; + } + } + } + } + + private void createSurfacePlanes() { + mPlanes = new SurfacePlane[ImageReader.this.mNumPlanes]; + for (int i = 0; i < ImageReader.this.mNumPlanes; i++) { + mPlanes[i] = nativeCreatePlane(i); + } + } + private class SurfacePlane implements android.media.Image.Plane { + // SurfacePlane instance is created by native code when a new SurfaceImage is created + private SurfacePlane(int index, int rowStride, int pixelStride) { + mIndex = index; + mRowStride = rowStride; + mPixelStride = pixelStride; + } + + @Override + public ByteBuffer getBuffer() { + if (SurfaceImage.this.isImageValid() == false) { + throw new IllegalStateException("Image is already released"); + } + if (mBuffer != null) { + return mBuffer; + } else { + mBuffer = SurfaceImage.this.nativeImageGetBuffer(mIndex); + // Set the byteBuffer order according to host endianness (native order), + // otherwise, the byteBuffer order defaults to ByteOrder.BIG_ENDIAN. + return mBuffer.order(ByteOrder.nativeOrder()); + } + } + + @Override + public int getPixelStride() { + if (SurfaceImage.this.isImageValid()) { + return mPixelStride; + } else { + throw new IllegalStateException("Image is already released"); + } + } + + @Override + public int getRowStride() { + if (SurfaceImage.this.isImageValid()) { + return mRowStride; + } else { + throw new IllegalStateException("Image is already released"); + } + } + + private void clearBuffer() { + mBuffer = null; + } + + final private int mIndex; + final private int mPixelStride; + final private int mRowStride; + + private ByteBuffer mBuffer; + } + + /** + * This field is used to keep track of native object and used by native code only. + * Don't modify. + */ + private long mLockedBuffer; + + /** + * This field is set by native code during nativeImageSetup(). + */ + private long mTimestamp; + + private SurfacePlane[] mPlanes; + private boolean mIsImageValid; + + private synchronized native ByteBuffer nativeImageGetBuffer(int idx); + private synchronized native SurfacePlane nativeCreatePlane(int idx); + } + + private synchronized native void nativeInit(Object weakSelf, int w, int h, + int fmt, int maxImgs); + private synchronized native void nativeClose(); + private synchronized native void nativeReleaseImage(Image i); + private synchronized native Surface nativeGetSurface(); + private synchronized native boolean nativeImageSetup(Image i); + + /* + * We use a class initializer to allow the native code to cache some + * field offsets. + */ + private static native void nativeClassInit(); + static { + System.loadLibrary("media_jni"); + nativeClassInit(); } } diff --git a/media/jni/Android.mk b/media/jni/Android.mk index 416a2a17d5bb..01b3174a4b5a 100644 --- a/media/jni/Android.mk +++ b/media/jni/Android.mk @@ -2,6 +2,7 @@ LOCAL_PATH:= $(call my-dir) include $(CLEAR_VARS) LOCAL_SRC_FILES:= \ + android_media_ImageReader.cpp \ android_media_MediaCrypto.cpp \ android_media_MediaCodec.cpp \ android_media_MediaCodecList.cpp \ diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp new file mode 100644 index 000000000000..bdb07a6543f9 --- /dev/null +++ b/media/jni/android_media_ImageReader.cpp @@ -0,0 +1,747 @@ +/* + * Copyright 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "ImageReader_JNI" +#include <utils/Log.h> +#include <utils/misc.h> +#include <utils/List.h> + +#include <cstdio> + +#include <gui/CpuConsumer.h> +#include <gui/Surface.h> + +#include <android_runtime/AndroidRuntime.h> +#include <android_runtime/android_view_Surface.h> + +#include <jni.h> +#include <JNIHelp.h> + +#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) ) + +#define ANDROID_MEDIA_IMAGEREADER_JNI_ID "mCpuConsumer" +#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext" +#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mLockedBuffer" +#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp" + +// ---------------------------------------------------------------------------- + +using namespace android; + +enum { + IMAGE_READER_MAX_NUM_PLANES = 3, +}; + +struct fields_t { + // For ImageReader class + jfieldID imageReaderContext; + jmethodID postEvent; + // For SurfaceImage class + jfieldID buffer; + jfieldID timeStamp; +}; + +struct classInfo_t { + jclass clazz; + jmethodID ctor; +}; + +static fields_t fields; +static classInfo_t surfPlaneClassInfo; + +// ---------------------------------------------------------------------------- + +class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener +{ +public: + JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages); + + virtual ~JNIImageReaderContext(); + + virtual void onFrameAvailable(); + + CpuConsumer::LockedBuffer* getLockedBuffer(); + + void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer); + + CpuConsumer* getCpuConsumer() { return mConsumer.get(); } + + void setCpuConsumer(sp<CpuConsumer> consumer) { mConsumer = consumer; } + + void setBufferFormat(int format) { mFormat = format; } + int getBufferFormat() { return mFormat; } + + void setBufferWidth(int width) { mWidth = width; } + int getBufferWidth() { return mWidth; } + + void setBufferHeight(int height) { mHeight = height; } + int getBufferHeight() { return mHeight; } + +private: + static JNIEnv* getJNIEnv(bool* needsDetach); + static void detachJNI(); + + List<CpuConsumer::LockedBuffer*> mBuffers; + sp<CpuConsumer> mConsumer; + jobject mWeakThiz; + jclass mClazz; + int mFormat; + int mWidth; + int mHeight; +}; + +JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env, + jobject weakThiz, jclass clazz, int maxImages) : + mWeakThiz(env->NewGlobalRef(weakThiz)), + mClazz((jclass)env->NewGlobalRef(clazz)) { + for (int i = 0; i < maxImages; i++) { + CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer; + mBuffers.push_back(buffer); + } +} + +JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) { + LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!"); + *needsDetach = false; + JNIEnv* env = AndroidRuntime::getJNIEnv(); + if (env == NULL) { + JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL}; + JavaVM* vm = AndroidRuntime::getJavaVM(); + int result = vm->AttachCurrentThread(&env, (void*) &args); + if (result != JNI_OK) { + ALOGE("thread attach failed: %#x", result); + return NULL; + } + *needsDetach = true; + } + return env; +} + +void JNIImageReaderContext::detachJNI() { + JavaVM* vm = AndroidRuntime::getJavaVM(); + int result = vm->DetachCurrentThread(); + if (result != JNI_OK) { + ALOGE("thread detach failed: %#x", result); + } +} + +CpuConsumer::LockedBuffer* JNIImageReaderContext::getLockedBuffer() { + if (mBuffers.empty()) { + return NULL; + } + // Return a LockedBuffer pointer and remove it from the list + List<CpuConsumer::LockedBuffer*>::iterator it = mBuffers.begin(); + CpuConsumer::LockedBuffer* buffer = *it; + mBuffers.erase(it); + return buffer; +} + +void JNIImageReaderContext::returnLockedBuffer(CpuConsumer::LockedBuffer * buffer) { + mBuffers.push_back(buffer); +} + +JNIImageReaderContext::~JNIImageReaderContext() { + bool needsDetach = false; + JNIEnv* env = getJNIEnv(&needsDetach); + if (env != NULL) { + env->DeleteGlobalRef(mWeakThiz); + env->DeleteGlobalRef(mClazz); + } else { + ALOGW("leaking JNI object references"); + } + if (needsDetach) { + detachJNI(); + } + + // Delete LockedBuffers + for (List<CpuConsumer::LockedBuffer *>::iterator it = mBuffers.begin(); + it != mBuffers.end(); it++) { + delete *it; + } + mBuffers.clear(); + mConsumer.clear(); +} + +void JNIImageReaderContext::onFrameAvailable() +{ + ALOGV("%s: frame available", __FUNCTION__); + bool needsDetach = false; + JNIEnv* env = getJNIEnv(&needsDetach); + if (env != NULL) { + env->CallStaticVoidMethod(mClazz, fields.postEvent, mWeakThiz); + } else { + ALOGW("onFrameAvailable event will not posted"); + } + if (needsDetach) { + detachJNI(); + } +} + +// ---------------------------------------------------------------------------- + +extern "C" { + +static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz) +{ + JNIImageReaderContext *ctx; + ctx = reinterpret_cast<JNIImageReaderContext *> + (env->GetLongField(thiz, fields.imageReaderContext)); + return ctx; +} + +static CpuConsumer* ImageReader_getCpuConsumer(JNIEnv* env, jobject thiz) +{ + ALOGV("%s:", __FUNCTION__); + JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); + if (ctx == NULL) { + jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); + return NULL; + } + return ctx->getCpuConsumer(); +} + +static void ImageReader_setNativeContext(JNIEnv* env, + jobject thiz, sp<JNIImageReaderContext> ctx) +{ + ALOGV("%s:", __FUNCTION__); + JNIImageReaderContext* const p = ImageReader_getContext(env, thiz); + if (ctx != 0) { + ctx->incStrong((void*)ImageReader_setNativeContext); + } + if (p) { + p->decStrong((void*)ImageReader_setNativeContext); + } + env->SetLongField(thiz, fields.imageReaderContext, reinterpret_cast<jlong>(ctx.get())); +} + +static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image) +{ + return reinterpret_cast<CpuConsumer::LockedBuffer*>(env->GetLongField(image, fields.buffer)); +} + +static void Image_setBuffer(JNIEnv* env, jobject thiz, + const CpuConsumer::LockedBuffer* buffer) +{ + env->SetLongField(thiz, fields.buffer, reinterpret_cast<jlong>(buffer)); +} + +// Some formats like JPEG defined with different values between android.graphics.ImageFormat and +// graphics.h, need convert to the one defined in graphics.h here. +static int Image_getPixelFormat(JNIEnv* env, int format) +{ + int jpegFormat, rawSensorFormat; + jfieldID fid; + + ALOGV("%s: format = 0x%x", __FUNCTION__, format); + + jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat"); + ALOG_ASSERT(imageFormatClazz != NULL); + + fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I"); + jpegFormat = env->GetStaticIntField(imageFormatClazz, fid); + fid = env->GetStaticFieldID(imageFormatClazz, "RAW_SENSOR", "I"); + rawSensorFormat = env->GetStaticIntField(imageFormatClazz, fid); + + // Translate the JPEG to BLOB for camera purpose, an add more if more mismatch is found. + if (format == jpegFormat) { + format = HAL_PIXEL_FORMAT_BLOB; + } + // Same thing for RAW_SENSOR format + if (format == rawSensorFormat) { + format = HAL_PIXEL_FORMAT_RAW_SENSOR; + } + + return format; +} + +static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, + uint8_t **base, uint32_t *size) +{ + ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); + ALOG_ASSERT(base != NULL, "base is NULL!!!"); + ALOG_ASSERT(size != NULL, "size is NULL!!!"); + ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0)); + + ALOGV("%s: buffer: 0x%p", __FUNCTION__, buffer); + + uint32_t dataSize, ySize, cSize, cStride; + uint8_t *cb, *cr; + uint8_t *pData = NULL; + + dataSize = ySize = cSize = cStride = 0; + int32_t fmt = buffer->format; + switch (fmt) { + case HAL_PIXEL_FORMAT_YCbCr_420_888: + pData = + (idx == 0) ? + buffer->data : + (idx == 1) ? + buffer->dataCb : + buffer->dataCr; + if (idx == 0) { + dataSize = buffer->stride * buffer->height; + } else { + dataSize = buffer->chromaStride * buffer->height / 2; + } + break; + // NV21 + case HAL_PIXEL_FORMAT_YCrCb_420_SP: + cr = buffer->data + (buffer->stride * buffer->height); + cb = cr + 1; + ySize = buffer->width * buffer->height; + cSize = buffer->width * buffer->height / 2; + + pData = + (idx == 0) ? + buffer->data : + (idx == 1) ? + cb: + cr; + + dataSize = (idx == 0) ? ySize : cSize; + break; + case HAL_PIXEL_FORMAT_YV12: + // Y and C stride need to be 16 pixel aligned. + LOG_ALWAYS_FATAL_IF(buffer->stride % 16, + "Stride is not 16 pixel aligned %d", buffer->stride); + + ySize = buffer->stride * buffer->height; + cStride = ALIGN(buffer->stride / 2, 16); + cr = buffer->data + ySize; + cSize = cStride * buffer->height / 2; + cb = cr + cSize; + + pData = + (idx == 0) ? + buffer->data : + (idx == 1) ? + cb : + cr; + dataSize = (idx == 0) ? ySize : cSize; + break; + case HAL_PIXEL_FORMAT_Y8: + // Single plane, 8bpp. + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + + pData = buffer->data; + dataSize = buffer->stride * buffer->height; + break; + case HAL_PIXEL_FORMAT_Y16: + // Single plane, 16bpp, strides are specified in pixels, not in bytes + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + + pData = buffer->data; + dataSize = buffer->stride * buffer->height * 2; + break; + case HAL_PIXEL_FORMAT_BLOB: + // Used for JPEG data, height must be 1, width == size, single plane. + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height); + + pData = buffer->data; + dataSize = buffer->width; + break; + case HAL_PIXEL_FORMAT_RAW_SENSOR: + // Single plane 16bpp bayer data. + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + pData = buffer->data; + dataSize = buffer->width * 2 * buffer->height; + break; + default: + jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", + "Pixel format: 0x%x is unsupported", fmt); + break; + } + + *base = pData; + *size = dataSize; +} + +static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx) +{ + ALOGV("%s: buffer index: %d", __FUNCTION__, idx); + ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx); + + int pixelStride = 0; + ALOG_ASSERT(buffer != NULL, "buffer is NULL"); + + int32_t fmt = buffer->format; + switch (fmt) { + case HAL_PIXEL_FORMAT_YCbCr_420_888: + pixelStride = (idx == 0) ? 1 : buffer->chromaStep; + break; + case HAL_PIXEL_FORMAT_YCrCb_420_SP: + pixelStride = (idx == 0) ? 1 : 2; + break; + case HAL_PIXEL_FORMAT_Y8: + // Single plane 8bpp data. + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + pixelStride; + break; + case HAL_PIXEL_FORMAT_YV12: + pixelStride = 1; + break; + case HAL_PIXEL_FORMAT_BLOB: + // Used for JPEG data, single plane, row and pixel strides are 0 + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + pixelStride = 0; + break; + case HAL_PIXEL_FORMAT_Y16: + case HAL_PIXEL_FORMAT_RAW_SENSOR: + // Single plane 16bpp data. + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + pixelStride = 2; + break; + default: + jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", + "Pixel format: 0x%x is unsupported", fmt); + break; + } + + return pixelStride; +} + +static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx) +{ + ALOGV("%s: buffer index: %d", __FUNCTION__, idx); + ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0)); + + int rowStride = 0; + ALOG_ASSERT(buffer != NULL, "buffer is NULL"); + + int32_t fmt = buffer->format; + + switch (fmt) { + case HAL_PIXEL_FORMAT_YCbCr_420_888: + rowStride = (idx == 0) ? buffer->stride : buffer->chromaStride; + break; + case HAL_PIXEL_FORMAT_YCrCb_420_SP: + rowStride = buffer->width; + break; + case HAL_PIXEL_FORMAT_YV12: + LOG_ALWAYS_FATAL_IF(buffer->stride % 16, + "Stride is not 16 pixel aligned %d", buffer->stride); + rowStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16); + break; + case HAL_PIXEL_FORMAT_BLOB: + // Used for JPEG data, single plane, row and pixel strides are 0 + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + rowStride = 0; + break; + case HAL_PIXEL_FORMAT_Y8: + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + LOG_ALWAYS_FATAL_IF(buffer->stride % 16, + "Stride is not 16 pixel aligned %d", buffer->stride); + rowStride = buffer->stride; + break; + case HAL_PIXEL_FORMAT_Y16: + case HAL_PIXEL_FORMAT_RAW_SENSOR: + // In native side, strides are specified in pixels, not in bytes. + // Single plane 16bpp bayer data. even width/height, + // row stride multiple of 16 pixels (32 bytes) + ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); + LOG_ALWAYS_FATAL_IF(buffer->stride % 16, + "Stride is not 16 pixel aligned %d", buffer->stride); + rowStride = buffer->stride * 2; + break; + default: + ALOGE("%s Pixel format: 0x%x is unsupported", __FUNCTION__, fmt); + jniThrowException(env, "java/lang/UnsupportedOperationException", + "unsupported buffer format"); + break; + } + + return rowStride; +} + +// ---------------------------------------------------------------------------- + +static void ImageReader_classInit(JNIEnv* env, jclass clazz) +{ + ALOGV("%s:", __FUNCTION__); + + jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage"); + LOG_ALWAYS_FATAL_IF(imageClazz == NULL, + "can't find android/graphics/ImageReader$SurfaceImage"); + fields.buffer = env->GetFieldID(imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J"); + LOG_ALWAYS_FATAL_IF(fields.buffer == NULL, + "can't find android/graphics/ImageReader.%s", + ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID); + + fields.timeStamp = env->GetFieldID(imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J"); + LOG_ALWAYS_FATAL_IF(fields.timeStamp == NULL, + "can't find android/graphics/ImageReader.%s", + ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID); + + fields.imageReaderContext = env->GetFieldID(clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J"); + LOG_ALWAYS_FATAL_IF(fields.imageReaderContext == NULL, + "can't find android/graphics/ImageReader.%s", + ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID); + + fields.postEvent = env->GetStaticMethodID(clazz, "postEventFromNative", + "(Ljava/lang/Object;)V"); + LOG_ALWAYS_FATAL_IF(fields.postEvent == NULL, + "can't find android/graphics/ImageReader.postEventFromNative"); + + jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane"); + LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class"); + // FindClass only gives a local reference of jclass object. + surfPlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); + surfPlaneClassInfo.ctor = env->GetMethodID(surfPlaneClassInfo.clazz, "<init>", + "(Landroid/media/ImageReader$SurfaceImage;III)V"); + LOG_ALWAYS_FATAL_IF(surfPlaneClassInfo.ctor == NULL, "Can not find SurfacePlane constructor"); +} + +static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, + jint width, jint height, jint format, jint maxImages) +{ + status_t res; + int nativeFormat; + + ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d", + __FUNCTION__, width, height, format, maxImages); + + nativeFormat = Image_getPixelFormat(env, format); + + sp<CpuConsumer> consumer = new CpuConsumer(maxImages); + // TODO: throw dvm exOutOfMemoryError? + if (consumer == NULL) { + jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer"); + return; + } + + jclass clazz = env->GetObjectClass(thiz); + if (clazz == NULL) { + jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader"); + return; + } + sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages)); + ctx->setCpuConsumer(consumer); + consumer->setFrameAvailableListener(ctx); + ImageReader_setNativeContext(env, thiz, ctx); + ctx->setBufferFormat(nativeFormat); + ctx->setBufferWidth(width); + ctx->setBufferHeight(height); + + // Set the width/height/format to the CpuConsumer + res = consumer->setDefaultBufferSize(width, height); + if (res != OK) { + jniThrowException(env, "java/lang/IllegalStateException", + "Failed to set CpuConsumer buffer size"); + return; + } + res = consumer->setDefaultBufferFormat(nativeFormat); + if (res != OK) { + jniThrowException(env, "java/lang/IllegalStateException", + "Failed to set CpuConsumer buffer format"); + } +} + +static void ImageReader_close(JNIEnv* env, jobject thiz) +{ + ALOGV("%s:", __FUNCTION__); + + JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); + if (ctx == NULL) { + // ImageReader is already closed. + return; + } + + CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz); + if (consumer != NULL) { + consumer->abandon(); + consumer->setFrameAvailableListener(NULL); + } + ImageReader_setNativeContext(env, thiz, NULL); +} + +static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image) +{ + ALOGV("%s:", __FUNCTION__); + JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); + if (ctx == NULL) { + ALOGW("ImageReader#close called before Image#close, consider calling Image#close first"); + return; + } + + CpuConsumer* consumer = ctx->getCpuConsumer(); + CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image); + if (!buffer) { + ALOGW("Image already released!!!"); + return; + } + consumer->unlockBuffer(*buffer); + Image_setBuffer(env, image, NULL); + ctx->returnLockedBuffer(buffer); +} + +static jboolean ImageReader_imageSetup(JNIEnv* env, jobject thiz, + jobject image) +{ + ALOGV("%s:", __FUNCTION__); + JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); + if (ctx == NULL) { + jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); + return false; + } + + CpuConsumer* consumer = ctx->getCpuConsumer(); + CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer(); + if (buffer == NULL) { + ALOGE("Unable to acquire a lockedBuffer, very likely client tries to lock more than" + "maxImages buffers"); + return false; + } + status_t res = consumer->lockNextBuffer(buffer); + if (res != NO_ERROR) { + ALOGE("%s Fail to lockNextBuffer with error: 0x%x ", __FUNCTION__, res); + return false; + } + + + // Check if the left-top corner of the crop rect is origin, we currently assume this point is + // zero, will revist this once this assumption turns out problematic. + Point lt = buffer->crop.leftTop(); + if (lt.x != 0 || lt.y != 0) { + ALOGE("crop left: %d, top = %d", lt.x, lt.y); + jniThrowException(env, "java/lang/UnsupportedOperationException", + "crop left top corner need to at origin"); + } + + // Check if the producer buffer configurations match what ImageReader configured. + // We want to fail for the very first image because this case is too bad. + int outputWidth = buffer->crop.getWidth() + 1; + int outputHeight = buffer->crop.getHeight() + 1; + int imageReaderWidth = ctx->getBufferWidth(); + int imageReaderHeight = ctx->getBufferHeight(); + if ((imageReaderWidth != outputWidth) || + (imageReaderHeight != outputHeight)) { + // Spew warning for now, since MediaCodec decoder has a bug to setup the right crop + // TODO: make it throw exception once the decoder bug is fixed. + ALOGW("Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d", + outputWidth, outputHeight, imageReaderWidth, imageReaderHeight); + } + + if (ctx->getBufferFormat() != buffer->format) { + // Return the buffer to the queue. + consumer->unlockBuffer(*buffer); + ctx->returnLockedBuffer(buffer); + + // Throw exception + ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x", + buffer->format, ctx->getBufferFormat()); + jniThrowException(env, "java/lang/UnsupportedOperationException", + "The producer output buffer configuration doesn't match the ImageReader" + "configured"); + return false; + } + // Set SurfaceImage instance member variables + Image_setBuffer(env, image, buffer); + env->SetLongField(image, fields.timeStamp, static_cast<jlong>(buffer->timestamp)); + + return true; +} + +static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz) +{ + ALOGV("%s: ", __FUNCTION__); + + CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz); + if (consumer == NULL) { + jniThrowRuntimeException(env, "CpuConsumer is uninitialized"); + return NULL; + } + + // Wrap the IGBP in a Java-language Surface. + return android_view_Surface_createFromIGraphicBufferProducer(env, + consumer->getProducerInterface()); +} + +static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx) +{ + int rowStride, pixelStride; + ALOGV("%s: buffer index: %d", __FUNCTION__, idx); + + CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); + + ALOG_ASSERT(buffer != NULL); + if (buffer == NULL) { + jniThrowException(env, "java/lang/IllegalStateException", "Image was released"); + } + rowStride = Image_imageGetRowStride(env, buffer, idx); + pixelStride = Image_imageGetPixelStride(env, buffer, idx); + + jobject surfPlaneObj = env->NewObject(surfPlaneClassInfo.clazz, surfPlaneClassInfo.ctor, + thiz, idx, rowStride, pixelStride); + + return surfPlaneObj; +} + +static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx) +{ + uint8_t *base = NULL; + uint32_t size = 0; + jobject byteBuffer; + + ALOGV("%s: buffer index: %d", __FUNCTION__, idx); + + CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); + + if (buffer == NULL) { + jniThrowException(env, "java/lang/IllegalStateException", "Image was released"); + } + + // Create byteBuffer from native buffer + Image_getLockedBufferInfo(env, buffer, idx, &base, &size); + byteBuffer = env->NewDirectByteBuffer(base, size); + // TODO: throw dvm exOutOfMemoryError? + if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { + jniThrowException(env, "java/lang/IllegalStateException", "Failed to allocate ByteBuffer"); + } + + return byteBuffer; +} + +} // extern "C" + +// ---------------------------------------------------------------------------- + +static JNINativeMethod gImageReaderMethods[] = { + {"nativeClassInit", "()V", (void*)ImageReader_classInit }, + {"nativeInit", "(Ljava/lang/Object;IIII)V", (void*)ImageReader_init }, + {"nativeClose", "()V", (void*)ImageReader_close }, + {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease }, + {"nativeImageSetup", "(Landroid/media/Image;)Z", (void*)ImageReader_imageSetup }, + {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface }, +}; + +static JNINativeMethod gImageMethods[] = { + {"nativeImageGetBuffer", "(I)Ljava/nio/ByteBuffer;", (void*)Image_getByteBuffer }, + {"nativeCreatePlane", "(I)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;", + (void*)Image_createSurfacePlane }, +}; + +int register_android_media_ImageReader(JNIEnv *env) { + + int ret1 = AndroidRuntime::registerNativeMethods(env, + "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods)); + + int ret2 = AndroidRuntime::registerNativeMethods(env, + "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods)); + + return (ret1 || ret2); +} diff --git a/media/jni/android_media_MediaPlayer.cpp b/media/jni/android_media_MediaPlayer.cpp index 0f786496453b..9b66c0677b6d 100644 --- a/media/jni/android_media_MediaPlayer.cpp +++ b/media/jni/android_media_MediaPlayer.cpp @@ -896,6 +896,7 @@ static int register_android_media_MediaPlayer(JNIEnv *env) "android/media/MediaPlayer", gMethods, NELEM(gMethods)); } +extern int register_android_media_ImageReader(JNIEnv *env); extern int register_android_media_Crypto(JNIEnv *env); extern int register_android_media_Drm(JNIEnv *env); extern int register_android_media_MediaCodec(JNIEnv *env); @@ -923,6 +924,11 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) } assert(env != NULL); + if (register_android_media_ImageReader(env) < 0) { + ALOGE("ERROR: ImageReader native registration failed"); + goto bail; + } + if (register_android_media_MediaPlayer(env) < 0) { ALOGE("ERROR: MediaPlayer native registration failed\n"); goto bail; |