Merge changes from topic "largeAudioBuffer" into main am: 5a20cf6f7c am: 610f44db26
Original change: https://android-review.googlesource.com/c/platform/frameworks/base/+/2896744
Change-Id: Ia11514bfc0bc573d36f0a8cafbafca3ce1f9236a
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/core/api/current.txt b/core/api/current.txt
index 89c7728..cb8db9e 100644
--- a/core/api/current.txt
+++ b/core/api/current.txt
@@ -22371,6 +22371,7 @@
method @NonNull public java.util.List<java.lang.String> getSupportedVendorParameters();
method @Nullable public static android.media.Image mapHardwareBuffer(@NonNull android.hardware.HardwareBuffer);
method public void queueInputBuffer(int, int, int, long, int) throws android.media.MediaCodec.CryptoException;
+ method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") public void queueInputBuffers(int, @NonNull java.util.ArrayDeque<android.media.MediaCodec.BufferInfo>);
method public void queueSecureInputBuffer(int, int, @NonNull android.media.MediaCodec.CryptoInfo, long, int) throws android.media.MediaCodec.CryptoException;
method public void release();
method public void releaseOutputBuffer(int, boolean);
@@ -22432,6 +22433,7 @@
method public abstract void onError(@NonNull android.media.MediaCodec, @NonNull android.media.MediaCodec.CodecException);
method public abstract void onInputBufferAvailable(@NonNull android.media.MediaCodec, int);
method public abstract void onOutputBufferAvailable(@NonNull android.media.MediaCodec, int, @NonNull android.media.MediaCodec.BufferInfo);
+ method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") public void onOutputBuffersAvailable(@NonNull android.media.MediaCodec, int, @NonNull java.util.ArrayDeque<android.media.MediaCodec.BufferInfo>);
method public abstract void onOutputFormatChanged(@NonNull android.media.MediaCodec, @NonNull android.media.MediaFormat);
}
@@ -22519,6 +22521,7 @@
}
public static final class MediaCodec.OutputFrame {
+ method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") @NonNull public java.util.ArrayDeque<android.media.MediaCodec.BufferInfo> getBufferInfos();
method @NonNull public java.util.Set<java.lang.String> getChangedKeys();
method public int getFlags();
method @NonNull public android.media.MediaFormat getFormat();
@@ -22534,6 +22537,7 @@
public final class MediaCodec.QueueRequest {
method public void queue();
+ method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") @NonNull public android.media.MediaCodec.QueueRequest setBufferInfos(@NonNull java.util.ArrayDeque<android.media.MediaCodec.BufferInfo>);
method @NonNull public android.media.MediaCodec.QueueRequest setByteBufferParameter(@NonNull String, @NonNull java.nio.ByteBuffer);
method @NonNull public android.media.MediaCodec.QueueRequest setEncryptedLinearBlock(@NonNull android.media.MediaCodec.LinearBlock, int, int, @NonNull android.media.MediaCodec.CryptoInfo);
method @NonNull public android.media.MediaCodec.QueueRequest setFlags(int);
diff --git a/media/java/android/media/MediaCodec.java b/media/java/android/media/MediaCodec.java
index 470a8ac..bfb4b42 100644
--- a/media/java/android/media/MediaCodec.java
+++ b/media/java/android/media/MediaCodec.java
@@ -17,6 +17,7 @@
package android.media;
import android.Manifest;
+import android.annotation.FlaggedApi;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
@@ -43,21 +44,25 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import static com.android.media.codec.flags.Flags.FLAG_LARGE_AUDIO_FRAME;
/**
MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components.
It is part of the Android low-level multimedia support infrastructure (normally used together
@@ -1824,6 +1829,7 @@
private static final String EOS_AND_DECODE_ONLY_ERROR_MESSAGE = "An input buffer cannot have "
+ "both BUFFER_FLAG_END_OF_STREAM and BUFFER_FLAG_DECODE_ONLY flags";
private static final int CB_CRYPTO_ERROR = 6;
+ private static final int CB_LARGE_FRAME_OUTPUT_AVAILABLE = 7;
private class EventHandler extends Handler {
private MediaCodec mCodec;
@@ -1945,6 +1951,39 @@
break;
}
+ case CB_LARGE_FRAME_OUTPUT_AVAILABLE:
+ {
+ int index = msg.arg2;
+ ArrayDeque<BufferInfo> infos = (ArrayDeque<BufferInfo>)msg.obj;
+ synchronized(mBufferLock) {
+ switch (mBufferMode) {
+ case BUFFER_MODE_LEGACY:
+ validateOutputByteBuffersLocked(mCachedOutputBuffers,
+ index, infos);
+ break;
+ case BUFFER_MODE_BLOCK:
+ while (mOutputFrames.size() <= index) {
+ mOutputFrames.add(null);
+ }
+ OutputFrame frame = mOutputFrames.get(index);
+ if (frame == null) {
+ frame = new OutputFrame(index);
+ mOutputFrames.set(index, frame);
+ }
+ frame.setBufferInfos(infos);
+ frame.setAccessible(true);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unrecognized buffer mode: for large frame output");
+ }
+ }
+ mCallback.onOutputBuffersAvailable(
+ mCodec, index, infos);
+
+ break;
+ }
+
case CB_ERROR:
{
mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj);
@@ -2836,11 +2875,72 @@
}
}
+ /**
+ * Submit multiple access units to the codec along with multiple
+ * {@link MediaCodec.BufferInfo} describing the contents of the buffer. This method
+ * is supported only in asynchronous mode. While this method can be used for all codecs,
+ * it is meant for buffer batching, which is only supported by codecs that advertise
+ * FEATURE_MultipleFrames. Other codecs will not output large output buffers via
+ * onOutputBuffersAvailable, and instead will output single-access-unit output via
+ * onOutputBufferAvailable.
+ * <p>
+ * Output buffer size can be configured using the following MediaFormat keys.
+ * {@link MediaFormat#KEY_BUFFER_BATCH_MAX_OUTPUT_SIZE} and
+ * {@link MediaFormat#KEY_BUFFER_BATCH_THRESHOLD_OUTPUT_SIZE}.
+ * Details for each access unit present in the buffer should be described using
+ * {@link MediaCodec.BufferInfo}. Access units must be laid out contiguously (without any gaps)
+ * and in order. Multiple access units in the output if present, will be available in
+ * {@link Callback#onOutputBuffersAvailable} or {@link Callback#onOutputBufferAvailable}
+ * in case of single-access-unit output or when output does not contain any buffers,
+ * such as flags.
+ * <p>
+ * All other details for populating {@link MediaCodec.BufferInfo} is the same as described in
+ * {@link #queueInputBuffer}.
+ *
+ * @param index The index of a client-owned input buffer previously returned
+ * in a call to {@link #dequeueInputBuffer}.
+ * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the
+ * contents in the buffer. The ArrayDeque and the BufferInfo objects provided
+ * can be recycled by the caller for re-use.
+ * @throws IllegalStateException if not in the Executing state or not in asynchronous mode.
+ * @throws MediaCodec.CodecException upon codec error.
+ * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the
+ * access units are not contiguous.
+ * @throws CryptoException if a crypto object has been specified in
+ * {@link #configure}
+ */
+ @FlaggedApi(FLAG_LARGE_AUDIO_FRAME)
+ public final void queueInputBuffers(
+ int index,
+ @NonNull ArrayDeque<BufferInfo> bufferInfos) {
+ synchronized(mBufferLock) {
+ if (mBufferMode == BUFFER_MODE_BLOCK) {
+ throw new IncompatibleWithBlockModelException("queueInputBuffers() "
+ + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. "
+ + "Please use getQueueRequest() to queue buffers");
+ }
+ invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */);
+ mDequeuedInputBuffers.remove(index);
+ }
+ try {
+ native_queueInputBuffers(
+ index, bufferInfos.toArray());
+ } catch (CryptoException | IllegalStateException | IllegalArgumentException e) {
+ revalidateByteBuffer(mCachedInputBuffers, index, true /* input */);
+ throw e;
+ }
+ }
+
private native final void native_queueInputBuffer(
int index,
int offset, int size, long presentationTimeUs, int flags)
throws CryptoException;
+ private native final void native_queueInputBuffers(
+ int index,
+ @NonNull Object[] infos)
+ throws CryptoException, CodecException;
+
public static final int CRYPTO_MODE_UNENCRYPTED = 0;
public static final int CRYPTO_MODE_AES_CTR = 1;
public static final int CRYPTO_MODE_AES_CBC = 2;
@@ -3464,6 +3564,26 @@
}
/**
+ * Sets MediaCodec.BufferInfo objects describing the access units
+ * contained in this queue request. Access units must be laid out
+ * contiguously without gaps and in order.
+ *
+ * @param infos Represents {@link MediaCodec.BufferInfo} objects to mark
+ * individual access-unit boundaries and the timestamps associated with it.
+ * The buffer is expected to contain the data in a continuous manner.
+ * @return this object
+ */
+ @FlaggedApi(FLAG_LARGE_AUDIO_FRAME)
+ public @NonNull QueueRequest setBufferInfos(@NonNull ArrayDeque<BufferInfo> infos) {
+ if (!isAccessible()) {
+ throw new IllegalStateException("The request is stale");
+ }
+ mBufferInfos.clear();
+ mBufferInfos.addAll(infos);
+ return this;
+ }
+
+ /**
* Add an integer parameter.
* See {@link MediaFormat} for an exhaustive list of supported keys with
* values of type int, that can also be set with {@link MediaFormat#setInteger}.
@@ -3579,10 +3699,18 @@
throw new IllegalStateException("No block is set");
}
setAccessible(false);
+ if (mBufferInfos.isEmpty()) {
+ BufferInfo info = new BufferInfo();
+ info.size = mSize;
+ info.offset = mOffset;
+ info.presentationTimeUs = mPresentationTimeUs;
+ info.flags = mFlags;
+ mBufferInfos.add(info);
+ }
if (mLinearBlock != null) {
mCodec.native_queueLinearBlock(
- mIndex, mLinearBlock, mOffset, mSize, mCryptoInfo,
- mPresentationTimeUs, mFlags,
+ mIndex, mLinearBlock, mCryptoInfo,
+ mBufferInfos.toArray(),
mTuningKeys, mTuningValues);
} else if (mHardwareBuffer != null) {
mCodec.native_queueHardwareBuffer(
@@ -3600,6 +3728,7 @@
mHardwareBuffer = null;
mPresentationTimeUs = 0;
mFlags = 0;
+ mBufferInfos.clear();
mTuningKeys.clear();
mTuningValues.clear();
return this;
@@ -3623,6 +3752,7 @@
private HardwareBuffer mHardwareBuffer = null;
private long mPresentationTimeUs = 0;
private @BufferFlag int mFlags = 0;
+ private final ArrayDeque<BufferInfo> mBufferInfos = new ArrayDeque<>();
private final ArrayList<String> mTuningKeys = new ArrayList<>();
private final ArrayList<Object> mTuningValues = new ArrayList<>();
@@ -3632,11 +3762,8 @@
private native void native_queueLinearBlock(
int index,
@NonNull LinearBlock block,
- int offset,
- int size,
@Nullable CryptoInfo cryptoInfo,
- long presentationTimeUs,
- int flags,
+ @NonNull Object[] bufferInfos,
@NonNull ArrayList<String> keys,
@NonNull ArrayList<Object> values);
@@ -4050,6 +4177,27 @@
}
}
+ private void validateOutputByteBuffersLocked(
+ @Nullable ByteBuffer[] buffers, int index, @NonNull ArrayDeque<BufferInfo> infoDeque) {
+ Optional<BufferInfo> minInfo = infoDeque.stream().min(
+ (info1, info2) -> Integer.compare(info1.offset, info2.offset));
+ Optional<BufferInfo> maxInfo = infoDeque.stream().max(
+ (info1, info2) -> Integer.compare(info1.offset, info2.offset));
+ if (buffers == null) {
+ if (index >= 0) {
+ mValidOutputIndices.set(index);
+ }
+ } else if (index >= 0 && index < buffers.length) {
+ ByteBuffer buffer = buffers[index];
+ if (buffer != null && minInfo.isPresent() && maxInfo.isPresent()) {
+ buffer.setAccessible(true);
+ buffer.limit(maxInfo.get().offset + maxInfo.get().size);
+ buffer.position(minInfo.get().offset);
+ }
+ }
+
+ }
+
private void validateOutputByteBufferLocked(
@Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) {
if (buffers == null) {
@@ -4407,6 +4555,22 @@
return mFlags;
}
+ /*
+ * Returns the BufferInfos associated with this OutputFrame. These BufferInfos
+ * describes the access units present in the OutputFrame. Access units are laid
+ * out contiguously without gaps and in order.
+ */
+ @FlaggedApi(FLAG_LARGE_AUDIO_FRAME)
+ public @NonNull ArrayDeque<BufferInfo> getBufferInfos() {
+ if (mBufferInfos.isEmpty()) {
+ // single BufferInfo could be present.
+ BufferInfo bufferInfo = new BufferInfo();
+ bufferInfo.set(0, 0, mPresentationTimeUs, mFlags);
+ mBufferInfos.add(bufferInfo);
+ }
+ return mBufferInfos;
+ }
+
/**
* Returns a read-only {@link MediaFormat} for this frame. The returned
* object is valid only until the client calls {@link MediaCodec#releaseOutputBuffer}.
@@ -4432,6 +4596,7 @@
mLinearBlock = null;
mHardwareBuffer = null;
mFormat = null;
+ mBufferInfos.clear();
mChangedKeys.clear();
mKeySet.clear();
mLoaded = false;
@@ -4450,6 +4615,11 @@
mFlags = info.flags;
}
+ void setBufferInfos(ArrayDeque<BufferInfo> infos) {
+ mBufferInfos.clear();
+ mBufferInfos.addAll(infos);
+ }
+
boolean isLoaded() {
return mLoaded;
}
@@ -4464,6 +4634,7 @@
private long mPresentationTimeUs = 0;
private @BufferFlag int mFlags = 0;
private MediaFormat mFormat = null;
+ private final ArrayDeque<BufferInfo> mBufferInfos = new ArrayDeque<>();
private final ArrayList<String> mChangedKeys = new ArrayList<>();
private final Set<String> mKeySet = new HashSet<>();
private boolean mAccessible = false;
@@ -5172,6 +5343,32 @@
@NonNull MediaCodec codec, int index, @NonNull BufferInfo info);
/**
+ * Called when multiple access-units are available in the output.
+ *
+ * @param codec The MediaCodec object.
+ * @param index The index of the available output buffer.
+ * @param infos Infos describing the available output buffer {@link MediaCodec.BufferInfo}.
+ * Access units present in the output buffer are laid out contiguously
+ * without gaps and in order.
+ */
+ @FlaggedApi(FLAG_LARGE_AUDIO_FRAME)
+ public void onOutputBuffersAvailable(
+ @NonNull MediaCodec codec, int index, @NonNull ArrayDeque<BufferInfo> infos) {
+ /*
+ * This callback returns multiple BufferInfos when codecs are configured to operate on
+ * large audio frame. Since at this point, we have a single large buffer, returning
+ * each BufferInfo using
+ * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} may cause the
+ * index to be released to the codec using {@link MediaCodec#releaseOutputBuffer}
+ * before all BuffersInfos can be returned to the client.
+ * Hence this callback is required to be implemented or else an exception is thrown.
+ */
+ throw new IllegalStateException(
+ "Client must override onOutputBuffersAvailable when codec is " +
+ "configured to operate with multiple access units");
+ }
+
+ /**
* Called when the MediaCodec encountered an error
*
* @param codec The MediaCodec object.
diff --git a/media/jni/android_media_MediaCodec.cpp b/media/jni/android_media_MediaCodec.cpp
index ef90bf9..8cdd59e 100644
--- a/media/jni/android_media_MediaCodec.cpp
+++ b/media/jni/android_media_MediaCodec.cpp
@@ -163,6 +163,13 @@
static struct {
jclass clazz;
jmethodID ctorId;
+ jmethodID sizeId;
+ jmethodID addId;
+} gArrayDequeInfo;
+
+static struct {
+ jclass clazz;
+ jmethodID ctorId;
jmethodID setInternalStateId;
jfieldID contextId;
jfieldID validId;
@@ -200,8 +207,14 @@
jfieldID queueRequestIndexID;
jfieldID outputFrameLinearBlockID;
jfieldID outputFrameHardwareBufferID;
+ jfieldID outputFramebufferInfosID;
jfieldID outputFrameChangedKeysID;
jfieldID outputFrameFormatID;
+ jfieldID bufferInfoFlags;
+ jfieldID bufferInfoOffset;
+ jfieldID bufferInfoSize;
+ jfieldID bufferInfoPresentationTimeUs;
+
};
static fields_t gFields;
@@ -412,6 +425,22 @@
index, offset, size, timeUs, flags, errorDetailMsg);
}
+status_t JMediaCodec::queueInputBuffers(
+ size_t index,
+ size_t offset,
+ size_t size,
+ const sp<RefBase> &infos,
+ AString *errorDetailMsg) {
+
+ sp<BufferInfosWrapper> auInfo((BufferInfosWrapper *)infos.get());
+ return mCodec->queueInputBuffers(
+ index,
+ offset,
+ size,
+ auInfo,
+ errorDetailMsg);
+}
+
status_t JMediaCodec::queueSecureInputBuffer(
size_t index,
size_t offset,
@@ -430,10 +459,11 @@
}
status_t JMediaCodec::queueBuffer(
- size_t index, const std::shared_ptr<C2Buffer> &buffer, int64_t timeUs,
- uint32_t flags, const sp<AMessage> &tunings, AString *errorDetailMsg) {
+ size_t index, const std::shared_ptr<C2Buffer> &buffer,
+ const sp<RefBase> &infos, const sp<AMessage> &tunings, AString *errorDetailMsg) {
+ sp<BufferInfosWrapper> auInfo((BufferInfosWrapper *)infos.get());
return mCodec->queueBuffer(
- index, buffer, timeUs, flags, tunings, errorDetailMsg);
+ index, buffer, auInfo, tunings, errorDetailMsg);
}
status_t JMediaCodec::queueEncryptedLinearBlock(
@@ -446,13 +476,13 @@
const uint8_t iv[16],
CryptoPlugin::Mode mode,
const CryptoPlugin::Pattern &pattern,
- int64_t presentationTimeUs,
- uint32_t flags,
+ const sp<RefBase> &infos,
const sp<AMessage> &tunings,
AString *errorDetailMsg) {
+ sp<BufferInfosWrapper> auInfo((BufferInfosWrapper *)infos.get());
return mCodec->queueEncryptedBuffer(
index, buffer, offset, subSamples, numSubSamples, key, iv, mode, pattern,
- presentationTimeUs, flags, tunings, errorDetailMsg);
+ auInfo, tunings, errorDetailMsg);
}
status_t JMediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
@@ -722,6 +752,42 @@
return OK;
}
+void maybeSetBufferInfos(JNIEnv *env, jobject &frame, const sp<BufferInfosWrapper> &bufInfos) {
+ if (!bufInfos) {
+ return;
+ }
+ std::vector<AccessUnitInfo> &infos = bufInfos.get()->value;
+ if (infos.empty()) {
+ return;
+ }
+ ScopedLocalRef<jobject> dequeObj{env, env->NewObject(
+ gArrayDequeInfo.clazz, gArrayDequeInfo.ctorId)};
+ jint offset = 0;
+ std::vector<jobject> jObjectInfos;
+ for (int i = 0 ; i < infos.size(); i++) {
+ jobject bufferInfo = env->NewObject(
+ gBufferInfo.clazz, gBufferInfo.ctorId);
+ if (bufferInfo != NULL) {
+ env->CallVoidMethod(bufferInfo, gBufferInfo.setId,
+ offset,
+ (jint)(infos)[i].mSize,
+ (infos)[i].mTimestamp,
+ (infos)[i].mFlags);
+ (void)env->CallBooleanMethod(
+ dequeObj.get(), gArrayDequeInfo.addId, bufferInfo);
+ offset += (infos)[i].mSize;
+ jObjectInfos.push_back(bufferInfo);
+ }
+ }
+ env->SetObjectField(
+ frame,
+ gFields.outputFramebufferInfosID,
+ dequeObj.get());
+ for (int i = 0; i < jObjectInfos.size(); i++) {
+ env->DeleteLocalRef(jObjectInfos[i]);
+ }
+}
+
status_t JMediaCodec::getOutputFrame(
JNIEnv *env, jobject frame, size_t index) const {
sp<MediaCodecBuffer> buffer;
@@ -732,6 +798,11 @@
}
if (buffer->size() > 0) {
+ sp<RefBase> obj;
+ sp<BufferInfosWrapper> bufInfos;
+ if (buffer->meta()->findObject("accessUnitInfo", &obj)) {
+ bufInfos = std::move(((decltype(bufInfos.get()))obj.get()));
+ }
std::shared_ptr<C2Buffer> c2Buffer = buffer->asC2Buffer();
if (c2Buffer) {
switch (c2Buffer->data().type()) {
@@ -747,6 +818,7 @@
(jlong)context.release(),
true);
env->SetObjectField(frame, gFields.outputFrameLinearBlockID, linearBlock.get());
+ maybeSetBufferInfos(env, frame, bufInfos);
break;
}
case C2BufferData::GRAPHIC: {
@@ -787,6 +859,7 @@
(jlong)context.release(),
true);
env->SetObjectField(frame, gFields.outputFrameLinearBlockID, linearBlock.get());
+ maybeSetBufferInfos(env, frame, bufInfos);
} else {
// No-op.
}
@@ -1250,6 +1323,7 @@
void JMediaCodec::handleCallback(const sp<AMessage> &msg) {
int32_t arg1, arg2 = 0;
jobject obj = NULL;
+ std::vector<jobject> jObjectInfos;
CHECK(msg->findInt32("callbackID", &arg1));
JNIEnv *env = AndroidRuntime::getJNIEnv();
@@ -1287,6 +1361,35 @@
break;
}
+ case MediaCodec::CB_LARGE_FRAME_OUTPUT_AVAILABLE:
+ {
+ sp<RefBase> spobj = nullptr;
+ CHECK(msg->findInt32("index", &arg2));
+ CHECK(msg->findObject("accessUnitInfo", &spobj));
+ if (spobj != nullptr) {
+ sp<BufferInfosWrapper> bufferInfoParamsWrapper {
+ (BufferInfosWrapper *)spobj.get()};
+ std::vector<AccessUnitInfo> &bufferInfoParams =
+ bufferInfoParamsWrapper.get()->value;
+ obj = env->NewObject(gArrayDequeInfo.clazz, gArrayDequeInfo.ctorId);
+ jint offset = 0;
+ for (int i = 0 ; i < bufferInfoParams.size(); i++) {
+ jobject bufferInfo = env->NewObject(gBufferInfo.clazz, gBufferInfo.ctorId);
+ if (bufferInfo != NULL) {
+ env->CallVoidMethod(bufferInfo, gBufferInfo.setId,
+ offset,
+ (jint)(bufferInfoParams)[i].mSize,
+ (bufferInfoParams)[i].mTimestamp,
+ (bufferInfoParams)[i].mFlags);
+ (void)env->CallBooleanMethod(obj, gArrayDequeInfo.addId, bufferInfo);
+ offset += (bufferInfoParams)[i].mSize;
+ jObjectInfos.push_back(bufferInfo);
+ }
+ }
+ }
+ break;
+ }
+
case MediaCodec::CB_CRYPTO_ERROR:
{
int32_t err, actionCode;
@@ -1346,6 +1449,9 @@
arg2,
obj);
+ for (int i = 0; i < jObjectInfos.size(); i++) {
+ env->DeleteLocalRef(jObjectInfos[i]);
+ }
env->DeleteLocalRef(obj);
}
@@ -1913,6 +2019,103 @@
codec->getExceptionMessage(errorDetailMsg.c_str()).c_str());
}
+static status_t extractInfosFromObject(
+ JNIEnv * const env,
+ jint * const initialOffset,
+ jint * const totalSize,
+ std::vector<AccessUnitInfo> * const infos,
+ const jobjectArray &objArray,
+ AString * const errorDetailMsg) {
+ if (totalSize == nullptr
+ || initialOffset == nullptr
+ || infos == nullptr) {
+ if (errorDetailMsg) {
+ *errorDetailMsg = "Error: Null arguments provided for extracting Access unit info";
+ }
+ return BAD_VALUE;
+ }
+ const jsize numEntries = env->GetArrayLength(objArray);
+ if (numEntries <= 0) {
+ if (errorDetailMsg) {
+ *errorDetailMsg = "Error: No BufferInfo found while queuing for large frame input";
+ }
+ return BAD_VALUE;
+ }
+ *initialOffset = 0;
+ *totalSize = 0;
+ for (jsize i = 0; i < numEntries; i++) {
+ jobject param = env->GetObjectArrayElement(objArray, i);
+ if (param == NULL) {
+ if (errorDetailMsg) {
+ *errorDetailMsg = "Error: Queuing a null BufferInfo";
+ }
+ return BAD_VALUE;
+ }
+ size_t offset = static_cast<size_t>(env->GetIntField(param, gFields.bufferInfoOffset));
+ size_t size = static_cast<size_t>(env->GetIntField(param, gFields.bufferInfoSize));
+ uint32_t flags = static_cast<uint32_t>(env->GetIntField(param, gFields.bufferInfoFlags));
+ if (flags == 0 && size == 0) {
+ if (errorDetailMsg) {
+ *errorDetailMsg = "Error: Queuing an empty BufferInfo";
+ }
+ return BAD_VALUE;
+ }
+ if (i == 0) {
+ *initialOffset = offset;
+ }
+ if (CC_UNLIKELY((offset > UINT32_MAX)
+ || ((long)(offset + size) > UINT32_MAX)
+ || ((offset - *initialOffset) != *totalSize))) {
+ if (errorDetailMsg) {
+ *errorDetailMsg = "Error: offset/size in BufferInfo";
+ }
+ return BAD_VALUE;
+ }
+ infos->emplace_back(
+ flags,
+ size,
+ env->GetLongField(param, gFields.bufferInfoPresentationTimeUs));
+ *totalSize += size;
+ }
+ return OK;
+}
+
+static void android_media_MediaCodec_queueInputBuffers(
+ JNIEnv *env,
+ jobject thiz,
+ jint index,
+ jobjectArray objArray) {
+ ALOGV("android_media_MediaCodec_queueInputBuffers");
+ sp<JMediaCodec> codec = getMediaCodec(env, thiz);
+ if (codec == NULL || codec->initCheck() != OK || objArray == NULL) {
+ throwExceptionAsNecessary(env, INVALID_OPERATION, codec);
+ return;
+ }
+ sp<BufferInfosWrapper> infoObj =
+ new BufferInfosWrapper{decltype(infoObj->value)()};
+ AString errorDetailMsg;
+ jint initialOffset = 0;
+ jint totalSize = 0;
+ status_t err = extractInfosFromObject(
+ env,
+ &initialOffset,
+ &totalSize,
+ &infoObj->value,
+ objArray,
+ &errorDetailMsg);
+ if (err == OK) {
+ err = codec->queueInputBuffers(
+ index,
+ initialOffset,
+ totalSize,
+ infoObj,
+ &errorDetailMsg);
+ }
+ throwExceptionAsNecessary(
+ env, err, ACTION_CODE_FATAL,
+ codec->getExceptionMessage(errorDetailMsg.c_str()).c_str());
+}
+
struct NativeCryptoInfo {
NativeCryptoInfo(JNIEnv *env, jobject cryptoInfoObj)
: mEnv{env},
@@ -2559,8 +2762,7 @@
static void android_media_MediaCodec_native_queueLinearBlock(
JNIEnv *env, jobject thiz, jint index, jobject bufferObj,
- jint offset, jint size, jobject cryptoInfoObj,
- jlong presentationTimeUs, jint flags, jobject keys, jobject values) {
+ jobject cryptoInfoObj, jobjectArray objArray, jobject keys, jobject values) {
ALOGV("android_media_MediaCodec_native_queueLinearBlock");
sp<JMediaCodec> codec = getMediaCodec(env, thiz);
@@ -2578,7 +2780,24 @@
"error occurred while converting tunings from Java to native");
return;
}
-
+ jint totalSize;
+ jint initialOffset;
+ std::vector<AccessUnitInfo> infoVec;
+ AString errorDetailMsg;
+ err = extractInfosFromObject(env,
+ &initialOffset,
+ &totalSize,
+ &infoVec,
+ objArray,
+ &errorDetailMsg);
+ if (err != OK) {
+ throwExceptionAsNecessary(
+ env, INVALID_OPERATION, ACTION_CODE_FATAL,
+ codec->getExceptionMessage(errorDetailMsg.c_str()).c_str());
+ return;
+ }
+ sp<BufferInfosWrapper> infos =
+ new BufferInfosWrapper{std::move(infoVec)};
std::shared_ptr<C2Buffer> buffer;
sp<hardware::HidlMemory> memory;
ScopedLocalRef<jobject> lock{env, env->GetObjectField(bufferObj, gLinearBlockInfo.lockId)};
@@ -2587,10 +2806,10 @@
JMediaCodecLinearBlock *context =
(JMediaCodecLinearBlock *)env->GetLongField(bufferObj, gLinearBlockInfo.contextId);
if (codec->hasCryptoOrDescrambler()) {
- extractMemoryFromContext(context, offset, size, &memory);
- offset += context->mHidlMemoryOffset;
+ extractMemoryFromContext(context, initialOffset, totalSize, &memory);
+ initialOffset += context->mHidlMemoryOffset;
} else {
- extractBufferFromContext(context, offset, size, &buffer);
+ extractBufferFromContext(context, initialOffset, totalSize, &buffer);
}
}
env->MonitorExit(lock.get());
@@ -2601,7 +2820,6 @@
return;
}
- AString errorDetailMsg;
if (codec->hasCryptoOrDescrambler()) {
if (!memory) {
// It means there was an unexpected failure in extractMemoryFromContext above
@@ -2615,7 +2833,7 @@
return;
}
auto cryptoInfo =
- cryptoInfoObj ? NativeCryptoInfo{env, cryptoInfoObj} : NativeCryptoInfo{size};
+ cryptoInfoObj ? NativeCryptoInfo{env, cryptoInfoObj} : NativeCryptoInfo{totalSize};
if (env->ExceptionCheck()) {
// Creation of cryptoInfo failed. Let the exception bubble up.
return;
@@ -2623,13 +2841,12 @@
err = codec->queueEncryptedLinearBlock(
index,
memory,
- offset,
+ initialOffset,
cryptoInfo.mSubSamples, cryptoInfo.mNumSubSamples,
(const uint8_t *)cryptoInfo.mKey, (const uint8_t *)cryptoInfo.mIv,
cryptoInfo.mMode,
cryptoInfo.mPattern,
- presentationTimeUs,
- flags,
+ infos,
tunings,
&errorDetailMsg);
ALOGI_IF(err != OK, "queueEncryptedLinearBlock returned err = %d", err);
@@ -2646,7 +2863,7 @@
return;
}
err = codec->queueBuffer(
- index, buffer, presentationTimeUs, flags, tunings, &errorDetailMsg);
+ index, buffer, infos, tunings, &errorDetailMsg);
}
throwExceptionAsNecessary(
env, err, ACTION_CODE_FATAL,
@@ -2704,8 +2921,11 @@
std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateGraphicBuffer(block->share(
block->crop(), C2Fence{}));
AString errorDetailMsg;
+ sp<BufferInfosWrapper> infos =
+ new BufferInfosWrapper{decltype(infos->value)()};
+ infos->value.emplace_back(flags, 0 /*not used*/, presentationTimeUs);
err = codec->queueBuffer(
- index, buffer, presentationTimeUs, flags, tunings, &errorDetailMsg);
+ index, buffer, infos, tunings, &errorDetailMsg);
throwExceptionAsNecessary(
env, err, ACTION_CODE_FATAL,
codec->getExceptionMessage(errorDetailMsg.c_str()).c_str());
@@ -3214,6 +3434,10 @@
env->GetFieldID(clazz.get(), "mLinearBlock", "Landroid/media/MediaCodec$LinearBlock;");
CHECK(gFields.outputFrameLinearBlockID != NULL);
+ gFields.outputFramebufferInfosID =
+ env->GetFieldID(clazz.get(), "mBufferInfos", "Ljava/util/ArrayDeque;");
+ CHECK(gFields.outputFramebufferInfosID != NULL);
+
gFields.outputFrameHardwareBufferID =
env->GetFieldID(clazz.get(), "mHardwareBuffer", "Landroid/hardware/HardwareBuffer;");
CHECK(gFields.outputFrameHardwareBufferID != NULL);
@@ -3401,6 +3625,19 @@
gArrayListInfo.addId = env->GetMethodID(clazz.get(), "add", "(Ljava/lang/Object;)Z");
CHECK(gArrayListInfo.addId != NULL);
+ clazz.reset(env->FindClass("java/util/ArrayDeque"));
+ CHECK(clazz.get() != NULL);
+ gArrayDequeInfo.clazz = (jclass)env->NewGlobalRef(clazz.get());
+
+ gArrayDequeInfo.ctorId = env->GetMethodID(clazz.get(), "<init>", "()V");
+ CHECK(gArrayDequeInfo.ctorId != NULL);
+
+ gArrayDequeInfo.sizeId = env->GetMethodID(clazz.get(), "size", "()I");
+ CHECK(gArrayDequeInfo.sizeId != NULL);
+
+ gArrayDequeInfo.addId = env->GetMethodID(clazz.get(), "add", "(Ljava/lang/Object;)Z");
+ CHECK(gArrayDequeInfo.addId != NULL);
+
clazz.reset(env->FindClass("android/media/MediaCodec$LinearBlock"));
CHECK(clazz.get() != NULL);
@@ -3444,6 +3681,12 @@
gBufferInfo.setId = env->GetMethodID(clazz.get(), "set", "(IIJI)V");
CHECK(gBufferInfo.setId != NULL);
+
+ gFields.bufferInfoSize = env->GetFieldID(clazz.get(), "size", "I");
+ gFields.bufferInfoFlags = env->GetFieldID(clazz.get(), "flags", "I");
+ gFields.bufferInfoOffset = env->GetFieldID(clazz.get(), "offset", "I");
+ gFields.bufferInfoPresentationTimeUs =
+ env->GetFieldID(clazz.get(), "presentationTimeUs", "J");
}
static void android_media_MediaCodec_native_setup(
@@ -3701,6 +3944,9 @@
{ "native_queueInputBuffer", "(IIIJI)V",
(void *)android_media_MediaCodec_queueInputBuffer },
+ { "native_queueInputBuffers", "(I[Ljava/lang/Object;)V",
+ (void *)android_media_MediaCodec_queueInputBuffers },
+
{ "native_queueSecureInputBuffer", "(IILandroid/media/MediaCodec$CryptoInfo;JI)V",
(void *)android_media_MediaCodec_queueSecureInputBuffer },
@@ -3711,8 +3957,8 @@
{ "native_closeMediaImage", "(J)V", (void *)android_media_MediaCodec_closeMediaImage },
{ "native_queueLinearBlock",
- "(ILandroid/media/MediaCodec$LinearBlock;IILandroid/media/MediaCodec$CryptoInfo;JI"
- "Ljava/util/ArrayList;Ljava/util/ArrayList;)V",
+ "(ILandroid/media/MediaCodec$LinearBlock;Landroid/media/MediaCodec$CryptoInfo;"
+ "[Ljava/lang/Object;Ljava/util/ArrayList;Ljava/util/ArrayList;)V",
(void *)android_media_MediaCodec_native_queueLinearBlock },
{ "native_queueHardwareBuffer",
diff --git a/media/jni/android_media_MediaCodec.h b/media/jni/android_media_MediaCodec.h
index fbaf64f..02708ef 100644
--- a/media/jni/android_media_MediaCodec.h
+++ b/media/jni/android_media_MediaCodec.h
@@ -35,6 +35,7 @@
namespace android {
struct ABuffer;
+struct AccessUnitInfo;
struct ALooper;
struct AMessage;
struct AString;
@@ -93,6 +94,13 @@
size_t offset, size_t size, int64_t timeUs, uint32_t flags,
AString *errorDetailMsg);
+ status_t queueInputBuffers(
+ size_t index,
+ size_t offset,
+ size_t size,
+ const sp<RefBase> &auInfo,
+ AString *errorDetailMsg = NULL);
+
status_t queueSecureInputBuffer(
size_t index,
size_t offset,
@@ -108,7 +116,7 @@
status_t queueBuffer(
size_t index, const std::shared_ptr<C2Buffer> &buffer,
- int64_t timeUs, uint32_t flags, const sp<AMessage> &tunings,
+ const sp<RefBase> &infos, const sp<AMessage> &tunings,
AString *errorDetailMsg);
status_t queueEncryptedLinearBlock(
@@ -121,8 +129,7 @@
const uint8_t iv[16],
CryptoPlugin::Mode mode,
const CryptoPlugin::Pattern &pattern,
- int64_t presentationTimeUs,
- uint32_t flags,
+ const sp<RefBase> &infos,
const sp<AMessage> &tunings,
AString *errorDetailMsg);