From 4ca490950b2600d836e113dea8c5058a74acc44c Mon Sep 17 00:00:00 2001 From: Arun Johnson Date: Thu, 1 Feb 2024 19:07:15 +0000 Subject: Add API queueSecureInputBuffers for large audio frames Bug: 298052174 API-Coverage-Bug: 309692716 Change-Id: I9c086d5bbb8fcaf79de211b8630ac0f49e6157cd --- core/api/current.txt | 1 + media/java/android/media/MediaCodec.java | 50 +++++++++ media/jni/android_media_MediaCodec.cpp | 173 ++++++++++++++++++++++++++++++- media/jni/android_media_MediaCodec.h | 8 ++ 4 files changed, 230 insertions(+), 2 deletions(-) diff --git a/core/api/current.txt b/core/api/current.txt index a1505351a410..28db56a69c36 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -22575,6 +22575,7 @@ package android.media { method public void queueInputBuffer(int, int, int, long, int) throws android.media.MediaCodec.CryptoException; method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") public void queueInputBuffers(int, @NonNull java.util.ArrayDeque); method public void queueSecureInputBuffer(int, int, @NonNull android.media.MediaCodec.CryptoInfo, long, int) throws android.media.MediaCodec.CryptoException; + method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") public void queueSecureInputBuffers(int, @NonNull java.util.ArrayDeque, @NonNull java.util.ArrayDeque); method public void release(); method public void releaseOutputBuffer(int, boolean); method public void releaseOutputBuffer(int, long); diff --git a/media/java/android/media/MediaCodec.java b/media/java/android/media/MediaCodec.java index 862ae8d7fcaf..ba0b77015802 100644 --- a/media/java/android/media/MediaCodec.java +++ b/media/java/android/media/MediaCodec.java @@ -3210,6 +3210,51 @@ final public class MediaCodec { } } + /** + * Similar to {@link #queueInputBuffers queueInputBuffers} but submits multiple access units + * in a buffer that is potentially encrypted. + * Check out further notes at {@link #queueInputBuffers queueInputBuffers}. + * + * @param index The index of a client-owned input buffer previously returned + * in a call to {@link #dequeueInputBuffer}. + * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the + * contents in the buffer. The ArrayDeque and the BufferInfo objects provided + * can be recycled by the caller for re-use. + * @param cryptoInfos ArrayDeque of {@link MediaCodec.CryptoInfo} objects to facilitate the + * decryption of the contents. The ArrayDeque and the CryptoInfo objects + * provided can be reused immediately after the call returns. These objects + * should correspond to bufferInfo objects to ensure correct decryption. + * @throws IllegalStateException if not in the Executing state or not in asynchronous mode. + * @throws MediaCodec.CodecException upon codec error. + * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the + * access units are not contiguous. + * @throws CryptoException if an error occurs while attempting to decrypt the buffer. + * An error code associated with the exception helps identify the + * reason for the failure. + */ + @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) + public final void queueSecureInputBuffers( + int index, + @NonNull ArrayDeque bufferInfos, + @NonNull ArrayDeque cryptoInfos) { + synchronized(mBufferLock) { + if (mBufferMode == BUFFER_MODE_BLOCK) { + throw new IncompatibleWithBlockModelException("queueSecureInputBuffers() " + + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " + + "Please use getQueueRequest() to queue buffers"); + } + invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); + mDequeuedInputBuffers.remove(index); + } + try { + native_queueSecureInputBuffers( + index, bufferInfos.toArray(), cryptoInfos.toArray()); + } catch (CryptoException | IllegalStateException | IllegalArgumentException e) { + revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); + throw e; + } + } + private native final void native_queueSecureInputBuffer( int index, int offset, @@ -3217,6 +3262,11 @@ final public class MediaCodec { long presentationTimeUs, int flags) throws CryptoException; + private native final void native_queueSecureInputBuffers( + int index, + @NonNull Object[] bufferInfos, + @NonNull Object[] cryptoInfos) throws CryptoException, CodecException; + /** * Returns the index of an input buffer to be filled with valid data * or -1 if no such buffer is currently available. diff --git a/media/jni/android_media_MediaCodec.cpp b/media/jni/android_media_MediaCodec.cpp index 8cdd59e51ffe..fb7d90515aae 100644 --- a/media/jni/android_media_MediaCodec.cpp +++ b/media/jni/android_media_MediaCodec.cpp @@ -458,6 +458,24 @@ status_t JMediaCodec::queueSecureInputBuffer( presentationTimeUs, flags, errorDetailMsg); } +status_t JMediaCodec::queueSecureInputBuffers( + size_t index, + size_t offset, + size_t size, + const sp &auInfos_, + const sp &cryptoInfos_, + AString *errorDetailMsg) { + sp auInfos((BufferInfosWrapper *)auInfos_.get()); + sp cryptoInfos((CryptoInfosWrapper *)cryptoInfos_.get()); + return mCodec->queueSecureInputBuffers( + index, + offset, + size, + auInfos, + cryptoInfos, + errorDetailMsg); +} + status_t JMediaCodec::queueBuffer( size_t index, const std::shared_ptr &buffer, const sp &infos, const sp &tunings, AString *errorDetailMsg) { @@ -2262,6 +2280,61 @@ struct NativeCryptoInfo { CryptoPlugin::Pattern mPattern; }; +// This class takes away all dependencies on java(env and jni) and +// could be used for taking cryptoInfo objects to MediaCodec. +struct MediaCodecCryptoInfo: public CodecCryptoInfo { + explicit MediaCodecCryptoInfo(const NativeCryptoInfo &cryptoInfo) { + if (cryptoInfo.mErr == OK) { + mNumSubSamples = cryptoInfo.mNumSubSamples; + mMode = cryptoInfo.mMode; + mPattern = cryptoInfo.mPattern; + if (cryptoInfo.mKey != nullptr) { + mKeyBuffer = ABuffer::CreateAsCopy(cryptoInfo.mKey, 16); + mKey = (uint8_t*)(mKeyBuffer.get() != nullptr ? mKeyBuffer.get()->data() : nullptr); + } + if (cryptoInfo.mIv != nullptr) { + mIvBuffer = ABuffer::CreateAsCopy(cryptoInfo.mIv, 16); + mIv = (uint8_t*)(mIvBuffer.get() != nullptr ? mIvBuffer.get()->data() : nullptr); + } + if (cryptoInfo.mSubSamples != nullptr) { + mSubSamplesBuffer = new ABuffer(sizeof(CryptoPlugin::SubSample) * mNumSubSamples); + if (mSubSamplesBuffer.get()) { + CryptoPlugin::SubSample * samples = + (CryptoPlugin::SubSample *)(mSubSamplesBuffer.get()->data()); + for (int s = 0 ; s < mNumSubSamples ; s++) { + samples[s].mNumBytesOfClearData = + cryptoInfo.mSubSamples[s].mNumBytesOfClearData; + samples[s].mNumBytesOfEncryptedData = + cryptoInfo.mSubSamples[s].mNumBytesOfEncryptedData; + } + mSubSamples = (CryptoPlugin::SubSample *)mSubSamplesBuffer.get()->data(); + } + } + + } + } + + explicit MediaCodecCryptoInfo(jint size) { + mSubSamplesBuffer = new ABuffer(sizeof(CryptoPlugin::SubSample) * 1); + mNumSubSamples = 1; + if (mSubSamplesBuffer.get()) { + CryptoPlugin::SubSample * samples = + (CryptoPlugin::SubSample *)(mSubSamplesBuffer.get()->data()); + samples[0].mNumBytesOfClearData = size; + samples[0].mNumBytesOfEncryptedData = 0; + mSubSamples = (CryptoPlugin::SubSample *)mSubSamplesBuffer.get()->data(); + } + } + ~MediaCodecCryptoInfo() {} + +protected: + // all backup buffers for the base object. + sp mKeyBuffer; + sp mIvBuffer; + sp mSubSamplesBuffer; + +}; + static void android_media_MediaCodec_queueSecureInputBuffer( JNIEnv *env, jobject thiz, @@ -2430,6 +2503,99 @@ static void android_media_MediaCodec_queueSecureInputBuffer( codec->getExceptionMessage(errorDetailMsg.c_str()).c_str(), codec->getCrypto()); } +static status_t extractCryptoInfosFromObjectArray(JNIEnv * const env, + jint * const totalSize, + std::vector> * const cryptoInfoObjs, + const jobjectArray &objArray, + AString * const errorDetailMsg) { + if (env == nullptr + || cryptoInfoObjs == nullptr + || totalSize == nullptr) { + if (errorDetailMsg) { + *errorDetailMsg = "Error: Null Parameters provided for extracting CryptoInfo"; + } + return BAD_VALUE; + } + const jsize numEntries = env->GetArrayLength(objArray); + if (numEntries <= 0) { + if (errorDetailMsg) { + *errorDetailMsg = "Error: No CryptoInfo found while queuing for large frame input"; + } + return BAD_VALUE; + } + cryptoInfoObjs->clear(); + *totalSize = 0; + jint size = 0; + for (jsize i = 0; i < numEntries ; i++) { + jobject param = env->GetObjectArrayElement(objArray, i); + if (param == NULL) { + if (errorDetailMsg) { + *errorDetailMsg = "Error: Null Parameters provided for extracting CryptoInfo"; + } + return BAD_VALUE; + } + NativeCryptoInfo nativeInfo(env, param); + std::unique_ptr info(new MediaCodecCryptoInfo(nativeInfo)); + for (int i = 0; i < info->mNumSubSamples; i++) { + size += info->mSubSamples[i].mNumBytesOfClearData; + size += info->mSubSamples[i].mNumBytesOfEncryptedData; + } + cryptoInfoObjs->push_back(std::move(info)); + } + *totalSize = size; + return OK; +} + + +static void android_media_MediaCodec_queueSecureInputBuffers( + JNIEnv *env, + jobject thiz, + jint index, + jobjectArray bufferInfosObjs, + jobjectArray cryptoInfoObjs) { + ALOGV("android_media_MediaCodec_queueSecureInputBuffers"); + + sp codec = getMediaCodec(env, thiz); + + if (codec == NULL || codec->initCheck() != OK) { + throwExceptionAsNecessary(env, INVALID_OPERATION, codec); + return; + } + sp auInfos = + new BufferInfosWrapper{decltype(auInfos->value)()}; + sp cryptoInfos = + new CryptoInfosWrapper{decltype(cryptoInfos->value)()}; + AString errorDetailMsg; + jint initialOffset = 0; + jint totalSize = 0; + status_t err = extractInfosFromObject( + env, + &initialOffset, + &totalSize, + &auInfos->value, + bufferInfosObjs, + &errorDetailMsg); + if (err == OK) { + err = extractCryptoInfosFromObjectArray(env, + &totalSize, + &cryptoInfos->value, + cryptoInfoObjs, + &errorDetailMsg); + } + if (err == OK) { + err = codec->queueSecureInputBuffers( + index, + initialOffset, + totalSize, + auInfos, + cryptoInfos, + &errorDetailMsg); + } + throwExceptionAsNecessary( + env, err, ACTION_CODE_FATAL, + codec->getExceptionMessage(errorDetailMsg.c_str()).c_str(), codec->getCrypto()); +} + static jobject android_media_MediaCodec_mapHardwareBuffer(JNIEnv *env, jclass, jobject bufferObj) { ALOGV("android_media_MediaCodec_mapHardwareBuffer"); AHardwareBuffer *hardwareBuffer = android_hardware_HardwareBuffer_getNativeHardwareBuffer( @@ -2780,8 +2946,8 @@ static void android_media_MediaCodec_native_queueLinearBlock( "error occurred while converting tunings from Java to native"); return; } - jint totalSize; - jint initialOffset; + jint totalSize = 0; + jint initialOffset = 0; std::vector infoVec; AString errorDetailMsg; err = extractInfosFromObject(env, @@ -3950,6 +4116,9 @@ static const JNINativeMethod gMethods[] = { { "native_queueSecureInputBuffer", "(IILandroid/media/MediaCodec$CryptoInfo;JI)V", (void *)android_media_MediaCodec_queueSecureInputBuffer }, + { "native_queueSecureInputBuffers", "(I[Ljava/lang/Object;[Ljava/lang/Object;)V", + (void *)android_media_MediaCodec_queueSecureInputBuffers }, + { "native_mapHardwareBuffer", "(Landroid/hardware/HardwareBuffer;)Landroid/media/Image;", (void *)android_media_MediaCodec_mapHardwareBuffer }, diff --git a/media/jni/android_media_MediaCodec.h b/media/jni/android_media_MediaCodec.h index 02708efdea3a..0663e4126b8a 100644 --- a/media/jni/android_media_MediaCodec.h +++ b/media/jni/android_media_MediaCodec.h @@ -114,6 +114,14 @@ struct JMediaCodec : public AHandler { uint32_t flags, AString *errorDetailMsg); + status_t queueSecureInputBuffers( + size_t index, + size_t offset, + size_t size, + const sp &auInfos, + const sp &cryptoInfos, + AString *errorDetailMsg); + status_t queueBuffer( size_t index, const std::shared_ptr &buffer, const sp &infos, const sp &tunings, -- cgit v1.2.3-59-g8ed1b From 677a881f474609c0d351b68ffe0e22e7c6373075 Mon Sep 17 00:00:00 2001 From: Arun Johnson Date: Tue, 6 Feb 2024 12:32:54 +0000 Subject: Add API setMultiFrameEncryptedLinearBlock for BLOCK_MODEL Bug: 298052174 API-Coverage-Bug: 309692716 Change-Id: I849dad124dc65cfd3a0c17d867a084778738fa42 --- core/api/current.txt | 1 + media/java/android/media/MediaCodec.java | 51 ++++++++++++++++++++++++++++---- media/jni/android_media_MediaCodec.cpp | 38 ++++++++++++++---------- media/jni/android_media_MediaCodec.h | 8 ++--- 4 files changed, 70 insertions(+), 28 deletions(-) diff --git a/core/api/current.txt b/core/api/current.txt index 28db56a69c36..9b272050ae49 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -22749,6 +22749,7 @@ package android.media { method @NonNull public android.media.MediaCodec.QueueRequest setIntegerParameter(@NonNull String, int); method @NonNull public android.media.MediaCodec.QueueRequest setLinearBlock(@NonNull android.media.MediaCodec.LinearBlock, int, int); method @NonNull public android.media.MediaCodec.QueueRequest setLongParameter(@NonNull String, long); + method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") @NonNull public android.media.MediaCodec.QueueRequest setMultiFrameEncryptedLinearBlock(@NonNull android.media.MediaCodec.LinearBlock, @NonNull java.util.ArrayDeque, @NonNull java.util.ArrayDeque); method @NonNull public android.media.MediaCodec.QueueRequest setPresentationTimeUs(long); method @NonNull public android.media.MediaCodec.QueueRequest setStringParameter(@NonNull String, @NonNull String); } diff --git a/media/java/android/media/MediaCodec.java b/media/java/android/media/MediaCodec.java index ba0b77015802..765145cb9b1d 100644 --- a/media/java/android/media/MediaCodec.java +++ b/media/java/android/media/MediaCodec.java @@ -3512,7 +3512,7 @@ final public class MediaCodec { mLinearBlock = block; mOffset = offset; mSize = size; - mCryptoInfo = null; + mCryptoInfos.clear(); return this; } @@ -3546,7 +3546,44 @@ final public class MediaCodec { mLinearBlock = block; mOffset = offset; mSize = size; - mCryptoInfo = cryptoInfo; + mCryptoInfos.clear(); + mCryptoInfos.add(cryptoInfo); + return this; + } + + /** + * Set an encrypted linear block to this queue request. Exactly one buffer must be + * set for a queue request before calling {@link #queue}. The block can contain multiple + * access units and if present should be laid out contiguously and without gaps. + * + * @param block The linear block object + * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the + * contents in the buffer. The ArrayDeque and the BufferInfo objects + * provided can be recycled by the caller for re-use. + * @param cryptoInfos ArrayDeque of {@link MediaCodec.CryptoInfo} that describes the + * structure of the encrypted input samples. The ArrayDeque and the + * BufferInfo objects provided can be recycled by the caller for re-use. + * @return this object + * @throws IllegalStateException if a buffer is already set + * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the + * access units are not contiguous. + */ + @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) + public @NonNull QueueRequest setMultiFrameEncryptedLinearBlock( + @NonNull LinearBlock block, + @NonNull ArrayDeque bufferInfos, + @NonNull ArrayDeque cryptoInfos) { + if (!isAccessible()) { + throw new IllegalStateException("The request is stale"); + } + if (mLinearBlock != null || mHardwareBuffer != null) { + throw new IllegalStateException("Cannot set block twice"); + } + mLinearBlock = block; + mBufferInfos.clear(); + mBufferInfos.addAll(bufferInfos); + mCryptoInfos.clear(); + mCryptoInfos.addAll(cryptoInfos); return this; } @@ -3756,8 +3793,10 @@ final public class MediaCodec { mBufferInfos.add(info); } if (mLinearBlock != null) { + mCodec.native_queueLinearBlock( - mIndex, mLinearBlock, mCryptoInfo, + mIndex, mLinearBlock, + mCryptoInfos.isEmpty() ? null : mCryptoInfos.toArray(), mBufferInfos.toArray(), mTuningKeys, mTuningValues); } else if (mHardwareBuffer != null) { @@ -3772,11 +3811,11 @@ final public class MediaCodec { mLinearBlock = null; mOffset = 0; mSize = 0; - mCryptoInfo = null; mHardwareBuffer = null; mPresentationTimeUs = 0; mFlags = 0; mBufferInfos.clear(); + mCryptoInfos.clear(); mTuningKeys.clear(); mTuningValues.clear(); return this; @@ -3796,11 +3835,11 @@ final public class MediaCodec { private LinearBlock mLinearBlock = null; private int mOffset = 0; private int mSize = 0; - private MediaCodec.CryptoInfo mCryptoInfo = null; private HardwareBuffer mHardwareBuffer = null; private long mPresentationTimeUs = 0; private @BufferFlag int mFlags = 0; private final ArrayDeque mBufferInfos = new ArrayDeque<>(); + private final ArrayDeque mCryptoInfos = new ArrayDeque<>(); private final ArrayList mTuningKeys = new ArrayList<>(); private final ArrayList mTuningValues = new ArrayList<>(); @@ -3810,7 +3849,7 @@ final public class MediaCodec { private native void native_queueLinearBlock( int index, @NonNull LinearBlock block, - @Nullable CryptoInfo cryptoInfo, + @Nullable Object[] cryptoInfos, @NonNull Object[] bufferInfos, @NonNull ArrayList keys, @NonNull ArrayList values); diff --git a/media/jni/android_media_MediaCodec.cpp b/media/jni/android_media_MediaCodec.cpp index fb7d90515aae..8396005b1b63 100644 --- a/media/jni/android_media_MediaCodec.cpp +++ b/media/jni/android_media_MediaCodec.cpp @@ -488,19 +488,16 @@ status_t JMediaCodec::queueEncryptedLinearBlock( size_t index, const sp &buffer, size_t offset, - const CryptoPlugin::SubSample *subSamples, - size_t numSubSamples, - const uint8_t key[16], - const uint8_t iv[16], - CryptoPlugin::Mode mode, - const CryptoPlugin::Pattern &pattern, + size_t size, const sp &infos, + const sp &cryptoInfos_, const sp &tunings, AString *errorDetailMsg) { sp auInfo((BufferInfosWrapper *)infos.get()); + sp cryptoInfos((CryptoInfosWrapper *)cryptoInfos_.get()); return mCodec->queueEncryptedBuffer( - index, buffer, offset, subSamples, numSubSamples, key, iv, mode, pattern, - auInfo, tunings, errorDetailMsg); + index, buffer, offset, size, auInfo, cryptoInfos, + tunings, errorDetailMsg); } status_t JMediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) { @@ -2928,7 +2925,7 @@ static void extractBufferFromContext( static void android_media_MediaCodec_native_queueLinearBlock( JNIEnv *env, jobject thiz, jint index, jobject bufferObj, - jobject cryptoInfoObj, jobjectArray objArray, jobject keys, jobject values) { + jobjectArray cryptoInfoArray, jobjectArray objArray, jobject keys, jobject values) { ALOGV("android_media_MediaCodec_native_queueLinearBlock"); sp codec = getMediaCodec(env, thiz); @@ -2998,8 +2995,19 @@ static void android_media_MediaCodec_native_queueLinearBlock( "MediaCodec.LinearBlock#obtain method to obtain a compatible buffer."); return; } - auto cryptoInfo = - cryptoInfoObj ? NativeCryptoInfo{env, cryptoInfoObj} : NativeCryptoInfo{totalSize}; + sp cryptoInfos = new CryptoInfosWrapper{decltype(cryptoInfos->value)()}; + jint sampleSize = 0; + if (cryptoInfoArray != nullptr) { + extractCryptoInfosFromObjectArray(env, + &sampleSize, + &cryptoInfos->value, + cryptoInfoArray, + &errorDetailMsg); + } else { + sampleSize = totalSize; + std::unique_ptr cryptoInfo{new MediaCodecCryptoInfo(totalSize)}; + cryptoInfos->value.push_back(std::move(cryptoInfo)); + } if (env->ExceptionCheck()) { // Creation of cryptoInfo failed. Let the exception bubble up. return; @@ -3008,11 +3016,9 @@ static void android_media_MediaCodec_native_queueLinearBlock( index, memory, initialOffset, - cryptoInfo.mSubSamples, cryptoInfo.mNumSubSamples, - (const uint8_t *)cryptoInfo.mKey, (const uint8_t *)cryptoInfo.mIv, - cryptoInfo.mMode, - cryptoInfo.mPattern, + sampleSize, infos, + cryptoInfos, tunings, &errorDetailMsg); ALOGI_IF(err != OK, "queueEncryptedLinearBlock returned err = %d", err); @@ -4126,7 +4132,7 @@ static const JNINativeMethod gMethods[] = { { "native_closeMediaImage", "(J)V", (void *)android_media_MediaCodec_closeMediaImage }, { "native_queueLinearBlock", - "(ILandroid/media/MediaCodec$LinearBlock;Landroid/media/MediaCodec$CryptoInfo;" + "(ILandroid/media/MediaCodec$LinearBlock;[Ljava/lang/Object;" "[Ljava/lang/Object;Ljava/util/ArrayList;Ljava/util/ArrayList;)V", (void *)android_media_MediaCodec_native_queueLinearBlock }, diff --git a/media/jni/android_media_MediaCodec.h b/media/jni/android_media_MediaCodec.h index 0663e4126b8a..abb23f516156 100644 --- a/media/jni/android_media_MediaCodec.h +++ b/media/jni/android_media_MediaCodec.h @@ -131,13 +131,9 @@ struct JMediaCodec : public AHandler { size_t index, const sp &buffer, size_t offset, - const CryptoPlugin::SubSample *subSamples, - size_t numSubSamples, - const uint8_t key[16], - const uint8_t iv[16], - CryptoPlugin::Mode mode, - const CryptoPlugin::Pattern &pattern, + size_t size, const sp &infos, + const sp &cryptoInfos, const sp &tunings, AString *errorDetailMsg); -- cgit v1.2.3-59-g8ed1b