diff options
46 files changed, 3180 insertions, 360 deletions
diff --git a/data/etc/Android.bp b/data/etc/Android.bp index 754e7b2ac0..226cae12aa 100644 --- a/data/etc/Android.bp +++ b/data/etc/Android.bp @@ -329,6 +329,12 @@ prebuilt_etc { } prebuilt_etc { + name: "android.software.opengles.deqp.level-2023-03-01.prebuilt.xml", + src: "android.software.opengles.deqp.level-2023-03-01.xml", + defaults: ["frameworks_native_data_etc_defaults"], +} + +prebuilt_etc { name: "android.software.sip.voip.prebuilt.xml", src: "android.software.sip.voip.xml", defaults: ["frameworks_native_data_etc_defaults"], @@ -353,6 +359,12 @@ prebuilt_etc { } prebuilt_etc { + name: "android.software.vulkan.deqp.level-2023-03-01.prebuilt.xml", + src: "android.software.vulkan.deqp.level-2023-03-01.xml", + defaults: ["frameworks_native_data_etc_defaults"], +} + +prebuilt_etc { name: "aosp_excluded_hardware.prebuilt.xml", src: "aosp_excluded_hardware.xml", defaults: ["frameworks_native_data_etc_defaults"], diff --git a/include/input/Input.h b/include/input/Input.h index fe0c775fd3..527a47741c 100644 --- a/include/input/Input.h +++ b/include/input/Input.h @@ -242,6 +242,19 @@ enum class ToolType { ftl_last = PALM, }; +/** + * The state of the key. This should have 1:1 correspondence with the values of anonymous enum + * defined in input.h + */ +enum class KeyState { + UNKNOWN = AKEY_STATE_UNKNOWN, + UP = AKEY_STATE_UP, + DOWN = AKEY_STATE_DOWN, + VIRTUAL = AKEY_STATE_VIRTUAL, + ftl_first = UNKNOWN, + ftl_last = VIRTUAL, +}; + bool isStylusToolType(ToolType toolType); /* diff --git a/libs/ui/Gralloc5.cpp b/libs/ui/Gralloc5.cpp index 21068394d2..c3b2d3d808 100644 --- a/libs/ui/Gralloc5.cpp +++ b/libs/ui/Gralloc5.cpp @@ -343,14 +343,17 @@ status_t Gralloc5Mapper::validateBufferSize(buffer_handle_t bufferHandle, uint32 return BAD_VALUE; } } - { - auto value = getStandardMetadata<StandardMetadataType::USAGE>(mMapper, bufferHandle); - if (static_cast<BufferUsage>(usage) != value) { - ALOGW("Usage didn't match, expected %" PRIu64 " got %" PRId64, usage, - static_cast<int64_t>(value.value_or(BufferUsage::CPU_READ_NEVER))); - return BAD_VALUE; - } - } + // TODO: This can false-positive fail if the allocator adjusted the USAGE bits internally + // Investigate further & re-enable or remove, but for now ignoring usage should be OK + (void)usage; + // { + // auto value = getStandardMetadata<StandardMetadataType::USAGE>(mMapper, bufferHandle); + // if (static_cast<BufferUsage>(usage) != value) { + // ALOGW("Usage didn't match, expected %" PRIu64 " got %" PRId64, usage, + // static_cast<int64_t>(value.value_or(BufferUsage::CPU_READ_NEVER))); + // return BAD_VALUE; + // } + // } { auto value = getStandardMetadata<StandardMetadataType::STRIDE>(mMapper, bufferHandle); if (stride != value) { diff --git a/libs/ultrahdr/fuzzer/Android.bp b/libs/ultrahdr/fuzzer/Android.bp new file mode 100644 index 0000000000..6c0a2f577c --- /dev/null +++ b/libs/ultrahdr/fuzzer/Android.bp @@ -0,0 +1,69 @@ +// Copyright 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "frameworks_native_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["frameworks_native_license"], +} + +cc_defaults { + name: "ultrahdr_fuzzer_defaults", + host_supported: true, + shared_libs: [ + "libimage_io", + "libjpeg", + ], + static_libs: [ + "libjpegdecoder", + "libjpegencoder", + "libultrahdr", + "libutils", + "liblog", + ], + target: { + darwin: { + enabled: false, + }, + }, + fuzz_config: { + cc: [ + "android-media-fuzzing-reports@google.com", + ], + description: "The fuzzers target the APIs of jpeg hdr", + service_privilege: "constrained", + users: "multi_user", + fuzzed_code_usage: "future_version", + vector: "local_no_privileges_required", + }, +} + +cc_fuzz { + name: "ultrahdr_enc_fuzzer", + defaults: ["ultrahdr_fuzzer_defaults"], + srcs: [ + "ultrahdr_enc_fuzzer.cpp", + ], +} + +cc_fuzz { + name: "ultrahdr_dec_fuzzer", + defaults: ["ultrahdr_fuzzer_defaults"], + srcs: [ + "ultrahdr_dec_fuzzer.cpp", + ], +} diff --git a/libs/ultrahdr/fuzzer/ultrahdr_dec_fuzzer.cpp b/libs/ultrahdr/fuzzer/ultrahdr_dec_fuzzer.cpp new file mode 100644 index 0000000000..ad1d57aaee --- /dev/null +++ b/libs/ultrahdr/fuzzer/ultrahdr_dec_fuzzer.cpp @@ -0,0 +1,73 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// System include files +#include <fuzzer/FuzzedDataProvider.h> +#include <iostream> +#include <vector> + +// User include files +#include "ultrahdr/jpegr.h" + +using namespace android::ultrahdr; + +// Transfer functions for image data, sync with ultrahdr.h +const int kOfMin = ULTRAHDR_OUTPUT_UNSPECIFIED + 1; +const int kOfMax = ULTRAHDR_OUTPUT_MAX; + +class UltraHdrDecFuzzer { +public: + UltraHdrDecFuzzer(const uint8_t* data, size_t size) : mFdp(data, size){}; + void process(); + +private: + FuzzedDataProvider mFdp; +}; + +void UltraHdrDecFuzzer::process() { + // hdr_of + auto of = static_cast<ultrahdr_output_format>(mFdp.ConsumeIntegralInRange<int>(kOfMin, kOfMax)); + auto buffer = mFdp.ConsumeRemainingBytes<uint8_t>(); + jpegr_compressed_struct jpegImgR{buffer.data(), (int)buffer.size(), (int)buffer.size(), + ULTRAHDR_COLORGAMUT_UNSPECIFIED}; + + std::vector<uint8_t> iccData(0); + std::vector<uint8_t> exifData(0); + jpegr_info_struct info{0, 0, &iccData, &exifData}; + JpegR jpegHdr; + (void)jpegHdr.getJPEGRInfo(&jpegImgR, &info); +//#define DUMP_PARAM +#ifdef DUMP_PARAM + std::cout << "input buffer size " << jpegImgR.length << std::endl; + std::cout << "image dimensions " << info.width << " x " << info.width << std::endl; +#endif + size_t outSize = info.width * info.height * ((of == ULTRAHDR_OUTPUT_SDR) ? 4 : 8); + jpegr_uncompressed_struct decodedJpegR; + auto decodedRaw = std::make_unique<uint8_t[]>(outSize); + decodedJpegR.data = decodedRaw.get(); + ultrahdr_metadata_struct metadata; + jpegr_uncompressed_struct decodedGainMap{}; + (void)jpegHdr.decodeJPEGR(&jpegImgR, &decodedJpegR, + mFdp.ConsumeFloatingPointInRange<float>(1.0, FLT_MAX), nullptr, of, + &decodedGainMap, &metadata); + if (decodedGainMap.data) free(decodedGainMap.data); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + UltraHdrDecFuzzer fuzzHandle(data, size); + fuzzHandle.process(); + return 0; +} diff --git a/libs/ultrahdr/fuzzer/ultrahdr_enc_fuzzer.cpp b/libs/ultrahdr/fuzzer/ultrahdr_enc_fuzzer.cpp new file mode 100644 index 0000000000..bbe58e0f2e --- /dev/null +++ b/libs/ultrahdr/fuzzer/ultrahdr_enc_fuzzer.cpp @@ -0,0 +1,303 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// System include files +#include <fuzzer/FuzzedDataProvider.h> +#include <algorithm> +#include <iostream> +#include <random> +#include <vector> + +// User include files +#include "ultrahdr/gainmapmath.h" +#include "ultrahdr/jpegencoderhelper.h" +#include "utils/Log.h" + +using namespace android::ultrahdr; + +// constants +const int kMinWidth = 8; +const int kMaxWidth = 7680; + +const int kMinHeight = 8; +const int kMaxHeight = 4320; + +const int kScaleFactor = 4; + +const int kJpegBlock = 16; + +// Color gamuts for image data, sync with ultrahdr.h +const int kCgMin = ULTRAHDR_COLORGAMUT_UNSPECIFIED + 1; +const int kCgMax = ULTRAHDR_COLORGAMUT_MAX; + +// Transfer functions for image data, sync with ultrahdr.h +const int kTfMin = ULTRAHDR_TF_UNSPECIFIED + 1; +const int kTfMax = ULTRAHDR_TF_PQ; + +// Transfer functions for image data, sync with ultrahdr.h +const int kOfMin = ULTRAHDR_OUTPUT_UNSPECIFIED + 1; +const int kOfMax = ULTRAHDR_OUTPUT_MAX; + +// quality factor +const int kQfMin = 0; +const int kQfMax = 100; + +class UltraHdrEncFuzzer { +public: + UltraHdrEncFuzzer(const uint8_t* data, size_t size) : mFdp(data, size){}; + void process(); + void fillP010Buffer(uint16_t* data, int width, int height, int stride); + void fill420Buffer(uint8_t* data, int size); + +private: + FuzzedDataProvider mFdp; +}; + +void UltraHdrEncFuzzer::fillP010Buffer(uint16_t* data, int width, int height, int stride) { + uint16_t* tmp = data; + std::vector<uint16_t> buffer(16); + for (int i = 0; i < buffer.size(); i++) { + buffer[i] = mFdp.ConsumeIntegralInRange<int>(0, (1 << 10) - 1); + } + for (int j = 0; j < height; j++) { + for (int i = 0; i < width; i += buffer.size()) { + memcpy(data + i, buffer.data(), std::min((int)buffer.size(), (width - i))); + std::shuffle(buffer.begin(), buffer.end(), + std::default_random_engine(std::random_device{}())); + } + tmp += stride; + } +} + +void UltraHdrEncFuzzer::fill420Buffer(uint8_t* data, int size) { + std::vector<uint8_t> buffer(16); + mFdp.ConsumeData(buffer.data(), buffer.size()); + for (int i = 0; i < size; i += buffer.size()) { + memcpy(data + i, buffer.data(), std::min((int)buffer.size(), (size - i))); + std::shuffle(buffer.begin(), buffer.end(), + std::default_random_engine(std::random_device{}())); + } +} + +void UltraHdrEncFuzzer::process() { + while (mFdp.remaining_bytes()) { + struct jpegr_uncompressed_struct p010Img {}; + struct jpegr_uncompressed_struct yuv420Img {}; + struct jpegr_uncompressed_struct grayImg {}; + struct jpegr_compressed_struct jpegImgR {}; + struct jpegr_compressed_struct jpegImg {}; + struct jpegr_compressed_struct jpegGainMap {}; + + // which encode api to select + int muxSwitch = mFdp.ConsumeIntegralInRange<int>(0, 4); + + // quality factor + int quality = mFdp.ConsumeIntegralInRange<int>(kQfMin, kQfMax); + + // hdr_tf + auto tf = static_cast<ultrahdr_transfer_function>( + mFdp.ConsumeIntegralInRange<int>(kTfMin, kTfMax)); + + // p010 Cg + auto p010Cg = + static_cast<ultrahdr_color_gamut>(mFdp.ConsumeIntegralInRange<int>(kCgMin, kCgMax)); + + // 420 Cg + auto yuv420Cg = + static_cast<ultrahdr_color_gamut>(mFdp.ConsumeIntegralInRange<int>(kCgMin, kCgMax)); + + // hdr_of + auto of = static_cast<ultrahdr_output_format>( + mFdp.ConsumeIntegralInRange<int>(kOfMin, kOfMax)); + + int width = mFdp.ConsumeIntegralInRange<int>(kMinWidth, kMaxWidth); + width = (width >> 1) << 1; + + int height = mFdp.ConsumeIntegralInRange<int>(kMinHeight, kMaxHeight); + height = (height >> 1) << 1; + + std::unique_ptr<uint16_t[]> bufferY = nullptr; + std::unique_ptr<uint16_t[]> bufferUV = nullptr; + std::unique_ptr<uint8_t[]> yuv420ImgRaw = nullptr; + std::unique_ptr<uint8_t[]> grayImgRaw = nullptr; + if (muxSwitch != 4) { + // init p010 image + bool isUVContiguous = mFdp.ConsumeBool(); + bool hasYStride = mFdp.ConsumeBool(); + int yStride = hasYStride ? mFdp.ConsumeIntegralInRange<int>(width, width + 128) : width; + p010Img.width = width; + p010Img.height = height; + p010Img.colorGamut = p010Cg; + p010Img.luma_stride = hasYStride ? yStride : 0; + int bppP010 = 2; + if (isUVContiguous) { + size_t p010Size = yStride * height * 3 / 2; + bufferY = std::make_unique<uint16_t[]>(p010Size); + p010Img.data = bufferY.get(); + p010Img.chroma_data = nullptr; + p010Img.chroma_stride = 0; + fillP010Buffer(bufferY.get(), width, height, yStride); + fillP010Buffer(bufferY.get() + yStride * height, width, height / 2, yStride); + } else { + int uvStride = mFdp.ConsumeIntegralInRange<int>(width, width + 128); + size_t p010YSize = yStride * height; + bufferY = std::make_unique<uint16_t[]>(p010YSize); + p010Img.data = bufferY.get(); + fillP010Buffer(bufferY.get(), width, height, yStride); + size_t p010UVSize = uvStride * p010Img.height / 2; + bufferUV = std::make_unique<uint16_t[]>(p010UVSize); + p010Img.chroma_data = bufferUV.get(); + p010Img.chroma_stride = uvStride; + fillP010Buffer(bufferUV.get(), width, height / 2, uvStride); + } + } else { + int map_width = width / kScaleFactor; + int map_height = height / kScaleFactor; + map_width = static_cast<size_t>(floor((map_width + kJpegBlock - 1) / kJpegBlock)) * + kJpegBlock; + map_height = ((map_height + 1) >> 1) << 1; + // init 400 image + grayImg.width = map_width; + grayImg.height = map_height; + grayImg.colorGamut = ULTRAHDR_COLORGAMUT_UNSPECIFIED; + + const size_t graySize = map_width * map_height; + grayImgRaw = std::make_unique<uint8_t[]>(graySize); + grayImg.data = grayImgRaw.get(); + fill420Buffer(grayImgRaw.get(), graySize); + grayImg.chroma_data = nullptr; + grayImg.luma_stride = 0; + grayImg.chroma_stride = 0; + } + + if (muxSwitch > 0) { + // init 420 image + yuv420Img.width = width; + yuv420Img.height = height; + yuv420Img.colorGamut = yuv420Cg; + + const size_t yuv420Size = (yuv420Img.width * yuv420Img.height * 3) / 2; + yuv420ImgRaw = std::make_unique<uint8_t[]>(yuv420Size); + yuv420Img.data = yuv420ImgRaw.get(); + fill420Buffer(yuv420ImgRaw.get(), yuv420Size); + yuv420Img.chroma_data = nullptr; + yuv420Img.luma_stride = 0; + yuv420Img.chroma_stride = 0; + } + + // dest + // 2 * p010 size as input data is random, DCT compression might not behave as expected + jpegImgR.maxLength = std::max(8 * 1024 /* min size 8kb */, width * height * 3 * 2); + auto jpegImgRaw = std::make_unique<uint8_t[]>(jpegImgR.maxLength); + jpegImgR.data = jpegImgRaw.get(); + +//#define DUMP_PARAM +#ifdef DUMP_PARAM + std::cout << "Api Select " << muxSwitch << std::endl; + std::cout << "image dimensions " << width << " x " << height << std::endl; + std::cout << "p010 color gamut " << p010Img.colorGamut << std::endl; + std::cout << "p010 luma stride " << p010Img.luma_stride << std::endl; + std::cout << "p010 chroma stride " << p010Img.chroma_stride << std::endl; + std::cout << "420 color gamut " << yuv420Img.colorGamut << std::endl; + std::cout << "quality factor " << quality << std::endl; +#endif + + JpegR jpegHdr; + android::status_t status = android::UNKNOWN_ERROR; + if (muxSwitch == 0) { // api 0 + jpegImgR.length = 0; + status = jpegHdr.encodeJPEGR(&p010Img, tf, &jpegImgR, quality, nullptr); + } else if (muxSwitch == 1) { // api 1 + jpegImgR.length = 0; + status = jpegHdr.encodeJPEGR(&p010Img, &yuv420Img, tf, &jpegImgR, quality, nullptr); + } else { + // compressed img + JpegEncoderHelper encoder; + if (encoder.compressImage(yuv420Img.data, yuv420Img.width, yuv420Img.height, quality, + nullptr, 0)) { + jpegImg.length = encoder.getCompressedImageSize(); + jpegImg.maxLength = jpegImg.length; + jpegImg.data = encoder.getCompressedImagePtr(); + jpegImg.colorGamut = yuv420Cg; + + if (muxSwitch == 2) { // api 2 + jpegImgR.length = 0; + status = jpegHdr.encodeJPEGR(&p010Img, &yuv420Img, &jpegImg, tf, &jpegImgR); + } else if (muxSwitch == 3) { // api 3 + jpegImgR.length = 0; + status = jpegHdr.encodeJPEGR(&p010Img, &jpegImg, tf, &jpegImgR); + } else if (muxSwitch == 4) { // api 4 + jpegImgR.length = 0; + JpegEncoderHelper gainMapEncoder; + if (gainMapEncoder.compressImage(grayImg.data, grayImg.width, grayImg.height, + quality, nullptr, 0, true)) { + jpegGainMap.length = gainMapEncoder.getCompressedImageSize(); + jpegGainMap.maxLength = jpegImg.length; + jpegGainMap.data = gainMapEncoder.getCompressedImagePtr(); + jpegGainMap.colorGamut = ULTRAHDR_COLORGAMUT_UNSPECIFIED; + ultrahdr_metadata_struct metadata; + metadata.version = "1.0"; + if (tf == ULTRAHDR_TF_HLG) { + metadata.maxContentBoost = kHlgMaxNits / kSdrWhiteNits; + } else if (tf == ULTRAHDR_TF_PQ) { + metadata.maxContentBoost = kPqMaxNits / kSdrWhiteNits; + } else { + metadata.maxContentBoost = 1.0f; + } + metadata.minContentBoost = 1.0f; + metadata.gamma = 1.0f; + metadata.offsetSdr = 0.0f; + metadata.offsetHdr = 0.0f; + metadata.hdrCapacityMin = 1.0f; + metadata.hdrCapacityMax = metadata.maxContentBoost; + status = jpegHdr.encodeJPEGR(&jpegImg, &jpegGainMap, &metadata, &jpegImgR); + } + } + } + } + if (status == android::OK) { + std::vector<uint8_t> iccData(0); + std::vector<uint8_t> exifData(0); + jpegr_info_struct info{0, 0, &iccData, &exifData}; + status = jpegHdr.getJPEGRInfo(&jpegImgR, &info); + if (status == android::OK) { + size_t outSize = info.width * info.height * ((of == ULTRAHDR_OUTPUT_SDR) ? 4 : 8); + jpegr_uncompressed_struct decodedJpegR; + auto decodedRaw = std::make_unique<uint8_t[]>(outSize); + decodedJpegR.data = decodedRaw.get(); + ultrahdr_metadata_struct metadata; + jpegr_uncompressed_struct decodedGainMap{}; + status = jpegHdr.decodeJPEGR(&jpegImgR, &decodedJpegR, + mFdp.ConsumeFloatingPointInRange<float>(1.0, FLT_MAX), + nullptr, of, &decodedGainMap, &metadata); + if (status != android::OK) { + ALOGE("encountered error during decoding %d", status); + } + if (decodedGainMap.data) free(decodedGainMap.data); + } else { + ALOGE("encountered error during get jpeg info %d", status); + } + } else { + ALOGE("encountered error during encoding %d", status); + } + } +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + UltraHdrEncFuzzer fuzzHandle(data, size); + fuzzHandle.process(); + return 0; +} diff --git a/libs/ultrahdr/gainmapmath.cpp b/libs/ultrahdr/gainmapmath.cpp index 37c3cf3d3b..ee15363b69 100644 --- a/libs/ultrahdr/gainmapmath.cpp +++ b/libs/ultrahdr/gainmapmath.cpp @@ -119,34 +119,39 @@ static float clampPixelFloat(float value) { return (value < 0.0f) ? 0.0f : (value > kMaxPixelFloat) ? kMaxPixelFloat : value; } -// See IEC 61966-2-1, Equation F.7. +// See IEC 61966-2-1/Amd 1:2003, Equation F.7. static const float kSrgbR = 0.2126f, kSrgbG = 0.7152f, kSrgbB = 0.0722f; float srgbLuminance(Color e) { return kSrgbR * e.r + kSrgbG * e.g + kSrgbB * e.b; } -// See ECMA TR/98, Section 7. -static const float kSrgbRCr = 1.402f, kSrgbGCb = 0.34414f, kSrgbGCr = 0.71414f, kSrgbBCb = 1.772f; +// See ITU-R BT.709-6, Section 3. +// Uses the same coefficients for deriving luma signal as +// IEC 61966-2-1/Amd 1:2003 states for luminance, so we reuse the luminance +// function above. +static const float kSrgbCb = 1.8556f, kSrgbCr = 1.5748f; -Color srgbYuvToRgb(Color e_gamma) { - return {{{ clampPixelFloat(e_gamma.y + kSrgbRCr * e_gamma.v), - clampPixelFloat(e_gamma.y - kSrgbGCb * e_gamma.u - kSrgbGCr * e_gamma.v), - clampPixelFloat(e_gamma.y + kSrgbBCb * e_gamma.u) }}}; +Color srgbRgbToYuv(Color e_gamma) { + float y_gamma = srgbLuminance(e_gamma); + return {{{ y_gamma, + (e_gamma.b - y_gamma) / kSrgbCb, + (e_gamma.r - y_gamma) / kSrgbCr }}}; } -// See ECMA TR/98, Section 7. -static const float kSrgbYR = 0.299f, kSrgbYG = 0.587f, kSrgbYB = 0.114f; -static const float kSrgbUR = -0.1687f, kSrgbUG = -0.3313f, kSrgbUB = 0.5f; -static const float kSrgbVR = 0.5f, kSrgbVG = -0.4187f, kSrgbVB = -0.0813f; +// See ITU-R BT.709-6, Section 3. +// Same derivation to BT.2100's YUV->RGB, below. Similar to srgbRgbToYuv, we +// can reuse the luminance coefficients since they are the same. +static const float kSrgbGCb = kSrgbB * kSrgbCb / kSrgbG; +static const float kSrgbGCr = kSrgbR * kSrgbCr / kSrgbG; -Color srgbRgbToYuv(Color e_gamma) { - return {{{ kSrgbYR * e_gamma.r + kSrgbYG * e_gamma.g + kSrgbYB * e_gamma.b, - kSrgbUR * e_gamma.r + kSrgbUG * e_gamma.g + kSrgbUB * e_gamma.b, - kSrgbVR * e_gamma.r + kSrgbVG * e_gamma.g + kSrgbVB * e_gamma.b }}}; +Color srgbYuvToRgb(Color e_gamma) { + return {{{ clampPixelFloat(e_gamma.y + kSrgbCr * e_gamma.v), + clampPixelFloat(e_gamma.y - kSrgbGCb * e_gamma.u - kSrgbGCr * e_gamma.v), + clampPixelFloat(e_gamma.y + kSrgbCb * e_gamma.u) }}}; } -// See IEC 61966-2-1, Equations F.5 and F.6. +// See IEC 61966-2-1/Amd 1:2003, Equations F.5 and F.6. float srgbInvOetf(float e_gamma) { if (e_gamma <= 0.04045f) { return e_gamma / 12.92f; @@ -178,13 +183,38 @@ Color srgbInvOetfLUT(Color e_gamma) { //////////////////////////////////////////////////////////////////////////////// // Display-P3 transformations -// See SMPTE EG 432-1, Table 7-2. +// See SMPTE EG 432-1, Equation 7-8. static const float kP3R = 0.20949f, kP3G = 0.72160f, kP3B = 0.06891f; float p3Luminance(Color e) { return kP3R * e.r + kP3G * e.g + kP3B * e.b; } +// See ITU-R BT.601-7, Sections 2.5.1 and 2.5.2. +// Unfortunately, calculation of luma signal differs from calculation of +// luminance for Display-P3, so we can't reuse p3Luminance here. +static const float kP3YR = 0.299f, kP3YG = 0.587f, kP3YB = 0.114f; +static const float kP3Cb = 1.772f, kP3Cr = 1.402f; + +Color p3RgbToYuv(Color e_gamma) { + float y_gamma = kP3YR * e_gamma.r + kP3YG * e_gamma.g + kP3YB * e_gamma.b; + return {{{ y_gamma, + (e_gamma.b - y_gamma) / kP3Cb, + (e_gamma.r - y_gamma) / kP3Cr }}}; +} + +// See ITU-R BT.601-7, Sections 2.5.1 and 2.5.2. +// Same derivation to BT.2100's YUV->RGB, below. Similar to p3RgbToYuv, we must +// use luma signal coefficients rather than the luminance coefficients. +static const float kP3GCb = kP3YB * kP3Cb / kP3YG; +static const float kP3GCr = kP3YR * kP3Cr / kP3YG; + +Color p3YuvToRgb(Color e_gamma) { + return {{{ clampPixelFloat(e_gamma.y + kP3Cr * e_gamma.v), + clampPixelFloat(e_gamma.y - kP3GCb * e_gamma.u - kP3GCr * e_gamma.v), + clampPixelFloat(e_gamma.y + kP3Cb * e_gamma.u) }}}; +} + //////////////////////////////////////////////////////////////////////////////// // BT.2100 transformations - according to ITU-R BT.2100-2 @@ -197,6 +227,8 @@ float bt2100Luminance(Color e) { } // See ITU-R BT.2100-2, Table 6, Derivation of colour difference signals. +// BT.2100 uses the same coefficients for calculating luma signal and luminance, +// so we reuse the luminance function here. static const float kBt2100Cb = 1.8814f, kBt2100Cr = 1.4746f; Color bt2100RgbToYuv(Color e_gamma) { @@ -206,6 +238,10 @@ Color bt2100RgbToYuv(Color e_gamma) { (e_gamma.r - y_gamma) / kBt2100Cr }}}; } +// See ITU-R BT.2100-2, Table 6, Derivation of colour difference signals. +// +// Similar to bt2100RgbToYuv above, we can reuse the luminance coefficients. +// // Derived by inversing bt2100RgbToYuv. The derivation for R and B are pretty // straight forward; we just invert the formulas for U and V above. But deriving // the formula for G is a bit more complicated: @@ -440,6 +476,85 @@ ColorTransformFn getHdrConversionFn(ultrahdr_color_gamut sdr_gamut, } } +// All of these conversions are derived from the respective input YUV->RGB conversion followed by +// the RGB->YUV for the receiving encoding. They are consistent with the RGB<->YUV functions in this +// file, given that we uses BT.709 encoding for sRGB and BT.601 encoding for Display-P3, to match +// DataSpace. + +Color yuv709To601(Color e_gamma) { + return {{{ 1.0f * e_gamma.y + 0.101579f * e_gamma.u + 0.196076f * e_gamma.v, + 0.0f * e_gamma.y + 0.989854f * e_gamma.u + -0.110653f * e_gamma.v, + 0.0f * e_gamma.y + -0.072453f * e_gamma.u + 0.983398f * e_gamma.v }}}; +} + +Color yuv709To2100(Color e_gamma) { + return {{{ 1.0f * e_gamma.y + -0.016969f * e_gamma.u + 0.096312f * e_gamma.v, + 0.0f * e_gamma.y + 0.995306f * e_gamma.u + -0.051192f * e_gamma.v, + 0.0f * e_gamma.y + 0.011507f * e_gamma.u + 1.002637f * e_gamma.v }}}; +} + +Color yuv601To709(Color e_gamma) { + return {{{ 1.0f * e_gamma.y + -0.118188f * e_gamma.u + -0.212685f * e_gamma.v, + 0.0f * e_gamma.y + 1.018640f * e_gamma.u + 0.114618f * e_gamma.v, + 0.0f * e_gamma.y + 0.075049f * e_gamma.u + 1.025327f * e_gamma.v }}}; +} + +Color yuv601To2100(Color e_gamma) { + return {{{ 1.0f * e_gamma.y + -0.128245f * e_gamma.u + -0.115879f * e_gamma.v, + 0.0f * e_gamma.y + 1.010016f * e_gamma.u + 0.061592f * e_gamma.v, + 0.0f * e_gamma.y + 0.086969f * e_gamma.u + 1.029350f * e_gamma.v }}}; +} + +Color yuv2100To709(Color e_gamma) { + return {{{ 1.0f * e_gamma.y + 0.018149f * e_gamma.u + -0.095132f * e_gamma.v, + 0.0f * e_gamma.y + 1.004123f * e_gamma.u + 0.051267f * e_gamma.v, + 0.0f * e_gamma.y + -0.011524f * e_gamma.u + 0.996782f * e_gamma.v }}}; +} + +Color yuv2100To601(Color e_gamma) { + return {{{ 1.0f * e_gamma.y + 0.117887f * e_gamma.u + 0.105521f * e_gamma.v, + 0.0f * e_gamma.y + 0.995211f * e_gamma.u + -0.059549f * e_gamma.v, + 0.0f * e_gamma.y + -0.084085f * e_gamma.u + 0.976518f * e_gamma.v }}}; +} + +void transformYuv420(jr_uncompressed_ptr image, size_t x_chroma, size_t y_chroma, + ColorTransformFn fn) { + Color yuv1 = getYuv420Pixel(image, x_chroma * 2, y_chroma * 2 ); + Color yuv2 = getYuv420Pixel(image, x_chroma * 2 + 1, y_chroma * 2 ); + Color yuv3 = getYuv420Pixel(image, x_chroma * 2, y_chroma * 2 + 1); + Color yuv4 = getYuv420Pixel(image, x_chroma * 2 + 1, y_chroma * 2 + 1); + + yuv1 = fn(yuv1); + yuv2 = fn(yuv2); + yuv3 = fn(yuv3); + yuv4 = fn(yuv4); + + Color new_uv = (yuv1 + yuv2 + yuv3 + yuv4) / 4.0f; + + size_t pixel_y1_idx = x_chroma * 2 + y_chroma * 2 * image->width; + size_t pixel_y2_idx = (x_chroma * 2 + 1) + y_chroma * 2 * image->width; + size_t pixel_y3_idx = x_chroma * 2 + (y_chroma * 2 + 1) * image->width; + size_t pixel_y4_idx = (x_chroma * 2 + 1) + (y_chroma * 2 + 1) * image->width; + + uint8_t& y1_uint = reinterpret_cast<uint8_t*>(image->data)[pixel_y1_idx]; + uint8_t& y2_uint = reinterpret_cast<uint8_t*>(image->data)[pixel_y2_idx]; + uint8_t& y3_uint = reinterpret_cast<uint8_t*>(image->data)[pixel_y3_idx]; + uint8_t& y4_uint = reinterpret_cast<uint8_t*>(image->data)[pixel_y4_idx]; + + size_t pixel_count = image->width * image->height; + size_t pixel_uv_idx = x_chroma + y_chroma * (image->width / 2); + + uint8_t& u_uint = reinterpret_cast<uint8_t*>(image->data)[pixel_count + pixel_uv_idx]; + uint8_t& v_uint = reinterpret_cast<uint8_t*>(image->data)[pixel_count * 5 / 4 + pixel_uv_idx]; + + y1_uint = static_cast<uint8_t>(floor(yuv1.y * 255.0f + 0.5f)); + y2_uint = static_cast<uint8_t>(floor(yuv2.y * 255.0f + 0.5f)); + y3_uint = static_cast<uint8_t>(floor(yuv3.y * 255.0f + 0.5f)); + y4_uint = static_cast<uint8_t>(floor(yuv4.y * 255.0f + 0.5f)); + + u_uint = static_cast<uint8_t>(floor(new_uv.u * 255.0f + 128.0f + 0.5f)); + v_uint = static_cast<uint8_t>(floor(new_uv.v * 255.0f + 128.0f + 0.5f)); +} //////////////////////////////////////////////////////////////////////////////// // Gain map calculations diff --git a/libs/ultrahdr/icc.cpp b/libs/ultrahdr/icc.cpp index c807705528..1ab3c7c793 100644 --- a/libs/ultrahdr/icc.cpp +++ b/libs/ultrahdr/icc.cpp @@ -14,6 +14,10 @@ * limitations under the License. */ +#ifndef USE_BIG_ENDIAN +#define USE_BIG_ENDIAN true +#endif + #include <ultrahdr/icc.h> #include <ultrahdr/gainmapmath.h> #include <vector> @@ -180,7 +184,7 @@ sp<DataStruct> IccHelper::write_text_tag(const char* text) { uint32_t total_length = text_length * 2 + sizeof(header); total_length = (((total_length + 2) >> 2) << 2); // 4 aligned - sp<DataStruct> dataStruct = new DataStruct(total_length); + sp<DataStruct> dataStruct = sp<DataStruct>::make(total_length); if (!dataStruct->write(header, sizeof(header))) { ALOGE("write_text_tag(): error in writing data"); @@ -204,7 +208,7 @@ sp<DataStruct> IccHelper::write_xyz_tag(float x, float y, float z) { static_cast<uint32_t>(Endian_SwapBE32(float_round_to_fixed(y))), static_cast<uint32_t>(Endian_SwapBE32(float_round_to_fixed(z))), }; - sp<DataStruct> dataStruct = new DataStruct(sizeof(data)); + sp<DataStruct> dataStruct = sp<DataStruct>::make(sizeof(data)); dataStruct->write(&data, sizeof(data)); return dataStruct; } @@ -212,7 +216,7 @@ sp<DataStruct> IccHelper::write_xyz_tag(float x, float y, float z) { sp<DataStruct> IccHelper::write_trc_tag(const int table_entries, const void* table_16) { int total_length = 4 + 4 + 4 + table_entries * 2; total_length = (((total_length + 2) >> 2) << 2); // 4 aligned - sp<DataStruct> dataStruct = new DataStruct(total_length); + sp<DataStruct> dataStruct = sp<DataStruct>::make(total_length); dataStruct->write32(Endian_SwapBE32(kTAG_CurveType)); // Type dataStruct->write32(0); // Reserved dataStruct->write32(Endian_SwapBE32(table_entries)); // Value count @@ -225,7 +229,7 @@ sp<DataStruct> IccHelper::write_trc_tag(const int table_entries, const void* tab sp<DataStruct> IccHelper::write_trc_tag_for_linear() { int total_length = 16; - sp<DataStruct> dataStruct = new DataStruct(total_length); + sp<DataStruct> dataStruct = sp<DataStruct>::make(total_length); dataStruct->write32(Endian_SwapBE32(kTAG_ParaCurveType)); // Type dataStruct->write32(0); // Reserved dataStruct->write32(Endian_SwapBE16(kExponential_ParaCurveType)); @@ -263,7 +267,7 @@ float IccHelper::compute_tone_map_gain(const ultrahdr_transfer_function tf, floa sp<DataStruct> IccHelper::write_cicp_tag(uint32_t color_primaries, uint32_t transfer_characteristics) { int total_length = 12; // 4 + 4 + 1 + 1 + 1 + 1 - sp<DataStruct> dataStruct = new DataStruct(total_length); + sp<DataStruct> dataStruct = sp<DataStruct>::make(total_length); dataStruct->write32(Endian_SwapBE32(kTAG_cicp)); // Type signature dataStruct->write32(0); // Reserved dataStruct->write8(color_primaries); // Color primaries @@ -314,7 +318,7 @@ sp<DataStruct> IccHelper::write_clut(const uint8_t* grid_points, const uint8_t* int total_length = 20 + 2 * value_count; total_length = (((total_length + 2) >> 2) << 2); // 4 aligned - sp<DataStruct> dataStruct = new DataStruct(total_length); + sp<DataStruct> dataStruct = sp<DataStruct>::make(total_length); for (size_t i = 0; i < 16; ++i) { dataStruct->write8(i < kNumChannels ? grid_points[i] : 0); // Grid size @@ -372,7 +376,7 @@ sp<DataStruct> IccHelper::write_mAB_or_mBA_tag(uint32_t type, total_length += a_curves_data[i]->getLength(); } } - sp<DataStruct> dataStruct = new DataStruct(total_length); + sp<DataStruct> dataStruct = sp<DataStruct>::make(total_length); dataStruct->write32(Endian_SwapBE32(type)); // Type signature dataStruct->write32(0); // Reserved dataStruct->write8(kNumChannels); // Input channels @@ -421,7 +425,7 @@ sp<DataStruct> IccHelper::writeIccProfile(ultrahdr_transfer_function tf, break; default: // Should not fall here. - return new DataStruct(0); + return nullptr; } // Compute primaries. @@ -540,13 +544,21 @@ sp<DataStruct> IccHelper::writeIccProfile(ultrahdr_transfer_function tf, size_t tag_table_size = kICCTagTableEntrySize * tags.size(); size_t profile_size = kICCHeaderSize + tag_table_size + tag_data_size; + sp<DataStruct> dataStruct = sp<DataStruct>::make(profile_size + kICCIdentifierSize); + + // Write identifier, chunk count, and chunk ID + if (!dataStruct->write(kICCIdentifier, sizeof(kICCIdentifier)) || + !dataStruct->write8(1) || !dataStruct->write8(1)) { + ALOGE("writeIccProfile(): error in identifier"); + return dataStruct; + } + // Write the header. header.data_color_space = Endian_SwapBE32(Signature_RGB); header.pcs = Endian_SwapBE32(tf == ULTRAHDR_TF_PQ ? Signature_Lab : Signature_XYZ); header.size = Endian_SwapBE32(profile_size); header.tag_count = Endian_SwapBE32(tags.size()); - sp<DataStruct> dataStruct = new DataStruct(profile_size); if (!dataStruct->write(&header, sizeof(header))) { ALOGE("writeIccProfile(): error in header"); return dataStruct; @@ -582,4 +594,84 @@ sp<DataStruct> IccHelper::writeIccProfile(ultrahdr_transfer_function tf, return dataStruct; } -} // namespace android::ultrahdr
\ No newline at end of file +bool IccHelper::tagsEqualToMatrix(const Matrix3x3& matrix, + const uint8_t* red_tag, + const uint8_t* green_tag, + const uint8_t* blue_tag) { + sp<DataStruct> red_tag_test = write_xyz_tag(matrix.vals[0][0], matrix.vals[1][0], + matrix.vals[2][0]); + sp<DataStruct> green_tag_test = write_xyz_tag(matrix.vals[0][1], matrix.vals[1][1], + matrix.vals[2][1]); + sp<DataStruct> blue_tag_test = write_xyz_tag(matrix.vals[0][2], matrix.vals[1][2], + matrix.vals[2][2]); + return memcmp(red_tag, red_tag_test->getData(), kColorantTagSize) == 0 && + memcmp(green_tag, green_tag_test->getData(), kColorantTagSize) == 0 && + memcmp(blue_tag, blue_tag_test->getData(), kColorantTagSize) == 0; +} + +ultrahdr_color_gamut IccHelper::readIccColorGamut(void* icc_data, size_t icc_size) { + // Each tag table entry consists of 3 fields of 4 bytes each. + static const size_t kTagTableEntrySize = 12; + + if (icc_data == nullptr || icc_size < sizeof(ICCHeader) + kICCIdentifierSize) { + return ULTRAHDR_COLORGAMUT_UNSPECIFIED; + } + + if (memcmp(icc_data, kICCIdentifier, sizeof(kICCIdentifier)) != 0) { + return ULTRAHDR_COLORGAMUT_UNSPECIFIED; + } + + uint8_t* icc_bytes = reinterpret_cast<uint8_t*>(icc_data) + kICCIdentifierSize; + + ICCHeader* header = reinterpret_cast<ICCHeader*>(icc_bytes); + + // Use 0 to indicate not found, since offsets are always relative to start + // of ICC data and therefore a tag offset of zero would never be valid. + size_t red_primary_offset = 0, green_primary_offset = 0, blue_primary_offset = 0; + size_t red_primary_size = 0, green_primary_size = 0, blue_primary_size = 0; + for (size_t tag_idx = 0; tag_idx < Endian_SwapBE32(header->tag_count); ++tag_idx) { + uint32_t* tag_entry_start = reinterpret_cast<uint32_t*>( + icc_bytes + sizeof(ICCHeader) + tag_idx * kTagTableEntrySize); + // first 4 bytes are the tag signature, next 4 bytes are the tag offset, + // last 4 bytes are the tag length in bytes. + if (red_primary_offset == 0 && *tag_entry_start == Endian_SwapBE32(kTAG_rXYZ)) { + red_primary_offset = Endian_SwapBE32(*(tag_entry_start+1)); + red_primary_size = Endian_SwapBE32(*(tag_entry_start+2)); + } else if (green_primary_offset == 0 && *tag_entry_start == Endian_SwapBE32(kTAG_gXYZ)) { + green_primary_offset = Endian_SwapBE32(*(tag_entry_start+1)); + green_primary_size = Endian_SwapBE32(*(tag_entry_start+2)); + } else if (blue_primary_offset == 0 && *tag_entry_start == Endian_SwapBE32(kTAG_bXYZ)) { + blue_primary_offset = Endian_SwapBE32(*(tag_entry_start+1)); + blue_primary_size = Endian_SwapBE32(*(tag_entry_start+2)); + } + } + + if (red_primary_offset == 0 || red_primary_size != kColorantTagSize || + kICCIdentifierSize + red_primary_offset + red_primary_size > icc_size || + green_primary_offset == 0 || green_primary_size != kColorantTagSize || + kICCIdentifierSize + green_primary_offset + green_primary_size > icc_size || + blue_primary_offset == 0 || blue_primary_size != kColorantTagSize || + kICCIdentifierSize + blue_primary_offset + blue_primary_size > icc_size) { + return ULTRAHDR_COLORGAMUT_UNSPECIFIED; + } + + uint8_t* red_tag = icc_bytes + red_primary_offset; + uint8_t* green_tag = icc_bytes + green_primary_offset; + uint8_t* blue_tag = icc_bytes + blue_primary_offset; + + // Serialize tags as we do on encode and compare what we find to that to + // determine the gamut (since we don't have a need yet for full deserialize). + if (tagsEqualToMatrix(kSRGB, red_tag, green_tag, blue_tag)) { + return ULTRAHDR_COLORGAMUT_BT709; + } else if (tagsEqualToMatrix(kDisplayP3, red_tag, green_tag, blue_tag)) { + return ULTRAHDR_COLORGAMUT_P3; + } else if (tagsEqualToMatrix(kRec2020, red_tag, green_tag, blue_tag)) { + return ULTRAHDR_COLORGAMUT_BT2100; + } + + // Didn't find a match to one of the profiles we write; indicate the gamut + // is unspecified since we don't understand it. + return ULTRAHDR_COLORGAMUT_UNSPECIFIED; +} + +} // namespace android::ultrahdr diff --git a/libs/ultrahdr/include/ultrahdr/gainmapmath.h b/libs/ultrahdr/include/ultrahdr/gainmapmath.h index abc93567f2..edf152d8ed 100644 --- a/libs/ultrahdr/include/ultrahdr/gainmapmath.h +++ b/libs/ultrahdr/include/ultrahdr/gainmapmath.h @@ -218,24 +218,30 @@ struct ShepardsIDW { // except for those concerning transfer functions. /* - * Calculate the luminance of a linear RGB sRGB pixel, according to IEC 61966-2-1. + * Calculate the luminance of a linear RGB sRGB pixel, according to + * IEC 61966-2-1/Amd 1:2003. * * [0.0, 1.0] range in and out. */ float srgbLuminance(Color e); /* - * Convert from OETF'd srgb YUV to RGB, according to ECMA TR/98. + * Convert from OETF'd srgb RGB to YUV, according to ITU-R BT.709-6. + * + * BT.709 YUV<->RGB matrix is used to match expectations for DataSpace. */ -Color srgbYuvToRgb(Color e_gamma); +Color srgbRgbToYuv(Color e_gamma); + /* - * Convert from OETF'd srgb RGB to YUV, according to ECMA TR/98. + * Convert from OETF'd srgb YUV to RGB, according to ITU-R BT.709-6. + * + * BT.709 YUV<->RGB matrix is used to match expectations for DataSpace. */ -Color srgbRgbToYuv(Color e_gamma); +Color srgbYuvToRgb(Color e_gamma); /* - * Convert from srgb to linear, according to IEC 61966-2-1. + * Convert from srgb to linear, according to IEC 61966-2-1/Amd 1:2003. * * [0.0, 1.0] range in and out. */ @@ -257,6 +263,20 @@ constexpr size_t kSrgbInvOETFNumEntries = 1 << kSrgbInvOETFPrecision; */ float p3Luminance(Color e); +/* + * Convert from OETF'd P3 RGB to YUV, according to ITU-R BT.601-7. + * + * BT.601 YUV<->RGB matrix is used to match expectations for DataSpace. + */ +Color p3RgbToYuv(Color e_gamma); + +/* + * Convert from OETF'd P3 YUV to RGB, according to ITU-R BT.601-7. + * + * BT.601 YUV<->RGB matrix is used to match expectations for DataSpace. + */ +Color p3YuvToRgb(Color e_gamma); + //////////////////////////////////////////////////////////////////////////////// // BT.2100 transformations - according to ITU-R BT.2100-2 @@ -269,12 +289,16 @@ float p3Luminance(Color e); float bt2100Luminance(Color e); /* - * Convert from OETF'd BT.2100 RGB to YUV. + * Convert from OETF'd BT.2100 RGB to YUV, according to ITU-R BT.2100-2. + * + * BT.2100 YUV<->RGB matrix is used to match expectations for DataSpace. */ Color bt2100RgbToYuv(Color e_gamma); /* - * Convert from OETF'd BT.2100 YUV to RGB. + * Convert from OETF'd BT.2100 YUV to RGB, according to ITU-R BT.2100-2. + * + * BT.2100 YUV<->RGB matrix is used to match expectations for DataSpace. */ Color bt2100YuvToRgb(Color e_gamma); @@ -358,6 +382,31 @@ inline Color identityConversion(Color e) { return e; } */ ColorTransformFn getHdrConversionFn(ultrahdr_color_gamut sdr_gamut, ultrahdr_color_gamut hdr_gamut); +/* + * Convert between YUV encodings, according to ITU-R BT.709-6, ITU-R BT.601-7, and ITU-R BT.2100-2. + * + * Bt.709 and Bt.2100 have well-defined YUV encodings; Display-P3's is less well defined, but is + * treated as Bt.601 by DataSpace, hence we do the same. + */ +Color yuv709To601(Color e_gamma); +Color yuv709To2100(Color e_gamma); +Color yuv601To709(Color e_gamma); +Color yuv601To2100(Color e_gamma); +Color yuv2100To709(Color e_gamma); +Color yuv2100To601(Color e_gamma); + +/* + * Performs a transformation at the chroma x and y coordinates provided on a YUV420 image. + * + * Apply the transformation by determining transformed YUV for each of the 4 Y + 1 UV; each Y gets + * this result, and UV gets the averaged result. + * + * x_chroma and y_chroma should be less than or equal to half the image's width and height + * respecitively, since input is 4:2:0 subsampled. + */ +void transformYuv420(jr_uncompressed_ptr image, size_t x_chroma, size_t y_chroma, + ColorTransformFn fn); + //////////////////////////////////////////////////////////////////////////////// // Gain map calculations @@ -365,6 +414,10 @@ ColorTransformFn getHdrConversionFn(ultrahdr_color_gamut sdr_gamut, ultrahdr_col /* * Calculate the 8-bit unsigned integer gain value for the given SDR and HDR * luminances in linear space, and the hdr ratio to encode against. + * + * Note: since this library always uses gamma of 1.0, offsetSdr of 0.0, and + * offsetHdr of 0.0, this function doesn't handle different metadata values for + * these fields. */ uint8_t encodeGain(float y_sdr, float y_hdr, ultrahdr_metadata_ptr metadata); uint8_t encodeGain(float y_sdr, float y_hdr, ultrahdr_metadata_ptr metadata, @@ -373,6 +426,10 @@ uint8_t encodeGain(float y_sdr, float y_hdr, ultrahdr_metadata_ptr metadata, /* * Calculates the linear luminance in nits after applying the given gain * value, with the given hdr ratio, to the given sdr input in the range [0, 1]. + * + * Note: similar to encodeGain(), this function only supports gamma 1.0, + * offsetSdr 0.0, offsetHdr 0.0, hdrCapacityMin 1.0, and hdrCapacityMax equal to + * gainMapMax, as this library encodes. */ Color applyGain(Color e, float gain, ultrahdr_metadata_ptr metadata); Color applyGain(Color e, float gain, ultrahdr_metadata_ptr metadata, float displayBoost); diff --git a/libs/ultrahdr/include/ultrahdr/icc.h b/libs/ultrahdr/include/ultrahdr/icc.h index 7f6ab882c6..7f047f8f5b 100644 --- a/libs/ultrahdr/include/ultrahdr/icc.h +++ b/libs/ultrahdr/include/ultrahdr/icc.h @@ -56,12 +56,16 @@ enum { Signature_XYZ = 0x58595A20, }; - typedef uint32_t FourByteTag; static inline constexpr FourByteTag SetFourByteTag(char a, char b, char c, char d) { return (((uint32_t)a << 24) | ((uint32_t)b << 16) | ((uint32_t)c << 8) | (uint32_t)d); } +static constexpr char kICCIdentifier[] = "ICC_PROFILE"; +// 12 for the actual identifier, +2 for the chunk count and chunk index which +// will always follow. +static constexpr size_t kICCIdentifierSize = 14; + // This is equal to the header size according to the ICC specification (128) // plus the size of the tag count (4). We include the tag count since we // always require it to be present anyway. @@ -70,6 +74,10 @@ static constexpr size_t kICCHeaderSize = 132; // Contains a signature (4), offset (4), and size (4). static constexpr size_t kICCTagTableEntrySize = 12; +// size should be 20; 4 bytes for type descriptor, 4 bytes reserved, 12 +// bytes for a single XYZ number type (4 bytes per coordinate). +static constexpr size_t kColorantTagSize = 20; + static constexpr uint32_t kDisplay_Profile = SetFourByteTag('m', 'n', 't', 'r'); static constexpr uint32_t kRGB_ColorSpace = SetFourByteTag('R', 'G', 'B', ' '); static constexpr uint32_t kXYZ_PCSSpace = SetFourByteTag('X', 'Y', 'Z', ' '); @@ -225,10 +233,23 @@ private: static void compute_lut_entry(const Matrix3x3& src_to_XYZD50, float rgb[3]); static sp<DataStruct> write_clut(const uint8_t* grid_points, const uint8_t* grid_16); + // Checks if a set of xyz tags is equivalent to a 3x3 Matrix. Each input + // tag buffer assumed to be at least kColorantTagSize in size. + static bool tagsEqualToMatrix(const Matrix3x3& matrix, + const uint8_t* red_tag, + const uint8_t* green_tag, + const uint8_t* blue_tag); + public: + // Output includes JPEG embedding identifier and chunk information, but not + // APPx information. static sp<DataStruct> writeIccProfile(const ultrahdr_transfer_function tf, const ultrahdr_color_gamut gamut); + // NOTE: this function is not robust; it can infer gamuts that IccHelper + // writes out but should not be considered a reference implementation for + // robust parsing of ICC profiles or their gamuts. + static ultrahdr_color_gamut readIccColorGamut(void* icc_data, size_t icc_size); }; } // namespace android::ultrahdr -#endif //ANDROID_ULTRAHDR_ICC_H
\ No newline at end of file +#endif //ANDROID_ULTRAHDR_ICC_H diff --git a/libs/ultrahdr/include/ultrahdr/jpegdecoderhelper.h b/libs/ultrahdr/include/ultrahdr/jpegdecoderhelper.h index f642bad89c..8b5499a2c0 100644 --- a/libs/ultrahdr/include/ultrahdr/jpegdecoderhelper.h +++ b/libs/ultrahdr/include/ultrahdr/jpegdecoderhelper.h @@ -25,6 +25,10 @@ extern "C" { } #include <utils/Errors.h> #include <vector> + +static const int kMaxWidth = 8192; +static const int kMaxHeight = 8192; + namespace android::ultrahdr { /* * Encapsulates a converter from JPEG to raw image (YUV420planer or grey-scale) format. @@ -79,11 +83,14 @@ public: */ size_t getEXIFSize(); /* - * Returns the position offset of EXIF package - * (4 bypes offset to FF sign, the byte after FF E1 XX XX <this byte>), - * or -1 if no EXIF exists. + * Returns the ICC data from the image. + */ + void* getICCPtr(); + /* + * Returns the decompressed ICC buffer size. This method must be called only after + * calling decompressImage() or getCompressedImageParameters(). */ - int getEXIFPos() { return mExifPos; } + size_t getICCSize(); /* * Decompresses metadata of the image. All vectors are owned by the caller. */ @@ -108,12 +115,12 @@ private: std::vector<JOCTET> mXMPBuffer; // The buffer that holds EXIF Data. std::vector<JOCTET> mEXIFBuffer; + // The buffer that holds ICC Data. + std::vector<JOCTET> mICCBuffer; // Resolution of the decompressed image. size_t mWidth; size_t mHeight; - // Position of EXIF package, default value is -1 which means no EXIF package appears. - size_t mExifPos; }; } /* namespace android::ultrahdr */ diff --git a/libs/ultrahdr/include/ultrahdr/jpegr.h b/libs/ultrahdr/include/ultrahdr/jpegr.h index 00b66aeaf6..a35fd30634 100644 --- a/libs/ultrahdr/include/ultrahdr/jpegr.h +++ b/libs/ultrahdr/include/ultrahdr/jpegr.h @@ -17,6 +17,7 @@ #ifndef ANDROID_ULTRAHDR_JPEGR_H #define ANDROID_ULTRAHDR_JPEGR_H +#include "jpegencoderhelper.h" #include "jpegrerrorcode.h" #include "ultrahdr.h" @@ -124,7 +125,7 @@ public: * * Generate gain map from the HDR and SDR inputs, compress SDR YUV to 8-bit JPEG and append * the gain map to the end of the compressed JPEG. HDR and SDR inputs must be the same - * resolution. + * resolution. SDR input is assumed to use the sRGB transfer function. * @param uncompressed_p010_image uncompressed HDR image in P010 color format * @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format * @param hdr_tf transfer function of the HDR image @@ -151,7 +152,9 @@ public: * This method requires HAL Hardware JPEG encoder. * * Generate gain map from the HDR and SDR inputs, append the gain map to the end of the - * compressed JPEG. HDR and SDR inputs must be the same resolution and color space. + * compressed JPEG. Adds an ICC profile if one isn't present in the input JPEG image. HDR and + * SDR inputs must be the same resolution and color space. SDR image is assumed to use the sRGB + * transfer function. * @param uncompressed_p010_image uncompressed HDR image in P010 color format * @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format * Note: the SDR image must be the decoded version of the JPEG @@ -177,8 +180,9 @@ public: * This method requires HAL Hardware JPEG encoder. * * Decode the compressed 8-bit JPEG image to YUV SDR, generate gain map from the HDR input - * and the decoded SDR result, append the gain map to the end of the compressed JPEG. HDR - * and SDR inputs must be the same resolution. + * and the decoded SDR result, append the gain map to the end of the compressed JPEG. Adds an + * ICC profile if one isn't present in the input JPEG image. HDR and SDR inputs must be the same + * resolution. JPEG image is assumed to use the sRGB transfer function. * @param uncompressed_p010_image uncompressed HDR image in P010 color format * @param compressed_jpeg_image compressed 8-bit JPEG image * @param hdr_tf transfer function of the HDR image @@ -197,7 +201,8 @@ public: * Encode API-4 * Assemble JPEGR image from SDR JPEG and gainmap JPEG. * - * Assemble the primary JPEG image, the gain map and the metadata to JPEG/R format. + * Assemble the primary JPEG image, the gain map and the metadata to JPEG/R format. Adds an ICC + * profile if one isn't present in the input JPEG image. * @param compressed_jpeg_image compressed 8-bit JPEG image * @param compressed_gainmap compressed 8-bit JPEG single channel image * @param metadata metadata to be written in XMP of the primary jpeg @@ -216,6 +221,13 @@ public: * Decode API * Decompress JPEGR image. * + * This method assumes that the JPEGR image contains an ICC profile with primaries that match + * those of a color gamut that this library is aware of; Bt.709, Display-P3, or Bt.2100. It also + * assumes the base image uses the sRGB transfer function. + * + * This method only supports single gain map metadata values for fields that allow multi-channel + * metadata values. + * * @param compressed_jpegr_image compressed JPEGR image. * @param dest destination of the uncompressed JPEGR image. * @param max_display_boost (optional) the maximum available boost supported by a display, @@ -257,6 +269,9 @@ public: /* * Gets Info from JPEGR file without decoding it. * + * This method only supports single gain map metadata values for fields that allow multi-channel + * metadata values. + * * The output is filled jpegr_info structure * @param compressed_jpegr_image compressed JPEGR image * @param jpegr_info pointer to output JPEGR info. Members of jpegr_info @@ -269,26 +284,30 @@ protected: /* * This method is called in the encoding pipeline. It will take the uncompressed 8-bit and * 10-bit yuv images as input, and calculate the uncompressed gain map. The input images - * must be the same resolution. + * must be the same resolution. The SDR input is assumed to use the sRGB transfer function. * * @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format * @param uncompressed_p010_image uncompressed HDR image in P010 color format * @param hdr_tf transfer function of the HDR image * @param dest gain map; caller responsible for memory of data * @param metadata max_content_boost is filled in + * @param sdr_is_601 if true, then use BT.601 decoding of YUV regardless of SDR image gamut * @return NO_ERROR if calculation succeeds, error code if error occurs. */ status_t generateGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, jr_uncompressed_ptr uncompressed_p010_image, ultrahdr_transfer_function hdr_tf, ultrahdr_metadata_ptr metadata, - jr_uncompressed_ptr dest); + jr_uncompressed_ptr dest, + bool sdr_is_601 = false); /* * This method is called in the decoding pipeline. It will take the uncompressed (decoded) * 8-bit yuv image, the uncompressed (decoded) gain map, and extracted JPEG/R metadata as * input, and calculate the 10-bit recovered image. The recovered output image is the same * color gamut as the SDR image, with HLG transfer function, and is in RGBA1010102 data format. + * The SDR image is assumed to use the sRGB transfer function. The SDR image is also assumed to + * be a decoded JPEG for the purpose of YUV interpration. * * @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format * @param uncompressed_gain_map uncompressed gain map @@ -312,11 +331,11 @@ private: * This method is called in the encoding pipeline. It will encode the gain map. * * @param uncompressed_gain_map uncompressed gain map - * @param dest encoded recover map + * @param resource to compress gain map * @return NO_ERROR if encoding succeeds, error code if error occurs. */ status_t compressGainMap(jr_uncompressed_ptr uncompressed_gain_map, - jr_compressed_ptr dest); + JpegEncoderHelper* jpeg_encoder); /* * This methoud is called to separate primary image and gain map image from JPEGR @@ -352,6 +371,8 @@ private: * @param compressed_jpeg_image compressed 8-bit JPEG image * @param compress_gain_map compressed recover map * @param (nullable) exif EXIF package + * @param (nullable) icc ICC package + * @param icc_size length in bytes of ICC package * @param metadata JPEG/R metadata to encode in XMP of the jpeg * @param dest compressed JPEGR image * @return NO_ERROR if calculation succeeds, error code if error occurs. @@ -359,6 +380,7 @@ private: status_t appendGainMap(jr_compressed_ptr compressed_jpeg_image, jr_compressed_ptr compressed_gain_map, jr_exif_ptr exif, + void* icc, size_t icc_size, ultrahdr_metadata_ptr metadata, jr_compressed_ptr dest); @@ -373,6 +395,22 @@ private: jr_uncompressed_ptr dest); /* + * This method will convert a YUV420 image from one YUV encoding to another in-place (eg. + * Bt.709 to Bt.601 YUV encoding). + * + * src_encoding and dest_encoding indicate the encoding via the YUV conversion defined for that + * gamut. P3 indicates Rec.601, since this is how DataSpace encodes Display-P3 YUV data. + * + * @param image the YUV420 image to convert + * @param src_encoding input YUV encoding + * @param dest_encoding output YUV encoding + * @return NO_ERROR if calculation succeeds, error code if error occurs. + */ + status_t convertYuv(jr_uncompressed_ptr image, + ultrahdr_color_gamut src_encoding, + ultrahdr_color_gamut dest_encoding); + + /* * This method will check the validity of the input arguments. * * @param uncompressed_p010_image uncompressed HDR image in P010 color format diff --git a/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h b/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h index 9f59c3eaf3..064123210f 100644 --- a/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h +++ b/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h @@ -42,6 +42,8 @@ enum { ERROR_JPEGR_BUFFER_TOO_SMALL = JPEGR_IO_ERROR_BASE - 4, ERROR_JPEGR_INVALID_COLORGAMUT = JPEGR_IO_ERROR_BASE - 5, ERROR_JPEGR_INVALID_TRANS_FUNC = JPEGR_IO_ERROR_BASE - 6, + ERROR_JPEGR_INVALID_METADATA = JPEGR_IO_ERROR_BASE - 7, + ERROR_JPEGR_UNSUPPORTED_METADATA = JPEGR_IO_ERROR_BASE - 8, JPEGR_RUNTIME_ERROR_BASE = -20000, ERROR_JPEGR_ENCODE_ERROR = JPEGR_RUNTIME_ERROR_BASE - 1, diff --git a/libs/ultrahdr/include/ultrahdr/ultrahdr.h b/libs/ultrahdr/include/ultrahdr/ultrahdr.h index e87a025867..17cc97173c 100644 --- a/libs/ultrahdr/include/ultrahdr/ultrahdr.h +++ b/libs/ultrahdr/include/ultrahdr/ultrahdr.h @@ -20,7 +20,7 @@ namespace android::ultrahdr { // Color gamuts for image data typedef enum { - ULTRAHDR_COLORGAMUT_UNSPECIFIED, + ULTRAHDR_COLORGAMUT_UNSPECIFIED = -1, ULTRAHDR_COLORGAMUT_BT709, ULTRAHDR_COLORGAMUT_P3, ULTRAHDR_COLORGAMUT_BT2100, @@ -39,22 +39,38 @@ typedef enum { // Target output formats for decoder typedef enum { + ULTRAHDR_OUTPUT_UNSPECIFIED = -1, ULTRAHDR_OUTPUT_SDR, // SDR in RGBA_8888 color format ULTRAHDR_OUTPUT_HDR_LINEAR, // HDR in F16 color format (linear) ULTRAHDR_OUTPUT_HDR_PQ, // HDR in RGBA_1010102 color format (PQ transfer function) ULTRAHDR_OUTPUT_HDR_HLG, // HDR in RGBA_1010102 color format (HLG transfer function) + ULTRAHDR_OUTPUT_MAX = ULTRAHDR_OUTPUT_HDR_HLG, } ultrahdr_output_format; /* * Holds information for gain map related metadata. + * + * Not: all values stored in linear. This differs from the metadata encoding in XMP, where + * maxContentBoost (aka gainMapMax), minContentBoost (aka gainMapMin), hdrCapacityMin, and + * hdrCapacityMax are stored in log2 space. */ struct ultrahdr_metadata_struct { - // Ultra HDR library version - const char* version; + // Ultra HDR format version + std::string version; // Max Content Boost for the map float maxContentBoost; // Min Content Boost for the map float minContentBoost; + // Gamma of the map data + float gamma; + // Offset for SDR data in map calculations + float offsetSdr; + // Offset for HDR data in map calculations + float offsetHdr; + // HDR capacity to apply the map at all + float hdrCapacityMin; + // HDR capacity to apply the map completely + float hdrCapacityMax; }; typedef struct ultrahdr_metadata_struct* ultrahdr_metadata_ptr; diff --git a/libs/ultrahdr/jpegdecoderhelper.cpp b/libs/ultrahdr/jpegdecoderhelper.cpp index 12217b7906..fef544452a 100644 --- a/libs/ultrahdr/jpegdecoderhelper.cpp +++ b/libs/ultrahdr/jpegdecoderhelper.cpp @@ -26,6 +26,8 @@ using namespace std; namespace android::ultrahdr { +#define ALIGNM(x, m) ((((x) + ((m) - 1)) / (m)) * (m)) + const uint32_t kAPP0Marker = JPEG_APP0; // JFIF const uint32_t kAPP1Marker = JPEG_APP0 + 1; // EXIF, XMP const uint32_t kAPP2Marker = JPEG_APP0 + 2; // ICC @@ -91,7 +93,6 @@ static void jpegrerror_exit(j_common_ptr cinfo) { } JpegDecoderHelper::JpegDecoderHelper() { - mExifPos = 0; } JpegDecoderHelper::~JpegDecoderHelper() { @@ -136,6 +137,14 @@ size_t JpegDecoderHelper::getEXIFSize() { return mEXIFBuffer.size(); } +void* JpegDecoderHelper::getICCPtr() { + return mICCBuffer.data(); +} + +size_t JpegDecoderHelper::getICCSize() { + return mICCBuffer.size(); +} + size_t JpegDecoderHelper::getDecompressedImageWidth() { return mWidth; } @@ -148,6 +157,7 @@ bool JpegDecoderHelper::decode(const void* image, int length, bool decodeToRGBA) jpeg_decompress_struct cinfo; jpegr_source_mgr mgr(static_cast<const uint8_t*>(image), length); jpegrerror_mgr myerr; + bool status = true; cinfo.err = jpeg_std_error(&myerr.pub); myerr.pub.error_exit = jpegrerror_exit; @@ -165,31 +175,21 @@ bool JpegDecoderHelper::decode(const void* image, int length, bool decodeToRGBA) cinfo.src = &mgr; jpeg_read_header(&cinfo, TRUE); - // Save XMP data and EXIF data. - // Here we only handle the first XMP / EXIF package. - // The parameter pos is used for capturing start offset of EXIF, which is hacky, but working... + // Save XMP data, EXIF data, and ICC data. + // Here we only handle the first XMP / EXIF / ICC package. // We assume that all packages are starting with two bytes marker (eg FF E1 for EXIF package), // two bytes of package length which is stored in marker->original_length, and the real data - // which is stored in marker->data. The pos is adding up all previous package lengths ( - // 4 bytes marker and length, marker->original_length) before EXIF appears. Note that here we - // we are using marker->original_length instead of marker->data_length because in case the real - // package length is larger than the limitation, jpeg-turbo will only copy the data within the - // limitation (represented by data_length) and this may vary from original_length / real offset. - // A better solution is making jpeg_marker_struct holding the offset, but currently it doesn't. + // which is stored in marker->data. bool exifAppears = false; bool xmpAppears = false; - size_t pos = 2; // position after SOI + bool iccAppears = false; for (jpeg_marker_struct* marker = cinfo.marker_list; - marker && !(exifAppears && xmpAppears); + marker && !(exifAppears && xmpAppears && iccAppears); marker = marker->next) { - pos += 4; - pos += marker->original_length; - - if (marker->marker != kAPP1Marker) { + if (marker->marker != kAPP1Marker && marker->marker != kAPP2Marker) { continue; } - const unsigned int len = marker->data_length; if (!xmpAppears && len > kXmpNameSpace.size() && @@ -207,24 +207,47 @@ bool JpegDecoderHelper::decode(const void* image, int length, bool decodeToRGBA) mEXIFBuffer.resize(len, 0); memcpy(static_cast<void*>(mEXIFBuffer.data()), marker->data, len); exifAppears = true; - mExifPos = pos - marker->original_length; + } else if (!iccAppears && + len > sizeof(kICCSig) && + !memcmp(marker->data, kICCSig, sizeof(kICCSig))) { + mICCBuffer.resize(len, 0); + memcpy(static_cast<void*>(mICCBuffer.data()), marker->data, len); + iccAppears = true; } } + if (cinfo.image_width > kMaxWidth || cinfo.image_height > kMaxHeight) { + // constraint on max width and max height is only due to alloc constraints + // tune these values basing on the target device + status = false; + goto CleanUp; + } + mWidth = cinfo.image_width; mHeight = cinfo.image_height; if (decodeToRGBA) { if (cinfo.jpeg_color_space == JCS_GRAYSCALE) { // We don't intend to support decoding grayscale to RGBA - return false; + status = false; + ALOGE("%s: decoding grayscale to RGBA is unsupported", __func__); + goto CleanUp; } // 4 bytes per pixel mResultBuffer.resize(cinfo.image_width * cinfo.image_height * 4); cinfo.out_color_space = JCS_EXT_RGBA; } else { if (cinfo.jpeg_color_space == JCS_YCbCr) { - // 1 byte per pixel for Y, 0.5 byte per pixel for U+V + if (cinfo.comp_info[0].h_samp_factor != 2 || + cinfo.comp_info[1].h_samp_factor != 1 || + cinfo.comp_info[2].h_samp_factor != 1 || + cinfo.comp_info[0].v_samp_factor != 2 || + cinfo.comp_info[1].v_samp_factor != 1 || + cinfo.comp_info[2].v_samp_factor != 1) { + status = false; + ALOGE("%s: decoding to YUV only supports 4:2:0 subsampling", __func__); + goto CleanUp; + } mResultBuffer.resize(cinfo.image_width * cinfo.image_height * 3 / 2, 0); } else if (cinfo.jpeg_color_space == JCS_GRAYSCALE) { mResultBuffer.resize(cinfo.image_width * cinfo.image_height, 0); @@ -239,13 +262,15 @@ bool JpegDecoderHelper::decode(const void* image, int length, bool decodeToRGBA) if (!decompress(&cinfo, static_cast<const uint8_t*>(mResultBuffer.data()), cinfo.jpeg_color_space == JCS_GRAYSCALE)) { - return false; + status = false; + goto CleanUp; } +CleanUp: jpeg_finish_decompress(&cinfo); jpeg_destroy_decompress(&cinfo); - return true; + return status; } bool JpegDecoderHelper::decompress(jpeg_decompress_struct* cinfo, const uint8_t* dest, @@ -283,8 +308,12 @@ bool JpegDecoderHelper::getCompressedImageParameters(const void* image, int leng return false; } - *pWidth = cinfo.image_width; - *pHeight = cinfo.image_height; + if (pWidth != nullptr) { + *pWidth = cinfo.image_width; + } + if (pHeight != nullptr) { + *pHeight = cinfo.image_height; + } if (iccData != nullptr) { for (jpeg_marker_struct* marker = cinfo.marker_list; marker; @@ -297,9 +326,7 @@ bool JpegDecoderHelper::getCompressedImageParameters(const void* image, int leng continue; } - const unsigned int len = marker->data_length - kICCMarkerHeaderSize; - const uint8_t *src = marker->data + kICCMarkerHeaderSize; - iccData->insert(iccData->end(), src, src+len); + iccData->insert(iccData->end(), marker->data, marker->data + marker->data_length); } } @@ -342,7 +369,6 @@ bool JpegDecoderHelper::decompressRGBA(jpeg_decompress_struct* cinfo, const uint } bool JpegDecoderHelper::decompressYUV(jpeg_decompress_struct* cinfo, const uint8_t* dest) { - JSAMPROW y[kCompressBatchSize]; JSAMPROW cb[kCompressBatchSize / 2]; JSAMPROW cr[kCompressBatchSize / 2]; @@ -353,9 +379,35 @@ bool JpegDecoderHelper::decompressYUV(jpeg_decompress_struct* cinfo, const uint8 uint8_t* y_plane = const_cast<uint8_t*>(dest); uint8_t* u_plane = const_cast<uint8_t*>(dest + y_plane_size); uint8_t* v_plane = const_cast<uint8_t*>(dest + y_plane_size + uv_plane_size); - std::unique_ptr<uint8_t[]> empty(new uint8_t[cinfo->image_width]); + std::unique_ptr<uint8_t[]> empty = std::make_unique<uint8_t[]>(cinfo->image_width); memset(empty.get(), 0, cinfo->image_width); + const int aligned_width = ALIGNM(cinfo->image_width, kCompressBatchSize); + bool is_width_aligned = (aligned_width == cinfo->image_width); + std::unique_ptr<uint8_t[]> buffer_intrm = nullptr; + uint8_t* y_plane_intrm = nullptr; + uint8_t* u_plane_intrm = nullptr; + uint8_t* v_plane_intrm = nullptr; + JSAMPROW y_intrm[kCompressBatchSize]; + JSAMPROW cb_intrm[kCompressBatchSize / 2]; + JSAMPROW cr_intrm[kCompressBatchSize / 2]; + JSAMPARRAY planes_intrm[3] {y_intrm, cb_intrm, cr_intrm}; + if (!is_width_aligned) { + size_t mcu_row_size = aligned_width * kCompressBatchSize * 3 / 2; + buffer_intrm = std::make_unique<uint8_t[]>(mcu_row_size); + y_plane_intrm = buffer_intrm.get(); + u_plane_intrm = y_plane_intrm + (aligned_width * kCompressBatchSize); + v_plane_intrm = u_plane_intrm + (aligned_width * kCompressBatchSize) / 4; + for (int i = 0; i < kCompressBatchSize; ++i) { + y_intrm[i] = y_plane_intrm + i * aligned_width; + } + for (int i = 0; i < kCompressBatchSize / 2; ++i) { + int offset_intrm = i * (aligned_width / 2); + cb_intrm[i] = u_plane_intrm + offset_intrm; + cr_intrm[i] = v_plane_intrm + offset_intrm; + } + } + while (cinfo->output_scanline < cinfo->image_height) { for (int i = 0; i < kCompressBatchSize; ++i) { size_t scanline = cinfo->output_scanline + i; @@ -377,11 +429,21 @@ bool JpegDecoderHelper::decompressYUV(jpeg_decompress_struct* cinfo, const uint8 } } - int processed = jpeg_read_raw_data(cinfo, planes, kCompressBatchSize); + int processed = jpeg_read_raw_data(cinfo, is_width_aligned ? planes : planes_intrm, + kCompressBatchSize); if (processed != kCompressBatchSize) { ALOGE("Number of processed lines does not equal input lines."); return false; } + if (!is_width_aligned) { + for (int i = 0; i < kCompressBatchSize; ++i) { + memcpy(y[i], y_intrm[i], cinfo->image_width); + } + for (int i = 0; i < kCompressBatchSize / 2; ++i) { + memcpy(cb[i], cb_intrm[i], cinfo->image_width / 2); + memcpy(cr[i], cr_intrm[i], cinfo->image_width / 2); + } + } } return true; } @@ -391,9 +453,24 @@ bool JpegDecoderHelper::decompressSingleChannel(jpeg_decompress_struct* cinfo, c JSAMPARRAY planes[1] {y}; uint8_t* y_plane = const_cast<uint8_t*>(dest); - std::unique_ptr<uint8_t[]> empty(new uint8_t[cinfo->image_width]); + std::unique_ptr<uint8_t[]> empty = std::make_unique<uint8_t[]>(cinfo->image_width); memset(empty.get(), 0, cinfo->image_width); + int aligned_width = ALIGNM(cinfo->image_width, kCompressBatchSize); + bool is_width_aligned = (aligned_width == cinfo->image_width); + std::unique_ptr<uint8_t[]> buffer_intrm = nullptr; + uint8_t* y_plane_intrm = nullptr; + JSAMPROW y_intrm[kCompressBatchSize]; + JSAMPARRAY planes_intrm[1] {y_intrm}; + if (!is_width_aligned) { + size_t mcu_row_size = aligned_width * kCompressBatchSize; + buffer_intrm = std::make_unique<uint8_t[]>(mcu_row_size); + y_plane_intrm = buffer_intrm.get(); + for (int i = 0; i < kCompressBatchSize; ++i) { + y_intrm[i] = y_plane_intrm + i * aligned_width; + } + } + while (cinfo->output_scanline < cinfo->image_height) { for (int i = 0; i < kCompressBatchSize; ++i) { size_t scanline = cinfo->output_scanline + i; @@ -404,11 +481,17 @@ bool JpegDecoderHelper::decompressSingleChannel(jpeg_decompress_struct* cinfo, c } } - int processed = jpeg_read_raw_data(cinfo, planes, kCompressBatchSize); + int processed = jpeg_read_raw_data(cinfo, is_width_aligned ? planes : planes_intrm, + kCompressBatchSize); if (processed != kCompressBatchSize / 2) { ALOGE("Number of processed lines does not equal input lines."); return false; } + if (!is_width_aligned) { + for (int i = 0; i < kCompressBatchSize; ++i) { + memcpy(y[i], y_intrm[i], cinfo->image_width); + } + } } return true; } diff --git a/libs/ultrahdr/jpegencoderhelper.cpp b/libs/ultrahdr/jpegencoderhelper.cpp index 10a763035f..a03547b538 100644 --- a/libs/ultrahdr/jpegencoderhelper.cpp +++ b/libs/ultrahdr/jpegencoderhelper.cpp @@ -22,6 +22,8 @@ namespace android::ultrahdr { +#define ALIGNM(x, m) ((((x) + ((m) - 1)) / (m)) * (m)) + // The destination manager that can access |mResultBuffer| in JpegEncoderHelper. struct destination_mgr { public: @@ -105,12 +107,11 @@ bool JpegEncoderHelper::encode(const void* image, int width, int height, int jpe jpeg_write_marker(&cinfo, JPEG_APP0 + 2, static_cast<const JOCTET*>(iccBuffer), iccSize); } - if (!compress(&cinfo, static_cast<const uint8_t*>(image), isSingleChannel)) { - return false; - } + bool status = compress(&cinfo, static_cast<const uint8_t*>(image), isSingleChannel); jpeg_finish_compress(&cinfo); jpeg_destroy_compress(&cinfo); - return true; + + return status; } void JpegEncoderHelper::setJpegDestination(jpeg_compress_struct* cinfo) { @@ -172,9 +173,40 @@ bool JpegEncoderHelper::compressYuv(jpeg_compress_struct* cinfo, const uint8_t* uint8_t* y_plane = const_cast<uint8_t*>(yuv); uint8_t* u_plane = const_cast<uint8_t*>(yuv + y_plane_size); uint8_t* v_plane = const_cast<uint8_t*>(yuv + y_plane_size + uv_plane_size); - std::unique_ptr<uint8_t[]> empty(new uint8_t[cinfo->image_width]); + std::unique_ptr<uint8_t[]> empty = std::make_unique<uint8_t[]>(cinfo->image_width); memset(empty.get(), 0, cinfo->image_width); + const int aligned_width = ALIGNM(cinfo->image_width, kCompressBatchSize); + const bool is_width_aligned = (aligned_width == cinfo->image_width); + std::unique_ptr<uint8_t[]> buffer_intrm = nullptr; + uint8_t* y_plane_intrm = nullptr; + uint8_t* u_plane_intrm = nullptr; + uint8_t* v_plane_intrm = nullptr; + JSAMPROW y_intrm[kCompressBatchSize]; + JSAMPROW cb_intrm[kCompressBatchSize / 2]; + JSAMPROW cr_intrm[kCompressBatchSize / 2]; + JSAMPARRAY planes_intrm[3]{y_intrm, cb_intrm, cr_intrm}; + if (!is_width_aligned) { + size_t mcu_row_size = aligned_width * kCompressBatchSize * 3 / 2; + buffer_intrm = std::make_unique<uint8_t[]>(mcu_row_size); + y_plane_intrm = buffer_intrm.get(); + u_plane_intrm = y_plane_intrm + (aligned_width * kCompressBatchSize); + v_plane_intrm = u_plane_intrm + (aligned_width * kCompressBatchSize) / 4; + for (int i = 0; i < kCompressBatchSize; ++i) { + y_intrm[i] = y_plane_intrm + i * aligned_width; + memset(y_intrm[i] + cinfo->image_width, 0, aligned_width - cinfo->image_width); + } + for (int i = 0; i < kCompressBatchSize / 2; ++i) { + int offset_intrm = i * (aligned_width / 2); + cb_intrm[i] = u_plane_intrm + offset_intrm; + cr_intrm[i] = v_plane_intrm + offset_intrm; + memset(cb_intrm[i] + cinfo->image_width / 2, 0, + (aligned_width - cinfo->image_width) / 2); + memset(cr_intrm[i] + cinfo->image_width / 2, 0, + (aligned_width - cinfo->image_width) / 2); + } + } + while (cinfo->next_scanline < cinfo->image_height) { for (int i = 0; i < kCompressBatchSize; ++i) { size_t scanline = cinfo->next_scanline + i; @@ -183,6 +215,9 @@ bool JpegEncoderHelper::compressYuv(jpeg_compress_struct* cinfo, const uint8_t* } else { y[i] = empty.get(); } + if (!is_width_aligned) { + memcpy(y_intrm[i], y[i], cinfo->image_width); + } } // cb, cr only have half scanlines for (int i = 0; i < kCompressBatchSize / 2; ++i) { @@ -194,9 +229,13 @@ bool JpegEncoderHelper::compressYuv(jpeg_compress_struct* cinfo, const uint8_t* } else { cb[i] = cr[i] = empty.get(); } + if (!is_width_aligned) { + memcpy(cb_intrm[i], cb[i], cinfo->image_width / 2); + memcpy(cr_intrm[i], cr[i], cinfo->image_width / 2); + } } - - int processed = jpeg_write_raw_data(cinfo, planes, kCompressBatchSize); + int processed = jpeg_write_raw_data(cinfo, is_width_aligned ? planes : planes_intrm, + kCompressBatchSize); if (processed != kCompressBatchSize) { ALOGE("Number of processed lines does not equal input lines."); return false; @@ -210,9 +249,26 @@ bool JpegEncoderHelper::compressSingleChannel(jpeg_compress_struct* cinfo, const JSAMPARRAY planes[1] {y}; uint8_t* y_plane = const_cast<uint8_t*>(image); - std::unique_ptr<uint8_t[]> empty(new uint8_t[cinfo->image_width]); + std::unique_ptr<uint8_t[]> empty = std::make_unique<uint8_t[]>(cinfo->image_width); memset(empty.get(), 0, cinfo->image_width); + const int aligned_width = ALIGNM(cinfo->image_width, kCompressBatchSize); + bool is_width_aligned = (aligned_width == cinfo->image_width); + std::unique_ptr<uint8_t[]> buffer_intrm = nullptr; + uint8_t* y_plane_intrm = nullptr; + uint8_t* u_plane_intrm = nullptr; + JSAMPROW y_intrm[kCompressBatchSize]; + JSAMPARRAY planes_intrm[]{y_intrm}; + if (!is_width_aligned) { + size_t mcu_row_size = aligned_width * kCompressBatchSize; + buffer_intrm = std::make_unique<uint8_t[]>(mcu_row_size); + y_plane_intrm = buffer_intrm.get(); + for (int i = 0; i < kCompressBatchSize; ++i) { + y_intrm[i] = y_plane_intrm + i * aligned_width; + memset(y_intrm[i] + cinfo->image_width, 0, aligned_width - cinfo->image_width); + } + } + while (cinfo->next_scanline < cinfo->image_height) { for (int i = 0; i < kCompressBatchSize; ++i) { size_t scanline = cinfo->next_scanline + i; @@ -221,8 +277,12 @@ bool JpegEncoderHelper::compressSingleChannel(jpeg_compress_struct* cinfo, const } else { y[i] = empty.get(); } + if (!is_width_aligned) { + memcpy(y_intrm[i], y[i], cinfo->image_width); + } } - int processed = jpeg_write_raw_data(cinfo, planes, kCompressBatchSize); + int processed = jpeg_write_raw_data(cinfo, is_width_aligned ? planes : planes_intrm, + kCompressBatchSize); if (processed != kCompressBatchSize / 2) { ALOGE("Number of processed lines does not equal input lines."); return false; diff --git a/libs/ultrahdr/jpegr.cpp b/libs/ultrahdr/jpegr.cpp index da257266ee..9c57f34c2a 100644 --- a/libs/ultrahdr/jpegr.cpp +++ b/libs/ultrahdr/jpegr.cpp @@ -65,13 +65,20 @@ static const char* const kJpegrVersion = "1.0"; // Map is quarter res / sixteenth size static const size_t kMapDimensionScaleFactor = 4; + +// Gain Map width is (image_width / kMapDimensionScaleFactor). If we were to +// compress 420 GainMap in jpeg, then we need at least 2 samples. For Grayscale +// 1 sample is sufficient. We are using 2 here anyways +static const int kMinWidth = 2 * kMapDimensionScaleFactor; +static const int kMinHeight = 2 * kMapDimensionScaleFactor; + // JPEG block size. // JPEG encoding / decoding will require block based DCT transform 16 x 16 for luma, // and 8 x 8 for chroma. // Width must be 16 dividable for luma, and 8 dividable for chroma. -// If this criteria is not ficilitated, we will pad zeros based on the required block size. +// If this criteria is not facilitated, we will pad zeros based to each line on the +// required block size. static const size_t kJpegBlock = JpegEncoderHelper::kCompressBatchSize; -static const size_t kJpegBlockSquare = kJpegBlock * kJpegBlock; // JPEG compress quality (0 ~ 100) for gain map static const int kMapCompressQuality = 85; @@ -105,10 +112,17 @@ status_t JpegR::areInputArgumentsValid(jr_uncompressed_ptr uncompressed_p010_ima return ERROR_JPEGR_INVALID_INPUT_TYPE; } - if (uncompressed_p010_image->width == 0 - || uncompressed_p010_image->height == 0) { - ALOGE("Image dimensions cannot be zero, image dimensions %dx%d", - uncompressed_p010_image->width, uncompressed_p010_image->height); + if (uncompressed_p010_image->width < kMinWidth + || uncompressed_p010_image->height < kMinHeight) { + ALOGE("Image dimensions cannot be less than %dx%d, image dimensions %dx%d", + kMinWidth, kMinHeight, uncompressed_p010_image->width, uncompressed_p010_image->height); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + + if (uncompressed_p010_image->width > kMaxWidth + || uncompressed_p010_image->height > kMaxHeight) { + ALOGE("Image dimensions cannot be larger than %dx%d, image dimensions %dx%d", + kMaxWidth, kMaxHeight, uncompressed_p010_image->width, uncompressed_p010_image->height); return ERROR_JPEGR_INVALID_INPUT_TYPE; } @@ -138,7 +152,8 @@ status_t JpegR::areInputArgumentsValid(jr_uncompressed_ptr uncompressed_p010_ima return ERROR_JPEGR_INVALID_NULL_PTR; } - if (hdr_tf <= ULTRAHDR_TF_UNSPECIFIED || hdr_tf > ULTRAHDR_TF_MAX) { + if (hdr_tf <= ULTRAHDR_TF_UNSPECIFIED || hdr_tf > ULTRAHDR_TF_MAX + || hdr_tf == ULTRAHDR_TF_SRGB) { ALOGE("Invalid hdr transfer function %d", hdr_tf); return ERROR_JPEGR_INVALID_INPUT_TYPE; } @@ -221,13 +236,8 @@ status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, metadata.version = kJpegrVersion; jpegr_uncompressed_struct uncompressed_yuv_420_image; - size_t gain_map_length = uncompressed_p010_image->width * uncompressed_p010_image->height * 3 / 2; - // Pad a pseudo chroma block (kJpegBlock / 2) x (kJpegBlock / 2) - // if width is not kJpegBlock aligned. - if (uncompressed_p010_image->width % kJpegBlock != 0) { - gain_map_length += kJpegBlockSquare / 4; - } - unique_ptr<uint8_t[]> uncompressed_yuv_420_image_data = make_unique<uint8_t[]>(gain_map_length); + unique_ptr<uint8_t[]> uncompressed_yuv_420_image_data = make_unique<uint8_t[]>( + uncompressed_p010_image->width * uncompressed_p010_image->height * 3 / 2); uncompressed_yuv_420_image.data = uncompressed_yuv_420_image_data.get(); JPEGR_CHECK(toneMap(uncompressed_p010_image, &uncompressed_yuv_420_image)); @@ -237,15 +247,21 @@ status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, std::unique_ptr<uint8_t[]> map_data; map_data.reset(reinterpret_cast<uint8_t*>(map.data)); + JpegEncoderHelper jpeg_encoder_gainmap; + JPEGR_CHECK(compressGainMap(&map, &jpeg_encoder_gainmap)); jpegr_compressed_struct compressed_map; - compressed_map.maxLength = map.width * map.height; - unique_ptr<uint8_t[]> compressed_map_data = make_unique<uint8_t[]>(compressed_map.maxLength); - compressed_map.data = compressed_map_data.get(); - JPEGR_CHECK(compressGainMap(&map, &compressed_map)); + compressed_map.maxLength = jpeg_encoder_gainmap.getCompressedImageSize(); + compressed_map.length = compressed_map.maxLength; + compressed_map.data = jpeg_encoder_gainmap.getCompressedImagePtr(); + compressed_map.colorGamut = ULTRAHDR_COLORGAMUT_UNSPECIFIED; sp<DataStruct> icc = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, uncompressed_yuv_420_image.colorGamut); + // Convert to Bt601 YUV encoding for JPEG encode + JPEGR_CHECK(convertYuv(&uncompressed_yuv_420_image, uncompressed_yuv_420_image.colorGamut, + ULTRAHDR_COLORGAMUT_P3)); + JpegEncoderHelper jpeg_encoder; if (!jpeg_encoder.compressImage(uncompressed_yuv_420_image.data, uncompressed_yuv_420_image.width, @@ -257,7 +273,9 @@ status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, jpeg.data = jpeg_encoder.getCompressedImagePtr(); jpeg.length = jpeg_encoder.getCompressedImageSize(); - JPEGR_CHECK(appendGainMap(&jpeg, &compressed_map, exif, &metadata, dest)); + // No ICC since JPEG encode already did it + JPEGR_CHECK(appendGainMap(&jpeg, &compressed_map, exif, /* icc */ nullptr, /* icc size */ 0, + &metadata, dest)); return NO_ERROR; } @@ -294,19 +312,33 @@ status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, std::unique_ptr<uint8_t[]> map_data; map_data.reset(reinterpret_cast<uint8_t*>(map.data)); + JpegEncoderHelper jpeg_encoder_gainmap; + JPEGR_CHECK(compressGainMap(&map, &jpeg_encoder_gainmap)); jpegr_compressed_struct compressed_map; - compressed_map.maxLength = map.width * map.height; - unique_ptr<uint8_t[]> compressed_map_data = make_unique<uint8_t[]>(compressed_map.maxLength); - compressed_map.data = compressed_map_data.get(); - JPEGR_CHECK(compressGainMap(&map, &compressed_map)); + compressed_map.maxLength = jpeg_encoder_gainmap.getCompressedImageSize(); + compressed_map.length = compressed_map.maxLength; + compressed_map.data = jpeg_encoder_gainmap.getCompressedImagePtr(); + compressed_map.colorGamut = ULTRAHDR_COLORGAMUT_UNSPECIFIED; sp<DataStruct> icc = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, uncompressed_yuv_420_image->colorGamut); + // Convert to Bt601 YUV encoding for JPEG encode; make a copy so as to no clobber client data + unique_ptr<uint8_t[]> yuv_420_bt601_data = make_unique<uint8_t[]>( + uncompressed_yuv_420_image->width * uncompressed_yuv_420_image->height * 3 / 2); + memcpy(yuv_420_bt601_data.get(), uncompressed_yuv_420_image->data, + uncompressed_yuv_420_image->width * uncompressed_yuv_420_image->height * 3 / 2); + + jpegr_uncompressed_struct yuv_420_bt601_image = { + yuv_420_bt601_data.get(), uncompressed_yuv_420_image->width, uncompressed_yuv_420_image->height, + uncompressed_yuv_420_image->colorGamut }; + JPEGR_CHECK(convertYuv(&yuv_420_bt601_image, yuv_420_bt601_image.colorGamut, + ULTRAHDR_COLORGAMUT_P3)); + JpegEncoderHelper jpeg_encoder; - if (!jpeg_encoder.compressImage(uncompressed_yuv_420_image->data, - uncompressed_yuv_420_image->width, - uncompressed_yuv_420_image->height, quality, + if (!jpeg_encoder.compressImage(yuv_420_bt601_image.data, + yuv_420_bt601_image.width, + yuv_420_bt601_image.height, quality, icc->getData(), icc->getLength())) { return ERROR_JPEGR_ENCODE_ERROR; } @@ -314,7 +346,9 @@ status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, jpeg.data = jpeg_encoder.getCompressedImagePtr(); jpeg.length = jpeg_encoder.getCompressedImageSize(); - JPEGR_CHECK(appendGainMap(&jpeg, &compressed_map, exif, &metadata, dest)); + // No ICC since jpeg encode already did it + JPEGR_CHECK(appendGainMap(&jpeg, &compressed_map, exif, /* icc */ nullptr, /* icc size */ 0, + &metadata, dest)); return NO_ERROR; } @@ -349,13 +383,32 @@ status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, std::unique_ptr<uint8_t[]> map_data; map_data.reset(reinterpret_cast<uint8_t*>(map.data)); + JpegEncoderHelper jpeg_encoder_gainmap; + JPEGR_CHECK(compressGainMap(&map, &jpeg_encoder_gainmap)); jpegr_compressed_struct compressed_map; - compressed_map.maxLength = map.width * map.height; - unique_ptr<uint8_t[]> compressed_map_data = make_unique<uint8_t[]>(compressed_map.maxLength); - compressed_map.data = compressed_map_data.get(); - JPEGR_CHECK(compressGainMap(&map, &compressed_map)); - - JPEGR_CHECK(appendGainMap(compressed_jpeg_image, &compressed_map, nullptr, &metadata, dest)); + compressed_map.maxLength = jpeg_encoder_gainmap.getCompressedImageSize(); + compressed_map.length = compressed_map.maxLength; + compressed_map.data = jpeg_encoder_gainmap.getCompressedImagePtr(); + compressed_map.colorGamut = ULTRAHDR_COLORGAMUT_UNSPECIFIED; + + // We just want to check if ICC is present, so don't do a full decode. Note, + // this doesn't verify that the ICC is valid. + JpegDecoderHelper decoder; + std::vector<uint8_t> icc; + decoder.getCompressedImageParameters(compressed_jpeg_image->data, compressed_jpeg_image->length, + /* pWidth */ nullptr, /* pHeight */ nullptr, + &icc, /* exifData */ nullptr); + + // Add ICC if not already present. + if (icc.size() > 0) { + JPEGR_CHECK(appendGainMap(compressed_jpeg_image, &compressed_map, /* exif */ nullptr, + /* icc */ nullptr, /* icc size */ 0, &metadata, dest)); + } else { + sp<DataStruct> newIcc = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, + uncompressed_yuv_420_image->colorGamut); + JPEGR_CHECK(appendGainMap(compressed_jpeg_image, &compressed_map, /* exif */ nullptr, + newIcc->getData(), newIcc->getLength(), &metadata, dest)); + } return NO_ERROR; } @@ -376,6 +429,7 @@ status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, return ret; } + // Note: output is Bt.601 YUV encoded regardless of gamut, due to jpeg decode. JpegDecoderHelper jpeg_decoder; if (!jpeg_decoder.decompressImage(compressed_jpeg_image->data, compressed_jpeg_image->length)) { return ERROR_JPEGR_DECODE_ERROR; @@ -395,18 +449,39 @@ status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, metadata.version = kJpegrVersion; jpegr_uncompressed_struct map; + // Indicate that the SDR image is Bt.601 YUV encoded. JPEGR_CHECK(generateGainMap( - &uncompressed_yuv_420_image, uncompressed_p010_image, hdr_tf, &metadata, &map)); + &uncompressed_yuv_420_image, uncompressed_p010_image, hdr_tf, &metadata, &map, + true /* sdr_is_601 */ )); std::unique_ptr<uint8_t[]> map_data; map_data.reset(reinterpret_cast<uint8_t*>(map.data)); + JpegEncoderHelper jpeg_encoder_gainmap; + JPEGR_CHECK(compressGainMap(&map, &jpeg_encoder_gainmap)); jpegr_compressed_struct compressed_map; - compressed_map.maxLength = map.width * map.height; - unique_ptr<uint8_t[]> compressed_map_data = make_unique<uint8_t[]>(compressed_map.maxLength); - compressed_map.data = compressed_map_data.get(); - JPEGR_CHECK(compressGainMap(&map, &compressed_map)); - - JPEGR_CHECK(appendGainMap(compressed_jpeg_image, &compressed_map, nullptr, &metadata, dest)); + compressed_map.maxLength = jpeg_encoder_gainmap.getCompressedImageSize(); + compressed_map.length = compressed_map.maxLength; + compressed_map.data = jpeg_encoder_gainmap.getCompressedImagePtr(); + compressed_map.colorGamut = ULTRAHDR_COLORGAMUT_UNSPECIFIED; + + // We just want to check if ICC is present, so don't do a full decode. Note, + // this doesn't verify that the ICC is valid. + JpegDecoderHelper decoder; + std::vector<uint8_t> icc; + decoder.getCompressedImageParameters(compressed_jpeg_image->data, compressed_jpeg_image->length, + /* pWidth */ nullptr, /* pHeight */ nullptr, + &icc, /* exifData */ nullptr); + + // Add ICC if not already present. + if (icc.size() > 0) { + JPEGR_CHECK(appendGainMap(compressed_jpeg_image, &compressed_map, /* exif */ nullptr, + /* icc */ nullptr, /* icc size */ 0, &metadata, dest)); + } else { + sp<DataStruct> newIcc = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, + uncompressed_yuv_420_image.colorGamut); + JPEGR_CHECK(appendGainMap(compressed_jpeg_image, &compressed_map, /* exif */ nullptr, + newIcc->getData(), newIcc->getLength(), &metadata, dest)); + } return NO_ERROR; } @@ -431,8 +506,25 @@ status_t JpegR::encodeJPEGR(jr_compressed_ptr compressed_jpeg_image, return ERROR_JPEGR_INVALID_NULL_PTR; } - JPEGR_CHECK(appendGainMap(compressed_jpeg_image, compressed_gainmap, /* exif */ nullptr, - metadata, dest)); + // We just want to check if ICC is present, so don't do a full decode. Note, + // this doesn't verify that the ICC is valid. + JpegDecoderHelper decoder; + std::vector<uint8_t> icc; + decoder.getCompressedImageParameters(compressed_jpeg_image->data, compressed_jpeg_image->length, + /* pWidth */ nullptr, /* pHeight */ nullptr, + &icc, /* exifData */ nullptr); + + // Add ICC if not already present. + if (icc.size() > 0) { + JPEGR_CHECK(appendGainMap(compressed_jpeg_image, compressed_gainmap, /* exif */ nullptr, + /* icc */ nullptr, /* icc size */ 0, metadata, dest)); + } else { + sp<DataStruct> newIcc = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, + compressed_jpeg_image->colorGamut); + JPEGR_CHECK(appendGainMap(compressed_jpeg_image, compressed_gainmap, /* exif */ nullptr, + newIcc->getData(), newIcc->getLength(), metadata, dest)); + } + return NO_ERROR; } @@ -469,12 +561,29 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, ultrahdr_output_format output_format, jr_uncompressed_ptr gain_map, ultrahdr_metadata_ptr metadata) { - if (compressed_jpegr_image == nullptr || dest == nullptr) { + if (compressed_jpegr_image == nullptr || compressed_jpegr_image->data == nullptr) { + ALOGE("received nullptr for compressed jpegr image"); + return ERROR_JPEGR_INVALID_NULL_PTR; + } + + if (dest == nullptr || dest->data == nullptr) { + ALOGE("received nullptr for dest image"); return ERROR_JPEGR_INVALID_NULL_PTR; } if (max_display_boost < 1.0f) { - return ERROR_JPEGR_INVALID_INPUT_TYPE; + ALOGE("received bad value for max_display_boost %f", max_display_boost); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + + if (exif != nullptr && exif->data == nullptr) { + ALOGE("received nullptr address for exif data"); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + + if (output_format <= ULTRAHDR_OUTPUT_UNSPECIFIED || output_format > ULTRAHDR_OUTPUT_MAX) { + ALOGE("received bad value for output format %d", output_format); + return ERROR_JPEGR_INVALID_INPUT_TYPE; } if (output_format == ULTRAHDR_OUTPUT_SDR) { @@ -518,6 +627,11 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, if (!gain_map_decoder.decompressImage(compressed_map.data, compressed_map.length)) { return ERROR_JPEGR_DECODE_ERROR; } + if ((gain_map_decoder.getDecompressedImageWidth() * + gain_map_decoder.getDecompressedImageHeight()) > + gain_map_decoder.getDecompressedImageSize()) { + return ERROR_JPEGR_CALCULATION_ERROR; + } if (gain_map != nullptr) { gain_map->width = gain_map_decoder.getDecompressedImageWidth(); @@ -530,13 +644,18 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, ultrahdr_metadata_struct uhdr_metadata; if (!getMetadataFromXMP(static_cast<uint8_t*>(gain_map_decoder.getXMPPtr()), gain_map_decoder.getXMPSize(), &uhdr_metadata)) { - return ERROR_JPEGR_DECODE_ERROR; + return ERROR_JPEGR_INVALID_METADATA; } if (metadata != nullptr) { metadata->version = uhdr_metadata.version; metadata->minContentBoost = uhdr_metadata.minContentBoost; metadata->maxContentBoost = uhdr_metadata.maxContentBoost; + metadata->gamma = uhdr_metadata.gamma; + metadata->offsetSdr = uhdr_metadata.offsetSdr; + metadata->offsetHdr = uhdr_metadata.offsetHdr; + metadata->hdrCapacityMin = uhdr_metadata.hdrCapacityMin; + metadata->hdrCapacityMax = uhdr_metadata.hdrCapacityMax; } if (output_format == ULTRAHDR_OUTPUT_SDR) { @@ -547,6 +666,11 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length)) { return ERROR_JPEGR_DECODE_ERROR; } + if ((jpeg_decoder.getDecompressedImageWidth() * + jpeg_decoder.getDecompressedImageHeight() * 3 / 2) > + jpeg_decoder.getDecompressedImageSize()) { + return ERROR_JPEGR_CALCULATION_ERROR; + } if (exif != nullptr) { if (exif->data == nullptr) { @@ -568,6 +692,8 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, uncompressed_yuv_420_image.data = jpeg_decoder.getDecompressedImagePtr(); uncompressed_yuv_420_image.width = jpeg_decoder.getDecompressedImageWidth(); uncompressed_yuv_420_image.height = jpeg_decoder.getDecompressedImageHeight(); + uncompressed_yuv_420_image.colorGamut = IccHelper::readIccColorGamut( + jpeg_decoder.getICCPtr(), jpeg_decoder.getICCSize()); JPEGR_CHECK(applyGainMap(&uncompressed_yuv_420_image, &map, &uhdr_metadata, output_format, max_display_boost, dest)); @@ -575,30 +701,22 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, } status_t JpegR::compressGainMap(jr_uncompressed_ptr uncompressed_gain_map, - jr_compressed_ptr dest) { - if (uncompressed_gain_map == nullptr || dest == nullptr) { + JpegEncoderHelper* jpeg_encoder) { + if (uncompressed_gain_map == nullptr || jpeg_encoder == nullptr) { return ERROR_JPEGR_INVALID_NULL_PTR; } - JpegEncoderHelper jpeg_encoder; - if (!jpeg_encoder.compressImage(uncompressed_gain_map->data, - uncompressed_gain_map->width, - uncompressed_gain_map->height, - kMapCompressQuality, - nullptr, - 0, - true /* isSingleChannel */)) { + // Don't need to convert YUV to Bt601 since single channel + if (!jpeg_encoder->compressImage(uncompressed_gain_map->data, + uncompressed_gain_map->width, + uncompressed_gain_map->height, + kMapCompressQuality, + nullptr, + 0, + true /* isSingleChannel */)) { return ERROR_JPEGR_ENCODE_ERROR; } - if (dest->maxLength < jpeg_encoder.getCompressedImageSize()) { - return ERROR_JPEGR_BUFFER_TOO_SMALL; - } - - memcpy(dest->data, jpeg_encoder.getCompressedImagePtr(), jpeg_encoder.getCompressedImageSize()); - dest->length = jpeg_encoder.getCompressedImageSize(); - dest->colorGamut = ULTRAHDR_COLORGAMUT_UNSPECIFIED; - return NO_ERROR; } @@ -664,7 +782,8 @@ status_t JpegR::generateGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, jr_uncompressed_ptr uncompressed_p010_image, ultrahdr_transfer_function hdr_tf, ultrahdr_metadata_ptr metadata, - jr_uncompressed_ptr dest) { + jr_uncompressed_ptr dest, + bool sdr_is_601) { if (uncompressed_yuv_420_image == nullptr || uncompressed_p010_image == nullptr || metadata == nullptr @@ -698,7 +817,7 @@ status_t JpegR::generateGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, map_data.reset(reinterpret_cast<uint8_t*>(dest->data)); ColorTransformFn hdrInvOetf = nullptr; - float hdr_white_nits = 0.0f; + float hdr_white_nits = kSdrWhiteNits; switch (hdr_tf) { case ULTRAHDR_TF_LINEAR: hdrInvOetf = identityConversion; @@ -726,6 +845,12 @@ status_t JpegR::generateGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, metadata->maxContentBoost = hdr_white_nits / kSdrWhiteNits; metadata->minContentBoost = 1.0f; + metadata->gamma = 1.0f; + metadata->offsetSdr = 0.0f; + metadata->offsetHdr = 0.0f; + metadata->hdrCapacityMin = 1.0f; + metadata->hdrCapacityMax = metadata->maxContentBoost; + float log2MinBoost = log2(metadata->minContentBoost); float log2MaxBoost = log2(metadata->maxContentBoost); @@ -733,15 +858,38 @@ status_t JpegR::generateGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, uncompressed_yuv_420_image->colorGamut, uncompressed_p010_image->colorGamut); ColorCalculationFn luminanceFn = nullptr; + ColorTransformFn sdrYuvToRgbFn = nullptr; switch (uncompressed_yuv_420_image->colorGamut) { case ULTRAHDR_COLORGAMUT_BT709: luminanceFn = srgbLuminance; + sdrYuvToRgbFn = srgbYuvToRgb; break; case ULTRAHDR_COLORGAMUT_P3: luminanceFn = p3Luminance; + sdrYuvToRgbFn = p3YuvToRgb; break; case ULTRAHDR_COLORGAMUT_BT2100: luminanceFn = bt2100Luminance; + sdrYuvToRgbFn = bt2100YuvToRgb; + break; + case ULTRAHDR_COLORGAMUT_UNSPECIFIED: + // Should be impossible to hit after input validation. + return ERROR_JPEGR_INVALID_COLORGAMUT; + } + if (sdr_is_601) { + sdrYuvToRgbFn = p3YuvToRgb; + } + + ColorTransformFn hdrYuvToRgbFn = nullptr; + switch (uncompressed_p010_image->colorGamut) { + case ULTRAHDR_COLORGAMUT_BT709: + hdrYuvToRgbFn = srgbYuvToRgb; + break; + case ULTRAHDR_COLORGAMUT_P3: + hdrYuvToRgbFn = p3YuvToRgb; + break; + case ULTRAHDR_COLORGAMUT_BT2100: + hdrYuvToRgbFn = bt2100YuvToRgb; break; case ULTRAHDR_COLORGAMUT_UNSPECIFIED: // Should be impossible to hit after input validation. @@ -755,8 +903,8 @@ status_t JpegR::generateGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, std::function<void()> generateMap = [uncompressed_yuv_420_image, uncompressed_p010_image, metadata, dest, hdrInvOetf, hdrGamutConversionFn, - luminanceFn, hdr_white_nits, log2MinBoost, log2MaxBoost, - &jobQueue]() -> void { + luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn, hdr_white_nits, + log2MinBoost, log2MaxBoost, &jobQueue]() -> void { size_t rowStart, rowEnd; size_t dest_map_width = uncompressed_yuv_420_image->width / kMapDimensionScaleFactor; size_t dest_map_stride = dest->width; @@ -765,7 +913,8 @@ status_t JpegR::generateGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, for (size_t x = 0; x < dest_map_width; ++x) { Color sdr_yuv_gamma = sampleYuv420(uncompressed_yuv_420_image, kMapDimensionScaleFactor, x, y); - Color sdr_rgb_gamma = srgbYuvToRgb(sdr_yuv_gamma); + Color sdr_rgb_gamma = sdrYuvToRgbFn(sdr_yuv_gamma); + // We are assuming the SDR input is always sRGB transfer. #if USE_SRGB_INVOETF_LUT Color sdr_rgb = srgbInvOetfLUT(sdr_rgb_gamma); #else @@ -774,7 +923,7 @@ status_t JpegR::generateGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, float sdr_y_nits = luminanceFn(sdr_rgb) * kSdrWhiteNits; Color hdr_yuv_gamma = sampleP010(uncompressed_p010_image, kMapDimensionScaleFactor, x, y); - Color hdr_rgb_gamma = bt2100YuvToRgb(hdr_yuv_gamma); + Color hdr_rgb_gamma = hdrYuvToRgbFn(hdr_yuv_gamma); Color hdr_rgb = hdrInvOetf(hdr_rgb_gamma); hdr_rgb = hdrGamutConversionFn(hdr_rgb); float hdr_y_nits = luminanceFn(hdr_rgb) * hdr_white_nits; @@ -820,6 +969,40 @@ status_t JpegR::applyGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, return ERROR_JPEGR_INVALID_NULL_PTR; } + if (metadata->version.compare("1.0")) { + ALOGE("Unsupported metadata version: %s", metadata->version.c_str()); + return ERROR_JPEGR_UNSUPPORTED_METADATA; + } + if (metadata->gamma != 1.0f) { + ALOGE("Unsupported metadata gamma: %f", metadata->gamma); + return ERROR_JPEGR_UNSUPPORTED_METADATA; + } + if (metadata->offsetSdr != 0.0f || metadata->offsetHdr != 0.0f) { + ALOGE("Unsupported metadata offset sdr, hdr: %f, %f", metadata->offsetSdr, + metadata->offsetHdr); + return ERROR_JPEGR_UNSUPPORTED_METADATA; + } + if (metadata->hdrCapacityMin != metadata->minContentBoost + || metadata->hdrCapacityMax != metadata->maxContentBoost) { + ALOGE("Unsupported metadata hdr capacity min, max: %f, %f", metadata->hdrCapacityMin, + metadata->hdrCapacityMax); + return ERROR_JPEGR_UNSUPPORTED_METADATA; + } + + // TODO: remove once map scaling factor is computed based on actual map dims + size_t image_width = uncompressed_yuv_420_image->width; + size_t image_height = uncompressed_yuv_420_image->height; + size_t map_width = image_width / kMapDimensionScaleFactor; + size_t map_height = image_height / kMapDimensionScaleFactor; + map_width = static_cast<size_t>( + floor((map_width + kJpegBlock - 1) / kJpegBlock)) * kJpegBlock; + map_height = ((map_height + 1) >> 1) << 1; + if (map_width != uncompressed_gain_map->width + || map_height != uncompressed_gain_map->height) { + ALOGE("gain map dimensions and primary image dimensions are not to scale"); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + dest->width = uncompressed_yuv_420_image->width; dest->height = uncompressed_yuv_420_image->height; ShepardsIDW idwTable(kMapDimensionScaleFactor); @@ -838,7 +1021,9 @@ status_t JpegR::applyGainMap(jr_uncompressed_ptr uncompressed_yuv_420_image, for (size_t y = rowStart; y < rowEnd; ++y) { for (size_t x = 0; x < width; ++x) { Color yuv_gamma_sdr = getYuv420Pixel(uncompressed_yuv_420_image, x, y); - Color rgb_gamma_sdr = srgbYuvToRgb(yuv_gamma_sdr); + // Assuming the sdr image is a decoded JPEG, we should always use Rec.601 YUV coefficients + Color rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr); + // We are assuming the SDR base image is always sRGB transfer. #if USE_SRGB_INVOETF_LUT Color rgb_sdr = srgbInvOetfLUT(rgb_gamma_sdr); #else @@ -1016,6 +1201,7 @@ status_t JpegR::extractGainMap(jr_compressed_ptr compressed_jpegr_image, status_t JpegR::appendGainMap(jr_compressed_ptr compressed_jpeg_image, jr_compressed_ptr compressed_gain_map, jr_exif_ptr exif, + void* icc, size_t icc_size, ultrahdr_metadata_ptr metadata, jr_compressed_ptr dest) { if (compressed_jpeg_image == nullptr @@ -1025,6 +1211,33 @@ status_t JpegR::appendGainMap(jr_compressed_ptr compressed_jpeg_image, return ERROR_JPEGR_INVALID_NULL_PTR; } + if (metadata->version.compare("1.0")) { + ALOGE("received bad value for version: %s", metadata->version.c_str()); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + if (metadata->maxContentBoost < metadata->minContentBoost) { + ALOGE("received bad value for content boost min %f, max %f", metadata->minContentBoost, + metadata->maxContentBoost); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + + if (metadata->hdrCapacityMax < metadata->hdrCapacityMin || metadata->hdrCapacityMin < 1.0f) { + ALOGE("received bad value for hdr capacity min %f, max %f", metadata->hdrCapacityMin, + metadata->hdrCapacityMax); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + + if (metadata->offsetSdr < 0.0f || metadata->offsetHdr < 0.0f) { + ALOGE("received bad value for offset sdr %f, hdr %f", metadata->offsetSdr, + metadata->offsetHdr); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + + if (metadata->gamma <= 0.0f) { + ALOGE("received bad value for gamma %f", metadata->gamma); + return ERROR_JPEGR_INVALID_INPUT_TYPE; + } + const string nameSpace = "http://ns.adobe.com/xap/1.0/"; const int nameSpaceLength = nameSpace.size() + 1; // need to count the null terminator @@ -1073,6 +1286,18 @@ status_t JpegR::appendGainMap(jr_compressed_ptr compressed_jpeg_image, JPEGR_CHECK(Write(dest, (void*)xmp_primary.c_str(), xmp_primary.size(), pos)); } + // Write ICC + if (icc != nullptr && icc_size > 0) { + const int length = icc_size + 2; + const uint8_t lengthH = ((length >> 8) & 0xff); + const uint8_t lengthL = (length & 0xff); + JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kStart, 1, pos)); + JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kAPP2, 1, pos)); + JPEGR_CHECK(Write(dest, &lengthH, 1, pos)); + JPEGR_CHECK(Write(dest, &lengthL, 1, pos)); + JPEGR_CHECK(Write(dest, icc, icc_size, pos)); + } + // Prepare and write MPF { const int length = 2 + calculateMpfSize(); @@ -1180,4 +1405,82 @@ status_t JpegR::toneMap(jr_uncompressed_ptr src, jr_uncompressed_ptr dest) { return NO_ERROR; } +status_t JpegR::convertYuv(jr_uncompressed_ptr image, + ultrahdr_color_gamut src_encoding, + ultrahdr_color_gamut dest_encoding) { + if (image == nullptr) { + return ERROR_JPEGR_INVALID_NULL_PTR; + } + + if (src_encoding == ULTRAHDR_COLORGAMUT_UNSPECIFIED + || dest_encoding == ULTRAHDR_COLORGAMUT_UNSPECIFIED) { + return ERROR_JPEGR_INVALID_COLORGAMUT; + } + + ColorTransformFn conversionFn = nullptr; + switch (src_encoding) { + case ULTRAHDR_COLORGAMUT_BT709: + switch (dest_encoding) { + case ULTRAHDR_COLORGAMUT_BT709: + return NO_ERROR; + case ULTRAHDR_COLORGAMUT_P3: + conversionFn = yuv709To601; + break; + case ULTRAHDR_COLORGAMUT_BT2100: + conversionFn = yuv709To2100; + break; + default: + // Should be impossible to hit after input validation + return ERROR_JPEGR_INVALID_COLORGAMUT; + } + break; + case ULTRAHDR_COLORGAMUT_P3: + switch (dest_encoding) { + case ULTRAHDR_COLORGAMUT_BT709: + conversionFn = yuv601To709; + break; + case ULTRAHDR_COLORGAMUT_P3: + return NO_ERROR; + case ULTRAHDR_COLORGAMUT_BT2100: + conversionFn = yuv601To2100; + break; + default: + // Should be impossible to hit after input validation + return ERROR_JPEGR_INVALID_COLORGAMUT; + } + break; + case ULTRAHDR_COLORGAMUT_BT2100: + switch (dest_encoding) { + case ULTRAHDR_COLORGAMUT_BT709: + conversionFn = yuv2100To709; + break; + case ULTRAHDR_COLORGAMUT_P3: + conversionFn = yuv2100To601; + break; + case ULTRAHDR_COLORGAMUT_BT2100: + return NO_ERROR; + default: + // Should be impossible to hit after input validation + return ERROR_JPEGR_INVALID_COLORGAMUT; + } + break; + default: + // Should be impossible to hit after input validation + return ERROR_JPEGR_INVALID_COLORGAMUT; + } + + if (conversionFn == nullptr) { + // Should be impossible to hit after input validation + return ERROR_JPEGR_INVALID_COLORGAMUT; + } + + for (size_t y = 0; y < image->height / 2; ++y) { + for (size_t x = 0; x < image->width / 2; ++x) { + transformYuv420(image, x, y, conversionFn); + } + } + + return NO_ERROR; +} + } // namespace android::ultrahdr diff --git a/libs/ultrahdr/jpegrutils.cpp b/libs/ultrahdr/jpegrutils.cpp index 6430af12c7..c434eb6459 100644 --- a/libs/ultrahdr/jpegrutils.cpp +++ b/libs/ultrahdr/jpegrutils.cpp @@ -113,6 +113,15 @@ public: XMPXmlHandler() : XmlHandler() { state = NotStrarted; + versionFound = false; + minContentBoostFound = false; + maxContentBoostFound = false; + gammaFound = false; + offsetSdrFound = false; + offsetHdrFound = false; + hdrCapacityMinFound = false; + hdrCapacityMaxFound = false; + baseRenditionIsHdrFound = false; } enum ParseState { @@ -147,10 +156,24 @@ public: string val; if (state == Started) { if (context.BuildTokenValue(&val)) { - if (!val.compare(maxContentBoostAttrName)) { + if (!val.compare(versionAttrName)) { + lastAttributeName = versionAttrName; + } else if (!val.compare(maxContentBoostAttrName)) { lastAttributeName = maxContentBoostAttrName; } else if (!val.compare(minContentBoostAttrName)) { lastAttributeName = minContentBoostAttrName; + } else if (!val.compare(gammaAttrName)) { + lastAttributeName = gammaAttrName; + } else if (!val.compare(offsetSdrAttrName)) { + lastAttributeName = offsetSdrAttrName; + } else if (!val.compare(offsetHdrAttrName)) { + lastAttributeName = offsetHdrAttrName; + } else if (!val.compare(hdrCapacityMinAttrName)) { + lastAttributeName = hdrCapacityMinAttrName; + } else if (!val.compare(hdrCapacityMaxAttrName)) { + lastAttributeName = hdrCapacityMaxAttrName; + } else if (!val.compare(baseRenditionIsHdrAttrName)) { + lastAttributeName = baseRenditionIsHdrAttrName; } else { lastAttributeName = ""; } @@ -163,18 +186,52 @@ public: string val; if (state == Started) { if (context.BuildTokenValue(&val, true)) { - if (!lastAttributeName.compare(maxContentBoostAttrName)) { + if (!lastAttributeName.compare(versionAttrName)) { + versionStr = val; + versionFound = true; + } else if (!lastAttributeName.compare(maxContentBoostAttrName)) { maxContentBoostStr = val; + maxContentBoostFound = true; } else if (!lastAttributeName.compare(minContentBoostAttrName)) { minContentBoostStr = val; + minContentBoostFound = true; + } else if (!lastAttributeName.compare(gammaAttrName)) { + gammaStr = val; + gammaFound = true; + } else if (!lastAttributeName.compare(offsetSdrAttrName)) { + offsetSdrStr = val; + offsetSdrFound = true; + } else if (!lastAttributeName.compare(offsetHdrAttrName)) { + offsetHdrStr = val; + offsetHdrFound = true; + } else if (!lastAttributeName.compare(hdrCapacityMinAttrName)) { + hdrCapacityMinStr = val; + hdrCapacityMinFound = true; + } else if (!lastAttributeName.compare(hdrCapacityMaxAttrName)) { + hdrCapacityMaxStr = val; + hdrCapacityMaxFound = true; + } else if (!lastAttributeName.compare(baseRenditionIsHdrAttrName)) { + baseRenditionIsHdrStr = val; + baseRenditionIsHdrFound = true; } } } return context.GetResult(); } - bool getMaxContentBoost(float* max_content_boost) { + bool getVersion(string* version, bool* present) { if (state == Done) { + *version = versionStr; + *present = versionFound; + return true; + } else { + return false; + } + } + + bool getMaxContentBoost(float* max_content_boost, bool* present) { + if (state == Done) { + *present = maxContentBoostFound; stringstream ss(maxContentBoostStr); float val; if (ss >> val) { @@ -188,8 +245,9 @@ public: } } - bool getMinContentBoost(float* min_content_boost) { + bool getMinContentBoost(float* min_content_boost, bool* present) { if (state == Done) { + *present = minContentBoostFound; stringstream ss(minContentBoostStr); float val; if (ss >> val) { @@ -203,12 +261,141 @@ public: } } + bool getGamma(float* gamma, bool* present) { + if (state == Done) { + *present = gammaFound; + stringstream ss(gammaStr); + float val; + if (ss >> val) { + *gamma = val; + return true; + } else { + return false; + } + } else { + return false; + } + } + + + bool getOffsetSdr(float* offset_sdr, bool* present) { + if (state == Done) { + *present = offsetSdrFound; + stringstream ss(offsetSdrStr); + float val; + if (ss >> val) { + *offset_sdr = val; + return true; + } else { + return false; + } + } else { + return false; + } + } + + + bool getOffsetHdr(float* offset_hdr, bool* present) { + if (state == Done) { + *present = offsetHdrFound; + stringstream ss(offsetHdrStr); + float val; + if (ss >> val) { + *offset_hdr = val; + return true; + } else { + return false; + } + } else { + return false; + } + } + + + bool getHdrCapacityMin(float* hdr_capacity_min, bool* present) { + if (state == Done) { + *present = hdrCapacityMinFound; + stringstream ss(hdrCapacityMinStr); + float val; + if (ss >> val) { + *hdr_capacity_min = exp2(val); + return true; + } else { + return false; + } + } else { + return false; + } + } + + + bool getHdrCapacityMax(float* hdr_capacity_max, bool* present) { + if (state == Done) { + *present = hdrCapacityMaxFound; + stringstream ss(hdrCapacityMaxStr); + float val; + if (ss >> val) { + *hdr_capacity_max = exp2(val); + return true; + } else { + return false; + } + } else { + return false; + } + } + + + bool getBaseRenditionIsHdr(bool* base_rendition_is_hdr, bool* present) { + if (state == Done) { + *present = baseRenditionIsHdrFound; + if (!baseRenditionIsHdrStr.compare("False")) { + *base_rendition_is_hdr = false; + return true; + } else if (!baseRenditionIsHdrStr.compare("True")) { + *base_rendition_is_hdr = true; + return true; + } else { + return false; + } + } else { + return false; + } + } + + + private: static const string containerName; + + static const string versionAttrName; + string versionStr; + bool versionFound; static const string maxContentBoostAttrName; string maxContentBoostStr; + bool maxContentBoostFound; static const string minContentBoostAttrName; string minContentBoostStr; + bool minContentBoostFound; + static const string gammaAttrName; + string gammaStr; + bool gammaFound; + static const string offsetSdrAttrName; + string offsetSdrStr; + bool offsetSdrFound; + static const string offsetHdrAttrName; + string offsetHdrStr; + bool offsetHdrFound; + static const string hdrCapacityMinAttrName; + string hdrCapacityMinStr; + bool hdrCapacityMinFound; + static const string hdrCapacityMaxAttrName; + string hdrCapacityMaxStr; + bool hdrCapacityMaxFound; + static const string baseRenditionIsHdrAttrName; + string baseRenditionIsHdrStr; + bool baseRenditionIsHdrFound; + string lastAttributeName; ParseState state; }; @@ -253,8 +440,15 @@ const string kMapHDRCapacityMax = Name(kGainMapPrefix, "HDRCapacityMax"); const string kMapBaseRenditionIsHDR = Name(kGainMapPrefix, "BaseRenditionIsHDR"); // GainMap XMP constants - names for XMP handlers +const string XMPXmlHandler::versionAttrName = kMapVersion; const string XMPXmlHandler::minContentBoostAttrName = kMapGainMapMin; const string XMPXmlHandler::maxContentBoostAttrName = kMapGainMapMax; +const string XMPXmlHandler::gammaAttrName = kMapGamma; +const string XMPXmlHandler::offsetSdrAttrName = kMapOffsetSdr; +const string XMPXmlHandler::offsetHdrAttrName = kMapOffsetHdr; +const string XMPXmlHandler::hdrCapacityMinAttrName = kMapHDRCapacityMin; +const string XMPXmlHandler::hdrCapacityMaxAttrName = kMapHDRCapacityMax; +const string XMPXmlHandler::baseRenditionIsHdrAttrName = kMapBaseRenditionIsHDR; bool getMetadataFromXMP(uint8_t* xmp_data, size_t xmp_size, ultrahdr_metadata_struct* metadata) { string nameSpace = "http://ns.adobe.com/xap/1.0/\0"; @@ -291,11 +485,48 @@ bool getMetadataFromXMP(uint8_t* xmp_data, size_t xmp_size, ultrahdr_metadata_st return false; } - if (!handler.getMaxContentBoost(&metadata->maxContentBoost)) { + // Apply default values to any not-present fields, except for Version, + // maxContentBoost, and hdrCapacityMax, which are required. Return false if + // we encounter a present field that couldn't be parsed, since this + // indicates it is invalid (eg. string where there should be a float). + bool present = false; + if (!handler.getVersion(&metadata->version, &present) || !present) { + return false; + } + if (!handler.getMaxContentBoost(&metadata->maxContentBoost, &present) || !present) { + return false; + } + if (!handler.getHdrCapacityMax(&metadata->hdrCapacityMax, &present) || !present) { return false; } + if (!handler.getMinContentBoost(&metadata->minContentBoost, &present)) { + if (present) return false; + metadata->minContentBoost = 1.0f; + } + if (!handler.getGamma(&metadata->gamma, &present)) { + if (present) return false; + metadata->gamma = 1.0f; + } + if (!handler.getOffsetSdr(&metadata->offsetSdr, &present)) { + if (present) return false; + metadata->offsetSdr = 1.0f / 64.0f; + } + if (!handler.getOffsetHdr(&metadata->offsetHdr, &present)) { + if (present) return false; + metadata->offsetHdr = 1.0f / 64.0f; + } + if (!handler.getHdrCapacityMin(&metadata->hdrCapacityMin, &present)) { + if (present) return false; + metadata->hdrCapacityMin = 1.0f; + } - if (!handler.getMinContentBoost(&metadata->minContentBoost)) { + bool base_rendition_is_hdr; + if (!handler.getBaseRenditionIsHdr(&base_rendition_is_hdr, &present)) { + if (present) return false; + base_rendition_is_hdr = false; + } + if (base_rendition_is_hdr) { + ALOGE("Base rendition of HDR is not supported!"); return false; } @@ -355,12 +586,11 @@ string generateXmpForSecondaryImage(ultrahdr_metadata_struct& metadata) { writer.WriteAttributeNameAndValue(kMapVersion, metadata.version); writer.WriteAttributeNameAndValue(kMapGainMapMin, log2(metadata.minContentBoost)); writer.WriteAttributeNameAndValue(kMapGainMapMax, log2(metadata.maxContentBoost)); - writer.WriteAttributeNameAndValue(kMapGamma, "1"); - writer.WriteAttributeNameAndValue(kMapOffsetSdr, "0"); - writer.WriteAttributeNameAndValue(kMapOffsetHdr, "0"); - writer.WriteAttributeNameAndValue( - kMapHDRCapacityMin, std::max(log2(metadata.minContentBoost), 0.0f)); - writer.WriteAttributeNameAndValue(kMapHDRCapacityMax, log2(metadata.maxContentBoost)); + writer.WriteAttributeNameAndValue(kMapGamma, metadata.gamma); + writer.WriteAttributeNameAndValue(kMapOffsetSdr, metadata.offsetSdr); + writer.WriteAttributeNameAndValue(kMapOffsetHdr, metadata.offsetHdr); + writer.WriteAttributeNameAndValue(kMapHDRCapacityMin, log2(metadata.hdrCapacityMin)); + writer.WriteAttributeNameAndValue(kMapHDRCapacityMax, log2(metadata.hdrCapacityMax)); writer.WriteAttributeNameAndValue(kMapBaseRenditionIsHDR, "False"); writer.FinishWriting(); diff --git a/libs/ultrahdr/multipictureformat.cpp b/libs/ultrahdr/multipictureformat.cpp index 7a265c61b7..f1679ef1b3 100644 --- a/libs/ultrahdr/multipictureformat.cpp +++ b/libs/ultrahdr/multipictureformat.cpp @@ -30,7 +30,7 @@ size_t calculateMpfSize() { sp<DataStruct> generateMpf(int primary_image_size, int primary_image_offset, int secondary_image_size, int secondary_image_offset) { size_t mpf_size = calculateMpfSize(); - sp<DataStruct> dataStruct = new DataStruct(mpf_size); + sp<DataStruct> dataStruct = sp<DataStruct>::make(mpf_size); dataStruct->write(static_cast<const void*>(kMpfSig), sizeof(kMpfSig)); #if USE_BIG_ENDIAN diff --git a/libs/ultrahdr/tests/Android.bp b/libs/ultrahdr/tests/Android.bp index 7dd9d04fbd..594413018c 100644 --- a/libs/ultrahdr/tests/Android.bp +++ b/libs/ultrahdr/tests/Android.bp @@ -25,8 +25,9 @@ cc_test { name: "libultrahdr_test", test_suites: ["device-tests"], srcs: [ - "jpegr_test.cpp", "gainmapmath_test.cpp", + "icchelper_test.cpp", + "jpegr_test.cpp", ], shared_libs: [ "libimage_io", @@ -72,5 +73,7 @@ cc_test { static_libs: [ "libgtest", "libjpegdecoder", + "libultrahdr", + "libutils", ], } diff --git a/libs/ultrahdr/tests/data/minnie-320x240-yuv-icc.jpg b/libs/ultrahdr/tests/data/minnie-320x240-yuv-icc.jpg Binary files differnew file mode 100644 index 0000000000..c7f4538534 --- /dev/null +++ b/libs/ultrahdr/tests/data/minnie-320x240-yuv-icc.jpg diff --git a/libs/ultrahdr/tests/gainmapmath_test.cpp b/libs/ultrahdr/tests/gainmapmath_test.cpp index c456653821..af90365e56 100644 --- a/libs/ultrahdr/tests/gainmapmath_test.cpp +++ b/libs/ultrahdr/tests/gainmapmath_test.cpp @@ -28,6 +28,7 @@ public: float ComparisonEpsilon() { return 1e-4f; } float LuminanceEpsilon() { return 1e-2f; } + float YuvConversionEpsilon() { return 1.0f / (255.0f * 2.0f); } Color Yuv420(uint8_t y, uint8_t u, uint8_t v) { return {{{ static_cast<float>(y) / 255.0f, @@ -63,9 +64,13 @@ public: Color YuvBlack() { return {{{ 0.0f, 0.0f, 0.0f }}}; } Color YuvWhite() { return {{{ 1.0f, 0.0f, 0.0f }}}; } - Color SrgbYuvRed() { return {{{ 0.299f, -0.1687f, 0.5f }}}; } - Color SrgbYuvGreen() { return {{{ 0.587f, -0.3313f, -0.4187f }}}; } - Color SrgbYuvBlue() { return {{{ 0.114f, 0.5f, -0.0813f }}}; } + Color SrgbYuvRed() { return {{{ 0.2126f, -0.11457f, 0.5f }}}; } + Color SrgbYuvGreen() { return {{{ 0.7152f, -0.38543f, -0.45415f }}}; } + Color SrgbYuvBlue() { return {{{ 0.0722f, 0.5f, -0.04585f }}}; } + + Color P3YuvRed() { return {{{ 0.299f, -0.16874f, 0.5f }}}; } + Color P3YuvGreen() { return {{{ 0.587f, -0.33126f, -0.41869f }}}; } + Color P3YuvBlue() { return {{{ 0.114f, 0.5f, -0.08131f }}}; } Color Bt2100YuvRed() { return {{{ 0.2627f, -0.13963f, 0.5f }}}; } Color Bt2100YuvGreen() { return {{{ 0.6780f, -0.36037f, -0.45979f }}}; } @@ -78,6 +83,13 @@ public: return luminance_scaled * kSdrWhiteNits; } + float P3YuvToLuminance(Color yuv_gamma, ColorCalculationFn luminanceFn) { + Color rgb_gamma = p3YuvToRgb(yuv_gamma); + Color rgb = srgbInvOetf(rgb_gamma); + float luminance_scaled = luminanceFn(rgb); + return luminance_scaled * kSdrWhiteNits; + } + float Bt2100YuvToLuminance(Color yuv_gamma, ColorTransformFn hdrInvOetf, ColorTransformFn gamutConversionFn, ColorCalculationFn luminanceFn, float scale_factor) { @@ -402,6 +414,56 @@ TEST_F(GainMapMathTest, P3Luminance) { EXPECT_FLOAT_EQ(p3Luminance(RgbBlue()), 0.06891f); } +TEST_F(GainMapMathTest, P3YuvToRgb) { + Color rgb_black = p3YuvToRgb(YuvBlack()); + EXPECT_RGB_NEAR(rgb_black, RgbBlack()); + + Color rgb_white = p3YuvToRgb(YuvWhite()); + EXPECT_RGB_NEAR(rgb_white, RgbWhite()); + + Color rgb_r = p3YuvToRgb(P3YuvRed()); + EXPECT_RGB_NEAR(rgb_r, RgbRed()); + + Color rgb_g = p3YuvToRgb(P3YuvGreen()); + EXPECT_RGB_NEAR(rgb_g, RgbGreen()); + + Color rgb_b = p3YuvToRgb(P3YuvBlue()); + EXPECT_RGB_NEAR(rgb_b, RgbBlue()); +} + +TEST_F(GainMapMathTest, P3RgbToYuv) { + Color yuv_black = p3RgbToYuv(RgbBlack()); + EXPECT_YUV_NEAR(yuv_black, YuvBlack()); + + Color yuv_white = p3RgbToYuv(RgbWhite()); + EXPECT_YUV_NEAR(yuv_white, YuvWhite()); + + Color yuv_r = p3RgbToYuv(RgbRed()); + EXPECT_YUV_NEAR(yuv_r, P3YuvRed()); + + Color yuv_g = p3RgbToYuv(RgbGreen()); + EXPECT_YUV_NEAR(yuv_g, P3YuvGreen()); + + Color yuv_b = p3RgbToYuv(RgbBlue()); + EXPECT_YUV_NEAR(yuv_b, P3YuvBlue()); +} + +TEST_F(GainMapMathTest, P3RgbYuvRoundtrip) { + Color rgb_black = p3YuvToRgb(p3RgbToYuv(RgbBlack())); + EXPECT_RGB_NEAR(rgb_black, RgbBlack()); + + Color rgb_white = p3YuvToRgb(p3RgbToYuv(RgbWhite())); + EXPECT_RGB_NEAR(rgb_white, RgbWhite()); + + Color rgb_r = p3YuvToRgb(p3RgbToYuv(RgbRed())); + EXPECT_RGB_NEAR(rgb_r, RgbRed()); + + Color rgb_g = p3YuvToRgb(p3RgbToYuv(RgbGreen())); + EXPECT_RGB_NEAR(rgb_g, RgbGreen()); + + Color rgb_b = p3YuvToRgb(p3RgbToYuv(RgbBlue())); + EXPECT_RGB_NEAR(rgb_b, RgbBlue()); +} TEST_F(GainMapMathTest, Bt2100Luminance) { EXPECT_FLOAT_EQ(bt2100Luminance(RgbBlack()), 0.0f); EXPECT_FLOAT_EQ(bt2100Luminance(RgbWhite()), 1.0f); @@ -461,6 +523,163 @@ TEST_F(GainMapMathTest, Bt2100RgbYuvRoundtrip) { EXPECT_RGB_NEAR(rgb_b, RgbBlue()); } +TEST_F(GainMapMathTest, Bt709ToBt601YuvConversion) { + Color yuv_black = srgbRgbToYuv(RgbBlack()); + EXPECT_YUV_NEAR(yuv709To601(yuv_black), YuvBlack()); + + Color yuv_white = srgbRgbToYuv(RgbWhite()); + EXPECT_YUV_NEAR(yuv709To601(yuv_white), YuvWhite()); + + Color yuv_r = srgbRgbToYuv(RgbRed()); + EXPECT_YUV_NEAR(yuv709To601(yuv_r), P3YuvRed()); + + Color yuv_g = srgbRgbToYuv(RgbGreen()); + EXPECT_YUV_NEAR(yuv709To601(yuv_g), P3YuvGreen()); + + Color yuv_b = srgbRgbToYuv(RgbBlue()); + EXPECT_YUV_NEAR(yuv709To601(yuv_b), P3YuvBlue()); +} + +TEST_F(GainMapMathTest, Bt709ToBt2100YuvConversion) { + Color yuv_black = srgbRgbToYuv(RgbBlack()); + EXPECT_YUV_NEAR(yuv709To2100(yuv_black), YuvBlack()); + + Color yuv_white = srgbRgbToYuv(RgbWhite()); + EXPECT_YUV_NEAR(yuv709To2100(yuv_white), YuvWhite()); + + Color yuv_r = srgbRgbToYuv(RgbRed()); + EXPECT_YUV_NEAR(yuv709To2100(yuv_r), Bt2100YuvRed()); + + Color yuv_g = srgbRgbToYuv(RgbGreen()); + EXPECT_YUV_NEAR(yuv709To2100(yuv_g), Bt2100YuvGreen()); + + Color yuv_b = srgbRgbToYuv(RgbBlue()); + EXPECT_YUV_NEAR(yuv709To2100(yuv_b), Bt2100YuvBlue()); +} + +TEST_F(GainMapMathTest, Bt601ToBt709YuvConversion) { + Color yuv_black = p3RgbToYuv(RgbBlack()); + EXPECT_YUV_NEAR(yuv601To709(yuv_black), YuvBlack()); + + Color yuv_white = p3RgbToYuv(RgbWhite()); + EXPECT_YUV_NEAR(yuv601To709(yuv_white), YuvWhite()); + + Color yuv_r = p3RgbToYuv(RgbRed()); + EXPECT_YUV_NEAR(yuv601To709(yuv_r), SrgbYuvRed()); + + Color yuv_g = p3RgbToYuv(RgbGreen()); + EXPECT_YUV_NEAR(yuv601To709(yuv_g), SrgbYuvGreen()); + + Color yuv_b = p3RgbToYuv(RgbBlue()); + EXPECT_YUV_NEAR(yuv601To709(yuv_b), SrgbYuvBlue()); +} + +TEST_F(GainMapMathTest, Bt601ToBt2100YuvConversion) { + Color yuv_black = p3RgbToYuv(RgbBlack()); + EXPECT_YUV_NEAR(yuv601To2100(yuv_black), YuvBlack()); + + Color yuv_white = p3RgbToYuv(RgbWhite()); + EXPECT_YUV_NEAR(yuv601To2100(yuv_white), YuvWhite()); + + Color yuv_r = p3RgbToYuv(RgbRed()); + EXPECT_YUV_NEAR(yuv601To2100(yuv_r), Bt2100YuvRed()); + + Color yuv_g = p3RgbToYuv(RgbGreen()); + EXPECT_YUV_NEAR(yuv601To2100(yuv_g), Bt2100YuvGreen()); + + Color yuv_b = p3RgbToYuv(RgbBlue()); + EXPECT_YUV_NEAR(yuv601To2100(yuv_b), Bt2100YuvBlue()); +} + +TEST_F(GainMapMathTest, Bt2100ToBt709YuvConversion) { + Color yuv_black = bt2100RgbToYuv(RgbBlack()); + EXPECT_YUV_NEAR(yuv2100To709(yuv_black), YuvBlack()); + + Color yuv_white = bt2100RgbToYuv(RgbWhite()); + EXPECT_YUV_NEAR(yuv2100To709(yuv_white), YuvWhite()); + + Color yuv_r = bt2100RgbToYuv(RgbRed()); + EXPECT_YUV_NEAR(yuv2100To709(yuv_r), SrgbYuvRed()); + + Color yuv_g = bt2100RgbToYuv(RgbGreen()); + EXPECT_YUV_NEAR(yuv2100To709(yuv_g), SrgbYuvGreen()); + + Color yuv_b = bt2100RgbToYuv(RgbBlue()); + EXPECT_YUV_NEAR(yuv2100To709(yuv_b), SrgbYuvBlue()); +} + +TEST_F(GainMapMathTest, Bt2100ToBt601YuvConversion) { + Color yuv_black = bt2100RgbToYuv(RgbBlack()); + EXPECT_YUV_NEAR(yuv2100To601(yuv_black), YuvBlack()); + + Color yuv_white = bt2100RgbToYuv(RgbWhite()); + EXPECT_YUV_NEAR(yuv2100To601(yuv_white), YuvWhite()); + + Color yuv_r = bt2100RgbToYuv(RgbRed()); + EXPECT_YUV_NEAR(yuv2100To601(yuv_r), P3YuvRed()); + + Color yuv_g = bt2100RgbToYuv(RgbGreen()); + EXPECT_YUV_NEAR(yuv2100To601(yuv_g), P3YuvGreen()); + + Color yuv_b = bt2100RgbToYuv(RgbBlue()); + EXPECT_YUV_NEAR(yuv2100To601(yuv_b), P3YuvBlue()); +} + +TEST_F(GainMapMathTest, TransformYuv420) { + ColorTransformFn transforms[] = { yuv709To601, yuv709To2100, yuv601To709, yuv601To2100, + yuv2100To709, yuv2100To601 }; + for (const ColorTransformFn& transform : transforms) { + jpegr_uncompressed_struct input = Yuv420Image(); + + size_t out_buf_size = input.width * input.height * 3 / 2; + std::unique_ptr<uint8_t[]> out_buf = std::make_unique<uint8_t[]>(out_buf_size); + memcpy(out_buf.get(), input.data, out_buf_size); + jpegr_uncompressed_struct output = Yuv420Image(); + output.data = out_buf.get(); + + transformYuv420(&output, 1, 1, transform); + + for (size_t y = 0; y < 4; ++y) { + for (size_t x = 0; x < 4; ++x) { + // Skip the last chroma sample, which we modified above + if (x >= 2 && y >= 2) { + continue; + } + + // All other pixels should remain unchanged + EXPECT_YUV_EQ(getYuv420Pixel(&input, x, y), getYuv420Pixel(&output, x, y)); + } + } + + // modified pixels should be updated as intended by the transformYuv420 algorithm + Color in1 = getYuv420Pixel(&input, 2, 2); + Color in2 = getYuv420Pixel(&input, 3, 2); + Color in3 = getYuv420Pixel(&input, 2, 3); + Color in4 = getYuv420Pixel(&input, 3, 3); + Color out1 = getYuv420Pixel(&output, 2, 2); + Color out2 = getYuv420Pixel(&output, 3, 2); + Color out3 = getYuv420Pixel(&output, 2, 3); + Color out4 = getYuv420Pixel(&output, 3, 3); + + EXPECT_NEAR(transform(in1).y, out1.y, YuvConversionEpsilon()); + EXPECT_NEAR(transform(in2).y, out2.y, YuvConversionEpsilon()); + EXPECT_NEAR(transform(in3).y, out3.y, YuvConversionEpsilon()); + EXPECT_NEAR(transform(in4).y, out4.y, YuvConversionEpsilon()); + + Color expect_uv = (transform(in1) + transform(in2) + transform(in3) + transform(in4)) / 4.0f; + + EXPECT_NEAR(expect_uv.u, out1.u, YuvConversionEpsilon()); + EXPECT_NEAR(expect_uv.u, out2.u, YuvConversionEpsilon()); + EXPECT_NEAR(expect_uv.u, out3.u, YuvConversionEpsilon()); + EXPECT_NEAR(expect_uv.u, out4.u, YuvConversionEpsilon()); + + EXPECT_NEAR(expect_uv.v, out1.v, YuvConversionEpsilon()); + EXPECT_NEAR(expect_uv.v, out2.v, YuvConversionEpsilon()); + EXPECT_NEAR(expect_uv.v, out3.v, YuvConversionEpsilon()); + EXPECT_NEAR(expect_uv.v, out4.v, YuvConversionEpsilon()); + } +} + TEST_F(GainMapMathTest, HlgOetf) { EXPECT_FLOAT_EQ(hlgOetf(0.0f), 0.0f); EXPECT_NEAR(hlgOetf(0.04167f), 0.35357f, ComparisonEpsilon()); @@ -693,7 +912,7 @@ TEST_F(GainMapMathTest, ColorConversionLookup) { TEST_F(GainMapMathTest, EncodeGain) { ultrahdr_metadata_struct metadata = { .maxContentBoost = 4.0f, - .minContentBoost = 1.0f / 4.0f }; + .minContentBoost = 1.0f / 4.0f }; EXPECT_EQ(encodeGain(0.0f, 0.0f, &metadata), 127); EXPECT_EQ(encodeGain(0.0f, 1.0f, &metadata), 127); @@ -751,7 +970,7 @@ TEST_F(GainMapMathTest, EncodeGain) { TEST_F(GainMapMathTest, ApplyGain) { ultrahdr_metadata_struct metadata = { .maxContentBoost = 4.0f, - .minContentBoost = 1.0f / 4.0f }; + .minContentBoost = 1.0f / 4.0f }; float displayBoost = metadata.maxContentBoost; EXPECT_RGB_NEAR(applyGain(RgbBlack(), 0.0f, &metadata), RgbBlack()); diff --git a/libs/ultrahdr/tests/icchelper_test.cpp b/libs/ultrahdr/tests/icchelper_test.cpp new file mode 100644 index 0000000000..ff61c08574 --- /dev/null +++ b/libs/ultrahdr/tests/icchelper_test.cpp @@ -0,0 +1,77 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> +#include <ultrahdr/icc.h> +#include <ultrahdr/ultrahdr.h> +#include <utils/Log.h> + +namespace android::ultrahdr { + +class IccHelperTest : public testing::Test { +public: + IccHelperTest(); + ~IccHelperTest(); +protected: + virtual void SetUp(); + virtual void TearDown(); +}; + +IccHelperTest::IccHelperTest() {} + +IccHelperTest::~IccHelperTest() {} + +void IccHelperTest::SetUp() {} + +void IccHelperTest::TearDown() {} + +TEST_F(IccHelperTest, iccWriteThenRead) { + sp<DataStruct> iccBt709 = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, + ULTRAHDR_COLORGAMUT_BT709); + ASSERT_NE(iccBt709->getLength(), 0); + ASSERT_NE(iccBt709->getData(), nullptr); + EXPECT_EQ(IccHelper::readIccColorGamut(iccBt709->getData(), iccBt709->getLength()), + ULTRAHDR_COLORGAMUT_BT709); + + sp<DataStruct> iccP3 = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, ULTRAHDR_COLORGAMUT_P3); + ASSERT_NE(iccP3->getLength(), 0); + ASSERT_NE(iccP3->getData(), nullptr); + EXPECT_EQ(IccHelper::readIccColorGamut(iccP3->getData(), iccP3->getLength()), + ULTRAHDR_COLORGAMUT_P3); + + sp<DataStruct> iccBt2100 = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, + ULTRAHDR_COLORGAMUT_BT2100); + ASSERT_NE(iccBt2100->getLength(), 0); + ASSERT_NE(iccBt2100->getData(), nullptr); + EXPECT_EQ(IccHelper::readIccColorGamut(iccBt2100->getData(), iccBt2100->getLength()), + ULTRAHDR_COLORGAMUT_BT2100); +} + +TEST_F(IccHelperTest, iccEndianness) { + sp<DataStruct> icc = IccHelper::writeIccProfile(ULTRAHDR_TF_SRGB, ULTRAHDR_COLORGAMUT_BT709); + size_t profile_size = icc->getLength() - kICCIdentifierSize; + + uint8_t* icc_bytes = reinterpret_cast<uint8_t*>(icc->getData()) + kICCIdentifierSize; + uint32_t encoded_size = static_cast<uint32_t>(icc_bytes[0]) << 24 | + static_cast<uint32_t>(icc_bytes[1]) << 16 | + static_cast<uint32_t>(icc_bytes[2]) << 8 | + static_cast<uint32_t>(icc_bytes[3]); + + EXPECT_EQ(static_cast<size_t>(encoded_size), profile_size); +} + +} // namespace android::ultrahdr + diff --git a/libs/ultrahdr/tests/jpegdecoderhelper_test.cpp b/libs/ultrahdr/tests/jpegdecoderhelper_test.cpp index c79dbe328b..e2da01c373 100644 --- a/libs/ultrahdr/tests/jpegdecoderhelper_test.cpp +++ b/libs/ultrahdr/tests/jpegdecoderhelper_test.cpp @@ -15,6 +15,7 @@ */ #include <ultrahdr/jpegdecoderhelper.h> +#include <ultrahdr/icc.h> #include <gtest/gtest.h> #include <utils/Log.h> @@ -22,11 +23,19 @@ namespace android::ultrahdr { +// No ICC or EXIF #define YUV_IMAGE "/sdcard/Documents/minnie-320x240-yuv.jpg" #define YUV_IMAGE_SIZE 20193 +// Has ICC and EXIF +#define YUV_ICC_IMAGE "/sdcard/Documents/minnie-320x240-yuv-icc.jpg" +#define YUV_ICC_IMAGE_SIZE 34266 +// No ICC or EXIF #define GREY_IMAGE "/sdcard/Documents/minnie-320x240-y.jpg" #define GREY_IMAGE_SIZE 20193 +#define IMAGE_WIDTH 320 +#define IMAGE_HEIGHT 240 + class JpegDecoderHelperTest : public testing::Test { public: struct Image { @@ -39,7 +48,7 @@ protected: virtual void SetUp(); virtual void TearDown(); - Image mYuvImage, mGreyImage; + Image mYuvImage, mYuvIccImage, mGreyImage; }; JpegDecoderHelperTest::JpegDecoderHelperTest() {} @@ -79,6 +88,10 @@ void JpegDecoderHelperTest::SetUp() { FAIL() << "Load file " << YUV_IMAGE << " failed"; } mYuvImage.size = YUV_IMAGE_SIZE; + if (!loadFile(YUV_ICC_IMAGE, &mYuvIccImage)) { + FAIL() << "Load file " << YUV_ICC_IMAGE << " failed"; + } + mYuvIccImage.size = YUV_ICC_IMAGE_SIZE; if (!loadFile(GREY_IMAGE, &mGreyImage)) { FAIL() << "Load file " << GREY_IMAGE << " failed"; } @@ -91,6 +104,16 @@ TEST_F(JpegDecoderHelperTest, decodeYuvImage) { JpegDecoderHelper decoder; EXPECT_TRUE(decoder.decompressImage(mYuvImage.buffer.get(), mYuvImage.size)); ASSERT_GT(decoder.getDecompressedImageSize(), static_cast<uint32_t>(0)); + EXPECT_EQ(IccHelper::readIccColorGamut(decoder.getICCPtr(), decoder.getICCSize()), + ULTRAHDR_COLORGAMUT_UNSPECIFIED); +} + +TEST_F(JpegDecoderHelperTest, decodeYuvIccImage) { + JpegDecoderHelper decoder; + EXPECT_TRUE(decoder.decompressImage(mYuvIccImage.buffer.get(), mYuvIccImage.size)); + ASSERT_GT(decoder.getDecompressedImageSize(), static_cast<uint32_t>(0)); + EXPECT_EQ(IccHelper::readIccColorGamut(decoder.getICCPtr(), decoder.getICCSize()), + ULTRAHDR_COLORGAMUT_BT709); } TEST_F(JpegDecoderHelperTest, decodeGreyImage) { @@ -99,4 +122,35 @@ TEST_F(JpegDecoderHelperTest, decodeGreyImage) { ASSERT_GT(decoder.getDecompressedImageSize(), static_cast<uint32_t>(0)); } -} // namespace android::ultrahdr
\ No newline at end of file +TEST_F(JpegDecoderHelperTest, getCompressedImageParameters) { + size_t width = 0, height = 0; + std::vector<uint8_t> icc, exif; + + JpegDecoderHelper decoder; + EXPECT_TRUE(decoder.getCompressedImageParameters(mYuvImage.buffer.get(), mYuvImage.size, + &width, &height, &icc, &exif)); + + EXPECT_EQ(width, IMAGE_WIDTH); + EXPECT_EQ(height, IMAGE_HEIGHT); + EXPECT_EQ(icc.size(), 0); + EXPECT_EQ(exif.size(), 0); +} + +TEST_F(JpegDecoderHelperTest, getCompressedImageParametersIcc) { + size_t width = 0, height = 0; + std::vector<uint8_t> icc, exif; + + JpegDecoderHelper decoder; + EXPECT_TRUE(decoder.getCompressedImageParameters(mYuvIccImage.buffer.get(), mYuvIccImage.size, + &width, &height, &icc, &exif)); + + EXPECT_EQ(width, IMAGE_WIDTH); + EXPECT_EQ(height, IMAGE_HEIGHT); + EXPECT_GT(icc.size(), 0); + EXPECT_GT(exif.size(), 0); + + EXPECT_EQ(IccHelper::readIccColorGamut(icc.data(), icc.size()), + ULTRAHDR_COLORGAMUT_BT709); +} + +} // namespace android::ultrahdr diff --git a/libs/ultrahdr/tests/jpegencoderhelper_test.cpp b/libs/ultrahdr/tests/jpegencoderhelper_test.cpp index 8f18ac0004..f0e1fa4968 100644 --- a/libs/ultrahdr/tests/jpegencoderhelper_test.cpp +++ b/libs/ultrahdr/tests/jpegencoderhelper_test.cpp @@ -108,18 +108,9 @@ TEST_F(JpegEncoderHelperTest, encodeAlignedImage) { ASSERT_GT(encoder.getCompressedImageSize(), static_cast<uint32_t>(0)); } -// The width of the "unaligned" image is not 16-aligned, and will fail if encoded directly. -// Should pass with the padding zero method. TEST_F(JpegEncoderHelperTest, encodeUnalignedImage) { JpegEncoderHelper encoder; - const size_t paddingZeroLength = JpegEncoderHelper::kCompressBatchSize - * JpegEncoderHelper::kCompressBatchSize / 4; - std::unique_ptr<uint8_t[]> imageWithPaddingZeros( - new uint8_t[UNALIGNED_IMAGE_WIDTH * UNALIGNED_IMAGE_HEIGHT * 3 / 2 - + paddingZeroLength]); - memcpy(imageWithPaddingZeros.get(), mUnalignedImage.buffer.get(), - UNALIGNED_IMAGE_WIDTH * UNALIGNED_IMAGE_HEIGHT * 3 / 2); - EXPECT_TRUE(encoder.compressImage(imageWithPaddingZeros.get(), mUnalignedImage.width, + EXPECT_TRUE(encoder.compressImage(mUnalignedImage.buffer.get(), mUnalignedImage.width, mUnalignedImage.height, JPEG_QUALITY, NULL, 0)); ASSERT_GT(encoder.getCompressedImageSize(), static_cast<uint32_t>(0)); } diff --git a/libs/ultrahdr/tests/jpegr_test.cpp b/libs/ultrahdr/tests/jpegr_test.cpp index ac358872b4..41d55ec497 100644 --- a/libs/ultrahdr/tests/jpegr_test.cpp +++ b/libs/ultrahdr/tests/jpegr_test.cpp @@ -89,6 +89,51 @@ static bool loadFile(const char filename[], void*& result, int* fileLength) { return true; } +static bool loadP010Image(const char *filename, jr_uncompressed_ptr img, + bool isUVContiguous) { + int fd = open(filename, O_CLOEXEC); + if (fd < 0) { + return false; + } + const int bpp = 2; + int lumaStride = img->luma_stride == 0 ? img->width : img->luma_stride; + int lumaSize = bpp * lumaStride * img->height; + int chromaSize = bpp * (img->height / 2) * + (isUVContiguous ? lumaStride : img->chroma_stride); + img->data = malloc(lumaSize + (isUVContiguous ? chromaSize : 0)); + if (img->data == nullptr) { + ALOGE("loadP010Image(): failed to allocate memory for luma data."); + return false; + } + uint8_t *mem = static_cast<uint8_t *>(img->data); + for (int i = 0; i < img->height; i++) { + if (read(fd, mem, img->width * bpp) != img->width * bpp) { + close(fd); + return false; + } + mem += lumaStride * bpp; + } + int chromaStride = lumaStride; + if (!isUVContiguous) { + img->chroma_data = malloc(chromaSize); + if (img->chroma_data == nullptr) { + ALOGE("loadP010Image(): failed to allocate memory for chroma data."); + return false; + } + mem = static_cast<uint8_t *>(img->chroma_data); + chromaStride = img->chroma_stride; + } + for (int i = 0; i < img->height / 2; i++) { + if (read(fd, mem, img->width * bpp) != img->width * bpp) { + close(fd); + return false; + } + mem += chromaStride * bpp; + } + close(fd); + return true; +} + class JpegRTest : public testing::Test { public: JpegRTest(); @@ -98,10 +143,11 @@ protected: virtual void SetUp(); virtual void TearDown(); - struct jpegr_uncompressed_struct mRawP010Image; - struct jpegr_uncompressed_struct mRawP010ImageWithStride; - struct jpegr_uncompressed_struct mRawYuv420Image; - struct jpegr_compressed_struct mJpegImage; + struct jpegr_uncompressed_struct mRawP010Image{}; + struct jpegr_uncompressed_struct mRawP010ImageWithStride{}; + struct jpegr_uncompressed_struct mRawP010ImageWithChromaData{}; + struct jpegr_uncompressed_struct mRawYuv420Image{}; + struct jpegr_compressed_struct mJpegImage{}; }; JpegRTest::JpegRTest() {} @@ -110,7 +156,11 @@ JpegRTest::~JpegRTest() {} void JpegRTest::SetUp() {} void JpegRTest::TearDown() { free(mRawP010Image.data); + free(mRawP010Image.chroma_data); free(mRawP010ImageWithStride.data); + free(mRawP010ImageWithStride.chroma_data); + free(mRawP010ImageWithChromaData.data); + free(mRawP010ImageWithChromaData.chroma_data); free(mRawYuv420Image.data); free(mJpegImage.data); } @@ -286,6 +336,8 @@ TEST_F(JpegRTest, encodeAPI0ForInvalidArgs) { &mRawP010ImageWithStride, ultrahdr_transfer_function::ULTRAHDR_TF_HLG, &jpegR, DEFAULT_JPEG_QUALITY, nullptr)) << "fail, API allows bad chroma stride"; + mRawP010ImageWithStride.chroma_data = nullptr; + free(jpegR.data); } @@ -734,6 +786,7 @@ TEST_F(JpegRTest, encodeAPI3ForInvalidArgs) { EXPECT_NE(OK, jpegRCodec.encodeJPEGR( &mRawP010ImageWithStride, &jpegR, ultrahdr_transfer_function::ULTRAHDR_TF_HLG, &jpegR)) << "fail, API allows bad chroma stride"; + mRawP010ImageWithStride.chroma_data = nullptr; // bad compressed image EXPECT_NE(OK, jpegRCodec.encodeJPEGR( @@ -766,14 +819,104 @@ TEST_F(JpegRTest, encodeAPI4ForInvalidArgs) { EXPECT_NE(OK, jpegRCodec.encodeJPEGR( &jpegR, nullptr, nullptr, &jpegR)) << "fail, API allows nullptr gainmap image"; + // test metadata + ultrahdr_metadata_struct good_metadata; + good_metadata.version = "1.0"; + good_metadata.minContentBoost = 1.0f; + good_metadata.maxContentBoost = 2.0f; + good_metadata.gamma = 1.0f; + good_metadata.offsetSdr = 0.0f; + good_metadata.offsetHdr = 0.0f; + good_metadata.hdrCapacityMin = 1.0f; + good_metadata.hdrCapacityMax = 2.0f; + + ultrahdr_metadata_struct metadata = good_metadata; + metadata.version = "1.1"; + EXPECT_NE(OK, jpegRCodec.encodeJPEGR( + &jpegR, nullptr, &metadata, &jpegR)) << "fail, API allows bad metadata version"; + + metadata = good_metadata; + metadata.minContentBoost = 3.0f; + EXPECT_NE(OK, jpegRCodec.encodeJPEGR( + &jpegR, nullptr, &metadata, &jpegR)) << "fail, API allows bad metadata content boost"; + + metadata = good_metadata; + metadata.gamma = -0.1f; + EXPECT_NE(OK, jpegRCodec.encodeJPEGR( + &jpegR, nullptr, &metadata, &jpegR)) << "fail, API allows bad metadata gamma"; + + metadata = good_metadata; + metadata.offsetSdr = -0.1f; + EXPECT_NE(OK, jpegRCodec.encodeJPEGR( + &jpegR, nullptr, &metadata, &jpegR)) << "fail, API allows bad metadata offset sdr"; + + metadata = good_metadata; + metadata.offsetHdr = -0.1f; + EXPECT_NE(OK, jpegRCodec.encodeJPEGR( + &jpegR, nullptr, &metadata, &jpegR)) << "fail, API allows bad metadata offset hdr"; + + metadata = good_metadata; + metadata.hdrCapacityMax = 0.5f; + EXPECT_NE(OK, jpegRCodec.encodeJPEGR( + &jpegR, nullptr, &metadata, &jpegR)) << "fail, API allows bad metadata hdr capacity max"; + + metadata = good_metadata; + metadata.hdrCapacityMin = 0.5f; + EXPECT_NE(OK, jpegRCodec.encodeJPEGR( + &jpegR, nullptr, &metadata, &jpegR)) << "fail, API allows bad metadata hdr capacity min"; + + free(jpegR.data); +} + +/* Test Decode API invalid arguments */ +TEST_F(JpegRTest, decodeAPIForInvalidArgs) { + int ret; + + // we are not really compressing anything so lets keep allocs to a minimum + jpegr_compressed_struct jpegR; + jpegR.maxLength = 16 * sizeof(uint8_t); + jpegR.data = malloc(jpegR.maxLength); + + // we are not really decoding anything so lets keep allocs to a minimum + mRawP010Image.data = malloc(16); + + JpegR jpegRCodec; + + // test jpegr image + EXPECT_NE(OK, jpegRCodec.decodeJPEGR( + nullptr, &mRawP010Image)) << "fail, API allows nullptr for jpegr img"; + + // test dest image + EXPECT_NE(OK, jpegRCodec.decodeJPEGR( + &jpegR, nullptr)) << "fail, API allows nullptr for dest"; + + // test max display boost + EXPECT_NE(OK, jpegRCodec.decodeJPEGR( + &jpegR, &mRawP010Image, 0.5)) << "fail, API allows invalid max display boost"; + + // test output format + EXPECT_NE(OK, jpegRCodec.decodeJPEGR( + &jpegR, &mRawP010Image, 0.5, nullptr, + static_cast<ultrahdr_output_format>(-1))) << "fail, API allows invalid output format"; + + EXPECT_NE(OK, jpegRCodec.decodeJPEGR( + &jpegR, &mRawP010Image, 0.5, nullptr, + static_cast<ultrahdr_output_format>(ULTRAHDR_OUTPUT_MAX + 1))) + << "fail, API allows invalid output format"; + free(jpegR.data); } TEST_F(JpegRTest, writeXmpThenRead) { ultrahdr_metadata_struct metadata_expected; metadata_expected.version = "1.0"; - metadata_expected.maxContentBoost = 1.25; - metadata_expected.minContentBoost = 0.75; + metadata_expected.maxContentBoost = 1.25f; + metadata_expected.minContentBoost = 0.75f; + metadata_expected.gamma = 1.0f; + metadata_expected.offsetSdr = 0.0f; + metadata_expected.offsetHdr = 0.0f; + metadata_expected.hdrCapacityMin = 1.0f; + metadata_expected.hdrCapacityMax = metadata_expected.maxContentBoost; const std::string nameSpace = "http://ns.adobe.com/xap/1.0/\0"; const int nameSpaceLength = nameSpace.size() + 1; // need to count the null terminator @@ -790,6 +933,86 @@ TEST_F(JpegRTest, writeXmpThenRead) { EXPECT_TRUE(getMetadataFromXMP(xmpData.data(), xmpData.size(), &metadata_read)); EXPECT_FLOAT_EQ(metadata_expected.maxContentBoost, metadata_read.maxContentBoost); EXPECT_FLOAT_EQ(metadata_expected.minContentBoost, metadata_read.minContentBoost); + EXPECT_FLOAT_EQ(metadata_expected.gamma, metadata_read.gamma); + EXPECT_FLOAT_EQ(metadata_expected.offsetSdr, metadata_read.offsetSdr); + EXPECT_FLOAT_EQ(metadata_expected.offsetHdr, metadata_read.offsetHdr); + EXPECT_FLOAT_EQ(metadata_expected.hdrCapacityMin, metadata_read.hdrCapacityMin); + EXPECT_FLOAT_EQ(metadata_expected.hdrCapacityMax, metadata_read.hdrCapacityMax); +} + +/* Test Encode API-0 */ +TEST_F(JpegRTest, encodeFromP010) { + int ret; + + mRawP010Image.width = TEST_IMAGE_WIDTH; + mRawP010Image.height = TEST_IMAGE_HEIGHT; + mRawP010Image.colorGamut = ultrahdr_color_gamut::ULTRAHDR_COLORGAMUT_BT2100; + // Load input files. + if (!loadP010Image(RAW_P010_IMAGE, &mRawP010Image, true)) { + FAIL() << "Load file " << RAW_P010_IMAGE << " failed"; + } + + JpegR jpegRCodec; + + jpegr_compressed_struct jpegR; + jpegR.maxLength = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * sizeof(uint8_t); + jpegR.data = malloc(jpegR.maxLength); + ret = jpegRCodec.encodeJPEGR( + &mRawP010Image, ultrahdr_transfer_function::ULTRAHDR_TF_HLG, &jpegR, DEFAULT_JPEG_QUALITY, + nullptr); + if (ret != OK) { + FAIL() << "Error code is " << ret; + } + + mRawP010ImageWithStride.width = TEST_IMAGE_WIDTH; + mRawP010ImageWithStride.height = TEST_IMAGE_HEIGHT; + mRawP010ImageWithStride.luma_stride = TEST_IMAGE_WIDTH + 128; + mRawP010ImageWithStride.colorGamut = ultrahdr_color_gamut::ULTRAHDR_COLORGAMUT_BT2100; + // Load input files. + if (!loadP010Image(RAW_P010_IMAGE, &mRawP010ImageWithStride, true)) { + FAIL() << "Load file " << RAW_P010_IMAGE << " failed"; + } + + jpegr_compressed_struct jpegRWithStride; + jpegRWithStride.maxLength = jpegR.length; + jpegRWithStride.data = malloc(jpegRWithStride.maxLength); + ret = jpegRCodec.encodeJPEGR( + &mRawP010ImageWithStride, ultrahdr_transfer_function::ULTRAHDR_TF_HLG, &jpegRWithStride, + DEFAULT_JPEG_QUALITY, nullptr); + if (ret != OK) { + FAIL() << "Error code is " << ret; + } + ASSERT_EQ(jpegR.length, jpegRWithStride.length) + << "Same input is yielding different output"; + ASSERT_EQ(0, memcmp(jpegR.data, jpegRWithStride.data, jpegR.length)) + << "Same input is yielding different output"; + + mRawP010ImageWithChromaData.width = TEST_IMAGE_WIDTH; + mRawP010ImageWithChromaData.height = TEST_IMAGE_HEIGHT; + mRawP010ImageWithChromaData.luma_stride = TEST_IMAGE_WIDTH + 64; + mRawP010ImageWithChromaData.chroma_stride = TEST_IMAGE_WIDTH + 256; + mRawP010ImageWithChromaData.colorGamut = ultrahdr_color_gamut::ULTRAHDR_COLORGAMUT_BT2100; + // Load input files. + if (!loadP010Image(RAW_P010_IMAGE, &mRawP010ImageWithChromaData, false)) { + FAIL() << "Load file " << RAW_P010_IMAGE << " failed"; + } + jpegr_compressed_struct jpegRWithChromaData; + jpegRWithChromaData.maxLength = jpegR.length; + jpegRWithChromaData.data = malloc(jpegRWithChromaData.maxLength); + ret = jpegRCodec.encodeJPEGR( + &mRawP010ImageWithChromaData, ultrahdr_transfer_function::ULTRAHDR_TF_HLG, + &jpegRWithChromaData, DEFAULT_JPEG_QUALITY, nullptr); + if (ret != OK) { + FAIL() << "Error code is " << ret; + } + ASSERT_EQ(jpegR.length, jpegRWithChromaData.length) + << "Same input is yielding different output"; + ASSERT_EQ(0, memcmp(jpegR.data, jpegRWithChromaData.data, jpegR.length)) + << "Same input is yielding different output"; + + free(jpegR.data); + free(jpegRWithStride.data); + free(jpegRWithChromaData.data); } /* Test Encode API-0 and decode */ @@ -1130,9 +1353,7 @@ TEST_F(JpegRTest, ProfileGainMapFuncs) { JpegRBenchmark benchmark; - ultrahdr_metadata_struct metadata = { .version = "1.0", - .maxContentBoost = 8.0f, - .minContentBoost = 1.0f / 8.0f }; + ultrahdr_metadata_struct metadata = { .version = "1.0" }; jpegr_uncompressed_struct map = { .data = NULL, .width = 0, diff --git a/services/inputflinger/dispatcher/InputDispatcher.cpp b/services/inputflinger/dispatcher/InputDispatcher.cpp index bdd45dc9b4..fbbb38835a 100644 --- a/services/inputflinger/dispatcher/InputDispatcher.cpp +++ b/services/inputflinger/dispatcher/InputDispatcher.cpp @@ -5659,14 +5659,6 @@ void InputDispatcher::dumpDispatchStateLocked(std::string& dump) const { } else { dump += INDENT "Displays: <none>\n"; } - dump += INDENT "Window Infos:\n"; - dump += StringPrintf(INDENT2 "vsync id: %" PRId64 "\n", mWindowInfosVsyncId); - dump += StringPrintf(INDENT2 "timestamp (ns): %" PRId64 "\n", mWindowInfosTimestamp); - dump += "\n"; - dump += StringPrintf(INDENT2 "max update delay (ns): %" PRId64 "\n", mMaxWindowInfosDelay); - dump += StringPrintf(INDENT2 "max update delay vsync id: %" PRId64 "\n", - mMaxWindowInfosDelayVsyncId); - dump += "\n"; if (!mGlobalMonitorsByDisplay.empty()) { for (const auto& [displayId, monitors] : mGlobalMonitorsByDisplay) { @@ -6709,14 +6701,12 @@ void InputDispatcher::onWindowInfosChanged(const gui::WindowInfosUpdate& update) setInputWindowsLocked(handles, displayId); } - mWindowInfosVsyncId = update.vsyncId; - mWindowInfosTimestamp = update.timestamp; - - int64_t delay = systemTime() - update.timestamp; - if (delay > mMaxWindowInfosDelay) { - mMaxWindowInfosDelay = delay; - mMaxWindowInfosDelayVsyncId = update.vsyncId; + if (update.vsyncId < mWindowInfosVsyncId) { + ALOGE("Received out of order window infos update. Last update vsync id: %" PRId64 + ", current update vsync id: %" PRId64, + mWindowInfosVsyncId, update.vsyncId); } + mWindowInfosVsyncId = update.vsyncId; } // Wake up poll loop since it may need to make new input dispatching choices. mLooper->wake(); diff --git a/services/inputflinger/dispatcher/InputDispatcher.h b/services/inputflinger/dispatcher/InputDispatcher.h index 0e9cfeffe4..6b22f2f24f 100644 --- a/services/inputflinger/dispatcher/InputDispatcher.h +++ b/services/inputflinger/dispatcher/InputDispatcher.h @@ -205,9 +205,6 @@ private: const IdGenerator mIdGenerator; int64_t mWindowInfosVsyncId GUARDED_BY(mLock); - int64_t mWindowInfosTimestamp GUARDED_BY(mLock); - int64_t mMaxWindowInfosDelay GUARDED_BY(mLock) = -1; - int64_t mMaxWindowInfosDelayVsyncId GUARDED_BY(mLock) = -1; // With each iteration, InputDispatcher nominally processes one queued event, // a timeout, or a response from an input consumer. diff --git a/services/inputflinger/tests/Android.bp b/services/inputflinger/tests/Android.bp index 52277ff078..569690ab78 100644 --- a/services/inputflinger/tests/Android.bp +++ b/services/inputflinger/tests/Android.bp @@ -40,6 +40,7 @@ cc_test { "AnrTracker_test.cpp", "BlockingQueue_test.cpp", "CapturedTouchpadEventConverter_test.cpp", + "CursorInputMapper_test.cpp", "EventHub_test.cpp", "FakeEventHub.cpp", "FakeInputReaderPolicy.cpp", @@ -58,6 +59,7 @@ cc_test { "PreferStylusOverTouch_test.cpp", "PropertyProvider_test.cpp", "TestInputListener.cpp", + "TouchpadInputMapper_test.cpp", "UinputDevice.cpp", "UnwantedInteractionBlocker_test.cpp", ], diff --git a/services/inputflinger/tests/CursorInputMapper_test.cpp b/services/inputflinger/tests/CursorInputMapper_test.cpp new file mode 100644 index 0000000000..6774b1793f --- /dev/null +++ b/services/inputflinger/tests/CursorInputMapper_test.cpp @@ -0,0 +1,105 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CursorInputMapper.h" + +#include <android-base/logging.h> +#include <gtest/gtest.h> + +#include "FakePointerController.h" +#include "InputMapperTest.h" +#include "InterfaceMocks.h" +#include "TestInputListenerMatchers.h" + +#define TAG "CursorInputMapper_test" + +namespace android { + +using testing::Return; +using testing::VariantWith; +constexpr auto ACTION_DOWN = AMOTION_EVENT_ACTION_DOWN; +constexpr auto ACTION_MOVE = AMOTION_EVENT_ACTION_MOVE; +constexpr auto ACTION_UP = AMOTION_EVENT_ACTION_UP; +constexpr auto BUTTON_PRESS = AMOTION_EVENT_ACTION_BUTTON_PRESS; +constexpr auto BUTTON_RELEASE = AMOTION_EVENT_ACTION_BUTTON_RELEASE; +constexpr auto HOVER_MOVE = AMOTION_EVENT_ACTION_HOVER_MOVE; + +/** + * Unit tests for CursorInputMapper. + * This class is named 'CursorInputMapperUnitTest' to avoid name collision with the existing + * 'CursorInputMapperTest'. If all of the CursorInputMapper tests are migrated here, the name + * can be simplified to 'CursorInputMapperTest'. + * TODO(b/283812079): move CursorInputMapper tests here. + */ +class CursorInputMapperUnitTest : public InputMapperUnitTest { +protected: + void SetUp() override { + InputMapperUnitTest::SetUp(); + + // Current scan code state - all keys are UP by default + setScanCodeState(KeyState::UP, + {BTN_LEFT, BTN_RIGHT, BTN_MIDDLE, BTN_BACK, BTN_SIDE, BTN_FORWARD, + BTN_EXTRA, BTN_TASK}); + EXPECT_CALL(mMockEventHub, hasRelativeAxis(EVENTHUB_ID, REL_WHEEL)) + .WillRepeatedly(Return(false)); + EXPECT_CALL(mMockEventHub, hasRelativeAxis(EVENTHUB_ID, REL_HWHEEL)) + .WillRepeatedly(Return(false)); + + EXPECT_CALL(mMockInputReaderContext, bumpGeneration()).WillRepeatedly(Return(1)); + + mMapper = createInputMapper<CursorInputMapper>(*mDeviceContext, mReaderConfiguration); + } +}; + +/** + * Move the mouse and then click the button. Check whether HOVER_EXIT is generated when hovering + * ends. Currently, it is not. + */ +TEST_F(CursorInputMapperUnitTest, HoverAndLeftButtonPress) { + std::list<NotifyArgs> args; + + // Move the cursor a little + args += process(EV_REL, REL_X, 10); + args += process(EV_REL, REL_Y, 20); + args += process(EV_SYN, SYN_REPORT, 0); + ASSERT_THAT(args, ElementsAre(VariantWith<NotifyMotionArgs>(WithMotionAction(HOVER_MOVE)))); + + // Now click the mouse button + args.clear(); + args += process(EV_KEY, BTN_LEFT, 1); + args += process(EV_SYN, SYN_REPORT, 0); + ASSERT_THAT(args, + ElementsAre(VariantWith<NotifyMotionArgs>(WithMotionAction(ACTION_DOWN)), + VariantWith<NotifyMotionArgs>(WithMotionAction(BUTTON_PRESS)))); + + // Move some more. + args.clear(); + args += process(EV_REL, REL_X, 10); + args += process(EV_REL, REL_Y, 20); + args += process(EV_SYN, SYN_REPORT, 0); + ASSERT_THAT(args, ElementsAre(VariantWith<NotifyMotionArgs>(WithMotionAction(ACTION_MOVE)))); + + // Release the button + args.clear(); + args += process(EV_KEY, BTN_LEFT, 0); + args += process(EV_SYN, SYN_REPORT, 0); + ASSERT_THAT(args, + ElementsAre(VariantWith<NotifyMotionArgs>(WithMotionAction(BUTTON_RELEASE)), + VariantWith<NotifyMotionArgs>(WithMotionAction(ACTION_UP)), + VariantWith<NotifyMotionArgs>(WithMotionAction(HOVER_MOVE)))); +} + +} // namespace android diff --git a/services/inputflinger/tests/InputMapperTest.cpp b/services/inputflinger/tests/InputMapperTest.cpp index ad48a79731..0eee2b9be4 100644 --- a/services/inputflinger/tests/InputMapperTest.cpp +++ b/services/inputflinger/tests/InputMapperTest.cpp @@ -22,6 +22,74 @@ namespace android { +using testing::Return; + +void InputMapperUnitTest::SetUp() { + mFakePointerController = std::make_shared<FakePointerController>(); + mFakePointerController->setBounds(0, 0, 800 - 1, 480 - 1); + mFakePointerController->setPosition(400, 240); + + EXPECT_CALL(mMockInputReaderContext, getPointerController(DEVICE_ID)) + .WillRepeatedly(Return(mFakePointerController)); + + EXPECT_CALL(mMockInputReaderContext, getEventHub()).WillRepeatedly(Return(&mMockEventHub)); + InputDeviceIdentifier identifier; + identifier.name = "device"; + identifier.location = "USB1"; + identifier.bus = 0; + + EXPECT_CALL(mMockEventHub, getDeviceIdentifier(EVENTHUB_ID)).WillRepeatedly(Return(identifier)); + mDevice = std::make_unique<InputDevice>(&mMockInputReaderContext, DEVICE_ID, + /*generation=*/2, identifier); + mDeviceContext = std::make_unique<InputDeviceContext>(*mDevice, EVENTHUB_ID); +} + +void InputMapperUnitTest::setupAxis(int axis, bool valid, int32_t min, int32_t max, + int32_t resolution) { + EXPECT_CALL(mMockEventHub, getAbsoluteAxisInfo(EVENTHUB_ID, axis, testing::_)) + .WillRepeatedly([=](int32_t, int32_t, RawAbsoluteAxisInfo* outAxisInfo) { + outAxisInfo->valid = valid; + outAxisInfo->minValue = min; + outAxisInfo->maxValue = max; + outAxisInfo->flat = 0; + outAxisInfo->fuzz = 0; + outAxisInfo->resolution = resolution; + return valid ? OK : -1; + }); +} + +void InputMapperUnitTest::expectScanCodes(bool present, std::set<int> scanCodes) { + for (const auto& scanCode : scanCodes) { + EXPECT_CALL(mMockEventHub, hasScanCode(EVENTHUB_ID, scanCode)) + .WillRepeatedly(testing::Return(present)); + } +} + +void InputMapperUnitTest::setScanCodeState(KeyState state, std::set<int> scanCodes) { + for (const auto& scanCode : scanCodes) { + EXPECT_CALL(mMockEventHub, getScanCodeState(EVENTHUB_ID, scanCode)) + .WillRepeatedly(testing::Return(static_cast<int>(state))); + } +} + +void InputMapperUnitTest::setKeyCodeState(KeyState state, std::set<int> keyCodes) { + for (const auto& keyCode : keyCodes) { + EXPECT_CALL(mMockEventHub, getKeyCodeState(EVENTHUB_ID, keyCode)) + .WillRepeatedly(testing::Return(static_cast<int>(state))); + } +} + +std::list<NotifyArgs> InputMapperUnitTest::process(int32_t type, int32_t code, int32_t value) { + RawEvent event; + event.when = systemTime(SYSTEM_TIME_MONOTONIC); + event.readTime = event.when; + event.deviceId = mMapper->getDeviceContext().getEventHubId(); + event.type = type; + event.code = code; + event.value = value; + return mMapper->process(&event); +} + const char* InputMapperTest::DEVICE_NAME = "device"; const char* InputMapperTest::DEVICE_LOCATION = "USB1"; const ftl::Flags<InputDeviceClass> InputMapperTest::DEVICE_CLASSES = diff --git a/services/inputflinger/tests/InputMapperTest.h b/services/inputflinger/tests/InputMapperTest.h index 2b6655c45e..909bd9c056 100644 --- a/services/inputflinger/tests/InputMapperTest.h +++ b/services/inputflinger/tests/InputMapperTest.h @@ -23,16 +23,48 @@ #include <InputMapper.h> #include <NotifyArgs.h> #include <ftl/flags.h> +#include <gmock/gmock.h> #include <utils/StrongPointer.h> #include "FakeEventHub.h" #include "FakeInputReaderPolicy.h" #include "InstrumentedInputReader.h" +#include "InterfaceMocks.h" #include "TestConstants.h" #include "TestInputListener.h" namespace android { +class InputMapperUnitTest : public testing::Test { +protected: + static constexpr int32_t EVENTHUB_ID = 1; + static constexpr int32_t DEVICE_ID = END_RESERVED_ID + 1000; + virtual void SetUp() override; + + void setupAxis(int axis, bool valid, int32_t min, int32_t max, int32_t resolution); + + void expectScanCodes(bool present, std::set<int> scanCodes); + + void setScanCodeState(KeyState state, std::set<int> scanCodes); + + void setKeyCodeState(KeyState state, std::set<int> keyCodes); + + std::list<NotifyArgs> process(int32_t type, int32_t code, int32_t value); + + MockEventHubInterface mMockEventHub; + std::shared_ptr<FakePointerController> mFakePointerController; + MockInputReaderContext mMockInputReaderContext; + std::unique_ptr<InputDevice> mDevice; + + std::unique_ptr<InputDeviceContext> mDeviceContext; + InputReaderConfiguration mReaderConfiguration; + // The mapper should be created by the subclasses. + std::unique_ptr<InputMapper> mMapper; +}; + +/** + * Deprecated - use InputMapperUnitTest instead. + */ class InputMapperTest : public testing::Test { protected: static const char* DEVICE_NAME; diff --git a/services/inputflinger/tests/InterfaceMocks.h b/services/inputflinger/tests/InterfaceMocks.h new file mode 100644 index 0000000000..d720a902dc --- /dev/null +++ b/services/inputflinger/tests/InterfaceMocks.h @@ -0,0 +1,146 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <android-base/logging.h> +#include <gmock/gmock.h> + +namespace android { + +class MockInputReaderContext : public InputReaderContext { +public: + MOCK_METHOD(void, updateGlobalMetaState, (), (override)); + int32_t getGlobalMetaState() override { return 0; }; + + MOCK_METHOD(void, disableVirtualKeysUntil, (nsecs_t time), (override)); + MOCK_METHOD(bool, shouldDropVirtualKey, (nsecs_t now, int32_t keyCode, int32_t scanCode), + (override)); + + MOCK_METHOD(void, fadePointer, (), (override)); + MOCK_METHOD(std::shared_ptr<PointerControllerInterface>, getPointerController, + (int32_t deviceId), (override)); + + MOCK_METHOD(void, requestTimeoutAtTime, (nsecs_t when), (override)); + MOCK_METHOD(int32_t, bumpGeneration, (), (override)); + + MOCK_METHOD(void, getExternalStylusDevices, (std::vector<InputDeviceInfo> & outDevices), + (override)); + MOCK_METHOD(std::list<NotifyArgs>, dispatchExternalStylusState, (const StylusState& outState), + (override)); + + MOCK_METHOD(InputReaderPolicyInterface*, getPolicy, (), (override)); + MOCK_METHOD(EventHubInterface*, getEventHub, (), (override)); + + int32_t getNextId() override { return 1; }; + + MOCK_METHOD(void, updateLedMetaState, (int32_t metaState), (override)); + MOCK_METHOD(int32_t, getLedMetaState, (), (override)); +}; + +class MockEventHubInterface : public EventHubInterface { +public: + MOCK_METHOD(ftl::Flags<InputDeviceClass>, getDeviceClasses, (int32_t deviceId), (const)); + MOCK_METHOD(InputDeviceIdentifier, getDeviceIdentifier, (int32_t deviceId), (const)); + MOCK_METHOD(int32_t, getDeviceControllerNumber, (int32_t deviceId), (const)); + MOCK_METHOD(std::optional<PropertyMap>, getConfiguration, (int32_t deviceId), (const)); + MOCK_METHOD(status_t, getAbsoluteAxisInfo, + (int32_t deviceId, int axis, RawAbsoluteAxisInfo* outAxisInfo), (const)); + MOCK_METHOD(bool, hasRelativeAxis, (int32_t deviceId, int axis), (const)); + MOCK_METHOD(bool, hasInputProperty, (int32_t deviceId, int property), (const)); + MOCK_METHOD(bool, hasMscEvent, (int32_t deviceId, int mscEvent), (const)); + MOCK_METHOD(void, addKeyRemapping, (int32_t deviceId, int fromKeyCode, int toKeyCode), (const)); + MOCK_METHOD(status_t, mapKey, + (int32_t deviceId, int scanCode, int usageCode, int32_t metaState, + int32_t* outKeycode, int32_t* outMetaState, uint32_t* outFlags), + (const)); + MOCK_METHOD(status_t, mapAxis, (int32_t deviceId, int scanCode, AxisInfo* outAxisInfo), + (const)); + MOCK_METHOD(void, setExcludedDevices, (const std::vector<std::string>& devices)); + MOCK_METHOD(std::vector<RawEvent>, getEvents, (int timeoutMillis)); + MOCK_METHOD(std::vector<TouchVideoFrame>, getVideoFrames, (int32_t deviceId)); + MOCK_METHOD((base::Result<std::pair<InputDeviceSensorType, int32_t>>), mapSensor, + (int32_t deviceId, int32_t absCode), (const, override)); + MOCK_METHOD(std::vector<int32_t>, getRawBatteryIds, (int32_t deviceId), (const, override)); + MOCK_METHOD(std::optional<RawBatteryInfo>, getRawBatteryInfo, + (int32_t deviceId, int32_t BatteryId), (const, override)); + MOCK_METHOD(std::vector<int32_t>, getRawLightIds, (int32_t deviceId), (const, override)); + MOCK_METHOD(std::optional<RawLightInfo>, getRawLightInfo, (int32_t deviceId, int32_t lightId), + (const, override)); + MOCK_METHOD(std::optional<int32_t>, getLightBrightness, (int32_t deviceId, int32_t lightId), + (const, override)); + MOCK_METHOD(void, setLightBrightness, (int32_t deviceId, int32_t lightId, int32_t brightness), + (override)); + MOCK_METHOD((std::optional<std::unordered_map<LightColor, int32_t>>), getLightIntensities, + (int32_t deviceId, int32_t lightId), (const, override)); + MOCK_METHOD(void, setLightIntensities, + (int32_t deviceId, int32_t lightId, + (std::unordered_map<LightColor, int32_t>)intensities), + (override)); + + MOCK_METHOD(std::optional<RawLayoutInfo>, getRawLayoutInfo, (int32_t deviceId), + (const, override)); + MOCK_METHOD(int32_t, getScanCodeState, (int32_t deviceId, int32_t scanCode), (const, override)); + MOCK_METHOD(int32_t, getKeyCodeState, (int32_t deviceId, int32_t keyCode), (const, override)); + MOCK_METHOD(int32_t, getSwitchState, (int32_t deviceId, int32_t sw), (const, override)); + + MOCK_METHOD(status_t, getAbsoluteAxisValue, (int32_t deviceId, int32_t axis, int32_t* outValue), + (const, override)); + MOCK_METHOD(int32_t, getKeyCodeForKeyLocation, (int32_t deviceId, int32_t locationKeyCode), + (const, override)); + MOCK_METHOD(bool, markSupportedKeyCodes, + (int32_t deviceId, const std::vector<int32_t>& keyCodes, uint8_t* outFlags), + (const, override)); + + MOCK_METHOD(bool, hasScanCode, (int32_t deviceId, int32_t scanCode), (const, override)); + + MOCK_METHOD(bool, hasKeyCode, (int32_t deviceId, int32_t keyCode), (const, override)); + + MOCK_METHOD(bool, hasLed, (int32_t deviceId, int32_t led), (const, override)); + + MOCK_METHOD(void, setLedState, (int32_t deviceId, int32_t led, bool on), (override)); + + MOCK_METHOD(void, getVirtualKeyDefinitions, + (int32_t deviceId, std::vector<VirtualKeyDefinition>& outVirtualKeys), + (const, override)); + + MOCK_METHOD(const std::shared_ptr<KeyCharacterMap>, getKeyCharacterMap, (int32_t deviceId), + (const, override)); + + MOCK_METHOD(bool, setKeyboardLayoutOverlay, + (int32_t deviceId, std::shared_ptr<KeyCharacterMap> map), (override)); + + MOCK_METHOD(void, vibrate, (int32_t deviceId, const VibrationElement& effect), (override)); + MOCK_METHOD(void, cancelVibrate, (int32_t deviceId), (override)); + + MOCK_METHOD(std::vector<int32_t>, getVibratorIds, (int32_t deviceId), (const, override)); + MOCK_METHOD(std::optional<int32_t>, getBatteryCapacity, (int32_t deviceId, int32_t batteryId), + (const, override)); + + MOCK_METHOD(std::optional<int32_t>, getBatteryStatus, (int32_t deviceId, int32_t batteryId), + (const, override)); + MOCK_METHOD(void, requestReopenDevices, (), (override)); + MOCK_METHOD(void, wake, (), (override)); + + MOCK_METHOD(void, dump, (std::string & dump), (const, override)); + MOCK_METHOD(void, monitor, (), (const, override)); + MOCK_METHOD(bool, isDeviceEnabled, (int32_t deviceId), (const, override)); + MOCK_METHOD(status_t, enableDevice, (int32_t deviceId), (override)); + MOCK_METHOD(status_t, disableDevice, (int32_t deviceId), (override)); + MOCK_METHOD(void, sysfsNodeChanged, (const std::string& sysfsNodePath), (override)); +}; + +} // namespace android diff --git a/services/inputflinger/tests/TouchpadInputMapper_test.cpp b/services/inputflinger/tests/TouchpadInputMapper_test.cpp new file mode 100644 index 0000000000..92cd462c9a --- /dev/null +++ b/services/inputflinger/tests/TouchpadInputMapper_test.cpp @@ -0,0 +1,155 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TouchpadInputMapper.h" + +#include <android-base/logging.h> +#include <gtest/gtest.h> + +#include <thread> +#include "FakePointerController.h" +#include "InputMapperTest.h" +#include "InterfaceMocks.h" +#include "TestInputListenerMatchers.h" + +#define TAG "TouchpadInputMapper_test" + +namespace android { + +using testing::Return; +using testing::VariantWith; +constexpr auto ACTION_DOWN = AMOTION_EVENT_ACTION_DOWN; +constexpr auto ACTION_UP = AMOTION_EVENT_ACTION_UP; +constexpr auto BUTTON_PRESS = AMOTION_EVENT_ACTION_BUTTON_PRESS; +constexpr auto BUTTON_RELEASE = AMOTION_EVENT_ACTION_BUTTON_RELEASE; +constexpr auto HOVER_MOVE = AMOTION_EVENT_ACTION_HOVER_MOVE; + +/** + * Unit tests for TouchpadInputMapper. + */ +class TouchpadInputMapperTest : public InputMapperUnitTest { +protected: + void SetUp() override { + InputMapperUnitTest::SetUp(); + + // Present scan codes: BTN_TOUCH and BTN_TOOL_FINGER + expectScanCodes(/*present=*/true, + {BTN_LEFT, BTN_RIGHT, BTN_TOOL_FINGER, BTN_TOOL_QUINTTAP, BTN_TOUCH, + BTN_TOOL_DOUBLETAP, BTN_TOOL_TRIPLETAP, BTN_TOOL_QUADTAP}); + // Missing scan codes that the mapper checks for. + expectScanCodes(/*present=*/false, + {BTN_TOOL_PEN, BTN_TOOL_RUBBER, BTN_TOOL_BRUSH, BTN_TOOL_PENCIL, + BTN_TOOL_AIRBRUSH}); + + // Current scan code state - all keys are UP by default + setScanCodeState(KeyState::UP, {BTN_TOUCH, BTN_STYLUS, + BTN_STYLUS2, BTN_0, + BTN_TOOL_FINGER, BTN_TOOL_PEN, + BTN_TOOL_RUBBER, BTN_TOOL_BRUSH, + BTN_TOOL_PENCIL, BTN_TOOL_AIRBRUSH, + BTN_TOOL_MOUSE, BTN_TOOL_LENS, + BTN_TOOL_DOUBLETAP, BTN_TOOL_TRIPLETAP, + BTN_TOOL_QUADTAP, BTN_TOOL_QUINTTAP, + BTN_LEFT, BTN_RIGHT, + BTN_MIDDLE, BTN_BACK, + BTN_SIDE, BTN_FORWARD, + BTN_EXTRA, BTN_TASK}); + + setKeyCodeState(KeyState::UP, + {AKEYCODE_STYLUS_BUTTON_PRIMARY, AKEYCODE_STYLUS_BUTTON_SECONDARY}); + + // Key mappings + EXPECT_CALL(mMockEventHub, + mapKey(EVENTHUB_ID, BTN_LEFT, /*usageCode=*/0, /*metaState=*/0, testing::_, + testing::_, testing::_)) + .WillRepeatedly(Return(NAME_NOT_FOUND)); + + // Input properties - only INPUT_PROP_BUTTONPAD + EXPECT_CALL(mMockEventHub, hasInputProperty(EVENTHUB_ID, INPUT_PROP_BUTTONPAD)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(mMockEventHub, hasInputProperty(EVENTHUB_ID, INPUT_PROP_SEMI_MT)) + .WillRepeatedly(Return(false)); + + // Axes that the device has + setupAxis(ABS_MT_SLOT, /*valid=*/true, /*min=*/0, /*max=*/4, /*resolution=*/0); + setupAxis(ABS_MT_POSITION_X, /*valid=*/true, /*min=*/0, /*max=*/2000, /*resolution=*/24); + setupAxis(ABS_MT_POSITION_Y, /*valid=*/true, /*min=*/0, /*max=*/1000, /*resolution=*/24); + setupAxis(ABS_MT_PRESSURE, /*valid=*/true, /*min*/ 0, /*max=*/255, /*resolution=*/0); + // Axes that the device does not have + setupAxis(ABS_MT_ORIENTATION, /*valid=*/false, /*min=*/0, /*max=*/0, /*resolution=*/0); + setupAxis(ABS_MT_TOUCH_MAJOR, /*valid=*/false, /*min=*/0, /*max=*/0, /*resolution=*/0); + setupAxis(ABS_MT_TOUCH_MINOR, /*valid=*/false, /*min=*/0, /*max=*/0, /*resolution=*/0); + setupAxis(ABS_MT_WIDTH_MAJOR, /*valid=*/false, /*min=*/0, /*max=*/0, /*resolution=*/0); + setupAxis(ABS_MT_WIDTH_MINOR, /*valid=*/false, /*min=*/0, /*max=*/0, /*resolution=*/0); + + EXPECT_CALL(mMockEventHub, getAbsoluteAxisValue(EVENTHUB_ID, ABS_MT_SLOT, testing::_)) + .WillRepeatedly([](int32_t eventHubId, int32_t, int32_t* outValue) { + *outValue = 0; + return OK; + }); + mMapper = createInputMapper<TouchpadInputMapper>(*mDeviceContext, mReaderConfiguration); + } +}; + +/** + * Start moving the finger and then click the left touchpad button. Check whether HOVER_EXIT is + * generated when hovering stops. Currently, it is not. + * In the current implementation, HOVER_MOVE and ACTION_DOWN events are not sent out right away, + * but only after the button is released. + */ +TEST_F(TouchpadInputMapperTest, HoverAndLeftButtonPress) { + std::list<NotifyArgs> args; + + args += process(EV_ABS, ABS_MT_TRACKING_ID, 1); + args += process(EV_KEY, BTN_TOUCH, 1); + setScanCodeState(KeyState::DOWN, {BTN_TOOL_FINGER}); + args += process(EV_KEY, BTN_TOOL_FINGER, 1); + args += process(EV_ABS, ABS_MT_POSITION_X, 50); + args += process(EV_ABS, ABS_MT_POSITION_Y, 50); + args += process(EV_ABS, ABS_MT_PRESSURE, 1); + args += process(EV_SYN, SYN_REPORT, 0); + ASSERT_THAT(args, testing::IsEmpty()); + + // Without this sleep, the test fails. + // TODO(b/284133337): Figure out whether this can be removed + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + + args += process(EV_KEY, BTN_LEFT, 1); + setScanCodeState(KeyState::DOWN, {BTN_LEFT}); + args += process(EV_SYN, SYN_REPORT, 0); + + args += process(EV_KEY, BTN_LEFT, 0); + setScanCodeState(KeyState::UP, {BTN_LEFT}); + args += process(EV_SYN, SYN_REPORT, 0); + ASSERT_THAT(args, + ElementsAre(VariantWith<NotifyMotionArgs>(WithMotionAction(HOVER_MOVE)), + VariantWith<NotifyMotionArgs>(WithMotionAction(ACTION_DOWN)), + VariantWith<NotifyMotionArgs>(WithMotionAction(BUTTON_PRESS)), + VariantWith<NotifyMotionArgs>(WithMotionAction(BUTTON_RELEASE)), + VariantWith<NotifyMotionArgs>(WithMotionAction(ACTION_UP)))); + + // Liftoff + args.clear(); + args += process(EV_ABS, ABS_MT_PRESSURE, 0); + args += process(EV_ABS, ABS_MT_TRACKING_ID, -1); + args += process(EV_KEY, BTN_TOUCH, 0); + setScanCodeState(KeyState::UP, {BTN_TOOL_FINGER}); + args += process(EV_KEY, BTN_TOOL_FINGER, 0); + args += process(EV_SYN, SYN_REPORT, 0); + ASSERT_THAT(args, testing::IsEmpty()); +} + +} // namespace android diff --git a/services/surfaceflinger/DisplayHardware/PowerAdvisor.cpp b/services/surfaceflinger/DisplayHardware/PowerAdvisor.cpp index 37b68c865e..f8b466c93c 100644 --- a/services/surfaceflinger/DisplayHardware/PowerAdvisor.cpp +++ b/services/surfaceflinger/DisplayHardware/PowerAdvisor.cpp @@ -223,7 +223,7 @@ void PowerAdvisor::updateTargetWorkDuration(Duration targetDuration) { } void PowerAdvisor::reportActualWorkDuration() { - if (!mBootFinished || !usePowerHintSession()) { + if (!mBootFinished || !sUseReportActualDuration || !usePowerHintSession()) { ALOGV("Actual work duration power hint cannot be sent, skipping"); return; } @@ -564,6 +564,9 @@ const Duration PowerAdvisor::sTargetSafetyMargin = std::chrono::microseconds( base::GetIntProperty<int64_t>("debug.sf.hint_margin_us", ticks<std::micro>(PowerAdvisor::kDefaultTargetSafetyMargin))); +const bool PowerAdvisor::sUseReportActualDuration = + base::GetBoolProperty(std::string("debug.adpf.use_report_actual_duration"), true); + power::PowerHalController& PowerAdvisor::getPowerHal() { static std::once_flag halFlag; std::call_once(halFlag, [this] { mPowerHal->init(); }); diff --git a/services/surfaceflinger/DisplayHardware/PowerAdvisor.h b/services/surfaceflinger/DisplayHardware/PowerAdvisor.h index 7a0d4267fe..f0d3fd8518 100644 --- a/services/surfaceflinger/DisplayHardware/PowerAdvisor.h +++ b/services/surfaceflinger/DisplayHardware/PowerAdvisor.h @@ -269,6 +269,9 @@ private: static const Duration sTargetSafetyMargin; static constexpr const Duration kDefaultTargetSafetyMargin{1ms}; + // Whether we should send reportActualWorkDuration calls + static const bool sUseReportActualDuration; + // How long we expect hwc to run after the present call until it waits for the fence static constexpr const Duration kFenceWaitStartDelayValidated{150us}; static constexpr const Duration kFenceWaitStartDelaySkippedValidate{250us}; diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h index 38590e6f20..f7596e20e5 100644 --- a/services/surfaceflinger/Layer.h +++ b/services/surfaceflinger/Layer.h @@ -877,6 +877,7 @@ public: // TODO(b/238781169) Remove direct calls to RenderEngine::drawLayers that don't go through // CompositionEngine to create a single path for composing layers. void updateSnapshot(bool updateGeometry); + void updateChildrenSnapshots(bool updateGeometry); void updateMetadataSnapshot(const LayerMetadata& parentMetadata); void updateRelativeMetadataSnapshot(const LayerMetadata& relativeLayerMetadata, std::unordered_set<Layer*>& visited); @@ -1134,8 +1135,6 @@ private: bool hasSomethingToDraw() const { return hasEffect() || hasBufferOrSidebandStream(); } - void updateChildrenSnapshots(bool updateGeometry); - // Fills the provided vector with the currently available JankData and removes the processed // JankData from the pending list. void transferAvailableJankData(const std::deque<sp<CallbackHandle>>& handles, diff --git a/services/surfaceflinger/LayerRenderArea.cpp b/services/surfaceflinger/LayerRenderArea.cpp index d606cffe40..51d4ff854f 100644 --- a/services/surfaceflinger/LayerRenderArea.cpp +++ b/services/surfaceflinger/LayerRenderArea.cpp @@ -116,6 +116,8 @@ void LayerRenderArea::render(std::function<void()> drawLayers) { mLayer->setChildrenDrawingParent(mLayer); } } + mLayer->updateSnapshot(/*updateGeometry=*/true); + mLayer->updateChildrenSnapshots(/*updateGeometry=*/true); } } // namespace android diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp index b1d4b3c837..79378befcc 100644 --- a/services/surfaceflinger/SurfaceFlinger.cpp +++ b/services/surfaceflinger/SurfaceFlinger.cpp @@ -2485,7 +2485,10 @@ bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expe mPowerAdvisor->setFrameDelay(frameDelay); mPowerAdvisor->setTotalFrameTargetWorkDuration(idealSfWorkDuration); - mPowerAdvisor->updateTargetWorkDuration(vsyncPeriod); + + const auto& display = FTL_FAKE_GUARD(mStateLock, getDefaultDisplayDeviceLocked()).get(); + const Period idealVsyncPeriod = display->getActiveMode().fps.getPeriod(); + mPowerAdvisor->updateTargetWorkDuration(idealVsyncPeriod); } if (mRefreshRateOverlaySpinner) { @@ -2547,7 +2550,7 @@ bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expe } updateCursorAsync(); - updateInputFlinger(vsyncId); + updateInputFlinger(vsyncId, frameTime); if (mLayerTracingEnabled && !mLayerTracing.flagIsSet(LayerTracing::TRACE_COMPOSITION)) { // This will block and tracing should only be enabled for debugging. @@ -3740,7 +3743,7 @@ void SurfaceFlinger::commitTransactionsLocked(uint32_t transactionFlags) { doCommitTransactions(); } -void SurfaceFlinger::updateInputFlinger(VsyncId vsyncId) { +void SurfaceFlinger::updateInputFlinger(VsyncId vsyncId, TimePoint frameTime) { if (!mInputFlinger || (!mUpdateInputInfo && mInputWindowCommands.empty())) { return; } @@ -3752,8 +3755,6 @@ void SurfaceFlinger::updateInputFlinger(VsyncId vsyncId) { if (mUpdateInputInfo) { mUpdateInputInfo = false; updateWindowInfo = true; - mLastInputFlingerUpdateVsyncId = vsyncId; - mLastInputFlingerUpdateTimestamp = systemTime(); buildWindowInfos(windowInfos, displayInfos); } @@ -3775,17 +3776,17 @@ void SurfaceFlinger::updateInputFlinger(VsyncId vsyncId) { inputWindowCommands = std::move(mInputWindowCommands), inputFlinger = mInputFlinger, this, - visibleWindowsChanged]() { + visibleWindowsChanged, vsyncId, frameTime]() { ATRACE_NAME("BackgroundExecutor::updateInputFlinger"); if (updateWindowInfo) { mWindowInfosListenerInvoker - ->windowInfosChanged(std::move(windowInfos), std::move(displayInfos), + ->windowInfosChanged(gui::WindowInfosUpdate{std::move(windowInfos), + std::move(displayInfos), + vsyncId.value, frameTime.ns()}, std::move( inputWindowCommands.windowInfosReportedListeners), /* forceImmediateCall= */ visibleWindowsChanged || - !inputWindowCommands.focusRequests.empty(), - mLastInputFlingerUpdateVsyncId, - mLastInputFlingerUpdateTimestamp); + !inputWindowCommands.focusRequests.empty()); } else { // If there are listeners but no changes to input windows, call the listeners // immediately. @@ -6152,27 +6153,14 @@ void SurfaceFlinger::dumpAllLocked(const DumpArgs& args, const std::string& comp result.append("\n"); result.append("Window Infos:\n"); - StringAppendF(&result, " input flinger update vsync id: %" PRId64 "\n", - mLastInputFlingerUpdateVsyncId.value); - StringAppendF(&result, " input flinger update timestamp (ns): %" PRId64 "\n", - mLastInputFlingerUpdateTimestamp); + auto windowInfosDebug = mWindowInfosListenerInvoker->getDebugInfo(); + StringAppendF(&result, " max send vsync id: %" PRId64 "\n", + windowInfosDebug.maxSendDelayVsyncId.value); + StringAppendF(&result, " max send delay (ns): %" PRId64 " ns\n", + windowInfosDebug.maxSendDelayDuration); + StringAppendF(&result, " unsent messages: %" PRIu32 "\n", + windowInfosDebug.pendingMessageCount); result.append("\n"); - - if (int64_t unsentVsyncId = mWindowInfosListenerInvoker->getUnsentMessageVsyncId().value; - unsentVsyncId != -1) { - StringAppendF(&result, " unsent input flinger update vsync id: %" PRId64 "\n", - unsentVsyncId); - StringAppendF(&result, " unsent input flinger update timestamp (ns): %" PRId64 "\n", - mWindowInfosListenerInvoker->getUnsentMessageTimestamp()); - result.append("\n"); - } - - if (uint32_t pendingMessages = mWindowInfosListenerInvoker->getPendingMessageCount(); - pendingMessages != 0) { - StringAppendF(&result, " pending input flinger calls: %" PRIu32 "\n", - mWindowInfosListenerInvoker->getPendingMessageCount()); - result.append("\n"); - } } mat4 SurfaceFlinger::calculateColorMatrix(float saturation) { diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h index e2691ab39a..0bc506f1fe 100644 --- a/services/surfaceflinger/SurfaceFlinger.h +++ b/services/surfaceflinger/SurfaceFlinger.h @@ -722,7 +722,7 @@ private: void updateLayerHistory(const frontend::LayerSnapshot& snapshot); frontend::Update flushLifecycleUpdates() REQUIRES(kMainThreadContext); - void updateInputFlinger(VsyncId); + void updateInputFlinger(VsyncId vsyncId, TimePoint frameTime); void persistDisplayBrightness(bool needsComposite) REQUIRES(kMainThreadContext); void buildWindowInfos(std::vector<gui::WindowInfo>& outWindowInfos, std::vector<gui::DisplayInfo>& outDisplayInfos); @@ -1259,9 +1259,6 @@ private: VsyncId mLastCommittedVsyncId; - VsyncId mLastInputFlingerUpdateVsyncId; - nsecs_t mLastInputFlingerUpdateTimestamp; - // If blurs should be enabled on this device. bool mSupportsBlur = false; std::atomic<uint32_t> mFrameMissedCount = 0; diff --git a/services/surfaceflinger/WindowInfosListenerInvoker.cpp b/services/surfaceflinger/WindowInfosListenerInvoker.cpp index 2b62638c61..20699ef123 100644 --- a/services/surfaceflinger/WindowInfosListenerInvoker.cpp +++ b/services/surfaceflinger/WindowInfosListenerInvoker.cpp @@ -16,8 +16,11 @@ #include <ftl/small_vector.h> #include <gui/ISurfaceComposer.h> +#include <gui/TraceUtils.h> #include <gui/WindowInfosUpdate.h> +#include <scheduler/Time.h> +#include "BackgroundExecutor.h" #include "WindowInfosListenerInvoker.h" namespace android { @@ -26,7 +29,7 @@ using gui::DisplayInfo; using gui::IWindowInfosListener; using gui::WindowInfo; -using WindowInfosListenerVector = ftl::SmallVector<const sp<IWindowInfosListener>, 3>; +using WindowInfosListenerVector = ftl::SmallVector<const sp<gui::IWindowInfosListener>, 3>; struct WindowInfosReportedListenerInvoker : gui::BnWindowInfosReportedListener, IBinder::DeathRecipient { @@ -86,45 +89,19 @@ void WindowInfosListenerInvoker::binderDied(const wp<IBinder>& who) { } void WindowInfosListenerInvoker::windowInfosChanged( - std::vector<WindowInfo> windowInfos, std::vector<DisplayInfo> displayInfos, - WindowInfosReportedListenerSet reportedListeners, bool forceImmediateCall, VsyncId vsyncId, - nsecs_t timestamp) { - reportedListeners.insert(sp<WindowInfosListenerInvoker>::fromExisting(this)); - auto callListeners = [this, windowInfos = std::move(windowInfos), - displayInfos = std::move(displayInfos), vsyncId, - timestamp](WindowInfosReportedListenerSet reportedListeners) mutable { - WindowInfosListenerVector windowInfosListeners; - { - std::scoped_lock lock(mListenersMutex); - for (const auto& [_, listener] : mWindowInfosListeners) { - windowInfosListeners.push_back(listener); - } - } - - auto reportedInvoker = - sp<WindowInfosReportedListenerInvoker>::make(windowInfosListeners, - std::move(reportedListeners)); - - gui::WindowInfosUpdate update(std::move(windowInfos), std::move(displayInfos), - vsyncId.value, timestamp); - - for (const auto& listener : windowInfosListeners) { - sp<IBinder> asBinder = IInterface::asBinder(listener); - - // linkToDeath is used here to ensure that the windowInfosReportedListeners - // are called even if one of the windowInfosListeners dies before - // calling onWindowInfosReported. - asBinder->linkToDeath(reportedInvoker); + gui::WindowInfosUpdate update, WindowInfosReportedListenerSet reportedListeners, + bool forceImmediateCall) { + WindowInfosListenerVector listeners; + { + std::scoped_lock lock{mMessagesMutex}; - auto status = listener->onWindowInfosChanged(update, reportedInvoker); - if (!status.isOk()) { - reportedInvoker->onWindowInfosReported(); - } + if (!mDelayInfo) { + mDelayInfo = DelayInfo{ + .vsyncId = update.vsyncId, + .frameTime = update.timestamp, + }; } - }; - { - std::scoped_lock lock(mMessagesMutex); // If there are unacked messages and this isn't a forced call, then return immediately. // If a forced window infos change doesn't happen first, the update will be sent after // the WindowInfosReportedListeners are called. If a forced window infos change happens or @@ -132,44 +109,87 @@ void WindowInfosListenerInvoker::windowInfosChanged( // will be dropped and the listeners will only be called with the latest info. This is done // to reduce the amount of binder memory used. if (mActiveMessageCount > 0 && !forceImmediateCall) { - mWindowInfosChangedDelayed = std::move(callListeners); - mUnsentVsyncId = vsyncId; - mUnsentTimestamp = timestamp; - mReportedListenersDelayed.merge(reportedListeners); + mDelayedUpdate = std::move(update); + mReportedListeners.merge(reportedListeners); + return; + } + + if (mDelayedUpdate) { + mDelayedUpdate.reset(); + } + + { + std::scoped_lock lock{mListenersMutex}; + for (const auto& [_, listener] : mWindowInfosListeners) { + listeners.push_back(listener); + } + } + if (CC_UNLIKELY(listeners.empty())) { + mReportedListeners.merge(reportedListeners); + mDelayInfo.reset(); return; } - mWindowInfosChangedDelayed = nullptr; - mUnsentVsyncId = {-1}; - mUnsentTimestamp = -1; - reportedListeners.merge(mReportedListenersDelayed); + reportedListeners.insert(sp<WindowInfosListenerInvoker>::fromExisting(this)); + reportedListeners.merge(mReportedListeners); + mReportedListeners.clear(); + mActiveMessageCount++; + updateMaxSendDelay(); + mDelayInfo.reset(); } - callListeners(std::move(reportedListeners)); -} -binder::Status WindowInfosListenerInvoker::onWindowInfosReported() { - std::function<void(WindowInfosReportedListenerSet)> callListeners; - WindowInfosReportedListenerSet reportedListeners; + auto reportedInvoker = + sp<WindowInfosReportedListenerInvoker>::make(listeners, std::move(reportedListeners)); - { - std::scoped_lock lock{mMessagesMutex}; - mActiveMessageCount--; - if (!mWindowInfosChangedDelayed || mActiveMessageCount > 0) { - return binder::Status::ok(); - } + for (const auto& listener : listeners) { + sp<IBinder> asBinder = IInterface::asBinder(listener); - mActiveMessageCount++; - callListeners = std::move(mWindowInfosChangedDelayed); - mWindowInfosChangedDelayed = nullptr; - mUnsentVsyncId = {-1}; - mUnsentTimestamp = -1; - reportedListeners = std::move(mReportedListenersDelayed); - mReportedListenersDelayed.clear(); + // linkToDeath is used here to ensure that the windowInfosReportedListeners + // are called even if one of the windowInfosListeners dies before + // calling onWindowInfosReported. + asBinder->linkToDeath(reportedInvoker); + + auto status = listener->onWindowInfosChanged(update, reportedInvoker); + if (!status.isOk()) { + reportedInvoker->onWindowInfosReported(); + } } +} - callListeners(std::move(reportedListeners)); +binder::Status WindowInfosListenerInvoker::onWindowInfosReported() { + BackgroundExecutor::getInstance().sendCallbacks({[this]() { + gui::WindowInfosUpdate update; + { + std::scoped_lock lock{mMessagesMutex}; + mActiveMessageCount--; + if (!mDelayedUpdate || mActiveMessageCount > 0) { + return; + } + update = std::move(*mDelayedUpdate); + mDelayedUpdate.reset(); + } + windowInfosChanged(std::move(update), {}, false); + }}); return binder::Status::ok(); } +WindowInfosListenerInvoker::DebugInfo WindowInfosListenerInvoker::getDebugInfo() { + std::scoped_lock lock{mMessagesMutex}; + updateMaxSendDelay(); + mDebugInfo.pendingMessageCount = mActiveMessageCount; + return mDebugInfo; +} + +void WindowInfosListenerInvoker::updateMaxSendDelay() { + if (!mDelayInfo) { + return; + } + nsecs_t delay = TimePoint::now().ns() - mDelayInfo->frameTime; + if (delay > mDebugInfo.maxSendDelayDuration) { + mDebugInfo.maxSendDelayDuration = delay; + mDebugInfo.maxSendDelayVsyncId = VsyncId{mDelayInfo->vsyncId}; + } +} + } // namespace android diff --git a/services/surfaceflinger/WindowInfosListenerInvoker.h b/services/surfaceflinger/WindowInfosListenerInvoker.h index e35d0564b5..bc465a3a2b 100644 --- a/services/surfaceflinger/WindowInfosListenerInvoker.h +++ b/services/surfaceflinger/WindowInfosListenerInvoker.h @@ -16,6 +16,7 @@ #pragma once +#include <optional> #include <unordered_set> #include <android/gui/BnWindowInfosReportedListener.h> @@ -40,26 +41,18 @@ public: void addWindowInfosListener(sp<gui::IWindowInfosListener>); void removeWindowInfosListener(const sp<gui::IWindowInfosListener>& windowInfosListener); - void windowInfosChanged(std::vector<gui::WindowInfo>, std::vector<gui::DisplayInfo>, + void windowInfosChanged(gui::WindowInfosUpdate update, WindowInfosReportedListenerSet windowInfosReportedListeners, - bool forceImmediateCall, VsyncId vsyncId, nsecs_t timestamp); + bool forceImmediateCall); binder::Status onWindowInfosReported() override; - VsyncId getUnsentMessageVsyncId() { - std::scoped_lock lock(mMessagesMutex); - return mUnsentVsyncId; - } - - nsecs_t getUnsentMessageTimestamp() { - std::scoped_lock lock(mMessagesMutex); - return mUnsentTimestamp; - } - - uint32_t getPendingMessageCount() { - std::scoped_lock lock(mMessagesMutex); - return mActiveMessageCount; - } + struct DebugInfo { + VsyncId maxSendDelayVsyncId; + nsecs_t maxSendDelayDuration; + uint32_t pendingMessageCount; + }; + DebugInfo getDebugInfo(); protected: void binderDied(const wp<IBinder>& who) override; @@ -73,11 +66,16 @@ private: std::mutex mMessagesMutex; uint32_t mActiveMessageCount GUARDED_BY(mMessagesMutex) = 0; - std::function<void(WindowInfosReportedListenerSet)> mWindowInfosChangedDelayed - GUARDED_BY(mMessagesMutex); - VsyncId mUnsentVsyncId GUARDED_BY(mMessagesMutex) = {-1}; - nsecs_t mUnsentTimestamp GUARDED_BY(mMessagesMutex) = -1; - WindowInfosReportedListenerSet mReportedListenersDelayed; + std::optional<gui::WindowInfosUpdate> mDelayedUpdate GUARDED_BY(mMessagesMutex); + WindowInfosReportedListenerSet mReportedListeners; + + DebugInfo mDebugInfo GUARDED_BY(mMessagesMutex); + struct DelayInfo { + int64_t vsyncId; + nsecs_t frameTime; + }; + std::optional<DelayInfo> mDelayInfo GUARDED_BY(mMessagesMutex); + void updateMaxSendDelay() REQUIRES(mMessagesMutex); }; } // namespace android diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h b/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h index da5ec480cc..4d03be04b3 100644 --- a/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h +++ b/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h @@ -590,7 +590,7 @@ public: mFlinger->binderDied(display); mFlinger->onFirstRef(); - mFlinger->updateInputFlinger(VsyncId{0}); + mFlinger->updateInputFlinger(VsyncId{}, TimePoint{}); mFlinger->updateCursorAsync(); mutableScheduler().setVsyncConfig({.sfOffset = mFdp.ConsumeIntegral<nsecs_t>(), diff --git a/services/surfaceflinger/tests/unittests/Android.bp b/services/surfaceflinger/tests/unittests/Android.bp index 881b362f1e..db81bad968 100644 --- a/services/surfaceflinger/tests/unittests/Android.bp +++ b/services/surfaceflinger/tests/unittests/Android.bp @@ -139,6 +139,7 @@ cc_test { "VSyncReactorTest.cpp", "VsyncConfigurationTest.cpp", "VsyncScheduleTest.cpp", + "WindowInfosListenerInvokerTest.cpp", ], } diff --git a/services/surfaceflinger/tests/unittests/WindowInfosListenerInvokerTest.cpp b/services/surfaceflinger/tests/unittests/WindowInfosListenerInvokerTest.cpp new file mode 100644 index 0000000000..af4971b063 --- /dev/null +++ b/services/surfaceflinger/tests/unittests/WindowInfosListenerInvokerTest.cpp @@ -0,0 +1,244 @@ +#include <android/gui/BnWindowInfosListener.h> +#include <gtest/gtest.h> +#include <gui/SurfaceComposerClient.h> +#include <gui/WindowInfosUpdate.h> +#include <condition_variable> + +#include "BackgroundExecutor.h" +#include "WindowInfosListenerInvoker.h" +#include "android/gui/IWindowInfosReportedListener.h" + +namespace android { + +class WindowInfosListenerInvokerTest : public testing::Test { +protected: + WindowInfosListenerInvokerTest() : mInvoker(sp<WindowInfosListenerInvoker>::make()) {} + + ~WindowInfosListenerInvokerTest() { + std::mutex mutex; + std::condition_variable cv; + bool flushComplete = false; + // Flush the BackgroundExecutor thread to ensure any scheduled tasks are complete. + // Otherwise, references those tasks hold may go out of scope before they are done + // executing. + BackgroundExecutor::getInstance().sendCallbacks({[&]() { + std::scoped_lock lock{mutex}; + flushComplete = true; + cv.notify_one(); + }}); + std::unique_lock<std::mutex> lock{mutex}; + cv.wait(lock, [&]() { return flushComplete; }); + } + + sp<WindowInfosListenerInvoker> mInvoker; +}; + +using WindowInfosUpdateConsumer = std::function<void(const gui::WindowInfosUpdate&, + const sp<gui::IWindowInfosReportedListener>&)>; + +class Listener : public gui::BnWindowInfosListener { +public: + Listener(WindowInfosUpdateConsumer consumer) : mConsumer(std::move(consumer)) {} + + binder::Status onWindowInfosChanged( + const gui::WindowInfosUpdate& update, + const sp<gui::IWindowInfosReportedListener>& reportedListener) override { + mConsumer(update, reportedListener); + return binder::Status::ok(); + } + +private: + WindowInfosUpdateConsumer mConsumer; +}; + +// Test that WindowInfosListenerInvoker#windowInfosChanged calls a single window infos listener. +TEST_F(WindowInfosListenerInvokerTest, callsSingleListener) { + std::mutex mutex; + std::condition_variable cv; + + int callCount = 0; + + mInvoker->addWindowInfosListener( + sp<Listener>::make([&](const gui::WindowInfosUpdate&, + const sp<gui::IWindowInfosReportedListener>& reportedListener) { + std::scoped_lock lock{mutex}; + callCount++; + cv.notify_one(); + + reportedListener->onWindowInfosReported(); + })); + + BackgroundExecutor::getInstance().sendCallbacks( + {[this]() { mInvoker->windowInfosChanged({}, {}, false); }}); + + std::unique_lock<std::mutex> lock{mutex}; + cv.wait(lock, [&]() { return callCount == 1; }); + EXPECT_EQ(callCount, 1); +} + +// Test that WindowInfosListenerInvoker#windowInfosChanged calls multiple window infos listeners. +TEST_F(WindowInfosListenerInvokerTest, callsMultipleListeners) { + std::mutex mutex; + std::condition_variable cv; + + int callCount = 0; + const int expectedCallCount = 3; + + for (int i = 0; i < expectedCallCount; i++) { + mInvoker->addWindowInfosListener(sp<Listener>::make( + [&](const gui::WindowInfosUpdate&, + const sp<gui::IWindowInfosReportedListener>& reportedListener) { + std::scoped_lock lock{mutex}; + callCount++; + if (callCount == expectedCallCount) { + cv.notify_one(); + } + + reportedListener->onWindowInfosReported(); + })); + } + + BackgroundExecutor::getInstance().sendCallbacks( + {[&]() { mInvoker->windowInfosChanged({}, {}, false); }}); + + std::unique_lock<std::mutex> lock{mutex}; + cv.wait(lock, [&]() { return callCount == expectedCallCount; }); + EXPECT_EQ(callCount, expectedCallCount); +} + +// Test that WindowInfosListenerInvoker#windowInfosChanged delays sending a second message until +// after the WindowInfosReportedListener is called. +TEST_F(WindowInfosListenerInvokerTest, delaysUnackedCall) { + std::mutex mutex; + std::condition_variable cv; + + int callCount = 0; + + // Simulate a slow ack by not calling the WindowInfosReportedListener. + mInvoker->addWindowInfosListener(sp<Listener>::make( + [&](const gui::WindowInfosUpdate&, const sp<gui::IWindowInfosReportedListener>&) { + std::scoped_lock lock{mutex}; + callCount++; + cv.notify_one(); + })); + + BackgroundExecutor::getInstance().sendCallbacks({[&]() { + mInvoker->windowInfosChanged({}, {}, false); + mInvoker->windowInfosChanged({}, {}, false); + }}); + + { + std::unique_lock lock{mutex}; + cv.wait(lock, [&]() { return callCount == 1; }); + } + EXPECT_EQ(callCount, 1); + + // Ack the first message. + mInvoker->onWindowInfosReported(); + + { + std::unique_lock lock{mutex}; + cv.wait(lock, [&]() { return callCount == 2; }); + } + EXPECT_EQ(callCount, 2); +} + +// Test that WindowInfosListenerInvoker#windowInfosChanged immediately sends a second message when +// forceImmediateCall is true. +TEST_F(WindowInfosListenerInvokerTest, sendsForcedMessage) { + std::mutex mutex; + std::condition_variable cv; + + int callCount = 0; + const int expectedCallCount = 2; + + // Simulate a slow ack by not calling the WindowInfosReportedListener. + mInvoker->addWindowInfosListener(sp<Listener>::make( + [&](const gui::WindowInfosUpdate&, const sp<gui::IWindowInfosReportedListener>&) { + std::scoped_lock lock{mutex}; + callCount++; + if (callCount == expectedCallCount) { + cv.notify_one(); + } + })); + + BackgroundExecutor::getInstance().sendCallbacks({[&]() { + mInvoker->windowInfosChanged({}, {}, false); + mInvoker->windowInfosChanged({}, {}, true); + }}); + + { + std::unique_lock lock{mutex}; + cv.wait(lock, [&]() { return callCount == expectedCallCount; }); + } + EXPECT_EQ(callCount, expectedCallCount); +} + +// Test that WindowInfosListenerInvoker#windowInfosChanged skips old messages when more than one +// message is delayed. +TEST_F(WindowInfosListenerInvokerTest, skipsDelayedMessage) { + std::mutex mutex; + std::condition_variable cv; + + int64_t lastUpdateId = -1; + + // Simulate a slow ack by not calling the WindowInfosReportedListener. + mInvoker->addWindowInfosListener( + sp<Listener>::make([&](const gui::WindowInfosUpdate& update, + const sp<gui::IWindowInfosReportedListener>&) { + std::scoped_lock lock{mutex}; + lastUpdateId = update.vsyncId; + cv.notify_one(); + })); + + BackgroundExecutor::getInstance().sendCallbacks({[&]() { + mInvoker->windowInfosChanged({{}, {}, /* vsyncId= */ 1, 0}, {}, false); + mInvoker->windowInfosChanged({{}, {}, /* vsyncId= */ 2, 0}, {}, false); + mInvoker->windowInfosChanged({{}, {}, /* vsyncId= */ 3, 0}, {}, false); + }}); + + { + std::unique_lock lock{mutex}; + cv.wait(lock, [&]() { return lastUpdateId == 1; }); + } + EXPECT_EQ(lastUpdateId, 1); + + // Ack the first message. The third update should be sent. + mInvoker->onWindowInfosReported(); + + { + std::unique_lock lock{mutex}; + cv.wait(lock, [&]() { return lastUpdateId == 3; }); + } + EXPECT_EQ(lastUpdateId, 3); +} + +// Test that WindowInfosListenerInvoker#windowInfosChanged immediately calls listener after a call +// where no listeners were configured. +TEST_F(WindowInfosListenerInvokerTest, noListeners) { + std::mutex mutex; + std::condition_variable cv; + + int callCount = 0; + + // Test that calling windowInfosChanged without any listeners doesn't cause the next call to be + // delayed. + BackgroundExecutor::getInstance().sendCallbacks({[&]() { + mInvoker->windowInfosChanged({}, {}, false); + mInvoker->addWindowInfosListener(sp<Listener>::make( + [&](const gui::WindowInfosUpdate&, const sp<gui::IWindowInfosReportedListener>&) { + std::scoped_lock lock{mutex}; + callCount++; + cv.notify_one(); + })); + mInvoker->windowInfosChanged({}, {}, false); + }}); + + { + std::unique_lock lock{mutex}; + cv.wait(lock, [&]() { return callCount == 1; }); + } + EXPECT_EQ(callCount, 1); +} + +} // namespace android diff --git a/vulkan/libvulkan/swapchain.cpp b/vulkan/libvulkan/swapchain.cpp index 5965953b38..af873065ff 100644 --- a/vulkan/libvulkan/swapchain.cpp +++ b/vulkan/libvulkan/swapchain.cpp @@ -877,6 +877,7 @@ VkResult GetPhysicalDeviceSurfaceCapabilities2KHR( int width, height; int transform_hint; int max_buffer_count; + int min_undequeued_buffers; if (surface == VK_NULL_HANDLE) { const InstanceData& instance_data = GetData(physicalDevice); ProcHook::Extension surfaceless = ProcHook::GOOGLE_surfaceless_query; @@ -929,17 +930,24 @@ VkResult GetPhysicalDeviceSurfaceCapabilities2KHR( return VK_ERROR_SURFACE_LOST_KHR; } + err = window->query(window, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, + &min_undequeued_buffers); + if (err != android::OK) { + ALOGE("NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS query failed: %s (%d)", + strerror(-err), err); + return VK_ERROR_SURFACE_LOST_KHR; + } + if (pPresentMode && IsSharedPresentMode(pPresentMode->presentMode)) { capabilities->minImageCount = 1; capabilities->maxImageCount = 1; } else if (pPresentMode && pPresentMode->presentMode == VK_PRESENT_MODE_MAILBOX_KHR) { - // TODO: use undequeued buffer requirement for more precise bound - capabilities->minImageCount = std::min(max_buffer_count, 4); + capabilities->minImageCount = + std::min(max_buffer_count, min_undequeued_buffers + 2); capabilities->maxImageCount = static_cast<uint32_t>(max_buffer_count); } else { - // TODO: if we're able to, provide better bounds on the number of buffers - // for other modes as well. - capabilities->minImageCount = std::min(max_buffer_count, 3); + capabilities->minImageCount = + std::min(max_buffer_count, min_undequeued_buffers + 1); capabilities->maxImageCount = static_cast<uint32_t>(max_buffer_count); } } |
