diff options
72 files changed, 1863 insertions, 843 deletions
diff --git a/cmds/dumpstate/dumpstate.cpp b/cmds/dumpstate/dumpstate.cpp index ecafcfc170..a48313a6fa 100644 --- a/cmds/dumpstate/dumpstate.cpp +++ b/cmds/dumpstate/dumpstate.cpp @@ -186,6 +186,7 @@ void add_mountinfo(); #define SYSTEM_TRACE_SNAPSHOT "/data/misc/perfetto-traces/bugreport/systrace.pftrace" #define CGROUPFS_DIR "/sys/fs/cgroup" #define SDK_EXT_INFO "/apex/com.android.sdkext/bin/derive_sdk" +#define DROPBOX_DIR "/data/system/dropbox" // TODO(narayan): Since this information has to be kept in sync // with tombstoned, we should just put it in a common header. @@ -526,6 +527,15 @@ static bool skip_not_stat(const char *path) { return strcmp(path + len - sizeof(stat) + 1, stat); /* .../stat? */ } +static bool skip_wtf_strictmode(const char *path) { + if (strstr(path, "_wtf")) { + return true; + } else if (strstr(path, "_strictmode")) { + return true; + } + return false; +} + static bool skip_none(const char* path __attribute__((unused))) { return false; } @@ -1895,6 +1905,11 @@ Dumpstate::RunStatus Dumpstate::DumpstateDefaultAfterCritical() { DumpIpTablesAsRoot(); DumpDynamicPartitionInfo(); ds.AddDir(OTA_METADATA_DIR, true); + if (!PropertiesHelper::IsUserBuild()) { + // Include dropbox entry files inside ZIP, but exclude + // noisy WTF and StrictMode entries + dump_files("", DROPBOX_DIR, skip_wtf_strictmode, _add_file_from_fd); + } // Capture any IPSec policies in play. No keys are exposed here. RunCommand("IP XFRM POLICY", {"ip", "xfrm", "policy"}, CommandOptions::WithTimeout(10).Build()); diff --git a/cmds/dumpstate/tests/dumpstate_test.cpp b/cmds/dumpstate/tests/dumpstate_test.cpp index 87f92544c0..aa5219bb5b 100644 --- a/cmds/dumpstate/tests/dumpstate_test.cpp +++ b/cmds/dumpstate/tests/dumpstate_test.cpp @@ -1051,7 +1051,8 @@ class ZippedBugReportStreamTest : public DumpstateBaseTest { }; // Generate a quick LimitedOnly report redirected to a file, open it and verify entry exist. -TEST_F(ZippedBugReportStreamTest, StreamLimitedOnlyReport) { +// TODO: broken test tracked in b/249983726 +TEST_F(ZippedBugReportStreamTest, DISABLED_StreamLimitedOnlyReport) { std::string out_path = kTestDataPath + "StreamLimitedOnlyReportOut.zip"; android::base::unique_fd out_fd; CreateFd(out_path, &out_fd); diff --git a/include/input/MotionPredictor.h b/include/input/MotionPredictor.h index 3fae4e6b68..68ebf75fc6 100644 --- a/include/input/MotionPredictor.h +++ b/include/input/MotionPredictor.h @@ -19,6 +19,7 @@ #include <cstdint> #include <memory> #include <mutex> +#include <string> #include <unordered_map> #include <android-base/thread_annotations.h> @@ -73,6 +74,7 @@ public: private: const nsecs_t mPredictionTimestampOffsetNanos; + const std::string mModelPath; const std::function<bool()> mCheckMotionPredictionEnabled; std::unique_ptr<TfLiteMotionPredictorModel> mModel; diff --git a/include/input/TfLiteMotionPredictor.h b/include/input/TfLiteMotionPredictor.h index 704349c2eb..54e2851a7a 100644 --- a/include/input/TfLiteMotionPredictor.h +++ b/include/input/TfLiteMotionPredictor.h @@ -22,8 +22,8 @@ #include <memory> #include <optional> #include <span> -#include <string> +#include <android-base/mapped_file.h> #include <input/RingBuffer.h> #include <tensorflow/lite/core/api/error_reporter.h> @@ -101,6 +101,8 @@ public: // Creates a model from an encoded Flatbuffer model. static std::unique_ptr<TfLiteMotionPredictorModel> create(const char* modelPath); + ~TfLiteMotionPredictorModel(); + // Returns the length of the model's input buffers. size_t inputLength() const; @@ -122,7 +124,7 @@ public: std::span<const float> outputPressure() const; private: - explicit TfLiteMotionPredictorModel(std::string model); + explicit TfLiteMotionPredictorModel(std::unique_ptr<android::base::MappedFile> model); void allocateTensors(); void attachInputTensors(); @@ -138,7 +140,7 @@ private: const TfLiteTensor* mOutputPhi = nullptr; const TfLiteTensor* mOutputPressure = nullptr; - std::string mFlatBuffer; + std::unique_ptr<android::base::MappedFile> mFlatBuffer; std::unique_ptr<tflite::ErrorReporter> mErrorReporter; std::unique_ptr<tflite::FlatBufferModel> mModel; std::unique_ptr<tflite::Interpreter> mInterpreter; diff --git a/libs/binder/Android.bp b/libs/binder/Android.bp index b84a9d3441..c4c8ffb656 100644 --- a/libs/binder/Android.bp +++ b/libs/binder/Android.bp @@ -518,6 +518,10 @@ aidl_interface { enabled: false, }, }, + visibility: [ + ":__subpackages__", + "//system/tools/aidl:__subpackages__", + ], } // TODO(b/184872979): remove once the Rust API is created. diff --git a/libs/binder/RpcState.cpp b/libs/binder/RpcState.cpp index 1ea13f9a1c..b27f1028d4 100644 --- a/libs/binder/RpcState.cpp +++ b/libs/binder/RpcState.cpp @@ -1036,8 +1036,8 @@ processTransactInternalTailCall: return DEAD_OBJECT; } - if (it->second.asyncTodo.size() != 0 && - it->second.asyncTodo.top().asyncNumber == it->second.asyncNumber) { + if (it->second.asyncTodo.size() == 0) return OK; + if (it->second.asyncTodo.top().asyncNumber == it->second.asyncNumber) { LOG_RPC_DETAIL("Found next async transaction %" PRIu64 " on %" PRIu64, it->second.asyncNumber, addr); diff --git a/libs/binder/rust/rpcbinder/Android.bp b/libs/binder/rust/rpcbinder/Android.bp index afb73e920f..38dd4fe187 100644 --- a/libs/binder/rust/rpcbinder/Android.bp +++ b/libs/binder/rust/rpcbinder/Android.bp @@ -23,7 +23,13 @@ rust_library { "liblibc", "liblog_rust", ], + visibility: [ + "//device/google/cuttlefish/shared/minidroid/sample", + "//packages/modules/Uwb", + "//packages/modules/Virtualization:__subpackages__", + ], apex_available: [ + "//apex_available:platform", "com.android.compos", "com.android.uwb", "com.android.virt", @@ -51,6 +57,7 @@ rust_library { "libutils", ], apex_available: [ + "//apex_available:platform", "com.android.compos", "com.android.uwb", "com.android.virt", @@ -84,6 +91,7 @@ rust_bindgen { "libutils", ], apex_available: [ + "//apex_available:platform", "com.android.compos", "com.android.uwb", "com.android.virt", diff --git a/libs/gui/BLASTBufferQueue.cpp b/libs/gui/BLASTBufferQueue.cpp index 66c0041965..9d82c143f5 100644 --- a/libs/gui/BLASTBufferQueue.cpp +++ b/libs/gui/BLASTBufferQueue.cpp @@ -20,6 +20,7 @@ #define ATRACE_TAG ATRACE_TAG_GRAPHICS //#define LOG_NDEBUG 0 +#include <cutils/atomic.h> #include <gui/BLASTBufferQueue.h> #include <gui/BufferItemConsumer.h> #include <gui/BufferQueueConsumer.h> @@ -157,11 +158,11 @@ BLASTBufferQueue::BLASTBufferQueue(const std::string& name, bool updateDestinati GraphicBuffer::USAGE_HW_COMPOSER | GraphicBuffer::USAGE_HW_TEXTURE, 1, false, this); - static int32_t id = 0; - mName = name + "#" + std::to_string(id); - auto consumerName = mName + "(BLAST Consumer)" + std::to_string(id); - mQueuedBufferTrace = "QueuedBuffer - " + mName + "BLAST#" + std::to_string(id); - id++; + static std::atomic<uint32_t> nextId = 0; + mProducerId = nextId++; + mName = name + "#" + std::to_string(mProducerId); + auto consumerName = mName + "(BLAST Consumer)" + std::to_string(mProducerId); + mQueuedBufferTrace = "QueuedBuffer - " + mName + "BLAST#" + std::to_string(mProducerId); mBufferItemConsumer->setName(String8(consumerName.c_str())); mBufferItemConsumer->setFrameAvailableListener(this); @@ -572,7 +573,8 @@ status_t BLASTBufferQueue::acquireNextBufferLocked( std::bind(releaseBufferCallbackThunk, wp<BLASTBufferQueue>(this) /* callbackContext */, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); sp<Fence> fence = bufferItem.mFence ? new Fence(bufferItem.mFence->dup()) : Fence::NO_FENCE; - t->setBuffer(mSurfaceControl, buffer, fence, bufferItem.mFrameNumber, releaseBufferCallback); + t->setBuffer(mSurfaceControl, buffer, fence, bufferItem.mFrameNumber, mProducerId, + releaseBufferCallback); t->setDataspace(mSurfaceControl, static_cast<ui::Dataspace>(bufferItem.mDataSpace)); t->setHdrMetadata(mSurfaceControl, bufferItem.mHdrMetadata); t->setSurfaceDamageRegion(mSurfaceControl, bufferItem.mSurfaceDamage); diff --git a/libs/gui/LayerState.cpp b/libs/gui/LayerState.cpp index 7772a65dae..a6276e500c 100644 --- a/libs/gui/LayerState.cpp +++ b/libs/gui/LayerState.cpp @@ -984,6 +984,7 @@ status_t BufferData::writeToParcel(Parcel* output) const { SAFE_PARCEL(output->writeUint64, cachedBuffer.id); SAFE_PARCEL(output->writeBool, hasBarrier); SAFE_PARCEL(output->writeUint64, barrierFrameNumber); + SAFE_PARCEL(output->writeUint32, producerId); return NO_ERROR; } @@ -1022,6 +1023,7 @@ status_t BufferData::readFromParcel(const Parcel* input) { SAFE_PARCEL(input->readBool, &hasBarrier); SAFE_PARCEL(input->readUint64, &barrierFrameNumber); + SAFE_PARCEL(input->readUint32, &producerId); return NO_ERROR; } diff --git a/libs/gui/SurfaceComposerClient.cpp b/libs/gui/SurfaceComposerClient.cpp index a34593825a..4596c8a02d 100644 --- a/libs/gui/SurfaceComposerClient.cpp +++ b/libs/gui/SurfaceComposerClient.cpp @@ -1632,7 +1632,7 @@ SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setBuffe SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setBuffer( const sp<SurfaceControl>& sc, const sp<GraphicBuffer>& buffer, const std::optional<sp<Fence>>& fence, const std::optional<uint64_t>& optFrameNumber, - ReleaseBufferCallback callback) { + uint32_t producerId, ReleaseBufferCallback callback) { layer_state_t* s = getLayerState(sc); if (!s) { mStatus = BAD_INDEX; @@ -1651,6 +1651,7 @@ SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setBuffe bufferData->buffer = buffer; uint64_t frameNumber = sc->resolveFrameNumber(optFrameNumber); bufferData->frameNumber = frameNumber; + bufferData->producerId = producerId; bufferData->flags |= BufferData::BufferDataChange::frameNumberChanged; if (fence) { bufferData->acquireFence = *fence; @@ -2662,6 +2663,12 @@ status_t SurfaceComposerClient::setHdrConversionStrategy( return statusTFromBinderStatus(status); } +status_t SurfaceComposerClient::getHdrOutputConversionSupport(bool* isSupported) { + binder::Status status = + ComposerServiceAIDL::getComposerService()->getHdrOutputConversionSupport(isSupported); + return statusTFromBinderStatus(status); +} + status_t SurfaceComposerClient::setOverrideFrameRate(uid_t uid, float frameRate) { binder::Status status = ComposerServiceAIDL::getComposerService()->setOverrideFrameRate(uid, frameRate); diff --git a/libs/gui/include/gui/BLASTBufferQueue.h b/libs/gui/include/gui/BLASTBufferQueue.h index 8d07162f1b..b9e06473bf 100644 --- a/libs/gui/include/gui/BLASTBufferQueue.h +++ b/libs/gui/include/gui/BLASTBufferQueue.h @@ -162,6 +162,11 @@ private: int32_t mNumFrameAvailable GUARDED_BY(mMutex) = 0; int32_t mNumAcquired GUARDED_BY(mMutex) = 0; + // A value used to identify if a producer has been changed for the same SurfaceControl. + // This is needed to know when the frame number has been reset to make sure we don't + // latch stale buffers and that we don't wait on barriers from an old producer. + uint32_t mProducerId = 0; + // Keep a reference to the submitted buffers so we can release when surfaceflinger drops the // buffer or the buffer has been presented and a new buffer is ready to be presented. std::unordered_map<ReleaseCallbackId, BufferItem, ReleaseBufferCallbackIdHash> mSubmitted diff --git a/libs/gui/include/gui/LayerState.h b/libs/gui/include/gui/LayerState.h index 03a2582589..ddaf473855 100644 --- a/libs/gui/include/gui/LayerState.h +++ b/libs/gui/include/gui/LayerState.h @@ -111,6 +111,7 @@ public: uint64_t frameNumber = 0; bool hasBarrier = false; uint64_t barrierFrameNumber = 0; + uint32_t producerId = 0; // Listens to when the buffer is safe to be released. This is used for blast // layers only. The callback includes a release fence as well as the graphic diff --git a/libs/gui/include/gui/SurfaceComposerClient.h b/libs/gui/include/gui/SurfaceComposerClient.h index 809ea5a30f..fcf8d645cd 100644 --- a/libs/gui/include/gui/SurfaceComposerClient.h +++ b/libs/gui/include/gui/SurfaceComposerClient.h @@ -194,6 +194,8 @@ public: static status_t getHdrConversionCapabilities(std::vector<gui::HdrConversionCapability>*); // Sets the HDR conversion strategy for the device static status_t setHdrConversionStrategy(gui::HdrConversionStrategy hdrConversionStrategy); + // Returns whether HDR conversion is supported by the device. + static status_t getHdrOutputConversionSupport(bool* isSupported); // Sets the frame rate of a particular app (uid). This is currently called // by GameManager. @@ -536,7 +538,7 @@ public: Transaction& setBuffer(const sp<SurfaceControl>& sc, const sp<GraphicBuffer>& buffer, const std::optional<sp<Fence>>& fence = std::nullopt, const std::optional<uint64_t>& frameNumber = std::nullopt, - ReleaseBufferCallback callback = nullptr); + uint32_t producerId = 0, ReleaseBufferCallback callback = nullptr); std::shared_ptr<BufferData> getAndClearBuffer(const sp<SurfaceControl>& sc); /** diff --git a/libs/input/Android.bp b/libs/input/Android.bp index 83392ec793..fd4fc16deb 100644 --- a/libs/input/Android.bp +++ b/libs/input/Android.bp @@ -73,11 +73,15 @@ cc_library { "liblog", "libPlatformProperties", "libvintf", - "libtflite", + ], + + ldflags: [ + "-Wl,--exclude-libs=libtflite_static.a", ], static_libs: [ "libui-types", + "libtflite_static", ], export_static_lib_headers: [ diff --git a/libs/input/MotionPredictor.cpp b/libs/input/MotionPredictor.cpp index 0f889e8128..7d11ef2575 100644 --- a/libs/input/MotionPredictor.cpp +++ b/libs/input/MotionPredictor.cpp @@ -65,9 +65,8 @@ TfLiteMotionPredictorSample::Point convertPrediction( MotionPredictor::MotionPredictor(nsecs_t predictionTimestampOffsetNanos, const char* modelPath, std::function<bool()> checkMotionPredictionEnabled) : mPredictionTimestampOffsetNanos(predictionTimestampOffsetNanos), - mCheckMotionPredictionEnabled(std::move(checkMotionPredictionEnabled)), - mModel(TfLiteMotionPredictorModel::create(modelPath == nullptr ? DEFAULT_MODEL_PATH - : modelPath)) {} + mModelPath(modelPath == nullptr ? DEFAULT_MODEL_PATH : modelPath), + mCheckMotionPredictionEnabled(std::move(checkMotionPredictionEnabled)) {} void MotionPredictor::record(const MotionEvent& event) { if (!isPredictionAvailable(event.getDeviceId(), event.getSource())) { @@ -76,6 +75,11 @@ void MotionPredictor::record(const MotionEvent& event) { return; } + // Initialise the model now that it's likely to be used. + if (!mModel) { + mModel = TfLiteMotionPredictorModel::create(mModelPath.c_str()); + } + TfLiteMotionPredictorBuffers& buffers = mDeviceBuffers.try_emplace(event.getDeviceId(), mModel->inputLength()).first->second; @@ -130,6 +134,7 @@ std::vector<std::unique_ptr<MotionEvent>> MotionPredictor::predict(nsecs_t times continue; } + LOG_ALWAYS_FATAL_IF(!mModel); buffer.copyTo(*mModel); LOG_ALWAYS_FATAL_IF(!mModel->invoke()); diff --git a/libs/input/TfLiteMotionPredictor.cpp b/libs/input/TfLiteMotionPredictor.cpp index 40653d36f7..10510d675c 100644 --- a/libs/input/TfLiteMotionPredictor.cpp +++ b/libs/input/TfLiteMotionPredictor.cpp @@ -17,27 +17,31 @@ #define LOG_TAG "TfLiteMotionPredictor" #include <input/TfLiteMotionPredictor.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <unistd.h> + #include <algorithm> #include <cmath> #include <cstddef> #include <cstdint> -#include <fstream> -#include <ios> -#include <iterator> #include <memory> #include <span> -#include <string> #include <type_traits> #include <utility> +#include <android-base/logging.h> +#include <android-base/mapped_file.h> #define ATRACE_TAG ATRACE_TAG_INPUT #include <cutils/trace.h> #include <log/log.h> #include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/core/api/op_resolver.h" #include "tensorflow/lite/interpreter.h" -#include "tensorflow/lite/kernels/register.h" +#include "tensorflow/lite/kernels/builtin_op_kernels.h" #include "tensorflow/lite/model.h" +#include "tensorflow/lite/mutable_op_resolver.h" namespace android { namespace { @@ -102,6 +106,15 @@ void checkTensor(const TfLiteTensor* tensor) { LOG_ALWAYS_FATAL_IF(buffer.empty(), "No buffer for tensor '%s'", tensor->name); } +std::unique_ptr<tflite::OpResolver> createOpResolver() { + auto resolver = std::make_unique<tflite::MutableOpResolver>(); + resolver->AddBuiltin(::tflite::BuiltinOperator_CONCATENATION, + ::tflite::ops::builtin::Register_CONCATENATION()); + resolver->AddBuiltin(::tflite::BuiltinOperator_FULLY_CONNECTED, + ::tflite::ops::builtin::Register_FULLY_CONNECTED()); + return resolver; +} + } // namespace TfLiteMotionPredictorBuffers::TfLiteMotionPredictorBuffers(size_t inputLength) @@ -195,27 +208,42 @@ void TfLiteMotionPredictorBuffers::pushSample(int64_t timestamp, std::unique_ptr<TfLiteMotionPredictorModel> TfLiteMotionPredictorModel::create( const char* modelPath) { - std::ifstream f(modelPath, std::ios::binary); - LOG_ALWAYS_FATAL_IF(!f, "Could not read model from %s", modelPath); + const int fd = open(modelPath, O_RDONLY); + if (fd == -1) { + PLOG(FATAL) << "Could not read model from " << modelPath; + } - std::string data; - data.assign(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>()); + const off_t fdSize = lseek(fd, 0, SEEK_END); + if (fdSize == -1) { + PLOG(FATAL) << "Failed to determine file size"; + } + + std::unique_ptr<android::base::MappedFile> modelBuffer = + android::base::MappedFile::FromFd(fd, /*offset=*/0, fdSize, PROT_READ); + if (!modelBuffer) { + PLOG(FATAL) << "Failed to mmap model"; + } + if (close(fd) == -1) { + PLOG(FATAL) << "Failed to close model fd"; + } return std::unique_ptr<TfLiteMotionPredictorModel>( - new TfLiteMotionPredictorModel(std::move(data))); + new TfLiteMotionPredictorModel(std::move(modelBuffer))); } -TfLiteMotionPredictorModel::TfLiteMotionPredictorModel(std::string model) +TfLiteMotionPredictorModel::TfLiteMotionPredictorModel( + std::unique_ptr<android::base::MappedFile> model) : mFlatBuffer(std::move(model)) { + CHECK(mFlatBuffer); mErrorReporter = std::make_unique<LoggingErrorReporter>(); - mModel = tflite::FlatBufferModel::VerifyAndBuildFromBuffer(mFlatBuffer.data(), - mFlatBuffer.length(), + mModel = tflite::FlatBufferModel::VerifyAndBuildFromBuffer(mFlatBuffer->data(), + mFlatBuffer->size(), /*extra_verifier=*/nullptr, mErrorReporter.get()); LOG_ALWAYS_FATAL_IF(!mModel); - tflite::ops::builtin::BuiltinOpResolver resolver; - tflite::InterpreterBuilder builder(*mModel, resolver); + auto resolver = createOpResolver(); + tflite::InterpreterBuilder builder(*mModel, *resolver); if (builder(&mInterpreter) != kTfLiteOk || !mInterpreter) { LOG_ALWAYS_FATAL("Failed to build interpreter"); @@ -227,6 +255,8 @@ TfLiteMotionPredictorModel::TfLiteMotionPredictorModel(std::string model) allocateTensors(); } +TfLiteMotionPredictorModel::~TfLiteMotionPredictorModel() {} + void TfLiteMotionPredictorModel::allocateTensors() { if (mRunner->AllocateTensors() != kTfLiteOk) { LOG_ALWAYS_FATAL("Failed to allocate tensors"); diff --git a/libs/input/tests/Android.bp b/libs/input/tests/Android.bp index f07164c87f..37faf91936 100644 --- a/libs/input/tests/Android.bp +++ b/libs/input/tests/Android.bp @@ -34,6 +34,7 @@ cc_test { "libgmock", "libgui_window_info_static", "libinput", + "libtflite_static", "libui-types", ], cflags: [ @@ -48,7 +49,6 @@ cc_test { "libcutils", "liblog", "libPlatformProperties", - "libtflite", "libutils", "libvintf", ], diff --git a/libs/jpegrecoverymap/Android.bp b/libs/jpegrecoverymap/Android.bp index 2c4b3bff77..78d1912d24 100644 --- a/libs/jpegrecoverymap/Android.bp +++ b/libs/jpegrecoverymap/Android.bp @@ -29,9 +29,9 @@ cc_library { local_include_dirs: ["include"], srcs: [ - "recoverymap.cpp", + "jpegr.cpp", "recoverymapmath.cpp", - "recoverymaputils.cpp", + "jpegrutils.cpp", ], shared_libs: [ @@ -57,7 +57,7 @@ cc_library { export_include_dirs: ["include"], srcs: [ - "jpegencoder.cpp", + "jpegencoderhelper.cpp", ], } @@ -73,6 +73,6 @@ cc_library { export_include_dirs: ["include"], srcs: [ - "jpegdecoder.cpp", + "jpegdecoderhelper.cpp", ], } diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegdecoder.h b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegdecoderhelper.h index 419b63d1de..874823709c 100644 --- a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegdecoder.h +++ b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegdecoderhelper.h @@ -1,4 +1,3 @@ - /* * Copyright 2022 The Android Open Source Project * @@ -15,8 +14,8 @@ * limitations under the License. */ -#ifndef ANDROID_JPEGRECOVERYMAP_JPEGDECODER_H -#define ANDROID_JPEGRECOVERYMAP_JPEGDECODER_H +#ifndef ANDROID_JPEGRECOVERYMAP_JPEGDECODERHELPER_H +#define ANDROID_JPEGRECOVERYMAP_JPEGDECODERHELPER_H // We must include cstdio before jpeglib.h. It is a requirement of libjpeg. #include <cstdio> @@ -26,15 +25,15 @@ extern "C" { } #include <utils/Errors.h> #include <vector> -namespace android::recoverymap { +namespace android::jpegrecoverymap { /* * Encapsulates a converter from JPEG to raw image (YUV420planer or grey-scale) format. * This class is not thread-safe. */ -class JpegDecoder { +class JpegDecoderHelper { public: - JpegDecoder(); - ~JpegDecoder(); + JpegDecoderHelper(); + ~JpegDecoderHelper(); /* * Decompresses JPEG image to raw image (YUV420planer, grey-scale or RGBA) format. After * calling this method, call getDecompressedImage() to get the image. @@ -116,6 +115,6 @@ private: // Position of EXIF package, default value is -1 which means no EXIF package appears. size_t mExifPos; }; -} /* namespace android */ +} /* namespace android::jpegrecoverymap */ -#endif // ANDROID_JPEGRECOVERYMAP_JPEGDECODER_H +#endif // ANDROID_JPEGRECOVERYMAP_JPEGDECODERHELPER_H diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegencoder.h b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegencoderhelper.h index 61aeb8ace7..8b82b2b00a 100644 --- a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegencoder.h +++ b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegencoderhelper.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ANDROID_JPEGRECOVERYMAP_JPEGENCODER_H -#define ANDROID_JPEGRECOVERYMAP_JPEGENCODER_H +#ifndef ANDROID_JPEGRECOVERYMAP_JPEGENCODERHELPER_H +#define ANDROID_JPEGRECOVERYMAP_JPEGENCODERHELPER_H // We must include cstdio before jpeglib.h. It is a requirement of libjpeg. #include <cstdio> @@ -28,16 +28,16 @@ extern "C" { #include <utils/Errors.h> #include <vector> -namespace android::recoverymap { +namespace android::jpegrecoverymap { /* * Encapsulates a converter from raw image (YUV420planer or grey-scale) to JPEG format. * This class is not thread-safe. */ -class JpegEncoder { +class JpegEncoderHelper { public: - JpegEncoder(); - ~JpegEncoder(); + JpegEncoderHelper(); + ~JpegEncoderHelper(); /* * Compresses YUV420Planer image to JPEG format. After calling this method, call @@ -90,6 +90,6 @@ private: std::vector<JOCTET> mResultBuffer; }; -} /* namespace android */ +} /* namespace android::jpegrecoverymap */ -#endif // ANDROID_JPEGRECOVERYMAP_JPEGENCODER_H +#endif // ANDROID_JPEGRECOVERYMAP_JPEGENCODERHELPER_H diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegr.h index aee6602aa4..5455ba61d6 100644 --- a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h +++ b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegr.h @@ -14,12 +14,12 @@ * limitations under the License. */ -#ifndef ANDROID_JPEGRECOVERYMAP_RECOVERYMAP_H -#define ANDROID_JPEGRECOVERYMAP_RECOVERYMAP_H +#ifndef ANDROID_JPEGRECOVERYMAP_JPEGR_H +#define ANDROID_JPEGRECOVERYMAP_JPEGR_H #include "jpegrerrorcode.h" -namespace android::recoverymap { +namespace android::jpegrecoverymap { // Color gamuts for image data typedef enum { @@ -88,6 +88,8 @@ struct jpegr_metadata { uint32_t version; // Max Content Boost for the map float maxContentBoost; + // Min Content Boost for the map + float minContentBoost; }; typedef struct jpegr_uncompressed_struct* jr_uncompressed_ptr; @@ -96,7 +98,7 @@ typedef struct jpegr_exif_struct* jr_exif_ptr; typedef struct jpegr_metadata* jr_metadata_ptr; typedef struct jpegr_info_struct* jr_info_ptr; -class RecoveryMap { +class JpegR { public: /* * Encode API-0 @@ -219,17 +221,7 @@ public: */ status_t getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, jr_info_ptr jpegr_info); -private: - /* - * This method is called in the encoding pipeline. It will encode the recovery map. - * - * @param uncompressed_recovery_map uncompressed recovery map - * @param dest encoded recover map - * @return NO_ERROR if encoding succeeds, error code if error occurs. - */ - status_t compressRecoveryMap(jr_uncompressed_ptr uncompressed_recovery_map, - jr_compressed_ptr dest); - +protected: /* * This method is called in the encoding pipeline. It will take the uncompressed 8-bit and * 10-bit yuv images as input, and calculate the uncompressed recovery map. The input images @@ -265,6 +257,17 @@ private: jr_metadata_ptr metadata, jr_uncompressed_ptr dest); +private: + /* + * This method is called in the encoding pipeline. It will encode the recovery map. + * + * @param uncompressed_recovery_map uncompressed recovery map + * @param dest encoded recover map + * @return NO_ERROR if encoding succeeds, error code if error occurs. + */ + status_t compressRecoveryMap(jr_uncompressed_ptr uncompressed_recovery_map, + jr_compressed_ptr dest); + /* * This methoud is called to separate primary image and recovery map image from JPEGR * @@ -320,6 +323,6 @@ private: jr_uncompressed_ptr dest); }; -} // namespace android::recoverymap +} // namespace android::jpegrecoverymap -#endif // ANDROID_JPEGRECOVERYMAP_RECOVERYMAP_H +#endif // ANDROID_JPEGRECOVERYMAP_JPEGR_H diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h index 699c0d3ca1..f73034338b 100644 --- a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h +++ b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h @@ -16,7 +16,7 @@ #include <utils/Errors.h> -namespace android::recoverymap { +namespace android::jpegrecoverymap { enum { // status_t map for errors in the media framework @@ -48,4 +48,4 @@ enum { ERROR_JPEGR_TONEMAP_ERROR = JPEGR_RUNTIME_ERROR_BASE - 5, }; -} // namespace android::recoverymap +} // namespace android::jpegrecoverymap diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymaputils.h b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrutils.h index de29a339ed..3a0f67de19 100644 --- a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymaputils.h +++ b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrutils.h @@ -14,17 +14,17 @@ * limitations under the License. */ -#ifndef ANDROID_JPEGRECOVERYMAP_RECOVERYMAPUTILS_H -#define ANDROID_JPEGRECOVERYMAP_RECOVERYMAPUTILS_H +#ifndef ANDROID_JPEGRECOVERYMAP_JPEGRUTILS_H +#define ANDROID_JPEGRECOVERYMAP_JPEGRUTILS_H -#include <jpegrecoverymap/recoverymap.h> +#include <jpegrecoverymap/jpegr.h> #include <sstream> #include <stdint.h> #include <string> #include <cstdio> -namespace android::recoverymap { +namespace android::jpegrecoverymap { struct jpegr_metadata; @@ -92,6 +92,6 @@ bool getMetadataFromXMP(uint8_t* xmp_data, size_t xmp_size, jpegr_metadata* meta * @return XMP metadata in type of string */ std::string generateXmp(int secondary_image_length, jpegr_metadata& metadata); -} +} // namespace android::jpegrecoverymap -#endif //ANDROID_JPEGRECOVERYMAP_RECOVERYMAPUTILS_H +#endif //ANDROID_JPEGRECOVERYMAP_JPEGRUTILS_H diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h index 0695bb74ac..c12cee9f19 100644 --- a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h +++ b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h @@ -20,9 +20,9 @@ #include <cmath> #include <stdint.h> -#include <jpegrecoverymap/recoverymap.h> +#include <jpegrecoverymap/jpegr.h> -namespace android::recoverymap { +namespace android::jpegrecoverymap { #define CLIP3(x, min, max) ((x) < (min)) ? (min) : ((x) > (max)) ? (max) : (x) @@ -118,11 +118,12 @@ inline Color operator/(const Color& lhs, const float rhs) { constexpr size_t kRecoveryFactorPrecision = 10; constexpr size_t kRecoveryFactorNumEntries = 1 << kRecoveryFactorPrecision; struct RecoveryLUT { - RecoveryLUT(float hdrRatio) { - float increment = 2.0 / kRecoveryFactorNumEntries; - float value = -1.0f; - for (int idx = 0; idx < kRecoveryFactorNumEntries; idx++, value += increment) { - mRecoveryTable[idx] = pow(hdrRatio, value); + RecoveryLUT(jr_metadata_ptr metadata) { + for (int idx = 0; idx < kRecoveryFactorNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kRecoveryFactorNumEntries - 1); + float logBoost = log2(metadata->minContentBoost) * (1.0f - value) + + log2(metadata->maxContentBoost) * value; + mRecoveryTable[idx] = exp2(logBoost); } } @@ -130,10 +131,10 @@ struct RecoveryLUT { } float getRecoveryFactor(float recovery) { - uint32_t value = static_cast<uint32_t>(((recovery + 1.0f) / 2.0f) * kRecoveryFactorNumEntries); + uint32_t idx = static_cast<uint32_t>(recovery * (kRecoveryFactorNumEntries - 1)); //TODO() : Remove once conversion modules have appropriate clamping in place - value = CLIP3(value, 0, kRecoveryFactorNumEntries - 1); - return mRecoveryTable[value]; + idx = CLIP3(idx, 0, kRecoveryFactorNumEntries - 1); + return mRecoveryTable[idx]; } private: @@ -219,6 +220,9 @@ Color srgbInvOetf(Color e_gamma); float srgbInvOetfLUT(float e_gamma); Color srgbInvOetfLUT(Color e_gamma); +constexpr size_t kSrgbInvOETFPrecision = 10; +constexpr size_t kSrgbInvOETFNumEntries = 1 << kSrgbInvOETFPrecision; + //////////////////////////////////////////////////////////////////////////////// // Display-P3 transformations @@ -260,6 +264,9 @@ Color hlgOetf(Color e); float hlgOetfLUT(float e); Color hlgOetfLUT(Color e); +constexpr size_t kHlgOETFPrecision = 10; +constexpr size_t kHlgOETFNumEntries = 1 << kHlgOETFPrecision; + /* * Convert from HLG to scene luminance. * @@ -270,6 +277,9 @@ Color hlgInvOetf(Color e_gamma); float hlgInvOetfLUT(float e_gamma); Color hlgInvOetfLUT(Color e_gamma); +constexpr size_t kHlgInvOETFPrecision = 10; +constexpr size_t kHlgInvOETFNumEntries = 1 << kHlgInvOETFPrecision; + /* * Convert from scene luminance to PQ. * @@ -280,6 +290,9 @@ Color pqOetf(Color e); float pqOetfLUT(float e); Color pqOetfLUT(Color e); +constexpr size_t kPqOETFPrecision = 10; +constexpr size_t kPqOETFNumEntries = 1 << kPqOETFPrecision; + /* * Convert from PQ to scene luminance in nits. * @@ -290,6 +303,9 @@ Color pqInvOetf(Color e_gamma); float pqInvOetfLUT(float e_gamma); Color pqInvOetfLUT(Color e_gamma); +constexpr size_t kPqInvOETFPrecision = 10; +constexpr size_t kPqInvOETFNumEntries = 1 << kPqInvOETFPrecision; + //////////////////////////////////////////////////////////////////////////////// // Color space conversions @@ -326,13 +342,13 @@ ColorTransformFn getHdrConversionFn(jpegr_color_gamut sdr_gamut, jpegr_color_gam * Calculate the 8-bit unsigned integer recovery value for the given SDR and HDR * luminances in linear space, and the hdr ratio to encode against. */ -uint8_t encodeRecovery(float y_sdr, float y_hdr, float hdr_ratio); +uint8_t encodeRecovery(float y_sdr, float y_hdr, jr_metadata_ptr metadata); /* * Calculates the linear luminance in nits after applying the given recovery * value, with the given hdr ratio, to the given sdr input in the range [0, 1]. */ -Color applyRecovery(Color e, float recovery, float hdr_ratio); +Color applyRecovery(Color e, float recovery, jr_metadata_ptr metadata); Color applyRecoveryLUT(Color e, float recovery, RecoveryLUT& recoveryLUT); /* @@ -376,6 +392,6 @@ float sampleMap(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, size */ uint32_t colorToRgba1010102(Color e_gamma); -} // namespace android::recoverymap +} // namespace android::jpegrecoverymap #endif // ANDROID_JPEGRECOVERYMAP_RECOVERYMAPMATH_H diff --git a/libs/jpegrecoverymap/jpegdecoder.cpp b/libs/jpegrecoverymap/jpegdecoderhelper.cpp index 1bf609a54c..d36bbf8165 100644 --- a/libs/jpegrecoverymap/jpegdecoder.cpp +++ b/libs/jpegrecoverymap/jpegdecoderhelper.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include <jpegrecoverymap/jpegdecoder.h> +#include <jpegrecoverymap/jpegdecoderhelper.h> #include <utils/Log.h> @@ -24,7 +24,7 @@ using namespace std; -namespace android::recoverymap { +namespace android::jpegrecoverymap { const uint32_t kAPP0Marker = JPEG_APP0; // JFIF const uint32_t kAPP1Marker = JPEG_APP0 + 1; // EXIF, XMP @@ -90,14 +90,14 @@ static void jpegrerror_exit(j_common_ptr cinfo) { longjmp(err->setjmp_buffer, 1); } -JpegDecoder::JpegDecoder() { +JpegDecoderHelper::JpegDecoderHelper() { mExifPos = 0; } -JpegDecoder::~JpegDecoder() { +JpegDecoderHelper::~JpegDecoderHelper() { } -bool JpegDecoder::decompressImage(const void* image, int length, bool decodeToRGBA) { +bool JpegDecoderHelper::decompressImage(const void* image, int length, bool decodeToRGBA) { if (image == nullptr || length <= 0) { ALOGE("Image size can not be handled: %d", length); return false; @@ -112,39 +112,39 @@ bool JpegDecoder::decompressImage(const void* image, int length, bool decodeToRG return true; } -void* JpegDecoder::getDecompressedImagePtr() { +void* JpegDecoderHelper::getDecompressedImagePtr() { return mResultBuffer.data(); } -size_t JpegDecoder::getDecompressedImageSize() { +size_t JpegDecoderHelper::getDecompressedImageSize() { return mResultBuffer.size(); } -void* JpegDecoder::getXMPPtr() { +void* JpegDecoderHelper::getXMPPtr() { return mXMPBuffer.data(); } -size_t JpegDecoder::getXMPSize() { +size_t JpegDecoderHelper::getXMPSize() { return mXMPBuffer.size(); } -void* JpegDecoder::getEXIFPtr() { +void* JpegDecoderHelper::getEXIFPtr() { return mEXIFBuffer.data(); } -size_t JpegDecoder::getEXIFSize() { +size_t JpegDecoderHelper::getEXIFSize() { return mEXIFBuffer.size(); } -size_t JpegDecoder::getDecompressedImageWidth() { +size_t JpegDecoderHelper::getDecompressedImageWidth() { return mWidth; } -size_t JpegDecoder::getDecompressedImageHeight() { +size_t JpegDecoderHelper::getDecompressedImageHeight() { return mHeight; } -bool JpegDecoder::decode(const void* image, int length, bool decodeToRGBA) { +bool JpegDecoderHelper::decode(const void* image, int length, bool decodeToRGBA) { jpeg_decompress_struct cinfo; jpegr_source_mgr mgr(static_cast<const uint8_t*>(image), length); jpegrerror_mgr myerr; @@ -248,7 +248,7 @@ bool JpegDecoder::decode(const void* image, int length, bool decodeToRGBA) { return true; } -bool JpegDecoder::decompress(jpeg_decompress_struct* cinfo, const uint8_t* dest, +bool JpegDecoderHelper::decompress(jpeg_decompress_struct* cinfo, const uint8_t* dest, bool isSingleChannel) { if (isSingleChannel) { return decompressSingleChannel(cinfo, dest); @@ -259,7 +259,7 @@ bool JpegDecoder::decompress(jpeg_decompress_struct* cinfo, const uint8_t* dest, return decompressYUV(cinfo, dest); } -bool JpegDecoder::getCompressedImageParameters(const void* image, int length, +bool JpegDecoderHelper::getCompressedImageParameters(const void* image, int length, size_t *pWidth, size_t *pHeight, std::vector<uint8_t> *iccData , std::vector<uint8_t> *exifData) { jpeg_decompress_struct cinfo; @@ -326,7 +326,7 @@ bool JpegDecoder::getCompressedImageParameters(const void* image, int length, return true; } -bool JpegDecoder::decompressRGBA(jpeg_decompress_struct* cinfo, const uint8_t* dest) { +bool JpegDecoderHelper::decompressRGBA(jpeg_decompress_struct* cinfo, const uint8_t* dest) { JSAMPLE* decodeDst = (JSAMPLE*) dest; uint32_t lines = 0; // TODO: use batches for more effectiveness @@ -341,7 +341,7 @@ bool JpegDecoder::decompressRGBA(jpeg_decompress_struct* cinfo, const uint8_t* d return lines == cinfo->image_height; } -bool JpegDecoder::decompressYUV(jpeg_decompress_struct* cinfo, const uint8_t* dest) { +bool JpegDecoderHelper::decompressYUV(jpeg_decompress_struct* cinfo, const uint8_t* dest) { JSAMPROW y[kCompressBatchSize]; JSAMPROW cb[kCompressBatchSize / 2]; @@ -386,7 +386,7 @@ bool JpegDecoder::decompressYUV(jpeg_decompress_struct* cinfo, const uint8_t* de return true; } -bool JpegDecoder::decompressSingleChannel(jpeg_decompress_struct* cinfo, const uint8_t* dest) { +bool JpegDecoderHelper::decompressSingleChannel(jpeg_decompress_struct* cinfo, const uint8_t* dest) { JSAMPROW y[kCompressBatchSize]; JSAMPARRAY planes[1] {y}; @@ -413,4 +413,4 @@ bool JpegDecoder::decompressSingleChannel(jpeg_decompress_struct* cinfo, const u return true; } -} // namespace android +} // namespace jpegrecoverymap diff --git a/libs/jpegrecoverymap/jpegencoder.cpp b/libs/jpegrecoverymap/jpegencoderhelper.cpp index 627dcdf6ee..586cd346e4 100644 --- a/libs/jpegrecoverymap/jpegencoder.cpp +++ b/libs/jpegrecoverymap/jpegencoderhelper.cpp @@ -14,28 +14,28 @@ * limitations under the License. */ -#include <jpegrecoverymap/jpegencoder.h> +#include <jpegrecoverymap/jpegencoderhelper.h> #include <utils/Log.h> #include <errno.h> -namespace android::recoverymap { +namespace android::jpegrecoverymap { -// The destination manager that can access |mResultBuffer| in JpegEncoder. +// The destination manager that can access |mResultBuffer| in JpegEncoderHelper. struct destination_mgr { public: struct jpeg_destination_mgr mgr; - JpegEncoder* encoder; + JpegEncoderHelper* encoder; }; -JpegEncoder::JpegEncoder() { +JpegEncoderHelper::JpegEncoderHelper() { } -JpegEncoder::~JpegEncoder() { +JpegEncoderHelper::~JpegEncoderHelper() { } -bool JpegEncoder::compressImage(const void* image, int width, int height, int quality, +bool JpegEncoderHelper::compressImage(const void* image, int width, int height, int quality, const void* iccBuffer, unsigned int iccSize, bool isSingleChannel) { if (width % 8 != 0 || height % 2 != 0) { @@ -52,15 +52,15 @@ bool JpegEncoder::compressImage(const void* image, int width, int height, int qu return true; } -void* JpegEncoder::getCompressedImagePtr() { +void* JpegEncoderHelper::getCompressedImagePtr() { return mResultBuffer.data(); } -size_t JpegEncoder::getCompressedImageSize() { +size_t JpegEncoderHelper::getCompressedImageSize() { return mResultBuffer.size(); } -void JpegEncoder::initDestination(j_compress_ptr cinfo) { +void JpegEncoderHelper::initDestination(j_compress_ptr cinfo) { destination_mgr* dest = reinterpret_cast<destination_mgr*>(cinfo->dest); std::vector<JOCTET>& buffer = dest->encoder->mResultBuffer; buffer.resize(kBlockSize); @@ -68,7 +68,7 @@ void JpegEncoder::initDestination(j_compress_ptr cinfo) { dest->mgr.free_in_buffer = buffer.size(); } -boolean JpegEncoder::emptyOutputBuffer(j_compress_ptr cinfo) { +boolean JpegEncoderHelper::emptyOutputBuffer(j_compress_ptr cinfo) { destination_mgr* dest = reinterpret_cast<destination_mgr*>(cinfo->dest); std::vector<JOCTET>& buffer = dest->encoder->mResultBuffer; size_t oldsize = buffer.size(); @@ -78,13 +78,13 @@ boolean JpegEncoder::emptyOutputBuffer(j_compress_ptr cinfo) { return true; } -void JpegEncoder::terminateDestination(j_compress_ptr cinfo) { +void JpegEncoderHelper::terminateDestination(j_compress_ptr cinfo) { destination_mgr* dest = reinterpret_cast<destination_mgr*>(cinfo->dest); std::vector<JOCTET>& buffer = dest->encoder->mResultBuffer; buffer.resize(buffer.size() - dest->mgr.free_in_buffer); } -void JpegEncoder::outputErrorMessage(j_common_ptr cinfo) { +void JpegEncoderHelper::outputErrorMessage(j_common_ptr cinfo) { char buffer[JMSG_LENGTH_MAX]; /* Create the message */ @@ -92,7 +92,7 @@ void JpegEncoder::outputErrorMessage(j_common_ptr cinfo) { ALOGE("%s\n", buffer); } -bool JpegEncoder::encode(const void* image, int width, int height, int jpegQuality, +bool JpegEncoderHelper::encode(const void* image, int width, int height, int jpegQuality, const void* iccBuffer, unsigned int iccSize, bool isSingleChannel) { jpeg_compress_struct cinfo; jpeg_error_mgr jerr; @@ -118,7 +118,7 @@ bool JpegEncoder::encode(const void* image, int width, int height, int jpegQuali return true; } -void JpegEncoder::setJpegDestination(jpeg_compress_struct* cinfo) { +void JpegEncoderHelper::setJpegDestination(jpeg_compress_struct* cinfo) { destination_mgr* dest = static_cast<struct destination_mgr *>((*cinfo->mem->alloc_small) ( (j_common_ptr) cinfo, JPOOL_PERMANENT, sizeof(destination_mgr))); dest->encoder = this; @@ -128,7 +128,7 @@ void JpegEncoder::setJpegDestination(jpeg_compress_struct* cinfo) { cinfo->dest = reinterpret_cast<struct jpeg_destination_mgr*>(dest); } -void JpegEncoder::setJpegCompressStruct(int width, int height, int quality, +void JpegEncoderHelper::setJpegCompressStruct(int width, int height, int quality, jpeg_compress_struct* cinfo, bool isSingleChannel) { cinfo->image_width = width; cinfo->image_height = height; @@ -158,7 +158,7 @@ void JpegEncoder::setJpegCompressStruct(int width, int height, int quality, } } -bool JpegEncoder::compress( +bool JpegEncoderHelper::compress( jpeg_compress_struct* cinfo, const uint8_t* image, bool isSingleChannel) { if (isSingleChannel) { return compressSingleChannel(cinfo, image); @@ -166,7 +166,7 @@ bool JpegEncoder::compress( return compressYuv(cinfo, image); } -bool JpegEncoder::compressYuv(jpeg_compress_struct* cinfo, const uint8_t* yuv) { +bool JpegEncoderHelper::compressYuv(jpeg_compress_struct* cinfo, const uint8_t* yuv) { JSAMPROW y[kCompressBatchSize]; JSAMPROW cb[kCompressBatchSize / 2]; JSAMPROW cr[kCompressBatchSize / 2]; @@ -210,7 +210,7 @@ bool JpegEncoder::compressYuv(jpeg_compress_struct* cinfo, const uint8_t* yuv) { return true; } -bool JpegEncoder::compressSingleChannel(jpeg_compress_struct* cinfo, const uint8_t* image) { +bool JpegEncoderHelper::compressSingleChannel(jpeg_compress_struct* cinfo, const uint8_t* image) { JSAMPROW y[kCompressBatchSize]; JSAMPARRAY planes[1] {y}; @@ -236,4 +236,4 @@ bool JpegEncoder::compressSingleChannel(jpeg_compress_struct* cinfo, const uint8 return true; } -} // namespace android +} // namespace jpegrecoverymap diff --git a/libs/jpegrecoverymap/recoverymap.cpp b/libs/jpegrecoverymap/jpegr.cpp index 8b8c2e7fe8..bd8874ed5d 100644 --- a/libs/jpegrecoverymap/recoverymap.cpp +++ b/libs/jpegrecoverymap/jpegr.cpp @@ -14,11 +14,11 @@ * limitations under the License. */ -#include <jpegrecoverymap/recoverymap.h> -#include <jpegrecoverymap/jpegencoder.h> -#include <jpegrecoverymap/jpegdecoder.h> +#include <jpegrecoverymap/jpegr.h> +#include <jpegrecoverymap/jpegencoderhelper.h> +#include <jpegrecoverymap/jpegdecoderhelper.h> #include <jpegrecoverymap/recoverymapmath.h> -#include <jpegrecoverymap/recoverymaputils.h> +#include <jpegrecoverymap/jpegrutils.h> #include <image_io/jpeg/jpeg_marker.h> #include <image_io/jpeg/jpeg_info.h> @@ -43,7 +43,7 @@ using namespace std; using namespace photos_editing_formats::image_io; -namespace android::recoverymap { +namespace android::jpegrecoverymap { #define USE_SRGB_INVOETF_LUT 1 #define USE_HLG_OETF_LUT 1 @@ -86,14 +86,15 @@ int GetCPUCoreCount() { return cpuCoreCount; } -static const map<recoverymap::jpegr_color_gamut, skcms_Matrix3x3> jrGamut_to_skGamut { +static const map<jpegrecoverymap::jpegr_color_gamut, skcms_Matrix3x3> jrGamut_to_skGamut { {JPEGR_COLORGAMUT_BT709, SkNamedGamut::kSRGB}, {JPEGR_COLORGAMUT_P3, SkNamedGamut::kDisplayP3}, {JPEGR_COLORGAMUT_BT2100, SkNamedGamut::kRec2020}, }; static const map< - recoverymap::jpegr_transfer_function, skcms_TransferFunction> jrTransFunc_to_skTransFunc { + jpegrecoverymap::jpegr_transfer_function, + skcms_TransferFunction> jrTransFunc_to_skTransFunc { {JPEGR_TF_SRGB, SkNamedTransferFn::kSRGB}, {JPEGR_TF_LINEAR, SkNamedTransferFn::kLinear}, {JPEGR_TF_HLG, SkNamedTransferFn::kHLG}, @@ -101,7 +102,7 @@ static const map< }; /* Encode API-0 */ -status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, +status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, jpegr_transfer_function hdr_tf, jr_compressed_ptr dest, int quality, @@ -146,7 +147,7 @@ status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, jrTransFunc_to_skTransFunc.at(JPEGR_TF_SRGB), jrGamut_to_skGamut.at(uncompressed_yuv_420_image.colorGamut)); - JpegEncoder jpeg_encoder; + JpegEncoderHelper jpeg_encoder; if (!jpeg_encoder.compressImage(uncompressed_yuv_420_image.data, uncompressed_yuv_420_image.width, uncompressed_yuv_420_image.height, quality, @@ -163,7 +164,7 @@ status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, } /* Encode API-1 */ -status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, +status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, jr_uncompressed_ptr uncompressed_yuv_420_image, jpegr_transfer_function hdr_tf, jr_compressed_ptr dest, @@ -210,7 +211,7 @@ status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, jrTransFunc_to_skTransFunc.at(JPEGR_TF_SRGB), jrGamut_to_skGamut.at(uncompressed_yuv_420_image->colorGamut)); - JpegEncoder jpeg_encoder; + JpegEncoderHelper jpeg_encoder; if (!jpeg_encoder.compressImage(uncompressed_yuv_420_image->data, uncompressed_yuv_420_image->width, uncompressed_yuv_420_image->height, quality, @@ -227,7 +228,7 @@ status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, } /* Encode API-2 */ -status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, +status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, jr_uncompressed_ptr uncompressed_yuv_420_image, jr_compressed_ptr compressed_jpeg_image, jpegr_transfer_function hdr_tf, @@ -272,7 +273,7 @@ status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, } /* Encode API-3 */ -status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, +status_t JpegR::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, jr_compressed_ptr compressed_jpeg_image, jpegr_transfer_function hdr_tf, jr_compressed_ptr dest) { @@ -289,7 +290,7 @@ status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, return ERROR_JPEGR_INVALID_INPUT_TYPE; } - JpegDecoder jpeg_decoder; + JpegDecoderHelper jpeg_decoder; if (!jpeg_decoder.decompressImage(compressed_jpeg_image->data, compressed_jpeg_image->length)) { return ERROR_JPEGR_DECODE_ERROR; } @@ -324,7 +325,7 @@ status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image, return NO_ERROR; } -status_t RecoveryMap::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, +status_t JpegR::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, jr_info_ptr jpegr_info) { if (compressed_jpegr_image == nullptr || jpegr_info == nullptr) { return ERROR_JPEGR_INVALID_NULL_PTR; @@ -334,7 +335,7 @@ status_t RecoveryMap::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, JPEGR_CHECK(extractPrimaryImageAndRecoveryMap(compressed_jpegr_image, &primary_image, &recovery_map)); - JpegDecoder jpeg_decoder; + JpegDecoderHelper jpeg_decoder; if (!jpeg_decoder.getCompressedImageParameters(primary_image.data, primary_image.length, &jpegr_info->width, &jpegr_info->height, jpegr_info->iccData, jpegr_info->exifData)) { @@ -345,7 +346,7 @@ status_t RecoveryMap::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, } /* Decode API */ -status_t RecoveryMap::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, +status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, jr_uncompressed_ptr dest, jr_exif_ptr exif, bool request_sdr) { @@ -356,7 +357,7 @@ status_t RecoveryMap::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, (void) exif; if (request_sdr) { - JpegDecoder jpeg_decoder; + JpegDecoderHelper jpeg_decoder; if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length, true)) { return ERROR_JPEGR_DECODE_ERROR; @@ -376,12 +377,12 @@ status_t RecoveryMap::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, jpegr_metadata metadata; JPEGR_CHECK(extractRecoveryMap(compressed_jpegr_image, &compressed_map)); - JpegDecoder jpeg_decoder; + JpegDecoderHelper jpeg_decoder; if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length)) { return ERROR_JPEGR_DECODE_ERROR; } - JpegDecoder recovery_map_decoder; + JpegDecoderHelper recovery_map_decoder; if (!recovery_map_decoder.decompressImage(compressed_map.data, compressed_map.length)) { return ERROR_JPEGR_DECODE_ERROR; } @@ -405,13 +406,13 @@ status_t RecoveryMap::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, return NO_ERROR; } -status_t RecoveryMap::compressRecoveryMap(jr_uncompressed_ptr uncompressed_recovery_map, +status_t JpegR::compressRecoveryMap(jr_uncompressed_ptr uncompressed_recovery_map, jr_compressed_ptr dest) { if (uncompressed_recovery_map == nullptr || dest == nullptr) { return ERROR_JPEGR_INVALID_NULL_PTR; } - JpegEncoder jpeg_encoder; + JpegEncoderHelper jpeg_encoder; if (!jpeg_encoder.compressImage(uncompressed_recovery_map->data, uncompressed_recovery_map->width, uncompressed_recovery_map->height, @@ -489,7 +490,7 @@ void JobQueue::reset() { mQueuedAllJobs = false; } -status_t RecoveryMap::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image, +status_t JpegR::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image, jr_uncompressed_ptr uncompressed_p010_image, jpegr_transfer_function hdr_tf, jr_metadata_ptr metadata, @@ -573,19 +574,20 @@ status_t RecoveryMap::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_4 } std::mutex mutex; - float hdr_y_nits_max = 0.0f; - double hdr_y_nits_avg = 0.0f; + float max_gain = 0.0f; + float min_gain = 1.0f; const int threads = std::clamp(GetCPUCoreCount(), 1, 4); size_t rowStep = threads == 1 ? image_height : kJobSzInRows; JobQueue jobQueue; - std::function<void()> computeMetadata = [uncompressed_p010_image, hdrInvOetf, - hdrGamutConversionFn, luminanceFn, hdr_white_nits, - threads, &mutex, &hdr_y_nits_avg, - &hdr_y_nits_max, &jobQueue]() -> void { + std::function<void()> computeMetadata = [uncompressed_p010_image, uncompressed_yuv_420_image, + hdrInvOetf, hdrGamutConversionFn, luminanceFn, + hdr_white_nits, threads, &mutex, &max_gain, &min_gain, + &jobQueue]() -> void { size_t rowStart, rowEnd; - float hdr_y_nits_max_th = 0.0f; - double hdr_y_nits_avg_th = 0.0f; + float max_gain_th = 0.0f; + float min_gain_th = 1.0f; + while (jobQueue.dequeueJob(rowStart, rowEnd)) { for (size_t y = rowStart; y < rowEnd; ++y) { for (size_t x = 0; x < uncompressed_p010_image->width; ++x) { @@ -595,16 +597,25 @@ status_t RecoveryMap::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_4 hdr_rgb = hdrGamutConversionFn(hdr_rgb); float hdr_y_nits = luminanceFn(hdr_rgb) * hdr_white_nits; - hdr_y_nits_avg_th += hdr_y_nits; - if (hdr_y_nits > hdr_y_nits_max_th) { - hdr_y_nits_max_th = hdr_y_nits; - } + Color sdr_yuv_gamma = + getYuv420Pixel(uncompressed_yuv_420_image, x, y); + Color sdr_rgb_gamma = srgbYuvToRgb(sdr_yuv_gamma); +#if USE_SRGB_INVOETF_LUT + Color sdr_rgb = srgbInvOetfLUT(sdr_rgb_gamma); +#else + Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma); +#endif + float sdr_y_nits = luminanceFn(sdr_rgb) * kSdrWhiteNits; + + float gain = hdr_y_nits / sdr_y_nits; + max_gain_th = std::max(max_gain_th, gain); + min_gain_th = std::min(min_gain_th, gain); } } } std::unique_lock<std::mutex> lock{mutex}; - hdr_y_nits_avg += hdr_y_nits_avg_th; - hdr_y_nits_max = std::max(hdr_y_nits_max, hdr_y_nits_max_th); + max_gain = std::max(max_gain, max_gain_th); + min_gain = std::min(min_gain, min_gain_th); }; std::function<void()> generateMap = [uncompressed_yuv_420_image, uncompressed_p010_image, @@ -634,7 +645,7 @@ status_t RecoveryMap::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_4 size_t pixel_idx = x + y * dest_map_stride; reinterpret_cast<uint8_t*>(dest->data)[pixel_idx] = - encodeRecovery(sdr_y_nits, hdr_y_nits, metadata->maxContentBoost); + encodeRecovery(sdr_y_nits, hdr_y_nits, metadata); } } } @@ -655,9 +666,9 @@ status_t RecoveryMap::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_4 computeMetadata(); std::for_each(workers.begin(), workers.end(), [](std::thread& t) { t.join(); }); workers.clear(); - hdr_y_nits_avg /= image_width * image_height; - metadata->maxContentBoost = hdr_y_nits_max / kSdrWhiteNits; + metadata->maxContentBoost = max_gain; + metadata->minContentBoost = min_gain; // generate map jobQueue.reset(); @@ -679,7 +690,7 @@ status_t RecoveryMap::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_4 return NO_ERROR; } -status_t RecoveryMap::applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image, +status_t JpegR::applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image, jr_uncompressed_ptr uncompressed_recovery_map, jr_metadata_ptr metadata, jr_uncompressed_ptr dest) { @@ -693,7 +704,7 @@ status_t RecoveryMap::applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_ dest->width = uncompressed_yuv_420_image->width; dest->height = uncompressed_yuv_420_image->height; ShepardsIDW idwTable(kMapDimensionScaleFactor); - RecoveryLUT recoveryLUT(metadata->maxContentBoost); + RecoveryLUT recoveryLUT(metadata); JobQueue jobQueue; std::function<void()> applyRecMap = [uncompressed_yuv_420_image, uncompressed_recovery_map, @@ -729,13 +740,12 @@ status_t RecoveryMap::applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_ if (map_scale_factor != floorf(map_scale_factor)) { recovery = sampleMap(uncompressed_recovery_map, map_scale_factor, x, y); } else { - recovery = sampleMap(uncompressed_recovery_map, map_scale_factor, x, y, - idwTable); + recovery = sampleMap(uncompressed_recovery_map, map_scale_factor, x, y, idwTable); } #if USE_APPLY_RECOVERY_LUT Color rgb_hdr = applyRecoveryLUT(rgb_sdr, recovery, recoveryLUT); #else - Color rgb_hdr = applyRecovery(rgb_sdr, recovery, hdr_ratio); + Color rgb_hdr = applyRecovery(rgb_sdr, recovery, metadata); #endif Color rgb_gamma_hdr = hdrOetf(rgb_hdr / metadata->maxContentBoost); uint32_t rgba1010102 = colorToRgba1010102(rgb_gamma_hdr); @@ -764,7 +774,7 @@ status_t RecoveryMap::applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_ return NO_ERROR; } -status_t RecoveryMap::extractPrimaryImageAndRecoveryMap(jr_compressed_ptr compressed_jpegr_image, +status_t JpegR::extractPrimaryImageAndRecoveryMap(jr_compressed_ptr compressed_jpegr_image, jr_compressed_ptr primary_image, jr_compressed_ptr recovery_map) { if (compressed_jpegr_image == nullptr) { @@ -814,7 +824,7 @@ status_t RecoveryMap::extractPrimaryImageAndRecoveryMap(jr_compressed_ptr compre } -status_t RecoveryMap::extractRecoveryMap(jr_compressed_ptr compressed_jpegr_image, +status_t JpegR::extractRecoveryMap(jr_compressed_ptr compressed_jpegr_image, jr_compressed_ptr dest) { if (compressed_jpegr_image == nullptr || dest == nullptr) { return ERROR_JPEGR_INVALID_NULL_PTR; @@ -845,7 +855,7 @@ status_t RecoveryMap::extractRecoveryMap(jr_compressed_ptr compressed_jpegr_imag // Exif 2.2 spec for EXIF marker // Adobe XMP spec part 3 for XMP marker // ICC v4.3 spec for ICC -status_t RecoveryMap::appendRecoveryMap(jr_compressed_ptr compressed_jpeg_image, +status_t JpegR::appendRecoveryMap(jr_compressed_ptr compressed_jpeg_image, jr_compressed_ptr compressed_recovery_map, jr_exif_ptr exif, jr_metadata_ptr metadata, @@ -908,7 +918,7 @@ status_t RecoveryMap::appendRecoveryMap(jr_compressed_ptr compressed_jpeg_image, return NO_ERROR; } -status_t RecoveryMap::toneMap(jr_uncompressed_ptr src, +status_t JpegR::toneMap(jr_uncompressed_ptr src, jr_uncompressed_ptr dest) { if (src == nullptr || dest == nullptr) { return ERROR_JPEGR_INVALID_NULL_PTR; @@ -945,4 +955,4 @@ status_t RecoveryMap::toneMap(jr_uncompressed_ptr src, return NO_ERROR; } -} // namespace android::recoverymap +} // namespace android::jpegrecoverymap diff --git a/libs/jpegrecoverymap/recoverymaputils.cpp b/libs/jpegrecoverymap/jpegrutils.cpp index 40956bdaba..bcca91af35 100644 --- a/libs/jpegrecoverymap/recoverymaputils.cpp +++ b/libs/jpegrecoverymap/jpegrutils.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include <jpegrecoverymap/recoverymaputils.h> +#include <jpegrecoverymap/jpegrutils.h> #include <image_io/xml/xml_reader.h> #include <image_io/xml/xml_writer.h> #include <image_io/base/message_handler.h> @@ -25,7 +25,7 @@ using namespace photos_editing_formats::image_io; using namespace std; -namespace android::recoverymap { +namespace android::jpegrecoverymap { /* * Helper function used for generating XMP metadata. @@ -247,4 +247,4 @@ string generateXmp(int secondary_image_length, jpegr_metadata& metadata) { return ss.str(); } -} // namespace android::recoverymap +} // namespace android::jpegrecoverymap diff --git a/libs/jpegrecoverymap/recoverymapmath.cpp b/libs/jpegrecoverymap/recoverymapmath.cpp index 4f21ac6372..7812e18b23 100644 --- a/libs/jpegrecoverymap/recoverymapmath.cpp +++ b/libs/jpegrecoverymap/recoverymapmath.cpp @@ -18,67 +18,48 @@ #include <vector> #include <jpegrecoverymap/recoverymapmath.h> -namespace android::recoverymap { - -constexpr size_t kPqOETFPrecision = 10; -constexpr size_t kPqOETFNumEntries = 1 << kPqOETFPrecision; +namespace android::jpegrecoverymap { static const std::vector<float> kPqOETF = [] { std::vector<float> result; - float increment = 1.0 / kPqOETFNumEntries; - float value = 0.0f; - for (int idx = 0; idx < kPqOETFNumEntries; idx++, value += increment) { + for (int idx = 0; idx < kPqOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kPqOETFNumEntries - 1); result.push_back(pqOetf(value)); } return result; }(); -constexpr size_t kPqInvOETFPrecision = 10; -constexpr size_t kPqInvOETFNumEntries = 1 << kPqInvOETFPrecision; - static const std::vector<float> kPqInvOETF = [] { std::vector<float> result; - float increment = 1.0 / kPqInvOETFNumEntries; - float value = 0.0f; - for (int idx = 0; idx < kPqInvOETFNumEntries; idx++, value += increment) { + for (int idx = 0; idx < kPqInvOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kPqInvOETFNumEntries - 1); result.push_back(pqInvOetf(value)); } return result; }(); -constexpr size_t kHlgOETFPrecision = 10; -constexpr size_t kHlgOETFNumEntries = 1 << kHlgOETFPrecision; - static const std::vector<float> kHlgOETF = [] { std::vector<float> result; - float increment = 1.0 / kHlgOETFNumEntries; - float value = 0.0f; - for (int idx = 0; idx < kHlgOETFNumEntries; idx++, value += increment) { + for (int idx = 0; idx < kHlgOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kHlgOETFNumEntries - 1); result.push_back(hlgOetf(value)); } return result; }(); -constexpr size_t kHlgInvOETFPrecision = 10; -constexpr size_t kHlgInvOETFNumEntries = 1 << kHlgInvOETFPrecision; - static const std::vector<float> kHlgInvOETF = [] { std::vector<float> result; - float increment = 1.0 / kHlgInvOETFNumEntries; - float value = 0.0f; - for (int idx = 0; idx < kHlgInvOETFNumEntries; idx++, value += increment) { + for (int idx = 0; idx < kHlgInvOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kHlgInvOETFNumEntries - 1); result.push_back(hlgInvOetf(value)); } return result; }(); -constexpr size_t kSRGBInvOETFPrecision = 10; -constexpr size_t kSRGBInvOETFNumEntries = 1 << kSRGBInvOETFPrecision; -static const std::vector<float> kSRGBInvOETF = [] { +static const std::vector<float> kSrgbInvOETF = [] { std::vector<float> result; - float increment = 1.0 / kSRGBInvOETFNumEntries; - float value = 0.0f; - for (int idx = 0; idx < kSRGBInvOETFNumEntries; idx++, value += increment) { + for (int idx = 0; idx < kSrgbInvOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kSrgbInvOETFNumEntries - 1); result.push_back(srgbInvOetf(value)); } return result; @@ -182,10 +163,10 @@ Color srgbInvOetf(Color e_gamma) { // See IEC 61966-2-1, Equations F.5 and F.6. float srgbInvOetfLUT(float e_gamma) { - uint32_t value = static_cast<uint32_t>(e_gamma * kSRGBInvOETFNumEntries); + uint32_t value = static_cast<uint32_t>(e_gamma * kSrgbInvOETFNumEntries); //TODO() : Remove once conversion modules have appropriate clamping in place - value = CLIP3(value, 0, kSRGBInvOETFNumEntries - 1); - return kSRGBInvOETF[value]; + value = CLIP3(value, 0, kSrgbInvOETFNumEntries - 1); + return kSrgbInvOETF[value]; } Color srgbInvOetfLUT(Color e_gamma) { @@ -461,21 +442,24 @@ ColorTransformFn getHdrConversionFn(jpegr_color_gamut sdr_gamut, jpegr_color_gam //////////////////////////////////////////////////////////////////////////////// // Recovery map calculations - -uint8_t encodeRecovery(float y_sdr, float y_hdr, float hdr_ratio) { +uint8_t encodeRecovery(float y_sdr, float y_hdr, jr_metadata_ptr metadata) { float gain = 1.0f; if (y_sdr > 0.0f) { gain = y_hdr / y_sdr; } - if (gain < (1.0f / hdr_ratio)) gain = 1.0f / hdr_ratio; - if (gain > hdr_ratio) gain = hdr_ratio; + if (gain < metadata->minContentBoost) gain = metadata->minContentBoost; + if (gain > metadata->maxContentBoost) gain = metadata->maxContentBoost; - return static_cast<uint8_t>(log2(gain) / log2(hdr_ratio) * 127.5f + 127.5f); + return static_cast<uint8_t>((log2(gain) - log2(metadata->minContentBoost)) + / (log2(metadata->maxContentBoost) - log2(metadata->minContentBoost)) + * 255.0f); } -Color applyRecovery(Color e, float recovery, float hdr_ratio) { - float recoveryFactor = pow(hdr_ratio, recovery); +Color applyRecovery(Color e, float recovery, jr_metadata_ptr metadata) { + float logBoost = log2(metadata->minContentBoost) * (1.0f - recovery) + + log2(metadata->maxContentBoost) * recovery; + float recoveryFactor = exp2(logBoost); return e * recoveryFactor; } @@ -550,7 +534,7 @@ static size_t clamp(const size_t& val, const size_t& low, const size_t& high) { } static float mapUintToFloat(uint8_t map_uint) { - return (static_cast<float>(map_uint) - 127.5f) / 127.5f; + return static_cast<float>(map_uint) / 255.0f; } static float pythDistance(float x_diff, float y_diff) { @@ -558,9 +542,9 @@ static float pythDistance(float x_diff, float y_diff) { } // TODO: If map_scale_factor is guaranteed to be an integer, then remove the following. -float sampleMap(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, size_t y) { - float x_map = static_cast<float>(x) / static_cast<float>(map_scale_factor); - float y_map = static_cast<float>(y) / static_cast<float>(map_scale_factor); +float sampleMap(jr_uncompressed_ptr map, float map_scale_factor, size_t x, size_t y) { + float x_map = static_cast<float>(x) / map_scale_factor; + float y_map = static_cast<float>(y) / map_scale_factor; size_t x_lower = static_cast<size_t>(floor(x_map)); size_t x_upper = x_lower + 1; @@ -647,4 +631,4 @@ uint32_t colorToRgba1010102(Color e_gamma) { | (0x3 << 30); // Set alpha to 1.0 } -} // namespace android::recoverymap +} // namespace android::jpegrecoverymap diff --git a/libs/jpegrecoverymap/tests/Android.bp b/libs/jpegrecoverymap/tests/Android.bp index e381caf025..5a4edb2a25 100644 --- a/libs/jpegrecoverymap/tests/Android.bp +++ b/libs/jpegrecoverymap/tests/Android.bp @@ -25,7 +25,7 @@ cc_test { name: "libjpegrecoverymap_test", test_suites: ["device-tests"], srcs: [ - "recoverymap_test.cpp", + "jpegr_test.cpp", "recoverymapmath_test.cpp", ], shared_libs: [ @@ -44,10 +44,10 @@ cc_test { } cc_test { - name: "libjpegencoder_test", + name: "libjpegencoderhelper_test", test_suites: ["device-tests"], srcs: [ - "jpegencoder_test.cpp", + "jpegencoderhelper_test.cpp", ], shared_libs: [ "libjpeg", @@ -60,10 +60,10 @@ cc_test { } cc_test { - name: "libjpegdecoder_test", + name: "libjpegdecoderhelper_test", test_suites: ["device-tests"], srcs: [ - "jpegdecoder_test.cpp", + "jpegdecoderhelper_test.cpp", ], shared_libs: [ "libjpeg", diff --git a/libs/jpegrecoverymap/tests/jpegdecoder_test.cpp b/libs/jpegrecoverymap/tests/jpegdecoderhelper_test.cpp index 8e013517fb..2f32a5685b 100644 --- a/libs/jpegrecoverymap/tests/jpegdecoder_test.cpp +++ b/libs/jpegrecoverymap/tests/jpegdecoderhelper_test.cpp @@ -14,27 +14,27 @@ * limitations under the License. */ -#include <jpegrecoverymap/jpegdecoder.h> +#include <jpegrecoverymap/jpegdecoderhelper.h> #include <gtest/gtest.h> #include <utils/Log.h> #include <fcntl.h> -namespace android::recoverymap { +namespace android::jpegrecoverymap { #define YUV_IMAGE "/sdcard/Documents/minnie-320x240-yuv.jpg" #define YUV_IMAGE_SIZE 20193 #define GREY_IMAGE "/sdcard/Documents/minnie-320x240-y.jpg" #define GREY_IMAGE_SIZE 20193 -class JpegDecoderTest : public testing::Test { +class JpegDecoderHelperTest : public testing::Test { public: struct Image { std::unique_ptr<uint8_t[]> buffer; size_t size; }; - JpegDecoderTest(); - ~JpegDecoderTest(); + JpegDecoderHelperTest(); + ~JpegDecoderHelperTest(); protected: virtual void SetUp(); virtual void TearDown(); @@ -42,9 +42,9 @@ protected: Image mYuvImage, mGreyImage; }; -JpegDecoderTest::JpegDecoderTest() {} +JpegDecoderHelperTest::JpegDecoderHelperTest() {} -JpegDecoderTest::~JpegDecoderTest() {} +JpegDecoderHelperTest::~JpegDecoderHelperTest() {} static size_t getFileSize(int fd) { struct stat st; @@ -55,7 +55,7 @@ static size_t getFileSize(int fd) { return st.st_size; // bytes } -static bool loadFile(const char filename[], JpegDecoderTest::Image* result) { +static bool loadFile(const char filename[], JpegDecoderHelperTest::Image* result) { int fd = open(filename, O_CLOEXEC); if (fd < 0) { return false; @@ -74,7 +74,7 @@ static bool loadFile(const char filename[], JpegDecoderTest::Image* result) { return true; } -void JpegDecoderTest::SetUp() { +void JpegDecoderHelperTest::SetUp() { if (!loadFile(YUV_IMAGE, &mYuvImage)) { FAIL() << "Load file " << YUV_IMAGE << " failed"; } @@ -85,18 +85,18 @@ void JpegDecoderTest::SetUp() { mGreyImage.size = GREY_IMAGE_SIZE; } -void JpegDecoderTest::TearDown() {} +void JpegDecoderHelperTest::TearDown() {} -TEST_F(JpegDecoderTest, decodeYuvImage) { - JpegDecoder decoder; +TEST_F(JpegDecoderHelperTest, decodeYuvImage) { + JpegDecoderHelper decoder; EXPECT_TRUE(decoder.decompressImage(mYuvImage.buffer.get(), mYuvImage.size)); ASSERT_GT(decoder.getDecompressedImageSize(), static_cast<uint32_t>(0)); } -TEST_F(JpegDecoderTest, decodeGreyImage) { - JpegDecoder decoder; +TEST_F(JpegDecoderHelperTest, decodeGreyImage) { + JpegDecoderHelper decoder; EXPECT_TRUE(decoder.decompressImage(mGreyImage.buffer.get(), mGreyImage.size)); ASSERT_GT(decoder.getDecompressedImageSize(), static_cast<uint32_t>(0)); } -}
\ No newline at end of file +} // namespace android::jpegrecoverymap
\ No newline at end of file diff --git a/libs/jpegrecoverymap/tests/jpegencoder_test.cpp b/libs/jpegrecoverymap/tests/jpegencoderhelper_test.cpp index 4cd2a5ef8c..095ac2fbf6 100644 --- a/libs/jpegrecoverymap/tests/jpegencoder_test.cpp +++ b/libs/jpegrecoverymap/tests/jpegencoderhelper_test.cpp @@ -14,13 +14,13 @@ * limitations under the License. */ -#include <jpegrecoverymap/jpegencoder.h> +#include <jpegrecoverymap/jpegencoderhelper.h> #include <gtest/gtest.h> #include <utils/Log.h> #include <fcntl.h> -namespace android::recoverymap { +namespace android::jpegrecoverymap { #define VALID_IMAGE "/sdcard/Documents/minnie-320x240.yu12" #define VALID_IMAGE_WIDTH 320 @@ -33,15 +33,15 @@ namespace android::recoverymap { #define INVALID_SIZE_IMAGE_HEIGHT 240 #define JPEG_QUALITY 90 -class JpegEncoderTest : public testing::Test { +class JpegEncoderHelperTest : public testing::Test { public: struct Image { std::unique_ptr<uint8_t[]> buffer; size_t width; size_t height; }; - JpegEncoderTest(); - ~JpegEncoderTest(); + JpegEncoderHelperTest(); + ~JpegEncoderHelperTest(); protected: virtual void SetUp(); virtual void TearDown(); @@ -49,9 +49,9 @@ protected: Image mValidImage, mInvalidSizeImage, mSingleChannelImage; }; -JpegEncoderTest::JpegEncoderTest() {} +JpegEncoderHelperTest::JpegEncoderHelperTest() {} -JpegEncoderTest::~JpegEncoderTest() {} +JpegEncoderHelperTest::~JpegEncoderHelperTest() {} static size_t getFileSize(int fd) { struct stat st; @@ -62,7 +62,7 @@ static size_t getFileSize(int fd) { return st.st_size; // bytes } -static bool loadFile(const char filename[], JpegEncoderTest::Image* result) { +static bool loadFile(const char filename[], JpegEncoderHelperTest::Image* result) { int fd = open(filename, O_CLOEXEC); if (fd < 0) { return false; @@ -81,7 +81,7 @@ static bool loadFile(const char filename[], JpegEncoderTest::Image* result) { return true; } -void JpegEncoderTest::SetUp() { +void JpegEncoderHelperTest::SetUp() { if (!loadFile(VALID_IMAGE, &mValidImage)) { FAIL() << "Load file " << VALID_IMAGE << " failed"; } @@ -99,27 +99,27 @@ void JpegEncoderTest::SetUp() { mSingleChannelImage.height = SINGLE_CHANNEL_IMAGE_HEIGHT; } -void JpegEncoderTest::TearDown() {} +void JpegEncoderHelperTest::TearDown() {} -TEST_F(JpegEncoderTest, validImage) { - JpegEncoder encoder; +TEST_F(JpegEncoderHelperTest, validImage) { + JpegEncoderHelper encoder; EXPECT_TRUE(encoder.compressImage(mValidImage.buffer.get(), mValidImage.width, mValidImage.height, JPEG_QUALITY, NULL, 0)); ASSERT_GT(encoder.getCompressedImageSize(), static_cast<uint32_t>(0)); } -TEST_F(JpegEncoderTest, invalidSizeImage) { - JpegEncoder encoder; +TEST_F(JpegEncoderHelperTest, invalidSizeImage) { + JpegEncoderHelper encoder; EXPECT_FALSE(encoder.compressImage(mInvalidSizeImage.buffer.get(), mInvalidSizeImage.width, mInvalidSizeImage.height, JPEG_QUALITY, NULL, 0)); } -TEST_F(JpegEncoderTest, singleChannelImage) { - JpegEncoder encoder; +TEST_F(JpegEncoderHelperTest, singleChannelImage) { + JpegEncoderHelper encoder; EXPECT_TRUE(encoder.compressImage(mSingleChannelImage.buffer.get(), mSingleChannelImage.width, mSingleChannelImage.height, JPEG_QUALITY, NULL, 0, true)); ASSERT_GT(encoder.getCompressedImageSize(), static_cast<uint32_t>(0)); } -} +} // namespace android::jpegrecoverymap diff --git a/libs/jpegrecoverymap/tests/recoverymap_test.cpp b/libs/jpegrecoverymap/tests/jpegr_test.cpp index 3e9a76d47a..c0347e39c7 100644 --- a/libs/jpegrecoverymap/tests/recoverymap_test.cpp +++ b/libs/jpegrecoverymap/tests/jpegr_test.cpp @@ -14,12 +14,13 @@ * limitations under the License. */ -#include <jpegrecoverymap/recoverymap.h> +#include <jpegrecoverymap/jpegr.h> +#include <jpegrecoverymap/jpegrutils.h> #include <jpegrecoverymap/recoverymapmath.h> -#include <jpegrecoverymap/recoverymaputils.h> #include <fcntl.h> #include <fstream> #include <gtest/gtest.h> +#include <sys/time.h> #include <utils/Log.h> #define RAW_P010_IMAGE "/sdcard/Documents/raw_p010_image.p010" @@ -33,29 +34,26 @@ #define SAVE_DECODING_RESULT true #define SAVE_INPUT_RGBA true -namespace android::recoverymap { +namespace android::jpegrecoverymap { -class RecoveryMapTest : public testing::Test { -public: - RecoveryMapTest(); - ~RecoveryMapTest(); -protected: - virtual void SetUp(); - virtual void TearDown(); - - struct jpegr_uncompressed_struct mRawP010Image; - struct jpegr_uncompressed_struct mRawYuv420Image; - struct jpegr_compressed_struct mJpegImage; +struct Timer { + struct timeval StartingTime; + struct timeval EndingTime; + struct timeval ElapsedMicroseconds; }; -RecoveryMapTest::RecoveryMapTest() {} -RecoveryMapTest::~RecoveryMapTest() {} +void timerStart(Timer *t) { + gettimeofday(&t->StartingTime, nullptr); +} -void RecoveryMapTest::SetUp() {} -void RecoveryMapTest::TearDown() { - free(mRawP010Image.data); - free(mRawYuv420Image.data); - free(mJpegImage.data); +void timerStop(Timer *t) { + gettimeofday(&t->EndingTime, nullptr); +} + +int64_t elapsedTime(Timer *t) { + t->ElapsedMicroseconds.tv_sec = t->EndingTime.tv_sec - t->StartingTime.tv_sec; + t->ElapsedMicroseconds.tv_usec = t->EndingTime.tv_usec - t->StartingTime.tv_usec; + return t->ElapsedMicroseconds.tv_sec * 1000000 + t->ElapsedMicroseconds.tv_usec; } static size_t getFileSize(int fd) { @@ -89,19 +87,93 @@ static bool loadFile(const char filename[], void*& result, int* fileLength) { return true; } -TEST_F(RecoveryMapTest, build) { +class JpegRTest : public testing::Test { +public: + JpegRTest(); + ~JpegRTest(); + +protected: + virtual void SetUp(); + virtual void TearDown(); + + struct jpegr_uncompressed_struct mRawP010Image; + struct jpegr_uncompressed_struct mRawYuv420Image; + struct jpegr_compressed_struct mJpegImage; +}; + +JpegRTest::JpegRTest() {} +JpegRTest::~JpegRTest() {} + +void JpegRTest::SetUp() {} +void JpegRTest::TearDown() { + free(mRawP010Image.data); + free(mRawYuv420Image.data); + free(mJpegImage.data); +} + +class JpegRBenchmark : public JpegR { +public: + void BenchmarkGenerateRecoveryMap(jr_uncompressed_ptr yuv420Image, jr_uncompressed_ptr p010Image, + jr_metadata_ptr metadata, jr_uncompressed_ptr map); + void BenchmarkApplyRecoveryMap(jr_uncompressed_ptr yuv420Image, jr_uncompressed_ptr map, + jr_metadata_ptr metadata, jr_uncompressed_ptr dest); +private: + const int kProfileCount = 10; +}; + +void JpegRBenchmark::BenchmarkGenerateRecoveryMap(jr_uncompressed_ptr yuv420Image, + jr_uncompressed_ptr p010Image, + jr_metadata_ptr metadata, + jr_uncompressed_ptr map) { + ASSERT_EQ(yuv420Image->width, p010Image->width); + ASSERT_EQ(yuv420Image->height, p010Image->height); + + Timer genRecMapTime; + + timerStart(&genRecMapTime); + for (auto i = 0; i < kProfileCount; i++) { + ASSERT_EQ(OK, generateRecoveryMap( + yuv420Image, p010Image, jpegr_transfer_function::JPEGR_TF_HLG, metadata, map)); + if (i != kProfileCount - 1) delete[] static_cast<uint8_t *>(map->data); + } + timerStop(&genRecMapTime); + + ALOGE("Generate Recovery Map:- Res = %i x %i, time = %f ms", + yuv420Image->width, yuv420Image->height, + elapsedTime(&genRecMapTime) / (kProfileCount * 1000.f)); + +} + +void JpegRBenchmark::BenchmarkApplyRecoveryMap(jr_uncompressed_ptr yuv420Image, + jr_uncompressed_ptr map, + jr_metadata_ptr metadata, + jr_uncompressed_ptr dest) { + Timer applyRecMapTime; + + timerStart(&applyRecMapTime); + for (auto i = 0; i < kProfileCount; i++) { + ASSERT_EQ(OK, applyRecoveryMap(yuv420Image, map, metadata, dest)); + } + timerStop(&applyRecMapTime); + + ALOGE("Apply Recovery Map:- Res = %i x %i, time = %f ms", + yuv420Image->width, yuv420Image->height, + elapsedTime(&applyRecMapTime) / (kProfileCount * 1000.f)); +} + +TEST_F(JpegRTest, build) { // Force all of the recovery map lib to be linked by calling all public functions. - RecoveryMap recovery_map; - recovery_map.encodeJPEGR(nullptr, static_cast<jpegr_transfer_function>(0), nullptr, 0, nullptr); - recovery_map.encodeJPEGR(nullptr, nullptr, static_cast<jpegr_transfer_function>(0), - nullptr, 0, nullptr); - recovery_map.encodeJPEGR(nullptr, nullptr, nullptr, static_cast<jpegr_transfer_function>(0), - nullptr); - recovery_map.encodeJPEGR(nullptr, nullptr, static_cast<jpegr_transfer_function>(0), nullptr); - recovery_map.decodeJPEGR(nullptr, nullptr, nullptr, false); + JpegR jpegRCodec; + jpegRCodec.encodeJPEGR(nullptr, static_cast<jpegr_transfer_function>(0), nullptr, 0, nullptr); + jpegRCodec.encodeJPEGR(nullptr, nullptr, static_cast<jpegr_transfer_function>(0), + nullptr, 0, nullptr); + jpegRCodec.encodeJPEGR(nullptr, nullptr, nullptr, static_cast<jpegr_transfer_function>(0), + nullptr); + jpegRCodec.encodeJPEGR(nullptr, nullptr, static_cast<jpegr_transfer_function>(0), nullptr); + jpegRCodec.decodeJPEGR(nullptr, nullptr, nullptr, false); } -TEST_F(RecoveryMapTest, writeXmpThenRead) { +TEST_F(JpegRTest, writeXmpThenRead) { jpegr_metadata metadata_expected; metadata_expected.maxContentBoost = 1.25; int length_expected = 1000; @@ -123,7 +195,7 @@ TEST_F(RecoveryMapTest, writeXmpThenRead) { } /* Test Encode API-0 and decode */ -TEST_F(RecoveryMapTest, encodeFromP010ThenDecode) { +TEST_F(JpegRTest, encodeFromP010ThenDecode) { int ret; // Load input files. @@ -134,12 +206,12 @@ TEST_F(RecoveryMapTest, encodeFromP010ThenDecode) { mRawP010Image.height = TEST_IMAGE_HEIGHT; mRawP010Image.colorGamut = jpegr_color_gamut::JPEGR_COLORGAMUT_BT2100; - RecoveryMap recoveryMap; + JpegR jpegRCodec; jpegr_compressed_struct jpegR; jpegR.maxLength = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * sizeof(uint8_t); jpegR.data = malloc(jpegR.maxLength); - ret = recoveryMap.encodeJPEGR( + ret = jpegRCodec.encodeJPEGR( &mRawP010Image, jpegr_transfer_function::JPEGR_TF_HLG, &jpegR, DEFAULT_JPEG_QUALITY, nullptr); if (ret != OK) { FAIL() << "Error code is " << ret; @@ -157,7 +229,7 @@ TEST_F(RecoveryMapTest, encodeFromP010ThenDecode) { jpegr_uncompressed_struct decodedJpegR; int decodedJpegRSize = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * 4; decodedJpegR.data = malloc(decodedJpegRSize); - ret = recoveryMap.decodeJPEGR(&jpegR, &decodedJpegR); + ret = jpegRCodec.decodeJPEGR(&jpegR, &decodedJpegR); if (ret != OK) { FAIL() << "Error code is " << ret; } @@ -176,7 +248,7 @@ TEST_F(RecoveryMapTest, encodeFromP010ThenDecode) { } /* Test Encode API-1 and decode */ -TEST_F(RecoveryMapTest, encodeFromRawHdrAndSdrThenDecode) { +TEST_F(JpegRTest, encodeFromRawHdrAndSdrThenDecode) { int ret; // Load input files. @@ -194,12 +266,12 @@ TEST_F(RecoveryMapTest, encodeFromRawHdrAndSdrThenDecode) { mRawYuv420Image.height = TEST_IMAGE_HEIGHT; mRawYuv420Image.colorGamut = jpegr_color_gamut::JPEGR_COLORGAMUT_BT709; - RecoveryMap recoveryMap; + JpegR jpegRCodec; jpegr_compressed_struct jpegR; jpegR.maxLength = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * sizeof(uint8_t); jpegR.data = malloc(jpegR.maxLength); - ret = recoveryMap.encodeJPEGR( + ret = jpegRCodec.encodeJPEGR( &mRawP010Image, &mRawYuv420Image, jpegr_transfer_function::JPEGR_TF_HLG, &jpegR, DEFAULT_JPEG_QUALITY, nullptr); if (ret != OK) { @@ -218,7 +290,7 @@ TEST_F(RecoveryMapTest, encodeFromRawHdrAndSdrThenDecode) { jpegr_uncompressed_struct decodedJpegR; int decodedJpegRSize = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * 4; decodedJpegR.data = malloc(decodedJpegRSize); - ret = recoveryMap.decodeJPEGR(&jpegR, &decodedJpegR); + ret = jpegRCodec.decodeJPEGR(&jpegR, &decodedJpegR); if (ret != OK) { FAIL() << "Error code is " << ret; } @@ -237,7 +309,7 @@ TEST_F(RecoveryMapTest, encodeFromRawHdrAndSdrThenDecode) { } /* Test Encode API-2 and decode */ -TEST_F(RecoveryMapTest, encodeFromRawHdrAndSdrAndJpegThenDecode) { +TEST_F(JpegRTest, encodeFromRawHdrAndSdrAndJpegThenDecode) { int ret; // Load input files. @@ -260,12 +332,12 @@ TEST_F(RecoveryMapTest, encodeFromRawHdrAndSdrAndJpegThenDecode) { } mJpegImage.colorGamut = jpegr_color_gamut::JPEGR_COLORGAMUT_BT709; - RecoveryMap recoveryMap; + JpegR jpegRCodec; jpegr_compressed_struct jpegR; jpegR.maxLength = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * sizeof(uint8_t); jpegR.data = malloc(jpegR.maxLength); - ret = recoveryMap.encodeJPEGR( + ret = jpegRCodec.encodeJPEGR( &mRawP010Image, &mRawYuv420Image, &mJpegImage, jpegr_transfer_function::JPEGR_TF_HLG, &jpegR); if (ret != OK) { FAIL() << "Error code is " << ret; @@ -283,7 +355,7 @@ TEST_F(RecoveryMapTest, encodeFromRawHdrAndSdrAndJpegThenDecode) { jpegr_uncompressed_struct decodedJpegR; int decodedJpegRSize = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * 4; decodedJpegR.data = malloc(decodedJpegRSize); - ret = recoveryMap.decodeJPEGR(&jpegR, &decodedJpegR); + ret = jpegRCodec.decodeJPEGR(&jpegR, &decodedJpegR); if (ret != OK) { FAIL() << "Error code is " << ret; } @@ -302,7 +374,7 @@ TEST_F(RecoveryMapTest, encodeFromRawHdrAndSdrAndJpegThenDecode) { } /* Test Encode API-3 and decode */ -TEST_F(RecoveryMapTest, encodeFromJpegThenDecode) { +TEST_F(JpegRTest, encodeFromJpegThenDecode) { int ret; // Load input files. @@ -341,12 +413,12 @@ TEST_F(RecoveryMapTest, encodeFromJpegThenDecode) { } mJpegImage.colorGamut = jpegr_color_gamut::JPEGR_COLORGAMUT_BT709; - RecoveryMap recoveryMap; + JpegR jpegRCodec; jpegr_compressed_struct jpegR; jpegR.maxLength = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * sizeof(uint8_t); jpegR.data = malloc(jpegR.maxLength); - ret = recoveryMap.encodeJPEGR( + ret = jpegRCodec.encodeJPEGR( &mRawP010Image, &mJpegImage, jpegr_transfer_function::JPEGR_TF_HLG, &jpegR); if (ret != OK) { FAIL() << "Error code is " << ret; @@ -364,7 +436,7 @@ TEST_F(RecoveryMapTest, encodeFromJpegThenDecode) { jpegr_uncompressed_struct decodedJpegR; int decodedJpegRSize = TEST_IMAGE_WIDTH * TEST_IMAGE_HEIGHT * 4; decodedJpegR.data = malloc(decodedJpegRSize); - ret = recoveryMap.decodeJPEGR(&jpegR, &decodedJpegR); + ret = jpegRCodec.decodeJPEGR(&jpegR, &decodedJpegR); if (ret != OK) { FAIL() << "Error code is " << ret; } @@ -382,4 +454,46 @@ TEST_F(RecoveryMapTest, encodeFromJpegThenDecode) { free(decodedJpegR.data); } -} // namespace android::recoverymap +TEST_F(JpegRTest, ProfileRecoveryMapFuncs) { + const size_t kWidth = TEST_IMAGE_WIDTH; + const size_t kHeight = TEST_IMAGE_HEIGHT; + + // Load input files. + if (!loadFile(RAW_P010_IMAGE, mRawP010Image.data, nullptr)) { + FAIL() << "Load file " << RAW_P010_IMAGE << " failed"; + } + mRawP010Image.width = kWidth; + mRawP010Image.height = kHeight; + mRawP010Image.colorGamut = jpegr_color_gamut::JPEGR_COLORGAMUT_BT2100; + + if (!loadFile(RAW_YUV420_IMAGE, mRawYuv420Image.data, nullptr)) { + FAIL() << "Load file " << RAW_P010_IMAGE << " failed"; + } + mRawYuv420Image.width = kWidth; + mRawYuv420Image.height = kHeight; + mRawYuv420Image.colorGamut = jpegr_color_gamut::JPEGR_COLORGAMUT_BT709; + + JpegRBenchmark benchmark; + + jpegr_metadata metadata = { .version = 1, + .maxContentBoost = 8.0f, + .minContentBoost = 1.0f / 8.0f }; + + jpegr_uncompressed_struct map = { .data = NULL, + .width = 0, + .height = 0, + .colorGamut = JPEGR_COLORGAMUT_UNSPECIFIED }; + + benchmark.BenchmarkGenerateRecoveryMap(&mRawYuv420Image, &mRawP010Image, &metadata, &map); + + const int dstSize = mRawYuv420Image.width * mRawYuv420Image.height * 4; + auto bufferDst = std::make_unique<uint8_t[]>(dstSize); + jpegr_uncompressed_struct dest = { .data = bufferDst.get(), + .width = 0, + .height = 0, + .colorGamut = JPEGR_COLORGAMUT_UNSPECIFIED }; + + benchmark.BenchmarkApplyRecoveryMap(&mRawYuv420Image, &map, &metadata, &dest); +} + +} // namespace android::recoverymap
\ No newline at end of file diff --git a/libs/jpegrecoverymap/tests/recoverymapmath_test.cpp b/libs/jpegrecoverymap/tests/recoverymapmath_test.cpp index 2eec95f01b..6c61ff13d7 100644 --- a/libs/jpegrecoverymap/tests/recoverymapmath_test.cpp +++ b/libs/jpegrecoverymap/tests/recoverymapmath_test.cpp @@ -19,7 +19,7 @@ #include <gmock/gmock.h> #include <jpegrecoverymap/recoverymapmath.h> -namespace android::recoverymap { +namespace android::jpegrecoverymap { class RecoveryMapMathTest : public testing::Test { public: @@ -42,7 +42,7 @@ public: } float Map(uint8_t e) { - return (static_cast<float>(e) - 127.5f) / 127.5f; + return static_cast<float>(e) / 255.0f; } Color ColorMin(Color e1, Color e2) { @@ -88,10 +88,10 @@ public: return luminance_scaled * scale_factor; } - Color Recover(Color yuv_gamma, float recovery, float max_content_boost) { + Color Recover(Color yuv_gamma, float recovery, jr_metadata_ptr metadata) { Color rgb_gamma = srgbYuvToRgb(yuv_gamma); Color rgb = srgbInvOetf(rgb_gamma); - return applyRecovery(rgb, recovery, max_content_boost); + return applyRecovery(rgb, recovery, metadata); } jpegr_uncompressed_struct Yuv420Image() { @@ -518,59 +518,95 @@ TEST_F(RecoveryMapMathTest, PqInvOetf) { } TEST_F(RecoveryMapMathTest, PqInvOetfLUT) { - float increment = 1.0 / 1024.0; - float value = 0.0f; - for (int idx = 0; idx < 1024; idx++, value += increment) { + for (int idx = 0; idx < kPqInvOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kPqInvOETFNumEntries - 1); EXPECT_FLOAT_EQ(pqInvOetf(value), pqInvOetfLUT(value)); } } TEST_F(RecoveryMapMathTest, HlgInvOetfLUT) { - float increment = 1.0 / 1024.0; - float value = 0.0f; - for (int idx = 0; idx < 1024; idx++, value += increment) { + for (int idx = 0; idx < kHlgInvOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kHlgInvOETFNumEntries - 1); EXPECT_FLOAT_EQ(hlgInvOetf(value), hlgInvOetfLUT(value)); } } TEST_F(RecoveryMapMathTest, pqOetfLUT) { - float increment = 1.0 / 1024.0; - float value = 0.0f; - for (int idx = 0; idx < 1024; idx++, value += increment) { + for (int idx = 0; idx < kPqOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kPqOETFNumEntries - 1); EXPECT_FLOAT_EQ(pqOetf(value), pqOetfLUT(value)); } } TEST_F(RecoveryMapMathTest, hlgOetfLUT) { - float increment = 1.0 / 1024.0; - float value = 0.0f; - for (int idx = 0; idx < 1024; idx++, value += increment) { + for (int idx = 0; idx < kHlgOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kHlgOETFNumEntries - 1); EXPECT_FLOAT_EQ(hlgOetf(value), hlgOetfLUT(value)); } } TEST_F(RecoveryMapMathTest, srgbInvOetfLUT) { - float increment = 1.0 / 1024.0; - float value = 0.0f; - for (int idx = 0; idx < 1024; idx++, value += increment) { + for (int idx = 0; idx < kSrgbInvOETFNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kSrgbInvOETFNumEntries - 1); EXPECT_FLOAT_EQ(srgbInvOetf(value), srgbInvOetfLUT(value)); } } TEST_F(RecoveryMapMathTest, applyRecoveryLUT) { - float increment = 2.0 / kRecoveryFactorNumEntries; - for (float hdrRatio = 1.0f; hdrRatio <= 10.0f; hdrRatio += 1.0f) { - RecoveryLUT recoveryLUT(hdrRatio); - for (float value = -1.0f; value <= -1.0f; value += increment) { - EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), value, hdrRatio), + for (int boost = 1; boost <= 10; boost++) { + jpegr_metadata metadata = { .maxContentBoost = static_cast<float>(boost), + .minContentBoost = 1.0f / static_cast<float>(boost) }; + RecoveryLUT recoveryLUT(&metadata); + for (int idx = 0; idx < kRecoveryFactorNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kRecoveryFactorNumEntries - 1); + EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), value, &metadata), applyRecoveryLUT(RgbBlack(), value, recoveryLUT)); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), value, hdrRatio), + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), value, &metadata), applyRecoveryLUT(RgbWhite(), value, recoveryLUT)); - EXPECT_RGB_NEAR(applyRecovery(RgbRed(), value, hdrRatio), + EXPECT_RGB_NEAR(applyRecovery(RgbRed(), value, &metadata), applyRecoveryLUT(RgbRed(), value, recoveryLUT)); - EXPECT_RGB_NEAR(applyRecovery(RgbGreen(), value, hdrRatio), + EXPECT_RGB_NEAR(applyRecovery(RgbGreen(), value, &metadata), applyRecoveryLUT(RgbGreen(), value, recoveryLUT)); - EXPECT_RGB_NEAR(applyRecovery(RgbBlue(), value, hdrRatio), + EXPECT_RGB_NEAR(applyRecovery(RgbBlue(), value, &metadata), + applyRecoveryLUT(RgbBlue(), value, recoveryLUT)); + } + } + + for (int boost = 1; boost <= 10; boost++) { + jpegr_metadata metadata = { .maxContentBoost = static_cast<float>(boost), + .minContentBoost = 1.0f }; + RecoveryLUT recoveryLUT(&metadata); + for (int idx = 0; idx < kRecoveryFactorNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kRecoveryFactorNumEntries - 1); + EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), value, &metadata), + applyRecoveryLUT(RgbBlack(), value, recoveryLUT)); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), value, &metadata), + applyRecoveryLUT(RgbWhite(), value, recoveryLUT)); + EXPECT_RGB_NEAR(applyRecovery(RgbRed(), value, &metadata), + applyRecoveryLUT(RgbRed(), value, recoveryLUT)); + EXPECT_RGB_NEAR(applyRecovery(RgbGreen(), value, &metadata), + applyRecoveryLUT(RgbGreen(), value, recoveryLUT)); + EXPECT_RGB_NEAR(applyRecovery(RgbBlue(), value, &metadata), + applyRecoveryLUT(RgbBlue(), value, recoveryLUT)); + } + } + + for (int boost = 1; boost <= 10; boost++) { + jpegr_metadata metadata = { .maxContentBoost = static_cast<float>(boost), + .minContentBoost = 1.0f / pow(static_cast<float>(boost), + 1.0f / 3.0f) }; + RecoveryLUT recoveryLUT(&metadata); + for (int idx = 0; idx < kRecoveryFactorNumEntries; idx++) { + float value = static_cast<float>(idx) / static_cast<float>(kRecoveryFactorNumEntries - 1); + EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), value, &metadata), + applyRecoveryLUT(RgbBlack(), value, recoveryLUT)); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), value, &metadata), + applyRecoveryLUT(RgbWhite(), value, recoveryLUT)); + EXPECT_RGB_NEAR(applyRecovery(RgbRed(), value, &metadata), + applyRecoveryLUT(RgbRed(), value, recoveryLUT)); + EXPECT_RGB_NEAR(applyRecovery(RgbGreen(), value, &metadata), + applyRecoveryLUT(RgbGreen(), value, recoveryLUT)); + EXPECT_RGB_NEAR(applyRecovery(RgbBlue(), value, &metadata), applyRecoveryLUT(RgbBlue(), value, recoveryLUT)); } } @@ -623,60 +659,121 @@ TEST_F(RecoveryMapMathTest, ColorConversionLookup) { } TEST_F(RecoveryMapMathTest, EncodeRecovery) { - EXPECT_EQ(encodeRecovery(0.0f, 0.0f, 4.0f), 127); - EXPECT_EQ(encodeRecovery(0.0f, 1.0f, 4.0f), 127); - EXPECT_EQ(encodeRecovery(1.0f, 0.0f, 4.0f), 0); - EXPECT_EQ(encodeRecovery(0.5f, 0.0f, 4.0f), 0); - - EXPECT_EQ(encodeRecovery(1.0f, 1.0f, 4.0f), 127); - EXPECT_EQ(encodeRecovery(1.0f, 4.0f, 4.0f), 255); - EXPECT_EQ(encodeRecovery(1.0f, 5.0f, 4.0f), 255); - EXPECT_EQ(encodeRecovery(4.0f, 1.0f, 4.0f), 0); - EXPECT_EQ(encodeRecovery(4.0f, 0.5f, 4.0f), 0); - EXPECT_EQ(encodeRecovery(1.0f, 2.0f, 4.0f), 191); - EXPECT_EQ(encodeRecovery(2.0f, 1.0f, 4.0f), 63); - - EXPECT_EQ(encodeRecovery(1.0f, 2.0f, 2.0f), 255); - EXPECT_EQ(encodeRecovery(2.0f, 1.0f, 2.0f), 0); - EXPECT_EQ(encodeRecovery(1.0f, 1.41421f, 2.0f), 191); - EXPECT_EQ(encodeRecovery(1.41421f, 1.0f, 2.0f), 63); - - EXPECT_EQ(encodeRecovery(1.0f, 8.0f, 8.0f), 255); - EXPECT_EQ(encodeRecovery(8.0f, 1.0f, 8.0f), 0); - EXPECT_EQ(encodeRecovery(1.0f, 2.82843f, 8.0f), 191); - EXPECT_EQ(encodeRecovery(2.82843f, 1.0f, 8.0f), 63); + jpegr_metadata metadata = { .maxContentBoost = 4.0f, + .minContentBoost = 1.0f / 4.0f }; + + EXPECT_EQ(encodeRecovery(0.0f, 0.0f, &metadata), 127); + EXPECT_EQ(encodeRecovery(0.0f, 1.0f, &metadata), 127); + EXPECT_EQ(encodeRecovery(1.0f, 0.0f, &metadata), 0); + EXPECT_EQ(encodeRecovery(0.5f, 0.0f, &metadata), 0); + + EXPECT_EQ(encodeRecovery(1.0f, 1.0f, &metadata), 127); + EXPECT_EQ(encodeRecovery(1.0f, 4.0f, &metadata), 255); + EXPECT_EQ(encodeRecovery(1.0f, 5.0f, &metadata), 255); + EXPECT_EQ(encodeRecovery(4.0f, 1.0f, &metadata), 0); + EXPECT_EQ(encodeRecovery(4.0f, 0.5f, &metadata), 0); + EXPECT_EQ(encodeRecovery(1.0f, 2.0f, &metadata), 191); + EXPECT_EQ(encodeRecovery(2.0f, 1.0f, &metadata), 63); + + metadata.maxContentBoost = 2.0f; + metadata.minContentBoost = 1.0f / 2.0f; + + EXPECT_EQ(encodeRecovery(1.0f, 2.0f, &metadata), 255); + EXPECT_EQ(encodeRecovery(2.0f, 1.0f, &metadata), 0); + EXPECT_EQ(encodeRecovery(1.0f, 1.41421f, &metadata), 191); + EXPECT_EQ(encodeRecovery(1.41421f, 1.0f, &metadata), 63); + + metadata.maxContentBoost = 8.0f; + metadata.minContentBoost = 1.0f / 8.0f; + + EXPECT_EQ(encodeRecovery(1.0f, 8.0f, &metadata), 255); + EXPECT_EQ(encodeRecovery(8.0f, 1.0f, &metadata), 0); + EXPECT_EQ(encodeRecovery(1.0f, 2.82843f, &metadata), 191); + EXPECT_EQ(encodeRecovery(2.82843f, 1.0f, &metadata), 63); + + metadata.maxContentBoost = 8.0f; + metadata.minContentBoost = 1.0f; + + EXPECT_EQ(encodeRecovery(0.0f, 0.0f, &metadata), 0); + EXPECT_EQ(encodeRecovery(1.0f, 0.0f, &metadata), 0); + + EXPECT_EQ(encodeRecovery(1.0f, 1.0f, &metadata), 0); + EXPECT_EQ(encodeRecovery(1.0f, 8.0f, &metadata), 255); + EXPECT_EQ(encodeRecovery(1.0f, 4.0f, &metadata), 170); + EXPECT_EQ(encodeRecovery(1.0f, 2.0f, &metadata), 85); + + metadata.maxContentBoost = 8.0f; + metadata.minContentBoost = 0.5f; + + EXPECT_EQ(encodeRecovery(0.0f, 0.0f, &metadata), 63); + EXPECT_EQ(encodeRecovery(1.0f, 0.0f, &metadata), 0); + + EXPECT_EQ(encodeRecovery(1.0f, 1.0f, &metadata), 63); + EXPECT_EQ(encodeRecovery(1.0f, 8.0f, &metadata), 255); + EXPECT_EQ(encodeRecovery(1.0f, 4.0f, &metadata), 191); + EXPECT_EQ(encodeRecovery(1.0f, 2.0f, &metadata), 127); + EXPECT_EQ(encodeRecovery(1.0f, 0.7071f, &metadata), 31); + EXPECT_EQ(encodeRecovery(1.0f, 0.5f, &metadata), 0); } TEST_F(RecoveryMapMathTest, ApplyRecovery) { - EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), -1.0f, 4.0f), RgbBlack()); - EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), 0.0f, 4.0f), RgbBlack()); - EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), 1.0f, 4.0f), RgbBlack()); - - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), -1.0f, 4.0f), RgbWhite() / 4.0f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), -0.5f, 4.0f), RgbWhite() / 2.0f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.0f, 4.0f), RgbWhite()); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.5f, 4.0f), RgbWhite() * 2.0f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f, 4.0f), RgbWhite() * 4.0f); - - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), -1.0f, 2.0f), RgbWhite() / 2.0f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), -0.5f, 2.0f), RgbWhite() / 1.41421f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.0f, 2.0f), RgbWhite()); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.5f, 2.0f), RgbWhite() * 1.41421f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f, 2.0f), RgbWhite() * 2.0f); - - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), -1.0f, 8.0f), RgbWhite() / 8.0f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), -0.5f, 8.0f), RgbWhite() / 2.82843f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.0f, 8.0f), RgbWhite()); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.5f, 8.0f), RgbWhite() * 2.82843f); - EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f, 8.0f), RgbWhite() * 8.0f); + jpegr_metadata metadata = { .maxContentBoost = 4.0f, + .minContentBoost = 1.0f / 4.0f }; + + EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), 0.0f, &metadata), RgbBlack()); + EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), 0.5f, &metadata), RgbBlack()); + EXPECT_RGB_NEAR(applyRecovery(RgbBlack(), 1.0f, &metadata), RgbBlack()); + + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.0f, &metadata), RgbWhite() / 4.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.25f, &metadata), RgbWhite() / 2.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.5f, &metadata), RgbWhite()); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.75f, &metadata), RgbWhite() * 2.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f, &metadata), RgbWhite() * 4.0f); + + metadata.maxContentBoost = 2.0f; + metadata.minContentBoost = 1.0f / 2.0f; + + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.0f, &metadata), RgbWhite() / 2.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.25f, &metadata), RgbWhite() / 1.41421f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.5f, &metadata), RgbWhite()); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.75f, &metadata), RgbWhite() * 1.41421f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f, &metadata), RgbWhite() * 2.0f); + + metadata.maxContentBoost = 8.0f; + metadata.minContentBoost = 1.0f / 8.0f; + + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.0f, &metadata), RgbWhite() / 8.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.25f, &metadata), RgbWhite() / 2.82843f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.5f, &metadata), RgbWhite()); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.75f, &metadata), RgbWhite() * 2.82843f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f, &metadata), RgbWhite() * 8.0f); + + metadata.maxContentBoost = 8.0f; + metadata.minContentBoost = 1.0f; + + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.0f, &metadata), RgbWhite()); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f / 3.0f, &metadata), RgbWhite() * 2.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 2.0f / 3.0f, &metadata), RgbWhite() * 4.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f, &metadata), RgbWhite() * 8.0f); + + metadata.maxContentBoost = 8.0f; + metadata.minContentBoost = 0.5f; + + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.0f, &metadata), RgbWhite() / 2.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.25f, &metadata), RgbWhite()); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.5f, &metadata), RgbWhite() * 2.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 0.75f, &metadata), RgbWhite() * 4.0f); + EXPECT_RGB_NEAR(applyRecovery(RgbWhite(), 1.0f, &metadata), RgbWhite() * 8.0f); Color e = {{{ 0.0f, 0.5f, 1.0f }}}; - - EXPECT_RGB_NEAR(applyRecovery(e, -1.0f, 4.0f), e / 4.0f); - EXPECT_RGB_NEAR(applyRecovery(e, -0.5f, 4.0f), e / 2.0f); - EXPECT_RGB_NEAR(applyRecovery(e, 0.0f, 4.0f), e); - EXPECT_RGB_NEAR(applyRecovery(e, 0.5f, 4.0f), e * 2.0f); - EXPECT_RGB_NEAR(applyRecovery(e, 1.0f, 4.0f), e * 4.0f); + metadata.maxContentBoost = 4.0f; + metadata.minContentBoost = 1.0f / 4.0f; + + EXPECT_RGB_NEAR(applyRecovery(e, 0.0f, &metadata), e / 4.0f); + EXPECT_RGB_NEAR(applyRecovery(e, 0.25f, &metadata), e / 2.0f); + EXPECT_RGB_NEAR(applyRecovery(e, 0.5f, &metadata), e); + EXPECT_RGB_NEAR(applyRecovery(e, 0.75f, &metadata), e * 2.0f); + EXPECT_RGB_NEAR(applyRecovery(e, 1.0f, &metadata), e * 4.0f); } TEST_F(RecoveryMapMathTest, GetYuv420Pixel) { @@ -785,8 +882,10 @@ TEST_F(RecoveryMapMathTest, SampleMap) { // Instead of reimplementing the sampling algorithm, confirm that the // sample output is within the range of the min and max of the nearest // points. - EXPECT_THAT(sampleMap(&image, kMapScaleFactor, x, y, idwTable), + EXPECT_THAT(sampleMap(&image, kMapScaleFactor, x, y), testing::AllOf(testing::Ge(min), testing::Le(max))); + EXPECT_EQ(sampleMap(&image, kMapScaleFactor, x, y, idwTable), + sampleMap(&image, kMapScaleFactor, x, y)); } } } @@ -882,60 +981,89 @@ TEST_F(RecoveryMapMathTest, GenerateMapLuminancePq) { } TEST_F(RecoveryMapMathTest, ApplyMap) { - EXPECT_RGB_EQ(Recover(YuvWhite(), 1.0f, 8.0f), + jpegr_metadata metadata = { .maxContentBoost = 8.0f, + .minContentBoost = 1.0f / 8.0f }; + + EXPECT_RGB_EQ(Recover(YuvWhite(), 1.0f, &metadata), RgbWhite() * 8.0f); - EXPECT_RGB_EQ(Recover(YuvBlack(), 1.0f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvBlack(), 1.0f, &metadata), RgbBlack()); - EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), 1.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), 1.0f, &metadata), RgbRed() * 8.0f); - EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), 1.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), 1.0f, &metadata), RgbGreen() * 8.0f); - EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), 1.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), 1.0f, &metadata), RgbBlue() * 8.0f); - EXPECT_RGB_EQ(Recover(YuvWhite(), 0.5f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.75f, &metadata), RgbWhite() * sqrt(8.0f)); - EXPECT_RGB_EQ(Recover(YuvBlack(), 0.5f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvBlack(), 0.75f, &metadata), RgbBlack()); - EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), 0.5f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), 0.75f, &metadata), RgbRed() * sqrt(8.0f)); - EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), 0.5f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), 0.75f, &metadata), RgbGreen() * sqrt(8.0f)); - EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), 0.5f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), 0.75f, &metadata), RgbBlue() * sqrt(8.0f)); - EXPECT_RGB_EQ(Recover(YuvWhite(), 0.0f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.5f, &metadata), RgbWhite()); - EXPECT_RGB_EQ(Recover(YuvBlack(), 0.0f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvBlack(), 0.5f, &metadata), RgbBlack()); - EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), 0.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), 0.5f, &metadata), RgbRed()); - EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), 0.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), 0.5f, &metadata), RgbGreen()); - EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), 0.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), 0.5f, &metadata), RgbBlue()); - EXPECT_RGB_EQ(Recover(YuvWhite(), -0.5f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.25f, &metadata), RgbWhite() / sqrt(8.0f)); - EXPECT_RGB_EQ(Recover(YuvBlack(), -0.5f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvBlack(), 0.25f, &metadata), RgbBlack()); - EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), -0.5f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), 0.25f, &metadata), RgbRed() / sqrt(8.0f)); - EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), -0.5f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), 0.25f, &metadata), RgbGreen() / sqrt(8.0f)); - EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), -0.5f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), 0.25f, &metadata), RgbBlue() / sqrt(8.0f)); - EXPECT_RGB_EQ(Recover(YuvWhite(), -1.0f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.0f, &metadata), RgbWhite() / 8.0f); - EXPECT_RGB_EQ(Recover(YuvBlack(), -1.0f, 8.0f), + EXPECT_RGB_EQ(Recover(YuvBlack(), 0.0f, &metadata), RgbBlack()); - EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), -1.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvRed(), 0.0f, &metadata), RgbRed() / 8.0f); - EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), -1.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvGreen(), 0.0f, &metadata), RgbGreen() / 8.0f); - EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), -1.0f, 8.0f), + EXPECT_RGB_CLOSE(Recover(SrgbYuvBlue(), 0.0f, &metadata), RgbBlue() / 8.0f); + + metadata.maxContentBoost = 8.0f; + metadata.minContentBoost = 1.0f; + + EXPECT_RGB_EQ(Recover(YuvWhite(), 1.0f, &metadata), + RgbWhite() * 8.0f); + EXPECT_RGB_EQ(Recover(YuvWhite(), 2.0f / 3.0f, &metadata), + RgbWhite() * 4.0f); + EXPECT_RGB_EQ(Recover(YuvWhite(), 1.0f / 3.0f, &metadata), + RgbWhite() * 2.0f); + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.0f, &metadata), + RgbWhite()); + + metadata.maxContentBoost = 8.0f; + metadata.minContentBoost = 0.5f;; + + EXPECT_RGB_EQ(Recover(YuvWhite(), 1.0f, &metadata), + RgbWhite() * 8.0f); + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.75, &metadata), + RgbWhite() * 4.0f); + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.5f, &metadata), + RgbWhite() * 2.0f); + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.25f, &metadata), + RgbWhite()); + EXPECT_RGB_EQ(Recover(YuvWhite(), 0.0f, &metadata), + RgbWhite() / 2.0f); } -} // namespace android::recoverymap +} // namespace android::jpegrecoverymap diff --git a/services/inputflinger/dispatcher/CancelationOptions.h b/services/inputflinger/dispatcher/CancelationOptions.h index 48f9f2b78a..83e6a60602 100644 --- a/services/inputflinger/dispatcher/CancelationOptions.h +++ b/services/inputflinger/dispatcher/CancelationOptions.h @@ -30,6 +30,7 @@ struct CancelationOptions { CANCEL_POINTER_EVENTS = 1, CANCEL_NON_POINTER_EVENTS = 2, CANCEL_FALLBACK_EVENTS = 3, + ftl_last = CANCEL_FALLBACK_EVENTS, }; // The criterion to use to determine which events should be canceled. diff --git a/services/inputflinger/dispatcher/InputDispatcher.cpp b/services/inputflinger/dispatcher/InputDispatcher.cpp index d14a781473..9d5bbbdd11 100644 --- a/services/inputflinger/dispatcher/InputDispatcher.cpp +++ b/services/inputflinger/dispatcher/InputDispatcher.cpp @@ -2185,8 +2185,9 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( const bool isFromMouse = isFromSource(entry.source, AINPUT_SOURCE_MOUSE); if (newGesture) { - bool down = maskedAction == AMOTION_EVENT_ACTION_DOWN; - if (switchedDevice && tempTouchState.isDown() && !down && !isHoverAction) { + // If pointers are already down, let's finish the current gesture and ignore the new events + // from another device. + if (switchedDevice && wasDown) { ALOGI("Dropping event because a pointer for a different device is already down " "in display %" PRId32, displayId); @@ -3761,9 +3762,9 @@ void InputDispatcher::synthesizeCancelationEventsForConnectionLocked( } if (DEBUG_OUTBOUND_EVENT_DETAILS) { ALOGD("channel '%s' ~ Synthesized %zu cancelation events to bring channel back in sync " - "with reality: %s, mode=%d.", + "with reality: %s, mode=%s.", connection->getInputChannelName().c_str(), cancelationEvents.size(), options.reason, - options.mode); + ftl::enum_string(options.mode).c_str()); } std::string reason = std::string("reason=").append(options.reason); diff --git a/services/inputflinger/tests/InputDispatcher_test.cpp b/services/inputflinger/tests/InputDispatcher_test.cpp index 3605be292b..e71cdce498 100644 --- a/services/inputflinger/tests/InputDispatcher_test.cpp +++ b/services/inputflinger/tests/InputDispatcher_test.cpp @@ -2461,6 +2461,86 @@ TEST_F(InputDispatcherTest, HoverTapAndSplitTouch) { } /** + * Start hovering with a stylus device, and then tap with a touch device. Ensure no crash occurs. + * While the touch is down, new hover events from the stylus device should be ignored. After the + * touch is gone, stylus hovering should start working again. + */ +TEST_F(InputDispatcherTest, StylusHoverAndTouchTap) { + std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); + sp<FakeWindowHandle> window = + sp<FakeWindowHandle>::make(application, mDispatcher, "Window", ADISPLAY_ID_DEFAULT); + window->setFrame(Rect(0, 0, 200, 200)); + + mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {window}}}); + + const int32_t stylusDeviceId = 5; + const int32_t touchDeviceId = 4; + // Start hovering with stylus + ASSERT_EQ(InputEventInjectionResult::SUCCEEDED, + injectMotionEvent(mDispatcher, + MotionEventBuilder(AMOTION_EVENT_ACTION_HOVER_ENTER, + AINPUT_SOURCE_STYLUS) + .deviceId(stylusDeviceId) + .pointer(PointerBuilder(0, AMOTION_EVENT_TOOL_TYPE_STYLUS) + .x(50) + .y(50)) + .build())); + window->consumeMotionEvent(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_ENTER)); + + // Finger down on the window + ASSERT_EQ(InputEventInjectionResult::SUCCEEDED, + injectMotionEvent(mDispatcher, + MotionEventBuilder(AMOTION_EVENT_ACTION_DOWN, + AINPUT_SOURCE_TOUCHSCREEN) + .deviceId(touchDeviceId) + .pointer(PointerBuilder(0, AMOTION_EVENT_TOOL_TYPE_FINGER) + .x(100) + .y(100)) + .build())); + window->consumeMotionEvent(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_EXIT)); + window->consumeMotionEvent(WithMotionAction(AMOTION_EVENT_ACTION_DOWN)); + + // Try to continue hovering with stylus. Since we are already down, injection should fail + ASSERT_EQ(InputEventInjectionResult::FAILED, + injectMotionEvent(mDispatcher, + MotionEventBuilder(AMOTION_EVENT_ACTION_HOVER_MOVE, + AINPUT_SOURCE_STYLUS) + .deviceId(stylusDeviceId) + .pointer(PointerBuilder(0, AMOTION_EVENT_TOOL_TYPE_STYLUS) + .x(50) + .y(50)) + .build())); + // No event should be sent. This event should be ignored because a pointer from another device + // is already down. + + // Lift up the finger + ASSERT_EQ(InputEventInjectionResult::SUCCEEDED, + injectMotionEvent(mDispatcher, + MotionEventBuilder(AMOTION_EVENT_ACTION_UP, + AINPUT_SOURCE_TOUCHSCREEN) + .deviceId(touchDeviceId) + .pointer(PointerBuilder(0, AMOTION_EVENT_TOOL_TYPE_FINGER) + .x(100) + .y(100)) + .build())); + window->consumeMotionEvent(WithMotionAction(AMOTION_EVENT_ACTION_UP)); + + // Now that the touch is gone, stylus hovering should start working again + ASSERT_EQ(InputEventInjectionResult::SUCCEEDED, + injectMotionEvent(mDispatcher, + MotionEventBuilder(AMOTION_EVENT_ACTION_HOVER_MOVE, + AINPUT_SOURCE_STYLUS) + .deviceId(stylusDeviceId) + .pointer(PointerBuilder(0, AMOTION_EVENT_TOOL_TYPE_STYLUS) + .x(50) + .y(50)) + .build())); + window->consumeMotionEvent(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_ENTER)); + // No more events + window->assertNoEvents(); +} + +/** * On the display, have a single window, and also an area where there's no window. * First pointer touches the "no window" area of the screen. Second pointer touches the window. * Make sure that the window receives the second pointer, and first pointer is simply ignored. diff --git a/services/sensorservice/SensorInterface.cpp b/services/sensorservice/SensorInterface.cpp index 398cdf9a0e..e9c8335963 100644 --- a/services/sensorservice/SensorInterface.cpp +++ b/services/sensorservice/SensorInterface.cpp @@ -87,14 +87,15 @@ VirtualSensor::VirtualSensor() : // --------------------------------------------------------------------------- -RuntimeSensor::RuntimeSensor(const sensor_t& sensor, sp<StateChangeCallback> callback) +RuntimeSensor::RuntimeSensor(const sensor_t& sensor, sp<SensorCallback> callback) : BaseSensor(sensor), mCallback(std::move(callback)) { } status_t RuntimeSensor::activate(void*, bool enabled) { if (enabled != mEnabled) { mEnabled = enabled; - mCallback->onStateChanged(mEnabled, mSamplingPeriodNs, mBatchReportLatencyNs); + return mCallback->onConfigurationChanged(mSensor.getHandle(), mEnabled, mSamplingPeriodNs, + mBatchReportLatencyNs); } return OK; } @@ -105,7 +106,8 @@ status_t RuntimeSensor::batch(void*, int, int, int64_t samplingPeriodNs, mSamplingPeriodNs = samplingPeriodNs; mBatchReportLatencyNs = maxBatchReportLatencyNs; if (mEnabled) { - mCallback->onStateChanged(mEnabled, mSamplingPeriodNs, mBatchReportLatencyNs); + return mCallback->onConfigurationChanged(mSensor.getHandle(), mEnabled, + mSamplingPeriodNs, mBatchReportLatencyNs); } } return OK; @@ -115,7 +117,8 @@ status_t RuntimeSensor::setDelay(void*, int, int64_t ns) { if (mSamplingPeriodNs != ns) { mSamplingPeriodNs = ns; if (mEnabled) { - mCallback->onStateChanged(mEnabled, mSamplingPeriodNs, mBatchReportLatencyNs); + return mCallback->onConfigurationChanged(mSensor.getHandle(), mEnabled, + mSamplingPeriodNs, mBatchReportLatencyNs); } } return OK; diff --git a/services/sensorservice/SensorInterface.h b/services/sensorservice/SensorInterface.h index 5ee5e1224a..c446d61f89 100644 --- a/services/sensorservice/SensorInterface.h +++ b/services/sensorservice/SensorInterface.h @@ -108,12 +108,12 @@ class RuntimeSensor : public BaseSensor { public: static constexpr int DEFAULT_DEVICE_ID = 0; - class StateChangeCallback : public virtual RefBase { + class SensorCallback : public virtual RefBase { public: - virtual void onStateChanged(bool enabled, int64_t samplingPeriodNs, - int64_t batchReportLatencyNs) = 0; + virtual status_t onConfigurationChanged(int handle, bool enabled, int64_t samplingPeriodNs, + int64_t batchReportLatencyNs) = 0; }; - RuntimeSensor(const sensor_t& sensor, sp<StateChangeCallback> callback); + RuntimeSensor(const sensor_t& sensor, sp<SensorCallback> callback); virtual status_t activate(void* ident, bool enabled) override; virtual status_t batch(void* ident, int handle, int flags, int64_t samplingPeriodNs, int64_t maxBatchReportLatencyNs) override; @@ -125,7 +125,7 @@ private: bool mEnabled = false; int64_t mSamplingPeriodNs = 0; int64_t mBatchReportLatencyNs = 0; - sp<StateChangeCallback> mCallback; + sp<SensorCallback> mCallback; }; // --------------------------------------------------------------------------- diff --git a/services/sensorservice/SensorService.cpp b/services/sensorservice/SensorService.cpp index 5c98614f1a..3a0329cd09 100644 --- a/services/sensorservice/SensorService.cpp +++ b/services/sensorservice/SensorService.cpp @@ -116,16 +116,17 @@ int32_t nextRuntimeSensorHandle() { return nextHandle++; } -class RuntimeSensorCallbackProxy : public RuntimeSensor::StateChangeCallback { +class RuntimeSensorCallbackProxy : public RuntimeSensor::SensorCallback { public: - RuntimeSensorCallbackProxy(sp<SensorService::RuntimeSensorStateChangeCallback> callback) + RuntimeSensorCallbackProxy(sp<SensorService::RuntimeSensorCallback> callback) : mCallback(std::move(callback)) {} - void onStateChanged(bool enabled, int64_t samplingPeriodNs, - int64_t batchReportLatencyNs) override { - mCallback->onStateChanged(enabled, samplingPeriodNs, batchReportLatencyNs); + status_t onConfigurationChanged(int handle, bool enabled, int64_t samplingPeriodNs, + int64_t batchReportLatencyNs) override { + return mCallback->onConfigurationChanged(handle, enabled, samplingPeriodNs, + batchReportLatencyNs); } private: - sp<SensorService::RuntimeSensorStateChangeCallback> mCallback; + sp<SensorService::RuntimeSensorCallback> mCallback; }; } // namespace @@ -166,7 +167,7 @@ SensorService::SensorService() } int SensorService::registerRuntimeSensor( - const sensor_t& sensor, int deviceId, sp<RuntimeSensorStateChangeCallback> callback) { + const sensor_t& sensor, int deviceId, sp<RuntimeSensorCallback> callback) { int handle = 0; while (handle == 0 || !mSensors.isNewHandle(handle)) { handle = nextRuntimeSensorHandle(); @@ -179,7 +180,7 @@ int SensorService::registerRuntimeSensor( ALOGI("Registering runtime sensor handle 0x%x, type %d, name %s", handle, sensor.type, sensor.name); - sp<RuntimeSensor::StateChangeCallback> runtimeSensorCallback( + sp<RuntimeSensor::SensorCallback> runtimeSensorCallback( new RuntimeSensorCallbackProxy(std::move(callback))); sensor_t runtimeSensor = sensor; // force the handle to be consistent diff --git a/services/sensorservice/SensorService.h b/services/sensorservice/SensorService.h index 0798279b6b..3f6a895b52 100644 --- a/services/sensorservice/SensorService.h +++ b/services/sensorservice/SensorService.h @@ -147,12 +147,13 @@ public: virtual void onProximityActive(bool isActive) = 0; }; - class RuntimeSensorStateChangeCallback : public virtual RefBase { + class RuntimeSensorCallback : public virtual RefBase { public: // Note that the callback is invoked from an async thread and can interact with the // SensorService directly. - virtual void onStateChanged(bool enabled, int64_t samplingPeriodNanos, - int64_t batchReportLatencyNanos) = 0; + virtual status_t onConfigurationChanged(int handle, bool enabled, + int64_t samplingPeriodNanos, + int64_t batchReportLatencyNanos) = 0; }; static char const* getServiceName() ANDROID_API { return "sensorservice"; } @@ -182,7 +183,7 @@ public: status_t removeProximityActiveListener(const sp<ProximityActiveListener>& callback) ANDROID_API; int registerRuntimeSensor(const sensor_t& sensor, int deviceId, - sp<RuntimeSensorStateChangeCallback> callback) ANDROID_API; + sp<RuntimeSensorCallback> callback) ANDROID_API; status_t unregisterRuntimeSensor(int handle) ANDROID_API; status_t sendRuntimeSensorEvent(const sensors_event_t& event) ANDROID_API; diff --git a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp index 6b69ce7941..1b86cd376a 100644 --- a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp +++ b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp @@ -585,6 +585,7 @@ void OutputLayer::writeOutputIndependentPerFrameStateToHWC( case Composition::CURSOR: case Composition::DEVICE: case Composition::DISPLAY_DECORATION: + case Composition::REFRESH_RATE_INDICATOR: writeBufferStateToHWC(hwcLayer, outputIndependentState, skipLayer); break; case Composition::INVALID: @@ -780,6 +781,7 @@ void OutputLayer::detectDisallowedCompositionTypeChange(Composition from, Compos case Composition::CURSOR: case Composition::SIDEBAND: case Composition::DISPLAY_DECORATION: + case Composition::REFRESH_RATE_INDICATOR: result = (to == Composition::CLIENT || to == Composition::DEVICE); break; } diff --git a/services/surfaceflinger/CompositionEngine/src/planner/Predictor.cpp b/services/surfaceflinger/CompositionEngine/src/planner/Predictor.cpp index 2fc029fdc6..6064126099 100644 --- a/services/surfaceflinger/CompositionEngine/src/planner/Predictor.cpp +++ b/services/surfaceflinger/CompositionEngine/src/planner/Predictor.cpp @@ -151,6 +151,10 @@ std::string to_string(const Plan& plan) { // A for "Alpha", since the decoration is an alpha layer. result.append("A"); break; + case aidl::android::hardware::graphics::composer3::Composition::REFRESH_RATE_INDICATOR: + // R for "Refresh", since the layer is Refresh rate overlay. + result.append("R"); + break; } } return result; diff --git a/services/surfaceflinger/DisplayHardware/AidlComposerHal.cpp b/services/surfaceflinger/DisplayHardware/AidlComposerHal.cpp index 9470552e41..ba9aed8786 100644 --- a/services/surfaceflinger/DisplayHardware/AidlComposerHal.cpp +++ b/services/surfaceflinger/DisplayHardware/AidlComposerHal.cpp @@ -208,6 +208,12 @@ public: return ::ndk::ScopedAStatus::ok(); } + ::ndk::ScopedAStatus onRefreshRateChangedDebug( + const RefreshRateChangedDebugData& refreshRateChangedDebugData) override { + mCallback.onRefreshRateChangedDebug(refreshRateChangedDebugData); + return ::ndk::ScopedAStatus::ok(); + } + private: HWC2::ComposerCallback& mCallback; }; diff --git a/services/surfaceflinger/DisplayHardware/HWC2.h b/services/surfaceflinger/DisplayHardware/HWC2.h index c1c7070a2d..23dd3e5016 100644 --- a/services/surfaceflinger/DisplayHardware/HWC2.h +++ b/services/surfaceflinger/DisplayHardware/HWC2.h @@ -44,6 +44,7 @@ #include <aidl/android/hardware/graphics/composer3/Composition.h> #include <aidl/android/hardware/graphics/composer3/DisplayCapability.h> #include <aidl/android/hardware/graphics/composer3/OverlayProperties.h> +#include <aidl/android/hardware/graphics/composer3/RefreshRateChangedDebugData.h> namespace android { @@ -63,6 +64,8 @@ class Layer; namespace hal = android::hardware::graphics::composer::hal; +using aidl::android::hardware::graphics::composer3::RefreshRateChangedDebugData; + // Implement this interface to receive hardware composer events. // // These callback functions will generally be called on a hwbinder thread, but @@ -77,6 +80,7 @@ struct ComposerCallback { const hal::VsyncPeriodChangeTimeline&) = 0; virtual void onComposerHalSeamlessPossible(hal::HWDisplayId) = 0; virtual void onComposerHalVsyncIdle(hal::HWDisplayId) = 0; + virtual void onRefreshRateChangedDebug(const RefreshRateChangedDebugData&) = 0; protected: ~ComposerCallback() = default; diff --git a/services/surfaceflinger/DisplayHardware/Hal.h b/services/surfaceflinger/DisplayHardware/Hal.h index 537d5450ad..bf3089f040 100644 --- a/services/surfaceflinger/DisplayHardware/Hal.h +++ b/services/surfaceflinger/DisplayHardware/Hal.h @@ -113,6 +113,8 @@ inline std::string to_string( return "Sideband"; case aidl::android::hardware::graphics::composer3::Composition::DISPLAY_DECORATION: return "DisplayDecoration"; + case aidl::android::hardware::graphics::composer3::Composition::REFRESH_RATE_INDICATOR: + return "RefreshRateIndicator"; default: return "Unknown"; } diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp index 6490476396..3ed24b2740 100644 --- a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp +++ b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp @@ -391,7 +391,9 @@ bool LayerSnapshotBuilder::tryFastUpdate(const Args& args) { void LayerSnapshotBuilder::updateSnapshots(const Args& args) { ATRACE_NAME("UpdateSnapshots"); - if (args.forceUpdate || args.displayChanges) { + if (args.parentCrop) { + mRootSnapshot.geomLayerBounds = *args.parentCrop; + } else if (args.forceUpdate || args.displayChanges) { mRootSnapshot.geomLayerBounds = getMaxDisplayBounds(args.displays); } if (args.displayChanges) { @@ -618,7 +620,8 @@ void LayerSnapshotBuilder::updateSnapshot(LayerSnapshot& snapshot, const Args& a RequestedLayerState::Changes::AffectsChildren); snapshot.changes = parentChanges | requested.changes; snapshot.isHiddenByPolicyFromParent = parentSnapshot.isHiddenByPolicyFromParent || - parentSnapshot.invalidTransform || requested.isHiddenByPolicy(); + parentSnapshot.invalidTransform || requested.isHiddenByPolicy() || + (args.excludeLayerIds.find(path.id) != args.excludeLayerIds.end()); snapshot.contentDirty = requested.what & layer_state_t::CONTENT_DIRTY; // TODO(b/238781169) scope down the changes to only buffer updates. snapshot.hasReadyFrame = @@ -983,6 +986,20 @@ void LayerSnapshotBuilder::forEachVisibleSnapshot(const ConstVisitor& visitor) c } } +// Visit each visible snapshot in z-order +void LayerSnapshotBuilder::forEachVisibleSnapshot(const ConstVisitor& visitor, + const LayerHierarchy& root) const { + root.traverseInZOrder( + [this, visitor](const LayerHierarchy&, + const LayerHierarchy::TraversalPath& traversalPath) -> bool { + LayerSnapshot* snapshot = getSnapshot(traversalPath); + if (snapshot && snapshot->isVisible) { + visitor(*snapshot); + } + return true; + }); +} + void LayerSnapshotBuilder::forEachVisibleSnapshot(const Visitor& visitor) { for (int i = 0; i < mNumInterestingSnapshots; i++) { std::unique_ptr<LayerSnapshot>& snapshot = mSnapshots.at((size_t)i); diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.h b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.h index abb7e668c3..f4544fd62f 100644 --- a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.h +++ b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.h @@ -36,7 +36,7 @@ namespace android::surfaceflinger::frontend { class LayerSnapshotBuilder { public: struct Args { - const LayerHierarchy& root; + LayerHierarchy root; const LayerLifecycleManager& layerLifecycleManager; bool forceUpdate = false; bool includeMetadata = false; @@ -46,6 +46,8 @@ public: const renderengine::ShadowSettings& globalShadowSettings; bool supportsBlur = true; bool forceFullDamage = false; + std::optional<FloatRect> parentCrop = std::nullopt; + std::unordered_set<uint32_t> excludeLayerIds; }; LayerSnapshotBuilder(); @@ -65,6 +67,9 @@ public: // Visit each visible snapshot in z-order void forEachVisibleSnapshot(const ConstVisitor& visitor) const; + // Visit each visible snapshot in z-order + void forEachVisibleSnapshot(const ConstVisitor& visitor, const LayerHierarchy& root) const; + typedef std::function<void(std::unique_ptr<LayerSnapshot>& snapshot)> Visitor; // Visit each visible snapshot in z-order and move the snapshot if needed void forEachVisibleSnapshot(const Visitor& visitor); diff --git a/services/surfaceflinger/FrontEnd/TransactionHandler.h b/services/surfaceflinger/FrontEnd/TransactionHandler.h index a06b870549..7fc825eba3 100644 --- a/services/surfaceflinger/FrontEnd/TransactionHandler.h +++ b/services/surfaceflinger/FrontEnd/TransactionHandler.h @@ -34,7 +34,7 @@ namespace surfaceflinger::frontend { class TransactionHandler { public: struct TransactionFlushState { - const TransactionState* transaction; + TransactionState* transaction; bool firstTransaction = true; nsecs_t queueProcessTime = 0; // Layer handles that have transactions with buffers that are ready to be applied. diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp index f2a8d44c10..704f336544 100644 --- a/services/surfaceflinger/Layer.cpp +++ b/services/surfaceflinger/Layer.cpp @@ -146,7 +146,7 @@ Layer::Layer(const LayerCreationArgs& args) mLayerCreationFlags(args.flags), mBorderEnabled(false), mTextureName(args.textureName), - mLayerFE(args.flinger->getFactory().createLayerFE(mName)) { + mLegacyLayerFE(args.flinger->getFactory().createLayerFE(mName)) { ALOGV("Creating Layer %s", getDebugName()); uint32_t layerFlags = 0; @@ -228,9 +228,7 @@ Layer::~Layer() { if (mBufferInfo.mBuffer != nullptr) { callReleaseBufferCallback(mDrawingState.releaseBufferListener, mBufferInfo.mBuffer->getBuffer(), mBufferInfo.mFrameNumber, - mBufferInfo.mFence, - mFlinger->getMaxAcquiredBufferCountForCurrentRefreshRate( - mOwnerUid)); + mBufferInfo.mFence); } if (!isClone()) { // The original layer and the clone layer share the same texture. Therefore, only one of @@ -2733,12 +2731,13 @@ void Layer::cloneDrawingState(const Layer* from) { void Layer::callReleaseBufferCallback(const sp<ITransactionCompletedListener>& listener, const sp<GraphicBuffer>& buffer, uint64_t framenumber, - const sp<Fence>& releaseFence, - uint32_t currentMaxAcquiredBufferCount) { + const sp<Fence>& releaseFence) { if (!listener) { return; } ATRACE_FORMAT_INSTANT("callReleaseBufferCallback %s - %" PRIu64, getDebugName(), framenumber); + uint32_t currentMaxAcquiredBufferCount = + mFlinger->getMaxAcquiredBufferCountForCurrentRefreshRate(mOwnerUid); listener->onReleaseBuffer({buffer->getId(), framenumber}, releaseFence ? releaseFence : Fence::NO_FENCE, currentMaxAcquiredBufferCount); @@ -2989,9 +2988,7 @@ bool Layer::setBuffer(std::shared_ptr<renderengine::ExternalTexture>& buffer, // call any release buffer callbacks if set. callReleaseBufferCallback(mDrawingState.releaseBufferListener, mDrawingState.buffer->getBuffer(), mDrawingState.frameNumber, - mDrawingState.acquireFence, - mFlinger->getMaxAcquiredBufferCountForCurrentRefreshRate( - mOwnerUid)); + mDrawingState.acquireFence); decrementPendingBufferCount(); if (mDrawingState.bufferSurfaceFrameTX != nullptr && mDrawingState.bufferSurfaceFrameTX->getPresentState() != PresentState::Presented) { @@ -3001,13 +2998,12 @@ bool Layer::setBuffer(std::shared_ptr<renderengine::ExternalTexture>& buffer, } else if (EARLY_RELEASE_ENABLED && mLastClientCompositionFence != nullptr) { callReleaseBufferCallback(mDrawingState.releaseBufferListener, mDrawingState.buffer->getBuffer(), mDrawingState.frameNumber, - mLastClientCompositionFence, - mFlinger->getMaxAcquiredBufferCountForCurrentRefreshRate( - mOwnerUid)); + mLastClientCompositionFence); mLastClientCompositionFence = nullptr; } } + mDrawingState.producerId = bufferData.producerId; mDrawingState.frameNumber = frameNumber; mDrawingState.releaseBufferListener = bufferData.releaseBufferListener; mDrawingState.buffer = std::move(buffer); @@ -3124,15 +3120,14 @@ bool Layer::setSidebandStream(const sp<NativeHandle>& sidebandStream) { return true; } -bool Layer::setTransactionCompletedListeners(const std::vector<sp<CallbackHandle>>& handles) { +bool Layer::setTransactionCompletedListeners(const std::vector<sp<CallbackHandle>>& handles, + bool willPresent) { // If there is no handle, we will not send a callback so reset mReleasePreviousBuffer and return if (handles.empty()) { mReleasePreviousBuffer = false; return false; } - const bool willPresent = willPresentCurrentTransaction(); - std::deque<sp<CallbackHandle>> remainingHandles; for (const auto& handle : handles) { // If this transaction set a buffer on this layer, release its previous buffer @@ -3215,11 +3210,10 @@ bool Layer::fenceHasSignaled() const { return fenceSignaled; } -bool Layer::onPreComposition(nsecs_t refreshStartTime) { +void Layer::onPreComposition(nsecs_t refreshStartTime) { for (const auto& handle : mDrawingState.callbackHandles) { handle->refreshStartTime = refreshStartTime; } - return hasReadyFrame(); } void Layer::setAutoRefresh(bool autoRefresh) { @@ -3605,7 +3599,7 @@ bool Layer::isHdrY410() const { sp<LayerFE> Layer::getCompositionEngineLayerFE() const { // There's no need to get a CE Layer if the layer isn't going to draw anything. - return hasSomethingToDraw() ? mLayerFE : nullptr; + return hasSomethingToDraw() ? mLegacyLayerFE : nullptr; } const LayerSnapshot* Layer::getLayerSnapshot() const { @@ -3616,16 +3610,36 @@ LayerSnapshot* Layer::editLayerSnapshot() { return mSnapshot.get(); } +std::unique_ptr<frontend::LayerSnapshot> Layer::stealLayerSnapshot() { + return std::move(mSnapshot); +} + +void Layer::updateLayerSnapshot(std::unique_ptr<frontend::LayerSnapshot> snapshot) { + mSnapshot = std::move(snapshot); +} + const compositionengine::LayerFECompositionState* Layer::getCompositionState() const { return mSnapshot.get(); } sp<LayerFE> Layer::copyCompositionEngineLayerFE() const { - auto result = mFlinger->getFactory().createLayerFE(mLayerFE->getDebugName()); + auto result = mFlinger->getFactory().createLayerFE(mName); result->mSnapshot = std::make_unique<LayerSnapshot>(*mSnapshot); return result; } +sp<LayerFE> Layer::getCompositionEngineLayerFE( + const frontend::LayerHierarchy::TraversalPath& path) { + for (auto& [p, layerFE] : mLayerFEs) { + if (p == path) { + return layerFE; + } + } + auto layerFE = mFlinger->getFactory().createLayerFE(mName); + mLayerFEs.emplace_back(path, layerFE); + return layerFE; +} + void Layer::useSurfaceDamage() { if (mFlinger->mForceFullDamage) { surfaceDamageRegion = Region::INVALID_REGION; @@ -4021,28 +4035,6 @@ void Layer::updateRelativeMetadataSnapshot(const LayerMetadata& relativeLayerMet } } -LayerSnapshotGuard::LayerSnapshotGuard(Layer* layer) : mLayer(layer) { - if (mLayer) { - mLayer->mLayerFE->mSnapshot = std::move(mLayer->mSnapshot); - } -} - -LayerSnapshotGuard::~LayerSnapshotGuard() { - if (mLayer) { - mLayer->mSnapshot = std::move(mLayer->mLayerFE->mSnapshot); - } -} - -LayerSnapshotGuard::LayerSnapshotGuard(LayerSnapshotGuard&& other) : mLayer(other.mLayer) { - other.mLayer = nullptr; -} - -LayerSnapshotGuard& LayerSnapshotGuard::operator=(LayerSnapshotGuard&& other) { - mLayer = other.mLayer; - other.mLayer = nullptr; - return *this; -} - void Layer::setTrustedPresentationInfo(TrustedPresentationThresholds const& thresholds, TrustedPresentationListener const& listener) { bool hadTrustedPresentationListener = hasTrustedPresentationListener(); diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h index 8424d7da9b..2955dafa0f 100644 --- a/services/surfaceflinger/Layer.h +++ b/services/surfaceflinger/Layer.h @@ -141,6 +141,8 @@ public: uint64_t frameNumber; ui::Transform transform; + + uint32_t producerId = 0; uint32_t bufferTransform; bool transformToDisplayInverse; Region transparentRegionHint; @@ -307,7 +309,8 @@ public: bool setSurfaceDamageRegion(const Region& /*surfaceDamage*/); bool setApi(int32_t /*api*/); bool setSidebandStream(const sp<NativeHandle>& /*sidebandStream*/); - bool setTransactionCompletedListeners(const std::vector<sp<CallbackHandle>>& /*handles*/); + bool setTransactionCompletedListeners(const std::vector<sp<CallbackHandle>>& /*handles*/, + bool willPresent); virtual bool setBackgroundColor(const half3& color, float alpha, ui::Dataspace dataspace); virtual bool setColorSpaceAgnostic(const bool agnostic); virtual bool setDimmingEnabled(const bool dimmingEnabled); @@ -328,9 +331,12 @@ public: virtual sp<LayerFE> getCompositionEngineLayerFE() const; virtual sp<LayerFE> copyCompositionEngineLayerFE() const; + sp<LayerFE> getCompositionEngineLayerFE(const frontend::LayerHierarchy::TraversalPath&); const frontend::LayerSnapshot* getLayerSnapshot() const; frontend::LayerSnapshot* editLayerSnapshot(); + std::unique_ptr<frontend::LayerSnapshot> stealLayerSnapshot(); + void updateLayerSnapshot(std::unique_ptr<frontend::LayerSnapshot> snapshot); // If we have received a new buffer this frame, we will pass its surface // damage down to hardware composer. Otherwise, we must send a region with @@ -512,7 +518,7 @@ public: // implements compositionengine::LayerFE const compositionengine::LayerFECompositionState* getCompositionState() const; bool fenceHasSignaled() const; - bool onPreComposition(nsecs_t refreshStartTime); + void onPreComposition(nsecs_t refreshStartTime); void onLayerDisplayed(ftl::SharedFuture<FenceResult>); void setWasClientComposed(const sp<Fence>& fence) { @@ -832,6 +838,11 @@ public: void updateMetadataSnapshot(const LayerMetadata& parentMetadata); void updateRelativeMetadataSnapshot(const LayerMetadata& relativeLayerMetadata, std::unordered_set<Layer*>& visited); + bool willPresentCurrentTransaction() const; + + void callReleaseBufferCallback(const sp<ITransactionCompletedListener>& listener, + const sp<GraphicBuffer>& buffer, uint64_t framenumber, + const sp<Fence>& releaseFence); protected: // For unit tests @@ -1037,13 +1048,15 @@ private: // Crop that applies to the buffer Rect computeBufferCrop(const State& s); - bool willPresentCurrentTransaction() const; - void callReleaseBufferCallback(const sp<ITransactionCompletedListener>& listener, const sp<GraphicBuffer>& buffer, uint64_t framenumber, const sp<Fence>& releaseFence, uint32_t currentMaxAcquiredBufferCount); + // Returns true if the transformed buffer size does not match the layer size and we need + // to apply filtering. + bool bufferNeedsFiltering() const; + // Returns true if there is a valid color to fill. bool fillsColor() const; // Returns true if this layer has a blur value. @@ -1151,34 +1164,10 @@ private: // not specify a destination frame. ui::Transform mRequestedTransform; - sp<LayerFE> mLayerFE; + sp<LayerFE> mLegacyLayerFE; + std::vector<std::pair<frontend::LayerHierarchy::TraversalPath, sp<LayerFE>>> mLayerFEs; std::unique_ptr<frontend::LayerSnapshot> mSnapshot = std::make_unique<frontend::LayerSnapshot>(); - - friend class LayerSnapshotGuard; -}; - -// LayerSnapshotGuard manages the movement of LayerSnapshot between a Layer and its corresponding -// LayerFE. This class must be used whenever LayerFEs are passed to CompositionEngine. Instances of -// LayerSnapshotGuard should only be constructed on the main thread and should not be moved outside -// the main thread. -// -// Moving the snapshot instead of sharing common state prevents use of LayerFE outside the main -// thread by making errors obvious (i.e. use outside the main thread results in SEGFAULTs due to -// nullptr dereference). -class LayerSnapshotGuard { -public: - LayerSnapshotGuard(Layer* layer) REQUIRES(kMainThreadContext); - ~LayerSnapshotGuard() REQUIRES(kMainThreadContext); - - LayerSnapshotGuard(const LayerSnapshotGuard&) = delete; - LayerSnapshotGuard& operator=(const LayerSnapshotGuard&) = delete; - - LayerSnapshotGuard(LayerSnapshotGuard&& other) REQUIRES(kMainThreadContext); - LayerSnapshotGuard& operator=(LayerSnapshotGuard&& other) REQUIRES(kMainThreadContext); - -private: - Layer* mLayer; }; std::ostream& operator<<(std::ostream& stream, const Layer::FrameRate& rate); diff --git a/services/surfaceflinger/LayerRenderArea.cpp b/services/surfaceflinger/LayerRenderArea.cpp index 2b4375b0fa..03a7f226ed 100644 --- a/services/surfaceflinger/LayerRenderArea.cpp +++ b/services/surfaceflinger/LayerRenderArea.cpp @@ -69,6 +69,14 @@ Rect LayerRenderArea::getSourceCrop() const { void LayerRenderArea::render(std::function<void()> drawLayers) { using namespace std::string_literals; + if (!mChildrenOnly) { + mTransform = mLayer->getTransform().inverse(); + } + + if (mFlinger.mLayerLifecycleManagerEnabled) { + drawLayers(); + return; + } // If layer is offscreen, update mirroring info if it exists if (mLayer->isRemovedFromCurrentState()) { mLayer->traverse(LayerVector::StateSet::Drawing, @@ -78,7 +86,6 @@ void LayerRenderArea::render(std::function<void()> drawLayers) { } if (!mChildrenOnly) { - mTransform = mLayer->getTransform().inverse(); // If the layer is offscreen, compute bounds since we don't compute bounds for offscreen // layers in a regular cycles. if (mLayer->isRemovedFromCurrentState()) { diff --git a/services/surfaceflinger/Scheduler/ISchedulerCallback.h b/services/surfaceflinger/Scheduler/ISchedulerCallback.h new file mode 100644 index 0000000000..c4de7497ac --- /dev/null +++ b/services/surfaceflinger/Scheduler/ISchedulerCallback.h @@ -0,0 +1,35 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <vector> + +#include "Display/DisplayModeRequest.h" + +namespace android::scheduler { + +struct ISchedulerCallback { + virtual void setVsyncEnabled(bool) = 0; + virtual void requestDisplayModes(std::vector<display::DisplayModeRequest>) = 0; + virtual void kernelTimerChanged(bool expired) = 0; + virtual void triggerOnFrameRateOverridesChanged() = 0; + +protected: + ~ISchedulerCallback() = default; +}; + +} // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/Scheduler.cpp b/services/surfaceflinger/Scheduler/Scheduler.cpp index 5fc250bd9f..17cdff9518 100644 --- a/services/surfaceflinger/Scheduler/Scheduler.cpp +++ b/services/surfaceflinger/Scheduler/Scheduler.cpp @@ -28,10 +28,10 @@ #include <ftl/enum.h> #include <ftl/fake_guard.h> #include <ftl/small_map.h> +#include <gui/TraceUtils.h> #include <gui/WindowInfo.h> #include <system/window.h> #include <utils/Timers.h> -#include <utils/Trace.h> #include <FrameTimeline/FrameTimeline.h> #include <scheduler/interface/ICompositor.h> @@ -155,7 +155,7 @@ void Scheduler::onFrameSignal(ICompositor& compositor, VsyncId vsyncId, } void Scheduler::createVsyncSchedule(FeatureFlags features) { - mVsyncSchedule.emplace(features); + mVsyncSchedule = std::make_unique<VsyncSchedule>(features); } std::optional<Fps> Scheduler::getFrameRateOverride(uid_t uid) const { @@ -171,6 +171,7 @@ bool Scheduler::isVsyncValid(TimePoint expectedVsyncTimestamp, uid_t uid) const return true; } + ATRACE_FORMAT("%s uid: %d frameRate: %s", __func__, uid, to_string(*frameRate).c_str()); return mVsyncSchedule->getTracker().isVSyncInPhase(expectedVsyncTimestamp.ns(), *frameRate); } @@ -393,38 +394,17 @@ void Scheduler::setVsyncConfig(const VsyncConfig& config, Period vsyncPeriod) { } void Scheduler::enableHardwareVsync() { - std::lock_guard<std::mutex> lock(mHWVsyncLock); - if (!mPrimaryHWVsyncEnabled && mHWVsyncAvailable) { - mVsyncSchedule->getTracker().resetModel(); - mSchedulerCallback.setVsyncEnabled(true); - mPrimaryHWVsyncEnabled = true; - } + mVsyncSchedule->enableHardwareVsync(mSchedulerCallback); } -void Scheduler::disableHardwareVsync(bool makeUnavailable) { - std::lock_guard<std::mutex> lock(mHWVsyncLock); - if (mPrimaryHWVsyncEnabled) { - mSchedulerCallback.setVsyncEnabled(false); - mPrimaryHWVsyncEnabled = false; - } - if (makeUnavailable) { - mHWVsyncAvailable = false; - } +void Scheduler::disableHardwareVsync(bool disallow) { + mVsyncSchedule->disableHardwareVsync(mSchedulerCallback, disallow); } -void Scheduler::resyncToHardwareVsync(bool makeAvailable, Fps refreshRate) { - { - std::lock_guard<std::mutex> lock(mHWVsyncLock); - if (makeAvailable) { - mHWVsyncAvailable = makeAvailable; - } else if (!mHWVsyncAvailable) { - // Hardware vsync is not currently available, so abort the resync - // attempt for now - return; - } +void Scheduler::resyncToHardwareVsync(bool allowToEnable, Fps refreshRate) { + if (mVsyncSchedule->isHardwareVsyncAllowed(allowToEnable) && refreshRate.isValid()) { + mVsyncSchedule->startPeriodTransition(mSchedulerCallback, refreshRate.getPeriod()); } - - setVsyncPeriod(refreshRate.getPeriodNsecs()); } void Scheduler::setRenderRate(Fps renderFrameRate) { @@ -457,37 +437,12 @@ void Scheduler::resync() { } } -void Scheduler::setVsyncPeriod(nsecs_t period) { - if (period <= 0) return; - - std::lock_guard<std::mutex> lock(mHWVsyncLock); - mVsyncSchedule->getController().startPeriodTransition(period); - - if (!mPrimaryHWVsyncEnabled) { - mVsyncSchedule->getTracker().resetModel(); - mSchedulerCallback.setVsyncEnabled(true); - mPrimaryHWVsyncEnabled = true; - } -} - -void Scheduler::addResyncSample(nsecs_t timestamp, std::optional<nsecs_t> hwcVsyncPeriod, - bool* periodFlushed) { - bool needsHwVsync = false; - *periodFlushed = false; - { // Scope for the lock - std::lock_guard<std::mutex> lock(mHWVsyncLock); - if (mPrimaryHWVsyncEnabled) { - needsHwVsync = - mVsyncSchedule->getController().addHwVsyncTimestamp(timestamp, hwcVsyncPeriod, - periodFlushed); - } - } - - if (needsHwVsync) { - enableHardwareVsync(); - } else { - disableHardwareVsync(false); - } +bool Scheduler::addResyncSample(nsecs_t timestamp, std::optional<nsecs_t> hwcVsyncPeriodIn) { + const auto hwcVsyncPeriod = ftl::Optional(hwcVsyncPeriodIn).transform([](nsecs_t nanos) { + return Period::fromNs(nanos); + }); + return mVsyncSchedule->addResyncSample(mSchedulerCallback, TimePoint::fromNs(timestamp), + hwcVsyncPeriod); } void Scheduler::addPresentFence(std::shared_ptr<FenceTime> fence) { @@ -635,14 +590,6 @@ void Scheduler::dump(utils::Dumper& dumper) const { mFrameRateOverrideMappings.dump(dumper); dumper.eol(); - - { - utils::Dumper::Section section(dumper, "Hardware VSYNC"sv); - - std::lock_guard lock(mHWVsyncLock); - dumper.dump("hwVsyncAvailable"sv, mHWVsyncAvailable); - dumper.dump("hwVsyncEnabled"sv, mPrimaryHWVsyncEnabled); - } } void Scheduler::dumpVsync(std::string& out) const { diff --git a/services/surfaceflinger/Scheduler/Scheduler.h b/services/surfaceflinger/Scheduler/Scheduler.h index dae9546c78..a340919a65 100644 --- a/services/surfaceflinger/Scheduler/Scheduler.h +++ b/services/surfaceflinger/Scheduler/Scheduler.h @@ -43,6 +43,7 @@ #include "Display/DisplayModeRequest.h" #include "EventThread.h" #include "FrameRateOverrideMappings.h" +#include "ISchedulerCallback.h" #include "LayerHistory.h" #include "MessageQueue.h" #include "OneShotTimer.h" @@ -92,16 +93,6 @@ namespace scheduler { using GlobalSignals = RefreshRateSelector::GlobalSignals; -struct ISchedulerCallback { - virtual void setVsyncEnabled(bool) = 0; - virtual void requestDisplayModes(std::vector<display::DisplayModeRequest>) = 0; - virtual void kernelTimerChanged(bool expired) = 0; - virtual void triggerOnFrameRateOverridesChanged() = 0; - -protected: - ~ISchedulerCallback() = default; -}; - class Scheduler : android::impl::MessageQueue { using Impl = android::impl::MessageQueue; @@ -192,20 +183,20 @@ public: void setRenderRate(Fps); void enableHardwareVsync(); - void disableHardwareVsync(bool makeUnavailable); + void disableHardwareVsync(bool disallow); // Resyncs the scheduler to hardware vsync. - // If makeAvailable is true, then hardware vsync will be turned on. + // If allowToEnable is true, then hardware vsync will be turned on. // Otherwise, if hardware vsync is not already enabled then this method will // no-op. - void resyncToHardwareVsync(bool makeAvailable, Fps refreshRate); + void resyncToHardwareVsync(bool allowToEnable, Fps refreshRate); void resync() EXCLUDES(mDisplayLock); void forceNextResync() { mLastResyncTime = 0; } - // Passes a vsync sample to VsyncController. periodFlushed will be true if - // VsyncController detected that the vsync period changed, and false otherwise. - void addResyncSample(nsecs_t timestamp, std::optional<nsecs_t> hwcVsyncPeriod, - bool* periodFlushed); + // Passes a vsync sample to VsyncController. Returns true if + // VsyncController detected that the vsync period changed and false + // otherwise. + bool addResyncSample(nsecs_t timestamp, std::optional<nsecs_t> hwcVsyncPeriod); void addPresentFence(std::shared_ptr<FenceTime>); // Layers are registered on creation, and unregistered when the weak reference expires. @@ -297,7 +288,6 @@ private: void touchTimerCallback(TimerState); void displayPowerTimerCallback(TimerState); - void setVsyncPeriod(nsecs_t period); void setVsyncConfig(const VsyncConfig&, Period vsyncPeriod); // Chooses a leader among the registered displays, unless `leaderIdOpt` is specified. The new @@ -362,14 +352,10 @@ private: ConnectionHandle mAppConnectionHandle; ConnectionHandle mSfConnectionHandle; - mutable std::mutex mHWVsyncLock; - bool mPrimaryHWVsyncEnabled GUARDED_BY(mHWVsyncLock) = false; - bool mHWVsyncAvailable GUARDED_BY(mHWVsyncLock) = false; - std::atomic<nsecs_t> mLastResyncTime = 0; const FeatureFlags mFeatures; - std::optional<VsyncSchedule> mVsyncSchedule; + std::unique_ptr<VsyncSchedule> mVsyncSchedule; // Shifts the VSYNC phase during certain transactions and refresh rate changes. const sp<VsyncModulator> mVsyncModulator; diff --git a/services/surfaceflinger/Scheduler/VSyncPredictor.cpp b/services/surfaceflinger/Scheduler/VSyncPredictor.cpp index 02e12fd942..f8cb3230af 100644 --- a/services/surfaceflinger/Scheduler/VSyncPredictor.cpp +++ b/services/surfaceflinger/Scheduler/VSyncPredictor.cpp @@ -31,8 +31,8 @@ #include <android-base/stringprintf.h> #include <cutils/compiler.h> #include <cutils/properties.h> +#include <gui/TraceUtils.h> #include <utils/Log.h> -#include <utils/Trace.h> #include "RefreshRateSelector.h" #include "VSyncPredictor.h" @@ -282,6 +282,13 @@ bool VSyncPredictor::isVSyncInPhase(nsecs_t timePoint, Fps frameRate) const { } bool VSyncPredictor::isVSyncInPhaseLocked(nsecs_t timePoint, unsigned divisor) const { + const TimePoint now = TimePoint::now(); + const auto getTimePointIn = [](TimePoint now, nsecs_t timePoint) -> float { + return ticks<std::milli, float>(TimePoint::fromNs(timePoint) - now); + }; + ATRACE_FORMAT("%s timePoint in: %.2f divisor: %zu", __func__, getTimePointIn(now, timePoint), + divisor); + struct VsyncError { nsecs_t vsyncTimestamp; float error; @@ -304,6 +311,7 @@ bool VSyncPredictor::isVSyncInPhaseLocked(nsecs_t timePoint, unsigned divisor) c if (knownTimestampIter == mRateDivisorKnownTimestampMap.end()) { const auto vsync = nextAnticipatedVSyncTimeFromLocked(justBeforeTimePoint); mRateDivisorKnownTimestampMap[dividedPeriod] = vsync; + ATRACE_FORMAT_INSTANT("(first) knownVsync in: %.2f", getTimePointIn(now, vsync)); return true; } @@ -323,6 +331,8 @@ bool VSyncPredictor::isVSyncInPhaseLocked(nsecs_t timePoint, unsigned divisor) c const auto minVsyncError = std::min_element(vsyncs.begin(), vsyncs.end()); mRateDivisorKnownTimestampMap[dividedPeriod] = minVsyncError->vsyncTimestamp; + ATRACE_FORMAT_INSTANT("knownVsync in: %.2f", + getTimePointIn(now, minVsyncError->vsyncTimestamp)); return std::abs(minVsyncError->vsyncTimestamp - timePoint) < period / 2; } diff --git a/services/surfaceflinger/Scheduler/VsyncSchedule.cpp b/services/surfaceflinger/Scheduler/VsyncSchedule.cpp index 95bc31f239..524555681b 100644 --- a/services/surfaceflinger/Scheduler/VsyncSchedule.cpp +++ b/services/surfaceflinger/Scheduler/VsyncSchedule.cpp @@ -16,11 +16,14 @@ #define ATRACE_TAG ATRACE_TAG_GRAPHICS +#include <ftl/fake_guard.h> #include <scheduler/Fps.h> #include <scheduler/Timer.h> #include "VsyncSchedule.h" +#include "ISchedulerCallback.h" +#include "Utils/Dumper.h" #include "VSyncDispatchTimerQueue.h" #include "VSyncPredictor.h" #include "VSyncReactor.h" @@ -54,18 +57,16 @@ private: VsyncSchedule::VsyncSchedule(FeatureFlags features) : mTracker(createTracker()), mDispatch(createDispatch(*mTracker)), - mController(createController(*mTracker, features)) { - if (features.test(Feature::kTracePredictedVsync)) { - mTracer = std::make_unique<PredictedVsyncTracer>(*mDispatch); - } -} + mController(createController(*mTracker, features)), + mTracer(features.test(Feature::kTracePredictedVsync) + ? std::make_unique<PredictedVsyncTracer>(*mDispatch) + : nullptr) {} VsyncSchedule::VsyncSchedule(TrackerPtr tracker, DispatchPtr dispatch, ControllerPtr controller) : mTracker(std::move(tracker)), mDispatch(std::move(dispatch)), mController(std::move(controller)) {} -VsyncSchedule::VsyncSchedule(VsyncSchedule&&) = default; VsyncSchedule::~VsyncSchedule() = default; Period VsyncSchedule::period() const { @@ -77,6 +78,16 @@ TimePoint VsyncSchedule::vsyncDeadlineAfter(TimePoint timePoint) const { } void VsyncSchedule::dump(std::string& out) const { + utils::Dumper dumper(out); + { + std::lock_guard<std::mutex> lock(mHwVsyncLock); + dumper.dump("hwVsyncState", ftl::enum_string(mHwVsyncState)); + + ftl::FakeGuard guard(kMainThreadContext); + dumper.dump("pendingHwVsyncState", ftl::enum_string(mPendingHwVsyncState)); + dumper.eol(); + } + out.append("VsyncController:\n"); mController->dump(out); @@ -120,4 +131,67 @@ VsyncSchedule::ControllerPtr VsyncSchedule::createController(VsyncTracker& track return reactor; } +void VsyncSchedule::startPeriodTransition(ISchedulerCallback& callback, Period period) { + std::lock_guard<std::mutex> lock(mHwVsyncLock); + mController->startPeriodTransition(period.ns()); + enableHardwareVsyncLocked(callback); +} + +bool VsyncSchedule::addResyncSample(ISchedulerCallback& callback, TimePoint timestamp, + ftl::Optional<Period> hwcVsyncPeriod) { + bool needsHwVsync = false; + bool periodFlushed = false; + { + std::lock_guard<std::mutex> lock(mHwVsyncLock); + if (mHwVsyncState == HwVsyncState::Enabled) { + needsHwVsync = mController->addHwVsyncTimestamp(timestamp.ns(), + hwcVsyncPeriod.transform(&Period::ns), + &periodFlushed); + } + } + if (needsHwVsync) { + enableHardwareVsync(callback); + } else { + disableHardwareVsync(callback, false /* disallow */); + } + return periodFlushed; +} + +void VsyncSchedule::enableHardwareVsync(ISchedulerCallback& callback) { + std::lock_guard<std::mutex> lock(mHwVsyncLock); + enableHardwareVsyncLocked(callback); +} + +void VsyncSchedule::enableHardwareVsyncLocked(ISchedulerCallback& callback) { + if (mHwVsyncState == HwVsyncState::Disabled) { + getTracker().resetModel(); + callback.setVsyncEnabled(true); + mHwVsyncState = HwVsyncState::Enabled; + } +} + +void VsyncSchedule::disableHardwareVsync(ISchedulerCallback& callback, bool disallow) { + std::lock_guard<std::mutex> lock(mHwVsyncLock); + if (mHwVsyncState == HwVsyncState::Enabled) { + callback.setVsyncEnabled(false); + } + mHwVsyncState = disallow ? HwVsyncState::Disallowed : HwVsyncState::Disabled; +} + +bool VsyncSchedule::isHardwareVsyncAllowed(bool makeAllowed) { + std::lock_guard<std::mutex> lock(mHwVsyncLock); + if (makeAllowed && mHwVsyncState == HwVsyncState::Disallowed) { + mHwVsyncState = HwVsyncState::Disabled; + } + return mHwVsyncState != HwVsyncState::Disallowed; +} + +void VsyncSchedule::setPendingHardwareVsyncState(bool enabled) { + mPendingHwVsyncState = enabled ? HwVsyncState::Enabled : HwVsyncState::Disabled; +} + +bool VsyncSchedule::getPendingHardwareVsyncState() const { + return mPendingHwVsyncState == HwVsyncState::Enabled; +} + } // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/VsyncSchedule.h b/services/surfaceflinger/Scheduler/VsyncSchedule.h index 173b1d00cf..d88f1d1f0b 100644 --- a/services/surfaceflinger/Scheduler/VsyncSchedule.h +++ b/services/surfaceflinger/Scheduler/VsyncSchedule.h @@ -19,6 +19,9 @@ #include <memory> #include <string> +#include <ThreadContext.h> +#include <ftl/enum.h> +#include <ftl/optional.h> #include <scheduler/Features.h> #include <scheduler/Time.h> @@ -32,6 +35,8 @@ class SchedulerFuzzer; namespace android::scheduler { +struct ISchedulerCallback; + // TODO(b/185535769): Rename classes, and remove aliases. class VSyncDispatch; class VSyncTracker; @@ -44,12 +49,24 @@ using VsyncTracker = VSyncTracker; class VsyncSchedule { public: explicit VsyncSchedule(FeatureFlags); - VsyncSchedule(VsyncSchedule&&); ~VsyncSchedule(); Period period() const; TimePoint vsyncDeadlineAfter(TimePoint) const; + // Inform the schedule that the period is changing and the schedule needs to recalibrate + // itself. The schedule will end the period transition internally. This will + // enable hardware VSYNCs in order to calibrate. + // + // \param [in] period The period that the system is changing into. + void startPeriodTransition(ISchedulerCallback&, Period period); + + // Pass a VSYNC sample to VsyncController. Return true if + // VsyncController detected that the VSYNC period changed. Enable or disable + // hardware VSYNCs depending on whether more samples are needed. + bool addResyncSample(ISchedulerCallback&, TimePoint timestamp, + ftl::Optional<Period> hwcVsyncPeriod); + // TODO(b/185535769): Hide behind API. const VsyncTracker& getTracker() const { return *mTracker; } VsyncTracker& getTracker() { return *mTracker; } @@ -60,6 +77,22 @@ public: void dump(std::string&) const; + // Turn on hardware VSYNCs, unless mHwVsyncState is Disallowed, in which + // case this call is ignored. + void enableHardwareVsync(ISchedulerCallback&) EXCLUDES(mHwVsyncLock); + + // Disable hardware VSYNCs. If `disallow` is true, future calls to + // enableHardwareVsync are ineffective until allowHardwareVsync is called. + void disableHardwareVsync(ISchedulerCallback&, bool disallow) EXCLUDES(mHwVsyncLock); + + // If true, enableHardwareVsync can enable hardware VSYNC (if not already + // enabled). If false, enableHardwareVsync does nothing. + bool isHardwareVsyncAllowed(bool makeAllowed) EXCLUDES(mHwVsyncLock); + + void setPendingHardwareVsyncState(bool enabled) REQUIRES(kMainThreadContext); + + bool getPendingHardwareVsyncState() const REQUIRES(kMainThreadContext); + private: friend class TestableScheduler; friend class android::EventThreadTest; @@ -76,14 +109,36 @@ private: static DispatchPtr createDispatch(VsyncTracker&); static ControllerPtr createController(VsyncTracker&, FeatureFlags); + void enableHardwareVsyncLocked(ISchedulerCallback&) REQUIRES(mHwVsyncLock); + + mutable std::mutex mHwVsyncLock; + enum class HwVsyncState { + // Hardware VSYNCs are currently enabled. + Enabled, + + // Hardware VSYNCs are currently disabled. They can be enabled by a call + // to `enableHardwareVsync`. + Disabled, + + // Hardware VSYNCs are not currently allowed (e.g. because the display + // is off). + Disallowed, + + ftl_last = Disallowed, + }; + HwVsyncState mHwVsyncState GUARDED_BY(mHwVsyncLock) = HwVsyncState::Disallowed; + + // Pending state, in case an attempt is made to set the state while the + // device is off. + HwVsyncState mPendingHwVsyncState GUARDED_BY(kMainThreadContext) = HwVsyncState::Disabled; + class PredictedVsyncTracer; using TracerPtr = std::unique_ptr<PredictedVsyncTracer>; - // Effectively const except in move constructor. - TrackerPtr mTracker; - DispatchPtr mDispatch; - ControllerPtr mController; - TracerPtr mTracer; + const TrackerPtr mTracker; + const DispatchPtr mDispatch; + const ControllerPtr mController; + const TracerPtr mTracer; }; } // namespace android::scheduler diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp index f223de61fe..b42576fd8e 100644 --- a/services/surfaceflinger/SurfaceFlinger.cpp +++ b/services/surfaceflinger/SurfaceFlinger.cpp @@ -173,6 +173,8 @@ using namespace std::string_view_literals; using namespace hardware::configstore; using namespace hardware::configstore::V1_0; using namespace sysprop; +using ftl::Flags; +using namespace ftl::flag_operators; using aidl::android::hardware::graphics::common::DisplayDecorationSupport; using aidl::android::hardware::graphics::composer3::Capability; @@ -470,6 +472,10 @@ SurfaceFlinger::SurfaceFlinger(Factory& factory) : SurfaceFlinger(factory, SkipI mPowerHintSessionMode = {.late = base::GetBoolProperty("debug.sf.send_late_power_session_hint"s, true), .early = base::GetBoolProperty("debug.sf.send_early_power_session_hint"s, false)}; + mLayerLifecycleManagerEnabled = + base::GetBoolProperty("debug.sf.enable_layer_lifecycle_manager"s, false); + mLegacyFrontEndEnabled = !mLayerLifecycleManagerEnabled || + base::GetBoolProperty("debug.sf.enable_legacy_frontend"s, true); } LatchUnsignaledConfig SurfaceFlinger::getLatchUnsignaledConfig() { @@ -2031,8 +2037,7 @@ void SurfaceFlinger::onComposerHalVsync(hal::HWDisplayId hwcDisplayId, int64_t t return; } - bool periodFlushed = false; - mScheduler->addResyncSample(timestamp, vsyncPeriod, &periodFlushed); + const bool periodFlushed = mScheduler->addResyncSample(timestamp, vsyncPeriod); if (periodFlushed) { mScheduler->modulateVsync(&VsyncModulator::onRefreshRateChangeCompleted); } @@ -2075,16 +2080,23 @@ void SurfaceFlinger::onComposerHalVsyncIdle(hal::HWDisplayId) { mScheduler->forceNextResync(); } +void SurfaceFlinger::onRefreshRateChangedDebug(const RefreshRateChangedDebugData&) { + // TODO(b/202734676) update refresh rate value on the RefreshRateOverlay +} + void SurfaceFlinger::setVsyncEnabled(bool enabled) { ATRACE_CALL(); // On main thread to avoid race conditions with display power state. static_cast<void>(mScheduler->schedule([=]() FTL_FAKE_GUARD(mStateLock) { - mHWCVsyncPendingState = enabled ? hal::Vsync::ENABLE : hal::Vsync::DISABLE; + { + ftl::FakeGuard guard(kMainThreadContext); + mScheduler->getVsyncSchedule().setPendingHardwareVsyncState(enabled); + } if (const auto display = getDefaultDisplayDeviceLocked(); display && display->isPoweredOn()) { - setHWCVsyncEnabled(display->getPhysicalId(), mHWCVsyncPendingState); + setHWCVsyncEnabled(display->getPhysicalId(), enabled); } })); } @@ -2127,6 +2139,110 @@ void SurfaceFlinger::configure() FTL_FAKE_GUARD(kMainThreadContext) { } } +bool SurfaceFlinger::updateLayerSnapshotsLegacy(VsyncId vsyncId, LifecycleUpdate& update, + bool transactionsFlushed, + bool& outTransactionsAreEmpty) { + bool needsTraversal = false; + if (transactionsFlushed) { + needsTraversal |= commitMirrorDisplays(vsyncId); + needsTraversal |= commitCreatedLayers(vsyncId, update.layerCreatedStates); + needsTraversal |= applyTransactions(update.transactions, vsyncId); + } + outTransactionsAreEmpty = !needsTraversal; + const bool shouldCommit = (getTransactionFlags() & ~eTransactionFlushNeeded) || needsTraversal; + if (shouldCommit) { + commitTransactions(); + } + + bool mustComposite = latchBuffers() || shouldCommit; + updateLayerGeometry(); + return mustComposite; +} + +bool SurfaceFlinger::updateLayerSnapshots(VsyncId vsyncId, LifecycleUpdate& update, + bool transactionsFlushed, bool& outTransactionsAreEmpty) { + using Changes = frontend::RequestedLayerState::Changes; + ATRACE_NAME("updateLayerSnapshots"); + { + mLayerLifecycleManager.addLayers(std::move(update.newLayers)); + mLayerLifecycleManager.applyTransactions(update.transactions); + mLayerLifecycleManager.onHandlesDestroyed(update.destroyedHandles); + for (auto& legacyLayer : update.layerCreatedStates) { + sp<Layer> layer = legacyLayer.layer.promote(); + if (layer) { + mLegacyLayers[layer->sequence] = layer; + } + } + } + if (mLayerLifecycleManager.getGlobalChanges().test(Changes::Hierarchy)) { + ATRACE_NAME("LayerHierarchyBuilder:update"); + mLayerHierarchyBuilder.update(mLayerLifecycleManager.getLayers(), + mLayerLifecycleManager.getDestroyedLayers()); + } + + applyAndCommitDisplayTransactionStates(update.transactions); + + { + ATRACE_NAME("LayerSnapshotBuilder:update"); + frontend::LayerSnapshotBuilder::Args args{.root = mLayerHierarchyBuilder.getHierarchy(), + .layerLifecycleManager = mLayerLifecycleManager, + .displays = mFrontEndDisplayInfos, + .displayChanges = mFrontEndDisplayInfosChanged, + .globalShadowSettings = + mDrawingState.globalShadowSettings, + .supportsBlur = mSupportsBlur, + .forceFullDamage = mForceFullDamage}; + mLayerSnapshotBuilder.update(args); + } + + if (mLayerLifecycleManager.getGlobalChanges().any(Changes::Geometry | Changes::Input | + Changes::Hierarchy)) { + mUpdateInputInfo = true; + } + if (mLayerLifecycleManager.getGlobalChanges().any(Changes::VisibleRegion | Changes::Hierarchy | + Changes::Visibility)) { + mVisibleRegionsDirty = true; + } + outTransactionsAreEmpty = mLayerLifecycleManager.getGlobalChanges().get() == 0; + const bool mustComposite = mLayerLifecycleManager.getGlobalChanges().get() != 0; + { + ATRACE_NAME("LLM:commitChanges"); + mLayerLifecycleManager.commitChanges(); + } + + if (!mLegacyFrontEndEnabled) { + ATRACE_NAME("DisplayCallbackAndStatsUpdates"); + applyTransactions(update.transactions, vsyncId); + + bool newDataLatched = false; + for (auto& snapshot : mLayerSnapshotBuilder.getSnapshots()) { + if (!snapshot->changes.test(Changes::Buffer)) continue; + auto it = mLegacyLayers.find(snapshot->sequence); + LOG_ALWAYS_FATAL_IF(it == mLegacyLayers.end(), "Couldnt find layer object for %s", + snapshot->getDebugString().c_str()); + mLayersWithQueuedFrames.emplace(it->second); + newDataLatched = true; + if (!snapshot->isVisible) break; + + Region visibleReg; + visibleReg.set(snapshot->transformedBoundsWithoutTransparentRegion); + invalidateLayerStack(snapshot->outputFilter, visibleReg); + } + + for (auto& destroyedLayer : mLayerLifecycleManager.getDestroyedLayers()) { + mLegacyLayers.erase(destroyedLayer->id); + } + + // enter boot animation on first buffer latch + if (CC_UNLIKELY(mBootStage == BootStage::BOOTLOADER && newDataLatched)) { + ALOGI("Enter boot animation"); + mBootStage = BootStage::BOOTANIMATION; + } + commitTransactions(); + } + return mustComposite; +} + bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expectedVsyncTime) FTL_FAKE_GUARD(kMainThreadContext) { // The expectedVsyncTime, which was predicted when this frame was scheduled, is normally in the @@ -2266,45 +2382,34 @@ bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expe mFrameTimeline->setSfWakeUp(vsyncId.value, frameTime.ns(), Fps::fromPeriodNsecs(vsyncPeriod.ns())); - bool needsTraversal = false; - if (clearTransactionFlags(eTransactionFlushNeeded)) { - // Locking: - // 1. to prevent onHandleDestroyed from being called while the state lock is held, - // we must keep a copy of the transactions (specifically the composer - // states) around outside the scope of the lock. - // 2. Transactions and created layers do not share a lock. To prevent applying - // transactions with layers still in the createdLayer queue, flush the transactions - // before committing the created layers. - std::vector<TransactionState> transactions = mTransactionHandler.flushTransactions(); - needsTraversal |= commitMirrorDisplays(vsyncId); - needsTraversal |= commitCreatedLayers(vsyncId); - needsTraversal |= applyTransactions(transactions, vsyncId); + const bool flushTransactions = clearTransactionFlags(eTransactionFlushNeeded); + LifecycleUpdate updates; + if (flushTransactions) { + updates = flushLifecycleUpdates(); } - - const bool shouldCommit = - (getTransactionFlags() & ~eTransactionFlushNeeded) || needsTraversal; - if (shouldCommit) { - commitTransactions(); + bool transactionsAreEmpty; + if (mLegacyFrontEndEnabled) { + mustComposite |= updateLayerSnapshotsLegacy(vsyncId, updates, flushTransactions, + transactionsAreEmpty); + } + if (mLayerLifecycleManagerEnabled) { + mustComposite |= + updateLayerSnapshots(vsyncId, updates, flushTransactions, transactionsAreEmpty); } if (transactionFlushNeeded()) { setTransactionFlags(eTransactionFlushNeeded); } - mustComposite |= shouldCommit; - mustComposite |= latchBuffers(); - // This has to be called after latchBuffers because we want to include the layers that have // been latched in the commit callback - if (!needsTraversal) { + if (transactionsAreEmpty) { // Invoke empty transaction callbacks early. mTransactionCallbackInvoker.sendCallbacks(false /* onCommitOnly */); } else { // Invoke OnCommit callbacks. mTransactionCallbackInvoker.sendCallbacks(true /* onCommitOnly */); } - - updateLayerGeometry(); } // Layers need to get updated (in the previous line) before we can use them for @@ -2391,15 +2496,6 @@ void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) refreshArgs.updatingOutputGeometryThisFrame = mVisibleRegionsDirty; refreshArgs.updatingGeometryThisFrame = mGeometryDirty.exchange(false) || mVisibleRegionsDirty; - std::vector<Layer*> layers; - - mDrawingState.traverseInZOrder([&refreshArgs, &layers](Layer* layer) { - if (auto layerFE = layer->getCompositionEngineLayerFE()) { - layer->updateSnapshot(refreshArgs.updatingGeometryThisFrame); - refreshArgs.layers.push_back(layerFE); - layers.push_back(layer); - } - }); refreshArgs.internalDisplayRotationFlags = DisplayDevice::getPrimaryDisplayRotationFlags(); if (CC_UNLIKELY(mDrawingState.colorMatrixChanged)) { @@ -2426,17 +2522,13 @@ void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) // the scheduler. const auto presentTime = systemTime(); - { - std::vector<LayerSnapshotGuard> layerSnapshotGuards; - for (Layer* layer : layers) { - layerSnapshotGuards.emplace_back(layer); - } - mCompositionEngine->present(refreshArgs); - } + std::vector<std::pair<Layer*, LayerFE*>> layers = + moveSnapshotsToCompositionArgs(refreshArgs, /*cursorOnly=*/false, vsyncId.value); + mCompositionEngine->present(refreshArgs); + moveSnapshotsFromCompositionArgs(refreshArgs, layers); - for (auto& layer : layers) { - CompositionResult compositionResult{ - layer->getCompositionEngineLayerFE()->stealCompositionResult()}; + for (auto [layer, layerFE] : layers) { + CompositionResult compositionResult{layerFE->stealCompositionResult()}; layer->onPreComposition(compositionResult.refreshStartTime); for (auto releaseFence : compositionResult.releaseFences) { layer->onLayerDisplayed(releaseFence); @@ -2530,7 +2622,7 @@ void SurfaceFlinger::updateLayerGeometry() { for (auto& layer : mLayersPendingRefresh) { Region visibleReg; visibleReg.set(layer->getScreenBounds()); - invalidateLayerStack(layer, visibleReg); + invalidateLayerStack(layer->getOutputFilter(), visibleReg); } mLayersPendingRefresh.clear(); } @@ -3387,7 +3479,8 @@ void SurfaceFlinger::processDisplayChangesLocked() { void SurfaceFlinger::commitTransactionsLocked(uint32_t transactionFlags) { // Commit display transactions. const bool displayTransactionNeeded = transactionFlags & eDisplayTransactionNeeded; - if (displayTransactionNeeded) { + mFrontEndDisplayInfosChanged = displayTransactionNeeded; + if (displayTransactionNeeded && !mLayerLifecycleManagerEnabled) { processDisplayChangesLocked(); mFrontEndDisplayInfos.clear(); for (const auto& [_, display] : mDisplays) { @@ -3478,7 +3571,7 @@ void SurfaceFlinger::commitTransactionsLocked(uint32_t transactionFlags) { // this layer is not visible anymore Region visibleReg; visibleReg.set(layer->getScreenBounds()); - invalidateLayerStack(sp<Layer>::fromExisting(layer), visibleReg); + invalidateLayerStack(layer->getOutputFilter(), visibleReg); } }); } @@ -3566,16 +3659,23 @@ void SurfaceFlinger::buildWindowInfos(std::vector<WindowInfo>& outWindowInfos, outWindowInfos.reserve(sNumWindowInfos); sNumWindowInfos = 0; - mDrawingState.traverseInReverseZOrder([&](Layer* layer) { - if (!layer->needsInputInfo()) return; - - const auto opt = mFrontEndDisplayInfos.get(layer->getLayerStack()) - .transform([](const frontend::DisplayInfo& info) { - return Layer::InputDisplayArgs{&info.transform, info.isSecure}; - }); + if (mLayerLifecycleManagerEnabled) { + mLayerSnapshotBuilder.forEachInputSnapshot( + [&outWindowInfos](const frontend::LayerSnapshot& snapshot) { + outWindowInfos.push_back(snapshot.inputInfo); + }); + } else { + mDrawingState.traverseInReverseZOrder([&](Layer* layer) { + if (!layer->needsInputInfo()) return; + const auto opt = + mFrontEndDisplayInfos.get(layer->getLayerStack()) + .transform([](const frontend::DisplayInfo& info) { + return Layer::InputDisplayArgs{&info.transform, info.isSecure}; + }); - outWindowInfos.push_back(layer->fillInputInfo(opt.value_or(Layer::InputDisplayArgs{}))); - }); + outWindowInfos.push_back(layer->fillInputInfo(opt.value_or(Layer::InputDisplayArgs{}))); + }); + } sNumWindowInfos = outWindowInfos.size(); @@ -3592,17 +3692,9 @@ void SurfaceFlinger::updateCursorAsync() { refreshArgs.outputs.push_back(display->getCompositionDisplay()); } } - - std::vector<LayerSnapshotGuard> layerSnapshotGuards; - mDrawingState.traverse([&layerSnapshotGuards](Layer* layer) { - if (layer->getLayerSnapshot()->compositionType == - aidl::android::hardware::graphics::composer3::Composition::CURSOR) { - layer->updateSnapshot(false /* updateGeometry */); - layerSnapshotGuards.emplace_back(layer); - } - }); - + auto layers = moveSnapshotsToCompositionArgs(refreshArgs, /*cursorOnly=*/true, 0); mCompositionEngine->updateCursorAsync(refreshArgs); + moveSnapshotsFromCompositionArgs(refreshArgs, layers); } void SurfaceFlinger::requestDisplayModes(std::vector<display::DisplayModeRequest> modeRequests) { @@ -3778,10 +3870,10 @@ void SurfaceFlinger::commitOffscreenLayers() { } } -void SurfaceFlinger::invalidateLayerStack(const sp<const Layer>& layer, const Region& dirty) { +void SurfaceFlinger::invalidateLayerStack(const ui::LayerFilter& layerFilter, const Region& dirty) { for (const auto& [token, displayDevice] : FTL_FAKE_GUARD(mStateLock, mDisplays)) { auto display = displayDevice->getCompositionDisplay(); - if (display->includesLayer(layer->getOutputFilter())) { + if (display->includesLayer(layerFilter)) { display->editState().dirtyRegion.orSelf(dirty); } } @@ -3901,6 +3993,7 @@ status_t SurfaceFlinger::addClientLayer(const LayerCreationArgs& args, const sp< { std::scoped_lock<std::mutex> lock(mCreatedLayersLock); mCreatedLayers.emplace_back(layer, parent, args.addToRoot); + mNewLayers.emplace_back(std::make_unique<frontend::RequestedLayerState>(args)); } setTransactionFlags(eTransactionNeeded); @@ -3971,16 +4064,30 @@ TransactionHandler::TransactionReadiness SurfaceFlinger::transactionReadyBufferC sp<Layer> layer = LayerHandle::getLayer(s.surface); const auto& transaction = *flushState.transaction; // check for barrier frames - if (s.bufferData->hasBarrier && - ((layer->getDrawingState().frameNumber) < s.bufferData->barrierFrameNumber)) { - const bool willApplyBarrierFrame = - flushState.bufferLayersReadyToPresent.contains(s.surface.get()) && - (flushState.bufferLayersReadyToPresent.get(s.surface.get()) >= - s.bufferData->barrierFrameNumber); - if (!willApplyBarrierFrame) { - ATRACE_NAME("NotReadyBarrier"); - ready = TransactionReadiness::NotReadyBarrier; - return false; + if (s.bufferData->hasBarrier) { + // The current producerId is already a newer producer than the buffer that has a + // barrier. This means the incoming buffer is older and we can release it here. We + // don't wait on the barrier since we know that's stale information. + if (layer->getDrawingState().producerId > s.bufferData->producerId) { + layer->callReleaseBufferCallback(s.bufferData->releaseBufferListener, + s.bufferData->buffer, s.bufferData->frameNumber, + s.bufferData->acquireFence); + // Delete the entire state at this point and not just release the buffer because + // everything associated with the Layer in this Transaction is now out of date. + ATRACE_NAME("DeleteStaleBuffer"); + return TraverseBuffersReturnValues::DELETE_AND_CONTINUE_TRAVERSAL; + } + + if (layer->getDrawingState().frameNumber < s.bufferData->barrierFrameNumber) { + const bool willApplyBarrierFrame = + flushState.bufferLayersReadyToPresent.contains(s.surface.get()) && + ((flushState.bufferLayersReadyToPresent.get(s.surface.get()) >= + s.bufferData->barrierFrameNumber)); + if (!willApplyBarrierFrame) { + ATRACE_NAME("NotReadyBarrier"); + ready = TransactionReadiness::NotReadyBarrier; + return TraverseBuffersReturnValues::STOP_TRAVERSAL; + } } } @@ -3991,7 +4098,7 @@ TransactionHandler::TransactionReadiness SurfaceFlinger::transactionReadyBufferC if (layer->backpressureEnabled() && hasPendingBuffer && transaction.isAutoTimestamp) { ATRACE_NAME("hasPendingBuffer"); ready = TransactionReadiness::NotReady; - return false; + return TraverseBuffersReturnValues::STOP_TRAVERSAL; } // check fence status @@ -4018,14 +4125,14 @@ TransactionHandler::TransactionReadiness SurfaceFlinger::transactionReadyBufferC "Buffer processing hung up due to stuck " "fence. Indicates GPU hang"); } - return false; + return TraverseBuffersReturnValues::STOP_TRAVERSAL; } ready = enableLatchUnsignaledConfig == LatchUnsignaledConfig::AutoSingleLayer ? TransactionReadiness::ReadyUnsignaledSingle : TransactionReadiness::ReadyUnsignaled; } - return true; + return TraverseBuffersReturnValues::CONTINUE_TRAVERSAL; }); ATRACE_INT("TransactionReadiness", static_cast<int>(ready)); return ready; @@ -4254,9 +4361,11 @@ bool SurfaceFlinger::applyTransactionState(const FrameTimelineInfo& frameTimelin const std::vector<ListenerCallbacks>& listenerCallbacks, int originPid, int originUid, uint64_t transactionId) { uint32_t transactionFlags = 0; - for (DisplayState& display : displays) { - display.sanitize(permissions); - transactionFlags |= setDisplayStateLocked(display); + if (!mLayerLifecycleManagerEnabled) { + for (DisplayState& display : displays) { + display.sanitize(permissions); + transactionFlags |= setDisplayStateLocked(display); + } } // start and end registration for listeners w/ no surface so they can get their callback. Note @@ -4268,9 +4377,16 @@ bool SurfaceFlinger::applyTransactionState(const FrameTimelineInfo& frameTimelin uint32_t clientStateFlags = 0; for (auto& resolvedState : states) { - clientStateFlags |= - setClientStateLocked(frameTimelineInfo, resolvedState, desiredPresentTime, - isAutoTimestamp, postTime, permissions, transactionId); + if (mLegacyFrontEndEnabled) { + clientStateFlags |= + setClientStateLocked(frameTimelineInfo, resolvedState, desiredPresentTime, + isAutoTimestamp, postTime, permissions, transactionId); + + } else /*mLayerLifecycleManagerEnabled*/ { + clientStateFlags |= updateLayerCallbacksAndStats(frameTimelineInfo, resolvedState, + desiredPresentTime, isAutoTimestamp, + postTime, permissions, transactionId); + } if ((flags & eAnimation) && resolvedState.state.surface) { if (const auto layer = LayerHandle::getLayer(resolvedState.state.surface)) { using LayerUpdateType = scheduler::LayerHistory::LayerUpdateType; @@ -4303,8 +4419,35 @@ bool SurfaceFlinger::applyTransactionState(const FrameTimelineInfo& frameTimelin bool needsTraversal = false; if (transactionFlags) { - // We are on the main thread, we are about to preform a traversal. Clear the traversal bit - // so we don't have to wake up again next frame to preform an unnecessary traversal. + // We are on the main thread, we are about to perform a traversal. Clear the traversal bit + // so we don't have to wake up again next frame to perform an unnecessary traversal. + if (transactionFlags & eTraversalNeeded) { + transactionFlags = transactionFlags & (~eTraversalNeeded); + needsTraversal = true; + } + if (transactionFlags) { + setTransactionFlags(transactionFlags); + } + } + + return needsTraversal; +} + +bool SurfaceFlinger::applyAndCommitDisplayTransactionStates( + std::vector<TransactionState>& transactions) { + Mutex::Autolock _l(mStateLock); + bool needsTraversal = false; + uint32_t transactionFlags = 0; + for (auto& transaction : transactions) { + for (DisplayState& display : transaction.displays) { + display.sanitize(transaction.permissions); + transactionFlags |= setDisplayStateLocked(display); + } + } + + if (transactionFlags) { + // We are on the main thread, we are about to perform a traversal. Clear the traversal bit + // so we don't have to wake up again next frame to perform an unnecessary traversal. if (transactionFlags & eTraversalNeeded) { transactionFlags = transactionFlags & (~eTraversalNeeded); needsTraversal = true; @@ -4314,6 +4457,15 @@ bool SurfaceFlinger::applyTransactionState(const FrameTimelineInfo& frameTimelin } } + mFrontEndDisplayInfosChanged = mTransactionFlags & eDisplayTransactionNeeded; + if (mFrontEndDisplayInfosChanged && !mLegacyFrontEndEnabled) { + processDisplayChangesLocked(); + mFrontEndDisplayInfos.clear(); + for (const auto& [_, display] : mDisplays) { + mFrontEndDisplayInfos.try_emplace(display->getLayerStack(), display->getFrontEndInfo()); + } + } + return needsTraversal; } @@ -4704,7 +4856,8 @@ uint32_t SurfaceFlinger::setClientStateLocked(const FrameTimelineInfo& frameTime // Do nothing. Processing the transaction completed listeners currently cause the flush. } - if (layer->setTransactionCompletedListeners(callbackHandles)) { + if (layer->setTransactionCompletedListeners(callbackHandles, + layer->willPresentCurrentTransaction())) { flags |= eTraversalNeeded; } @@ -4720,6 +4873,96 @@ uint32_t SurfaceFlinger::setClientStateLocked(const FrameTimelineInfo& frameTime return flags; } +uint32_t SurfaceFlinger::updateLayerCallbacksAndStats(const FrameTimelineInfo& frameTimelineInfo, + ResolvedComposerState& composerState, + int64_t desiredPresentTime, + bool isAutoTimestamp, int64_t postTime, + uint32_t permissions, + uint64_t transactionId) { + layer_state_t& s = composerState.state; + s.sanitize(permissions); + const nsecs_t latchTime = systemTime(); + bool unused; + + std::vector<ListenerCallbacks> filteredListeners; + for (auto& listener : s.listeners) { + // Starts a registration but separates the callback ids according to callback type. This + // allows the callback invoker to send on latch callbacks earlier. + // note that startRegistration will not re-register if the listener has + // already be registered for a prior surface control + + ListenerCallbacks onCommitCallbacks = listener.filter(CallbackId::Type::ON_COMMIT); + if (!onCommitCallbacks.callbackIds.empty()) { + filteredListeners.push_back(onCommitCallbacks); + } + + ListenerCallbacks onCompleteCallbacks = listener.filter(CallbackId::Type::ON_COMPLETE); + if (!onCompleteCallbacks.callbackIds.empty()) { + filteredListeners.push_back(onCompleteCallbacks); + } + } + + const uint64_t what = s.what; + uint32_t flags = 0; + sp<Layer> layer = nullptr; + if (s.surface) { + layer = LayerHandle::getLayer(s.surface); + } else { + // The client may provide us a null handle. Treat it as if the layer was removed. + ALOGW("Attempt to set client state with a null layer handle"); + } + if (layer == nullptr) { + for (auto& [listener, callbackIds] : s.listeners) { + mTransactionCallbackInvoker.addCallbackHandle(sp<CallbackHandle>::make(listener, + callbackIds, + s.surface), + std::vector<JankData>()); + } + return 0; + } + if (what & layer_state_t::eProducerDisconnect) { + layer->onDisconnect(); + } + std::optional<nsecs_t> dequeueBufferTimestamp; + if (what & layer_state_t::eMetadataChanged) { + dequeueBufferTimestamp = s.metadata.getInt64(gui::METADATA_DEQUEUE_TIME); + } + + std::vector<sp<CallbackHandle>> callbackHandles; + if ((what & layer_state_t::eHasListenerCallbacksChanged) && (!filteredListeners.empty())) { + for (auto& [listener, callbackIds] : filteredListeners) { + callbackHandles.emplace_back( + sp<CallbackHandle>::make(listener, callbackIds, s.surface)); + } + } + if (what & layer_state_t::eSidebandStreamChanged) { + if (layer->setSidebandStream(s.sidebandStream)) flags |= eTraversalNeeded; + } + if (what & layer_state_t::eBufferChanged) { + if (layer->setBuffer(composerState.externalTexture, *s.bufferData, postTime, + desiredPresentTime, isAutoTimestamp, dequeueBufferTimestamp, + frameTimelineInfo)) { + layer->latchBuffer(unused, latchTime); + flags |= eTraversalNeeded; + } + mLayersWithQueuedFrames.emplace(layer); + } else if (frameTimelineInfo.vsyncId != FrameTimelineInfo::INVALID_VSYNC_ID) { + layer->setFrameTimelineVsyncForBufferlessTransaction(frameTimelineInfo, postTime); + } + + if (what & layer_state_t::eTrustedPresentationInfoChanged) { + layer->setTrustedPresentationInfo(s.trustedPresentationThresholds, + s.trustedPresentationListener); + } + + const auto& snapshot = mLayerSnapshotBuilder.getSnapshot(layer->getSequence()); + bool willPresentCurrentTransaction = + snapshot && (snapshot->hasReadyFrame || snapshot->sidebandStreamHasFrame); + if (layer->setTransactionCompletedListeners(callbackHandles, willPresentCurrentTransaction)) + flags |= eTraversalNeeded; + return flags; +} + uint32_t SurfaceFlinger::addInputWindowCommands(const InputWindowCommands& inputWindowCommands) { bool hasChanges = mInputWindowCommands.merge(inputWindowCommands); return hasChanges ? eTraversalNeeded : 0; @@ -4788,6 +5031,7 @@ status_t SurfaceFlinger::mirrorDisplay(DisplayId displayId, const LayerCreationA LayerCreationArgs mirrorArgs(args); mirrorArgs.flags |= ISurfaceComposerClient::eNoColorFill; mirrorArgs.addToRoot = true; + mirrorArgs.layerStackToMirror = layerStack; result = createEffectLayer(mirrorArgs, &outResult.handle, &rootMirrorLayer); outResult.layerId = rootMirrorLayer->sequence; outResult.layerName = String16(rootMirrorLayer->getDebugName()); @@ -4890,7 +5134,12 @@ void SurfaceFlinger::markLayerPendingRemovalLocked(const sp<Layer>& layer) { setTransactionFlags(eTransactionNeeded); } -void SurfaceFlinger::onHandleDestroyed(BBinder* handle, sp<Layer>& layer, uint32_t /* layerId */) { +void SurfaceFlinger::onHandleDestroyed(BBinder* handle, sp<Layer>& layer, uint32_t layerId) { + { + std::scoped_lock<std::mutex> lock(mCreatedLayersLock); + mDestroyedHandles.emplace_back(layerId); + } + Mutex::Autolock lock(mStateLock); markLayerPendingRemovalLocked(layer); mBufferCountTracker.remove(handle); @@ -4898,6 +5147,8 @@ void SurfaceFlinger::onHandleDestroyed(BBinder* handle, sp<Layer>& layer, uint32 if (mTransactionTracing) { mTransactionTracing->onHandleRemoved(handle); } + + setTransactionFlags(eTransactionFlushNeeded); } void SurfaceFlinger::onInitializeDisplays() { @@ -4998,7 +5249,8 @@ void SurfaceFlinger::setPowerModeInternal(const sp<DisplayDevice>& display, hal: } getHwComposer().setPowerMode(displayId, mode); if (isActiveDisplay && mode != hal::PowerMode::DOZE_SUSPEND) { - setHWCVsyncEnabled(displayId, mHWCVsyncPendingState); + setHWCVsyncEnabled(displayId, + mScheduler->getVsyncSchedule().getPendingHardwareVsyncState()); mScheduler->onScreenAcquired(mAppConnectionHandle); mScheduler->resyncToHardwareVsync(true, refreshRate); } @@ -5019,7 +5271,7 @@ void SurfaceFlinger::setPowerModeInternal(const sp<DisplayDevice>& display, hal: } // Make sure HWVsync is disabled before turning off the display - setHWCVsyncEnabled(displayId, hal::Vsync::DISABLE); + setHWCVsyncEnabled(displayId, false); getHwComposer().setPowerMode(displayId, mode); mVisibleRegionsDirty = true; @@ -5228,14 +5480,6 @@ void SurfaceFlinger::dumpScheduler(std::string& result) const { mScheduler->dump(dumper); - // TODO(b/241286146): Move to Scheduler. - { - utils::Dumper::Indent indent(dumper); - dumper.dump("lastHwcVsyncState"sv, mLastHWCVsyncState); - dumper.dump("pendingHwcVsyncState"sv, mHWCVsyncPendingState); - } - dumper.eol(); - // TODO(b/241285876): Move to DisplayModeController. dumper.dump("debugDisplayModeSetByBackdoor"sv, mDebugDisplayModeSetByBackdoor); dumper.eol(); @@ -6467,10 +6711,15 @@ status_t SurfaceFlinger::captureDisplay(const DisplayCaptureArgs& args, args.useIdentityTransform, args.captureSecureLayers); }); - auto traverseLayers = [this, args, layerStack](const LayerVector::Visitor& visitor) { - traverseLayersInLayerStack(layerStack, args.uid, visitor); - }; - auto getLayerSnapshots = RenderArea::fromTraverseLayersLambda(traverseLayers); + GetLayerSnapshotsFunction getLayerSnapshots; + if (mLayerLifecycleManagerEnabled) { + getLayerSnapshots = getLayerSnapshotsForScreenshots(layerStack, args.uid); + } else { + auto traverseLayers = [this, args, layerStack](const LayerVector::Visitor& visitor) { + traverseLayersInLayerStack(layerStack, args.uid, visitor); + }; + getLayerSnapshots = RenderArea::fromTraverseLayersLambda(traverseLayers); + } auto future = captureScreenCommon(std::move(renderAreaFuture), getLayerSnapshots, reqSize, args.pixelFormat, args.allowProtected, args.grayscale, @@ -6504,10 +6753,15 @@ status_t SurfaceFlinger::captureDisplay(DisplayId displayId, false /* captureSecureLayers */); }); - auto traverseLayers = [this, layerStack](const LayerVector::Visitor& visitor) { - traverseLayersInLayerStack(layerStack, CaptureArgs::UNSET_UID, visitor); - }; - auto getLayerSnapshots = RenderArea::fromTraverseLayersLambda(traverseLayers); + GetLayerSnapshotsFunction getLayerSnapshots; + if (mLayerLifecycleManagerEnabled) { + getLayerSnapshots = getLayerSnapshotsForScreenshots(layerStack, CaptureArgs::UNSET_UID); + } else { + auto traverseLayers = [this, layerStack](const LayerVector::Visitor& visitor) { + traverseLayersInLayerStack(layerStack, CaptureArgs::UNSET_UID, visitor); + }; + getLayerSnapshots = RenderArea::fromTraverseLayersLambda(traverseLayers); + } if (captureListener == nullptr) { ALOGE("capture screen must provide a capture listener callback"); @@ -6602,29 +6856,37 @@ status_t SurfaceFlinger::captureLayers(const LayerCaptureArgs& args, return std::make_unique<LayerRenderArea>(*this, parent, crop, reqSize, dataspace, childrenOnly, args.captureSecureLayers); }); - - auto traverseLayers = [parent, args, excludeLayerIds](const LayerVector::Visitor& visitor) { - parent->traverseChildrenInZOrder(LayerVector::StateSet::Drawing, [&](Layer* layer) { - if (!layer->isVisible()) { - return; - } else if (args.childrenOnly && layer == parent.get()) { - return; - } else if (args.uid != CaptureArgs::UNSET_UID && args.uid != layer->getOwnerUid()) { - return; - } - - auto p = sp<Layer>::fromExisting(layer); - while (p != nullptr) { - if (excludeLayerIds.count(p->sequence) != 0) { + GetLayerSnapshotsFunction getLayerSnapshots; + if (mLayerLifecycleManagerEnabled) { + FloatRect parentCrop = crop.isEmpty() ? FloatRect(0, 0, reqSize.width, reqSize.height) + : crop.toFloatRect(); + getLayerSnapshots = getLayerSnapshotsForScreenshots(parent->sequence, args.uid, + std::move(excludeLayerIds), + args.childrenOnly, parentCrop); + } else { + auto traverseLayers = [parent, args, excludeLayerIds](const LayerVector::Visitor& visitor) { + parent->traverseChildrenInZOrder(LayerVector::StateSet::Drawing, [&](Layer* layer) { + if (!layer->isVisible()) { + return; + } else if (args.childrenOnly && layer == parent.get()) { + return; + } else if (args.uid != CaptureArgs::UNSET_UID && args.uid != layer->getOwnerUid()) { return; } - p = p->getParent(); - } - visitor(layer); - }); - }; - auto getLayerSnapshots = RenderArea::fromTraverseLayersLambda(traverseLayers); + auto p = sp<Layer>::fromExisting(layer); + while (p != nullptr) { + if (excludeLayerIds.count(p->sequence) != 0) { + return; + } + p = p->getParent(); + } + + visitor(layer); + }); + }; + getLayerSnapshots = RenderArea::fromTraverseLayersLambda(traverseLayers); + } if (captureListener == nullptr) { ALOGE("capture screen must provide a capture listener callback"); @@ -7406,24 +7668,18 @@ bool SurfaceFlinger::commitMirrorDisplays(VsyncId vsyncId) { return true; } -bool SurfaceFlinger::commitCreatedLayers(VsyncId vsyncId) { - std::vector<LayerCreatedState> createdLayers; - { - std::scoped_lock<std::mutex> lock(mCreatedLayersLock); - createdLayers = std::move(mCreatedLayers); - mCreatedLayers.clear(); - if (createdLayers.size() == 0) { - return false; - } +bool SurfaceFlinger::commitCreatedLayers(VsyncId vsyncId, + std::vector<LayerCreatedState>& createdLayers) { + if (createdLayers.size() == 0) { + return false; } Mutex::Autolock _l(mStateLock); for (const auto& createdLayer : createdLayers) { handleLayerCreatedLocked(createdLayer, vsyncId); } - createdLayers.clear(); mLayersAdded = true; - return true; + return mLayersAdded; } void SurfaceFlinger::updateLayerMetadataSnapshot() { @@ -7451,6 +7707,150 @@ void SurfaceFlinger::updateLayerMetadataSnapshot() { }); } +void SurfaceFlinger::moveSnapshotsFromCompositionArgs( + compositionengine::CompositionRefreshArgs& refreshArgs, + std::vector<std::pair<Layer*, LayerFE*>>& layers) { + if (mLayerLifecycleManagerEnabled) { + std::vector<std::unique_ptr<frontend::LayerSnapshot>>& snapshots = + mLayerSnapshotBuilder.getSnapshots(); + for (auto [_, layerFE] : layers) { + auto i = layerFE->mSnapshot->globalZ; + snapshots[i] = std::move(layerFE->mSnapshot); + } + } + if (mLegacyFrontEndEnabled && !mLayerLifecycleManagerEnabled) { + for (auto [layer, layerFE] : layers) { + layer->updateLayerSnapshot(std::move(layerFE->mSnapshot)); + } + } +} + +std::vector<std::pair<Layer*, LayerFE*>> SurfaceFlinger::moveSnapshotsToCompositionArgs( + compositionengine::CompositionRefreshArgs& refreshArgs, bool cursorOnly, int64_t vsyncId) { + std::vector<std::pair<Layer*, LayerFE*>> layers; + if (mLayerLifecycleManagerEnabled) { + mLayerSnapshotBuilder.forEachVisibleSnapshot( + [&](std::unique_ptr<frontend::LayerSnapshot>& snapshot) { + if (cursorOnly && + snapshot->compositionType != + aidl::android::hardware::graphics::composer3::Composition::CURSOR) { + return; + } + + if (!snapshot->hasSomethingToDraw()) { + return; + } + + auto it = mLegacyLayers.find(snapshot->sequence); + LOG_ALWAYS_FATAL_IF(it == mLegacyLayers.end(), + "Couldnt find layer object for %s", + snapshot->getDebugString().c_str()); + auto& legacyLayer = it->second; + sp<LayerFE> layerFE = legacyLayer->getCompositionEngineLayerFE(snapshot->path); + layerFE->mSnapshot = std::move(snapshot); + refreshArgs.layers.push_back(layerFE); + layers.emplace_back(legacyLayer.get(), layerFE.get()); + }); + } + if (mLegacyFrontEndEnabled && !mLayerLifecycleManagerEnabled) { + mDrawingState.traverseInZOrder([&refreshArgs, cursorOnly, &layers](Layer* layer) { + if (const auto& layerFE = layer->getCompositionEngineLayerFE()) { + if (cursorOnly && + layer->getLayerSnapshot()->compositionType != + aidl::android::hardware::graphics::composer3::Composition::CURSOR) + return; + layer->updateSnapshot(refreshArgs.updatingGeometryThisFrame); + layerFE->mSnapshot = layer->stealLayerSnapshot(); + refreshArgs.layers.push_back(layerFE); + layers.emplace_back(layer, layerFE.get()); + } + }); + } + + return layers; +} + +std::function<std::vector<std::pair<Layer*, sp<LayerFE>>>()> +SurfaceFlinger::getLayerSnapshotsForScreenshots(std::optional<ui::LayerStack> layerStack, + uint32_t uid) { + return [this, layerStack, uid]() { + std::vector<std::pair<Layer*, sp<LayerFE>>> layers; + for (auto& snapshot : mLayerSnapshotBuilder.getSnapshots()) { + if (layerStack && snapshot->outputFilter.layerStack != *layerStack) { + continue; + } + if (uid != CaptureArgs::UNSET_UID && snapshot->inputInfo.ownerUid != uid) { + continue; + } + if (!snapshot->isVisible || !snapshot->hasSomethingToDraw()) { + continue; + } + + auto it = mLegacyLayers.find(snapshot->sequence); + LOG_ALWAYS_FATAL_IF(it == mLegacyLayers.end(), "Couldnt find layer object for %s", + snapshot->getDebugString().c_str()); + auto& legacyLayer = it->second; + sp<LayerFE> layerFE = getFactory().createLayerFE(legacyLayer->getName()); + layerFE->mSnapshot = std::make_unique<frontend::LayerSnapshot>(*snapshot); + layers.emplace_back(legacyLayer.get(), std::move(layerFE)); + } + + return layers; + }; +} + +std::function<std::vector<std::pair<Layer*, sp<LayerFE>>>()> +SurfaceFlinger::getLayerSnapshotsForScreenshots(uint32_t rootLayerId, uint32_t uid, + std::unordered_set<uint32_t> excludeLayerIds, + bool childrenOnly, const FloatRect& parentCrop) { + return [this, excludeLayerIds = std::move(excludeLayerIds), uid, rootLayerId, childrenOnly, + parentCrop]() { + frontend::LayerSnapshotBuilder::Args + args{.root = mLayerHierarchyBuilder.getPartialHierarchy(rootLayerId, childrenOnly), + .layerLifecycleManager = mLayerLifecycleManager, + .displays = mFrontEndDisplayInfos, + .displayChanges = true, + .globalShadowSettings = mDrawingState.globalShadowSettings, + .supportsBlur = mSupportsBlur, + .forceFullDamage = mForceFullDamage, + .parentCrop = {parentCrop}, + .excludeLayerIds = std::move(excludeLayerIds)}; + mLayerSnapshotBuilder.update(args); + + auto getLayerSnapshotsFn = getLayerSnapshotsForScreenshots({}, uid); + std::vector<std::pair<Layer*, sp<LayerFE>>> layers = getLayerSnapshotsFn(); + args.root = mLayerHierarchyBuilder.getHierarchy(); + args.parentCrop.reset(); + args.excludeLayerIds.clear(); + mLayerSnapshotBuilder.update(args); + return layers; + }; +} + +SurfaceFlinger::LifecycleUpdate SurfaceFlinger::flushLifecycleUpdates() { + LifecycleUpdate update; + ATRACE_NAME("TransactionHandler:flushTransactions"); + // Locking: + // 1. to prevent onHandleDestroyed from being called while the state lock is held, + // we must keep a copy of the transactions (specifically the composer + // states) around outside the scope of the lock. + // 2. Transactions and created layers do not share a lock. To prevent applying + // transactions with layers still in the createdLayer queue, flush the transactions + // before committing the created layers. + update.transactions = mTransactionHandler.flushTransactions(); + { + // TODO(b/238781169) lockless queue this and keep order. + std::scoped_lock<std::mutex> lock(mCreatedLayersLock); + update.layerCreatedStates = std::move(mCreatedLayers); + mCreatedLayers.clear(); + update.newLayers = std::move(mNewLayers); + mNewLayers.clear(); + update.destroyedHandles = std::move(mDestroyedHandles); + mDestroyedHandles.clear(); + } + return update; +} + // gui::ISurfaceComposer binder::Status SurfaceComposerAIDL::bootFinished() { diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h index 97469f4253..aded52a8c9 100644 --- a/services/surfaceflinger/SurfaceFlinger.h +++ b/services/surfaceflinger/SurfaceFlinger.h @@ -71,9 +71,12 @@ #include "FlagManager.h" #include "FrontEnd/DisplayInfo.h" #include "FrontEnd/LayerCreationArgs.h" +#include "FrontEnd/LayerLifecycleManager.h" #include "FrontEnd/LayerSnapshot.h" +#include "FrontEnd/LayerSnapshotBuilder.h" #include "FrontEnd/TransactionHandler.h" #include "LayerVector.h" +#include "Scheduler/ISchedulerCallback.h" #include "Scheduler/RefreshRateSelector.h" #include "Scheduler/RefreshRateStats.h" #include "Scheduler/Scheduler.h" @@ -102,6 +105,7 @@ #include <vector> #include <aidl/android/hardware/graphics/common/DisplayDecorationSupport.h> +#include <aidl/android/hardware/graphics/composer3/RefreshRateChangedDebugData.h> #include "Client.h" using namespace android::surfaceflinger; @@ -125,6 +129,7 @@ class FrameTracer; class ScreenCapturer; class WindowInfosListenerInvoker; +using ::aidl::android::hardware::graphics::composer3::RefreshRateChangedDebugData; using frontend::TransactionHandler; using gui::CaptureArgs; using gui::DisplayCaptureArgs; @@ -449,6 +454,26 @@ private: FINISHED, }; + struct LayerCreatedState { + LayerCreatedState(const wp<Layer>& layer, const wp<Layer>& parent, bool addToRoot) + : layer(layer), initialParent(parent), addToRoot(addToRoot) {} + wp<Layer> layer; + // Indicates the initial parent of the created layer, only used for creating layer in + // SurfaceFlinger. If nullptr, it may add the created layer into the current root layers. + wp<Layer> initialParent; + // Indicates whether the layer getting created should be added at root if there's no parent + // and has permission ACCESS_SURFACE_FLINGER. If set to false and no parent, the layer will + // be added offscreen. + bool addToRoot; + }; + + struct LifecycleUpdate { + std::vector<TransactionState> transactions; + std::vector<LayerCreatedState> layerCreatedStates; + std::vector<std::unique_ptr<frontend::RequestedLayerState>> newLayers; + std::vector<uint32_t> destroyedHandles; + }; + template <typename F, std::enable_if_t<!std::is_member_function_pointer_v<F>>* = nullptr> static Dumper dumper(F&& dump) { using namespace std::placeholders; @@ -608,6 +633,7 @@ private: const hal::VsyncPeriodChangeTimeline&) override; void onComposerHalSeamlessPossible(hal::HWDisplayId) override; void onComposerHalVsyncIdle(hal::HWDisplayId) override; + void onRefreshRateChangedDebug(const RefreshRateChangedDebugData&) override; // ICompositor overrides: void configure() override; @@ -688,6 +714,17 @@ private: void updateLayerGeometry(); void updateLayerMetadataSnapshot(); + std::vector<std::pair<Layer*, LayerFE*>> moveSnapshotsToCompositionArgs( + compositionengine::CompositionRefreshArgs& refreshArgs, bool cursorOnly, + int64_t vsyncId); + void moveSnapshotsFromCompositionArgs(compositionengine::CompositionRefreshArgs& refreshArgs, + std::vector<std::pair<Layer*, LayerFE*>>& layers); + bool updateLayerSnapshotsLegacy(VsyncId vsyncId, LifecycleUpdate& update, + bool transactionsFlushed, bool& out) + REQUIRES(kMainThreadContext); + bool updateLayerSnapshots(VsyncId vsyncId, LifecycleUpdate& update, bool transactionsFlushed, + bool& out) REQUIRES(kMainThreadContext); + LifecycleUpdate flushLifecycleUpdates() REQUIRES(kMainThreadContext); void updateInputFlinger(); void persistDisplayBrightness(bool needsComposite) REQUIRES(kMainThreadContext); @@ -715,6 +752,8 @@ private: bool flushTransactionQueues(VsyncId) REQUIRES(kMainThreadContext); bool applyTransactions(std::vector<TransactionState>&, VsyncId) REQUIRES(kMainThreadContext); + bool applyAndCommitDisplayTransactionStates(std::vector<TransactionState>& transactions) + REQUIRES(kMainThreadContext); // Returns true if there is at least one transaction that needs to be flushed bool transactionFlushNeeded(); @@ -730,7 +769,10 @@ private: int64_t desiredPresentTime, bool isAutoTimestamp, int64_t postTime, uint32_t permissions, uint64_t transactionId) REQUIRES(mStateLock); - + uint32_t updateLayerCallbacksAndStats(const FrameTimelineInfo&, ResolvedComposerState&, + int64_t desiredPresentTime, bool isAutoTimestamp, + int64_t postTime, uint32_t permissions, + uint64_t transactionId) REQUIRES(mStateLock); uint32_t getTransactionFlags() const; // Sets the masked bits, and schedules a commit if needed. @@ -888,7 +930,7 @@ private: // mark a region of a layer stack dirty. this updates the dirty // region of all screens presenting this layer stack. - void invalidateLayerStack(const sp<const Layer>& layer, const Region& dirty); + void invalidateLayerStack(const ui::LayerFilter& layerFilter, const Region& dirty); ui::LayerFilter makeLayerFilterForDisplay(DisplayId displayId, ui::LayerStack layerStack) REQUIRES(mStateLock) { @@ -954,9 +996,9 @@ private: */ nsecs_t getVsyncPeriodFromHWC() const REQUIRES(mStateLock); - void setHWCVsyncEnabled(PhysicalDisplayId id, hal::Vsync enabled) { - mLastHWCVsyncState = enabled; - getHwComposer().setVsyncEnabled(id, enabled); + void setHWCVsyncEnabled(PhysicalDisplayId id, bool enabled) { + hal::Vsync halState = enabled ? hal::Vsync::ENABLE : hal::Vsync::DISABLE; + getHwComposer().setVsyncEnabled(id, halState); } using FenceTimePtr = std::shared_ptr<FenceTime>; @@ -1146,6 +1188,7 @@ private: // Set if LayerMetadata has changed since the last LayerMetadata snapshot. bool mLayerMetadataSnapshotNeeded = false; + // TODO(b/238781169) validate these on composition // Tracks layers that have pending frames which are candidates for being // latched. std::unordered_set<sp<Layer>, SpHash<Layer>> mLayersWithQueuedFrames; @@ -1282,9 +1325,6 @@ private: TimePoint mScheduledPresentTime GUARDED_BY(kMainThreadContext); TimePoint mExpectedPresentTime GUARDED_BY(kMainThreadContext); - hal::Vsync mHWCVsyncPendingState = hal::Vsync::DISABLE; - hal::Vsync mLastHWCVsyncState = hal::Vsync::DISABLE; - // below flags are set by main thread only bool mSetActiveModePending = false; @@ -1320,23 +1360,11 @@ private: GUARDED_BY(mStateLock); mutable std::mutex mCreatedLayersLock; - struct LayerCreatedState { - LayerCreatedState(const wp<Layer>& layer, const wp<Layer> parent, bool addToRoot) - : layer(layer), initialParent(parent), addToRoot(addToRoot) {} - wp<Layer> layer; - // Indicates the initial parent of the created layer, only used for creating layer in - // SurfaceFlinger. If nullptr, it may add the created layer into the current root layers. - wp<Layer> initialParent; - // Indicates whether the layer getting created should be added at root if there's no parent - // and has permission ACCESS_SURFACE_FLINGER. If set to false and no parent, the layer will - // be added offscreen. - bool addToRoot; - }; // A temporay pool that store the created layers and will be added to current state in main // thread. std::vector<LayerCreatedState> mCreatedLayers GUARDED_BY(mCreatedLayersLock); - bool commitCreatedLayers(VsyncId); + bool commitCreatedLayers(VsyncId, std::vector<LayerCreatedState>& createdLayers); void handleLayerCreatedLocked(const LayerCreatedState&, VsyncId) REQUIRES(mStateLock); mutable std::mutex mMirrorDisplayLock; @@ -1358,6 +1386,11 @@ private: return hasDisplay( [](const auto& display) { return display.isRefreshRateOverlayEnabled(); }); } + std::function<std::vector<std::pair<Layer*, sp<LayerFE>>>()> getLayerSnapshotsForScreenshots( + std::optional<ui::LayerStack> layerStack, uint32_t uid); + std::function<std::vector<std::pair<Layer*, sp<LayerFE>>>()> getLayerSnapshotsForScreenshots( + uint32_t rootLayerId, uint32_t uid, std::unordered_set<uint32_t> excludeLayerIds, + bool childrenOnly, const FloatRect& parentCrop); const sp<WindowInfosListenerInvoker> mWindowInfosListenerInvoker; @@ -1370,6 +1403,18 @@ private: bool mPowerHintSessionEnabled; + bool mLayerLifecycleManagerEnabled = false; + bool mLegacyFrontEndEnabled = true; + + frontend::LayerLifecycleManager mLayerLifecycleManager; + frontend::LayerHierarchyBuilder mLayerHierarchyBuilder{{}}; + frontend::LayerSnapshotBuilder mLayerSnapshotBuilder; + + std::vector<uint32_t> mDestroyedHandles; + std::vector<std::unique_ptr<frontend::RequestedLayerState>> mNewLayers; + // These classes do not store any client state but help with managing transaction callbacks + // and stats. + std::unordered_map<uint32_t, sp<Layer>> mLegacyLayers; struct { bool late = false; bool early = false; @@ -1377,6 +1422,7 @@ private: TransactionHandler mTransactionHandler; display::DisplayMap<ui::LayerStack, frontend::DisplayInfo> mFrontEndDisplayInfos; + bool mFrontEndDisplayInfosChanged = false; }; class SurfaceComposerAIDL : public gui::BnSurfaceComposer { diff --git a/services/surfaceflinger/TransactionState.h b/services/surfaceflinger/TransactionState.h index 5025c4935c..6c5a8b213d 100644 --- a/services/surfaceflinger/TransactionState.h +++ b/services/surfaceflinger/TransactionState.h @@ -27,6 +27,12 @@ namespace android { +enum TraverseBuffersReturnValues { + CONTINUE_TRAVERSAL, + STOP_TRAVERSAL, + DELETE_AND_CONTINUE_TRAVERSAL, +}; + // Extends the client side composer state by resolving buffer. class ResolvedComposerState : public ComposerState { public: @@ -75,12 +81,18 @@ struct TransactionState { } template <typename Visitor> - void traverseStatesWithBuffersWhileTrue(Visitor&& visitor) const { - for (const auto& state : states) { - if (state.state.hasBufferChanges() && state.state.hasValidBuffer() && - state.state.surface) { - if (!visitor(state.state)) return; + void traverseStatesWithBuffersWhileTrue(Visitor&& visitor) { + for (auto state = states.begin(); state != states.end();) { + if (state->state.hasBufferChanges() && state->state.hasValidBuffer() && + state->state.surface) { + int result = visitor(state->state); + if (result == STOP_TRAVERSAL) return; + if (result == DELETE_AND_CONTINUE_TRAVERSAL) { + state = states.erase(state); + continue; + } } + state++; } } diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_displayhardware_fuzzer_utils.h b/services/surfaceflinger/fuzzer/surfaceflinger_displayhardware_fuzzer_utils.h index 6a6e3db733..1a951b34ac 100644 --- a/services/surfaceflinger/fuzzer/surfaceflinger_displayhardware_fuzzer_utils.h +++ b/services/surfaceflinger/fuzzer/surfaceflinger_displayhardware_fuzzer_utils.h @@ -41,6 +41,7 @@ class SurfaceComposerClient; namespace android::hardware::graphics::composer::hal { +using aidl::android::hardware::graphics::composer3::RefreshRateChangedDebugData; using ::android::hardware::Return; using ::android::hardware::Void; using ::android::HWC2::ComposerCallback; @@ -99,6 +100,7 @@ struct TestHWC2ComposerCallback : public HWC2::ComposerCallback { void onComposerHalVsyncPeriodTimingChanged(HWDisplayId, const VsyncPeriodChangeTimeline&) {} void onComposerHalSeamlessPossible(HWDisplayId) {} void onComposerHalVsyncIdle(HWDisplayId) {} + void onRefreshRateChangedDebug(const RefreshRateChangedDebugData&) {} }; } // namespace android::hardware::graphics::composer::hal diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h b/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h index cdffbb4724..609fd33a3f 100644 --- a/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h +++ b/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h @@ -234,7 +234,8 @@ public: std::shared_ptr<RefreshRateSelector> selectorPtr, sp<VsyncModulator> modulatorPtr, ISchedulerCallback& callback) : Scheduler(*this, callback, Feature::kContentDetection, std::move(modulatorPtr)) { - mVsyncSchedule.emplace(VsyncSchedule(std::move(tracker), nullptr, std::move(controller))); + mVsyncSchedule = std::unique_ptr<VsyncSchedule>( + new VsyncSchedule(std::move(tracker), nullptr, std::move(controller))); const auto displayId = selectorPtr->getActiveMode().modePtr->getPhysicalDisplayId(); registerDisplay(displayId, std::move(selectorPtr)); @@ -244,9 +245,6 @@ public: return Scheduler::createConnection(std::move(eventThread)); } - auto &mutablePrimaryHWVsyncEnabled() { return mPrimaryHWVsyncEnabled; } - auto &mutableHWVsyncAvailable() { return mHWVsyncAvailable; } - auto &mutableLayerHistory() { return mLayerHistory; } auto refreshRateSelector() { return leaderSelectorPtr(); } diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_layer_fuzzer.cpp b/services/surfaceflinger/fuzzer/surfaceflinger_layer_fuzzer.cpp index 11719c435e..c088e7b40c 100644 --- a/services/surfaceflinger/fuzzer/surfaceflinger_layer_fuzzer.cpp +++ b/services/surfaceflinger/fuzzer/surfaceflinger_layer_fuzzer.cpp @@ -148,7 +148,7 @@ void LayerFuzzer::invokeBufferStateLayer() { layer->fenceHasSignaled(); layer->onPreComposition(mFdp.ConsumeIntegral<int64_t>()); const std::vector<sp<CallbackHandle>> callbacks; - layer->setTransactionCompletedListeners(callbacks); + layer->setTransactionCompletedListeners(callbacks, mFdp.ConsumeBool()); std::shared_ptr<renderengine::ExternalTexture> texture = std::make_shared< renderengine::mock::FakeExternalTexture>(mFdp.ConsumeIntegral<uint32_t>(), diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_scheduler_fuzzer.cpp b/services/surfaceflinger/fuzzer/surfaceflinger_scheduler_fuzzer.cpp index 44805dba82..61fb29a964 100644 --- a/services/surfaceflinger/fuzzer/surfaceflinger_scheduler_fuzzer.cpp +++ b/services/surfaceflinger/fuzzer/surfaceflinger_scheduler_fuzzer.cpp @@ -76,7 +76,7 @@ private: FuzzedDataProvider mFdp; - std::optional<scheduler::VsyncSchedule> mVsyncSchedule; + std::unique_ptr<scheduler::VsyncSchedule> mVsyncSchedule; }; PhysicalDisplayId SchedulerFuzzer::getPhysicalDisplayId() { @@ -90,9 +90,9 @@ PhysicalDisplayId SchedulerFuzzer::getPhysicalDisplayId() { } void SchedulerFuzzer::fuzzEventThread() { - mVsyncSchedule.emplace(scheduler::VsyncSchedule(std::make_unique<mock::VSyncTracker>(), - std::make_unique<mock::VSyncDispatch>(), - nullptr)); + mVsyncSchedule = std::unique_ptr<scheduler::VsyncSchedule>( + new scheduler::VsyncSchedule(std::make_unique<mock::VSyncTracker>(), + std::make_unique<mock::VSyncDispatch>(), nullptr)); const auto getVsyncPeriod = [](uid_t /* uid */) { return kSyncPeriod.count(); }; std::unique_ptr<android::impl::EventThread> thread = std::make_unique< android::impl::EventThread>("fuzzer", *mVsyncSchedule, nullptr, nullptr, getVsyncPeriod, diff --git a/services/surfaceflinger/main_surfaceflinger.cpp b/services/surfaceflinger/main_surfaceflinger.cpp index fedd71e921..049567878f 100644 --- a/services/surfaceflinger/main_surfaceflinger.cpp +++ b/services/surfaceflinger/main_surfaceflinger.cpp @@ -139,11 +139,6 @@ int main(int, char**) { set_sched_policy(0, SP_FOREGROUND); - // Put most SurfaceFlinger threads in the system-background cpuset - // Keeps us from unnecessarily using big cores - // Do this after the binder thread pool init - if (cpusets_enabled()) set_cpuset_policy(0, SP_SYSTEM); - // initialize before clients can connect flinger->init(); diff --git a/services/surfaceflinger/tests/ReleaseBufferCallback_test.cpp b/services/surfaceflinger/tests/ReleaseBufferCallback_test.cpp index 16076eaac9..c23fb9bd23 100644 --- a/services/surfaceflinger/tests/ReleaseBufferCallback_test.cpp +++ b/services/surfaceflinger/tests/ReleaseBufferCallback_test.cpp @@ -85,7 +85,8 @@ public: sp<Fence> fence, CallbackHelper& callback, const ReleaseCallbackId& id, ReleaseBufferCallbackHelper& releaseCallback) { Transaction t; - t.setBuffer(layer, buffer, fence, id.framenumber, releaseCallback.getCallback()); + t.setBuffer(layer, buffer, fence, id.framenumber, 0 /* producerId */, + releaseCallback.getCallback()); t.addTransactionCompletedCallback(callback.function, callback.getContext()); t.apply(); } @@ -301,7 +302,7 @@ TEST_F(ReleaseBufferCallbackTest, DISABLED_FrameDropping) { Transaction t; t.setBuffer(layer, firstBuffer, std::nullopt, firstBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); t.addTransactionCompletedCallback(transactionCallback.function, transactionCallback.getContext()); t.setDesiredPresentTime(time); @@ -317,7 +318,7 @@ TEST_F(ReleaseBufferCallbackTest, DISABLED_FrameDropping) { sp<GraphicBuffer> secondBuffer = getBuffer(); ReleaseCallbackId secondBufferCallbackId(secondBuffer->getId(), generateFrameNumber()); t.setBuffer(layer, secondBuffer, std::nullopt, secondBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); t.addTransactionCompletedCallback(transactionCallback.function, transactionCallback.getContext()); t.setDesiredPresentTime(time); @@ -362,7 +363,7 @@ TEST_F(ReleaseBufferCallbackTest, DISABLED_Merge_Different_Processes) { Transaction transaction1; transaction1.setBuffer(layer, secondBuffer, std::nullopt, secondBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); transaction1.addTransactionCompletedCallback(callback1.function, callback1.getContext()); // Set a different TransactionCompletedListener to mimic a second process @@ -397,14 +398,14 @@ TEST_F(ReleaseBufferCallbackTest, DISABLED_SetBuffer_OverwriteBuffers) { // Create transaction with a buffer. Transaction transaction; transaction.setBuffer(layer, firstBuffer, std::nullopt, firstBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); sp<GraphicBuffer> secondBuffer = getBuffer(); ReleaseCallbackId secondBufferCallbackId(secondBuffer->getId(), generateFrameNumber()); // Call setBuffer on the same transaction with a different buffer. transaction.setBuffer(layer, secondBuffer, std::nullopt, secondBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); ASSERT_NO_FATAL_FAILURE(waitForReleaseBufferCallback(*releaseCallback, firstBufferCallbackId)); } @@ -419,7 +420,7 @@ TEST_F(ReleaseBufferCallbackTest, DISABLED_Merge_Transactions_OverwriteBuffers) // Create transaction with a buffer. Transaction transaction1; transaction1.setBuffer(layer, firstBuffer, std::nullopt, firstBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); sp<GraphicBuffer> secondBuffer = getBuffer(); ReleaseCallbackId secondBufferCallbackId(secondBuffer->getId(), generateFrameNumber()); @@ -427,7 +428,7 @@ TEST_F(ReleaseBufferCallbackTest, DISABLED_Merge_Transactions_OverwriteBuffers) // Create a second transaction with a new buffer for the same layer. Transaction transaction2; transaction2.setBuffer(layer, secondBuffer, std::nullopt, secondBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); // merge transaction1 into transaction2 so ensure we get a proper buffer release callback. transaction1.merge(std::move(transaction2)); @@ -450,7 +451,7 @@ TEST_F(ReleaseBufferCallbackTest, DISABLED_MergeBuffers_Different_Processes) { Transaction transaction1; transaction1.setBuffer(layer, firstBuffer, std::nullopt, firstBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); // Sent a second buffer to allow the first buffer to get released. sp<GraphicBuffer> secondBuffer = getBuffer(); @@ -458,7 +459,7 @@ TEST_F(ReleaseBufferCallbackTest, DISABLED_MergeBuffers_Different_Processes) { Transaction transaction2; transaction2.setBuffer(layer, secondBuffer, std::nullopt, secondBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); // Set a different TransactionCompletedListener to mimic a second process TransactionCompletedListener::setInstance(secondCompletedListener); @@ -479,10 +480,11 @@ TEST_F(ReleaseBufferCallbackTest, SetBuffer_OverwriteBuffersWithNull) { // Create transaction with a buffer. Transaction transaction; transaction.setBuffer(layer, firstBuffer, std::nullopt, firstBufferCallbackId.framenumber, - releaseCallback->getCallback()); + 0 /* producerId */, releaseCallback->getCallback()); // Call setBuffer on the same transaction with a null buffer. - transaction.setBuffer(layer, nullptr, std::nullopt, 0, releaseCallback->getCallback()); + transaction.setBuffer(layer, nullptr, std::nullopt, 0, 0 /* producerId */, + releaseCallback->getCallback()); ASSERT_NO_FATAL_FAILURE(waitForReleaseBufferCallback(*releaseCallback, firstBufferCallbackId)); } diff --git a/services/surfaceflinger/tests/unittests/EventThreadTest.cpp b/services/surfaceflinger/tests/unittests/EventThreadTest.cpp index b3aba377ee..f6bcadc586 100644 --- a/services/surfaceflinger/tests/unittests/EventThreadTest.cpp +++ b/services/surfaceflinger/tests/unittests/EventThreadTest.cpp @@ -125,7 +125,7 @@ protected: ConnectionEventRecorder mConnectionEventCallRecorder{0}; ConnectionEventRecorder mThrottledConnectionEventCallRecorder{0}; - std::optional<scheduler::VsyncSchedule> mVsyncSchedule; + std::unique_ptr<scheduler::VsyncSchedule> mVsyncSchedule; std::unique_ptr<impl::EventThread> mThread; sp<MockEventThreadConnection> mConnection; sp<MockEventThreadConnection> mThrottledConnection; @@ -140,9 +140,9 @@ EventThreadTest::EventThreadTest() { ::testing::UnitTest::GetInstance()->current_test_info(); ALOGD("**** Setting up for %s.%s\n", test_info->test_case_name(), test_info->name()); - mVsyncSchedule.emplace(scheduler::VsyncSchedule(std::make_unique<mock::VSyncTracker>(), - std::make_unique<mock::VSyncDispatch>(), - nullptr)); + mVsyncSchedule = std::unique_ptr<scheduler::VsyncSchedule>( + new scheduler::VsyncSchedule(std::make_unique<mock::VSyncTracker>(), + std::make_unique<mock::VSyncDispatch>(), nullptr)); mock::VSyncDispatch& mockDispatch = *static_cast<mock::VSyncDispatch*>(&mVsyncSchedule->getDispatch()); diff --git a/services/surfaceflinger/tests/unittests/HWComposerTest.cpp b/services/surfaceflinger/tests/unittests/HWComposerTest.cpp index 9534f3b548..da00377cae 100644 --- a/services/surfaceflinger/tests/unittests/HWComposerTest.cpp +++ b/services/surfaceflinger/tests/unittests/HWComposerTest.cpp @@ -52,6 +52,7 @@ namespace aidl = aidl::android::hardware::graphics::composer3; using Hwc2::Config; +using ::aidl::android::hardware::graphics::composer3::RefreshRateChangedDebugData; using ::testing::_; using ::testing::DoAll; using ::testing::ElementsAreArray; @@ -155,6 +156,7 @@ struct MockHWC2ComposerCallback final : StrictMock<HWC2::ComposerCallback> { void(hal::HWDisplayId, const hal::VsyncPeriodChangeTimeline&)); MOCK_METHOD1(onComposerHalSeamlessPossible, void(hal::HWDisplayId)); MOCK_METHOD1(onComposerHalVsyncIdle, void(hal::HWDisplayId)); + MOCK_METHOD(void, onRefreshRateChangedDebug, (const RefreshRateChangedDebugData&), (override)); }; struct HWComposerSetCallbackTest : HWComposerTest { diff --git a/services/surfaceflinger/tests/unittests/SurfaceFlinger_SetPowerModeInternalTest.cpp b/services/surfaceflinger/tests/unittests/SurfaceFlinger_SetPowerModeInternalTest.cpp index ab732ed485..88ddb0f45b 100644 --- a/services/surfaceflinger/tests/unittests/SurfaceFlinger_SetPowerModeInternalTest.cpp +++ b/services/surfaceflinger/tests/unittests/SurfaceFlinger_SetPowerModeInternalTest.cpp @@ -262,8 +262,8 @@ struct DisplayPowerCase { return display; } - static void setInitialPrimaryHWVsyncEnabled(DisplayTransactionTest* test, bool enabled) { - test->mFlinger.scheduler()->mutablePrimaryHWVsyncEnabled() = enabled; + static void setInitialHwVsyncEnabled(DisplayTransactionTest* test, bool enabled) { + test->mFlinger.scheduler()->setInitialHwVsyncEnabled(enabled); } static void setupRepaintEverythingCallExpectations(DisplayTransactionTest* test) { @@ -329,9 +329,9 @@ void SetPowerModeInternalTest::transitionDisplayCommon() { Case::Doze::setupComposerCallExpectations(this); auto display = Case::injectDisplayWithInitialPowerMode(this, Case::Transition::INITIAL_POWER_MODE); - Case::setInitialPrimaryHWVsyncEnabled(this, - PowerModeInitialVSyncEnabled< - Case::Transition::INITIAL_POWER_MODE>::value); + Case::setInitialHwVsyncEnabled(this, + PowerModeInitialVSyncEnabled< + Case::Transition::INITIAL_POWER_MODE>::value); // -------------------------------------------------------------------- // Call Expectations diff --git a/services/surfaceflinger/tests/unittests/TestableScheduler.h b/services/surfaceflinger/tests/unittests/TestableScheduler.h index 6cf61416c0..bd3f3cae98 100644 --- a/services/surfaceflinger/tests/unittests/TestableScheduler.h +++ b/services/surfaceflinger/tests/unittests/TestableScheduler.h @@ -44,9 +44,9 @@ public: std::unique_ptr<VSyncTracker> tracker, RefreshRateSelectorPtr selectorPtr, sp<VsyncModulator> modulatorPtr, ISchedulerCallback& callback) : Scheduler(*this, callback, Feature::kContentDetection, std::move(modulatorPtr)) { - mVsyncSchedule.emplace(VsyncSchedule(std::move(tracker), - std::make_unique<mock::VSyncDispatch>(), - std::move(controller))); + mVsyncSchedule = std::unique_ptr<VsyncSchedule>( + new VsyncSchedule(std::move(tracker), std::make_unique<mock::VSyncDispatch>(), + std::move(controller))); const auto displayId = selectorPtr->getActiveMode().modePtr->getPhysicalDisplayId(); registerDisplay(displayId, std::move(selectorPtr)); @@ -66,13 +66,6 @@ public: return Scheduler::createConnection(std::move(eventThread)); } - /* ------------------------------------------------------------------------ - * Read-write access to private data to set up preconditions and assert - * post-conditions. - */ - auto& mutablePrimaryHWVsyncEnabled() { return mPrimaryHWVsyncEnabled; } - auto& mutableHWVsyncAvailable() { return mHWVsyncAvailable; } - auto refreshRateSelector() { return leaderSelectorPtr(); } const auto& refreshRateSelectors() const NO_THREAD_SAFETY_ANALYSIS { @@ -157,6 +150,12 @@ public: Scheduler::onNonPrimaryDisplayModeChanged(handle, mode); } + void setInitialHwVsyncEnabled(bool enabled) { + std::lock_guard<std::mutex> lock(mVsyncSchedule->mHwVsyncLock); + mVsyncSchedule->mHwVsyncState = enabled ? VsyncSchedule::HwVsyncState::Enabled + : VsyncSchedule::HwVsyncState::Disabled; + } + private: // ICompositor overrides: void configure() override {} diff --git a/vulkan/vkjson/Android.bp b/vulkan/vkjson/Android.bp index b6d3a0b45f..b544245a7a 100644 --- a/vulkan/vkjson/Android.bp +++ b/vulkan/vkjson/Android.bp @@ -25,10 +25,8 @@ cc_library_shared { ".", ], shared_libs: [ - "libvulkan", - ], - whole_static_libs: [ "libjsoncpp", + "libvulkan", ], export_shared_lib_headers: [ "libvulkan", |