diff options
Diffstat (limited to 'libs')
188 files changed, 9147 insertions, 1672 deletions
diff --git a/libs/graphicsenv/GraphicsEnv.cpp b/libs/graphicsenv/GraphicsEnv.cpp index 4c3f4a6428..d1a56635f5 100644 --- a/libs/graphicsenv/GraphicsEnv.cpp +++ b/libs/graphicsenv/GraphicsEnv.cpp @@ -401,18 +401,10 @@ void GraphicsEnv::setDriverToLoad(GpuStatsInfo::Driver driver) { switch (driver) { case GpuStatsInfo::Driver::GL: case GpuStatsInfo::Driver::GL_UPDATED: - case GpuStatsInfo::Driver::ANGLE: { - if (mGpuStats.glDriverToLoad == GpuStatsInfo::Driver::NONE || - mGpuStats.glDriverToLoad == GpuStatsInfo::Driver::GL) { - mGpuStats.glDriverToLoad = driver; - break; - } - - if (mGpuStats.glDriverFallback == GpuStatsInfo::Driver::NONE) { - mGpuStats.glDriverFallback = driver; - } + case GpuStatsInfo::Driver::ANGLE: + mGpuStats.glDriverToLoad = driver; break; - } + case GpuStatsInfo::Driver::VULKAN: case GpuStatsInfo::Driver::VULKAN_UPDATED: { if (mGpuStats.vkDriverToLoad == GpuStatsInfo::Driver::NONE || @@ -561,8 +553,7 @@ void GraphicsEnv::sendGpuStatsLocked(GpuStatsInfo::Api api, bool isDriverLoaded, bool isIntendedDriverLoaded = false; if (api == GpuStatsInfo::Api::API_GL) { driver = mGpuStats.glDriverToLoad; - isIntendedDriverLoaded = - isDriverLoaded && (mGpuStats.glDriverFallback == GpuStatsInfo::Driver::NONE); + isIntendedDriverLoaded = isDriverLoaded; } else { driver = mGpuStats.vkDriverToLoad; isIntendedDriverLoaded = diff --git a/libs/graphicsenv/include/graphicsenv/GpuStatsInfo.h b/libs/graphicsenv/include/graphicsenv/GpuStatsInfo.h index 23f583bda0..72f29c6b0b 100644 --- a/libs/graphicsenv/include/graphicsenv/GpuStatsInfo.h +++ b/libs/graphicsenv/include/graphicsenv/GpuStatsInfo.h @@ -125,6 +125,11 @@ public: VULKAN_DEVICE_EXTENSION = 9, }; + enum GLTelemetryHints { + NO_HINT = 0, + SKIP_TELEMETRY = 1, + }; + GpuStatsInfo() = default; GpuStatsInfo(const GpuStatsInfo&) = default; virtual ~GpuStatsInfo() = default; @@ -136,7 +141,6 @@ public: std::string appPackageName = ""; int32_t vulkanVersion = 0; Driver glDriverToLoad = Driver::NONE; - Driver glDriverFallback = Driver::NONE; Driver vkDriverToLoad = Driver::NONE; Driver vkDriverFallback = Driver::NONE; bool glDriverToSend = false; diff --git a/libs/gui/Android.bp b/libs/gui/Android.bp index 51d2e5305a..1243b214d3 100644 --- a/libs/gui/Android.bp +++ b/libs/gui/Android.bp @@ -255,6 +255,7 @@ filegroup { "BitTube.cpp", "BLASTBufferQueue.cpp", "BufferItemConsumer.cpp", + "BufferReleaseChannel.cpp", "Choreographer.cpp", "CompositorTiming.cpp", "ConsumerBase.cpp", diff --git a/libs/gui/BLASTBufferQueue.cpp b/libs/gui/BLASTBufferQueue.cpp index 044170c378..25e6a52ed1 100644 --- a/libs/gui/BLASTBufferQueue.cpp +++ b/libs/gui/BLASTBufferQueue.cpp @@ -20,12 +20,16 @@ #define ATRACE_TAG ATRACE_TAG_GRAPHICS //#define LOG_NDEBUG 0 +#include <com_android_graphics_libgui_flags.h> #include <cutils/atomic.h> +#include <ftl/fake_guard.h> #include <gui/BLASTBufferQueue.h> #include <gui/BufferItemConsumer.h> #include <gui/BufferQueueConsumer.h> #include <gui/BufferQueueCore.h> #include <gui/BufferQueueProducer.h> +#include <sys/epoll.h> +#include <sys/eventfd.h> #include <gui/FrameRateUtils.h> #include <gui/GLConsumer.h> @@ -39,7 +43,6 @@ #include <private/gui/ComposerServiceAIDL.h> #include <android-base/thread_annotations.h> -#include <chrono> #include <com_android_graphics_libgui_flags.h> @@ -74,6 +77,12 @@ namespace android { std::unique_lock _lock{mutex}; \ base::ScopedLockAssertion assumeLocked(mutex); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL) +static ReleaseBufferCallback EMPTY_RELEASE_CALLBACK = + [](const ReleaseCallbackId&, const sp<Fence>& /*releaseFence*/, + std::optional<uint32_t> /*currentMaxAcquiredBufferCount*/) {}; +#endif + void BLASTBufferItemConsumer::onDisconnect() { Mutex::Autolock lock(mMutex); mPreviouslyConnected = mCurrentlyConnected; @@ -175,16 +184,21 @@ BLASTBufferQueue::BLASTBufferQueue(const std::string& name, bool updateDestinati mSyncTransaction(nullptr), mUpdateDestinationFrame(updateDestinationFrame) { createBufferQueue(&mProducer, &mConsumer); - // since the adapter is in the client process, set dequeue timeout - // explicitly so that dequeueBuffer will block - mProducer->setDequeueTimeout(std::numeric_limits<int64_t>::max()); - - // safe default, most producers are expected to override this - mProducer->setMaxDequeuedBufferCount(2); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + mBufferItemConsumer = new BLASTBufferItemConsumer(mProducer, mConsumer, + GraphicBuffer::USAGE_HW_COMPOSER | + GraphicBuffer::USAGE_HW_TEXTURE, + 1, false, this); +#else mBufferItemConsumer = new BLASTBufferItemConsumer(mConsumer, GraphicBuffer::USAGE_HW_COMPOSER | GraphicBuffer::USAGE_HW_TEXTURE, 1, false, this); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // since the adapter is in the client process, set dequeue timeout + // explicitly so that dequeueBuffer will block + mProducer->setDequeueTimeout(std::numeric_limits<int64_t>::max()); + static std::atomic<uint32_t> nextId = 0; mProducerId = nextId++; mName = name + "#" + std::to_string(mProducerId); @@ -210,6 +224,12 @@ BLASTBufferQueue::BLASTBufferQueue(const std::string& name, bool updateDestinati }, this); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL) + std::unique_ptr<gui::BufferReleaseChannel::ConsumerEndpoint> bufferReleaseConsumer; + gui::BufferReleaseChannel::open(mName, bufferReleaseConsumer, mBufferReleaseProducer); + mBufferReleaseReader = std::make_shared<BufferReleaseReader>(std::move(bufferReleaseConsumer)); +#endif + BQA_LOGV("BLASTBufferQueue created"); } @@ -236,6 +256,14 @@ BLASTBufferQueue::~BLASTBufferQueue() { } } +void BLASTBufferQueue::onFirstRef() { + // safe default, most producers are expected to override this + mProducer->setMaxDequeuedBufferCount(2); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL) + mBufferReleaseThread.start(sp<BLASTBufferQueue>::fromExisting(this)); +#endif +} + void BLASTBufferQueue::update(const sp<SurfaceControl>& surface, uint32_t width, uint32_t height, int32_t format) { LOG_ALWAYS_FATAL_IF(surface == nullptr, "BLASTBufferQueue: mSurfaceControl must not be NULL"); @@ -259,6 +287,9 @@ void BLASTBufferQueue::update(const sp<SurfaceControl>& surface, uint32_t width, if (surfaceControlChanged) { t.setFlags(mSurfaceControl, layer_state_t::eEnableBackpressure, layer_state_t::eEnableBackpressure); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL) + t.setBufferReleaseChannel(mSurfaceControl, mBufferReleaseProducer); +#endif applyTransaction = true; } mTransformHint = mSurfaceControl->getTransformHint(); @@ -296,14 +327,12 @@ static std::optional<SurfaceControlStats> findMatchingStat( return std::nullopt; } -static void transactionCommittedCallbackThunk(void* context, nsecs_t latchTime, - const sp<Fence>& presentFence, - const std::vector<SurfaceControlStats>& stats) { - if (context == nullptr) { - return; - } - sp<BLASTBufferQueue> bq = static_cast<BLASTBufferQueue*>(context); - bq->transactionCommittedCallback(latchTime, presentFence, stats); +TransactionCompletedCallbackTakesContext BLASTBufferQueue::makeTransactionCommittedCallbackThunk() { + return [bbq = sp<BLASTBufferQueue>::fromExisting( + this)](void* /*context*/, nsecs_t latchTime, const sp<Fence>& presentFence, + const std::vector<SurfaceControlStats>& stats) { + bbq->transactionCommittedCallback(latchTime, presentFence, stats); + }; } void BLASTBufferQueue::transactionCommittedCallback(nsecs_t /*latchTime*/, @@ -336,18 +365,15 @@ void BLASTBufferQueue::transactionCommittedCallback(nsecs_t /*latchTime*/, BQA_LOGE("No matching SurfaceControls found: mSurfaceControlsWithPendingCallback was " "empty."); } - decStrong((void*)transactionCommittedCallbackThunk); } } -static void transactionCallbackThunk(void* context, nsecs_t latchTime, - const sp<Fence>& presentFence, - const std::vector<SurfaceControlStats>& stats) { - if (context == nullptr) { - return; - } - sp<BLASTBufferQueue> bq = static_cast<BLASTBufferQueue*>(context); - bq->transactionCallback(latchTime, presentFence, stats); +TransactionCompletedCallbackTakesContext BLASTBufferQueue::makeTransactionCallbackThunk() { + return [bbq = sp<BLASTBufferQueue>::fromExisting( + this)](void* /*context*/, nsecs_t latchTime, const sp<Fence>& presentFence, + const std::vector<SurfaceControlStats>& stats) { + bbq->transactionCallback(latchTime, presentFence, stats); + }; } void BLASTBufferQueue::transactionCallback(nsecs_t /*latchTime*/, const sp<Fence>& /*presentFence*/, @@ -381,6 +407,7 @@ void BLASTBufferQueue::transactionCallback(nsecs_t /*latchTime*/, const sp<Fence stat.latchTime, stat.frameEventStats.dequeueReadyTime); } +#if !COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL) auto currFrameNumber = stat.frameEventStats.frameNumber; std::vector<ReleaseCallbackId> staleReleases; for (const auto& [key, value]: mSubmitted) { @@ -396,6 +423,7 @@ void BLASTBufferQueue::transactionCallback(nsecs_t /*latchTime*/, const sp<Fence stat.currentMaxAcquiredBufferCount, true /* fakeRelease */); } +#endif } else { BQA_LOGE("Failed to find matching SurfaceControl in transactionCallback"); } @@ -403,23 +431,6 @@ void BLASTBufferQueue::transactionCallback(nsecs_t /*latchTime*/, const sp<Fence BQA_LOGE("No matching SurfaceControls found: mSurfaceControlsWithPendingCallback was " "empty."); } - - decStrong((void*)transactionCallbackThunk); - } -} - -// Unlike transactionCallbackThunk the release buffer callback does not extend the life of the -// BBQ. This is because if the BBQ is destroyed, then the buffers will be released by the client. -// So we pass in a weak pointer to the BBQ and if it still alive, then we release the buffer. -// Otherwise, this is a no-op. -static void releaseBufferCallbackThunk(wp<BLASTBufferQueue> context, const ReleaseCallbackId& id, - const sp<Fence>& releaseFence, - std::optional<uint32_t> currentMaxAcquiredBufferCount) { - sp<BLASTBufferQueue> blastBufferQueue = context.promote(); - if (blastBufferQueue) { - blastBufferQueue->releaseBufferCallback(id, releaseFence, currentMaxAcquiredBufferCount); - } else { - ALOGV("releaseBufferCallbackThunk %s blastBufferQueue is dead", id.to_string().c_str()); } } @@ -432,6 +443,23 @@ void BLASTBufferQueue::flushShadowQueue() { } } +// Unlike transactionCallbackThunk the release buffer callback does not extend the life of the +// BBQ. This is because if the BBQ is destroyed, then the buffers will be released by the client. +// So we pass in a weak pointer to the BBQ and if it still alive, then we release the buffer. +// Otherwise, this is a no-op. +ReleaseBufferCallback BLASTBufferQueue::makeReleaseBufferCallbackThunk() { + return [weakBbq = wp<BLASTBufferQueue>::fromExisting( + this)](const ReleaseCallbackId& id, const sp<Fence>& releaseFence, + std::optional<uint32_t> currentMaxAcquiredBufferCount) { + sp<BLASTBufferQueue> bbq = weakBbq.promote(); + if (!bbq) { + ALOGV("releaseBufferCallbackThunk %s blastBufferQueue is dead", id.to_string().c_str()); + return; + } + bbq->releaseBufferCallback(id, releaseFence, currentMaxAcquiredBufferCount); + }; +} + void BLASTBufferQueue::releaseBufferCallback( const ReleaseCallbackId& id, const sp<Fence>& releaseFence, std::optional<uint32_t> currentMaxAcquiredBufferCount) { @@ -594,9 +622,6 @@ status_t BLASTBufferQueue::acquireNextBufferLocked( t->notifyProducerDisconnect(mSurfaceControl); } - // Ensure BLASTBufferQueue stays alive until we receive the transaction complete callback. - incStrong((void*)transactionCallbackThunk); - // Only update mSize for destination bounds if the incoming buffer matches the requested size. // Otherwise, it could cause stretching since the destination bounds will update before the // buffer with the new size is acquired. @@ -609,9 +634,12 @@ status_t BLASTBufferQueue::acquireNextBufferLocked( bufferItem.mGraphicBuffer->getHeight(), bufferItem.mTransform, bufferItem.mScalingMode, crop); - auto releaseBufferCallback = - std::bind(releaseBufferCallbackThunk, wp<BLASTBufferQueue>(this) /* callbackContext */, - std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL) + ReleaseBufferCallback releaseBufferCallback = + applyTransaction ? EMPTY_RELEASE_CALLBACK : makeReleaseBufferCallbackThunk(); +#else + auto releaseBufferCallback = makeReleaseBufferCallbackThunk(); +#endif sp<Fence> fence = bufferItem.mFence ? new Fence(bufferItem.mFence->dup()) : Fence::NO_FENCE; nsecs_t dequeueTime = -1; @@ -629,7 +657,7 @@ status_t BLASTBufferQueue::acquireNextBufferLocked( t->setDataspace(mSurfaceControl, static_cast<ui::Dataspace>(bufferItem.mDataSpace)); t->setHdrMetadata(mSurfaceControl, bufferItem.mHdrMetadata); t->setSurfaceDamageRegion(mSurfaceControl, bufferItem.mSurfaceDamage); - t->addTransactionCompletedCallback(transactionCallbackThunk, static_cast<void*>(this)); + t->addTransactionCompletedCallback(makeTransactionCallbackThunk(), nullptr); mSurfaceControlsWithPendingCallback.push(mSurfaceControl); @@ -786,9 +814,9 @@ void BLASTBufferQueue::onFrameAvailable(const BufferItem& item) { // Only need a commit callback when syncing to ensure the buffer that's synced has been // sent to SF - incStrong((void*)transactionCommittedCallbackThunk); - mSyncTransaction->addTransactionCommittedCallback(transactionCommittedCallbackThunk, - static_cast<void*>(this)); + mSyncTransaction + ->addTransactionCommittedCallback(makeTransactionCommittedCallbackThunk(), + nullptr); if (mAcquireSingleBuffer) { prevCallback = mTransactionReadyCallback; prevTransaction = mSyncTransaction; @@ -817,7 +845,7 @@ void BLASTBufferQueue::onFrameDequeued(const uint64_t bufferId) { void BLASTBufferQueue::onFrameCancelled(const uint64_t bufferId) { std::lock_guard _lock{mTimestampMutex}; mDequeueTimestamps.erase(bufferId); -}; +} bool BLASTBufferQueue::syncNextTransaction( std::function<void(SurfaceComposerClient::Transaction*)> callback, @@ -1113,9 +1141,9 @@ public: // can be non-blocking when the producer is in the client process. class BBQBufferQueueProducer : public BufferQueueProducer { public: - BBQBufferQueueProducer(const sp<BufferQueueCore>& core, wp<BLASTBufferQueue> bbq) + BBQBufferQueueProducer(const sp<BufferQueueCore>& core, const wp<BLASTBufferQueue>& bbq) : BufferQueueProducer(core, false /* consumerIsSurfaceFlinger*/), - mBLASTBufferQueue(std::move(bbq)) {} + mBLASTBufferQueue(bbq) {} status_t connect(const sp<IProducerListener>& listener, int api, bool producerControlledByApp, QueueBufferOutput* output) override { @@ -1130,27 +1158,32 @@ public: // We want to resize the frame history when changing the size of the buffer queue status_t setMaxDequeuedBufferCount(int maxDequeuedBufferCount) override { int maxBufferCount; - status_t status = BufferQueueProducer::setMaxDequeuedBufferCount(maxDequeuedBufferCount, - &maxBufferCount); + if (status_t status = BufferQueueProducer::setMaxDequeuedBufferCount(maxDequeuedBufferCount, + &maxBufferCount); + status != OK) { + return status; + } + + sp<BLASTBufferQueue> bbq = mBLASTBufferQueue.promote(); + if (!bbq) { + return OK; + } + // if we can't determine the max buffer count, then just skip growing the history size - if (status == OK) { - size_t newFrameHistorySize = maxBufferCount + 2; // +2 because triple buffer rendering - // optimize away resizing the frame history unless it will grow - if (newFrameHistorySize > FrameEventHistory::INITIAL_MAX_FRAME_HISTORY) { - sp<BLASTBufferQueue> bbq = mBLASTBufferQueue.promote(); - if (bbq != nullptr) { - ALOGV("increasing frame history size to %zu", newFrameHistorySize); - bbq->resizeFrameEventHistory(newFrameHistorySize); - } - } + size_t newFrameHistorySize = maxBufferCount + 2; // +2 because triple buffer rendering + // optimize away resizing the frame history unless it will grow + if (newFrameHistorySize > FrameEventHistory::INITIAL_MAX_FRAME_HISTORY) { + ALOGV("increasing frame history size to %zu", newFrameHistorySize); + bbq->resizeFrameEventHistory(newFrameHistorySize); } - return status; + + return OK; } int query(int what, int* value) override { if (what == NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER) { *value = 1; - return NO_ERROR; + return OK; } return BufferQueueProducer::query(what, value); } @@ -1230,7 +1263,125 @@ bool BLASTBufferQueue::isSameSurfaceControl(const sp<SurfaceControl>& surfaceCon void BLASTBufferQueue::setTransactionHangCallback( std::function<void(const std::string&)> callback) { std::lock_guard _lock{mMutex}; - mTransactionHangCallback = callback; + mTransactionHangCallback = std::move(callback); +} + +void BLASTBufferQueue::setApplyToken(sp<IBinder> applyToken) { + std::lock_guard _lock{mMutex}; + mApplyToken = std::move(applyToken); +} + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL) + +BLASTBufferQueue::BufferReleaseReader::BufferReleaseReader( + std::unique_ptr<gui::BufferReleaseChannel::ConsumerEndpoint> endpoint) + : mEndpoint{std::move(endpoint)} { + mEpollFd = android::base::unique_fd{epoll_create1(0)}; + LOG_ALWAYS_FATAL_IF(!mEpollFd.ok(), + "Failed to create buffer release epoll file descriptor. errno=%d " + "message='%s'", + errno, strerror(errno)); + + epoll_event registerEndpointFd{}; + registerEndpointFd.events = EPOLLIN; + registerEndpointFd.data.fd = mEndpoint->getFd(); + status_t status = + epoll_ctl(mEpollFd.get(), EPOLL_CTL_ADD, mEndpoint->getFd(), ®isterEndpointFd); + LOG_ALWAYS_FATAL_IF(status == -1, + "Failed to register buffer release consumer file descriptor with epoll. " + "errno=%d message='%s'", + errno, strerror(errno)); + + mEventFd = android::base::unique_fd(eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK)); + LOG_ALWAYS_FATAL_IF(!mEventFd.ok(), + "Failed to create buffer release event file descriptor. errno=%d " + "message='%s'", + errno, strerror(errno)); + + epoll_event registerEventFd{}; + registerEventFd.events = EPOLLIN; + registerEventFd.data.fd = mEventFd.get(); + status = epoll_ctl(mEpollFd.get(), EPOLL_CTL_ADD, mEventFd.get(), ®isterEventFd); + LOG_ALWAYS_FATAL_IF(status == -1, + "Failed to register buffer release event file descriptor with epoll. " + "errno=%d message='%s'", + errno, strerror(errno)); +} + +BLASTBufferQueue::BufferReleaseReader& BLASTBufferQueue::BufferReleaseReader::operator=( + BufferReleaseReader&& other) { + if (this != &other) { + ftl::FakeGuard guard{mMutex}; + ftl::FakeGuard otherGuard{other.mMutex}; + mEndpoint = std::move(other.mEndpoint); + mEpollFd = std::move(other.mEpollFd); + mEventFd = std::move(other.mEventFd); + } + return *this; } +status_t BLASTBufferQueue::BufferReleaseReader::readBlocking(ReleaseCallbackId& outId, + sp<Fence>& outFence, + uint32_t& outMaxAcquiredBufferCount) { + epoll_event event{}; + while (true) { + int eventCount = epoll_wait(mEpollFd.get(), &event, 1 /* maxevents */, -1 /* timeout */); + if (eventCount == 1) { + break; + } + if (eventCount == -1 && errno != EINTR) { + ALOGE("epoll_wait error while waiting for buffer release. errno=%d message='%s'", errno, + strerror(errno)); + } + } + + if (event.data.fd == mEventFd.get()) { + uint64_t value; + if (read(mEventFd.get(), &value, sizeof(uint64_t)) == -1 && errno != EWOULDBLOCK) { + ALOGE("error while reading from eventfd. errno=%d message='%s'", errno, + strerror(errno)); + } + return WOULD_BLOCK; + } + + std::lock_guard lock{mMutex}; + return mEndpoint->readReleaseFence(outId, outFence, outMaxAcquiredBufferCount); +} + +void BLASTBufferQueue::BufferReleaseReader::interruptBlockingRead() { + uint64_t value = 1; + if (write(mEventFd.get(), &value, sizeof(uint64_t)) == -1) { + ALOGE("failed to notify dequeue event. errno=%d message='%s'", errno, strerror(errno)); + } +} + +void BLASTBufferQueue::BufferReleaseThread::start(const sp<BLASTBufferQueue>& bbq) { + mRunning = std::make_shared<std::atomic_bool>(true); + mReader = bbq->mBufferReleaseReader; + std::thread([running = mRunning, reader = mReader, weakBbq = wp<BLASTBufferQueue>(bbq)]() { + pthread_setname_np(pthread_self(), "BufferReleaseThread"); + while (*running) { + ReleaseCallbackId id; + sp<Fence> fence; + uint32_t maxAcquiredBufferCount; + if (status_t status = reader->readBlocking(id, fence, maxAcquiredBufferCount); + status != OK) { + continue; + } + sp<BLASTBufferQueue> bbq = weakBbq.promote(); + if (!bbq) { + return; + } + bbq->releaseBufferCallback(id, fence, maxAcquiredBufferCount); + } + }).detach(); +} + +BLASTBufferQueue::BufferReleaseThread::~BufferReleaseThread() { + *mRunning = false; + mReader->interruptBlockingRead(); +} + +#endif + } // namespace android diff --git a/libs/gui/BufferItemConsumer.cpp b/libs/gui/BufferItemConsumer.cpp index e6331e7282..bfe3d6e023 100644 --- a/libs/gui/BufferItemConsumer.cpp +++ b/libs/gui/BufferItemConsumer.cpp @@ -21,8 +21,11 @@ #include <inttypes.h> +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferItem.h> #include <gui/BufferItemConsumer.h> +#include <ui/BufferQueueDefs.h> +#include <ui/GraphicBuffer.h> #define BI_LOGV(x, ...) ALOGV("[%s] " x, mName.c_str(), ##__VA_ARGS__) // #define BI_LOGD(x, ...) ALOGD("[%s] " x, mName.c_str(), ##__VA_ARGS__) @@ -32,18 +35,37 @@ namespace android { +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +BufferItemConsumer::BufferItemConsumer(uint64_t consumerUsage, int bufferCount, + bool controlledByApp, bool isConsumerSurfaceFlinger) + : ConsumerBase(controlledByApp, isConsumerSurfaceFlinger) { + initialize(consumerUsage, bufferCount); +} + +BufferItemConsumer::BufferItemConsumer(const sp<IGraphicBufferProducer>& producer, + const sp<IGraphicBufferConsumer>& consumer, + uint64_t consumerUsage, int bufferCount, + bool controlledByApp) + : ConsumerBase(producer, consumer, controlledByApp) { + initialize(consumerUsage, bufferCount); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + BufferItemConsumer::BufferItemConsumer( const sp<IGraphicBufferConsumer>& consumer, uint64_t consumerUsage, int bufferCount, bool controlledByApp) : ConsumerBase(consumer, controlledByApp) { + initialize(consumerUsage, bufferCount); +} + +void BufferItemConsumer::initialize(uint64_t consumerUsage, int bufferCount) { status_t err = mConsumer->setConsumerUsageBits(consumerUsage); - LOG_ALWAYS_FATAL_IF(err != OK, - "Failed to set consumer usage bits to %#" PRIx64, consumerUsage); + LOG_ALWAYS_FATAL_IF(err != OK, "Failed to set consumer usage bits to %#" PRIx64, consumerUsage); if (bufferCount != DEFAULT_MAX_BUFFERS) { err = mConsumer->setMaxAcquiredBufferCount(bufferCount); - LOG_ALWAYS_FATAL_IF(err != OK, - "Failed to set max acquired buffer count to %d", bufferCount); + LOG_ALWAYS_FATAL_IF(err != OK, "Failed to set max acquired buffer count to %d", + bufferCount); } } @@ -87,17 +109,38 @@ status_t BufferItemConsumer::acquireBuffer(BufferItem *item, status_t BufferItemConsumer::releaseBuffer(const BufferItem &item, const sp<Fence>& releaseFence) { - status_t err; + Mutex::Autolock _l(mMutex); + return releaseBufferSlotLocked(item.mSlot, item.mGraphicBuffer, releaseFence); +} +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) +status_t BufferItemConsumer::releaseBuffer(const sp<GraphicBuffer>& buffer, + const sp<Fence>& releaseFence) { Mutex::Autolock _l(mMutex); - err = addReleaseFenceLocked(item.mSlot, item.mGraphicBuffer, releaseFence); + if (buffer == nullptr) { + return BAD_VALUE; + } + + int slotIndex = getSlotForBufferLocked(buffer); + if (slotIndex == INVALID_BUFFER_SLOT) { + return BAD_VALUE; + } + + return releaseBufferSlotLocked(slotIndex, buffer, releaseFence); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + +status_t BufferItemConsumer::releaseBufferSlotLocked(int slotIndex, const sp<GraphicBuffer>& buffer, + const sp<Fence>& releaseFence) { + status_t err; + + err = addReleaseFenceLocked(slotIndex, buffer, releaseFence); if (err != OK) { BI_LOGE("Failed to addReleaseFenceLocked"); } - err = releaseBufferLocked(item.mSlot, item.mGraphicBuffer, EGL_NO_DISPLAY, - EGL_NO_SYNC_KHR); + err = releaseBufferLocked(slotIndex, buffer, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR); if (err != OK && err != IGraphicBufferConsumer::STALE_BUFFER_SLOT) { BI_LOGE("Failed to release buffer: %s (%d)", strerror(-err), err); diff --git a/libs/gui/BufferQueueProducer.cpp b/libs/gui/BufferQueueProducer.cpp index ead9f0f014..831b2ee4be 100644 --- a/libs/gui/BufferQueueProducer.cpp +++ b/libs/gui/BufferQueueProducer.cpp @@ -45,7 +45,10 @@ #include <system/window.h> +#include <com_android_graphics_libgui_flags.h> + namespace android { +using namespace com::android::graphics::libgui; // Macros for include BufferQueueCore information in log messages #define BQ_LOGV(x, ...) \ @@ -924,6 +927,7 @@ status_t BufferQueueProducer::queueBuffer(int slot, uint64_t currentFrameNumber = 0; BufferItem item; int connectedApi; + bool enableEglCpuThrottling = true; sp<Fence> lastQueuedFence; { // Autolock scope @@ -1097,6 +1101,9 @@ status_t BufferQueueProducer::queueBuffer(int slot, VALIDATE_CONSISTENCY(); connectedApi = mCore->mConnectedApi; + if (flags::bq_producer_throttles_only_async_mode()) { + enableEglCpuThrottling = mCore->mAsyncMode || mCore->mDequeueBufferCannotBlock; + } lastQueuedFence = std::move(mLastQueueBufferFence); mLastQueueBufferFence = std::move(acquireFence); @@ -1142,7 +1149,7 @@ status_t BufferQueueProducer::queueBuffer(int slot, } // Wait without lock held - if (connectedApi == NATIVE_WINDOW_API_EGL) { + if (connectedApi == NATIVE_WINDOW_API_EGL && enableEglCpuThrottling) { // Waiting here allows for two full buffers to be queued but not a // third. In the event that frames take varying time, this makes a // small trade-off in favor of latency rather than throughput. diff --git a/libs/gui/BufferReleaseChannel.cpp b/libs/gui/BufferReleaseChannel.cpp new file mode 100644 index 0000000000..27367aa83f --- /dev/null +++ b/libs/gui/BufferReleaseChannel.cpp @@ -0,0 +1,358 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "BufferReleaseChannel" + +#include <fcntl.h> +#include <sys/socket.h> +#include <sys/uio.h> + +#include <android-base/result.h> +#include <android/binder_status.h> +#include <binder/Parcel.h> +#include <utils/Flattenable.h> + +#include <gui/BufferReleaseChannel.h> +#include <private/gui/ParcelUtils.h> + +using android::base::Result; + +namespace android::gui { + +namespace { + +template <typename T> +static void readAligned(const void*& buffer, size_t& size, T& value) { + size -= FlattenableUtils::align<alignof(T)>(buffer); + FlattenableUtils::read(buffer, size, value); +} + +template <typename T> +static void writeAligned(void*& buffer, size_t& size, T value) { + size -= FlattenableUtils::align<alignof(T)>(buffer); + FlattenableUtils::write(buffer, size, value); +} + +template <typename T> +static void addAligned(size_t& size, T /* value */) { + size = FlattenableUtils::align<sizeof(T)>(size); + size += sizeof(T); +} + +template <typename T> +static inline constexpr uint32_t low32(const T n) { + return static_cast<uint32_t>(static_cast<uint64_t>(n)); +} + +template <typename T> +static inline constexpr uint32_t high32(const T n) { + return static_cast<uint32_t>(static_cast<uint64_t>(n) >> 32); +} + +template <typename T> +static inline constexpr T to64(const uint32_t lo, const uint32_t hi) { + return static_cast<T>(static_cast<uint64_t>(hi) << 32 | lo); +} + +} // namespace + +size_t BufferReleaseChannel::Message::getPodSize() const { + size_t size = 0; + addAligned(size, low32(releaseCallbackId.bufferId)); + addAligned(size, high32(releaseCallbackId.bufferId)); + addAligned(size, low32(releaseCallbackId.framenumber)); + addAligned(size, high32(releaseCallbackId.framenumber)); + addAligned(size, maxAcquiredBufferCount); + return size; +} + +size_t BufferReleaseChannel::Message::getFlattenedSize() const { + size_t size = releaseFence->getFlattenedSize(); + size = FlattenableUtils::align<4>(size); + size += getPodSize(); + return size; +} + +status_t BufferReleaseChannel::Message::flatten(void*& buffer, size_t& size, int*& fds, + size_t& count) const { + if (status_t err = releaseFence->flatten(buffer, size, fds, count); err != OK) { + return err; + } + size -= FlattenableUtils::align<4>(buffer); + + // Check we still have enough space + if (size < getPodSize()) { + return NO_MEMORY; + } + + writeAligned(buffer, size, low32(releaseCallbackId.bufferId)); + writeAligned(buffer, size, high32(releaseCallbackId.bufferId)); + writeAligned(buffer, size, low32(releaseCallbackId.framenumber)); + writeAligned(buffer, size, high32(releaseCallbackId.framenumber)); + writeAligned(buffer, size, maxAcquiredBufferCount); + return OK; +} + +status_t BufferReleaseChannel::Message::unflatten(void const*& buffer, size_t& size, + int const*& fds, size_t& count) { + releaseFence = new Fence(); + if (status_t err = releaseFence->unflatten(buffer, size, fds, count); err != OK) { + return err; + } + size -= FlattenableUtils::align<4>(buffer); + + // Check we still have enough space + if (size < getPodSize()) { + return OK; + } + + uint32_t bufferIdLo = 0, bufferIdHi = 0; + uint32_t frameNumberLo = 0, frameNumberHi = 0; + + readAligned(buffer, size, bufferIdLo); + readAligned(buffer, size, bufferIdHi); + releaseCallbackId.bufferId = to64<int64_t>(bufferIdLo, bufferIdHi); + readAligned(buffer, size, frameNumberLo); + readAligned(buffer, size, frameNumberHi); + releaseCallbackId.framenumber = to64<uint64_t>(frameNumberLo, frameNumberHi); + readAligned(buffer, size, maxAcquiredBufferCount); + + return OK; +} + +status_t BufferReleaseChannel::ConsumerEndpoint::readReleaseFence( + ReleaseCallbackId& outReleaseCallbackId, sp<Fence>& outReleaseFence, + uint32_t& outMaxAcquiredBufferCount) { + Message message; + mFlattenedBuffer.resize(message.getFlattenedSize()); + std::array<uint8_t, CMSG_SPACE(sizeof(int))> controlMessageBuffer; + + iovec iov{ + .iov_base = mFlattenedBuffer.data(), + .iov_len = mFlattenedBuffer.size(), + }; + + msghdr msg{ + .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = controlMessageBuffer.data(), + .msg_controllen = controlMessageBuffer.size(), + }; + + int result; + do { + result = recvmsg(mFd, &msg, 0); + } while (result == -1 && errno == EINTR); + if (result == -1) { + if (errno == EWOULDBLOCK || errno == EAGAIN) { + return WOULD_BLOCK; + } + ALOGE("Error reading release fence from socket: error %#x (%s)", errno, strerror(errno)); + return UNKNOWN_ERROR; + } + + if (msg.msg_iovlen != 1) { + ALOGE("Error reading release fence from socket: bad data length"); + return UNKNOWN_ERROR; + } + + if (msg.msg_controllen % sizeof(int) != 0) { + ALOGE("Error reading release fence from socket: bad fd length"); + return UNKNOWN_ERROR; + } + + size_t dataLen = msg.msg_iov->iov_len; + const void* data = static_cast<const void*>(msg.msg_iov->iov_base); + if (!data) { + ALOGE("Error reading release fence from socket: no buffer data"); + return UNKNOWN_ERROR; + } + + size_t fdCount = 0; + const int* fdData = nullptr; + if (cmsghdr* cmsg = CMSG_FIRSTHDR(&msg)) { + fdData = reinterpret_cast<const int*>(CMSG_DATA(cmsg)); + fdCount = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int); + } + + if (status_t err = message.unflatten(data, dataLen, fdData, fdCount); err != OK) { + return err; + } + + outReleaseCallbackId = message.releaseCallbackId; + outReleaseFence = std::move(message.releaseFence); + outMaxAcquiredBufferCount = message.maxAcquiredBufferCount; + + return OK; +} + +int BufferReleaseChannel::ProducerEndpoint::writeReleaseFence(const ReleaseCallbackId& callbackId, + const sp<Fence>& fence, + uint32_t maxAcquiredBufferCount) { + Message message{callbackId, fence ? fence : Fence::NO_FENCE, maxAcquiredBufferCount}; + mFlattenedBuffer.resize(message.getFlattenedSize()); + int flattenedFd; + { + // Make copies of needed items since flatten modifies them, and we don't + // want to send anything if there's an error during flatten. + void* flattenedBufferPtr = mFlattenedBuffer.data(); + size_t flattenedBufferSize = mFlattenedBuffer.size(); + int* flattenedFdPtr = &flattenedFd; + size_t flattenedFdCount = 1; + if (status_t err = message.flatten(flattenedBufferPtr, flattenedBufferSize, flattenedFdPtr, + flattenedFdCount); + err != OK) { + ALOGE("Failed to flatten BufferReleaseChannel message."); + return err; + } + } + + iovec iov{ + .iov_base = mFlattenedBuffer.data(), + .iov_len = mFlattenedBuffer.size(), + }; + + msghdr msg{ + .msg_iov = &iov, + .msg_iovlen = 1, + }; + + std::array<uint8_t, CMSG_SPACE(sizeof(int))> controlMessageBuffer; + if (fence && fence->isValid()) { + msg.msg_control = controlMessageBuffer.data(); + msg.msg_controllen = controlMessageBuffer.size(); + + cmsghdr* cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(cmsg), &flattenedFd, sizeof(int)); + } + + int result; + do { + result = sendmsg(mFd, &msg, 0); + } while (result == -1 && errno == EINTR); + if (result == -1) { + ALOGD("Error writing release fence to socket: error %#x (%s)", errno, strerror(errno)); + return -errno; + } + + return OK; +} + +status_t BufferReleaseChannel::ProducerEndpoint::readFromParcel(const android::Parcel* parcel) { + if (!parcel) return STATUS_BAD_VALUE; + SAFE_PARCEL(parcel->readUtf8FromUtf16, &mName); + SAFE_PARCEL(parcel->readUniqueFileDescriptor, &mFd); + return STATUS_OK; +} + +status_t BufferReleaseChannel::ProducerEndpoint::writeToParcel(android::Parcel* parcel) const { + if (!parcel) return STATUS_BAD_VALUE; + SAFE_PARCEL(parcel->writeUtf8AsUtf16, mName); + SAFE_PARCEL(parcel->writeUniqueFileDescriptor, mFd); + return STATUS_OK; +} + +status_t BufferReleaseChannel::open(std::string name, + std::unique_ptr<ConsumerEndpoint>& outConsumer, + std::shared_ptr<ProducerEndpoint>& outProducer) { + outConsumer.reset(); + outProducer.reset(); + + int sockets[2]; + if (socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sockets)) { + ALOGE("[%s] Failed to create socket pair. errorno=%d message='%s'", name.c_str(), errno, + strerror(errno)); + return -errno; + } + + android::base::unique_fd consumerFd(sockets[0]); + android::base::unique_fd producerFd(sockets[1]); + + // Socket buffer size. The default is typically about 128KB, which is much larger than + // we really need. + size_t bufferSize = 32 * 1024; + if (setsockopt(consumerFd.get(), SOL_SOCKET, SO_SNDBUF, &bufferSize, sizeof(bufferSize)) == + -1) { + ALOGE("[%s] Failed to set consumer socket send buffer size. errno=%d message='%s'", + name.c_str(), errno, strerror(errno)); + return -errno; + } + if (setsockopt(consumerFd.get(), SOL_SOCKET, SO_RCVBUF, &bufferSize, sizeof(bufferSize)) == + -1) { + ALOGE("[%s] Failed to set consumer socket receive buffer size. errno=%d " + "message='%s'", + name.c_str(), errno, strerror(errno)); + return -errno; + } + if (setsockopt(producerFd.get(), SOL_SOCKET, SO_SNDBUF, &bufferSize, sizeof(bufferSize)) == + -1) { + ALOGE("[%s] Failed to set producer socket send buffer size. errno=%d message='%s'", + name.c_str(), errno, strerror(errno)); + return -errno; + } + if (setsockopt(producerFd.get(), SOL_SOCKET, SO_RCVBUF, &bufferSize, sizeof(bufferSize)) == + -1) { + ALOGE("[%s] Failed to set producer socket receive buffer size. errno=%d " + "message='%s'", + name.c_str(), errno, strerror(errno)); + return -errno; + } + + // Configure the consumer socket to be non-blocking. + int flags = fcntl(consumerFd.get(), F_GETFL, 0); + if (flags == -1) { + ALOGE("[%s] Failed to get consumer socket flags. errno=%d message='%s'", name.c_str(), + errno, strerror(errno)); + return -errno; + } + if (fcntl(consumerFd.get(), F_SETFL, flags | O_NONBLOCK) == -1) { + ALOGE("[%s] Failed to set consumer socket to non-blocking mode. errno=%d " + "message='%s'", + name.c_str(), errno, strerror(errno)); + return -errno; + } + + // Configure a timeout for the producer socket. + const timeval timeout{.tv_sec = 1, .tv_usec = 0}; + if (setsockopt(producerFd.get(), SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeval)) == -1) { + ALOGE("[%s] Failed to set producer socket timeout. errno=%d message='%s'", name.c_str(), + errno, strerror(errno)); + return -errno; + } + + // Make the consumer read-only + if (shutdown(consumerFd.get(), SHUT_WR) == -1) { + ALOGE("[%s] Failed to shutdown writing on consumer socket. errno=%d message='%s'", + name.c_str(), errno, strerror(errno)); + return -errno; + } + + // Make the producer write-only + if (shutdown(producerFd.get(), SHUT_RD) == -1) { + ALOGE("[%s] Failed to shutdown reading on producer socket. errno=%d message='%s'", + name.c_str(), errno, strerror(errno)); + return -errno; + } + + outConsumer = std::make_unique<ConsumerEndpoint>(name, std::move(consumerFd)); + outProducer = std::make_shared<ProducerEndpoint>(std::move(name), std::move(producerFd)); + return STATUS_OK; +} + +} // namespace android::gui
\ No newline at end of file diff --git a/libs/gui/ConsumerBase.cpp b/libs/gui/ConsumerBase.cpp index b625c3f75e..602bba8dab 100644 --- a/libs/gui/ConsumerBase.cpp +++ b/libs/gui/ConsumerBase.cpp @@ -14,8 +14,6 @@ * limitations under the License. */ -#include <inttypes.h> - #define LOG_TAG "ConsumerBase" #define ATRACE_TAG ATRACE_TAG_GRAPHICS //#define LOG_NDEBUG 0 @@ -29,17 +27,23 @@ #include <cutils/atomic.h> +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferItem.h> +#include <gui/BufferQueue.h> +#include <gui/ConsumerBase.h> #include <gui/ISurfaceComposer.h> +#include <gui/Surface.h> #include <gui/SurfaceComposerClient.h> -#include <gui/ConsumerBase.h> #include <private/gui/ComposerService.h> +#include <log/log.h> #include <utils/Log.h> #include <utils/String8.h> #include <utils/Trace.h> +#include <inttypes.h> + // Macros for including the ConsumerBase name in log messages #define CB_LOGV(x, ...) ALOGV("[%s] " x, mName.c_str(), ##__VA_ARGS__) // #define CB_LOGD(x, ...) ALOGD("[%s] " x, mName.c_str(), ##__VA_ARGS__) @@ -59,6 +63,30 @@ ConsumerBase::ConsumerBase(const sp<IGraphicBufferConsumer>& bufferQueue, bool c mAbandoned(false), mConsumer(bufferQueue), mPrevFinalReleaseFence(Fence::NO_FENCE) { + initialize(controlledByApp); +} + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +ConsumerBase::ConsumerBase(bool controlledByApp, bool consumerIsSurfaceFlinger) + : mAbandoned(false), mPrevFinalReleaseFence(Fence::NO_FENCE) { + sp<IGraphicBufferProducer> producer; + BufferQueue::createBufferQueue(&producer, &mConsumer, consumerIsSurfaceFlinger); + mSurface = sp<Surface>::make(producer, controlledByApp); + initialize(controlledByApp); +} + +ConsumerBase::ConsumerBase(const sp<IGraphicBufferProducer>& producer, + const sp<IGraphicBufferConsumer>& consumer, bool controlledByApp) + : mAbandoned(false), + mConsumer(consumer), + mSurface(sp<Surface>::make(producer, controlledByApp)), + mPrevFinalReleaseFence(Fence::NO_FENCE) { + initialize(controlledByApp); +} + +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + +void ConsumerBase::initialize(bool controlledByApp) { // Choose a name using the PID and a process-unique ID. mName = String8::format("unnamed-%d-%d", getpid(), createProcessUniqueId()); @@ -96,6 +124,35 @@ void ConsumerBase::onLastStrongRef(const void* id __attribute__((unused))) { abandon(); } +int ConsumerBase::getSlotForBufferLocked(const sp<GraphicBuffer>& buffer) { + if (!buffer) { + return BufferQueue::INVALID_BUFFER_SLOT; + } + + uint64_t id = buffer->getId(); + for (int i = 0; i < BufferQueueDefs::NUM_BUFFER_SLOTS; i++) { + auto& slot = mSlots[i]; + if (slot.mGraphicBuffer && slot.mGraphicBuffer->getId() == id) { + return i; + } + } + + return BufferQueue::INVALID_BUFFER_SLOT; +} + +status_t ConsumerBase::detachBufferLocked(int slotIndex) { + status_t result = mConsumer->detachBuffer(slotIndex); + + if (result != NO_ERROR) { + CB_LOGE("Failed to detach buffer: %d", result); + return result; + } + + freeBufferLocked(slotIndex); + + return result; +} + void ConsumerBase::freeBufferLocked(int slotIndex) { CB_LOGV("freeBufferLocked: slotIndex=%d", slotIndex); mSlots[slotIndex].mGraphicBuffer = nullptr; @@ -252,16 +309,30 @@ status_t ConsumerBase::detachBuffer(int slot) { return NO_INIT; } - status_t result = mConsumer->detachBuffer(slot); - if (result != NO_ERROR) { - CB_LOGE("Failed to detach buffer: %d", result); - return result; + return detachBufferLocked(slot); +} + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) +status_t ConsumerBase::detachBuffer(const sp<GraphicBuffer>& buffer) { + CB_LOGV("detachBuffer"); + Mutex::Autolock lock(mMutex); + + if (mAbandoned) { + CB_LOGE("detachBuffer: ConsumerBase is abandoned!"); + return NO_INIT; + } + if (buffer == nullptr) { + return BAD_VALUE; } - freeBufferLocked(slot); + int slotIndex = getSlotForBufferLocked(buffer); + if (slotIndex == BufferQueue::INVALID_BUFFER_SLOT) { + return BAD_VALUE; + } - return result; + return detachBufferLocked(slotIndex); } +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) status_t ConsumerBase::setDefaultBufferSize(uint32_t width, uint32_t height) { Mutex::Autolock _l(mMutex); @@ -309,6 +380,17 @@ status_t ConsumerBase::setTransformHint(uint32_t hint) { return mConsumer->setTransformHint(hint); } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +status_t ConsumerBase::setMaxBufferCount(int bufferCount) { + Mutex::Autolock lock(mMutex); + if (mAbandoned) { + CB_LOGE("setMaxBufferCount: ConsumerBase is abandoned!"); + return NO_INIT; + } + return mConsumer->setMaxBufferCount(bufferCount); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + status_t ConsumerBase::setMaxAcquiredBufferCount(int maxAcquiredBuffers) { Mutex::Autolock lock(mMutex); if (mAbandoned) { @@ -318,6 +400,17 @@ status_t ConsumerBase::setMaxAcquiredBufferCount(int maxAcquiredBuffers) { return mConsumer->setMaxAcquiredBufferCount(maxAcquiredBuffers); } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +status_t ConsumerBase::setConsumerIsProtected(bool isProtected) { + Mutex::Autolock lock(mMutex); + if (mAbandoned) { + CB_LOGE("setConsumerIsProtected: ConsumerBase is abandoned!"); + return NO_INIT; + } + return mConsumer->setConsumerIsProtected(isProtected); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + sp<NativeHandle> ConsumerBase::getSidebandStream() const { Mutex::Autolock _l(mMutex); if (mAbandoned) { @@ -384,6 +477,19 @@ void ConsumerBase::dumpLocked(String8& result, const char* prefix) const { } } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +sp<Surface> ConsumerBase::getSurface() const { + LOG_ALWAYS_FATAL_IF(mSurface == nullptr, + "It's illegal to get the surface of a Consumer that does not own it. This " + "should be impossible once the old CTOR is removed."); + return mSurface; +} + +sp<IGraphicBufferConsumer> ConsumerBase::getIGraphicBufferConsumer() const { + return mConsumer; +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + status_t ConsumerBase::acquireBufferLocked(BufferItem *item, nsecs_t presentWhen, uint64_t maxFrameNumber) { if (mAbandoned) { diff --git a/libs/gui/CpuConsumer.cpp b/libs/gui/CpuConsumer.cpp index 3031fa11fc..23b432e1f4 100644 --- a/libs/gui/CpuConsumer.cpp +++ b/libs/gui/CpuConsumer.cpp @@ -18,9 +18,9 @@ #define LOG_TAG "CpuConsumer" //#define ATRACE_TAG ATRACE_TAG_GRAPHICS -#include <gui/CpuConsumer.h> - +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferItem.h> +#include <gui/CpuConsumer.h> #include <utils/Log.h> #define CC_LOGV(x, ...) ALOGV("[%s] " x, mName.c_str(), ##__VA_ARGS__) @@ -31,12 +31,25 @@ namespace android { -CpuConsumer::CpuConsumer(const sp<IGraphicBufferConsumer>& bq, - size_t maxLockedBuffers, bool controlledByApp) : - ConsumerBase(bq, controlledByApp), - mMaxLockedBuffers(maxLockedBuffers), - mCurrentLockedBuffers(0) -{ +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +CpuConsumer::CpuConsumer(size_t maxLockedBuffers, bool controlledByApp, + bool isConsumerSurfaceFlinger) + : ConsumerBase(controlledByApp, isConsumerSurfaceFlinger), + mMaxLockedBuffers(maxLockedBuffers), + mCurrentLockedBuffers(0) { + // Create tracking entries for locked buffers + mAcquiredBuffers.insertAt(0, maxLockedBuffers); + + mConsumer->setConsumerUsageBits(GRALLOC_USAGE_SW_READ_OFTEN); + mConsumer->setMaxAcquiredBufferCount(static_cast<int32_t>(maxLockedBuffers)); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + +CpuConsumer::CpuConsumer(const sp<IGraphicBufferConsumer>& bq, size_t maxLockedBuffers, + bool controlledByApp) + : ConsumerBase(bq, controlledByApp), + mMaxLockedBuffers(maxLockedBuffers), + mCurrentLockedBuffers(0) { // Create tracking entries for locked buffers mAcquiredBuffers.insertAt(0, maxLockedBuffers); diff --git a/libs/gui/GLConsumer.cpp b/libs/gui/GLConsumer.cpp index d49489c5a8..95cce5c1df 100644 --- a/libs/gui/GLConsumer.cpp +++ b/libs/gui/GLConsumer.cpp @@ -101,6 +101,34 @@ static bool hasEglProtectedContent() { return hasIt; } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +GLConsumer::GLConsumer(uint32_t tex, uint32_t texTarget, bool useFenceSync, bool isControlledByApp) + : ConsumerBase(isControlledByApp, /* isConsumerSurfaceFlinger */ false), + mCurrentCrop(Rect::EMPTY_RECT), + mCurrentTransform(0), + mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE), + mCurrentFence(Fence::NO_FENCE), + mCurrentTimestamp(0), + mCurrentDataSpace(HAL_DATASPACE_UNKNOWN), + mCurrentFrameNumber(0), + mDefaultWidth(1), + mDefaultHeight(1), + mFilteringEnabled(true), + mTexName(tex), + mUseFenceSync(useFenceSync), + mTexTarget(texTarget), + mEglDisplay(EGL_NO_DISPLAY), + mEglContext(EGL_NO_CONTEXT), + mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), + mAttached(true) { + GLC_LOGV("GLConsumer"); + + memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix)); + + mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + GLConsumer::GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t texTarget, bool useFenceSync, bool isControlledByApp) : ConsumerBase(bq, isControlledByApp), @@ -130,27 +158,54 @@ GLConsumer::GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS); } -GLConsumer::GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t texTarget, - bool useFenceSync, bool isControlledByApp) : - ConsumerBase(bq, isControlledByApp), - mCurrentCrop(Rect::EMPTY_RECT), - mCurrentTransform(0), - mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE), - mCurrentFence(Fence::NO_FENCE), - mCurrentTimestamp(0), - mCurrentDataSpace(HAL_DATASPACE_UNKNOWN), - mCurrentFrameNumber(0), - mDefaultWidth(1), - mDefaultHeight(1), - mFilteringEnabled(true), - mTexName(0), - mUseFenceSync(useFenceSync), - mTexTarget(texTarget), - mEglDisplay(EGL_NO_DISPLAY), - mEglContext(EGL_NO_CONTEXT), - mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), - mAttached(false) -{ +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +GLConsumer::GLConsumer(uint32_t texTarget, bool useFenceSync, bool isControlledByApp) + : ConsumerBase(isControlledByApp, /* isConsumerSurfaceFlinger */ false), + mCurrentCrop(Rect::EMPTY_RECT), + mCurrentTransform(0), + mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE), + mCurrentFence(Fence::NO_FENCE), + mCurrentTimestamp(0), + mCurrentDataSpace(HAL_DATASPACE_UNKNOWN), + mCurrentFrameNumber(0), + mDefaultWidth(1), + mDefaultHeight(1), + mFilteringEnabled(true), + mTexName(0), + mUseFenceSync(useFenceSync), + mTexTarget(texTarget), + mEglDisplay(EGL_NO_DISPLAY), + mEglContext(EGL_NO_CONTEXT), + mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), + mAttached(false) { + GLC_LOGV("GLConsumer"); + + memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix)); + + mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + +GLConsumer::GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t texTarget, bool useFenceSync, + bool isControlledByApp) + : ConsumerBase(bq, isControlledByApp), + mCurrentCrop(Rect::EMPTY_RECT), + mCurrentTransform(0), + mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE), + mCurrentFence(Fence::NO_FENCE), + mCurrentTimestamp(0), + mCurrentDataSpace(HAL_DATASPACE_UNKNOWN), + mCurrentFrameNumber(0), + mDefaultWidth(1), + mDefaultHeight(1), + mFilteringEnabled(true), + mTexName(0), + mUseFenceSync(useFenceSync), + mTexTarget(texTarget), + mEglDisplay(EGL_NO_DISPLAY), + mEglContext(EGL_NO_CONTEXT), + mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), + mAttached(false) { GLC_LOGV("GLConsumer"); memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), diff --git a/libs/gui/ISurfaceComposer.cpp b/libs/gui/ISurfaceComposer.cpp index ff6b558d41..269936858a 100644 --- a/libs/gui/ISurfaceComposer.cpp +++ b/libs/gui/ISurfaceComposer.cpp @@ -62,7 +62,7 @@ public: status_t setTransactionState( const FrameTimelineInfo& frameTimelineInfo, Vector<ComposerState>& state, - const Vector<DisplayState>& displays, uint32_t flags, const sp<IBinder>& applyToken, + Vector<DisplayState>& displays, uint32_t flags, const sp<IBinder>& applyToken, InputWindowCommands commands, int64_t desiredPresentTime, bool isAutoTimestamp, const std::vector<client_cache_t>& uncacheBuffers, bool hasListenerCallbacks, const std::vector<ListenerCallbacks>& listenerCallbacks, uint64_t transactionId, diff --git a/libs/gui/ITransactionCompletedListener.cpp b/libs/gui/ITransactionCompletedListener.cpp index f5d19aac78..83fc827c5f 100644 --- a/libs/gui/ITransactionCompletedListener.cpp +++ b/libs/gui/ITransactionCompletedListener.cpp @@ -43,12 +43,6 @@ enum class Tag : uint32_t { } // Anonymous namespace -namespace { // Anonymous - -constexpr int32_t kSerializedCallbackTypeOnCompelteWithJankData = 2; - -} // Anonymous namespace - status_t FrameEventHistoryStats::writeToParcel(Parcel* output) const { status_t err = output->writeUint64(frameNumber); if (err != NO_ERROR) return err; @@ -119,23 +113,6 @@ status_t FrameEventHistoryStats::readFromParcel(const Parcel* input) { return err; } -JankData::JankData() - : frameVsyncId(FrameTimelineInfo::INVALID_VSYNC_ID), jankType(JankType::None) {} - -status_t JankData::writeToParcel(Parcel* output) const { - SAFE_PARCEL(output->writeInt64, frameVsyncId); - SAFE_PARCEL(output->writeInt32, jankType); - SAFE_PARCEL(output->writeInt64, frameIntervalNs); - return NO_ERROR; -} - -status_t JankData::readFromParcel(const Parcel* input) { - SAFE_PARCEL(input->readInt64, &frameVsyncId); - SAFE_PARCEL(input->readInt32, &jankType); - SAFE_PARCEL(input->readInt64, &frameIntervalNs); - return NO_ERROR; -} - status_t SurfaceStats::writeToParcel(Parcel* output) const { SAFE_PARCEL(output->writeStrongBinder, surfaceControl); if (const auto* acquireFence = std::get_if<sp<Fence>>(&acquireTimeOrFence)) { @@ -160,10 +137,6 @@ status_t SurfaceStats::writeToParcel(Parcel* output) const { SAFE_PARCEL(output->writeUint32, currentMaxAcquiredBufferCount); SAFE_PARCEL(output->writeParcelable, eventStats); - SAFE_PARCEL(output->writeInt32, static_cast<int32_t>(jankData.size())); - for (const auto& data : jankData) { - SAFE_PARCEL(output->writeParcelable, data); - } SAFE_PARCEL(output->writeParcelable, previousReleaseCallbackId); return NO_ERROR; } @@ -200,13 +173,6 @@ status_t SurfaceStats::readFromParcel(const Parcel* input) { SAFE_PARCEL(input->readUint32, ¤tMaxAcquiredBufferCount); SAFE_PARCEL(input->readParcelable, &eventStats); - int32_t jankData_size = 0; - SAFE_PARCEL_READ_SIZE(input->readInt32, &jankData_size, input->dataSize()); - for (int i = 0; i < jankData_size; i++) { - JankData data; - SAFE_PARCEL(input->readParcelable, &data); - jankData.push_back(data); - } SAFE_PARCEL(input->readParcelable, &previousReleaseCallbackId); return NO_ERROR; } @@ -371,11 +337,7 @@ ListenerCallbacks ListenerCallbacks::filter(CallbackId::Type type) const { status_t CallbackId::writeToParcel(Parcel* output) const { SAFE_PARCEL(output->writeInt64, id); - if (type == Type::ON_COMPLETE && includeJankData) { - SAFE_PARCEL(output->writeInt32, kSerializedCallbackTypeOnCompelteWithJankData); - } else { - SAFE_PARCEL(output->writeInt32, static_cast<int32_t>(type)); - } + SAFE_PARCEL(output->writeInt32, static_cast<int32_t>(type)); return NO_ERROR; } @@ -383,13 +345,7 @@ status_t CallbackId::readFromParcel(const Parcel* input) { SAFE_PARCEL(input->readInt64, &id); int32_t typeAsInt; SAFE_PARCEL(input->readInt32, &typeAsInt); - if (typeAsInt == kSerializedCallbackTypeOnCompelteWithJankData) { - type = Type::ON_COMPLETE; - includeJankData = true; - } else { - type = static_cast<CallbackId::Type>(typeAsInt); - includeJankData = false; - } + type = static_cast<CallbackId::Type>(typeAsInt); return NO_ERROR; } diff --git a/libs/gui/LayerState.cpp b/libs/gui/LayerState.cpp index 3745805aa3..b10996951b 100644 --- a/libs/gui/LayerState.cpp +++ b/libs/gui/LayerState.cpp @@ -17,7 +17,6 @@ #define LOG_TAG "LayerState" #include <cinttypes> -#include <cmath> #include <android/gui/ISurfaceComposerClient.h> #include <android/native_window.h> @@ -177,6 +176,7 @@ status_t layer_state_t::write(Parcel& output) const } SAFE_PARCEL(output.write, stretchEffect); + SAFE_PARCEL(output.writeParcelable, edgeExtensionParameters); SAFE_PARCEL(output.write, bufferCrop); SAFE_PARCEL(output.write, destinationFrame); SAFE_PARCEL(output.writeInt32, static_cast<uint32_t>(trustedOverlay)); @@ -193,6 +193,13 @@ status_t layer_state_t::write(Parcel& output) const SAFE_PARCEL(output.writeFloat, currentHdrSdrRatio); SAFE_PARCEL(output.writeFloat, desiredHdrSdrRatio); SAFE_PARCEL(output.writeInt32, static_cast<int32_t>(cachingHint)); + + const bool hasBufferReleaseChannel = (bufferReleaseChannel != nullptr); + SAFE_PARCEL(output.writeBool, hasBufferReleaseChannel); + if (hasBufferReleaseChannel) { + SAFE_PARCEL(output.writeParcelable, *bufferReleaseChannel); + } + return NO_ERROR; } @@ -306,6 +313,7 @@ status_t layer_state_t::read(const Parcel& input) } SAFE_PARCEL(input.read, stretchEffect); + SAFE_PARCEL(input.readParcelable, &edgeExtensionParameters); SAFE_PARCEL(input.read, bufferCrop); SAFE_PARCEL(input.read, destinationFrame); uint32_t trustedOverlayInt; @@ -337,6 +345,13 @@ status_t layer_state_t::read(const Parcel& input) SAFE_PARCEL(input.readInt32, &tmpInt32); cachingHint = static_cast<gui::CachingHint>(tmpInt32); + bool hasBufferReleaseChannel; + SAFE_PARCEL(input.readBool, &hasBufferReleaseChannel); + if (hasBufferReleaseChannel) { + bufferReleaseChannel = std::make_shared<gui::BufferReleaseChannel::ProducerEndpoint>(); + SAFE_PARCEL(input.readParcelable, bufferReleaseChannel.get()); + } + return NO_ERROR; } @@ -682,6 +697,10 @@ void layer_state_t::merge(const layer_state_t& other) { what |= eStretchChanged; stretchEffect = other.stretchEffect; } + if (other.what & eEdgeExtensionChanged) { + what |= eEdgeExtensionChanged; + edgeExtensionParameters = other.edgeExtensionParameters; + } if (other.what & eBufferCropChanged) { what |= eBufferCropChanged; bufferCrop = other.bufferCrop; @@ -712,6 +731,10 @@ void layer_state_t::merge(const layer_state_t& other) { if (other.what & eFlushJankData) { what |= eFlushJankData; } + if (other.what & eBufferReleaseChannelChanged) { + what |= eBufferReleaseChannelChanged; + bufferReleaseChannel = other.bufferReleaseChannel; + } if ((other.what & what) != other.what) { ALOGE("Unmerged SurfaceComposer Transaction properties. LayerState::merge needs updating? " "other.what=0x%" PRIX64 " what=0x%" PRIX64 " unmerged flags=0x%" PRIX64, @@ -783,6 +806,7 @@ uint64_t layer_state_t::diff(const layer_state_t& other) const { CHECK_DIFF(diff, eAutoRefreshChanged, other, autoRefresh); CHECK_DIFF(diff, eTrustedOverlayChanged, other, trustedOverlay); CHECK_DIFF(diff, eStretchChanged, other, stretchEffect); + CHECK_DIFF(diff, eEdgeExtensionChanged, other, edgeExtensionParameters); CHECK_DIFF(diff, eBufferCropChanged, other, bufferCrop); CHECK_DIFF(diff, eDestinationFrameChanged, other, destinationFrame); if (other.what & eProducerDisconnect) diff |= eProducerDisconnect; @@ -790,6 +814,7 @@ uint64_t layer_state_t::diff(const layer_state_t& other) const { CHECK_DIFF(diff, eColorChanged, other, color.rgb); CHECK_DIFF(diff, eColorSpaceAgnosticChanged, other, colorSpaceAgnostic); CHECK_DIFF(diff, eDimmingEnabledChanged, other, dimmingEnabled); + if (other.what & eBufferReleaseChannelChanged) diff |= eBufferReleaseChannelChanged; return diff; } @@ -867,88 +892,6 @@ status_t InputWindowCommands::read(const Parcel& input) { // ---------------------------------------------------------------------------- -namespace gui { - -status_t CaptureArgs::writeToParcel(Parcel* output) const { - SAFE_PARCEL(output->writeInt32, static_cast<int32_t>(pixelFormat)); - SAFE_PARCEL(output->write, sourceCrop); - SAFE_PARCEL(output->writeFloat, frameScaleX); - SAFE_PARCEL(output->writeFloat, frameScaleY); - SAFE_PARCEL(output->writeBool, captureSecureLayers); - SAFE_PARCEL(output->writeInt32, uid); - SAFE_PARCEL(output->writeInt32, static_cast<int32_t>(dataspace)); - SAFE_PARCEL(output->writeBool, allowProtected); - SAFE_PARCEL(output->writeBool, grayscale); - SAFE_PARCEL(output->writeInt32, excludeHandles.size()); - for (auto& excludeHandle : excludeHandles) { - SAFE_PARCEL(output->writeStrongBinder, excludeHandle); - } - SAFE_PARCEL(output->writeBool, hintForSeamlessTransition); - return NO_ERROR; -} - -status_t CaptureArgs::readFromParcel(const Parcel* input) { - int32_t value = 0; - SAFE_PARCEL(input->readInt32, &value); - pixelFormat = static_cast<ui::PixelFormat>(value); - SAFE_PARCEL(input->read, sourceCrop); - SAFE_PARCEL(input->readFloat, &frameScaleX); - SAFE_PARCEL(input->readFloat, &frameScaleY); - SAFE_PARCEL(input->readBool, &captureSecureLayers); - SAFE_PARCEL(input->readInt32, &uid); - SAFE_PARCEL(input->readInt32, &value); - dataspace = static_cast<ui::Dataspace>(value); - SAFE_PARCEL(input->readBool, &allowProtected); - SAFE_PARCEL(input->readBool, &grayscale); - int32_t numExcludeHandles = 0; - SAFE_PARCEL_READ_SIZE(input->readInt32, &numExcludeHandles, input->dataSize()); - excludeHandles.reserve(numExcludeHandles); - for (int i = 0; i < numExcludeHandles; i++) { - sp<IBinder> binder; - SAFE_PARCEL(input->readStrongBinder, &binder); - excludeHandles.emplace(binder); - } - SAFE_PARCEL(input->readBool, &hintForSeamlessTransition); - return NO_ERROR; -} - -status_t DisplayCaptureArgs::writeToParcel(Parcel* output) const { - SAFE_PARCEL(CaptureArgs::writeToParcel, output); - - SAFE_PARCEL(output->writeStrongBinder, displayToken); - SAFE_PARCEL(output->writeUint32, width); - SAFE_PARCEL(output->writeUint32, height); - return NO_ERROR; -} - -status_t DisplayCaptureArgs::readFromParcel(const Parcel* input) { - SAFE_PARCEL(CaptureArgs::readFromParcel, input); - - SAFE_PARCEL(input->readStrongBinder, &displayToken); - SAFE_PARCEL(input->readUint32, &width); - SAFE_PARCEL(input->readUint32, &height); - return NO_ERROR; -} - -status_t LayerCaptureArgs::writeToParcel(Parcel* output) const { - SAFE_PARCEL(CaptureArgs::writeToParcel, output); - - SAFE_PARCEL(output->writeStrongBinder, layerHandle); - SAFE_PARCEL(output->writeBool, childrenOnly); - return NO_ERROR; -} - -status_t LayerCaptureArgs::readFromParcel(const Parcel* input) { - SAFE_PARCEL(CaptureArgs::readFromParcel, input); - - SAFE_PARCEL(input->readStrongBinder, &layerHandle); - - SAFE_PARCEL(input->readBool, &childrenOnly); - return NO_ERROR; -} - -}; // namespace gui - ReleaseCallbackId BufferData::generateReleaseCallbackId() const { uint64_t bufferId; if (buffer) { diff --git a/libs/gui/ScreenCaptureResults.cpp b/libs/gui/ScreenCaptureResults.cpp index 601a5f9b33..2de023e5b2 100644 --- a/libs/gui/ScreenCaptureResults.cpp +++ b/libs/gui/ScreenCaptureResults.cpp @@ -40,6 +40,13 @@ status_t ScreenCaptureResults::writeToParcel(android::Parcel* parcel) const { SAFE_PARCEL(parcel->writeBool, capturedSecureLayers); SAFE_PARCEL(parcel->writeBool, capturedHdrLayers); SAFE_PARCEL(parcel->writeUint32, static_cast<uint32_t>(capturedDataspace)); + if (optionalGainMap != nullptr) { + SAFE_PARCEL(parcel->writeBool, true); + SAFE_PARCEL(parcel->write, *optionalGainMap); + } else { + SAFE_PARCEL(parcel->writeBool, false); + } + SAFE_PARCEL(parcel->writeFloat, hdrSdrRatio); return NO_ERROR; } @@ -68,6 +75,14 @@ status_t ScreenCaptureResults::readFromParcel(const android::Parcel* parcel) { uint32_t dataspace = 0; SAFE_PARCEL(parcel->readUint32, &dataspace); capturedDataspace = static_cast<ui::Dataspace>(dataspace); + + bool hasGainmap; + SAFE_PARCEL(parcel->readBool, &hasGainmap); + if (hasGainmap) { + optionalGainMap = new GraphicBuffer(); + SAFE_PARCEL(parcel->read, *optionalGainMap); + } + SAFE_PARCEL(parcel->readFloat, &hdrSdrRatio); return NO_ERROR; } diff --git a/libs/gui/Surface.cpp b/libs/gui/Surface.cpp index 87fd448f0c..66e7ddd915 100644 --- a/libs/gui/Surface.cpp +++ b/libs/gui/Surface.cpp @@ -21,6 +21,8 @@ #include <gui/Surface.h> #include <condition_variable> +#include <cstddef> +#include <cstdint> #include <deque> #include <mutex> #include <thread> @@ -41,11 +43,9 @@ #include <ui/GraphicBuffer.h> #include <ui/Region.h> -#include <gui/AidlStatusUtil.h> +#include <gui/AidlUtil.h> #include <gui/BufferItem.h> -#include <gui/IProducerListener.h> - #include <gui/ISurfaceComposer.h> #include <gui/LayerState.h> #include <private/gui/ComposerService.h> @@ -77,9 +77,28 @@ bool isInterceptorRegistrationOp(int op) { } // namespace +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) +Surface::ProducerDeathListenerProxy::ProducerDeathListenerProxy(wp<SurfaceListener> surfaceListener) + : mSurfaceListener(surfaceListener) {} + +void Surface::ProducerDeathListenerProxy::binderDied(const wp<IBinder>&) { + sp<SurfaceListener> surfaceListener = mSurfaceListener.promote(); + if (!surfaceListener) { + return; + } + + if (surfaceListener->needsDeathNotify()) { + surfaceListener->onRemoteDied(); + } +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + Surface::Surface(const sp<IGraphicBufferProducer>& bufferProducer, bool controlledByApp, const sp<IBinder>& surfaceControlHandle) : mGraphicBufferProducer(bufferProducer), +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + mSurfaceDeathListener(nullptr), +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) mCrop(Rect::EMPTY_RECT), mBufferAge(0), mGenerationNumber(0), @@ -134,6 +153,12 @@ Surface::~Surface() { if (mConnectedToCpu) { Surface::disconnect(NATIVE_WINDOW_API_CPU); } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + if (mSurfaceDeathListener != nullptr) { + IInterface::asBinder(mGraphicBufferProducer)->unlinkToDeath(mSurfaceDeathListener); + mSurfaceDeathListener = nullptr; + } +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) } sp<ISurfaceComposer> Surface::composerService() const { @@ -163,6 +188,12 @@ void Surface::allocateBuffers() { mReqFormat, mReqUsage); } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) +status_t Surface::allowAllocation(bool allowAllocation) { + return mGraphicBufferProducer->allowAllocation(allowAllocation); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + status_t Surface::setGenerationNumber(uint32_t generation) { status_t result = mGraphicBufferProducer->setGenerationNumber(generation); if (result == NO_ERROR) { @@ -695,6 +726,51 @@ int Surface::dequeueBuffer(android_native_buffer_t** buffer, int* fenceFd) { return OK; } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + +status_t Surface::dequeueBuffer(sp<GraphicBuffer>* buffer, sp<Fence>* outFence) { + if (buffer == nullptr || outFence == nullptr) { + return BAD_VALUE; + } + + android_native_buffer_t* anb; + int fd = -1; + status_t res = dequeueBuffer(&anb, &fd); + *buffer = GraphicBuffer::from(anb); + *outFence = sp<Fence>::make(fd); + return res; +} + +status_t Surface::queueBuffer(const sp<GraphicBuffer>& buffer, const sp<Fence>& fd, + SurfaceQueueBufferOutput* output) { + if (buffer == nullptr) { + return BAD_VALUE; + } + return queueBuffer(buffer.get(), fd ? fd->get() : -1, output); +} + +status_t Surface::detachBuffer(const sp<GraphicBuffer>& buffer) { + if (nullptr == buffer) { + return BAD_VALUE; + } + + Mutex::Autolock lock(mMutex); + + uint64_t bufferId = buffer->getId(); + for (int slot = 0; slot < Surface::NUM_BUFFER_SLOTS; ++slot) { + auto& bufferSlot = mSlots[slot]; + if (bufferSlot.buffer != nullptr && bufferSlot.buffer->getId() == bufferId) { + bufferSlot.buffer = nullptr; + bufferSlot.dirtyRegion = Region::INVALID_REGION; + return mGraphicBufferProducer->detachBuffer(slot); + } + } + + return BAD_VALUE; +} + +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + int Surface::dequeueBuffers(std::vector<BatchBuffer>* buffers) { using DequeueBufferInput = IGraphicBufferProducer::DequeueBufferInput; using DequeueBufferOutput = IGraphicBufferProducer::DequeueBufferOutput; @@ -1118,6 +1194,140 @@ void Surface::onBufferQueuedLocked(int slot, sp<Fence> fence, } } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + +int Surface::queueBuffer(android_native_buffer_t* buffer, int fenceFd, + SurfaceQueueBufferOutput* surfaceOutput) { + ATRACE_CALL(); + ALOGV("Surface::queueBuffer"); + + IGraphicBufferProducer::QueueBufferOutput output; + IGraphicBufferProducer::QueueBufferInput input; + int slot; + sp<Fence> fence; + { + Mutex::Autolock lock(mMutex); + + slot = getSlotFromBufferLocked(buffer); + if (slot < 0) { + if (fenceFd >= 0) { + close(fenceFd); + } + return slot; + } + if (mSharedBufferSlot == slot && mSharedBufferHasBeenQueued) { + if (fenceFd >= 0) { + close(fenceFd); + } + return OK; + } + + getQueueBufferInputLocked(buffer, fenceFd, mTimestamp, &input); + applyGrallocMetadataLocked(buffer, input); + fence = input.fence; + } + nsecs_t now = systemTime(); + // Drop the lock temporarily while we touch the underlying producer. In the case of a local + // BufferQueue, the following should be allowable: + // + // Surface::queueBuffer + // -> IConsumerListener::onFrameAvailable callback triggers automatically + // -> implementation calls IGraphicBufferConsumer::acquire/release immediately + // -> SurfaceListener::onBufferRelesed callback triggers automatically + // -> implementation calls Surface::dequeueBuffer + status_t err = mGraphicBufferProducer->queueBuffer(slot, input, &output); + { + Mutex::Autolock lock(mMutex); + + mLastQueueDuration = systemTime() - now; + if (err != OK) { + ALOGE("queueBuffer: error queuing buffer, %d", err); + } + + onBufferQueuedLocked(slot, fence, output); + } + + if (surfaceOutput != nullptr) { + *surfaceOutput = {.bufferReplaced = output.bufferReplaced}; + } + + return err; +} + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) +int Surface::queueBuffers(const std::vector<BatchQueuedBuffer>& buffers, + std::vector<SurfaceQueueBufferOutput>* queueBufferOutputs) +#else +int Surface::queueBuffers(const std::vector<BatchQueuedBuffer>& buffers) +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) +{ + ATRACE_CALL(); + ALOGV("Surface::queueBuffers"); + + size_t numBuffers = buffers.size(); + std::vector<IGraphicBufferProducer::QueueBufferInput> igbpQueueBufferInputs(numBuffers); + std::vector<IGraphicBufferProducer::QueueBufferOutput> igbpQueueBufferOutputs; + std::vector<int> bufferSlots(numBuffers, -1); + std::vector<sp<Fence>> bufferFences(numBuffers); + + int err; + { + Mutex::Autolock lock(mMutex); + + if (mSharedBufferMode) { + ALOGE("%s: batched operation is not supported in shared buffer mode", __FUNCTION__); + return INVALID_OPERATION; + } + + for (size_t batchIdx = 0; batchIdx < numBuffers; batchIdx++) { + int i = getSlotFromBufferLocked(buffers[batchIdx].buffer); + if (i < 0) { + if (buffers[batchIdx].fenceFd >= 0) { + close(buffers[batchIdx].fenceFd); + } + return i; + } + bufferSlots[batchIdx] = i; + + IGraphicBufferProducer::QueueBufferInput input; + getQueueBufferInputLocked(buffers[batchIdx].buffer, buffers[batchIdx].fenceFd, + buffers[batchIdx].timestamp, &input); + input.slot = i; + bufferFences[batchIdx] = input.fence; + igbpQueueBufferInputs[batchIdx] = input; + } + } + nsecs_t now = systemTime(); + err = mGraphicBufferProducer->queueBuffers(igbpQueueBufferInputs, &igbpQueueBufferOutputs); + { + Mutex::Autolock lock(mMutex); + mLastQueueDuration = systemTime() - now; + if (err != OK) { + ALOGE("%s: error queuing buffer, %d", __FUNCTION__, err); + } + + for (size_t batchIdx = 0; batchIdx < numBuffers; batchIdx++) { + onBufferQueuedLocked(bufferSlots[batchIdx], bufferFences[batchIdx], + igbpQueueBufferOutputs[batchIdx]); + } + } + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + if (queueBufferOutputs != nullptr) { + queueBufferOutputs->clear(); + queueBufferOutputs->resize(numBuffers); + for (size_t batchIdx = 0; batchIdx < numBuffers; batchIdx++) { + (*queueBufferOutputs)[batchIdx].bufferReplaced = + igbpQueueBufferOutputs[batchIdx].bufferReplaced; + } + } +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + + return err; +} + +#else + int Surface::queueBuffer(android_native_buffer_t* buffer, int fenceFd) { ATRACE_CALL(); ALOGV("Surface::queueBuffer"); @@ -1205,6 +1415,8 @@ int Surface::queueBuffers(const std::vector<BatchQueuedBuffer>& buffers) { return err; } +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + void Surface::querySupportedTimestampsLocked() const { // mMutex must be locked when calling this method. @@ -1860,30 +2072,23 @@ bool Surface::transformToDisplayInverse() const { } int Surface::connect(int api) { - static sp<IProducerListener> listener = new StubProducerListener(); + static sp<SurfaceListener> listener = new StubSurfaceListener(); return connect(api, listener); } -int Surface::connect(int api, const sp<IProducerListener>& listener) { - return connect(api, listener, false); -} - -int Surface::connect( - int api, bool reportBufferRemoval, const sp<SurfaceListener>& sListener) { - if (sListener != nullptr) { - mListenerProxy = new ProducerListenerProxy(this, sListener); - } - return connect(api, mListenerProxy, reportBufferRemoval); -} - -int Surface::connect( - int api, const sp<IProducerListener>& listener, bool reportBufferRemoval) { +int Surface::connect(int api, const sp<SurfaceListener>& listener, bool reportBufferRemoval) { ATRACE_CALL(); ALOGV("Surface::connect"); Mutex::Autolock lock(mMutex); IGraphicBufferProducer::QueueBufferOutput output; mReportRemovedBuffers = reportBufferRemoval; - int err = mGraphicBufferProducer->connect(listener, api, mProducerControlledByApp, &output); + + if (listener != nullptr) { + mListenerProxy = new ProducerListenerProxy(this, listener); + } + + int err = + mGraphicBufferProducer->connect(mListenerProxy, api, mProducerControlledByApp, &output); if (err == NO_ERROR) { mDefaultWidth = output.width; mDefaultHeight = output.height; @@ -1898,6 +2103,13 @@ int Surface::connect( } mConsumerRunningBehind = (output.numPendingBuffers >= 2); + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + if (listener && listener->needsDeathNotify()) { + mSurfaceDeathListener = sp<ProducerDeathListenerProxy>::make(listener); + IInterface::asBinder(mGraphicBufferProducer)->linkToDeath(mSurfaceDeathListener); + } +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) } if (!err && api == NATIVE_WINDOW_API_CPU) { mConnectedToCpu = true; @@ -1911,7 +2123,6 @@ int Surface::connect( return err; } - int Surface::disconnect(int api, IGraphicBufferProducer::DisconnectMode mode) { ATRACE_CALL(); ALOGV("Surface::disconnect"); @@ -1939,6 +2150,14 @@ int Surface::disconnect(int api, IGraphicBufferProducer::DisconnectMode mode) { mConnectedToCpu = false; } } + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + if (mSurfaceDeathListener != nullptr) { + IInterface::asBinder(mGraphicBufferProducer)->unlinkToDeath(mSurfaceDeathListener); + mSurfaceDeathListener = nullptr; + } +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + return err; } diff --git a/libs/gui/SurfaceComposerClient.cpp b/libs/gui/SurfaceComposerClient.cpp index af91bb3ae2..df58df43be 100644 --- a/libs/gui/SurfaceComposerClient.cpp +++ b/libs/gui/SurfaceComposerClient.cpp @@ -20,8 +20,11 @@ #include <stdint.h> #include <sys/types.h> +#include <com_android_graphics_libgui_flags.h> + #include <android/gui/BnWindowInfosReportedListener.h> #include <android/gui/DisplayState.h> +#include <android/gui/EdgeExtensionParameters.h> #include <android/gui/ISurfaceComposerClient.h> #include <android/gui/IWindowInfosListener.h> #include <android/gui/TrustedPresentationThresholds.h> @@ -40,7 +43,7 @@ #include <system/graphics.h> -#include <gui/AidlStatusUtil.h> +#include <gui/AidlUtil.h> #include <gui/BufferItemConsumer.h> #include <gui/CpuConsumer.h> #include <gui/IGraphicBufferProducer.h> @@ -86,7 +89,8 @@ int64_t generateId() { return (((int64_t)getpid()) << 32) | ++idCounter; } -void emptyCallback(nsecs_t, const sp<Fence>&, const std::vector<SurfaceControlStats>&) {} +constexpr int64_t INVALID_VSYNC = -1; + } // namespace const std::string SurfaceComposerClient::kEmpty{}; @@ -207,9 +211,168 @@ sp<SurfaceComposerClient> SurfaceComposerClient::getDefault() { return DefaultComposerClient::getComposerClient(); } +// --------------------------------------------------------------------------- + JankDataListener::~JankDataListener() { } +status_t JankDataListener::flushJankData() { + if (mLayerId == -1) { + return INVALID_OPERATION; + } + + binder::Status status = ComposerServiceAIDL::getComposerService()->flushJankData(mLayerId); + return statusTFromBinderStatus(status); +} + +std::mutex JankDataListenerFanOut::sFanoutInstanceMutex; +std::unordered_map<int32_t, sp<JankDataListenerFanOut>> JankDataListenerFanOut::sFanoutInstances; + +binder::Status JankDataListenerFanOut::onJankData(const std::vector<gui::JankData>& jankData) { + // Find the highest VSync ID. + int64_t lastVsync = jankData.empty() + ? 0 + : std::max_element(jankData.begin(), jankData.end(), + [](const gui::JankData& jd1, const gui::JankData& jd2) { + return jd1.frameVsyncId < jd2.frameVsyncId; + }) + ->frameVsyncId; + + // Fan out the jank data callback. + std::vector<wp<JankDataListener>> listenersToRemove; + for (auto listener : getActiveListeners()) { + if (!listener->onJankDataAvailable(jankData) || + (listener->mRemoveAfter >= 0 && listener->mRemoveAfter <= lastVsync)) { + listenersToRemove.push_back(listener); + } + } + + return removeListeners(listenersToRemove) + ? binder::Status::ok() + : binder::Status::fromExceptionCode(binder::Status::EX_NULL_POINTER); +} + +status_t JankDataListenerFanOut::addListener(sp<SurfaceControl> sc, sp<JankDataListener> listener) { + sp<IBinder> layer = sc->getHandle(); + if (layer == nullptr) { + return UNEXPECTED_NULL; + } + int32_t layerId = sc->getLayerId(); + + sFanoutInstanceMutex.lock(); + auto it = sFanoutInstances.find(layerId); + bool registerNeeded = it == sFanoutInstances.end(); + sp<JankDataListenerFanOut> fanout; + if (registerNeeded) { + fanout = sp<JankDataListenerFanOut>::make(layerId); + sFanoutInstances.insert({layerId, fanout}); + } else { + fanout = it->second; + } + + fanout->mMutex.lock(); + fanout->mListeners.insert(listener); + fanout->mMutex.unlock(); + + sFanoutInstanceMutex.unlock(); + + if (registerNeeded) { + binder::Status status = + ComposerServiceAIDL::getComposerService()->addJankListener(layer, fanout); + return statusTFromBinderStatus(status); + } + return OK; +} + +status_t JankDataListenerFanOut::removeListener(sp<JankDataListener> listener) { + int32_t layerId = listener->mLayerId; + if (layerId == -1) { + return INVALID_OPERATION; + } + + int64_t removeAfter = INVALID_VSYNC; + sp<JankDataListenerFanOut> fanout; + + sFanoutInstanceMutex.lock(); + auto it = sFanoutInstances.find(layerId); + if (it != sFanoutInstances.end()) { + fanout = it->second; + removeAfter = fanout->updateAndGetRemovalVSync(); + } + + if (removeAfter != INVALID_VSYNC) { + // Remove this instance from the map, so that no new listeners are added + // while we're scheduled to be removed. + sFanoutInstances.erase(layerId); + } + sFanoutInstanceMutex.unlock(); + + if (removeAfter < 0) { + return OK; + } + + binder::Status status = + ComposerServiceAIDL::getComposerService()->removeJankListener(layerId, fanout, + removeAfter); + return statusTFromBinderStatus(status); +} + +std::vector<sp<JankDataListener>> JankDataListenerFanOut::getActiveListeners() { + std::scoped_lock<std::mutex> lock(mMutex); + + std::vector<sp<JankDataListener>> listeners; + for (auto it = mListeners.begin(); it != mListeners.end();) { + auto listener = it->promote(); + if (!listener) { + it = mListeners.erase(it); + } else { + listeners.push_back(std::move(listener)); + it++; + } + } + return listeners; +} + +bool JankDataListenerFanOut::removeListeners(const std::vector<wp<JankDataListener>>& listeners) { + std::scoped_lock<std::mutex> fanoutLock(sFanoutInstanceMutex); + std::scoped_lock<std::mutex> listenersLock(mMutex); + + for (auto listener : listeners) { + mListeners.erase(listener); + } + + if (mListeners.empty()) { + sFanoutInstances.erase(mLayerId); + return false; + } + return true; +} + +int64_t JankDataListenerFanOut::updateAndGetRemovalVSync() { + std::scoped_lock<std::mutex> lock(mMutex); + if (mRemoveAfter >= 0) { + // We've already been scheduled to be removed. Don't schedule again. + return INVALID_VSYNC; + } + + int64_t removeAfter = 0; + for (auto it = mListeners.begin(); it != mListeners.end();) { + auto listener = it->promote(); + if (!listener) { + it = mListeners.erase(it); + } else if (listener->mRemoveAfter < 0) { + // We have at least one listener that's still interested. Don't remove. + return INVALID_VSYNC; + } else { + removeAfter = std::max(removeAfter, listener->mRemoveAfter); + it++; + } + } + + mRemoveAfter = removeAfter; + return removeAfter; +} + // --------------------------------------------------------------------------- // TransactionCompletedListener does not use ANDROID_SINGLETON_STATIC_INSTANCE because it needs @@ -256,14 +419,6 @@ CallbackId TransactionCompletedListener::addCallbackFunction( surfaceControls, CallbackId::Type callbackType) { std::lock_guard<std::mutex> lock(mMutex); - return addCallbackFunctionLocked(callbackFunction, surfaceControls, callbackType); -} - -CallbackId TransactionCompletedListener::addCallbackFunctionLocked( - const TransactionCompletedCallback& callbackFunction, - const std::unordered_set<sp<SurfaceControl>, SurfaceComposerClient::SCHash>& - surfaceControls, - CallbackId::Type callbackType) { startListeningLocked(); CallbackId callbackId(getNextIdLocked(), callbackType); @@ -272,33 +427,11 @@ CallbackId TransactionCompletedListener::addCallbackFunctionLocked( for (const auto& surfaceControl : surfaceControls) { callbackSurfaceControls[surfaceControl->getHandle()] = surfaceControl; - - if (callbackType == CallbackId::Type::ON_COMPLETE && - mJankListeners.count(surfaceControl->getLayerId()) != 0) { - callbackId.includeJankData = true; - } } return callbackId; } -void TransactionCompletedListener::addJankListener(const sp<JankDataListener>& listener, - sp<SurfaceControl> surfaceControl) { - std::lock_guard<std::mutex> lock(mMutex); - mJankListeners.insert({surfaceControl->getLayerId(), listener}); -} - -void TransactionCompletedListener::removeJankListener(const sp<JankDataListener>& listener) { - std::lock_guard<std::mutex> lock(mMutex); - for (auto it = mJankListeners.begin(); it != mJankListeners.end();) { - if (it->second == listener) { - it = mJankListeners.erase(it); - } else { - it++; - } - } -} - void TransactionCompletedListener::setReleaseBufferCallback(const ReleaseCallbackId& callbackId, ReleaseBufferCallback listener) { std::scoped_lock<std::mutex> lock(mMutex); @@ -325,32 +458,20 @@ void TransactionCompletedListener::removeSurfaceStatsListener(void* context, voi } void TransactionCompletedListener::addSurfaceControlToCallbacks( - SurfaceComposerClient::CallbackInfo& callbackInfo, - const sp<SurfaceControl>& surfaceControl) { + const sp<SurfaceControl>& surfaceControl, + const std::unordered_set<CallbackId, CallbackIdHash>& callbackIds) { std::lock_guard<std::mutex> lock(mMutex); - bool includingJankData = false; - for (auto callbackId : callbackInfo.callbackIds) { + for (auto callbackId : callbackIds) { mCallbacks[callbackId].surfaceControls.emplace(std::piecewise_construct, std::forward_as_tuple( surfaceControl->getHandle()), std::forward_as_tuple(surfaceControl)); - includingJankData = includingJankData || callbackId.includeJankData; - } - - // If no registered callback is requesting jank data, but there is a jank listener registered - // on the new surface control, add a synthetic callback that requests the jank data. - if (!includingJankData && mJankListeners.count(surfaceControl->getLayerId()) != 0) { - CallbackId callbackId = - addCallbackFunctionLocked(&emptyCallback, callbackInfo.surfaceControls, - CallbackId::Type::ON_COMPLETE); - callbackInfo.callbackIds.emplace(callbackId); } } void TransactionCompletedListener::onTransactionCompleted(ListenerStats listenerStats) { std::unordered_map<CallbackId, CallbackTranslation, CallbackIdHash> callbacksMap; - std::multimap<int32_t, sp<JankDataListener>> jankListenersMap; { std::lock_guard<std::mutex> lock(mMutex); @@ -366,7 +487,6 @@ void TransactionCompletedListener::onTransactionCompleted(ListenerStats listener * sp<SurfaceControl> that could possibly exist for the callbacks. */ callbacksMap = mCallbacks; - jankListenersMap = mJankListeners; for (const auto& transactionStats : listenerStats.transactionStats) { for (auto& callbackId : transactionStats.callbackIds) { mCallbacks.erase(callbackId); @@ -486,12 +606,6 @@ void TransactionCompletedListener::onTransactionCompleted(ListenerStats listener transactionStats.presentFence, surfaceStats); } } - - if (surfaceStats.jankData.empty()) continue; - auto jankRange = jankListenersMap.equal_range(layerId); - for (auto it = jankRange.first; it != jankRange.second; it++) { - it->second->onJankDataAvailable(surfaceStats.jankData); - } } } } @@ -713,7 +827,6 @@ SurfaceComposerClient::Transaction::Transaction() { SurfaceComposerClient::Transaction::Transaction(const Transaction& other) : mId(other.mId), - mTransactionNestCount(other.mTransactionNestCount), mAnimation(other.mAnimation), mEarlyWakeupStart(other.mEarlyWakeupStart), mEarlyWakeupEnd(other.mEarlyWakeupEnd), @@ -753,7 +866,6 @@ SurfaceComposerClient::Transaction::createFromParcel(const Parcel* parcel) { status_t SurfaceComposerClient::Transaction::readFromParcel(const Parcel* parcel) { const uint64_t transactionId = parcel->readUint64(); - const uint32_t transactionNestCount = parcel->readUint32(); const bool animation = parcel->readBool(); const bool earlyWakeupStart = parcel->readBool(); const bool earlyWakeupEnd = parcel->readBool(); @@ -850,7 +962,6 @@ status_t SurfaceComposerClient::Transaction::readFromParcel(const Parcel* parcel // Parsing was successful. Update the object. mId = transactionId; - mTransactionNestCount = transactionNestCount; mAnimation = animation; mEarlyWakeupStart = earlyWakeupStart; mEarlyWakeupEnd = earlyWakeupEnd; @@ -882,7 +993,6 @@ status_t SurfaceComposerClient::Transaction::writeToParcel(Parcel* parcel) const const_cast<SurfaceComposerClient::Transaction*>(this)->cacheBuffers(); parcel->writeUint64(mId); - parcel->writeUint32(mTransactionNestCount); parcel->writeBool(mAnimation); parcel->writeBool(mEarlyWakeupStart); parcel->writeBool(mEarlyWakeupEnd); @@ -1004,8 +1114,9 @@ SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::merge(Tr // register all surface controls for all callbackIds for this listener that is merging for (const auto& surfaceControl : currentProcessCallbackInfo.surfaceControls) { - mTransactionCompletedListener->addSurfaceControlToCallbacks(currentProcessCallbackInfo, - surfaceControl); + mTransactionCompletedListener + ->addSurfaceControlToCallbacks(surfaceControl, + currentProcessCallbackInfo.callbackIds); } } @@ -1033,7 +1144,6 @@ void SurfaceComposerClient::Transaction::clear() { mInputWindowCommands.clear(); mUncacheBuffers.clear(); mMayContainBuffer = false; - mTransactionNestCount = 0; mAnimation = false; mEarlyWakeupStart = false; mEarlyWakeupEnd = false; @@ -1059,7 +1169,8 @@ void SurfaceComposerClient::doUncacheBufferTransaction(uint64_t cacheId) { uncacheBuffer.token = BufferCache::getInstance().getToken(); uncacheBuffer.id = cacheId; Vector<ComposerState> composerStates; - status_t status = sf->setTransactionState(FrameTimelineInfo{}, composerStates, {}, + Vector<DisplayState> displayStates; + status_t status = sf->setTransactionState(FrameTimelineInfo{}, composerStates, displayStates, ISurfaceComposer::eOneWay, Transaction::getDefaultApplyToken(), {}, systemTime(), true, {uncacheBuffer}, false, {}, generateId(), {}); @@ -1361,7 +1472,7 @@ void SurfaceComposerClient::Transaction::registerSurfaceControlForCallback( auto& callbackInfo = mListenerCallbacks[TransactionCompletedListener::getIInstance()]; callbackInfo.surfaceControls.insert(sc); - mTransactionCompletedListener->addSurfaceControlToCallbacks(callbackInfo, sc); + mTransactionCompletedListener->addSurfaceControlToCallbacks(sc, callbackInfo.callbackIds); } SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setPosition( @@ -1940,8 +2051,9 @@ SurfaceComposerClient::Transaction::setFrameRateSelectionPriority(const sp<Surfa SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::addTransactionCallback( TransactionCompletedCallbackTakesContext callback, void* callbackContext, CallbackId::Type callbackType) { - auto callbackWithContext = std::bind(callback, callbackContext, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3); + auto callbackWithContext = + std::bind(std::move(callback), callbackContext, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3); const auto& surfaceControls = mListenerCallbacks[mTransactionCompletedListener].surfaceControls; CallbackId callbackId = @@ -1955,13 +2067,15 @@ SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::addTrans SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::addTransactionCompletedCallback( TransactionCompletedCallbackTakesContext callback, void* callbackContext) { - return addTransactionCallback(callback, callbackContext, CallbackId::Type::ON_COMPLETE); + return addTransactionCallback(std::move(callback), callbackContext, + CallbackId::Type::ON_COMPLETE); } SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::addTransactionCommittedCallback( TransactionCompletedCallbackTakesContext callback, void* callbackContext) { - return addTransactionCallback(callback, callbackContext, CallbackId::Type::ON_COMMIT); + return addTransactionCallback(std::move(callback), callbackContext, + CallbackId::Type::ON_COMMIT); } SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::notifyProducerDisconnect( @@ -2217,6 +2331,23 @@ SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setStret return *this; } +bool SurfaceComposerClient::flagEdgeExtensionEffectUseShader() { + return com::android::graphics::libgui::flags::edge_extension_shader(); +} + +SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setEdgeExtensionEffect( + const sp<SurfaceControl>& sc, const gui::EdgeExtensionParameters& effect) { + layer_state_t* s = getLayerState(sc); + if (!s) { + mStatus = BAD_INDEX; + return *this; + } + + s->what |= layer_state_t::eEdgeExtensionChanged; + s->edgeExtensionParameters = effect; + return *this; +} + SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setBufferCrop( const sp<SurfaceControl>& sc, const Rect& bufferCrop) { layer_state_t* s = getLayerState(sc); @@ -2262,6 +2393,22 @@ SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setDropI return *this; } +SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setBufferReleaseChannel( + const sp<SurfaceControl>& sc, + const std::shared_ptr<gui::BufferReleaseChannel::ProducerEndpoint>& channel) { + layer_state_t* s = getLayerState(sc); + if (!s) { + mStatus = BAD_INDEX; + return *this; + } + + s->what |= layer_state_t::eBufferReleaseChannelChanged; + s->bufferReleaseChannel = channel; + + registerSurfaceControlForCallback(sc); + return *this; +} + // --------------------------------------------------------------------------- DisplayState& SurfaceComposerClient::Transaction::getDisplayState(const sp<IBinder>& token) { diff --git a/libs/gui/SurfaceControl.cpp b/libs/gui/SurfaceControl.cpp index c5f9c38ca3..f126c0be2f 100644 --- a/libs/gui/SurfaceControl.cpp +++ b/libs/gui/SurfaceControl.cpp @@ -139,9 +139,9 @@ sp<Surface> SurfaceControl::generateSurfaceLocked() uint32_t ignore; auto flags = mCreateFlags & (ISurfaceComposerClient::eCursorWindow | ISurfaceComposerClient::eOpaque); - mBbqChild = mClient->createSurface(String8("bbq-wrapper"), 0, 0, mFormat, + mBbqChild = mClient->createSurface(String8::format("[BBQ] %s", mName.c_str()), 0, 0, mFormat, flags, mHandle, {}, &ignore); - mBbq = sp<BLASTBufferQueue>::make("bbq-adapter", mBbqChild, mWidth, mHeight, mFormat); + mBbq = sp<BLASTBufferQueue>::make("[BBQ]" + mName, mBbqChild, mWidth, mHeight, mFormat); // This surface is always consumed by SurfaceFlinger, so the // producerControlledByApp value doesn't matter; using false. diff --git a/libs/gui/WindowInfosListenerReporter.cpp b/libs/gui/WindowInfosListenerReporter.cpp index 0929b8e120..91c9a85149 100644 --- a/libs/gui/WindowInfosListenerReporter.cpp +++ b/libs/gui/WindowInfosListenerReporter.cpp @@ -15,7 +15,7 @@ */ #include <android/gui/ISurfaceComposer.h> -#include <gui/AidlStatusUtil.h> +#include <gui/AidlUtil.h> #include <gui/WindowInfosListenerReporter.h> #include "gui/WindowInfosUpdate.h" diff --git a/libs/gui/aidl/Android.bp b/libs/gui/aidl/Android.bp index 8ed08c2644..fd035f60f5 100644 --- a/libs/gui/aidl/Android.bp +++ b/libs/gui/aidl/Android.bp @@ -28,9 +28,6 @@ filegroup { ":libgui_extra_unstructured_aidl_files", "android/gui/BitTube.aidl", - "android/gui/CaptureArgs.aidl", - "android/gui/DisplayCaptureArgs.aidl", - "android/gui/LayerCaptureArgs.aidl", "android/gui/LayerMetadata.aidl", "android/gui/ParcelableVsyncEventData.aidl", "android/gui/ScreenCaptureResults.aidl", diff --git a/libs/gui/aidl/android/gui/CaptureArgs.aidl b/libs/gui/aidl/android/gui/CaptureArgs.aidl index 9f198cae10..4920344e0e 100644 --- a/libs/gui/aidl/android/gui/CaptureArgs.aidl +++ b/libs/gui/aidl/android/gui/CaptureArgs.aidl @@ -16,4 +16,63 @@ package android.gui; -parcelable CaptureArgs cpp_header "gui/DisplayCaptureArgs.h" rust_type "gui_aidl_types_rs::CaptureArgs"; +import android.gui.ARect; + +// Common arguments for capturing content on-screen +parcelable CaptureArgs { + const int UNSET_UID = -1; + + // Desired pixel format of the final screenshotted buffer + int /*ui::PixelFormat*/ pixelFormat = 1; + + // Crop in layer space: all content outside of the crop will not be captured. + ARect sourceCrop; + + // Scale in the x-direction for the screenshotted result. + float frameScaleX = 1.0f; + + // Scale in the y-direction for the screenshotted result. + float frameScaleY = 1.0f; + + // True if capturing secure layers is permitted + boolean captureSecureLayers = false; + + // UID whose content we want to screenshot + int uid = UNSET_UID; + + // Force capture to be in a color space. If the value is ui::Dataspace::UNKNOWN, the captured + // result will be in a colorspace appropriate for capturing the display contents + // The display may use non-RGB dataspace (ex. displayP3) that could cause pixel data could be + // different from SRGB (byte per color), and failed when checking colors in tests. + // NOTE: In normal cases, we want the screen to be captured in display's colorspace. + int /*ui::Dataspace*/ dataspace = 0; + + // The receiver of the capture can handle protected buffer. A protected buffer has + // GRALLOC_USAGE_PROTECTED usage bit and must not be accessed unprotected behaviour. + // Any read/write access from unprotected context will result in undefined behaviour. + // Protected contents are typically DRM contents. This has no direct implication to the + // secure property of the surface, which is specified by the application explicitly to avoid + // the contents being accessed/captured by screenshot or unsecure display. + boolean allowProtected = false; + + // True if the content should be captured in grayscale + boolean grayscale = false; + + // List of layers to exclude capturing from + IBinder[] excludeHandles; + + // Hint that the caller will use the screenshot animation as part of a transition animation. + // The canonical example would be screen rotation - in such a case any color shift in the + // screenshot is a detractor so composition in the display's colorspace is required. + // Otherwise, the system may choose a colorspace that is more appropriate for use-cases + // such as file encoding or for blending HDR content into an ap's UI, where the display's + // exact colorspace is not an appropriate intermediate result. + // Note that if the caller is requesting a specific dataspace, this hint does nothing. + boolean hintForSeamlessTransition = false; + + // Allows the screenshot to attach a gainmap, which allows for a per-pixel + // transformation of the screenshot to another luminance range, typically + // mapping an SDR base image into HDR. + boolean attachGainmap = false; +} + diff --git a/libs/gui/aidl/android/gui/DisplayCaptureArgs.aidl b/libs/gui/aidl/android/gui/DisplayCaptureArgs.aidl index fc97dbf03d..e00a2dfa82 100644 --- a/libs/gui/aidl/android/gui/DisplayCaptureArgs.aidl +++ b/libs/gui/aidl/android/gui/DisplayCaptureArgs.aidl @@ -16,5 +16,18 @@ package android.gui; -parcelable DisplayCaptureArgs cpp_header "gui/DisplayCaptureArgs.h" rust_type "gui_aidl_types_rs::DisplayCaptureArgs"; +import android.gui.CaptureArgs; + +// Arguments for screenshotting an entire display +parcelable DisplayCaptureArgs { + CaptureArgs captureArgs; + + // The display that we want to screenshot + IBinder displayToken; + + // The width of the render area when we screenshot + int width = 0; + // The length of the render area when we screenshot + int height = 0; +} diff --git a/libs/tracing_perfetto/include/trace_result.h b/libs/gui/aidl/android/gui/EdgeExtensionParameters.aidl index f7581fc0fb..44f4259f74 100644 --- a/libs/tracing_perfetto/include/trace_result.h +++ b/libs/gui/aidl/android/gui/EdgeExtensionParameters.aidl @@ -14,17 +14,14 @@ * limitations under the License. */ -#ifndef TRACE_RESULT_H -#define TRACE_RESULT_H +package android.gui; -namespace tracing_perfetto { -enum class Result { - SUCCESS, - NOT_SUPPORTED, - INVALID_INPUT, -}; - -} - -#endif // TRACE_RESULT_H +/** @hide */ +parcelable EdgeExtensionParameters { + // These represent the translation of the window as requested by the animation + boolean extendRight; + boolean extendLeft; + boolean extendTop; + boolean extendBottom; +}
\ No newline at end of file diff --git a/libs/gui/aidl/android/gui/IJankListener.aidl b/libs/gui/aidl/android/gui/IJankListener.aidl new file mode 100644 index 0000000000..2bfd1af69d --- /dev/null +++ b/libs/gui/aidl/android/gui/IJankListener.aidl @@ -0,0 +1,29 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.gui; + +import android.gui.JankData; + +/** @hide */ +interface IJankListener { + + /** + * Callback reporting jank data of the most recent frames. + * @See {@link ISurfaceComposer#addJankListener(IBinder, IJankListener)} + */ + void onJankData(in JankData[] data); +} diff --git a/libs/gui/aidl/android/gui/ISurfaceComposer.aidl b/libs/gui/aidl/android/gui/ISurfaceComposer.aidl index 6d018ea7ef..ac14138e8c 100644 --- a/libs/gui/aidl/android/gui/ISurfaceComposer.aidl +++ b/libs/gui/aidl/android/gui/ISurfaceComposer.aidl @@ -42,6 +42,7 @@ import android.gui.ISurfaceComposerClient; import android.gui.ITunnelModeEnabledListener; import android.gui.IWindowInfosListener; import android.gui.IWindowInfosPublisher; +import android.gui.IJankListener; import android.gui.LayerCaptureArgs; import android.gui.OverlayProperties; import android.gui.PullAtomData; @@ -580,4 +581,22 @@ interface ISurfaceComposer { * This method should not block the ShutdownThread therefore it's handled asynchronously. */ oneway void notifyShutdown(); + + /** + * Registers the jank listener on the given layer to receive jank data of future frames. + */ + void addJankListener(IBinder layer, IJankListener listener); + + /** + * Flushes any pending jank data on the given layer to any registered listeners on that layer. + */ + oneway void flushJankData(int layerId); + + /** + * Schedules the removal of the jank listener from the given layer after the VSync with the + * specified ID. Use a value <= 0 for afterVsync to remove the listener immediately. The given + * listener will not be removed before the given VSync, but may still receive data for frames + * past the provided VSync. + */ + oneway void removeJankListener(int layerId, IJankListener listener, long afterVsync); } diff --git a/libs/gui/aidl/android/gui/JankData.aidl b/libs/gui/aidl/android/gui/JankData.aidl new file mode 100644 index 0000000000..ec13681182 --- /dev/null +++ b/libs/gui/aidl/android/gui/JankData.aidl @@ -0,0 +1,45 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package android.gui; + + /** @hide */ +parcelable JankData { + /** + * Identifier for the frame submitted with Transaction.setFrameTimelineVsyncId + */ + long frameVsyncId; + + /** + * Bitmask of jank types that occurred. + */ + int jankType; + + /** + * Time between frames in nanoseconds. + */ + long frameIntervalNs; + + /** + * Time allocated to the application to render this frame. + */ + long scheduledAppFrameTimeNs; + + /** + * Time taken by the application to render this frame. + */ + long actualAppFrameTimeNs; +} diff --git a/libs/gui/aidl/android/gui/LayerCaptureArgs.aidl b/libs/gui/aidl/android/gui/LayerCaptureArgs.aidl index 18d293f211..004c35a5ce 100644 --- a/libs/gui/aidl/android/gui/LayerCaptureArgs.aidl +++ b/libs/gui/aidl/android/gui/LayerCaptureArgs.aidl @@ -16,4 +16,15 @@ package android.gui; -parcelable LayerCaptureArgs cpp_header "gui/LayerCaptureArgs.h" rust_type "gui_aidl_types_rs::LayerCaptureArgs"; +import android.gui.CaptureArgs; + +// Arguments for capturing a layer and/or its children +parcelable LayerCaptureArgs { + CaptureArgs captureArgs; + + // The Layer that we may want to capture. We would also capture its children + IBinder layerHandle; + // True if we don't actually want to capture the layer and want to capture + // its children instead. + boolean childrenOnly = false; +} diff --git a/libs/gui/aidl/android/gui/ScreenCaptureResults.aidl b/libs/gui/aidl/android/gui/ScreenCaptureResults.aidl index 97a903515b..f4ef16dc71 100644 --- a/libs/gui/aidl/android/gui/ScreenCaptureResults.aidl +++ b/libs/gui/aidl/android/gui/ScreenCaptureResults.aidl @@ -16,4 +16,4 @@ package android.gui; -parcelable ScreenCaptureResults cpp_header "gui/ScreenCaptureResults.h" rust_type "gui_aidl_types_rs::ScreenCaptureResults";
\ No newline at end of file +parcelable ScreenCaptureResults cpp_header "gui/ScreenCaptureResults.h" rust_type "gui_aidl_types_rs::ScreenCaptureResults"; diff --git a/libs/gui/include/gui/AidlStatusUtil.h b/libs/gui/include/gui/AidlUtil.h index 55be27bf35..a3ecd84ba1 100644 --- a/libs/gui/include/gui/AidlStatusUtil.h +++ b/libs/gui/include/gui/AidlUtil.h @@ -16,9 +16,11 @@ #pragma once +#include <android/gui/ARect.h> #include <binder/Status.h> +#include <ui/Rect.h> -// Extracted from frameworks/av/media/libaudioclient/include/media/AidlConversionUtil.h +// Originally extracted from frameworks/av/media/libaudioclient/include/media/AidlConversionUtil.h namespace android::gui::aidl_utils { /** @@ -68,7 +70,7 @@ static inline status_t statusTFromExceptionCode(int32_t exceptionCode) { * * return_type method(type0 param0, ...) */ -static inline status_t statusTFromBinderStatus(const ::android::binder::Status &status) { +static inline status_t statusTFromBinderStatus(const ::android::binder::Status& status) { return status.isOk() ? OK // check OK, : status.serviceSpecificErrorCode() // service-side error, not standard Java exception // (fromServiceSpecificError) @@ -84,8 +86,8 @@ static inline status_t statusTFromBinderStatus(const ::android::binder::Status & * where Java callers expect an exception, not an integer return value. */ static inline ::android::binder::Status binderStatusFromStatusT( - status_t status, const char *optionalMessage = nullptr) { - const char *const emptyIfNull = optionalMessage == nullptr ? "" : optionalMessage; + status_t status, const char* optionalMessage = nullptr) { + const char* const emptyIfNull = optionalMessage == nullptr ? "" : optionalMessage; // From binder::Status instructions: // Prefer a generic exception code when possible, then a service specific // code, and finally a status_t for low level failures or legacy support. @@ -111,4 +113,26 @@ static inline ::android::binder::Status binderStatusFromStatusT( return Status::fromServiceSpecificError(status, emptyIfNull); } +static inline Rect fromARect(ARect rect) { + return Rect(rect.left, rect.top, rect.right, rect.bottom); +} + +static inline ARect toARect(Rect rect) { + ARect aRect; + + aRect.left = rect.left; + aRect.top = rect.top; + aRect.right = rect.right; + aRect.bottom = rect.bottom; + return aRect; +} + +static inline ARect toARect(int32_t left, int32_t top, int32_t right, int32_t bottom) { + return toARect(Rect(left, top, right, bottom)); +} + +static inline ARect toARect(int32_t width, int32_t height) { + return toARect(Rect(width, height)); +} + } // namespace android::gui::aidl_utils diff --git a/libs/gui/include/gui/BLASTBufferQueue.h b/libs/gui/include/gui/BLASTBufferQueue.h index 0e1a505c69..8592cffd15 100644 --- a/libs/gui/include/gui/BLASTBufferQueue.h +++ b/libs/gui/include/gui/BLASTBufferQueue.h @@ -17,9 +17,10 @@ #ifndef ANDROID_GUI_BLAST_BUFFER_QUEUE_H #define ANDROID_GUI_BLAST_BUFFER_QUEUE_H +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferItem.h> #include <gui/BufferItemConsumer.h> - +#include <gui/IGraphicBufferConsumer.h> #include <gui/IGraphicBufferProducer.h> #include <gui/SurfaceComposerClient.h> @@ -28,7 +29,6 @@ #include <utils/RefBase.h> #include <system/window.h> -#include <thread> #include <queue> #include <com_android_graphics_libgui_flags.h> @@ -40,12 +40,20 @@ class BufferItemConsumer; class BLASTBufferItemConsumer : public BufferItemConsumer { public: +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + BLASTBufferItemConsumer(const sp<IGraphicBufferProducer>& producer, + const sp<IGraphicBufferConsumer>& consumer, uint64_t consumerUsage, + int bufferCount, bool controlledByApp, wp<BLASTBufferQueue> bbq) + : BufferItemConsumer(producer, consumer, consumerUsage, bufferCount, controlledByApp), +#else BLASTBufferItemConsumer(const sp<IGraphicBufferConsumer>& consumer, uint64_t consumerUsage, int bufferCount, bool controlledByApp, wp<BLASTBufferQueue> bbq) : BufferItemConsumer(consumer, consumerUsage, bufferCount, controlledByApp), +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) mBLASTBufferQueue(std::move(bbq)), mCurrentlyConnected(false), - mPreviouslyConnected(false) {} + mPreviouslyConnected(false) { + } void onDisconnect() override EXCLUDES(mMutex); void addAndGetFrameTimestamps(const NewFrameEventsEntry* newTimestamps, @@ -95,15 +103,21 @@ public: void onFrameDequeued(const uint64_t) override; void onFrameCancelled(const uint64_t) override; + TransactionCompletedCallbackTakesContext makeTransactionCommittedCallbackThunk(); void transactionCommittedCallback(nsecs_t latchTime, const sp<Fence>& presentFence, const std::vector<SurfaceControlStats>& stats); + + TransactionCompletedCallbackTakesContext makeTransactionCallbackThunk(); virtual void transactionCallback(nsecs_t latchTime, const sp<Fence>& presentFence, const std::vector<SurfaceControlStats>& stats); + + ReleaseBufferCallback makeReleaseBufferCallbackThunk(); void releaseBufferCallback(const ReleaseCallbackId& id, const sp<Fence>& releaseFence, std::optional<uint32_t> currentMaxAcquiredBufferCount); void releaseBufferCallbackLocked(const ReleaseCallbackId& id, const sp<Fence>& releaseFence, std::optional<uint32_t> currentMaxAcquiredBufferCount, bool fakeRelease) REQUIRES(mMutex); + bool syncNextTransaction(std::function<void(SurfaceComposerClient::Transaction*)> callback, bool acquireSingleBuffer = true); void stopContinuousSyncTransaction(); @@ -128,9 +142,11 @@ public: * indicates the reason for the hang. */ void setTransactionHangCallback(std::function<void(const std::string&)> callback); - + void setApplyToken(sp<IBinder>); virtual ~BLASTBufferQueue(); + void onFirstRef() override; + private: friend class BLASTBufferQueueHelper; friend class BBQBufferQueueProducer; @@ -170,8 +186,7 @@ private: // BufferQueue internally allows 1 more than // the max to be acquired - int32_t mMaxAcquiredBuffers = 1; - + int32_t mMaxAcquiredBuffers GUARDED_BY(mMutex) = 1; int32_t mNumFrameAvailable GUARDED_BY(mMutex) = 0; int32_t mNumAcquired GUARDED_BY(mMutex) = 0; @@ -256,7 +271,7 @@ private: // Queues up transactions using this token in SurfaceFlinger. This prevents queued up // transactions from other parts of the client from blocking this transaction. - const sp<IBinder> mApplyToken GUARDED_BY(mMutex) = sp<BBinder>::make(); + sp<IBinder> mApplyToken GUARDED_BY(mMutex) = sp<BBinder>::make(); // Guards access to mDequeueTimestamps since we cannot hold to mMutex in onFrameDequeued or // we will deadlock. @@ -300,6 +315,51 @@ private: std::function<void(const std::string&)> mTransactionHangCallback; std::unordered_set<uint64_t> mSyncedFrameNumbers GUARDED_BY(mMutex); + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BUFFER_RELEASE_CHANNEL) + class BufferReleaseReader { + public: + BufferReleaseReader() = default; + BufferReleaseReader(std::unique_ptr<gui::BufferReleaseChannel::ConsumerEndpoint>); + BufferReleaseReader& operator=(BufferReleaseReader&&); + + // Block until we can read a buffer release message. + // + // Returns: + // * OK if a ReleaseCallbackId and Fence were successfully read. + // * WOULD_BLOCK if the blocking read was interrupted by interruptBlockingRead. + // * UNKNOWN_ERROR if something went wrong. + status_t readBlocking(ReleaseCallbackId& outId, sp<Fence>& outReleaseFence, + uint32_t& outMaxAcquiredBufferCount); + + // Signals the reader's eventfd to wake up any threads waiting on readBlocking. + void interruptBlockingRead(); + + private: + std::mutex mMutex; + std::unique_ptr<gui::BufferReleaseChannel::ConsumerEndpoint> mEndpoint GUARDED_BY(mMutex); + android::base::unique_fd mEpollFd; + android::base::unique_fd mEventFd; + }; + + // BufferReleaseChannel is used to communicate buffer releases from SurfaceFlinger to + // the client. See BBQBufferQueueProducer::dequeueBuffer for details. + std::shared_ptr<BufferReleaseReader> mBufferReleaseReader; + std::shared_ptr<gui::BufferReleaseChannel::ProducerEndpoint> mBufferReleaseProducer; + + class BufferReleaseThread { + public: + BufferReleaseThread() = default; + ~BufferReleaseThread(); + void start(const sp<BLASTBufferQueue>&); + + private: + std::shared_ptr<std::atomic_bool> mRunning; + std::shared_ptr<BufferReleaseReader> mReader; + }; + + BufferReleaseThread mBufferReleaseThread; +#endif }; } // namespace android diff --git a/libs/gui/include/gui/BufferItemConsumer.h b/libs/gui/include/gui/BufferItemConsumer.h index a905610ee2..6810edaf7c 100644 --- a/libs/gui/include/gui/BufferItemConsumer.h +++ b/libs/gui/include/gui/BufferItemConsumer.h @@ -17,13 +17,15 @@ #ifndef ANDROID_GUI_BUFFERITEMCONSUMER_H #define ANDROID_GUI_BUFFERITEMCONSUMER_H -#include <gui/ConsumerBase.h> +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferQueue.h> +#include <gui/ConsumerBase.h> #define ANDROID_GRAPHICS_BUFFERITEMCONSUMER_JNI_ID "mBufferItemConsumer" namespace android { +class GraphicBuffer; class String8; /** @@ -51,9 +53,17 @@ class BufferItemConsumer: public ConsumerBase // access at the same time. // controlledByApp tells whether this consumer is controlled by the // application. +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + BufferItemConsumer(uint64_t consumerUsage, int bufferCount = DEFAULT_MAX_BUFFERS, + bool controlledByApp = false, bool isConsumerSurfaceFlinger = false); + BufferItemConsumer(const sp<IGraphicBufferConsumer>& consumer, uint64_t consumerUsage, + int bufferCount = DEFAULT_MAX_BUFFERS, bool controlledByApp = false) + __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); +#else BufferItemConsumer(const sp<IGraphicBufferConsumer>& consumer, uint64_t consumerUsage, int bufferCount = DEFAULT_MAX_BUFFERS, bool controlledByApp = false); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) ~BufferItemConsumer() override; @@ -85,7 +95,25 @@ class BufferItemConsumer: public ConsumerBase status_t releaseBuffer(const BufferItem &item, const sp<Fence>& releaseFence = Fence::NO_FENCE); - private: +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + status_t releaseBuffer(const sp<GraphicBuffer>& buffer, + const sp<Fence>& releaseFence = Fence::NO_FENCE); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + +protected: +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // This should only be used by BLASTBufferQueue: + BufferItemConsumer(const sp<IGraphicBufferProducer>& producer, + const sp<IGraphicBufferConsumer>& consumer, uint64_t consumerUsage, + int bufferCount = DEFAULT_MAX_BUFFERS, bool controlledByApp = false); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + +private: + void initialize(uint64_t consumerUsage, int bufferCount); + + status_t releaseBufferSlotLocked(int slotIndex, const sp<GraphicBuffer>& buffer, + const sp<Fence>& releaseFence); + void freeBufferLocked(int slotIndex) override; // mBufferFreedListener is the listener object that will be called when diff --git a/libs/gui/include/gui/BufferReleaseChannel.h b/libs/gui/include/gui/BufferReleaseChannel.h new file mode 100644 index 0000000000..51fe0b6fab --- /dev/null +++ b/libs/gui/include/gui/BufferReleaseChannel.h @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <string> +#include <vector> + +#include <android-base/unique_fd.h> + +#include <binder/Parcelable.h> +#include <gui/ITransactionCompletedListener.h> +#include <ui/Fence.h> +#include <utils/Errors.h> + +namespace android::gui { + +/** + * IPC wrapper to pass release fences from SurfaceFlinger to apps via a local unix domain socket. + */ +class BufferReleaseChannel { +private: + class Endpoint { + public: + Endpoint(std::string name, android::base::unique_fd fd) + : mName(std::move(name)), mFd(std::move(fd)) {} + Endpoint() {} + + Endpoint(Endpoint&&) noexcept = default; + Endpoint& operator=(Endpoint&&) noexcept = default; + + Endpoint(const Endpoint&) = delete; + void operator=(const Endpoint&) = delete; + + const android::base::unique_fd& getFd() const { return mFd; } + + protected: + std::string mName; + android::base::unique_fd mFd; + }; + +public: + class ConsumerEndpoint : public Endpoint { + public: + ConsumerEndpoint(std::string name, android::base::unique_fd fd) + : Endpoint(std::move(name), std::move(fd)) {} + + /** + * Reads a release fence from the BufferReleaseChannel. + * + * Returns OK on success. + * Returns WOULD_BLOCK if there is no fence present. + * Other errors probably indicate that the channel is broken. + */ + status_t readReleaseFence(ReleaseCallbackId& outReleaseCallbackId, + sp<Fence>& outReleaseFence, uint32_t& maxAcquiredBufferCount); + + private: + std::vector<uint8_t> mFlattenedBuffer; + }; + + class ProducerEndpoint : public Endpoint, public Parcelable { + public: + ProducerEndpoint(std::string name, android::base::unique_fd fd) + : Endpoint(std::move(name), std::move(fd)) {} + ProducerEndpoint() {} + + status_t readFromParcel(const android::Parcel* parcel) override; + status_t writeToParcel(android::Parcel* parcel) const override; + + status_t writeReleaseFence(const ReleaseCallbackId&, const sp<Fence>& releaseFence, + uint32_t maxAcquiredBufferCount); + + private: + std::vector<uint8_t> mFlattenedBuffer; + }; + + /** + * Create two endpoints that make up the BufferReleaseChannel. + * + * Return OK on success. + */ + static status_t open(const std::string name, std::unique_ptr<ConsumerEndpoint>& outConsumer, + std::shared_ptr<ProducerEndpoint>& outProducer); + + struct Message : public Flattenable<Message> { + ReleaseCallbackId releaseCallbackId; + sp<Fence> releaseFence = Fence::NO_FENCE; + uint32_t maxAcquiredBufferCount; + + Message() = default; + Message(ReleaseCallbackId releaseCallbackId, sp<Fence> releaseFence, + uint32_t maxAcquiredBufferCount) + : releaseCallbackId{releaseCallbackId}, + releaseFence{std::move(releaseFence)}, + maxAcquiredBufferCount{maxAcquiredBufferCount} {} + + // Flattenable protocol + size_t getFlattenedSize() const; + + size_t getFdCount() const { return releaseFence->getFdCount(); } + + status_t flatten(void*& buffer, size_t& size, int*& fds, size_t& count) const; + + status_t unflatten(void const*& buffer, size_t& size, int const*& fds, size_t& count); + + private: + size_t getPodSize() const; + }; +}; + +} // namespace android::gui diff --git a/libs/gui/include/gui/ConsumerBase.h b/libs/gui/include/gui/ConsumerBase.h index 8ff0cd0f6e..e976aa48be 100644 --- a/libs/gui/include/gui/ConsumerBase.h +++ b/libs/gui/include/gui/ConsumerBase.h @@ -17,18 +17,17 @@ #ifndef ANDROID_GUI_CONSUMERBASE_H #define ANDROID_GUI_CONSUMERBASE_H +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferQueueDefs.h> #include <gui/IConsumerListener.h> #include <gui/IGraphicBufferConsumer.h> +#include <gui/IGraphicBufferProducer.h> #include <gui/OccupancyTracker.h> - #include <ui/PixelFormat.h> - #include <utils/String8.h> #include <utils/Vector.h> #include <utils/threads.h> - namespace android { // ---------------------------------------------------------------------------- @@ -76,12 +75,28 @@ public: void dumpState(String8& result) const; void dumpState(String8& result, const char* prefix) const; +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // Returns a Surface that can be used as the producer for this consumer. + sp<Surface> getSurface() const; + + // DEPRECATED, DO NOT USE. Returns the underlying IGraphicBufferConsumer + // that backs this ConsumerBase. + sp<IGraphicBufferConsumer> getIGraphicBufferConsumer() const + __attribute((deprecated("DO NOT USE: Temporary hack for refactoring"))); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // setFrameAvailableListener sets the listener object that will be notified // when a new frame becomes available. void setFrameAvailableListener(const wp<FrameAvailableListener>& listener); // See IGraphicBufferConsumer::detachBuffer - status_t detachBuffer(int slot); + status_t detachBuffer(int slot) __attribute(( + deprecated("Please use the GraphicBuffer variant--slots are deprecated."))); + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + // See IGraphicBufferConsumer::detachBuffer + status_t detachBuffer(const sp<GraphicBuffer>& buffer); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) // See IGraphicBufferConsumer::setDefaultBufferSize status_t setDefaultBufferSize(uint32_t width, uint32_t height); @@ -98,9 +113,18 @@ public: // See IGraphicBufferConsumer::setTransformHint status_t setTransformHint(uint32_t hint); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // See IGraphicBufferConsumer::setMaxBufferCount + status_t setMaxBufferCount(int bufferCount); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // See IGraphicBufferConsumer::setMaxAcquiredBufferCount status_t setMaxAcquiredBufferCount(int maxAcquiredBuffers); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + status_t setConsumerIsProtected(bool isProtected); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // See IGraphicBufferConsumer::getSidebandStream sp<NativeHandle> getSidebandStream() const; @@ -115,12 +139,24 @@ private: ConsumerBase(const ConsumerBase&); void operator=(const ConsumerBase&); + void initialize(bool controlledByApp); + protected: // ConsumerBase constructs a new ConsumerBase object to consume image // buffers from the given IGraphicBufferConsumer. // The controlledByApp flag indicates that this consumer is under the application's // control. +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + explicit ConsumerBase(bool controlledByApp = false, bool consumerIsSurfaceFlinger = false); + explicit ConsumerBase(const sp<IGraphicBufferProducer>& producer, + const sp<IGraphicBufferConsumer>& consumer, bool controlledByApp = false); + + explicit ConsumerBase(const sp<IGraphicBufferConsumer>& consumer, bool controlledByApp = false) + __attribute((deprecated("ConsumerBase should own its own producer, and constructing it " + "without one is fragile! This method is going away soon."))); +#else explicit ConsumerBase(const sp<IGraphicBufferConsumer>& consumer, bool controlledByApp = false); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) // onLastStrongRef gets called by RefBase just before the dtor of the most // derived class. It is used to clean up the buffers so that ConsumerBase @@ -150,6 +186,10 @@ protected: virtual void onBuffersReleased() override; virtual void onSidebandStreamChanged() override; + virtual int getSlotForBufferLocked(const sp<GraphicBuffer>& buffer); + + virtual status_t detachBufferLocked(int slotIndex); + // freeBufferLocked frees up the given buffer slot. If the slot has been // initialized this will release the reference to the GraphicBuffer in that // slot. Otherwise it has no effect. @@ -264,10 +304,16 @@ protected: Mutex mFrameAvailableMutex; wp<FrameAvailableListener> mFrameAvailableListener; - // The ConsumerBase has-a BufferQueue and is responsible for creating this object - // if none is supplied + // The ConsumerBase has-a BufferQueue and is responsible for creating these + // objects if not supplied. sp<IGraphicBufferConsumer> mConsumer; +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // This Surface wraps the IGraphicBufferConsumer created for this + // ConsumerBase. + sp<Surface> mSurface; +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + // The final release fence of the most recent buffer released by // releaseBufferLocked. sp<Fence> mPrevFinalReleaseFence; diff --git a/libs/gui/include/gui/CpuConsumer.h b/libs/gui/include/gui/CpuConsumer.h index 806fbe8aa0..2bba61bbe8 100644 --- a/libs/gui/include/gui/CpuConsumer.h +++ b/libs/gui/include/gui/CpuConsumer.h @@ -19,8 +19,9 @@ #include <system/window.h> -#include <gui/ConsumerBase.h> +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferQueue.h> +#include <gui/ConsumerBase.h> #include <utils/Vector.h> @@ -91,8 +92,17 @@ class CpuConsumer : public ConsumerBase // Create a new CPU consumer. The maxLockedBuffers parameter specifies // how many buffers can be locked for user access at the same time. +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + CpuConsumer(size_t maxLockedBuffers, bool controlledByApp = false, + bool isConsumerSurfaceFlinger = false); + + CpuConsumer(const sp<IGraphicBufferConsumer>& bq, size_t maxLockedBuffers, + bool controlledByApp = false) + __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); +#else CpuConsumer(const sp<IGraphicBufferConsumer>& bq, size_t maxLockedBuffers, bool controlledByApp = false); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) // Gets the next graphics buffer from the producer and locks it for CPU use, // filling out the passed-in locked buffer structure with the native pointer diff --git a/libs/gui/include/gui/DisplayCaptureArgs.h b/libs/gui/include/gui/DisplayCaptureArgs.h deleted file mode 100644 index e29ce41bd5..0000000000 --- a/libs/gui/include/gui/DisplayCaptureArgs.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <stdint.h> -#include <sys/types.h> - -#include <binder/IBinder.h> -#include <binder/Parcel.h> -#include <binder/Parcelable.h> -#include <gui/SpHash.h> -#include <ui/GraphicTypes.h> -#include <ui/PixelFormat.h> -#include <ui/Rect.h> -#include <unordered_set> - -namespace android::gui { - -struct CaptureArgs : public Parcelable { - const static int32_t UNSET_UID = -1; - virtual ~CaptureArgs() = default; - - ui::PixelFormat pixelFormat{ui::PixelFormat::RGBA_8888}; - Rect sourceCrop; - float frameScaleX{1}; - float frameScaleY{1}; - bool captureSecureLayers{false}; - int32_t uid{UNSET_UID}; - // Force capture to be in a color space. If the value is ui::Dataspace::UNKNOWN, the captured - // result will be in a colorspace appropriate for capturing the display contents - // The display may use non-RGB dataspace (ex. displayP3) that could cause pixel data could be - // different from SRGB (byte per color), and failed when checking colors in tests. - // NOTE: In normal cases, we want the screen to be captured in display's colorspace. - ui::Dataspace dataspace = ui::Dataspace::UNKNOWN; - - // The receiver of the capture can handle protected buffer. A protected buffer has - // GRALLOC_USAGE_PROTECTED usage bit and must not be accessed unprotected behaviour. - // Any read/write access from unprotected context will result in undefined behaviour. - // Protected contents are typically DRM contents. This has no direct implication to the - // secure property of the surface, which is specified by the application explicitly to avoid - // the contents being accessed/captured by screenshot or unsecure display. - bool allowProtected = false; - - bool grayscale = false; - - std::unordered_set<sp<IBinder>, SpHash<IBinder>> excludeHandles; - - // Hint that the caller will use the screenshot animation as part of a transition animation. - // The canonical example would be screen rotation - in such a case any color shift in the - // screenshot is a detractor so composition in the display's colorspace is required. - // Otherwise, the system may choose a colorspace that is more appropriate for use-cases - // such as file encoding or for blending HDR content into an ap's UI, where the display's - // exact colorspace is not an appropriate intermediate result. - // Note that if the caller is requesting a specific dataspace, this hint does nothing. - bool hintForSeamlessTransition = false; - - virtual status_t writeToParcel(Parcel* output) const; - virtual status_t readFromParcel(const Parcel* input); -}; - -struct DisplayCaptureArgs : CaptureArgs { - sp<IBinder> displayToken; - uint32_t width{0}; - uint32_t height{0}; - - status_t writeToParcel(Parcel* output) const override; - status_t readFromParcel(const Parcel* input) override; -}; - -}; // namespace android::gui diff --git a/libs/gui/include/gui/LayerCaptureArgs.h b/libs/gui/include/gui/Flags.h index fae2bcc787..735375a1e7 100644 --- a/libs/gui/include/gui/LayerCaptureArgs.h +++ b/libs/gui/include/gui/Flags.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 The Android Open Source Project + * Copyright 2024 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,19 +16,9 @@ #pragma once -#include <stdint.h> -#include <sys/types.h> +#include <com_android_graphics_libgui_flags.h> -#include <gui/DisplayCaptureArgs.h> - -namespace android::gui { - -struct LayerCaptureArgs : CaptureArgs { - sp<IBinder> layerHandle; - bool childrenOnly{false}; - - status_t writeToParcel(Parcel* output) const override; - status_t readFromParcel(const Parcel* input) override; -}; - -}; // namespace android::gui +#define WB_CAMERA3_AND_PROCESSORS_WITH_DEPENDENCIES \ + (COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CAMERA3_AND_PROCESSORS) && \ + COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) && \ + COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS))
\ No newline at end of file diff --git a/libs/gui/include/gui/FrameTimestamps.h b/libs/gui/include/gui/FrameTimestamps.h index 3d1be4d2eb..462081bfd2 100644 --- a/libs/gui/include/gui/FrameTimestamps.h +++ b/libs/gui/include/gui/FrameTimestamps.h @@ -116,7 +116,7 @@ public: // Public for testing. static nsecs_t snapToNextTick( nsecs_t timestamp, nsecs_t tickPhase, nsecs_t tickInterval); - nsecs_t getReportedCompositeDeadline() const { return mCompositorTiming.deadline; }; + nsecs_t getReportedCompositeDeadline() const { return mCompositorTiming.deadline; } nsecs_t getNextCompositeDeadline(const nsecs_t now) const; nsecs_t getCompositeInterval() const { return mCompositorTiming.interval; } diff --git a/libs/gui/include/gui/GLConsumer.h b/libs/gui/include/gui/GLConsumer.h index ba268ab17a..bfe3eb31e8 100644 --- a/libs/gui/include/gui/GLConsumer.h +++ b/libs/gui/include/gui/GLConsumer.h @@ -20,6 +20,7 @@ #include <EGL/egl.h> #include <EGL/eglext.h> +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferQueueDefs.h> #include <gui/ConsumerBase.h> @@ -82,12 +83,25 @@ public: // If the constructor without the tex parameter is used, the GLConsumer is // created in a detached state, and attachToContext must be called before // calls to updateTexImage. - GLConsumer(const sp<IGraphicBufferConsumer>& bq, - uint32_t tex, uint32_t texureTarget, bool useFenceSync, - bool isControlledByApp); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + GLConsumer(uint32_t tex, uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); - GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t texureTarget, - bool useFenceSync, bool isControlledByApp); + GLConsumer(uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); + + GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, + bool useFenceSync, bool isControlledByApp) + __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); + + GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, + bool isControlledByApp) + __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); +#else + GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, + bool useFenceSync, bool isControlledByApp); + + GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, + bool isControlledByApp); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) // updateTexImage acquires the most recently queued buffer, and sets the // image contents of the target texture to it. diff --git a/libs/gui/include/gui/IGraphicBufferProducer.h b/libs/gui/include/gui/IGraphicBufferProducer.h index 84a1730d7a..197e792367 100644 --- a/libs/gui/include/gui/IGraphicBufferProducer.h +++ b/libs/gui/include/gui/IGraphicBufferProducer.h @@ -867,6 +867,6 @@ class BnGraphicBufferProducer : public IGraphicBufferProducer { #endif // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android #endif // ANDROID_GUI_IGRAPHICBUFFERPRODUCER_H diff --git a/libs/gui/include/gui/ISurfaceComposer.h b/libs/gui/include/gui/ISurfaceComposer.h index eb4a802c17..9a422fd808 100644 --- a/libs/gui/include/gui/ISurfaceComposer.h +++ b/libs/gui/include/gui/ISurfaceComposer.h @@ -18,6 +18,7 @@ #include <android/gui/CachingHint.h> #include <android/gui/DisplayBrightness.h> +#include <android/gui/DisplayCaptureArgs.h> #include <android/gui/FrameTimelineInfo.h> #include <android/gui/IDisplayEventConnection.h> #include <android/gui/IFpsListener.h> @@ -27,6 +28,7 @@ #include <android/gui/ITunnelModeEnabledListener.h> #include <android/gui/IWindowInfosListener.h> #include <android/gui/IWindowInfosPublisher.h> +#include <android/gui/LayerCaptureArgs.h> #include <binder/IBinder.h> #include <binder/IInterface.h> #include <gui/ITransactionCompletedListener.h> @@ -70,13 +72,6 @@ using gui::IRegionSamplingListener; using gui::IScreenCaptureListener; using gui::SpHash; -namespace gui { - -struct DisplayCaptureArgs; -struct LayerCaptureArgs; - -} // namespace gui - namespace ui { struct DisplayMode; @@ -112,7 +107,7 @@ public: /* open/close transactions. requires ACCESS_SURFACE_FLINGER permission */ virtual status_t setTransactionState( const FrameTimelineInfo& frameTimelineInfo, Vector<ComposerState>& state, - const Vector<DisplayState>& displays, uint32_t flags, const sp<IBinder>& applyToken, + Vector<DisplayState>& displays, uint32_t flags, const sp<IBinder>& applyToken, InputWindowCommands inputWindowCommands, int64_t desiredPresentTime, bool isAutoTimestamp, const std::vector<client_cache_t>& uncacheBuffer, bool hasListenerCallbacks, const std::vector<ListenerCallbacks>& listenerCallbacks, diff --git a/libs/gui/include/gui/ITransactionCompletedListener.h b/libs/gui/include/gui/ITransactionCompletedListener.h index bc97cd0828..014029b257 100644 --- a/libs/gui/include/gui/ITransactionCompletedListener.h +++ b/libs/gui/include/gui/ITransactionCompletedListener.h @@ -16,8 +16,6 @@ #pragma once -#include "JankInfo.h" - #include <binder/IInterface.h> #include <binder/Parcel.h> #include <binder/Parcelable.h> @@ -40,15 +38,10 @@ class ListenerCallbacks; class CallbackId : public Parcelable { public: int64_t id; - enum class Type : int32_t { - ON_COMPLETE = 0, - ON_COMMIT = 1, - /*reserved for serialization = 2*/ - } type; - bool includeJankData; // Only respected for ON_COMPLETE callbacks. + enum class Type : int32_t { ON_COMPLETE = 0, ON_COMMIT = 1 } type; CallbackId() {} - CallbackId(int64_t id, Type type) : id(id), type(type), includeJankData(false) {} + CallbackId(int64_t id, Type type) : id(id), type(type) {} status_t writeToParcel(Parcel* output) const override; status_t readFromParcel(const Parcel* input) override; @@ -113,29 +106,6 @@ public: nsecs_t dequeueReadyTime; }; -/** - * Jank information representing SurfaceFlinger's jank classification about frames for a specific - * surface. - */ -class JankData : public Parcelable { -public: - status_t writeToParcel(Parcel* output) const override; - status_t readFromParcel(const Parcel* input) override; - - JankData(); - JankData(int64_t frameVsyncId, int32_t jankType, nsecs_t frameIntervalNs) - : frameVsyncId(frameVsyncId), jankType(jankType), frameIntervalNs(frameIntervalNs) {} - - // Identifier for the frame submitted with Transaction.setFrameTimelineVsyncId - int64_t frameVsyncId; - - // Bitmask of janks that occurred - int32_t jankType; - - // Expected duration of the frame - nsecs_t frameIntervalNs; -}; - class SurfaceStats : public Parcelable { public: status_t writeToParcel(Parcel* output) const override; @@ -145,14 +115,13 @@ public: SurfaceStats(const sp<IBinder>& sc, std::variant<nsecs_t, sp<Fence>> acquireTimeOrFence, const sp<Fence>& prevReleaseFence, std::optional<uint32_t> hint, uint32_t currentMaxAcquiredBuffersCount, FrameEventHistoryStats frameEventStats, - std::vector<JankData> jankData, ReleaseCallbackId previousReleaseCallbackId) + ReleaseCallbackId previousReleaseCallbackId) : surfaceControl(sc), acquireTimeOrFence(std::move(acquireTimeOrFence)), previousReleaseFence(prevReleaseFence), transformHint(hint), currentMaxAcquiredBufferCount(currentMaxAcquiredBuffersCount), eventStats(frameEventStats), - jankData(std::move(jankData)), previousReleaseCallbackId(previousReleaseCallbackId) {} sp<IBinder> surfaceControl; @@ -161,7 +130,6 @@ public: std::optional<uint32_t> transformHint = 0; uint32_t currentMaxAcquiredBufferCount = 0; FrameEventHistoryStats eventStats; - std::vector<JankData> jankData; ReleaseCallbackId previousReleaseCallbackId; }; diff --git a/libs/gui/include/gui/LayerMetadata.h b/libs/gui/include/gui/LayerMetadata.h index 9cf62bc7d6..7ee291df4c 100644 --- a/libs/gui/include/gui/LayerMetadata.h +++ b/libs/gui/include/gui/LayerMetadata.h @@ -74,6 +74,7 @@ enum class GameMode : int32_t { } // namespace android::gui using android::gui::METADATA_ACCESSIBILITY_ID; +using android::gui::METADATA_CALLING_UID; using android::gui::METADATA_DEQUEUE_TIME; using android::gui::METADATA_GAME_MODE; using android::gui::METADATA_MOUSE_CURSOR; diff --git a/libs/gui/include/gui/LayerState.h b/libs/gui/include/gui/LayerState.h index 5f2f8dc013..2cdde3255e 100644 --- a/libs/gui/include/gui/LayerState.h +++ b/libs/gui/include/gui/LayerState.h @@ -21,7 +21,9 @@ #include <stdint.h> #include <sys/types.h> +#include <android/gui/DisplayCaptureArgs.h> #include <android/gui/IWindowInfosReportedListener.h> +#include <android/gui/LayerCaptureArgs.h> #include <android/gui/TrustedPresentationThresholds.h> #include <android/native_window.h> #include <gui/IGraphicBufferProducer.h> @@ -29,13 +31,13 @@ #include <math/mat4.h> #include <android/gui/DropInputMode.h> +#include <android/gui/EdgeExtensionParameters.h> #include <android/gui/FocusRequest.h> #include <android/gui/TrustedOverlay.h> #include <ftl/flags.h> -#include <gui/DisplayCaptureArgs.h> +#include <gui/BufferReleaseChannel.h> #include <gui/ISurfaceComposer.h> -#include <gui/LayerCaptureArgs.h> #include <gui/LayerMetadata.h> #include <gui/SpHash.h> #include <gui/SurfaceControl.h> @@ -218,6 +220,8 @@ struct layer_state_t { eTrustedOverlayChanged = 0x4000'00000000, eDropInputModeChanged = 0x8000'00000000, eExtendedRangeBrightnessChanged = 0x10000'00000000, + eEdgeExtensionChanged = 0x20000'00000000, + eBufferReleaseChannelChanged = 0x40000'00000000, }; layer_state_t(); @@ -241,7 +245,7 @@ struct layer_state_t { layer_state_t::eCropChanged | layer_state_t::eDestinationFrameChanged | layer_state_t::eMatrixChanged | layer_state_t::ePositionChanged | layer_state_t::eTransformToDisplayInverseChanged | - layer_state_t::eTransparentRegionChanged; + layer_state_t::eTransparentRegionChanged | layer_state_t::eEdgeExtensionChanged; // Buffer and related updates. static constexpr uint64_t BUFFER_CHANGES = layer_state_t::eApiChanged | @@ -393,6 +397,9 @@ struct layer_state_t { // Stretch effect to be applied to this layer StretchEffect stretchEffect; + // Edge extension effect to be applied to this layer + gui::EdgeExtensionParameters edgeExtensionParameters; + Rect bufferCrop; Rect destinationFrame; @@ -407,6 +414,8 @@ struct layer_state_t { TrustedPresentationThresholds trustedPresentationThresholds; TrustedPresentationListener trustedPresentationListener; + + std::shared_ptr<gui::BufferReleaseChannel::ProducerEndpoint> bufferReleaseChannel; }; class ComposerState { diff --git a/libs/gui/include/gui/ScreenCaptureResults.h b/libs/gui/include/gui/ScreenCaptureResults.h index 6e17791a29..f176f48fb4 100644 --- a/libs/gui/include/gui/ScreenCaptureResults.h +++ b/libs/gui/include/gui/ScreenCaptureResults.h @@ -36,6 +36,11 @@ public: bool capturedSecureLayers{false}; bool capturedHdrLayers{false}; ui::Dataspace capturedDataspace{ui::Dataspace::V0_SRGB}; + // A gainmap that can be used to "lift" the screenshot into HDR + sp<GraphicBuffer> optionalGainMap; + // HDR/SDR ratio value that fully applies the gainmap. + // Note that we use 1/64 epsilon offsets to eliminate precision issues + float hdrSdrRatio{1.0f}; }; } // namespace android::gui diff --git a/libs/gui/include/gui/Surface.h b/libs/gui/include/gui/Surface.h index bdcaaf2866..14a351316d 100644 --- a/libs/gui/include/gui/Surface.h +++ b/libs/gui/include/gui/Surface.h @@ -18,6 +18,7 @@ #define ANDROID_GUI_SURFACE_H #include <android/gui/FrameTimelineInfo.h> +#include <com_android_graphics_libgui_flags.h> #include <gui/BufferQueueDefs.h> #include <gui/HdrMetadata.h> #include <gui/IGraphicBufferProducer.h> @@ -35,6 +36,8 @@ namespace android { +class GraphicBuffer; + namespace gui { class ISurfaceComposer; } // namespace gui @@ -56,8 +59,41 @@ public: virtual bool needsReleaseNotify() = 0; virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& buffers) = 0; + virtual void onBufferDetached(int slot) = 0; +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_CONSUMER_ATTACH_CALLBACK) + virtual void onBufferAttached() {} + virtual bool needsAttachNotify() { return false; } +#endif + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + // Called if this Surface is connected to a remote implementation and it + // dies or becomes unavailable. + virtual void onRemoteDied() {} + + // Clients will overwrite this if they want to receive a notification + // via onRemoteDied. This should return a constant value. + virtual bool needsDeathNotify() { return false; } +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) }; +class StubSurfaceListener : public SurfaceListener { +public: + virtual ~StubSurfaceListener() {} + virtual void onBufferReleased() override {} + virtual bool needsReleaseNotify() { return false; } + virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& /*buffers*/) override {} + virtual void onBufferDetached(int /*slot*/) override {} +}; + +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) +// Contains additional data from the queueBuffer operation. +struct SurfaceQueueBufferOutput { + // True if this queueBuffer caused a buffer to be replaced in the queue + // (and therefore not will not be acquired) + bool bufferReplaced = false; +}; +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + /* * An implementation of ANativeWindow that feeds graphics buffers into a * BufferQueue. @@ -154,6 +190,11 @@ public: */ virtual void allocateBuffers(); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + // See IGraphicBufferProducer::allowAllocation + status_t allowAllocation(bool allowAllocation); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + /* Sets the generation number on the IGraphicBufferProducer and updates the * generation number on any buffers attached to the Surface after this call. * See IGBP::setGenerationNumber for more information. */ @@ -170,6 +211,14 @@ public: * in <system/window.h>. */ int setScalingMode(int mode); + virtual int setBuffersTimestamp(int64_t timestamp); + virtual int setBuffersDataSpace(ui::Dataspace dataSpace); + virtual int setCrop(Rect const* rect); + virtual int setBuffersTransform(uint32_t transform); + virtual int setBuffersStickyTransform(uint32_t transform); + virtual int setBuffersFormat(PixelFormat format); + virtual int setUsage(uint64_t reqUsage); + // See IGraphicBufferProducer::setDequeueTimeout status_t setDequeueTimeout(nsecs_t timeout); @@ -321,7 +370,12 @@ private: protected: virtual int dequeueBuffer(ANativeWindowBuffer** buffer, int* fenceFd); virtual int cancelBuffer(ANativeWindowBuffer* buffer, int fenceFd); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + virtual int queueBuffer(ANativeWindowBuffer* buffer, int fenceFd, + SurfaceQueueBufferOutput* surfaceOutput = nullptr); +#else virtual int queueBuffer(ANativeWindowBuffer* buffer, int fenceFd); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) virtual int perform(int operation, va_list args); virtual int setSwapInterval(int interval); @@ -330,16 +384,9 @@ protected: virtual int connect(int api); virtual int setBufferCount(int bufferCount); virtual int setBuffersUserDimensions(uint32_t width, uint32_t height); - virtual int setBuffersFormat(PixelFormat format); - virtual int setBuffersTransform(uint32_t transform); - virtual int setBuffersStickyTransform(uint32_t transform); - virtual int setBuffersTimestamp(int64_t timestamp); - virtual int setBuffersDataSpace(ui::Dataspace dataSpace); virtual int setBuffersSmpte2086Metadata(const android_smpte2086_metadata* metadata); virtual int setBuffersCta8613Metadata(const android_cta861_3_metadata* metadata); virtual int setBuffersHdr10PlusMetadata(const size_t size, const uint8_t* metadata); - virtual int setCrop(Rect const* rect); - virtual int setUsage(uint64_t reqUsage); virtual void setSurfaceDamage(android_native_rect_t* rects, size_t numRects); public: @@ -357,22 +404,15 @@ public: virtual int unlockAndPost(); virtual int query(int what, int* value) const; - virtual int connect(int api, const sp<IProducerListener>& listener); - // When reportBufferRemoval is true, clients must call getAndFlushRemovedBuffers to fetch // GraphicBuffers removed from this surface after a dequeueBuffer, detachNextBuffer or // attachBuffer call. This allows clients with their own buffer caches to free up buffers no // longer in use by this surface. - virtual int connect( - int api, const sp<IProducerListener>& listener, - bool reportBufferRemoval); - virtual int detachNextBuffer(sp<GraphicBuffer>* outBuffer, - sp<Fence>* outFence); + virtual int connect(int api, const sp<SurfaceListener>& listener, + bool reportBufferRemoval = false); + virtual int detachNextBuffer(sp<GraphicBuffer>* outBuffer, sp<Fence>* outFence); virtual int attachBuffer(ANativeWindowBuffer*); - virtual int connect( - int api, bool reportBufferRemoval, - const sp<SurfaceListener>& sListener); virtual void destroy(); // When client connects to Surface with reportBufferRemoval set to true, any buffers removed @@ -387,6 +427,21 @@ public: static status_t attachAndQueueBufferWithDataspace(Surface* surface, sp<GraphicBuffer> buffer, ui::Dataspace dataspace); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + // Dequeues a buffer and its outFence, which must be signalled before the buffer can be used. + status_t dequeueBuffer(sp<GraphicBuffer>* buffer, sp<Fence>* outFence); + + // Queues a buffer, with an optional fd fence that captures pending work on the buffer. This + // buffer must have been returned by dequeueBuffer or associated with this Surface via an + // attachBuffer operation. + status_t queueBuffer(const sp<GraphicBuffer>& buffer, const sp<Fence>& fd = Fence::NO_FENCE, + SurfaceQueueBufferOutput* output = nullptr); + + // Detaches this buffer, dissociating it from this Surface. This buffer must have been returned + // by queueBuffer or associated with this Surface via an attachBuffer operation. + status_t detachBuffer(const sp<GraphicBuffer>& buffer); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + // Batch version of dequeueBuffer, cancelBuffer and queueBuffer // Note that these batched operations are not supported when shared buffer mode is being used. struct BatchBuffer { @@ -401,8 +456,13 @@ public: int fenceFd = -1; nsecs_t timestamp = NATIVE_WINDOW_TIMESTAMP_AUTO; }; +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + virtual int queueBuffers(const std::vector<BatchQueuedBuffer>& buffers, + std::vector<SurfaceQueueBufferOutput>* queueBufferOutputs = nullptr); +#else virtual int queueBuffers( const std::vector<BatchQueuedBuffer>& buffers); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) protected: enum { NUM_BUFFER_SLOTS = BufferQueueDefs::NUM_BUFFER_SLOTS }; @@ -422,12 +482,38 @@ protected: return mSurfaceListener->needsReleaseNotify(); } + virtual void onBufferDetached(int slot) { mSurfaceListener->onBufferDetached(slot); } + virtual void onBuffersDiscarded(const std::vector<int32_t>& slots); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_CONSUMER_ATTACH_CALLBACK) + virtual void onBufferAttached() { + mSurfaceListener->onBufferAttached(); + } + + virtual bool needsAttachNotify() { + return mSurfaceListener->needsAttachNotify(); + } +#endif private: wp<Surface> mParent; sp<SurfaceListener> mSurfaceListener; }; +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + class ProducerDeathListenerProxy : public IBinder::DeathRecipient { + public: + ProducerDeathListenerProxy(wp<SurfaceListener> surfaceListener); + ProducerDeathListenerProxy(ProducerDeathListenerProxy&) = delete; + + // IBinder::DeathRecipient + virtual void binderDied(const wp<IBinder>&) override; + + private: + wp<SurfaceListener> mSurfaceListener; + }; + friend class ProducerDeathListenerProxy; +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + void querySupportedTimestampsLocked() const; void freeAllBuffers(); @@ -459,6 +545,13 @@ protected: // TODO: rename to mBufferProducer sp<IGraphicBufferProducer> mGraphicBufferProducer; +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + // mSurfaceDeathListener gets registered as mGraphicBufferProducer's + // DeathRecipient when SurfaceListener::needsDeathNotify returns true and + // gets notified when it dies. + sp<ProducerDeathListenerProxy> mSurfaceDeathListener; +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + // mSlots stores the buffers that have been allocated for each buffer slot. // It is initialized to null pointers, and gets filled in with the result of // IGraphicBufferProducer::requestBuffer when the client dequeues a buffer from a diff --git a/libs/gui/include/gui/SurfaceComposerClient.h b/libs/gui/include/gui/SurfaceComposerClient.h index 0862e03c44..4f9af16826 100644 --- a/libs/gui/include/gui/SurfaceComposerClient.h +++ b/libs/gui/include/gui/SurfaceComposerClient.h @@ -35,14 +35,17 @@ #include <ui/BlurRegion.h> #include <ui/ConfigStoreTypes.h> #include <ui/DisplayedFrameStats.h> +#include <ui/EdgeExtensionEffect.h> #include <ui/FrameStats.h> #include <ui/GraphicTypes.h> #include <ui/PixelFormat.h> #include <ui/Rotation.h> #include <ui/StaticDisplayInfo.h> +#include <android/gui/BnJankListener.h> #include <android/gui/ISurfaceComposerClient.h> +#include <gui/BufferReleaseChannel.h> #include <gui/CpuConsumer.h> #include <gui/ISurfaceComposer.h> #include <gui/ITransactionCompletedListener.h> @@ -337,6 +340,8 @@ public: static std::optional<aidl::android::hardware::graphics::common::DisplayDecorationSupport> getDisplayDecorationSupport(const sp<IBinder>& displayToken); + static bool flagEdgeExtensionEffectUseShader(); + // ------------------------------------------------------------------------ // surface creation / destruction @@ -447,7 +452,6 @@ public: uint64_t mId; - uint32_t mTransactionNestCount = 0; bool mAnimation = false; bool mEarlyWakeupStart = false; bool mEarlyWakeupEnd = false; @@ -743,11 +747,26 @@ public: Transaction& setStretchEffect(const sp<SurfaceControl>& sc, const StretchEffect& stretchEffect); + /** + * Provides the edge extension effect configured on a container that the + * surface is rendered within. + * @param sc target surface the edge extension should be applied to + * @param effect the corresponding EdgeExtensionParameters to be applied + * to the surface. + * @return The transaction being constructed + */ + Transaction& setEdgeExtensionEffect(const sp<SurfaceControl>& sc, + const gui::EdgeExtensionParameters& effect); + Transaction& setBufferCrop(const sp<SurfaceControl>& sc, const Rect& bufferCrop); Transaction& setDestinationFrame(const sp<SurfaceControl>& sc, const Rect& destinationFrame); Transaction& setDropInputMode(const sp<SurfaceControl>& sc, gui::DropInputMode mode); + Transaction& setBufferReleaseChannel( + const sp<SurfaceControl>& sc, + const std::shared_ptr<gui::BufferReleaseChannel::ProducerEndpoint>& channel); + status_t setDisplaySurface(const sp<IBinder>& token, const sp<IGraphicBufferProducer>& bufferProducer); @@ -864,12 +883,82 @@ public: // --------------------------------------------------------------------------- -class JankDataListener : public VirtualLightRefBase { +class JankDataListener; + +// Acts as a representative listener to the composer for a single layer and +// forwards any received jank data to multiple listeners. Will remove itself +// from the composer only once the last listener is removed. +class JankDataListenerFanOut : public gui::BnJankListener { +public: + JankDataListenerFanOut(int32_t layerId) : mLayerId(layerId) {} + + binder::Status onJankData(const std::vector<gui::JankData>& jankData) override; + + static status_t addListener(sp<SurfaceControl> sc, sp<JankDataListener> listener); + static status_t removeListener(sp<JankDataListener> listener); + +private: + std::vector<sp<JankDataListener>> getActiveListeners(); + bool removeListeners(const std::vector<wp<JankDataListener>>& listeners); + int64_t updateAndGetRemovalVSync(); + + struct WpJDLHash { + std::size_t operator()(const wp<JankDataListener>& listener) const { + return std::hash<JankDataListener*>{}(listener.unsafe_get()); + } + }; + + std::mutex mMutex; + std::unordered_set<wp<JankDataListener>, WpJDLHash> mListeners GUARDED_BY(mMutex); + int32_t mLayerId; + int64_t mRemoveAfter = -1; + + static std::mutex sFanoutInstanceMutex; + static std::unordered_map<int32_t, sp<JankDataListenerFanOut>> sFanoutInstances; +}; + +// Base class for client listeners interested in jank classification data from +// the composer. Subclasses should override onJankDataAvailable and call the add +// and removal methods to receive jank data. +class JankDataListener : public virtual RefBase { public: - virtual ~JankDataListener() = 0; - virtual void onJankDataAvailable(const std::vector<JankData>& jankData) = 0; + JankDataListener() {} + virtual ~JankDataListener(); + + virtual bool onJankDataAvailable(const std::vector<gui::JankData>& jankData) = 0; + + status_t addListener(sp<SurfaceControl> sc) { + if (mLayerId != -1) { + removeListener(0); + mLayerId = -1; + } + + int32_t layerId = sc->getLayerId(); + status_t status = + JankDataListenerFanOut::addListener(std::move(sc), + sp<JankDataListener>::fromExisting(this)); + if (status == OK) { + mLayerId = layerId; + } + return status; + } + + status_t removeListener(int64_t afterVsync) { + mRemoveAfter = std::max(static_cast<int64_t>(0), afterVsync); + return JankDataListenerFanOut::removeListener(sp<JankDataListener>::fromExisting(this)); + } + + status_t flushJankData(); + + friend class JankDataListenerFanOut; + +private: + int32_t mLayerId = -1; + int64_t mRemoveAfter = -1; }; +// --------------------------------------------------------------------------- + class TransactionCompletedListener : public BnTransactionCompletedListener { public: TransactionCompletedListener(); @@ -904,7 +993,6 @@ protected: std::unordered_map<CallbackId, CallbackTranslation, CallbackIdHash> mCallbacks GUARDED_BY(mMutex); - std::multimap<int32_t, sp<JankDataListener>> mJankListeners GUARDED_BY(mMutex); std::unordered_map<ReleaseCallbackId, ReleaseBufferCallback, ReleaseBufferCallbackIdHash> mReleaseBufferCallbacks GUARDED_BY(mMutex); @@ -927,14 +1015,10 @@ public: const std::unordered_set<sp<SurfaceControl>, SurfaceComposerClient::SCHash>& surfaceControls, CallbackId::Type callbackType); - CallbackId addCallbackFunctionLocked( - const TransactionCompletedCallback& callbackFunction, - const std::unordered_set<sp<SurfaceControl>, SurfaceComposerClient::SCHash>& - surfaceControls, - CallbackId::Type callbackType) REQUIRES(mMutex); - void addSurfaceControlToCallbacks(SurfaceComposerClient::CallbackInfo& callbackInfo, - const sp<SurfaceControl>& surfaceControl); + void addSurfaceControlToCallbacks( + const sp<SurfaceControl>& surfaceControl, + const std::unordered_set<CallbackId, CallbackIdHash>& callbackIds); void addQueueStallListener(std::function<void(const std::string&)> stallListener, void* id); void removeQueueStallListener(void *id); @@ -943,18 +1027,6 @@ public: TrustedPresentationCallback tpc, int id, void* context); void clearTrustedPresentationCallback(int id); - /* - * Adds a jank listener to be informed about SurfaceFlinger's jank classification for a specific - * surface. Jank classifications arrive as part of the transaction callbacks about previous - * frames submitted to this Surface. - */ - void addJankListener(const sp<JankDataListener>& listener, sp<SurfaceControl> surfaceControl); - - /** - * Removes a jank listener previously added to addJankCallback. - */ - void removeJankListener(const sp<JankDataListener>& listener); - void addSurfaceStatsListener(void* context, void* cookie, sp<SurfaceControl> surfaceControl, SurfaceStatsCallback listener); void removeSurfaceStatsListener(void* context, void* cookie); diff --git a/libs/gui/include/gui/view/Surface.h b/libs/gui/include/gui/view/Surface.h index b7aba2b9dc..7ddac8139a 100644 --- a/libs/gui/include/gui/view/Surface.h +++ b/libs/gui/include/gui/view/Surface.h @@ -59,8 +59,9 @@ class Surface : public Parcelable { // of the full parceling to happen on its native side. status_t readFromParcel(const Parcel* parcel, bool nameAlreadyRead); - private: + std::string toString() const; +private: static String16 readMaybeEmptyString16(const Parcel* parcel); }; diff --git a/libs/gui/libgui_flags.aconfig b/libs/gui/libgui_flags.aconfig index 87cef087db..d3f2899ba3 100644 --- a/libs/gui/libgui_flags.aconfig +++ b/libs/gui/libgui_flags.aconfig @@ -43,3 +43,75 @@ flag { purpose: PURPOSE_BUGFIX } } # trace_frame_rate_override + +flag { + name: "wb_consumer_base_owns_bq" + namespace: "core_graphics" + description: "ConsumerBase-based classes now own their own bufferqueue" + bug: "340933754" + is_fixed_read_only: true +} # wb_consumer_base_owns_bq + +flag { + name: "wb_platform_api_improvements" + namespace: "core_graphics" + description: "Simple improvements to Surface and ConsumerBase classes" + bug: "340933794" + is_fixed_read_only: true +} # wb_platform_api_improvements + +flag { + name: "wb_stream_splitter" + namespace: "core_graphics" + description: "Removes IGBP/IGBCs from Camera3StreamSplitter as part of BufferQueue refactors" + bug: "340933206" + is_fixed_read_only: true +} # wb_stream_splitter + +flag { + name: "edge_extension_shader" + namespace: "windowing_frontend" + description: "Enable edge extension via shader" + bug: "322036393" + is_fixed_read_only: true +} # edge_extension_shader + +flag { + name: "buffer_release_channel" + namespace: "window_surfaces" + description: "Enable BufferReleaseChannel to optimize buffer releases" + bug: "294133380" + is_fixed_read_only: true +} # buffer_release_channel + +flag { + name: "wb_ring_buffer" + namespace: "core_graphics" + description: "Remove slot dependency in the Ring Buffer Consumer." + bug: "342197847" + is_fixed_read_only: true +} # wb_ring_buffer + +flag { + name: "wb_camera3_and_processors" + namespace: "core_graphics" + description: "Remove usage of IGBPs in the *Processor and Camera3*" + bug: "342199002" + is_fixed_read_only: true +} # wb_camera3_and_processors + +flag { + name: "wb_libcameraservice" + namespace: "core_graphics" + description: "Remove usage of IGBPs in the libcameraservice." + bug: "342197849" + is_fixed_read_only: true +} # wb_libcameraservice + +flag { + name: "bq_producer_throttles_only_async_mode" + namespace: "core_graphics" + description: "BufferQueueProducer only CPU throttle on queueBuffer() in async mode." + bug: "359252619" + is_fixed_read_only: true +} # bq_producer_throttles_only_async_mode diff --git a/libs/gui/rust/aidl_types/src/lib.rs b/libs/gui/rust/aidl_types/src/lib.rs index fead018bbf..2351df0318 100644 --- a/libs/gui/rust/aidl_types/src/lib.rs +++ b/libs/gui/rust/aidl_types/src/lib.rs @@ -42,10 +42,7 @@ macro_rules! stub_unstructured_parcelable { } stub_unstructured_parcelable!(BitTube); -stub_unstructured_parcelable!(CaptureArgs); -stub_unstructured_parcelable!(DisplayCaptureArgs); stub_unstructured_parcelable!(DisplayInfo); -stub_unstructured_parcelable!(LayerCaptureArgs); stub_unstructured_parcelable!(LayerDebugInfo); stub_unstructured_parcelable!(LayerMetadata); stub_unstructured_parcelable!(ParcelableVsyncEventData); diff --git a/libs/gui/tests/Android.bp b/libs/gui/tests/Android.bp index ea8acbbb72..f07747f32f 100644 --- a/libs/gui/tests/Android.bp +++ b/libs/gui/tests/Android.bp @@ -12,6 +12,34 @@ package { default_applicable_licenses: ["frameworks_native_license"], } +aidl_interface { + name: "libgui_test_server_aidl", + unstable: true, + srcs: ["testserver/aidl/**/*.aidl"], + local_include_dir: "testserver/aidl", + include_dirs: [ + "frameworks/native/aidl/gui", + ], + backend: { + cpp: { + enabled: true, + additional_shared_libraries: [ + "libgui", + "libui", + ], + }, + java: { + enabled: false, + }, + ndk: { + enabled: false, + }, + rust: { + enabled: false, + }, + }, +} + cc_test { name: "libgui_test", test_suites: ["device-tests"], @@ -25,34 +53,41 @@ cc_test { "-Wthread-safety", "-DCOM_ANDROID_GRAPHICS_LIBGUI_FLAGS_BQ_SETFRAMERATE=true", "-DCOM_ANDROID_GRAPHICS_LIBGUI_FLAGS_BQ_EXTENDEDALLOCATE=true", + "-DCOM_ANDROID_GRAPHICS_LIBGUI_FLAGS_WB_CONSUMER_BASE_OWNS_BQ=true", + "-DCOM_ANDROID_GRAPHICS_LIBGUI_FLAGS_WB_PLATFORM_API_IMPROVEMENTS=true", ], srcs: [ - "LibGuiMain.cpp", // Custom gtest entrypoint "BLASTBufferQueue_test.cpp", "BufferItemConsumer_test.cpp", "BufferQueue_test.cpp", + "BufferReleaseChannel_test.cpp", "Choreographer_test.cpp", "CompositorTiming_test.cpp", "CpuConsumer_test.cpp", - "EndToEndNativeInputTest.cpp", - "FrameRateUtilsTest.cpp", - "DisplayInfo_test.cpp", "DisplayedContentSampling_test.cpp", + "DisplayInfo_test.cpp", + "EndToEndNativeInputTest.cpp", "FillBuffer.cpp", + "FrameRateUtilsTest.cpp", "GLTest.cpp", "IGraphicBufferProducer_test.cpp", + "LibGuiMain.cpp", // Custom gtest entrypoint "Malicious.cpp", "MultiTextureConsumer_test.cpp", "RegionSampling_test.cpp", "StreamSplitter_test.cpp", + "Surface_test.cpp", "SurfaceTextureClient_test.cpp", "SurfaceTextureFBO_test.cpp", + "SurfaceTextureGL_test.cpp", "SurfaceTextureGLThreadToGL_test.cpp", "SurfaceTextureGLToGL_test.cpp", - "SurfaceTextureGL_test.cpp", "SurfaceTextureMultiContextGL_test.cpp", - "Surface_test.cpp", + "TestServer_test.cpp", + "testserver/TestServer.cpp", + "testserver/TestServerClient.cpp", + "testserver/TestServerHost.cpp", "TextureRenderer.cpp", "VsyncEventData_test.cpp", "WindowInfo_test.cpp", @@ -63,10 +98,17 @@ cc_test { "android.hardware.configstore-utils", "libSurfaceFlingerProp", "libGLESv1_CM", + "libgui_test_server_aidl-cpp", "libinput", "libnativedisplay", ], + // This needs to get copied over for the test since it's not part of the + // platform. + data_libs: [ + "libgui_test_server_aidl-cpp", + ], + static_libs: [ "libgmock", ], diff --git a/libs/gui/tests/BLASTBufferQueue_test.cpp b/libs/gui/tests/BLASTBufferQueue_test.cpp index 946ff058cf..53f4a36c42 100644 --- a/libs/gui/tests/BLASTBufferQueue_test.cpp +++ b/libs/gui/tests/BLASTBufferQueue_test.cpp @@ -20,7 +20,7 @@ #include <android-base/thread_annotations.h> #include <android/hardware/graphics/common/1.2/types.h> -#include <gui/AidlStatusUtil.h> +#include <gui/AidlUtil.h> #include <gui/BufferQueueCore.h> #include <gui/BufferQueueProducer.h> #include <gui/FrameTimestamps.h> @@ -186,6 +186,10 @@ public: mBlastBufferQueueAdapter->mergeWithNextTransaction(merge, frameNumber); } + void setApplyToken(sp<IBinder> applyToken) { + mBlastBufferQueueAdapter->setApplyToken(std::move(applyToken)); + } + private: sp<TestBLASTBufferQueue> mBlastBufferQueueAdapter; }; @@ -229,7 +233,8 @@ protected: ISurfaceComposerClient::eFXSurfaceBufferState, /*parent*/ mRootSurfaceControl->getHandle()); - mCaptureArgs.sourceCrop = Rect(ui::Size(mDisplayWidth, mDisplayHeight)); + mCaptureArgs.captureArgs.sourceCrop = + gui::aidl_utils::toARect(mDisplayWidth, mDisplayHeight); mCaptureArgs.layerHandle = mRootSurfaceControl->getHandle(); } @@ -510,6 +515,69 @@ TEST_F(BLASTBufferQueueTest, TripleBuffering) { adapter.waitForCallbacks(); } +class WaitForCommittedCallback { +public: + WaitForCommittedCallback() = default; + ~WaitForCommittedCallback() = default; + + void wait() { + std::unique_lock lock(mMutex); + cv.wait(lock, [this] { return mCallbackReceived; }); + } + + void notify() { + std::unique_lock lock(mMutex); + mCallbackReceived = true; + cv.notify_one(); + mCallbackReceivedTimeStamp = std::chrono::system_clock::now(); + } + auto getCallback() { + return [this](void* /* unused context */, nsecs_t /* latchTime */, + const sp<Fence>& /* presentFence */, + const std::vector<SurfaceControlStats>& /* stats */) { notify(); }; + } + std::chrono::time_point<std::chrono::system_clock> mCallbackReceivedTimeStamp; + +private: + std::mutex mMutex; + std::condition_variable cv; + bool mCallbackReceived = false; +}; + +TEST_F(BLASTBufferQueueTest, setApplyToken) { + sp<IBinder> applyToken = sp<BBinder>::make(); + WaitForCommittedCallback firstTransaction; + WaitForCommittedCallback secondTransaction; + { + BLASTBufferQueueHelper adapter(mSurfaceControl, mDisplayWidth, mDisplayHeight); + adapter.setApplyToken(applyToken); + sp<IGraphicBufferProducer> igbProducer; + setUpProducer(adapter, igbProducer); + + Transaction t; + t.addTransactionCommittedCallback(firstTransaction.getCallback(), nullptr); + adapter.mergeWithNextTransaction(&t, 1); + queueBuffer(igbProducer, 127, 127, 127, + /*presentTimeDelay*/ std::chrono::nanoseconds(500ms).count()); + } + { + BLASTBufferQueueHelper adapter(mSurfaceControl, mDisplayWidth, mDisplayHeight); + adapter.setApplyToken(applyToken); + sp<IGraphicBufferProducer> igbProducer; + setUpProducer(adapter, igbProducer); + + Transaction t; + t.addTransactionCommittedCallback(secondTransaction.getCallback(), nullptr); + adapter.mergeWithNextTransaction(&t, 1); + queueBuffer(igbProducer, 127, 127, 127, /*presentTimeDelay*/ 0); + } + + firstTransaction.wait(); + secondTransaction.wait(); + EXPECT_GT(secondTransaction.mCallbackReceivedTimeStamp, + firstTransaction.mCallbackReceivedTimeStamp); +} + TEST_F(BLASTBufferQueueTest, SetCrop_Item) { uint8_t r = 255; uint8_t g = 0; @@ -1269,6 +1337,20 @@ public: } }; +class TestSurfaceListener : public SurfaceListener { +public: + sp<IGraphicBufferProducer> mIgbp; + TestSurfaceListener(const sp<IGraphicBufferProducer>& igbp) : mIgbp(igbp) {} + void onBufferReleased() override { + sp<GraphicBuffer> buffer; + sp<Fence> fence; + mIgbp->detachNextBuffer(&buffer, &fence); + } + bool needsReleaseNotify() override { return true; } + void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& /*buffers*/) override {} + void onBufferDetached(int /*slot*/) {} +}; + TEST_F(BLASTBufferQueueTest, CustomProducerListener) { BLASTBufferQueueHelper adapter(mSurfaceControl, mDisplayWidth, mDisplayHeight); sp<IGraphicBufferProducer> igbProducer = adapter.getIGraphicBufferProducer(); @@ -1327,7 +1409,7 @@ TEST_F(BLASTBufferQueueTest, TransformHint) { ASSERT_EQ(ui::Transform::ROT_0, static_cast<ui::Transform::RotationFlags>(transformHint)); ASSERT_EQ(NO_ERROR, - surface->connect(NATIVE_WINDOW_API_CPU, new TestProducerListener(igbProducer))); + surface->connect(NATIVE_WINDOW_API_CPU, new TestSurfaceListener(igbProducer))); // After connecting to the surface, we should get the correct hint. surface->query(NATIVE_WINDOW_TRANSFORM_HINT, &transformHint); diff --git a/libs/gui/tests/BufferItemConsumer_test.cpp b/libs/gui/tests/BufferItemConsumer_test.cpp index 656453411d..3b6a66efe9 100644 --- a/libs/gui/tests/BufferItemConsumer_test.cpp +++ b/libs/gui/tests/BufferItemConsumer_test.cpp @@ -17,10 +17,12 @@ #define LOG_TAG "BufferItemConsumer_test" //#define LOG_NDEBUG 0 +#include <gmock/gmock.h> #include <gtest/gtest.h> #include <gui/BufferItemConsumer.h> #include <gui/IProducerListener.h> #include <gui/Surface.h> +#include <ui/GraphicBuffer.h> namespace android { @@ -43,15 +45,26 @@ class BufferItemConsumerTest : public ::testing::Test { BufferItemConsumerTest* mTest; }; + struct TrackingProducerListener : public BnProducerListener { + TrackingProducerListener(BufferItemConsumerTest* test) : mTest(test) {} + + virtual void onBufferReleased() override {} + virtual bool needsReleaseNotify() override { return true; } + virtual void onBuffersDiscarded(const std::vector<int32_t>&) override {} + virtual void onBufferDetached(int slot) override { mTest->HandleBufferDetached(slot); } + + BufferItemConsumerTest* mTest; + }; + void SetUp() override { - BufferQueue::createBufferQueue(&mProducer, &mConsumer); - mBIC = new BufferItemConsumer(mConsumer, kUsage, kMaxLockedBuffers, true); + mBIC = new BufferItemConsumer(kUsage, kMaxLockedBuffers, true); String8 name("BufferItemConsumer_Under_Test"); mBIC->setName(name); mBFL = new BufferFreedListener(this); mBIC->setBufferFreedListener(mBFL); - sp<IProducerListener> producerListener = new StubProducerListener(); + sp<IProducerListener> producerListener = new TrackingProducerListener(this); + mProducer = mBIC->getSurface()->getIGraphicBufferProducer(); IGraphicBufferProducer::QueueBufferOutput bufferOutput; ASSERT_EQ(NO_ERROR, mProducer->connect(producerListener, NATIVE_WINDOW_API_CPU, @@ -71,6 +84,13 @@ class BufferItemConsumerTest : public ::testing::Test { ALOGD("HandleBufferFreed, mFreedBufferCount=%d", mFreedBufferCount); } + void HandleBufferDetached(int slot) { + std::lock_guard<std::mutex> lock(mMutex); + mDetachedBufferSlots.push_back(slot); + ALOGD("HandleBufferDetached, slot=%d mDetachedBufferSlots-count=%zu", slot, + mDetachedBufferSlots.size()); + } + void DequeueBuffer(int* outSlot) { ASSERT_NE(outSlot, nullptr); @@ -120,6 +140,7 @@ class BufferItemConsumerTest : public ::testing::Test { std::mutex mMutex; int mFreedBufferCount{0}; + std::vector<int> mDetachedBufferSlots = {}; sp<BufferItemConsumer> mBIC; sp<BufferFreedListener> mBFL; @@ -203,4 +224,19 @@ TEST_F(BufferItemConsumerTest, TriggerBufferFreed_DeleteBufferItemConsumer) { ASSERT_EQ(1, GetFreedBufferCount()); } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) +// Test that delete BufferItemConsumer triggers onBufferFreed. +TEST_F(BufferItemConsumerTest, DetachBufferWithBuffer) { + int slot; + // Let buffer go through the cycle at least once. + DequeueBuffer(&slot); + QueueBuffer(slot); + AcquireBuffer(&slot); + + sp<GraphicBuffer> buffer = mBuffers[slot]; + EXPECT_EQ(OK, mBIC->detachBuffer(buffer)); + EXPECT_THAT(mDetachedBufferSlots, testing::ElementsAre(slot)); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + } // namespace android diff --git a/libs/gui/tests/BufferQueue_test.cpp b/libs/gui/tests/BufferQueue_test.cpp index 590e2c87c9..2e6ffcb57f 100644 --- a/libs/gui/tests/BufferQueue_test.cpp +++ b/libs/gui/tests/BufferQueue_test.cpp @@ -1430,19 +1430,15 @@ TEST_F(BufferQueueTest, TestBqSetFrameRateFlagBuildTimeIsSet) { } struct BufferItemConsumerSetFrameRateListener : public BufferItemConsumer { - BufferItemConsumerSetFrameRateListener(const sp<IGraphicBufferConsumer>& consumer) - : BufferItemConsumer(consumer, GRALLOC_USAGE_SW_READ_OFTEN, 1) {} + BufferItemConsumerSetFrameRateListener() : BufferItemConsumer(GRALLOC_USAGE_SW_READ_OFTEN, 1) {} MOCK_METHOD(void, onSetFrameRate, (float, int8_t, int8_t), (override)); }; TEST_F(BufferQueueTest, TestSetFrameRate) { - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<BufferItemConsumerSetFrameRateListener> bufferConsumer = - sp<BufferItemConsumerSetFrameRateListener>::make(consumer); + sp<BufferItemConsumerSetFrameRateListener>::make(); + sp<IGraphicBufferProducer> producer = bufferConsumer->getSurface()->getIGraphicBufferProducer(); EXPECT_CALL(*bufferConsumer, onSetFrameRate(12.34f, 1, 0)).Times(1); producer->setFrameRate(12.34f, 1, 0); @@ -1493,14 +1489,10 @@ struct OneshotOnDequeuedListener final : public BufferItemConsumer::FrameAvailab // See b/270004534 TEST(BufferQueueThreading, TestProducerDequeueConsumerDestroy) { - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<BufferItemConsumer> bufferConsumer = - sp<BufferItemConsumer>::make(consumer, GRALLOC_USAGE_SW_READ_OFTEN, 2); + sp<BufferItemConsumer>::make(GRALLOC_USAGE_SW_READ_OFTEN, 2); ASSERT_NE(nullptr, bufferConsumer.get()); - sp<Surface> surface = sp<Surface>::make(producer); + sp<Surface> surface = bufferConsumer->getSurface(); native_window_set_buffers_format(surface.get(), PIXEL_FORMAT_RGBA_8888); native_window_set_buffers_dimensions(surface.get(), 100, 100); @@ -1531,14 +1523,10 @@ TEST(BufferQueueThreading, TestProducerDequeueConsumerDestroy) { } TEST_F(BufferQueueTest, TestAdditionalOptions) { - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<BufferItemConsumer> bufferConsumer = - sp<BufferItemConsumer>::make(consumer, GRALLOC_USAGE_SW_READ_OFTEN, 2); + sp<BufferItemConsumer>::make(GRALLOC_USAGE_SW_READ_OFTEN, 2); ASSERT_NE(nullptr, bufferConsumer.get()); - sp<Surface> surface = sp<Surface>::make(producer); + sp<Surface> surface = bufferConsumer->getSurface(); native_window_set_buffers_format(surface.get(), PIXEL_FORMAT_RGBA_8888); native_window_set_buffers_dimensions(surface.get(), 100, 100); diff --git a/libs/gui/tests/BufferReleaseChannel_test.cpp b/libs/gui/tests/BufferReleaseChannel_test.cpp new file mode 100644 index 0000000000..11d122b525 --- /dev/null +++ b/libs/gui/tests/BufferReleaseChannel_test.cpp @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <vector> + +#include <gtest/gtest.h> +#include <gui/BufferReleaseChannel.h> + +using namespace std::string_literals; +using android::gui::BufferReleaseChannel; + +namespace android { + +namespace { + +// Helper function to check if two file descriptors point to the same file. +bool is_same_file(int fd1, int fd2) { + struct stat stat1; + if (fstat(fd1, &stat1) != 0) { + return false; + } + struct stat stat2; + if (fstat(fd2, &stat2) != 0) { + return false; + } + return (stat1.st_dev == stat2.st_dev) && (stat1.st_ino == stat2.st_ino); +} + +} // namespace + +TEST(BufferReleaseChannelTest, MessageFlattenable) { + ReleaseCallbackId releaseCallbackId{1, 2}; + sp<Fence> releaseFence = sp<Fence>::make(memfd_create("fake-fence-fd", 0)); + uint32_t maxAcquiredBufferCount = 5; + + std::vector<uint8_t> dataBuffer; + std::vector<int> fdBuffer; + + // Verify that we can flatten a message + { + BufferReleaseChannel::Message message{releaseCallbackId, releaseFence, + maxAcquiredBufferCount}; + + dataBuffer.resize(message.getFlattenedSize()); + void* dataPtr = dataBuffer.data(); + size_t dataSize = dataBuffer.size(); + + fdBuffer.resize(message.getFdCount()); + int* fdPtr = fdBuffer.data(); + size_t fdSize = fdBuffer.size(); + + ASSERT_EQ(OK, message.flatten(dataPtr, dataSize, fdPtr, fdSize)); + + // Fence's unique_fd uses fdsan to check ownership of the file descriptor. Normally the file + // descriptor is passed through the Unix socket and duplicated (and sent to another process) + // so there's no problem with duplicate file descriptor ownership. For this unit test, we + // need to set up a duplicate file descriptor to avoid crashing due to duplicate ownership. + ASSERT_EQ(releaseFence->get(), fdBuffer[0]); + fdBuffer[0] = message.releaseFence->dup(); + } + + // Verify that we can unflatten a message + { + BufferReleaseChannel::Message message; + + const void* dataPtr = dataBuffer.data(); + size_t dataSize = dataBuffer.size(); + + const int* fdPtr = fdBuffer.data(); + size_t fdSize = fdBuffer.size(); + + ASSERT_EQ(OK, message.unflatten(dataPtr, dataSize, fdPtr, fdSize)); + ASSERT_EQ(releaseCallbackId, message.releaseCallbackId); + ASSERT_TRUE(is_same_file(releaseFence->get(), message.releaseFence->get())); + ASSERT_EQ(maxAcquiredBufferCount, message.maxAcquiredBufferCount); + } +} + +// Verify that the BufferReleaseChannel consume returns WOULD_BLOCK when there's no message +// available. +TEST(BufferReleaseChannelTest, ConsumerEndpointIsNonBlocking) { + std::unique_ptr<BufferReleaseChannel::ConsumerEndpoint> consumer; + std::shared_ptr<BufferReleaseChannel::ProducerEndpoint> producer; + ASSERT_EQ(OK, BufferReleaseChannel::open("test-channel"s, consumer, producer)); + + ReleaseCallbackId releaseCallbackId; + sp<Fence> releaseFence; + uint32_t maxAcquiredBufferCount; + ASSERT_EQ(WOULD_BLOCK, + consumer->readReleaseFence(releaseCallbackId, releaseFence, maxAcquiredBufferCount)); +} + +// Verify that we can write a message to the BufferReleaseChannel producer and read that message +// using the BufferReleaseChannel consumer. +TEST(BufferReleaseChannelTest, ProduceAndConsume) { + std::unique_ptr<BufferReleaseChannel::ConsumerEndpoint> consumer; + std::shared_ptr<BufferReleaseChannel::ProducerEndpoint> producer; + ASSERT_EQ(OK, BufferReleaseChannel::open("test-channel"s, consumer, producer)); + + sp<Fence> fence = sp<Fence>::make(memfd_create("fake-fence-fd", 0)); + + for (uint64_t i = 0; i < 64; i++) { + ReleaseCallbackId producerId{i, i + 1}; + uint32_t maxAcquiredBufferCount = i + 2; + ASSERT_EQ(OK, producer->writeReleaseFence(producerId, fence, maxAcquiredBufferCount)); + } + + for (uint64_t i = 0; i < 64; i++) { + ReleaseCallbackId expectedId{i, i + 1}; + uint32_t expectedMaxAcquiredBufferCount = i + 2; + + ReleaseCallbackId consumerId; + sp<Fence> consumerFence; + uint32_t maxAcquiredBufferCount; + ASSERT_EQ(OK, + consumer->readReleaseFence(consumerId, consumerFence, maxAcquiredBufferCount)); + + ASSERT_EQ(expectedId, consumerId); + ASSERT_TRUE(is_same_file(fence->get(), consumerFence->get())); + ASSERT_EQ(expectedMaxAcquiredBufferCount, maxAcquiredBufferCount); + } +} + +} // namespace android
\ No newline at end of file diff --git a/libs/gui/tests/Choreographer_test.cpp b/libs/gui/tests/Choreographer_test.cpp index 2ac2550f07..8db48d2eb0 100644 --- a/libs/gui/tests/Choreographer_test.cpp +++ b/libs/gui/tests/Choreographer_test.cpp @@ -52,25 +52,23 @@ TEST_F(ChoreographerTest, InputCallbackBeforeAnimation) { sp<Looper> looper = Looper::prepare(0); Choreographer* choreographer = Choreographer::getForThread(); VsyncCallback animationCb; - VsyncCallback inputCb; - choreographer->postFrameCallbackDelayed(nullptr, nullptr, vsyncCallback, &animationCb, 0, CALLBACK_ANIMATION); + VsyncCallback inputCb; choreographer->postFrameCallbackDelayed(nullptr, nullptr, vsyncCallback, &inputCb, 0, CALLBACK_INPUT); - - nsecs_t startTime = systemTime(SYSTEM_TIME_MONOTONIC); - nsecs_t currTime; - int pollResult; + auto startTime = std::chrono::system_clock::now(); do { - pollResult = looper->pollOnce(16); - currTime = systemTime(SYSTEM_TIME_MONOTONIC); - } while (!(inputCb.callbackReceived() && animationCb.callbackReceived()) && - (pollResult != Looper::POLL_TIMEOUT && pollResult != Looper::POLL_ERROR) && - (currTime - startTime < 3000)); - - ASSERT_TRUE(inputCb.callbackReceived()) << "did not receive input callback"; - ASSERT_TRUE(animationCb.callbackReceived()) << "did not receive animation callback"; + static constexpr int32_t timeoutMs = 1000; + int pollResult = looper->pollOnce(timeoutMs); + ASSERT_TRUE((pollResult != Looper::POLL_TIMEOUT) && (pollResult != Looper::POLL_ERROR)) + << "Failed to poll looper. Poll result = " << pollResult; + auto elapsedMs = std::chrono::duration_cast<std::chrono::milliseconds>( + std::chrono::system_clock::now() - startTime); + ASSERT_LE(elapsedMs.count(), timeoutMs) + << "Timed out waiting for callbacks. inputCb=" << inputCb.callbackReceived() + << " animationCb=" << animationCb.callbackReceived(); + } while (!(inputCb.callbackReceived() && animationCb.callbackReceived())); ASSERT_EQ(inputCb.frameTime, animationCb.frameTime) << android::base::StringPrintf("input and animation callback frame times don't match. " diff --git a/libs/gui/tests/CpuConsumer_test.cpp b/libs/gui/tests/CpuConsumer_test.cpp index d80bd9c62a..f4239cb69e 100644 --- a/libs/gui/tests/CpuConsumer_test.cpp +++ b/libs/gui/tests/CpuConsumer_test.cpp @@ -66,13 +66,10 @@ protected: test_info->name(), params.width, params.height, params.maxLockedBuffers, params.format); - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - mCC = new CpuConsumer(consumer, params.maxLockedBuffers); + mCC = new CpuConsumer(params.maxLockedBuffers); String8 name("CpuConsumer_Under_Test"); mCC->setName(name); - mSTC = new Surface(producer); + mSTC = mCC->getSurface(); mANW = mSTC; } diff --git a/libs/gui/tests/LibGuiMain.cpp b/libs/gui/tests/LibGuiMain.cpp index 10f7207588..7c7c2cc30f 100644 --- a/libs/gui/tests/LibGuiMain.cpp +++ b/libs/gui/tests/LibGuiMain.cpp @@ -14,8 +14,15 @@ * limitations under the License. */ -#include "gtest/gtest.h" -#include "log/log.h" +#include <android-base/unique_fd.h> +#include <gtest/gtest.h> +#include <log/log.h> + +#include "testserver/TestServer.h" +#include "testserver/TestServerClient.h" +#include "testserver/TestServerHost.h" + +using namespace android; namespace { @@ -32,7 +39,34 @@ class TestCaseLogger : public ::testing::EmptyTestEventListener { } // namespace int main(int argc, char** argv) { + // There are three modes that we can run in to support the libgui TestServer: + // + // - libgui_test : normal mode, runs tests and fork/execs the testserver host process + // - libgui_test --test-server-host $recvPipeFd $sendPipeFd : TestServerHost mode, listens on + // $recvPipeFd for commands and sends responses over $sendPipeFd + // - libgui_test --test-server $name : TestServer mode, starts a ITestService binder service + // under $name + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + if (arg == "--test-server-host") { + LOG_ALWAYS_FATAL_IF(argc < (i + 2), "--test-server-host requires two pipe fds"); + // Note that the send/recv are from our perspective. + base::unique_fd recvPipeFd = base::unique_fd(atoi(argv[i + 1])); + base::unique_fd sendPipeFd = base::unique_fd(atoi(argv[i + 2])); + return TestServerHostMain(argv[0], std::move(sendPipeFd), std::move(recvPipeFd)); + } + if (arg == "--test-server") { + LOG_ALWAYS_FATAL_IF(argc < (i + 1), "--test-server requires a name"); + return TestServerMain(argv[i + 1]); + } + } testing::InitGoogleTest(&argc, argv); testing::UnitTest::GetInstance()->listeners().Append(new TestCaseLogger()); + + // This has to be run *before* any test initialization, because it fork/execs a TestServerHost, + // which will later create new binder service. You can't do that in a forked thread after you've + // initialized any binder stuff, which some tests do. + TestServerClient::InitializeOrDie(argv[0]); + return RUN_ALL_TESTS(); }
\ No newline at end of file diff --git a/libs/gui/tests/MultiTextureConsumer_test.cpp b/libs/gui/tests/MultiTextureConsumer_test.cpp index 7d3d4aa412..2428bb3110 100644 --- a/libs/gui/tests/MultiTextureConsumer_test.cpp +++ b/libs/gui/tests/MultiTextureConsumer_test.cpp @@ -34,12 +34,8 @@ protected: virtual void SetUp() { GLTest::SetUp(); - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - mGlConsumer = new GLConsumer(consumer, TEX_ID, - GLConsumer::TEXTURE_EXTERNAL, true, false); - mSurface = new Surface(producer); + mGlConsumer = new GLConsumer(TEX_ID, GLConsumer::TEXTURE_EXTERNAL, true, false); + mSurface = mGlConsumer->getSurface(); mANW = mSurface.get(); } diff --git a/libs/gui/tests/RegionSampling_test.cpp b/libs/gui/tests/RegionSampling_test.cpp index 223e4b6cbd..a0d8c53385 100644 --- a/libs/gui/tests/RegionSampling_test.cpp +++ b/libs/gui/tests/RegionSampling_test.cpp @@ -19,7 +19,7 @@ #include <android/gui/BnRegionSamplingListener.h> #include <binder/ProcessState.h> -#include <gui/AidlStatusUtil.h> +#include <gui/AidlUtil.h> #include <gui/DisplayEventReceiver.h> #include <gui/ISurfaceComposer.h> #include <gui/Surface.h> diff --git a/libs/gui/tests/SurfaceTextureClient_test.cpp b/libs/gui/tests/SurfaceTextureClient_test.cpp index b28dca8ab4..59d05b673c 100644 --- a/libs/gui/tests/SurfaceTextureClient_test.cpp +++ b/libs/gui/tests/SurfaceTextureClient_test.cpp @@ -40,12 +40,8 @@ protected: } virtual void SetUp() { - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - mST = new GLConsumer(consumer, 123, GLConsumer::TEXTURE_EXTERNAL, true, - false); - mSTC = new Surface(producer); + mST = new GLConsumer(123, GLConsumer::TEXTURE_EXTERNAL, true, false); + mSTC = mST->getSurface(); mANW = mSTC; // We need a valid GL context so we can test updateTexImage() @@ -731,12 +727,8 @@ protected: ASSERT_NE(EGL_NO_CONTEXT, mEglContext); for (int i = 0; i < NUM_SURFACE_TEXTURES; i++) { - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<GLConsumer> st(new GLConsumer(consumer, i, - GLConsumer::TEXTURE_EXTERNAL, true, false)); - sp<Surface> stc(new Surface(producer)); + sp<GLConsumer> st(new GLConsumer(i, GLConsumer::TEXTURE_EXTERNAL, true, false)); + sp<Surface> stc = st->getSurface(); mEglSurfaces[i] = eglCreateWindowSurface(mEglDisplay, myConfig, static_cast<ANativeWindow*>(stc.get()), nullptr); ASSERT_EQ(EGL_SUCCESS, eglGetError()); diff --git a/libs/gui/tests/SurfaceTextureGL.h b/libs/gui/tests/SurfaceTextureGL.h index 9d8af5d0f5..1309635afd 100644 --- a/libs/gui/tests/SurfaceTextureGL.h +++ b/libs/gui/tests/SurfaceTextureGL.h @@ -38,11 +38,8 @@ protected: void SetUp() { GLTest::SetUp(); - sp<IGraphicBufferProducer> producer; - BufferQueue::createBufferQueue(&producer, &mConsumer); - mST = new GLConsumer(mConsumer, TEX_ID, GLConsumer::TEXTURE_EXTERNAL, - true, false); - mSTC = new Surface(producer); + mST = new GLConsumer(TEX_ID, GLConsumer::TEXTURE_EXTERNAL, true, false); + mSTC = mST->getSurface(); mANW = mSTC; ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(), TEST_PRODUCER_USAGE_BITS)); mTextureRenderer = new TextureRenderer(TEX_ID, mST); @@ -63,7 +60,6 @@ protected: mTextureRenderer->drawTexture(); } - sp<IGraphicBufferConsumer> mConsumer; sp<GLConsumer> mST; sp<Surface> mSTC; sp<ANativeWindow> mANW; diff --git a/libs/gui/tests/SurfaceTextureGL_test.cpp b/libs/gui/tests/SurfaceTextureGL_test.cpp index f76c0be265..449533aa57 100644 --- a/libs/gui/tests/SurfaceTextureGL_test.cpp +++ b/libs/gui/tests/SurfaceTextureGL_test.cpp @@ -480,8 +480,8 @@ TEST_F(SurfaceTextureGLTest, DisconnectStressTest) { }; sp<DisconnectWaiter> dw(new DisconnectWaiter()); - mConsumer->consumerConnect(dw, false); - + sp<IGraphicBufferConsumer> consumer = mST->getIGraphicBufferConsumer(); + consumer->consumerConnect(dw, false); sp<Thread> pt(new ProducerThread(mANW)); pt->run("ProducerThread"); diff --git a/libs/gui/tests/Surface_test.cpp b/libs/gui/tests/Surface_test.cpp index 43cd0f8a7f..88893b64ba 100644 --- a/libs/gui/tests/Surface_test.cpp +++ b/libs/gui/tests/Surface_test.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "gui/view/Surface.h" #include "Constants.h" #include "MockConsumer.h" @@ -23,28 +24,41 @@ #include <android/gui/IDisplayEventConnection.h> #include <android/gui/ISurfaceComposer.h> #include <android/hardware/configstore/1.0/ISurfaceFlingerConfigs.h> +#include <android/hardware_buffer.h> #include <binder/ProcessState.h> +#include <com_android_graphics_libgui_flags.h> #include <configstore/Utils.h> -#include <gui/AidlStatusUtil.h> +#include <gui/AidlUtil.h> #include <gui/BufferItemConsumer.h> -#include <gui/IProducerListener.h> +#include <gui/BufferQueue.h> +#include <gui/CpuConsumer.h> +#include <gui/IConsumerListener.h> +#include <gui/IGraphicBufferConsumer.h> +#include <gui/IGraphicBufferProducer.h> #include <gui/ISurfaceComposer.h> #include <gui/Surface.h> #include <gui/SurfaceComposerClient.h> #include <gui/SyncScreenCaptureListener.h> -#include <inttypes.h> #include <private/gui/ComposerService.h> #include <private/gui/ComposerServiceAIDL.h> #include <sys/types.h> +#include <system/window.h> #include <ui/BufferQueueDefs.h> #include <ui/DisplayMode.h> +#include <ui/GraphicBuffer.h> #include <ui/Rect.h> #include <utils/Errors.h> #include <utils/String8.h> +#include <chrono> +#include <cstddef> +#include <cstdint> +#include <future> #include <limits> #include <thread> +#include "testserver/TestServerClient.h" + namespace android { using namespace std::chrono_literals; @@ -82,7 +96,7 @@ public: virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& buffers) { mDiscardedBuffers.insert(mDiscardedBuffers.end(), buffers.begin(), buffers.end()); } - + virtual void onBufferDetached(int /*slot*/) {} int getReleaseNotifyCount() const { return mBuffersReleased; } @@ -97,6 +111,18 @@ private: std::vector<sp<GraphicBuffer>> mDiscardedBuffers; }; +class DeathWatcherListener : public StubSurfaceListener { +public: + virtual void onRemoteDied() { mDiedPromise.set_value(true); } + + virtual bool needsDeathNotify() { return true; } + + std::future<bool> getDiedFuture() { return mDiedPromise.get_future(); } + +private: + std::promise<bool> mDiedPromise; +}; + class SurfaceTest : public ::testing::Test { protected: SurfaceTest() { @@ -143,10 +169,10 @@ protected: if (hasSurfaceListener) { listener = new FakeSurfaceListener(enableReleasedCb); } - ASSERT_EQ(OK, surface->connect( - NATIVE_WINDOW_API_CPU, - /*reportBufferRemoval*/true, - /*listener*/listener)); + ASSERT_EQ(OK, + surface->connect(NATIVE_WINDOW_API_CPU, + /*listener*/ listener, + /*reportBufferRemoval*/ true)); const int BUFFER_COUNT = 4 + extraDiscardedBuffers; ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(window.get(), BUFFER_COUNT)); ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); @@ -264,13 +290,9 @@ TEST_F(SurfaceTest, LayerCountIsOne) { TEST_F(SurfaceTest, QueryConsumerUsage) { const int TEST_USAGE_FLAGS = GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_HW_RENDER; - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<BufferItemConsumer> c = new BufferItemConsumer(consumer, - TEST_USAGE_FLAGS); - sp<Surface> s = new Surface(producer); + sp<BufferItemConsumer> c = new BufferItemConsumer(TEST_USAGE_FLAGS); + sp<Surface> s = c->getSurface(); sp<ANativeWindow> anw(s); int flags = -1; @@ -282,15 +304,11 @@ TEST_F(SurfaceTest, QueryConsumerUsage) { TEST_F(SurfaceTest, QueryDefaultBuffersDataSpace) { const android_dataspace TEST_DATASPACE = HAL_DATASPACE_V0_SRGB; - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<CpuConsumer> cpuConsumer = new CpuConsumer(consumer, 1); + sp<CpuConsumer> cpuConsumer = new CpuConsumer(1); cpuConsumer->setDefaultBufferDataSpace(TEST_DATASPACE); - sp<Surface> s = new Surface(producer); - + sp<Surface> s = cpuConsumer->getSurface(); sp<ANativeWindow> anw(s); android_dataspace dataSpace; @@ -303,11 +321,8 @@ TEST_F(SurfaceTest, QueryDefaultBuffersDataSpace) { } TEST_F(SurfaceTest, SettingGenerationNumber) { - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<CpuConsumer> cpuConsumer = new CpuConsumer(consumer, 1); - sp<Surface> surface = new Surface(producer); + sp<CpuConsumer> cpuConsumer = new CpuConsumer(1); + sp<Surface> surface = cpuConsumer->getSurface(); sp<ANativeWindow> window(surface); // Allocate a buffer with a generation number of 0 @@ -491,11 +506,11 @@ TEST_F(SurfaceTest, GetAndFlushRemovedBuffers) { sp<Surface> surface = new Surface(producer); sp<ANativeWindow> window(surface); - sp<StubProducerListener> listener = new StubProducerListener(); - ASSERT_EQ(OK, surface->connect( - NATIVE_WINDOW_API_CPU, - /*listener*/listener, - /*reportBufferRemoval*/true)); + sp<StubSurfaceListener> listener = new StubSurfaceListener(); + ASSERT_EQ(OK, + surface->connect(NATIVE_WINDOW_API_CPU, + /*listener*/ listener, + /*reportBufferRemoval*/ true)); const int BUFFER_COUNT = 4; ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(window.get(), BUFFER_COUNT)); ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); @@ -636,7 +651,7 @@ public: status_t setTransactionState( const FrameTimelineInfo& /*frameTimelineInfo*/, Vector<ComposerState>& /*state*/, - const Vector<DisplayState>& /*displays*/, uint32_t /*flags*/, + Vector<DisplayState>& /*displays*/, uint32_t /*flags*/, const sp<IBinder>& /*applyToken*/, InputWindowCommands /*inputWindowCommands*/, int64_t /*desiredPresentTime*/, bool /*isAutoTimestamp*/, const std::vector<client_cache_t>& /*cachedBuffer*/, bool /*hasListenerCallbacks*/, @@ -987,6 +1002,19 @@ public: binder::Status notifyShutdown() override { return binder::Status::ok(); } + binder::Status addJankListener(const sp<IBinder>& /*layer*/, + const sp<gui::IJankListener>& /*listener*/) override { + return binder::Status::ok(); + } + + binder::Status flushJankData(int32_t /*layerId*/) override { return binder::Status::ok(); } + + binder::Status removeJankListener(int32_t /*layerId*/, + const sp<gui::IJankListener>& /*listener*/, + int64_t /*afterVsync*/) override { + return binder::Status::ok(); + } + protected: IBinder* onAsBinder() override { return nullptr; } @@ -2134,14 +2162,11 @@ TEST_F(SurfaceTest, DefaultMaxBufferCountSetAndUpdated) { TEST_F(SurfaceTest, BatchOperations) { const int BUFFER_COUNT = 16; const int BATCH_SIZE = 8; - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<CpuConsumer> cpuConsumer = new CpuConsumer(consumer, 1); - sp<Surface> surface = new Surface(producer); + sp<CpuConsumer> cpuConsumer = new CpuConsumer(1); + sp<Surface> surface = cpuConsumer->getSurface(); sp<ANativeWindow> window(surface); - sp<StubProducerListener> listener = new StubProducerListener(); + sp<StubSurfaceListener> listener = new StubSurfaceListener(); ASSERT_EQ(OK, surface->connect(NATIVE_WINDOW_API_CPU, /*listener*/listener, /*reportBufferRemoval*/false)); @@ -2186,14 +2211,11 @@ TEST_F(SurfaceTest, BatchOperations) { TEST_F(SurfaceTest, BatchIllegalOperations) { const int BUFFER_COUNT = 16; const int BATCH_SIZE = 8; - sp<IGraphicBufferProducer> producer; - sp<IGraphicBufferConsumer> consumer; - BufferQueue::createBufferQueue(&producer, &consumer); - sp<CpuConsumer> cpuConsumer = new CpuConsumer(consumer, 1); - sp<Surface> surface = new Surface(producer); + sp<CpuConsumer> cpuConsumer = new CpuConsumer(1); + sp<Surface> surface = cpuConsumer->getSurface(); sp<ANativeWindow> window(surface); - sp<StubProducerListener> listener = new StubProducerListener(); + sp<StubSurfaceListener> listener = new StubSurfaceListener(); ASSERT_EQ(OK, surface->connect(NATIVE_WINDOW_API_CPU, /*listener*/listener, /*reportBufferRemoval*/false)); @@ -2213,4 +2235,288 @@ TEST_F(SurfaceTest, BatchIllegalOperations) { ASSERT_EQ(NO_ERROR, surface->disconnect(NATIVE_WINDOW_API_CPU)); } +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + +TEST_F(SurfaceTest, PlatformBufferMethods) { + sp<CpuConsumer> cpuConsumer = sp<CpuConsumer>::make(1); + sp<Surface> surface = cpuConsumer->getSurface(); + sp<StubSurfaceListener> listener = sp<StubSurfaceListener>::make(); + sp<GraphicBuffer> buffer; + sp<Fence> fence; + + EXPECT_EQ(OK, + surface->connect(NATIVE_WINDOW_API_CPU, listener, /* reportBufferRemoval */ false)); + + // + // Verify nullptrs are handled safely: + // + + EXPECT_EQ(BAD_VALUE, surface->dequeueBuffer((sp<GraphicBuffer>*)nullptr, nullptr)); + EXPECT_EQ(BAD_VALUE, surface->dequeueBuffer((sp<GraphicBuffer>*)nullptr, &fence)); + EXPECT_EQ(BAD_VALUE, surface->dequeueBuffer(&buffer, nullptr)); + EXPECT_EQ(BAD_VALUE, surface->queueBuffer(nullptr, nullptr)); + EXPECT_EQ(BAD_VALUE, surface->detachBuffer(nullptr)); + + // + // Verify dequeue/queue: + // + + EXPECT_EQ(OK, surface->dequeueBuffer(&buffer, &fence)); + EXPECT_NE(nullptr, buffer); + EXPECT_EQ(OK, surface->queueBuffer(buffer, fence)); + + // + // Verify dequeue/detach: + // + + wp<GraphicBuffer> weakBuffer; + { + EXPECT_EQ(OK, surface->dequeueBuffer(&buffer, &fence)); + + EXPECT_EQ(OK, surface->detachBuffer(buffer)); + + weakBuffer = buffer; + buffer = nullptr; + } + EXPECT_EQ(nullptr, weakBuffer.promote()) << "Weak buffer still held by Surface."; + + // + // Verify detach without borrowing the buffer does not work: + // + + sp<GraphicBuffer> heldTooLongBuffer; + EXPECT_EQ(OK, surface->dequeueBuffer(&heldTooLongBuffer, &fence)); + EXPECT_EQ(OK, surface->queueBuffer(heldTooLongBuffer)); + EXPECT_EQ(BAD_VALUE, surface->detachBuffer(heldTooLongBuffer)); +} + +TEST_F(SurfaceTest, AllowAllocation) { + // controlledByApp must be true to disable blocking + sp<CpuConsumer> cpuConsumer = sp<CpuConsumer>::make(1, /*controlledByApp*/ true); + sp<Surface> surface = cpuConsumer->getSurface(); + sp<StubSurfaceListener> listener = sp<StubSurfaceListener>::make(); + sp<GraphicBuffer> buffer; + sp<Fence> fence; + + EXPECT_EQ(OK, + surface->connect(NATIVE_WINDOW_API_CPU, listener, /* reportBufferRemoval */ false)); + EXPECT_EQ(OK, surface->allowAllocation(false)); + + EXPECT_EQ(OK, surface->setDequeueTimeout(-1)); + EXPECT_EQ(WOULD_BLOCK, surface->dequeueBuffer(&buffer, &fence)); + + EXPECT_EQ(OK, surface->setDequeueTimeout(10)); + EXPECT_EQ(TIMED_OUT, surface->dequeueBuffer(&buffer, &fence)); + + EXPECT_EQ(OK, surface->allowAllocation(true)); + EXPECT_EQ(OK, surface->dequeueBuffer(&buffer, &fence)); +} + +TEST_F(SurfaceTest, QueueAcquireReleaseDequeue_CalledInStack_DoesNotDeadlock) { + class DequeuingSurfaceListener : public SurfaceListener { + public: + DequeuingSurfaceListener(const wp<Surface>& surface) : mSurface(surface) {} + + virtual void onBufferReleased() override { + sp<Surface> surface = mSurface.promote(); + ASSERT_NE(nullptr, surface); + EXPECT_EQ(OK, surface->dequeueBuffer(&mBuffer, &mFence)); + } + + virtual bool needsReleaseNotify() override { return true; } + virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>&) override {} + virtual void onBufferDetached(int) override {} + + sp<GraphicBuffer> mBuffer; + sp<Fence> mFence; + + private: + wp<Surface> mSurface; + }; + + class ImmediateReleaseConsumerListener : public BufferItemConsumer::FrameAvailableListener { + public: + ImmediateReleaseConsumerListener(const wp<BufferItemConsumer>& consumer) + : mConsumer(consumer) {} + + virtual void onFrameAvailable(const BufferItem&) override { + sp<BufferItemConsumer> consumer = mConsumer.promote(); + ASSERT_NE(nullptr, consumer); + + mCalls += 1; + + BufferItem buffer; + EXPECT_EQ(OK, consumer->acquireBuffer(&buffer, 0)); + EXPECT_EQ(OK, consumer->releaseBuffer(buffer)); + } + + size_t mCalls = 0; + + private: + wp<BufferItemConsumer> mConsumer; + }; + + sp<IGraphicBufferProducer> bqProducer; + sp<IGraphicBufferConsumer> bqConsumer; + BufferQueue::createBufferQueue(&bqProducer, &bqConsumer); + + sp<BufferItemConsumer> consumer = sp<BufferItemConsumer>::make(bqConsumer, 3); + sp<Surface> surface = sp<Surface>::make(bqProducer); + sp<ImmediateReleaseConsumerListener> consumerListener = + sp<ImmediateReleaseConsumerListener>::make(consumer); + consumer->setFrameAvailableListener(consumerListener); + + sp<DequeuingSurfaceListener> surfaceListener = sp<DequeuingSurfaceListener>::make(surface); + EXPECT_EQ(OK, surface->connect(NATIVE_WINDOW_API_CPU, surfaceListener, false)); + + EXPECT_EQ(OK, surface->setMaxDequeuedBufferCount(2)); + + sp<GraphicBuffer> buffer; + sp<Fence> fence; + EXPECT_EQ(OK, surface->dequeueBuffer(&buffer, &fence)); + EXPECT_EQ(OK, surface->queueBuffer(buffer, fence)); + + EXPECT_EQ(1u, consumerListener->mCalls); + EXPECT_NE(nullptr, surfaceListener->mBuffer); + + EXPECT_EQ(OK, surface->disconnect(NATIVE_WINDOW_API_CPU)); +} + +TEST_F(SurfaceTest, ViewSurface_toString) { + view::Surface surface{}; + EXPECT_EQ("", surface.toString()); + + surface.name = String16("name"); + EXPECT_EQ("name", surface.toString()); +} + +TEST_F(SurfaceTest, TestRemoteSurfaceDied_CallbackCalled) { + sp<TestServerClient> testServer = TestServerClient::Create(); + sp<IGraphicBufferProducer> producer = testServer->CreateProducer(); + EXPECT_NE(nullptr, producer); + + sp<Surface> surface = sp<Surface>::make(producer); + sp<DeathWatcherListener> deathWatcher = sp<DeathWatcherListener>::make(); + EXPECT_EQ(OK, surface->connect(NATIVE_WINDOW_API_CPU, deathWatcher)); + + auto diedFuture = deathWatcher->getDiedFuture(); + EXPECT_EQ(OK, testServer->Kill()); + + diedFuture.wait(); + EXPECT_TRUE(diedFuture.get()); +} + +TEST_F(SurfaceTest, TestRemoteSurfaceDied_Disconnect_CallbackNotCalled) { + sp<TestServerClient> testServer = TestServerClient::Create(); + sp<IGraphicBufferProducer> producer = testServer->CreateProducer(); + EXPECT_NE(nullptr, producer); + + sp<Surface> surface = sp<Surface>::make(producer); + sp<DeathWatcherListener> deathWatcher = sp<DeathWatcherListener>::make(); + EXPECT_EQ(OK, surface->connect(NATIVE_WINDOW_API_CPU, deathWatcher)); + EXPECT_EQ(OK, surface->disconnect(NATIVE_WINDOW_API_CPU)); + + auto watcherDiedFuture = deathWatcher->getDiedFuture(); + EXPECT_EQ(OK, testServer->Kill()); + + std::future_status status = watcherDiedFuture.wait_for(std::chrono::seconds(1)); + EXPECT_EQ(std::future_status::timeout, status); +} + +TEST_F(SurfaceTest, QueueBufferOutput_TracksReplacements) { + sp<BufferItemConsumer> consumer = sp<BufferItemConsumer>::make(GRALLOC_USAGE_SW_READ_OFTEN); + ASSERT_EQ(OK, consumer->setMaxBufferCount(3)); + ASSERT_EQ(OK, consumer->setMaxAcquiredBufferCount(1)); + + sp<Surface> surface = consumer->getSurface(); + sp<StubSurfaceListener> listener = sp<StubSurfaceListener>::make(); + + // Async mode sets up an extra buffer so the surface can queue it without waiting. + ASSERT_EQ(OK, surface->setMaxDequeuedBufferCount(1)); + ASSERT_EQ(OK, surface->setAsyncMode(true)); + ASSERT_EQ(OK, surface->connect(NATIVE_WINDOW_API_CPU, listener)); + + sp<GraphicBuffer> buffer; + sp<Fence> fence; + SurfaceQueueBufferOutput output; + BufferItem item; + + // We can queue directly, without an output arg. + EXPECT_EQ(OK, surface->dequeueBuffer(&buffer, &fence)); + EXPECT_EQ(OK, surface->queueBuffer(buffer, fence)); + EXPECT_EQ(OK, consumer->acquireBuffer(&item, 0)); + EXPECT_EQ(OK, consumer->releaseBuffer(item)); + + // We can queue with an output arg, and that we don't expect to see a replacement. + EXPECT_EQ(OK, surface->dequeueBuffer(&buffer, &fence)); + EXPECT_EQ(OK, surface->queueBuffer(buffer, fence, &output)); + EXPECT_FALSE(output.bufferReplaced); + + // We expect see a replacement when we queue a second buffer in async mode, and the consumer + // hasn't acquired the first one yet. + EXPECT_EQ(OK, surface->dequeueBuffer(&buffer, &fence)); + EXPECT_EQ(OK, surface->queueBuffer(buffer, fence, &output)); + EXPECT_TRUE(output.bufferReplaced); +} + +TEST_F(SurfaceTest, QueueBufferOutput_TracksReplacements_Plural) { + sp<BufferItemConsumer> consumer = sp<BufferItemConsumer>::make(GRALLOC_USAGE_SW_READ_OFTEN); + ASSERT_EQ(OK, consumer->setMaxBufferCount(4)); + ASSERT_EQ(OK, consumer->setMaxAcquiredBufferCount(1)); + + sp<Surface> surface = consumer->getSurface(); + consumer->setName(String8("TRPTest")); + sp<StubSurfaceListener> listener = sp<StubSurfaceListener>::make(); + + // Async mode sets up an extra buffer so the surface can queue it without waiting. + ASSERT_EQ(OK, surface->setMaxDequeuedBufferCount(2)); + ASSERT_EQ(OK, surface->setAsyncMode(true)); + ASSERT_EQ(OK, surface->connect(NATIVE_WINDOW_API_CPU, listener)); + + // dequeueBuffers requires a vector of a certain size: + std::vector<Surface::BatchBuffer> buffers(2); + std::vector<Surface::BatchQueuedBuffer> queuedBuffers; + std::vector<SurfaceQueueBufferOutput> outputs; + BufferItem item; + + auto moveBuffersToQueuedBuffers = [&]() { + EXPECT_EQ(2u, buffers.size()); + EXPECT_NE(nullptr, buffers[0].buffer); + EXPECT_NE(nullptr, buffers[1].buffer); + + queuedBuffers.clear(); + for (auto& buffer : buffers) { + auto& queuedBuffer = queuedBuffers.emplace_back(); + queuedBuffer.buffer = buffer.buffer; + queuedBuffer.fenceFd = buffer.fenceFd; + queuedBuffer.timestamp = NATIVE_WINDOW_TIMESTAMP_AUTO; + } + buffers = {{}, {}}; + }; + + // We can queue directly, without an output arg. + EXPECT_EQ(OK, surface->dequeueBuffers(&buffers)); + moveBuffersToQueuedBuffers(); + EXPECT_EQ(OK, surface->queueBuffers(queuedBuffers)); + EXPECT_EQ(OK, consumer->acquireBuffer(&item, 0)); + EXPECT_EQ(OK, consumer->releaseBuffer(item)); + + // We can queue with an output arg. Only the second one should be replaced. + EXPECT_EQ(OK, surface->dequeueBuffers(&buffers)); + moveBuffersToQueuedBuffers(); + EXPECT_EQ(OK, surface->queueBuffers(queuedBuffers, &outputs)); + EXPECT_EQ(2u, outputs.size()); + EXPECT_FALSE(outputs[0].bufferReplaced); + EXPECT_TRUE(outputs[1].bufferReplaced); + + // Since we haven't acquired anything, both queued buffers will replace the original one. + EXPECT_EQ(OK, surface->dequeueBuffers(&buffers)); + moveBuffersToQueuedBuffers(); + EXPECT_EQ(OK, surface->queueBuffers(queuedBuffers, &outputs)); + EXPECT_EQ(2u, outputs.size()); + EXPECT_TRUE(outputs[0].bufferReplaced); + EXPECT_TRUE(outputs[1].bufferReplaced); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_PLATFORM_API_IMPROVEMENTS) + } // namespace android diff --git a/libs/gui/tests/TestServer_test.cpp b/libs/gui/tests/TestServer_test.cpp new file mode 100644 index 0000000000..d6407820ff --- /dev/null +++ b/libs/gui/tests/TestServer_test.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include <SurfaceFlingerProperties.h> +#include <android/gui/IDisplayEventConnection.h> +#include <android/gui/ISurfaceComposer.h> +#include <android/hardware/configstore/1.0/ISurfaceFlingerConfigs.h> +#include <android/hardware_buffer.h> +#include <binder/ProcessState.h> +#include <com_android_graphics_libgui_flags.h> +#include <configstore/Utils.h> +#include <gui/AidlUtil.h> +#include <gui/BufferItemConsumer.h> +#include <gui/BufferQueue.h> +#include <gui/CpuConsumer.h> +#include <gui/IConsumerListener.h> +#include <gui/IGraphicBufferConsumer.h> +#include <gui/IGraphicBufferProducer.h> +#include <gui/ISurfaceComposer.h> +#include <gui/Surface.h> +#include <gui/SurfaceComposerClient.h> +#include <gui/SyncScreenCaptureListener.h> +#include <private/gui/ComposerService.h> +#include <private/gui/ComposerServiceAIDL.h> +#include <sys/types.h> +#include <system/window.h> +#include <ui/BufferQueueDefs.h> +#include <ui/DisplayMode.h> +#include <ui/GraphicBuffer.h> +#include <ui/Rect.h> +#include <utils/Errors.h> +#include <utils/String8.h> + +#include <cstddef> +#include <limits> +#include <thread> + +#include "binder/IInterface.h" +#include "testserver/TestServerClient.h" + +namespace android { + +namespace { + +class TestServerTest : public ::testing::Test { +protected: + TestServerTest() { ProcessState::self()->startThreadPool(); } +}; + +} // namespace + +TEST_F(TestServerTest, Create) { + EXPECT_NE(nullptr, TestServerClient::Create()); +} + +TEST_F(TestServerTest, CreateProducer) { + sp<TestServerClient> client = TestServerClient::Create(); + EXPECT_NE(nullptr, client->CreateProducer()); +} + +TEST_F(TestServerTest, KillServer) { + class DeathWaiter : public IBinder::DeathRecipient { + public: + virtual void binderDied(const wp<IBinder>&) override { mPromise.set_value(true); } + std::future<bool> getFuture() { return mPromise.get_future(); } + + std::promise<bool> mPromise; + }; + + sp<TestServerClient> client = TestServerClient::Create(); + sp<IGraphicBufferProducer> producer = client->CreateProducer(); + EXPECT_NE(nullptr, producer); + + sp<DeathWaiter> deathWaiter = sp<DeathWaiter>::make(); + EXPECT_EQ(OK, IInterface::asBinder(producer)->linkToDeath(deathWaiter)); + + auto deathWaiterFuture = deathWaiter->getFuture(); + EXPECT_EQ(OK, client->Kill()); + EXPECT_EQ(nullptr, client->CreateProducer()); + + EXPECT_TRUE(deathWaiterFuture.get()); +} + +} // namespace android diff --git a/libs/gui/tests/testserver/TestServer.cpp b/libs/gui/tests/testserver/TestServer.cpp new file mode 100644 index 0000000000..cd8824e355 --- /dev/null +++ b/libs/gui/tests/testserver/TestServer.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "TestServer" + +#include <android-base/stringprintf.h> +#include <binder/IInterface.h> +#include <binder/IPCThreadState.h> +#include <binder/IServiceManager.h> +#include <binder/ProcessState.h> +#include <binder/Status.h> +#include <gui/BufferQueue.h> +#include <gui/IConsumerListener.h> +#include <gui/IGraphicBufferConsumer.h> +#include <gui/IGraphicBufferProducer.h> +#include <gui/view/Surface.h> +#include <libgui_test_server/BnTestServer.h> +#include <log/log.h> +#include <utils/Errors.h> + +#include <cstdint> +#include <cstdlib> +#include <memory> +#include <mutex> +#include <vector> + +#include <fcntl.h> +#include <unistd.h> + +#include "TestServer.h" + +namespace android { + +namespace { +class TestConsumerListener : public BnConsumerListener { + virtual void onFrameAvailable(const BufferItem&) override {} + virtual void onBuffersReleased() override {} + virtual void onSidebandStreamChanged() override {} +}; + +class TestServiceImpl : public libgui_test_server::BnTestServer { +public: + TestServiceImpl(const char* name) : mName(name) {} + + virtual binder::Status createProducer(view::Surface* out) override { + std::lock_guard<std::mutex> lock(mMutex); + + BufferQueueHolder bq; + BufferQueue::createBufferQueue(&bq.producer, &bq.consumer); + sp<TestConsumerListener> listener = sp<TestConsumerListener>::make(); + bq.consumer->consumerConnect(listener, /*controlledByApp*/ true); + + uint64_t id = 0; + bq.producer->getUniqueId(&id); + std::string name = base::StringPrintf("%s-%" PRIu64, mName, id); + + out->name = String16(name.c_str()); + out->graphicBufferProducer = bq.producer; + mBqs.push_back(std::move(bq)); + + return binder::Status::ok(); + } + + virtual binder::Status killNow() override { + ALOGE("LibGUI Test Service %s dying in response to killNow", mName); + _exit(0); + // Not reached: + return binder::Status::ok(); + } + +private: + std::mutex mMutex; + const char* mName; + + struct BufferQueueHolder { + sp<IGraphicBufferProducer> producer; + sp<IGraphicBufferConsumer> consumer; + }; + + std::vector<BufferQueueHolder> mBqs; +}; +} // namespace + +int TestServerMain(const char* name) { + ProcessState::self()->startThreadPool(); + + sp<TestServiceImpl> testService = sp<TestServiceImpl>::make(name); + ALOGE("service"); + sp<IServiceManager> serviceManager(defaultServiceManager()); + LOG_ALWAYS_FATAL_IF(OK != serviceManager->addService(String16(name), testService)); + + ALOGD("LibGUI Test Service %s STARTED", name); + + IPCThreadState::self()->joinThreadPool(); + + ALOGW("LibGUI Test Service %s DIED", name); + + return 0; +} + +} // namespace android
\ No newline at end of file diff --git a/libs/gui/tests/testserver/TestServer.h b/libs/gui/tests/testserver/TestServer.h new file mode 100644 index 0000000000..4226f1bb09 --- /dev/null +++ b/libs/gui/tests/testserver/TestServer.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace android { + +/* + * Main method for a libgui ITestServer server. + * + * This must be called without any binder setup having been done, because you can't fork and do + * binder things once ProcessState is set up. + * @param name The service name of the test server to start. + * @return retcode + */ +int TestServerMain(const char* name); + +} // namespace android diff --git a/libs/gui/tests/testserver/TestServerClient.cpp b/libs/gui/tests/testserver/TestServerClient.cpp new file mode 100644 index 0000000000..e388074675 --- /dev/null +++ b/libs/gui/tests/testserver/TestServerClient.cpp @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <sys/wait.h> +#include <cerrno> +#define LOG_TAG "TestServerClient" + +#include <android-base/stringprintf.h> +#include <binder/IServiceManager.h> +#include <binder/ProcessState.h> +#include <libgui_test_server/ITestServer.h> +#include <log/log.h> +#include <utils/Errors.h> + +#include <errno.h> +#include <fcntl.h> +#include <signal.h> +#include <sys/types.h> +#include <unistd.h> + +#include <atomic> +#include <csignal> +#include <cstdlib> +#include <mutex> +#include <string> + +#include "TestServerClient.h" +#include "TestServerCommon.h" + +namespace android { + +namespace { + +std::string GetUniqueServiceName() { + static std::atomic<int> uniqueId = 1; + + pid_t pid = getpid(); + int id = uniqueId++; + return base::StringPrintf("Libgui-TestServer-%d-%d", pid, id); +} + +struct RemoteTestServerHostHolder { + RemoteTestServerHostHolder(pid_t pid, int sendFd, int recvFd) + : mPid(pid), mSendFd(sendFd), mRecvFd(recvFd) {} + ~RemoteTestServerHostHolder() { + std::lock_guard lock(mMutex); + + kill(mPid, SIGKILL); + close(mSendFd); + close(mRecvFd); + } + + pid_t CreateTestServerOrDie(std::string name) { + std::lock_guard lock(mMutex); + + CreateServerRequest request; + strlcpy(request.name, name.c_str(), sizeof(request.name) / sizeof(request.name[0])); + + ssize_t bytes = write(mSendFd, &request, sizeof(request)); + LOG_ALWAYS_FATAL_IF(bytes != sizeof(request)); + + CreateServerResponse response; + bytes = read(mRecvFd, &response, sizeof(response)); + LOG_ALWAYS_FATAL_IF(bytes != sizeof(response)); + + return response.pid; + } + +private: + std::mutex mMutex; + + pid_t mPid; + int mSendFd; + int mRecvFd; +}; + +std::unique_ptr<RemoteTestServerHostHolder> g_remoteTestServerHostHolder = nullptr; + +} // namespace + +void TestServerClient::InitializeOrDie(const char* filename) { + int sendPipeFds[2]; + int ret = pipe(sendPipeFds); + LOG_ALWAYS_FATAL_IF(ret, "Unable to create subprocess send pipe"); + + int recvPipeFds[2]; + ret = pipe(recvPipeFds); + LOG_ALWAYS_FATAL_IF(ret, "Unable to create subprocess recv pipe"); + + pid_t childPid = fork(); + LOG_ALWAYS_FATAL_IF(childPid < 0, "Unable to fork child process"); + + if (childPid == 0) { + // We forked! + close(sendPipeFds[1]); + close(recvPipeFds[0]); + + // We'll be reading from the parent's "send" and writing to the parent's "recv". + std::string sendPipe = std::to_string(sendPipeFds[0]); + std::string recvPipe = std::to_string(recvPipeFds[1]); + char* args[] = { + const_cast<char*>(filename), + const_cast<char*>("--test-server-host"), + const_cast<char*>(sendPipe.c_str()), + const_cast<char*>(recvPipe.c_str()), + nullptr, + }; + + ret = execv(filename, args); + ALOGE("Failed to exec libguiTestServer. ret=%d errno=%d (%s)", ret, errno, strerror(errno)); + status_t status = -errno; + write(recvPipeFds[1], &status, sizeof(status)); + _exit(EXIT_FAILURE); + } + + close(sendPipeFds[0]); + close(recvPipeFds[1]); + + // Check for an OK status that the host started. If so, we're good to go. + status_t status; + ret = read(recvPipeFds[0], &status, sizeof(status)); + LOG_ALWAYS_FATAL_IF(ret != sizeof(status), "Unable to read from pipe: %d", ret); + LOG_ALWAYS_FATAL_IF(OK != status, "Pipe returned failed status: %d", status); + + g_remoteTestServerHostHolder = + std::make_unique<RemoteTestServerHostHolder>(childPid, sendPipeFds[1], recvPipeFds[0]); +} + +sp<TestServerClient> TestServerClient::Create() { + std::string serviceName = GetUniqueServiceName(); + + pid_t childPid = g_remoteTestServerHostHolder->CreateTestServerOrDie(serviceName); + ALOGD("Created child server %s with pid %d", serviceName.c_str(), childPid); + + sp<libgui_test_server::ITestServer> server = + waitForService<libgui_test_server::ITestServer>(String16(serviceName.c_str())); + LOG_ALWAYS_FATAL_IF(server == nullptr); + ALOGD("Created connected to child server %s", serviceName.c_str()); + + return sp<TestServerClient>::make(server); +} + +TestServerClient::TestServerClient(const sp<libgui_test_server::ITestServer>& server) + : mServer(server) {} + +TestServerClient::~TestServerClient() { + Kill(); +} + +sp<IGraphicBufferProducer> TestServerClient::CreateProducer() { + std::lock_guard<std::mutex> lock(mMutex); + + if (!mIsAlive) { + return nullptr; + } + + view::Surface surface; + binder::Status status = mServer->createProducer(&surface); + + if (!status.isOk()) { + ALOGE("Failed to create remote producer. Error: %s", status.exceptionMessage().c_str()); + return nullptr; + } + + if (!surface.graphicBufferProducer) { + ALOGE("Remote producer returned no IGBP."); + return nullptr; + } + + return surface.graphicBufferProducer; +} + +status_t TestServerClient::Kill() { + std::lock_guard<std::mutex> lock(mMutex); + if (!mIsAlive) { + return DEAD_OBJECT; + } + + mServer->killNow(); + mServer = nullptr; + mIsAlive = false; + + return OK; +} + +} // namespace android diff --git a/libs/gui/tests/testserver/TestServerClient.h b/libs/gui/tests/testserver/TestServerClient.h new file mode 100644 index 0000000000..53296344a3 --- /dev/null +++ b/libs/gui/tests/testserver/TestServerClient.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <libgui_test_server/ITestServer.h> +#include <utils/RefBase.h> + +namespace android { + +class TestServerClient : public RefBase { +public: + static void InitializeOrDie(const char* filename); + static sp<TestServerClient> Create(); + + TestServerClient(const sp<libgui_test_server::ITestServer>& server); + virtual ~TestServerClient() override; + + sp<IGraphicBufferProducer> CreateProducer(); + status_t Kill(); + +private: + std::mutex mMutex; + + sp<libgui_test_server::ITestServer> mServer; + bool mIsAlive = true; +}; + +} // namespace android diff --git a/libs/gui/tests/testserver/TestServerCommon.h b/libs/gui/tests/testserver/TestServerCommon.h new file mode 100644 index 0000000000..7370f20ef8 --- /dev/null +++ b/libs/gui/tests/testserver/TestServerCommon.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <fcntl.h> + +namespace android { + +/* + * Test -> TestServerHost Request to create a new ITestServer fork. + */ +struct CreateServerRequest { + /* + * Service name for new ITestServer. + */ + char name[128]; +}; + +/* + * TestServerHost -> Test Response for creating an ITestServer fork. + */ +struct CreateServerResponse { + /* + * pid of new ITestServer. + */ + pid_t pid; +}; + +} // namespace android
\ No newline at end of file diff --git a/libs/gui/tests/testserver/TestServerHost.cpp b/libs/gui/tests/testserver/TestServerHost.cpp new file mode 100644 index 0000000000..696c3b9817 --- /dev/null +++ b/libs/gui/tests/testserver/TestServerHost.cpp @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "TestServerHost" + +#include <android-base/unique_fd.h> +#include <binder/IInterface.h> +#include <binder/IPCThreadState.h> +#include <binder/IServiceManager.h> +#include <binder/ProcessState.h> +#include <binder/Status.h> +#include <gui/BufferQueue.h> +#include <gui/IConsumerListener.h> +#include <gui/IGraphicBufferConsumer.h> +#include <gui/IGraphicBufferProducer.h> +#include <libgui_test_server/BnTestServer.h> +#include <log/log.h> +#include <utils/Errors.h> + +#include <memory> +#include <vector> + +#include <fcntl.h> +#include <unistd.h> +#include <cstddef> +#include <cstdlib> + +#include "TestServerCommon.h" +#include "TestServerHost.h" + +namespace android { + +namespace { + +pid_t ForkTestServer(const char* filename, char* name) { + pid_t childPid = fork(); + LOG_ALWAYS_FATAL_IF(childPid == -1); + + if (childPid != 0) { + return childPid; + } + + // We forked! + const char* test_server_flag = "--test-server"; + char* args[] = { + const_cast<char*>(filename), + const_cast<char*>(test_server_flag), + name, + nullptr, + }; + + int ret = execv(filename, args); + ALOGE("Failed to exec libgui_test as a TestServer. ret=%d errno=%d (%s)", ret, errno, + strerror(errno)); + _exit(EXIT_FAILURE); +} + +} // namespace + +int TestServerHostMain(const char* filename, base::unique_fd sendPipeFd, + base::unique_fd recvPipeFd) { + status_t status = OK; + LOG_ALWAYS_FATAL_IF(sizeof(status) != write(sendPipeFd.get(), &status, sizeof(status))); + + ALOGE("Launched TestServerHost"); + + while (true) { + CreateServerRequest request = {}; + ssize_t bytes = read(recvPipeFd.get(), &request, sizeof(request)); + LOG_ALWAYS_FATAL_IF(bytes != sizeof(request)); + pid_t childPid = ForkTestServer(filename, request.name); + + CreateServerResponse response = {}; + response.pid = childPid; + bytes = write(sendPipeFd.get(), &response, sizeof(response)); + LOG_ALWAYS_FATAL_IF(bytes != sizeof(response)); + } + + return 0; +} + +} // namespace android
\ No newline at end of file diff --git a/libs/gui/tests/testserver/TestServerHost.h b/libs/gui/tests/testserver/TestServerHost.h new file mode 100644 index 0000000000..df22c0c3fe --- /dev/null +++ b/libs/gui/tests/testserver/TestServerHost.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <android-base/unique_fd.h> + +#include <string> + +namespace android { + +/* + * Main method for a host process for TestServers. + * + * This must be called without any binder setup having been done, because you can't fork and do + * binder things once ProcessState is set up. + * @param filename File name of this binary / the binary to execve into + * @param sendPipeFd Pipe FD to send data to. + * @param recvPipeFd Pipe FD to receive data from. + * @return retcode + */ +int TestServerHostMain(const char* filename, base::unique_fd sendPipeFd, + base::unique_fd recvPipeFd); + +} // namespace android diff --git a/libs/gui/tests/testserver/aidl/libgui_test_server/ITestServer.aidl b/libs/gui/tests/testserver/aidl/libgui_test_server/ITestServer.aidl new file mode 100644 index 0000000000..c939ea00c1 --- /dev/null +++ b/libs/gui/tests/testserver/aidl/libgui_test_server/ITestServer.aidl @@ -0,0 +1,12 @@ +package libgui_test_server; + +import android.view.Surface; + +// Test server for libgui_test +interface ITestServer { + // Create a new producer. The server will have connected to the consumer. + Surface createProducer(); + + // Kills the server immediately. + void killNow(); +} diff --git a/libs/gui/view/Surface.cpp b/libs/gui/view/Surface.cpp index 7c15e7cf92..84c2a6ac71 100644 --- a/libs/gui/view/Surface.cpp +++ b/libs/gui/view/Surface.cpp @@ -121,5 +121,11 @@ String16 Surface::readMaybeEmptyString16(const Parcel* parcel) { return str.value_or(String16()); } +std::string Surface::toString() const { + std::stringstream out; + out << name; + return out.str(); +} + } // namespace view } // namespace android diff --git a/libs/input/Android.bp b/libs/input/Android.bp index d782f42071..e4e81adf58 100644 --- a/libs/input/Android.bp +++ b/libs/input/Android.bp @@ -30,6 +30,7 @@ filegroup { "android/os/InputEventInjectionResult.aidl", "android/os/InputEventInjectionSync.aidl", "android/os/InputConfig.aidl", + "android/os/MotionEventFlag.aidl", "android/os/PointerIconType.aidl", ], } @@ -231,6 +232,7 @@ cc_library { "MotionPredictorMetricsManager.cpp", "PrintTools.cpp", "PropertyMap.cpp", + "Resampler.cpp", "TfLiteMotionPredictor.cpp", "TouchVideoFrame.cpp", "VelocityControl.cpp", @@ -257,6 +259,7 @@ cc_library { ], shared_libs: [ + "android.companion.virtualdevice.flags-aconfig-cc", "libbase", "libbinder", "libbinder_ndk", diff --git a/libs/input/Input.cpp b/libs/input/Input.cpp index b09814797f..a2bb3453fe 100644 --- a/libs/input/Input.cpp +++ b/libs/input/Input.cpp @@ -580,7 +580,7 @@ void MotionEvent::initialize(int32_t id, int32_t deviceId, uint32_t source, &pointerProperties[pointerCount]); mSampleEventTimes.clear(); mSamplePointerCoords.clear(); - addSample(eventTime, pointerCoords); + addSample(eventTime, pointerCoords, mId); } void MotionEvent::copyFrom(const MotionEvent* other, bool keepHistory) { @@ -640,9 +640,9 @@ void MotionEvent::splitFrom(const android::MotionEvent& other, mSampleEventTimes = other.mSampleEventTimes; } -void MotionEvent::addSample( - int64_t eventTime, - const PointerCoords* pointerCoords) { +void MotionEvent::addSample(int64_t eventTime, const PointerCoords* pointerCoords, + int32_t eventId) { + mId = eventId; mSampleEventTimes.push_back(eventTime); mSamplePointerCoords.insert(mSamplePointerCoords.end(), &pointerCoords[0], &pointerCoords[getPointerCount()]); diff --git a/libs/input/InputConsumer.cpp b/libs/input/InputConsumer.cpp index fcf490d5f9..1eeb4e678c 100644 --- a/libs/input/InputConsumer.cpp +++ b/libs/input/InputConsumer.cpp @@ -135,7 +135,7 @@ void addSample(MotionEvent& event, const InputMessage& msg) { } event.setMetaState(event.getMetaState() | msg.body.motion.metaState); - event.addSample(msg.body.motion.eventTime, pointerCoords); + event.addSample(msg.body.motion.eventTime, pointerCoords, msg.body.motion.eventId); } void initializeTouchModeEvent(TouchModeEvent& event, const InputMessage& msg) { @@ -235,8 +235,9 @@ status_t InputConsumer::consume(InputEventFactoryInterface* factory, bool consum mMsgDeferred = false; } else { // Receive a fresh message. - status_t result = mChannel->receiveMessage(&mMsg); - if (result == OK) { + android::base::Result<InputMessage> result = mChannel->receiveMessage(); + if (result.ok()) { + mMsg = std::move(result.value()); const auto [_, inserted] = mConsumeTimes.emplace(mMsg.header.seq, systemTime(SYSTEM_TIME_MONOTONIC)); LOG_ALWAYS_FATAL_IF(!inserted, "Already have a consume time for seq=%" PRIu32, @@ -244,11 +245,11 @@ status_t InputConsumer::consume(InputEventFactoryInterface* factory, bool consum // Trace the event processing timeline - event was just read from the socket ATRACE_ASYNC_BEGIN(mProcessingTraceTag.c_str(), /*cookie=*/mMsg.header.seq); - } - if (result) { + } else { // Consume the next batched event unless batches are being held for later. - if (consumeBatches || result != WOULD_BLOCK) { - result = consumeBatch(factory, frameTime, outSeq, outEvent); + if (consumeBatches || result.error().code() != WOULD_BLOCK) { + result = android::base::Error( + consumeBatch(factory, frameTime, outSeq, outEvent)); if (*outEvent) { ALOGD_IF(DEBUG_TRANSPORT_CONSUMER, "channel '%s' consumer ~ consumed batch event, seq=%u", @@ -256,7 +257,7 @@ status_t InputConsumer::consume(InputEventFactoryInterface* factory, bool consum break; } } - return result; + return result.error().code(); } } @@ -696,7 +697,7 @@ void InputConsumer::resampleTouchState(nsecs_t sampleTime, MotionEvent* event, currentCoords.getY(), otherCoords.getX(), otherCoords.getY(), alpha); } - event->addSample(sampleTime, touchState.lastResample.pointers); + event->addSample(sampleTime, touchState.lastResample.pointers, event->getId()); } status_t InputConsumer::sendFinishedSignal(uint32_t seq, bool handled) { diff --git a/libs/input/InputConsumerNoResampling.cpp b/libs/input/InputConsumerNoResampling.cpp index 15d992f9f3..cdbc1869c3 100644 --- a/libs/input/InputConsumerNoResampling.cpp +++ b/libs/input/InputConsumerNoResampling.cpp @@ -14,9 +14,11 @@ * limitations under the License. */ -#define LOG_TAG "InputTransport" +#define LOG_TAG "InputConsumerNoResampling" #define ATRACE_TAG ATRACE_TAG_INPUT +#include <chrono> + #include <inttypes.h> #include <android-base/logging.h> @@ -31,12 +33,12 @@ #include <input/PrintTools.h> #include <input/TraceTools.h> -namespace input_flags = com::android::input::flags; - namespace android { namespace { +using std::chrono::nanoseconds; + /** * Log debug messages relating to the consumer end of the transport channel. * Enable this via "adb shell setprop log.tag.InputTransportConsumer DEBUG" (requires restart) @@ -114,7 +116,7 @@ void addSample(MotionEvent& event, const InputMessage& msg) { // TODO(b/329770983): figure out if it's safe to combine events with mismatching metaState event.setMetaState(event.getMetaState() | msg.body.motion.metaState); - event.addSample(msg.body.motion.eventTime, pointerCoords.data()); + event.addSample(msg.body.motion.eventTime, pointerCoords.data(), msg.body.motion.eventId); } std::unique_ptr<TouchModeEvent> createTouchModeEvent(const InputMessage& msg) { @@ -168,17 +170,24 @@ InputMessage createTimelineMessage(int32_t inputEventId, nsecs_t gpuCompletedTim return msg; } +bool isPointerEvent(const MotionEvent& motionEvent) { + return (motionEvent.getSource() & AINPUT_SOURCE_CLASS_POINTER) == AINPUT_SOURCE_CLASS_POINTER; +} } // namespace using android::base::Result; -using android::base::StringPrintf; // --- InputConsumerNoResampling --- InputConsumerNoResampling::InputConsumerNoResampling(const std::shared_ptr<InputChannel>& channel, sp<Looper> looper, - InputConsumerCallbacks& callbacks) - : mChannel(channel), mLooper(looper), mCallbacks(callbacks), mFdEvents(0) { + InputConsumerCallbacks& callbacks, + std::unique_ptr<Resampler> resampler) + : mChannel{channel}, + mLooper{looper}, + mCallbacks{callbacks}, + mResampler{std::move(resampler)}, + mFdEvents(0) { LOG_ALWAYS_FATAL_IF(mLooper == nullptr); mCallback = sp<LooperEventCallback>::make( std::bind(&InputConsumerNoResampling::handleReceiveCallback, this, @@ -190,7 +199,7 @@ InputConsumerNoResampling::InputConsumerNoResampling(const std::shared_ptr<Input InputConsumerNoResampling::~InputConsumerNoResampling() { ensureCalledOnLooperThread(__func__); - consumeBatchedInputEvents(std::nullopt); + consumeBatchedInputEvents(/*requestedFrameTime=*/std::nullopt); while (!mOutboundQueue.empty()) { processOutboundEvents(); // This is our last chance to ack the events. If we don't ack them here, we will get an ANR, @@ -216,8 +225,7 @@ int InputConsumerNoResampling::handleReceiveCallback(int events) { int handledEvents = 0; if (events & ALOOPER_EVENT_INPUT) { - std::vector<InputMessage> messages = readAllMessages(); - handleMessages(std::move(messages)); + handleMessages(readAllMessages()); handledEvents |= ALOOPER_EVENT_INPUT; } @@ -325,10 +333,8 @@ void InputConsumerNoResampling::handleMessages(std::vector<InputMessage>&& messa // add it to batch mBatches[deviceId].emplace(msg); } else { - // consume all pending batches for this event immediately - // TODO(b/329776327): figure out if this could be smarter by limiting the - // consumption only to the current device. - consumeBatchedInputEvents(std::nullopt); + // consume all pending batches for this device immediately + consumeBatchedInputEvents(deviceId, /*requestedFrameTime=*/std::nullopt); handleMessage(msg); } } else { @@ -362,36 +368,36 @@ void InputConsumerNoResampling::handleMessages(std::vector<InputMessage>&& messa std::vector<InputMessage> InputConsumerNoResampling::readAllMessages() { std::vector<InputMessage> messages; while (true) { - InputMessage msg; - status_t result = mChannel->receiveMessage(&msg); - switch (result) { - case OK: { - const auto [_, inserted] = - mConsumeTimes.emplace(msg.header.seq, systemTime(SYSTEM_TIME_MONOTONIC)); - LOG_ALWAYS_FATAL_IF(!inserted, "Already have a consume time for seq=%" PRIu32, - msg.header.seq); - - // Trace the event processing timeline - event was just read from the socket - // TODO(b/329777420): distinguish between multiple instances of InputConsumer - // in the same process. - ATRACE_ASYNC_BEGIN("InputConsumer processing", /*cookie=*/msg.header.seq); - messages.push_back(msg); - break; - } - case WOULD_BLOCK: { - return messages; - } - case DEAD_OBJECT: { - LOG(FATAL) << "Got a dead object for " << mChannel->getName(); - break; - } - case BAD_VALUE: { - LOG(FATAL) << "Got a bad value for " << mChannel->getName(); - break; - } - default: { - LOG(FATAL) << "Unexpected error: " << result; - break; + android::base::Result<InputMessage> result = mChannel->receiveMessage(); + if (result.ok()) { + const InputMessage& msg = *result; + const auto [_, inserted] = + mConsumeTimes.emplace(msg.header.seq, systemTime(SYSTEM_TIME_MONOTONIC)); + LOG_ALWAYS_FATAL_IF(!inserted, "Already have a consume time for seq=%" PRIu32, + msg.header.seq); + + // Trace the event processing timeline - event was just read from the socket + // TODO(b/329777420): distinguish between multiple instances of InputConsumer + // in the same process. + ATRACE_ASYNC_BEGIN("InputConsumer processing", /*cookie=*/msg.header.seq); + messages.push_back(msg); + } else { // !result.ok() + switch (result.error().code()) { + case WOULD_BLOCK: { + return messages; + } + case DEAD_OBJECT: { + LOG(FATAL) << "Got a dead object for " << mChannel->getName(); + break; + } + case BAD_VALUE: { + LOG(FATAL) << "Got a bad value for " << mChannel->getName(); + break; + } + default: { + LOG(FATAL) << "Unexpected error: " << result.error().message(); + break; + } } } } @@ -445,51 +451,81 @@ void InputConsumerNoResampling::handleMessage(const InputMessage& msg) const { } } +std::pair<std::unique_ptr<MotionEvent>, std::optional<uint32_t>> +InputConsumerNoResampling::createBatchedMotionEvent(const nsecs_t requestedFrameTime, + std::queue<InputMessage>& messages) { + std::unique_ptr<MotionEvent> motionEvent; + std::optional<uint32_t> firstSeqForBatch; + const nanoseconds resampleLatency = + (mResampler != nullptr) ? mResampler->getResampleLatency() : nanoseconds{0}; + const nanoseconds adjustedFrameTime = nanoseconds{requestedFrameTime} - resampleLatency; + + while (!messages.empty() && + (messages.front().body.motion.eventTime <= adjustedFrameTime.count())) { + if (motionEvent == nullptr) { + motionEvent = createMotionEvent(messages.front()); + firstSeqForBatch = messages.front().header.seq; + const auto [_, inserted] = mBatchedSequenceNumbers.insert({*firstSeqForBatch, {}}); + LOG_IF(FATAL, !inserted) + << "The sequence " << messages.front().header.seq << " was already present!"; + } else { + addSample(*motionEvent, messages.front()); + mBatchedSequenceNumbers[*firstSeqForBatch].push_back(messages.front().header.seq); + } + messages.pop(); + } + // Check if resampling should be performed. + if (motionEvent != nullptr && isPointerEvent(*motionEvent) && mResampler != nullptr) { + InputMessage* futureSample = nullptr; + if (!messages.empty()) { + futureSample = &messages.front(); + } + mResampler->resampleMotionEvent(nanoseconds{requestedFrameTime}, *motionEvent, + futureSample); + } + return std::make_pair(std::move(motionEvent), firstSeqForBatch); +} + bool InputConsumerNoResampling::consumeBatchedInputEvents( - std::optional<nsecs_t> requestedFrameTime) { + std::optional<DeviceId> deviceId, std::optional<nsecs_t> requestedFrameTime) { ensureCalledOnLooperThread(__func__); // When batching is not enabled, we want to consume all events. That's equivalent to having an - // infinite frameTime. - const nsecs_t frameTime = requestedFrameTime.value_or(std::numeric_limits<nsecs_t>::max()); + // infinite requestedFrameTime. + requestedFrameTime = requestedFrameTime.value_or(std::numeric_limits<nsecs_t>::max()); bool producedEvents = false; - for (auto& [deviceId, messages] : mBatches) { - std::unique_ptr<MotionEvent> motion; - std::optional<uint32_t> firstSeqForBatch; - std::vector<uint32_t> sequences; - while (!messages.empty()) { - const InputMessage& msg = messages.front(); - if (msg.body.motion.eventTime > frameTime) { - break; - } - if (motion == nullptr) { - motion = createMotionEvent(msg); - firstSeqForBatch = msg.header.seq; - const auto [_, inserted] = mBatchedSequenceNumbers.insert({*firstSeqForBatch, {}}); - if (!inserted) { - LOG(FATAL) << "The sequence " << msg.header.seq << " was already present!"; - } - } else { - addSample(*motion, msg); - mBatchedSequenceNumbers[*firstSeqForBatch].push_back(msg.header.seq); - } - messages.pop(); - } + + for (auto deviceIdIter = (deviceId.has_value()) ? (mBatches.find(*deviceId)) + : (mBatches.begin()); + deviceIdIter != mBatches.cend(); ++deviceIdIter) { + std::queue<InputMessage>& messages = deviceIdIter->second; + auto [motion, firstSeqForBatch] = createBatchedMotionEvent(*requestedFrameTime, messages); if (motion != nullptr) { LOG_ALWAYS_FATAL_IF(!firstSeqForBatch.has_value()); mCallbacks.onMotionEvent(std::move(motion), *firstSeqForBatch); producedEvents = true; } else { - // This is OK, it just means that the frameTime is too old (all events that we have - // pending are in the future of the frametime). Maybe print a - // warning? If there are multiple devices active though, this might be normal and can - // just be ignored, unless none of them resulted in any consumption (in that case, this - // function would already return "false" so we could just leave it up to the caller). + // This is OK, it just means that the requestedFrameTime is too old (all events that we + // have pending are in the future of the requestedFrameTime). Maybe print a warning? If + // there are multiple devices active though, this might be normal and can just be + // ignored, unless none of them resulted in any consumption (in that case, this function + // would already return "false" so we could just leave it up to the caller). + } + + if (deviceId.has_value()) { + // We already consumed events for this device. Break here to prevent iterating over the + // other devices. + break; } } std::erase_if(mBatches, [](const auto& pair) { return pair.second.empty(); }); return producedEvents; } +bool InputConsumerNoResampling::consumeBatchedInputEvents( + std::optional<nsecs_t> requestedFrameTime) { + return consumeBatchedInputEvents(/*deviceId=*/std::nullopt, requestedFrameTime); +} + void InputConsumerNoResampling::ensureCalledOnLooperThread(const char* func) const { sp<Looper> callingThreadLooper = Looper::getForThread(); if (callingThreadLooper != mLooper) { diff --git a/libs/input/InputDevice.cpp b/libs/input/InputDevice.cpp index 9333ab83a6..c9030312f9 100644 --- a/libs/input/InputDevice.cpp +++ b/libs/input/InputDevice.cpp @@ -20,6 +20,7 @@ #include <unistd.h> #include <ctype.h> +#include <android-base/logging.h> #include <android-base/properties.h> #include <android-base/stringprintf.h> #include <ftl/enum.h> @@ -31,6 +32,9 @@ using android::base::StringPrintf; namespace android { +// Set to true to log detailed debugging messages about IDC file probing. +static constexpr bool DEBUG_PROBE = false; + static const char* CONFIGURATION_FILE_DIR[] = { "idc/", "keylayout/", @@ -114,15 +118,18 @@ std::string getInputDeviceConfigurationFilePathByName( for (const auto& prefix : pathPrefixes) { path = prefix; appendInputDeviceConfigurationFileRelativePath(path, name, type); -#if DEBUG_PROBE - ALOGD("Probing for system provided input device configuration file: path='%s'", - path.c_str()); -#endif if (!access(path.c_str(), R_OK)) { -#if DEBUG_PROBE - ALOGD("Found"); -#endif + LOG_IF(INFO, DEBUG_PROBE) + << "Found system-provided input device configuration file at " << path; return path; + } else if (errno != ENOENT) { + LOG(WARNING) << "Couldn't find a system-provided input device configuration file at " + << path << " due to error " << errno << " (" << strerror(errno) + << "); there may be an IDC file there that cannot be loaded."; + } else { + LOG_IF(ERROR, DEBUG_PROBE) + << "Didn't find system-provided input device configuration file at " << path + << ": " << strerror(errno); } } @@ -135,21 +142,22 @@ std::string getInputDeviceConfigurationFilePathByName( } path += "/system/devices/"; appendInputDeviceConfigurationFileRelativePath(path, name, type); -#if DEBUG_PROBE - ALOGD("Probing for system user input device configuration file: path='%s'", path.c_str()); -#endif if (!access(path.c_str(), R_OK)) { -#if DEBUG_PROBE - ALOGD("Found"); -#endif + LOG_IF(INFO, DEBUG_PROBE) << "Found system user input device configuration file at " + << path; return path; + } else if (errno != ENOENT) { + LOG(WARNING) << "Couldn't find a system user input device configuration file at " << path + << " due to error " << errno << " (" << strerror(errno) + << "); there may be an IDC file there that cannot be loaded."; + } else { + LOG_IF(ERROR, DEBUG_PROBE) << "Didn't find system user input device configuration file at " + << path << ": " << strerror(errno); } // Not found. -#if DEBUG_PROBE - ALOGD("Probe failed to find input device configuration file: name='%s', type=%d", - name.c_str(), type); -#endif + LOG_IF(INFO, DEBUG_PROBE) << "Probe failed to find input device configuration file with name '" + << name << "' and type " << ftl::enum_string(type); return ""; } diff --git a/libs/input/InputTransport.cpp b/libs/input/InputTransport.cpp index 47b422857e..77dcaa9ef2 100644 --- a/libs/input/InputTransport.cpp +++ b/libs/input/InputTransport.cpp @@ -375,13 +375,11 @@ status_t InputChannel::openInputChannelPair(const std::string& name, sp<IBinder> token = sp<BBinder>::make(); - std::string serverChannelName = name + " (server)"; android::base::unique_fd serverFd(sockets[0]); - outServerChannel = InputChannel::create(serverChannelName, std::move(serverFd), token); + outServerChannel = InputChannel::create(name, std::move(serverFd), token); - std::string clientChannelName = name + " (client)"; android::base::unique_fd clientFd(sockets[1]); - outClientChannel = InputChannel::create(clientChannelName, std::move(clientFd), token); + outClientChannel = InputChannel::create(name, std::move(clientFd), token); return OK; } @@ -424,10 +422,11 @@ status_t InputChannel::sendMessage(const InputMessage* msg) { return OK; } -status_t InputChannel::receiveMessage(InputMessage* msg) { +android::base::Result<InputMessage> InputChannel::receiveMessage() { ssize_t nRead; + InputMessage msg; do { - nRead = ::recv(getFd(), msg, sizeof(InputMessage), MSG_DONTWAIT); + nRead = ::recv(getFd(), &msg, sizeof(InputMessage), MSG_DONTWAIT); } while (nRead == -1 && errno == EINTR); if (nRead < 0) { @@ -435,36 +434,36 @@ status_t InputChannel::receiveMessage(InputMessage* msg) { ALOGD_IF(DEBUG_CHANNEL_MESSAGES, "channel '%s' ~ receive message failed, errno=%d", name.c_str(), errno); if (error == EAGAIN || error == EWOULDBLOCK) { - return WOULD_BLOCK; + return android::base::Error(WOULD_BLOCK); } if (error == EPIPE || error == ENOTCONN || error == ECONNREFUSED) { - return DEAD_OBJECT; + return android::base::Error(DEAD_OBJECT); } - return -error; + return android::base::Error(-error); } if (nRead == 0) { // check for EOF ALOGD_IF(DEBUG_CHANNEL_MESSAGES, "channel '%s' ~ receive message failed because peer was closed", name.c_str()); - return DEAD_OBJECT; + return android::base::Error(DEAD_OBJECT); } - if (!msg->isValid(nRead)) { + if (!msg.isValid(nRead)) { ALOGE("channel '%s' ~ received invalid message of size %zd", name.c_str(), nRead); - return BAD_VALUE; + return android::base::Error(BAD_VALUE); } ALOGD_IF(DEBUG_CHANNEL_MESSAGES, "channel '%s' ~ received message of type %s", name.c_str(), - ftl::enum_string(msg->header.type).c_str()); + ftl::enum_string(msg.header.type).c_str()); if (ATRACE_ENABLED()) { // Add an additional trace point to include data about the received message. std::string message = StringPrintf("receiveMessage(inputChannel=%s, seq=0x%" PRIx32 ", type=%s)", - name.c_str(), msg->header.seq, - ftl::enum_string(msg->header.type).c_str()); + name.c_str(), msg.header.seq, + ftl::enum_string(msg.header.type).c_str()); ATRACE_NAME(message.c_str()); } - return OK; + return msg; } bool InputChannel::probablyHasInput() const { @@ -589,7 +588,8 @@ status_t InputPublisher::publishMotionEvent( mInputVerifier.processMovement(deviceId, source, action, pointerCount, pointerProperties, pointerCoords, flags); if (!result.ok()) { - LOG(FATAL) << "Bad stream: " << result.error(); + LOG(ERROR) << "Bad stream: " << result.error(); + return BAD_VALUE; } } if (debugTransportPublisher()) { @@ -729,15 +729,16 @@ status_t InputPublisher::publishTouchModeEvent(uint32_t seq, int32_t eventId, bo } android::base::Result<InputPublisher::ConsumerResponse> InputPublisher::receiveConsumerResponse() { - InputMessage msg; - status_t result = mChannel->receiveMessage(&msg); - if (result) { - if (debugTransportPublisher() && result != WOULD_BLOCK) { + android::base::Result<InputMessage> result = mChannel->receiveMessage(); + if (!result.ok()) { + if (debugTransportPublisher() && result.error().code() != WOULD_BLOCK) { LOG(INFO) << "channel '" << mChannel->getName() << "' publisher ~ " << __func__ << ": " - << strerror(result); + << result.error().message(); } - return android::base::Error(result); + return result.error(); } + + const InputMessage& msg = *result; if (msg.header.type == InputMessage::Type::FINISHED) { ALOGD_IF(debugTransportPublisher(), "channel '%s' publisher ~ %s: finished: seq=%u, handled=%s", diff --git a/libs/input/KeyCharacterMap.cpp b/libs/input/KeyCharacterMap.cpp index 1cf5612d45..b0563abaf7 100644 --- a/libs/input/KeyCharacterMap.cpp +++ b/libs/input/KeyCharacterMap.cpp @@ -317,19 +317,8 @@ bool KeyCharacterMap::getEvents(int32_t deviceId, const char16_t* chars, size_t return true; } -void KeyCharacterMap::addKeyRemapping(int32_t fromKeyCode, int32_t toKeyCode) { - if (fromKeyCode == toKeyCode) { - mKeyRemapping.erase(fromKeyCode); -#if DEBUG_MAPPING - ALOGD("addKeyRemapping: Cleared remapping forKeyCode=%d ~ Result Successful.", fromKeyCode); -#endif - return; - } - mKeyRemapping.insert_or_assign(fromKeyCode, toKeyCode); -#if DEBUG_MAPPING - ALOGD("addKeyRemapping: fromKeyCode=%d, toKeyCode=%d ~ Result Successful.", fromKeyCode, - toKeyCode); -#endif +void KeyCharacterMap::setKeyRemapping(const std::map<int32_t, int32_t>& keyRemapping) { + mKeyRemapping = keyRemapping; } status_t KeyCharacterMap::mapKey(int32_t scanCode, int32_t usageCode, int32_t* outKeyCode) const { diff --git a/libs/input/KeyLayoutMap.cpp b/libs/input/KeyLayoutMap.cpp index 508818852e..f3241c9c32 100644 --- a/libs/input/KeyLayoutMap.cpp +++ b/libs/input/KeyLayoutMap.cpp @@ -240,8 +240,9 @@ const KeyLayoutMap::Key* KeyLayoutMap::getKey(int32_t scanCode, int32_t usageCod std::vector<int32_t> KeyLayoutMap::findScanCodesForKey(int32_t keyCode) const { std::vector<int32_t> scanCodes; + // b/354333072: Only consider keys without FUNCTION flag for (const auto& [scanCode, key] : mKeysByScanCode) { - if (keyCode == key.keyCode) { + if (keyCode == key.keyCode && !(key.flags & POLICY_FLAG_FUNCTION)) { scanCodes.push_back(scanCode); } } diff --git a/libs/input/MotionPredictor.cpp b/libs/input/MotionPredictor.cpp index 5b61d3953f..c61d3943e0 100644 --- a/libs/input/MotionPredictor.cpp +++ b/libs/input/MotionPredictor.cpp @@ -72,9 +72,13 @@ float normalizeRange(float x, float min, float max) { // --- JerkTracker --- -JerkTracker::JerkTracker(bool normalizedDt) : mNormalizedDt(normalizedDt) {} +JerkTracker::JerkTracker(bool normalizedDt, float alpha) + : mNormalizedDt(normalizedDt), mAlpha(alpha) {} void JerkTracker::pushSample(int64_t timestamp, float xPos, float yPos) { + // If we previously had full samples, we have a previous jerk calculation + // to do weighted smoothing. + const bool applySmoothing = mTimestamps.size() == mTimestamps.capacity(); mTimestamps.pushBack(timestamp); const int numSamples = mTimestamps.size(); @@ -115,6 +119,16 @@ void JerkTracker::pushSample(int64_t timestamp, float xPos, float yPos) { } } + if (numSamples == static_cast<int>(mTimestamps.capacity())) { + float newJerkMagnitude = std::hypot(newXDerivatives[3], newYDerivatives[3]); + ALOGD_IF(isDebug(), "raw jerk: %f", newJerkMagnitude); + if (applySmoothing) { + mJerkMagnitude = mJerkMagnitude + (mAlpha * (newJerkMagnitude - mJerkMagnitude)); + } else { + mJerkMagnitude = newJerkMagnitude; + } + } + std::swap(newXDerivatives, mXDerivatives); std::swap(newYDerivatives, mYDerivatives); } @@ -125,7 +139,7 @@ void JerkTracker::reset() { std::optional<float> JerkTracker::jerkMagnitude() const { if (mTimestamps.size() == mTimestamps.capacity()) { - return std::hypot(mXDerivatives[3], mYDerivatives[3]); + return mJerkMagnitude; } return std::nullopt; } @@ -139,6 +153,24 @@ MotionPredictor::MotionPredictor(nsecs_t predictionTimestampOffsetNanos, mCheckMotionPredictionEnabled(std::move(checkMotionPredictionEnabled)), mReportAtomFunction(reportAtomFunction) {} +void MotionPredictor::initializeObjects() { + mModel = TfLiteMotionPredictorModel::create(); + LOG_ALWAYS_FATAL_IF(!mModel); + + // mJerkTracker assumes normalized dt = 1 between recorded samples because + // the underlying mModel input also assumes fixed-interval samples. + // Normalized dt as 1 is also used to correspond with the similar Jank + // implementation from the JetPack MotionPredictor implementation. + mJerkTracker = std::make_unique<JerkTracker>(/*normalizedDt=*/true, mModel->config().jerkAlpha); + + mBuffers = std::make_unique<TfLiteMotionPredictorBuffers>(mModel->inputLength()); + + mMetricsManager = + std::make_unique<MotionPredictorMetricsManager>(mModel->config().predictionInterval, + mModel->outputLength(), + mReportAtomFunction); +} + android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { if (mLastEvent && mLastEvent->getDeviceId() != event.getDeviceId()) { // We still have an active gesture for another device. The provided MotionEvent is not @@ -155,28 +187,18 @@ android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { return {}; } - // Initialise the model now that it's likely to be used. if (!mModel) { - mModel = TfLiteMotionPredictorModel::create(); - LOG_ALWAYS_FATAL_IF(!mModel); - } - - if (!mBuffers) { - mBuffers = std::make_unique<TfLiteMotionPredictorBuffers>(mModel->inputLength()); + initializeObjects(); } // Pass input event to the MetricsManager. - if (!mMetricsManager) { - mMetricsManager.emplace(mModel->config().predictionInterval, mModel->outputLength(), - mReportAtomFunction); - } mMetricsManager->onRecord(event); const int32_t action = event.getActionMasked(); if (action == AMOTION_EVENT_ACTION_UP || action == AMOTION_EVENT_ACTION_CANCEL) { ALOGD_IF(isDebug(), "End of event stream"); mBuffers->reset(); - mJerkTracker.reset(); + mJerkTracker->reset(); mLastEvent.reset(); return {}; } else if (action != AMOTION_EVENT_ACTION_DOWN && action != AMOTION_EVENT_ACTION_MOVE) { @@ -211,9 +233,9 @@ android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { 0, i), .orientation = event.getHistoricalOrientation(0, i), }); - mJerkTracker.pushSample(event.getHistoricalEventTime(i), - coords->getAxisValue(AMOTION_EVENT_AXIS_X), - coords->getAxisValue(AMOTION_EVENT_AXIS_Y)); + mJerkTracker->pushSample(event.getHistoricalEventTime(i), + coords->getAxisValue(AMOTION_EVENT_AXIS_X), + coords->getAxisValue(AMOTION_EVENT_AXIS_Y)); } if (!mLastEvent) { @@ -261,7 +283,7 @@ std::unique_ptr<MotionEvent> MotionPredictor::predict(nsecs_t timestamp) { int64_t predictionTime = mBuffers->lastTimestamp(); const int64_t futureTime = timestamp + mPredictionTimestampOffsetNanos; - const float jerkMagnitude = mJerkTracker.jerkMagnitude().value_or(0); + const float jerkMagnitude = mJerkTracker->jerkMagnitude().value_or(0); const float fractionKept = 1 - normalizeRange(jerkMagnitude, mModel->config().lowJerk, mModel->config().highJerk); // float to ensure proper division below. @@ -323,7 +345,7 @@ std::unique_ptr<MotionEvent> MotionPredictor::predict(nsecs_t timestamp) { event.getRawTransform(), event.getDownTime(), predictionTime, event.getPointerCount(), event.getPointerProperties(), &coords); } else { - prediction->addSample(predictionTime, &coords); + prediction->addSample(predictionTime, &coords, prediction->getId()); } axisFrom = axisTo; diff --git a/libs/input/Resampler.cpp b/libs/input/Resampler.cpp new file mode 100644 index 0000000000..51fadf8ec1 --- /dev/null +++ b/libs/input/Resampler.cpp @@ -0,0 +1,266 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "LegacyResampler" + +#include <algorithm> +#include <chrono> + +#include <android-base/logging.h> +#include <android-base/properties.h> +#include <ftl/enum.h> + +#include <input/Resampler.h> +#include <utils/Timers.h> + +using std::chrono::nanoseconds; + +namespace android { + +namespace { + +const bool IS_DEBUGGABLE_BUILD = +#if defined(__ANDROID__) + android::base::GetBoolProperty("ro.debuggable", false); +#else + true; +#endif + +bool debugResampling() { + if (!IS_DEBUGGABLE_BUILD) { + static const bool DEBUG_TRANSPORT_RESAMPLING = + __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", + ANDROID_LOG_INFO); + return DEBUG_TRANSPORT_RESAMPLING; + } + return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", ANDROID_LOG_INFO); +} + +constexpr std::chrono::milliseconds RESAMPLE_LATENCY{5}; + +constexpr std::chrono::milliseconds RESAMPLE_MIN_DELTA{2}; + +constexpr std::chrono::milliseconds RESAMPLE_MAX_DELTA{20}; + +constexpr std::chrono::milliseconds RESAMPLE_MAX_PREDICTION{8}; + +bool canResampleTool(ToolType toolType) { + return toolType == ToolType::FINGER || toolType == ToolType::MOUSE || + toolType == ToolType::STYLUS || toolType == ToolType::UNKNOWN; +} + +inline float lerp(float a, float b, float alpha) { + return a + alpha * (b - a); +} + +PointerCoords calculateResampledCoords(const PointerCoords& a, const PointerCoords& b, + float alpha) { + // We use the value of alpha to initialize resampledCoords with the latest sample information. + PointerCoords resampledCoords = (alpha < 1.0f) ? a : b; + resampledCoords.isResampled = true; + resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_X, lerp(a.getX(), b.getX(), alpha)); + resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, lerp(a.getY(), b.getY(), alpha)); + return resampledCoords; +} +} // namespace + +void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) { + const size_t numSamples = motionEvent.getHistorySize() + 1; + const size_t latestIndex = numSamples - 1; + const size_t secondToLatestIndex = (latestIndex > 0) ? (latestIndex - 1) : 0; + for (size_t sampleIndex = secondToLatestIndex; sampleIndex < numSamples; ++sampleIndex) { + std::vector<Pointer> pointers; + const size_t numPointers = motionEvent.getPointerCount(); + for (size_t pointerIndex = 0; pointerIndex < numPointers; ++pointerIndex) { + // getSamplePointerCoords is the vector representation of a getHistorySize by + // getPointerCount matrix. + const PointerCoords& pointerCoords = + motionEvent.getSamplePointerCoords()[sampleIndex * numPointers + pointerIndex]; + pointers.push_back( + Pointer{*motionEvent.getPointerProperties(pointerIndex), pointerCoords}); + } + mLatestSamples.pushBack( + Sample{nanoseconds{motionEvent.getHistoricalEventTime(sampleIndex)}, pointers}); + } +} + +LegacyResampler::Sample LegacyResampler::messageToSample(const InputMessage& message) { + std::vector<Pointer> pointers; + for (uint32_t i = 0; i < message.body.motion.pointerCount; ++i) { + pointers.push_back(Pointer{message.body.motion.pointers[i].properties, + message.body.motion.pointers[i].coords}); + } + return Sample{nanoseconds{message.body.motion.eventTime}, pointers}; +} + +bool LegacyResampler::pointerPropertiesResampleable(const Sample& target, const Sample& auxiliary) { + if (target.pointers.size() > auxiliary.pointers.size()) { + LOG_IF(INFO, debugResampling()) + << "Not resampled. Auxiliary sample has fewer pointers than target sample."; + return false; + } + for (size_t i = 0; i < target.pointers.size(); ++i) { + if (target.pointers[i].properties.id != auxiliary.pointers[i].properties.id) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Pointer ID mismatch."; + return false; + } + if (target.pointers[i].properties.toolType != auxiliary.pointers[i].properties.toolType) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Pointer ToolType mismatch."; + return false; + } + if (!canResampleTool(target.pointers[i].properties.toolType)) { + LOG_IF(INFO, debugResampling()) + << "Not resampled. Cannot resample " + << ftl::enum_string(target.pointers[i].properties.toolType) << " ToolType."; + return false; + } + } + return true; +} + +bool LegacyResampler::canInterpolate(const InputMessage& message) const { + LOG_IF(FATAL, mLatestSamples.empty()) + << "Not resampled. mLatestSamples must not be empty to interpolate."; + + const Sample& pastSample = *(mLatestSamples.end() - 1); + const Sample& futureSample = messageToSample(message); + + if (!pointerPropertiesResampleable(pastSample, futureSample)) { + return false; + } + + const nanoseconds delta = futureSample.eventTime - pastSample.eventTime; + if (delta < RESAMPLE_MIN_DELTA) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns."; + return false; + } + return true; +} + +std::optional<LegacyResampler::Sample> LegacyResampler::attemptInterpolation( + nanoseconds resampleTime, const InputMessage& futureSample) const { + if (!canInterpolate(futureSample)) { + return std::nullopt; + } + LOG_IF(FATAL, mLatestSamples.empty()) + << "Not resampled. mLatestSamples must not be empty to interpolate."; + + const Sample& pastSample = *(mLatestSamples.end() - 1); + + const nanoseconds delta = + nanoseconds{futureSample.body.motion.eventTime} - pastSample.eventTime; + const float alpha = + std::chrono::duration<float, std::milli>(resampleTime - pastSample.eventTime) / delta; + + std::vector<Pointer> resampledPointers; + for (size_t i = 0; i < pastSample.pointers.size(); ++i) { + const PointerCoords& resampledCoords = + calculateResampledCoords(pastSample.pointers[i].coords, + futureSample.body.motion.pointers[i].coords, alpha); + resampledPointers.push_back(Pointer{pastSample.pointers[i].properties, resampledCoords}); + } + return Sample{resampleTime, resampledPointers}; +} + +bool LegacyResampler::canExtrapolate() const { + if (mLatestSamples.size() < 2) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Not enough data."; + return false; + } + + const Sample& pastSample = *(mLatestSamples.end() - 2); + const Sample& presentSample = *(mLatestSamples.end() - 1); + + if (!pointerPropertiesResampleable(presentSample, pastSample)) { + return false; + } + + const nanoseconds delta = presentSample.eventTime - pastSample.eventTime; + if (delta < RESAMPLE_MIN_DELTA) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns."; + return false; + } else if (delta > RESAMPLE_MAX_DELTA) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too large: " << delta << "ns."; + return false; + } + return true; +} + +std::optional<LegacyResampler::Sample> LegacyResampler::attemptExtrapolation( + nanoseconds resampleTime) const { + if (!canExtrapolate()) { + return std::nullopt; + } + LOG_IF(FATAL, mLatestSamples.size() < 2) + << "Not resampled. mLatestSamples must have at least two samples to extrapolate."; + + const Sample& pastSample = *(mLatestSamples.end() - 2); + const Sample& presentSample = *(mLatestSamples.end() - 1); + + const nanoseconds delta = presentSample.eventTime - pastSample.eventTime; + // The farthest future time to which we can extrapolate. If the given resampleTime exceeds this, + // we use this value as the resample time target. + const nanoseconds farthestPrediction = + presentSample.eventTime + std::min<nanoseconds>(delta / 2, RESAMPLE_MAX_PREDICTION); + const nanoseconds newResampleTime = + (resampleTime > farthestPrediction) ? (farthestPrediction) : (resampleTime); + LOG_IF(INFO, debugResampling() && newResampleTime == farthestPrediction) + << "Resample time is too far in the future. Adjusting prediction from " + << (resampleTime - presentSample.eventTime) << " to " + << (farthestPrediction - presentSample.eventTime) << "ns."; + const float alpha = + std::chrono::duration<float, std::milli>(newResampleTime - pastSample.eventTime) / + delta; + + std::vector<Pointer> resampledPointers; + for (size_t i = 0; i < presentSample.pointers.size(); ++i) { + const PointerCoords& resampledCoords = + calculateResampledCoords(pastSample.pointers[i].coords, + presentSample.pointers[i].coords, alpha); + resampledPointers.push_back(Pointer{presentSample.pointers[i].properties, resampledCoords}); + } + return Sample{newResampleTime, resampledPointers}; +} + +inline void LegacyResampler::addSampleToMotionEvent(const Sample& sample, + MotionEvent& motionEvent) { + motionEvent.addSample(sample.eventTime.count(), sample.asPointerCoords().data(), + motionEvent.getId()); +} + +nanoseconds LegacyResampler::getResampleLatency() const { + return RESAMPLE_LATENCY; +} + +void LegacyResampler::resampleMotionEvent(nanoseconds frameTime, MotionEvent& motionEvent, + const InputMessage* futureSample) { + if (mPreviousDeviceId && *mPreviousDeviceId != motionEvent.getDeviceId()) { + mLatestSamples.clear(); + } + mPreviousDeviceId = motionEvent.getDeviceId(); + + const nanoseconds resampleTime = frameTime - RESAMPLE_LATENCY; + + updateLatestSamples(motionEvent); + + const std::optional<Sample> sample = (futureSample != nullptr) + ? (attemptInterpolation(resampleTime, *futureSample)) + : (attemptExtrapolation(resampleTime)); + if (sample.has_value()) { + addSampleToMotionEvent(*sample, motionEvent); + } +} +} // namespace android diff --git a/libs/input/TfLiteMotionPredictor.cpp b/libs/input/TfLiteMotionPredictor.cpp index b843a4bbf6..5250a9d2db 100644 --- a/libs/input/TfLiteMotionPredictor.cpp +++ b/libs/input/TfLiteMotionPredictor.cpp @@ -283,6 +283,7 @@ std::unique_ptr<TfLiteMotionPredictorModel> TfLiteMotionPredictorModel::create() .distanceNoiseFloor = parseXMLFloat(*configRoot, "distance-noise-floor"), .lowJerk = parseXMLFloat(*configRoot, "low-jerk"), .highJerk = parseXMLFloat(*configRoot, "high-jerk"), + .jerkAlpha = parseXMLFloat(*configRoot, "jerk-alpha"), }; return std::unique_ptr<TfLiteMotionPredictorModel>( diff --git a/libs/input/VirtualInputDevice.cpp b/libs/input/VirtualInputDevice.cpp index eea06f1720..51edbf1533 100644 --- a/libs/input/VirtualInputDevice.cpp +++ b/libs/input/VirtualInputDevice.cpp @@ -16,30 +16,267 @@ #define LOG_TAG "VirtualInputDevice" +#include <android-base/logging.h> #include <android/input.h> #include <android/keycodes.h> +#include <android_companion_virtualdevice_flags.h> #include <fcntl.h> #include <input/Input.h> #include <input/VirtualInputDevice.h> #include <linux/uinput.h> -#include <math.h> -#include <utils/Log.h> -#include <map> #include <string> using android::base::unique_fd; +namespace { + /** * Log debug messages about native virtual input devices. * Enable this via "adb shell setprop log.tag.VirtualInputDevice DEBUG" */ -static bool isDebug() { +bool isDebug() { return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG, ANDROID_LOG_INFO); } +unique_fd invalidFd() { + return unique_fd(-1); +} + +} // namespace + namespace android { +namespace vd_flags = android::companion::virtualdevice::flags; + +/** Creates a new uinput device and assigns a file descriptor. */ +unique_fd openUinput(const char* readableName, int32_t vendorId, int32_t productId, + const char* phys, DeviceType deviceType, int32_t screenHeight, + int32_t screenWidth) { + unique_fd fd(TEMP_FAILURE_RETRY(::open("/dev/uinput", O_WRONLY | O_NONBLOCK))); + if (fd < 0) { + ALOGE("Error creating uinput device: %s", strerror(errno)); + return invalidFd(); + } + + ioctl(fd, UI_SET_PHYS, phys); + + ioctl(fd, UI_SET_EVBIT, EV_KEY); + ioctl(fd, UI_SET_EVBIT, EV_SYN); + switch (deviceType) { + case DeviceType::DPAD: + for (const auto& [_, keyCode] : VirtualDpad::DPAD_KEY_CODE_MAPPING) { + ioctl(fd, UI_SET_KEYBIT, keyCode); + } + break; + case DeviceType::KEYBOARD: + for (const auto& [_, keyCode] : VirtualKeyboard::KEY_CODE_MAPPING) { + ioctl(fd, UI_SET_KEYBIT, keyCode); + } + break; + case DeviceType::MOUSE: + ioctl(fd, UI_SET_EVBIT, EV_REL); + ioctl(fd, UI_SET_KEYBIT, BTN_LEFT); + ioctl(fd, UI_SET_KEYBIT, BTN_RIGHT); + ioctl(fd, UI_SET_KEYBIT, BTN_MIDDLE); + ioctl(fd, UI_SET_KEYBIT, BTN_BACK); + ioctl(fd, UI_SET_KEYBIT, BTN_FORWARD); + ioctl(fd, UI_SET_RELBIT, REL_X); + ioctl(fd, UI_SET_RELBIT, REL_Y); + ioctl(fd, UI_SET_RELBIT, REL_WHEEL); + ioctl(fd, UI_SET_RELBIT, REL_HWHEEL); + if (vd_flags::high_resolution_scroll()) { + ioctl(fd, UI_SET_RELBIT, REL_WHEEL_HI_RES); + ioctl(fd, UI_SET_RELBIT, REL_HWHEEL_HI_RES); + } + break; + case DeviceType::TOUCHSCREEN: + ioctl(fd, UI_SET_EVBIT, EV_ABS); + ioctl(fd, UI_SET_KEYBIT, BTN_TOUCH); + ioctl(fd, UI_SET_ABSBIT, ABS_MT_SLOT); + ioctl(fd, UI_SET_ABSBIT, ABS_MT_POSITION_X); + ioctl(fd, UI_SET_ABSBIT, ABS_MT_POSITION_Y); + ioctl(fd, UI_SET_ABSBIT, ABS_MT_TRACKING_ID); + ioctl(fd, UI_SET_ABSBIT, ABS_MT_TOOL_TYPE); + ioctl(fd, UI_SET_ABSBIT, ABS_MT_TOUCH_MAJOR); + ioctl(fd, UI_SET_ABSBIT, ABS_MT_PRESSURE); + ioctl(fd, UI_SET_PROPBIT, INPUT_PROP_DIRECT); + break; + case DeviceType::STYLUS: + ioctl(fd, UI_SET_EVBIT, EV_ABS); + ioctl(fd, UI_SET_KEYBIT, BTN_TOUCH); + ioctl(fd, UI_SET_KEYBIT, BTN_STYLUS); + ioctl(fd, UI_SET_KEYBIT, BTN_STYLUS2); + ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_PEN); + ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_RUBBER); + ioctl(fd, UI_SET_ABSBIT, ABS_X); + ioctl(fd, UI_SET_ABSBIT, ABS_Y); + ioctl(fd, UI_SET_ABSBIT, ABS_TILT_X); + ioctl(fd, UI_SET_ABSBIT, ABS_TILT_Y); + ioctl(fd, UI_SET_ABSBIT, ABS_PRESSURE); + ioctl(fd, UI_SET_PROPBIT, INPUT_PROP_DIRECT); + break; + case DeviceType::ROTARY_ENCODER: + ioctl(fd, UI_SET_EVBIT, EV_REL); + ioctl(fd, UI_SET_RELBIT, REL_WHEEL); + if (vd_flags::high_resolution_scroll()) { + ioctl(fd, UI_SET_RELBIT, REL_WHEEL_HI_RES); + } + break; + default: + ALOGE("Invalid input device type %d", static_cast<int32_t>(deviceType)); + return invalidFd(); + } + + int version; + if (ioctl(fd, UI_GET_VERSION, &version) == 0 && version >= 5) { + uinput_setup setup; + memset(&setup, 0, sizeof(setup)); + std::strncpy(setup.name, readableName, UINPUT_MAX_NAME_SIZE); + setup.id.version = 1; + setup.id.bustype = BUS_VIRTUAL; + setup.id.vendor = vendorId; + setup.id.product = productId; + if (deviceType == DeviceType::TOUCHSCREEN) { + uinput_abs_setup xAbsSetup; + xAbsSetup.code = ABS_MT_POSITION_X; + xAbsSetup.absinfo.maximum = screenWidth - 1; + xAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &xAbsSetup) != 0) { + ALOGE("Error creating touchscreen uinput x axis: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup yAbsSetup; + yAbsSetup.code = ABS_MT_POSITION_Y; + yAbsSetup.absinfo.maximum = screenHeight - 1; + yAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &yAbsSetup) != 0) { + ALOGE("Error creating touchscreen uinput y axis: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup majorAbsSetup; + majorAbsSetup.code = ABS_MT_TOUCH_MAJOR; + majorAbsSetup.absinfo.maximum = screenWidth - 1; + majorAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &majorAbsSetup) != 0) { + ALOGE("Error creating touchscreen uinput major axis: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup pressureAbsSetup; + pressureAbsSetup.code = ABS_MT_PRESSURE; + pressureAbsSetup.absinfo.maximum = 255; + pressureAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &pressureAbsSetup) != 0) { + ALOGE("Error creating touchscreen uinput pressure axis: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup slotAbsSetup; + slotAbsSetup.code = ABS_MT_SLOT; + slotAbsSetup.absinfo.maximum = MAX_POINTERS - 1; + slotAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &slotAbsSetup) != 0) { + ALOGE("Error creating touchscreen uinput slots: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup trackingIdAbsSetup; + trackingIdAbsSetup.code = ABS_MT_TRACKING_ID; + trackingIdAbsSetup.absinfo.maximum = MAX_POINTERS - 1; + trackingIdAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &trackingIdAbsSetup) != 0) { + ALOGE("Error creating touchscreen uinput tracking ids: %s", strerror(errno)); + return invalidFd(); + } + } else if (deviceType == DeviceType::STYLUS) { + uinput_abs_setup xAbsSetup; + xAbsSetup.code = ABS_X; + xAbsSetup.absinfo.maximum = screenWidth - 1; + xAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &xAbsSetup) != 0) { + ALOGE("Error creating stylus uinput x axis: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup yAbsSetup; + yAbsSetup.code = ABS_Y; + yAbsSetup.absinfo.maximum = screenHeight - 1; + yAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &yAbsSetup) != 0) { + ALOGE("Error creating stylus uinput y axis: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup tiltXAbsSetup; + tiltXAbsSetup.code = ABS_TILT_X; + tiltXAbsSetup.absinfo.maximum = 90; + tiltXAbsSetup.absinfo.minimum = -90; + if (ioctl(fd, UI_ABS_SETUP, &tiltXAbsSetup) != 0) { + ALOGE("Error creating stylus uinput tilt x axis: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup tiltYAbsSetup; + tiltYAbsSetup.code = ABS_TILT_Y; + tiltYAbsSetup.absinfo.maximum = 90; + tiltYAbsSetup.absinfo.minimum = -90; + if (ioctl(fd, UI_ABS_SETUP, &tiltYAbsSetup) != 0) { + ALOGE("Error creating stylus uinput tilt y axis: %s", strerror(errno)); + return invalidFd(); + } + uinput_abs_setup pressureAbsSetup; + pressureAbsSetup.code = ABS_PRESSURE; + pressureAbsSetup.absinfo.maximum = 255; + pressureAbsSetup.absinfo.minimum = 0; + if (ioctl(fd, UI_ABS_SETUP, &pressureAbsSetup) != 0) { + ALOGE("Error creating touchscreen uinput pressure axis: %s", strerror(errno)); + return invalidFd(); + } + } + if (ioctl(fd, UI_DEV_SETUP, &setup) != 0) { + ALOGE("Error creating uinput device: %s", strerror(errno)); + return invalidFd(); + } + } else { + // UI_DEV_SETUP was not introduced until version 5. Try setting up manually. + ALOGI("Falling back to version %d manual setup", version); + uinput_user_dev fallback; + memset(&fallback, 0, sizeof(fallback)); + std::strncpy(fallback.name, readableName, UINPUT_MAX_NAME_SIZE); + fallback.id.version = 1; + fallback.id.bustype = BUS_VIRTUAL; + fallback.id.vendor = vendorId; + fallback.id.product = productId; + if (deviceType == DeviceType::TOUCHSCREEN) { + fallback.absmin[ABS_MT_POSITION_X] = 0; + fallback.absmax[ABS_MT_POSITION_X] = screenWidth - 1; + fallback.absmin[ABS_MT_POSITION_Y] = 0; + fallback.absmax[ABS_MT_POSITION_Y] = screenHeight - 1; + fallback.absmin[ABS_MT_TOUCH_MAJOR] = 0; + fallback.absmax[ABS_MT_TOUCH_MAJOR] = screenWidth - 1; + fallback.absmin[ABS_MT_PRESSURE] = 0; + fallback.absmax[ABS_MT_PRESSURE] = 255; + } else if (deviceType == DeviceType::STYLUS) { + fallback.absmin[ABS_X] = 0; + fallback.absmax[ABS_X] = screenWidth - 1; + fallback.absmin[ABS_Y] = 0; + fallback.absmax[ABS_Y] = screenHeight - 1; + fallback.absmin[ABS_TILT_X] = -90; + fallback.absmax[ABS_TILT_X] = 90; + fallback.absmin[ABS_TILT_Y] = -90; + fallback.absmax[ABS_TILT_Y] = 90; + fallback.absmin[ABS_PRESSURE] = 0; + fallback.absmax[ABS_PRESSURE] = 255; + } + if (TEMP_FAILURE_RETRY(write(fd, &fallback, sizeof(fallback))) != sizeof(fallback)) { + ALOGE("Error creating uinput device: %s", strerror(errno)); + return invalidFd(); + } + } + + if (ioctl(fd, UI_DEV_CREATE) != 0) { + ALOGE("Error creating uinput device: %s", strerror(errno)); + return invalidFd(); + } + + return fd; +} + VirtualInputDevice::VirtualInputDevice(unique_fd fd) : mFd(std::move(fd)) {} VirtualInputDevice::~VirtualInputDevice() { @@ -253,7 +490,10 @@ const std::map<int, int> VirtualMouse::BUTTON_CODE_MAPPING = { // clang-format on }; -VirtualMouse::VirtualMouse(unique_fd fd) : VirtualInputDevice(std::move(fd)) {} +VirtualMouse::VirtualMouse(unique_fd fd) + : VirtualInputDevice(std::move(fd)), + mAccumulatedHighResScrollX(0), + mAccumulatedHighResScrollY(0) {} VirtualMouse::~VirtualMouse() {} @@ -272,9 +512,47 @@ bool VirtualMouse::writeRelativeEvent(float relativeX, float relativeY, bool VirtualMouse::writeScrollEvent(float xAxisMovement, float yAxisMovement, std::chrono::nanoseconds eventTime) { - return writeInputEvent(EV_REL, REL_HWHEEL, xAxisMovement, eventTime) && - writeInputEvent(EV_REL, REL_WHEEL, yAxisMovement, eventTime) && - writeInputEvent(EV_SYN, SYN_REPORT, 0, eventTime); + if (!vd_flags::high_resolution_scroll()) { + return writeInputEvent(EV_REL, REL_HWHEEL, static_cast<int32_t>(xAxisMovement), + eventTime) && + writeInputEvent(EV_REL, REL_WHEEL, static_cast<int32_t>(yAxisMovement), + eventTime) && + writeInputEvent(EV_SYN, SYN_REPORT, 0, eventTime); + } + + const auto highResScrollX = + static_cast<int32_t>(xAxisMovement * kEvdevHighResScrollUnitsPerDetent); + const auto highResScrollY = + static_cast<int32_t>(yAxisMovement * kEvdevHighResScrollUnitsPerDetent); + bool highResScrollResult = + writeInputEvent(EV_REL, REL_HWHEEL_HI_RES, highResScrollX, eventTime) && + writeInputEvent(EV_REL, REL_WHEEL_HI_RES, highResScrollY, eventTime); + if (!highResScrollResult) { + return false; + } + + // According to evdev spec, a high-resolution mouse needs to emit REL_WHEEL / REL_HWHEEL events + // in addition to high-res scroll events. Regular scroll events can approximate high-res scroll + // events, so we send a regular scroll event when the accumulated scroll motion reaches a detent + // (single mouse wheel click). + mAccumulatedHighResScrollX += highResScrollX; + mAccumulatedHighResScrollY += highResScrollY; + const int32_t scrollX = mAccumulatedHighResScrollX / kEvdevHighResScrollUnitsPerDetent; + const int32_t scrollY = mAccumulatedHighResScrollY / kEvdevHighResScrollUnitsPerDetent; + if (scrollX != 0) { + if (!writeInputEvent(EV_REL, REL_HWHEEL, scrollX, eventTime)) { + return false; + } + mAccumulatedHighResScrollX %= kEvdevHighResScrollUnitsPerDetent; + } + if (scrollY != 0) { + if (!writeInputEvent(EV_REL, REL_WHEEL, scrollY, eventTime)) { + return false; + } + mAccumulatedHighResScrollY %= kEvdevHighResScrollUnitsPerDetent; + } + + return writeInputEvent(EV_SYN, SYN_REPORT, 0, eventTime); } // --- VirtualTouchscreen --- @@ -509,4 +787,39 @@ bool VirtualStylus::handleStylusUp(uint16_t tool, std::chrono::nanoseconds event return true; } +// --- VirtualRotaryEncoder --- +VirtualRotaryEncoder::VirtualRotaryEncoder(unique_fd fd) + : VirtualInputDevice(std::move(fd)), mAccumulatedHighResScrollAmount(0) {} + +VirtualRotaryEncoder::~VirtualRotaryEncoder() {} + +bool VirtualRotaryEncoder::writeScrollEvent(float scrollAmount, + std::chrono::nanoseconds eventTime) { + if (!vd_flags::high_resolution_scroll()) { + return writeInputEvent(EV_REL, REL_WHEEL, static_cast<int32_t>(scrollAmount), eventTime) && + writeInputEvent(EV_SYN, SYN_REPORT, 0, eventTime); + } + + const auto highResScrollAmount = + static_cast<int32_t>(scrollAmount * kEvdevHighResScrollUnitsPerDetent); + if (!writeInputEvent(EV_REL, REL_WHEEL_HI_RES, highResScrollAmount, eventTime)) { + return false; + } + + // According to evdev spec, a high-resolution scroll device needs to emit REL_WHEEL / REL_HWHEEL + // events in addition to high-res scroll events. Regular scroll events can approximate high-res + // scroll events, so we send a regular scroll event when the accumulated scroll motion reaches a + // detent (single wheel click). + mAccumulatedHighResScrollAmount += highResScrollAmount; + const int32_t scroll = mAccumulatedHighResScrollAmount / kEvdevHighResScrollUnitsPerDetent; + if (scroll != 0) { + if (!writeInputEvent(EV_REL, REL_WHEEL, scroll, eventTime)) { + return false; + } + mAccumulatedHighResScrollAmount %= kEvdevHighResScrollUnitsPerDetent; + } + + return writeInputEvent(EV_SYN, SYN_REPORT, 0, eventTime); +} + } // namespace android diff --git a/libs/input/android/os/IInputConstants.aidl b/libs/input/android/os/IInputConstants.aidl index a77dfa59fe..e23fc94c5e 100644 --- a/libs/input/android/os/IInputConstants.aidl +++ b/libs/input/android/os/IInputConstants.aidl @@ -49,130 +49,24 @@ interface IInputConstants const int POLICY_FLAG_INJECTED_FROM_ACCESSIBILITY = 0x20000; /** - * This flag indicates that the window that received this motion event is partly - * or wholly obscured by another visible window above it and the event directly passed through - * the obscured area. - * - * A security sensitive application can check this flag to identify situations in which - * a malicious application may have covered up part of its content for the purpose - * of misleading the user or hijacking touches. An appropriate response might be - * to drop the suspect touches or to take additional precautions to confirm the user's - * actual intent. - */ - const int MOTION_EVENT_FLAG_WINDOW_IS_OBSCURED = 0x1; - - /** - * This flag indicates that the window that received this motion event is partly - * or wholly obscured by another visible window above it and the event did not directly pass - * through the obscured area. - * - * A security sensitive application can check this flag to identify situations in which - * a malicious application may have covered up part of its content for the purpose - * of misleading the user or hijacking touches. An appropriate response might be - * to drop the suspect touches or to take additional precautions to confirm the user's - * actual intent. - * - * Unlike FLAG_WINDOW_IS_OBSCURED, this is only true if the window that received this event is - * obstructed in areas other than the touched location. - */ - const int MOTION_EVENT_FLAG_WINDOW_IS_PARTIALLY_OBSCURED = 0x2; - - /** - * This private flag is only set on {@link #ACTION_HOVER_MOVE} events and indicates that - * this event will be immediately followed by a {@link #ACTION_HOVER_EXIT}. It is used to - * prevent generating redundant {@link #ACTION_HOVER_ENTER} events. - * @hide - */ - const int MOTION_EVENT_FLAG_HOVER_EXIT_PENDING = 0x4; - - /** - * This flag indicates that the event has been generated by a gesture generator. It - * provides a hint to the GestureDetector to not apply any touch slop. - * - * @hide - */ - const int MOTION_EVENT_FLAG_IS_GENERATED_GESTURE = 0x8; - - /** - * This flag is only set for events with {@link #ACTION_POINTER_UP} and {@link #ACTION_CANCEL}. - * It indicates that the pointer going up was an unintentional user touch. When FLAG_CANCELED - * is set, the typical actions that occur in response for a pointer going up (such as click - * handlers, end of drawing) should be aborted. This flag is typically set when the user was - * accidentally touching the screen, such as by gripping the device, or placing the palm on the - * screen. - * - * @see #ACTION_POINTER_UP - * @see #ACTION_CANCEL + * Common input event flag used for both motion and key events for a gesture or pointer being + * canceled. */ const int INPUT_EVENT_FLAG_CANCELED = 0x20; /** - * This flag indicates that the event will not cause a focus change if it is directed to an - * unfocused window, even if it an {@link #ACTION_DOWN}. This is typically used with pointer - * gestures to allow the user to direct gestures to an unfocused window without bringing the - * window into focus. - * @hide - */ - const int MOTION_EVENT_FLAG_NO_FOCUS_CHANGE = 0x40; - - /** - * This flag indicates that the event has a valid value for AXIS_ORIENTATION. - * - * This is a private flag that is not used in Java. - * @hide - */ - const int MOTION_EVENT_PRIVATE_FLAG_SUPPORTS_ORIENTATION = 0x80; - - /** - * This flag indicates that the pointers' AXIS_ORIENTATION can be used to precisely determine - * the direction in which the tool is pointing. The value of the orientation axis will be in - * the range [-pi, pi], which represents a full circle. This is usually supported by devices - * like styluses. - * - * Conversely, AXIS_ORIENTATION cannot be used to tell which direction the tool is pointing - * when this flag is not set. In this case, the axis value will have a range of [-pi/2, pi/2], - * which represents half a circle. This is usually the case for devices like touchscreens and - * touchpads, for which it is difficult to tell which direction along the major axis of the - * touch ellipse the finger is pointing. - * - * This is a private flag that is not used in Java. - * @hide - */ - const int MOTION_EVENT_PRIVATE_FLAG_SUPPORTS_DIRECTIONAL_ORIENTATION = 0x100; - - /** - * The input event was generated or modified by accessibility service. - * Shared by both KeyEvent and MotionEvent flags, so this value should not overlap with either - * set of flags, including in input/Input.h and in android/input.h. + * Common input event flag used for both motion and key events, indicating that the event + * was generated or modified by accessibility service. */ const int INPUT_EVENT_FLAG_IS_ACCESSIBILITY_EVENT = 0x800; /** - * Private flag that indicates when the system has detected that this motion event - * may be inconsistent with respect to the sequence of previously delivered motion events, - * such as when a pointer move event is sent but the pointer is not down. - * - * @hide - * @see #isTainted - * @see #setTainted + * Common input event flag used for both motion and key events, indicating that the system has + * detected this event may be inconsistent with the current event sequence or gesture, such as + * when a pointer move event is sent but the pointer is not down. */ const int INPUT_EVENT_FLAG_TAINTED = 0x80000000; - /** - * Private flag indicating that this event was synthesized by the system and should be delivered - * to the accessibility focused view first. When being dispatched such an event is not handled - * by predecessors of the accessibility focused view and after the event reaches that view the - * flag is cleared and normal event dispatch is performed. This ensures that the platform can - * click on any view that has accessibility focus which is semantically equivalent to asking the - * view to perform a click accessibility action but more generic as views not implementing click - * action correctly can still be activated. - * - * @hide - * @see #isTargetAccessibilityFocus() - * @see #setTargetAccessibilityFocus(boolean) - */ - const int MOTION_EVENT_FLAG_TARGET_ACCESSIBILITY_FOCUS = 0x40000000; - /* The default pointer acceleration value. */ const int DEFAULT_POINTER_ACCELERATION = 3; diff --git a/libs/input/android/os/MotionEventFlag.aidl b/libs/input/android/os/MotionEventFlag.aidl new file mode 100644 index 0000000000..2093b0636a --- /dev/null +++ b/libs/input/android/os/MotionEventFlag.aidl @@ -0,0 +1,152 @@ +/** + * Copyright 2024, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.os; + +import android.os.IInputConstants; + +/** + * Flag definitions for MotionEvents. + * @hide + */ +@Backing(type="int") +enum MotionEventFlag { + + /** + * This flag indicates that the window that received this motion event is partly + * or wholly obscured by another visible window above it and the event directly passed through + * the obscured area. + * + * A security sensitive application can check this flag to identify situations in which + * a malicious application may have covered up part of its content for the purpose + * of misleading the user or hijacking touches. An appropriate response might be + * to drop the suspect touches or to take additional precautions to confirm the user's + * actual intent. + */ + WINDOW_IS_OBSCURED = 0x1, + + /** + * This flag indicates that the window that received this motion event is partly + * or wholly obscured by another visible window above it and the event did not directly pass + * through the obscured area. + * + * A security sensitive application can check this flag to identify situations in which + * a malicious application may have covered up part of its content for the purpose + * of misleading the user or hijacking touches. An appropriate response might be + * to drop the suspect touches or to take additional precautions to confirm the user's + * actual intent. + * + * Unlike FLAG_WINDOW_IS_OBSCURED, this is only true if the window that received this event is + * obstructed in areas other than the touched location. + */ + WINDOW_IS_PARTIALLY_OBSCURED = 0x2, + + /** + * This private flag is only set on {@link #ACTION_HOVER_MOVE} events and indicates that + * this event will be immediately followed by a {@link #ACTION_HOVER_EXIT}. It is used to + * prevent generating redundant {@link #ACTION_HOVER_ENTER} events. + * @hide + */ + HOVER_EXIT_PENDING = 0x4, + + /** + * This flag indicates that the event has been generated by a gesture generator. It + * provides a hint to the GestureDetector to not apply any touch slop. + * + * @hide + */ + IS_GENERATED_GESTURE = 0x8, + + /** + * This flag is only set for events with {@link #ACTION_POINTER_UP} and {@link #ACTION_CANCEL}. + * It indicates that the pointer going up was an unintentional user touch. When FLAG_CANCELED + * is set, the typical actions that occur in response for a pointer going up (such as click + * handlers, end of drawing) should be aborted. This flag is typically set when the user was + * accidentally touching the screen, such as by gripping the device, or placing the palm on the + * screen. + * + * @see #ACTION_POINTER_UP + * @see #ACTION_CANCEL + */ + CANCELED = IInputConstants.INPUT_EVENT_FLAG_CANCELED, + + /** + * This flag indicates that the event will not cause a focus change if it is directed to an + * unfocused window, even if it an {@link #ACTION_DOWN}. This is typically used with pointer + * gestures to allow the user to direct gestures to an unfocused window without bringing the + * window into focus. + * @hide + */ + NO_FOCUS_CHANGE = 0x40, + + /** + * This flag indicates that the event has a valid value for AXIS_ORIENTATION. + * + * This is a private flag that is not used in Java. + * @hide + */ + PRIVATE_FLAG_SUPPORTS_ORIENTATION = 0x80, + + /** + * This flag indicates that the pointers' AXIS_ORIENTATION can be used to precisely determine + * the direction in which the tool is pointing. The value of the orientation axis will be in + * the range [-pi, pi], which represents a full circle. This is usually supported by devices + * like styluses. + * + * Conversely, AXIS_ORIENTATION cannot be used to tell which direction the tool is pointing + * when this flag is not set. In this case, the axis value will have a range of [-pi/2, pi/2], + * which represents half a circle. This is usually the case for devices like touchscreens and + * touchpads, for which it is difficult to tell which direction along the major axis of the + * touch ellipse the finger is pointing. + * + * This is a private flag that is not used in Java. + * @hide + */ + PRIVATE_FLAG_SUPPORTS_DIRECTIONAL_ORIENTATION = 0x100, + + /** + * The input event was generated or modified by accessibility service. + * Shared by both KeyEvent and MotionEvent flags, so this value should not overlap with either + * set of flags, including in input/Input.h and in android/input.h. + */ + IS_ACCESSIBILITY_EVENT = IInputConstants.INPUT_EVENT_FLAG_IS_ACCESSIBILITY_EVENT, + + /** + * Private flag that indicates when the system has detected that this motion event + * may be inconsistent with respect to the sequence of previously delivered motion events, + * such as when a pointer move event is sent but the pointer is not down. + * + * @hide + * @see #isTainted + * @see #setTainted + */ + TAINTED = IInputConstants.INPUT_EVENT_FLAG_TAINTED, + + /** + * Private flag indicating that this event was synthesized by the system and should be delivered + * to the accessibility focused view first. When being dispatched such an event is not handled + * by predecessors of the accessibility focused view and after the event reaches that view the + * flag is cleared and normal event dispatch is performed. This ensures that the platform can + * click on any view that has accessibility focus which is semantically equivalent to asking the + * view to perform a click accessibility action but more generic as views not implementing click + * action correctly can still be activated. + * + * @hide + * @see #isTargetAccessibilityFocus() + * @see #setTargetAccessibilityFocus(boolean) + */ + TARGET_ACCESSIBILITY_FOCUS = 0x40000000, +} diff --git a/libs/input/input_flags.aconfig b/libs/input/input_flags.aconfig index a2192cbdc4..60fb00e128 100644 --- a/libs/input/input_flags.aconfig +++ b/libs/input/input_flags.aconfig @@ -16,13 +16,13 @@ flag { } flag { - name: "enable_gestures_library_timer_provider" + name: "remove_input_channel_from_windowstate" namespace: "input" - description: "Set to true to enable timer support for the touchpad Gestures library" - bug: "297192727" - } + description: "Do not store a copy of input channel inside WindowState." + bug: "323450804" +} - flag { +flag { name: "enable_input_event_tracing" namespace: "input" description: "Set to true to enable input event tracing, including always-on tracing on non-user builds" @@ -37,6 +37,13 @@ flag { } flag { + name: "split_all_touches" + namespace: "input" + description: "Set FLAG_SPLIT_TOUCHES to true for all windows, regardless of what they specify. This is essentially deprecating this flag by forcefully enabling the split functionality" + bug: "239934827" +} + +flag { name: "a11y_crash_on_inconsistent_event_stream" namespace: "accessibility" description: "Brings back fatal logging for inconsistent event streams originating from accessibility." @@ -87,13 +94,6 @@ flag { } flag { - name: "remove_pointer_event_tracking_in_wm" - namespace: "input" - description: "Remove pointer event tracking in WM after the Pointer Icon Refactor" - bug: "315321016" -} - -flag { name: "enable_new_mouse_pointer_ballistics" namespace: "input" description: "Change the acceleration curves for mouse pointer movements to match the touchpad ones" @@ -157,3 +157,53 @@ flag { description: "Keyboard classifier that classifies all keyboards into alphabetic or non-alphabetic" bug: "263559234" } + +flag { + name: "show_pointers_for_partial_screenshare" + namespace: "input" + description: "Show touch and pointer indicators when mirroring a single task" + bug: "310179437" +} + +flag { + name: "include_relative_axis_values_for_captured_touchpads" + namespace: "input" + description: "Include AXIS_RELATIVE_X and AXIS_RELATIVE_Y values when reporting touches from captured touchpads." + bug: "330522990" +} + +flag { + name: "enable_per_device_input_latency_metrics" + namespace: "input" + description: "Capture input latency metrics on a per device granular level using histograms." + bug: "270049345" +} + +flag { + name: "collect_palm_rejection_quality_metrics" + namespace: "input" + description: "Collect quality metrics on framework palm rejection." + bug: "341717757" +} + +flag { + name: "enable_touchpad_no_focus_change" + namespace: "input" + description: "Prevents touchpad gesture changing window focus." + bug: "364460018" +} + +flag { + name: "enable_input_policy_profile" + namespace: "input" + description: "Apply input policy profile for input threads." + bug: "347122505" + is_fixed_read_only: true +} + +flag { + name: "keyboard_repeat_keys" + namespace: "input" + description: "Allow user to enable key repeats or configure timeout before key repeat and key repeat delay rates." + bug: "336585002" +} diff --git a/libs/input/rust/Android.bp b/libs/input/rust/Android.bp index 018d199ce2..63853f77fa 100644 --- a/libs/input/rust/Android.bp +++ b/libs/input/rust/Android.bp @@ -24,6 +24,8 @@ rust_defaults { "liblogger", "liblog_rust", "inputconstants-rust", + "libserde", + "libserde_json", ], whole_static_libs: [ "libinput_from_rust_to_cpp", diff --git a/libs/input/rust/data_store.rs b/libs/input/rust/data_store.rs new file mode 100644 index 0000000000..6bdcefda36 --- /dev/null +++ b/libs/input/rust/data_store.rs @@ -0,0 +1,232 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Contains the DataStore, used to store input related data in a persistent way. + +use crate::input::KeyboardType; +use log::{debug, error}; +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::{Read, Write}; +use std::path::Path; +use std::sync::{Arc, RwLock}; + +/// Data store to be used to store information that persistent across device reboots. +pub struct DataStore { + file_reader_writer: Box<dyn FileReaderWriter>, + inner: Arc<RwLock<DataStoreInner>>, +} + +#[derive(Default)] +struct DataStoreInner { + is_loaded: bool, + data: Data, +} + +#[derive(Default, Serialize, Deserialize)] +struct Data { + // Map storing data for keyboard classification for specific devices. + #[serde(default)] + keyboard_classifications: Vec<KeyboardClassification>, + // NOTE: Important things to consider: + // - Add any data that needs to be persisted here in this struct. + // - Mark all new fields with "#[serde(default)]" for backward compatibility. + // - Also, you can't modify the already added fields. + // - Can add new nested fields to existing structs. e.g. Add another field to the struct + // KeyboardClassification and mark it "#[serde(default)]". +} + +#[derive(Default, Serialize, Deserialize)] +struct KeyboardClassification { + descriptor: String, + keyboard_type: KeyboardType, + is_finalized: bool, +} + +impl DataStore { + /// Creates a new instance of Data store + pub fn new(file_reader_writer: Box<dyn FileReaderWriter>) -> Self { + Self { file_reader_writer, inner: Default::default() } + } + + fn load(&mut self) { + if self.inner.read().unwrap().is_loaded { + return; + } + self.load_internal(); + } + + fn load_internal(&mut self) { + let s = self.file_reader_writer.read(); + let data: Data = if !s.is_empty() { + let deserialize: Data = match serde_json::from_str(&s) { + Ok(deserialize) => deserialize, + Err(msg) => { + error!("Unable to deserialize JSON data into struct: {:?} -> {:?}", msg, s); + Default::default() + } + }; + deserialize + } else { + Default::default() + }; + + let mut inner = self.inner.write().unwrap(); + inner.data = data; + inner.is_loaded = true; + } + + fn save(&mut self) { + let string_to_save; + { + let inner = self.inner.read().unwrap(); + string_to_save = serde_json::to_string(&inner.data).unwrap(); + } + self.file_reader_writer.write(string_to_save); + } + + /// Get keyboard type of the device (as stored in the data store) + pub fn get_keyboard_type(&mut self, descriptor: &String) -> Option<(KeyboardType, bool)> { + self.load(); + let data = &self.inner.read().unwrap().data; + for keyboard_classification in data.keyboard_classifications.iter() { + if keyboard_classification.descriptor == *descriptor { + return Some(( + keyboard_classification.keyboard_type, + keyboard_classification.is_finalized, + )); + } + } + None + } + + /// Save keyboard type of the device in the data store + pub fn set_keyboard_type( + &mut self, + descriptor: &String, + keyboard_type: KeyboardType, + is_finalized: bool, + ) { + { + let data = &mut self.inner.write().unwrap().data; + data.keyboard_classifications + .retain(|classification| classification.descriptor != *descriptor); + data.keyboard_classifications.push(KeyboardClassification { + descriptor: descriptor.to_string(), + keyboard_type, + is_finalized, + }) + } + self.save(); + } +} + +pub trait FileReaderWriter { + fn read(&self) -> String; + fn write(&self, to_write: String); +} + +/// Default file reader writer implementation +pub struct DefaultFileReaderWriter { + filepath: String, +} + +impl DefaultFileReaderWriter { + /// Creates a new instance of Default file reader writer that can read and write string to a + /// particular file in the filesystem + pub fn new(filepath: String) -> Self { + Self { filepath } + } +} + +impl FileReaderWriter for DefaultFileReaderWriter { + fn read(&self) -> String { + let path = Path::new(&self.filepath); + let mut fs_string = String::new(); + match File::open(path) { + Err(e) => error!("couldn't open {:?}: {}", path, e), + Ok(mut file) => match file.read_to_string(&mut fs_string) { + Err(e) => error!("Couldn't read from {:?}: {}", path, e), + Ok(_) => debug!("Successfully read from file {:?}", path), + }, + }; + fs_string + } + + fn write(&self, to_write: String) { + let path = Path::new(&self.filepath); + match File::create(path) { + Err(e) => error!("couldn't create {:?}: {}", path, e), + Ok(mut file) => match file.write_all(to_write.as_bytes()) { + Err(e) => error!("Couldn't write to {:?}: {}", path, e), + Ok(_) => debug!("Successfully saved to file {:?}", path), + }, + }; + } +} + +#[cfg(test)] +mod tests { + use crate::data_store::{ + test_file_reader_writer::TestFileReaderWriter, DataStore, FileReaderWriter, + }; + use crate::input::KeyboardType; + + #[test] + fn test_backward_compatibility_version_1() { + // This test tests JSON string that will be created by the first version of data store + // This test SHOULD NOT be modified + let test_reader_writer = TestFileReaderWriter::new(); + test_reader_writer.write(r#"{"keyboard_classifications":[{"descriptor":"descriptor","keyboard_type":{"type":"Alphabetic"},"is_finalized":true}]}"#.to_string()); + + let mut data_store = DataStore::new(Box::new(test_reader_writer)); + let (keyboard_type, is_finalized) = + data_store.get_keyboard_type(&"descriptor".to_string()).unwrap(); + assert_eq!(keyboard_type, KeyboardType::Alphabetic); + assert!(is_finalized); + } +} + +#[cfg(test)] +pub mod test_file_reader_writer { + + use crate::data_store::FileReaderWriter; + use std::sync::{Arc, RwLock}; + + #[derive(Default)] + struct TestFileReaderWriterInner { + fs_string: String, + } + + #[derive(Default, Clone)] + pub struct TestFileReaderWriter(Arc<RwLock<TestFileReaderWriterInner>>); + + impl TestFileReaderWriter { + pub fn new() -> Self { + Default::default() + } + } + + impl FileReaderWriter for TestFileReaderWriter { + fn read(&self) -> String { + self.0.read().unwrap().fs_string.clone() + } + + fn write(&self, fs_string: String) { + self.0.write().unwrap().fs_string = fs_string; + } + } +} diff --git a/libs/input/rust/input.rs b/libs/input/rust/input.rs index 564d94dbb4..90f509d97f 100644 --- a/libs/input/rust/input.rs +++ b/libs/input/rust/input.rs @@ -19,6 +19,8 @@ use crate::ffi::RustInputDeviceIdentifier; use bitflags::bitflags; use inputconstants::aidl::android::os::IInputConstants; +use inputconstants::aidl::android::os::MotionEventFlag::MotionEventFlag; +use serde::{Deserialize, Serialize}; use std::fmt; /// The InputDevice ID. @@ -193,31 +195,34 @@ impl MotionAction { bitflags! { /// MotionEvent flags. + /// The source of truth for the flag definitions are the MotionEventFlag AIDL enum. + /// The flag values are redefined here as a bitflags API. #[derive(Debug)] pub struct MotionFlags: u32 { /// FLAG_WINDOW_IS_OBSCURED - const WINDOW_IS_OBSCURED = IInputConstants::MOTION_EVENT_FLAG_WINDOW_IS_OBSCURED as u32; + const WINDOW_IS_OBSCURED = MotionEventFlag::WINDOW_IS_OBSCURED.0 as u32; /// FLAG_WINDOW_IS_PARTIALLY_OBSCURED - const WINDOW_IS_PARTIALLY_OBSCURED = IInputConstants::MOTION_EVENT_FLAG_WINDOW_IS_PARTIALLY_OBSCURED as u32; + const WINDOW_IS_PARTIALLY_OBSCURED = MotionEventFlag::WINDOW_IS_PARTIALLY_OBSCURED.0 as u32; /// FLAG_HOVER_EXIT_PENDING - const HOVER_EXIT_PENDING = IInputConstants::MOTION_EVENT_FLAG_HOVER_EXIT_PENDING as u32; + const HOVER_EXIT_PENDING = MotionEventFlag::HOVER_EXIT_PENDING.0 as u32; /// FLAG_IS_GENERATED_GESTURE - const IS_GENERATED_GESTURE = IInputConstants::MOTION_EVENT_FLAG_IS_GENERATED_GESTURE as u32; + const IS_GENERATED_GESTURE = MotionEventFlag::IS_GENERATED_GESTURE.0 as u32; /// FLAG_CANCELED - const CANCELED = IInputConstants::INPUT_EVENT_FLAG_CANCELED as u32; + const CANCELED = MotionEventFlag::CANCELED.0 as u32; /// FLAG_NO_FOCUS_CHANGE - const NO_FOCUS_CHANGE = IInputConstants::MOTION_EVENT_FLAG_NO_FOCUS_CHANGE as u32; + const NO_FOCUS_CHANGE = MotionEventFlag::NO_FOCUS_CHANGE.0 as u32; /// PRIVATE_FLAG_SUPPORTS_ORIENTATION - const PRIVATE_SUPPORTS_ORIENTATION = IInputConstants::MOTION_EVENT_PRIVATE_FLAG_SUPPORTS_ORIENTATION as u32; + const PRIVATE_FLAG_SUPPORTS_ORIENTATION = + MotionEventFlag::PRIVATE_FLAG_SUPPORTS_ORIENTATION.0 as u32; /// PRIVATE_FLAG_SUPPORTS_DIRECTIONAL_ORIENTATION - const PRIVATE_SUPPORTS_DIRECTIONAL_ORIENTATION = - IInputConstants::MOTION_EVENT_PRIVATE_FLAG_SUPPORTS_DIRECTIONAL_ORIENTATION as u32; + const PRIVATE_FLAG_SUPPORTS_DIRECTIONAL_ORIENTATION = + MotionEventFlag::PRIVATE_FLAG_SUPPORTS_DIRECTIONAL_ORIENTATION.0 as u32; /// FLAG_IS_ACCESSIBILITY_EVENT - const IS_ACCESSIBILITY_EVENT = IInputConstants::INPUT_EVENT_FLAG_IS_ACCESSIBILITY_EVENT as u32; + const IS_ACCESSIBILITY_EVENT = MotionEventFlag::IS_ACCESSIBILITY_EVENT.0 as u32; /// FLAG_TAINTED - const TAINTED = IInputConstants::INPUT_EVENT_FLAG_TAINTED as u32; + const TAINTED = MotionEventFlag::TAINTED.0 as u32; /// FLAG_TARGET_ACCESSIBILITY_FOCUS - const TARGET_ACCESSIBILITY_FOCUS = IInputConstants::MOTION_EVENT_FLAG_TARGET_ACCESSIBILITY_FOCUS as u32; + const TARGET_ACCESSIBILITY_FOCUS = MotionEventFlag::TARGET_ACCESSIBILITY_FOCUS.0 as u32; } } @@ -320,9 +325,11 @@ bitflags! { /// A rust enum representation of a Keyboard type. #[repr(u32)] -#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] pub enum KeyboardType { /// KEYBOARD_TYPE_NONE + #[default] None = input_bindgen::AINPUT_KEYBOARD_TYPE_NONE, /// KEYBOARD_TYPE_NON_ALPHABETIC NonAlphabetic = input_bindgen::AINPUT_KEYBOARD_TYPE_NON_ALPHABETIC, @@ -333,10 +340,24 @@ pub enum KeyboardType { #[cfg(test)] mod tests { use crate::input::SourceClass; + use crate::MotionFlags; use crate::Source; + use inputconstants::aidl::android::os::MotionEventFlag::MotionEventFlag; + #[test] fn convert_source_class_pointer() { let source = Source::from_bits(input_bindgen::AINPUT_SOURCE_CLASS_POINTER).unwrap(); assert!(source.is_from_class(SourceClass::Pointer)); } + + /// Ensure all MotionEventFlag constants are re-defined in rust. + #[test] + fn all_motion_event_flags_defined() { + for flag in MotionEventFlag::enum_values() { + assert!( + MotionFlags::from_bits(flag.0 as u32).is_some(), + "MotionEventFlag value {flag:?} is not redefined as a rust MotionFlags" + ); + } + } } diff --git a/libs/input/rust/keyboard_classification_config.rs b/libs/input/rust/keyboard_classification_config.rs new file mode 100644 index 0000000000..ab74efb674 --- /dev/null +++ b/libs/input/rust/keyboard_classification_config.rs @@ -0,0 +1,127 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::input::KeyboardType; + +// TODO(b/263559234): Categorize some of these to KeyboardType::None based on ability to produce +// key events at all. (Requires setup allowing InputDevice to dynamically add/remove +// KeyboardInputMapper based on blocklist and KeyEvents in case a KeyboardType::None device ends +// up producing a key event) +pub static CLASSIFIED_DEVICES: &[( + /* vendorId */ u16, + /* productId */ u16, + KeyboardType, + /* is_finalized */ bool, +)] = &[ + // HP X4000 Wireless Mouse + (0x03f0, 0xa407, KeyboardType::NonAlphabetic, true), + // Microsoft Wireless Mobile Mouse 6000 + (0x045e, 0x0745, KeyboardType::NonAlphabetic, true), + // Microsoft Surface Precision Mouse + (0x045e, 0x0821, KeyboardType::NonAlphabetic, true), + // Microsoft Pro IntelliMouse + (0x045e, 0x082a, KeyboardType::NonAlphabetic, true), + // Microsoft Bluetooth Mouse + (0x045e, 0x082f, KeyboardType::NonAlphabetic, true), + // Xbox One Elite Series 2 gamepad + (0x045e, 0x0b05, KeyboardType::NonAlphabetic, true), + // Logitech T400 + (0x046d, 0x4026, KeyboardType::NonAlphabetic, true), + // Logitech M720 Triathlon (Unifying) + (0x046d, 0x405e, KeyboardType::NonAlphabetic, true), + // Logitech MX Master 2S (Unifying) + (0x046d, 0x4069, KeyboardType::NonAlphabetic, true), + // Logitech M585 (Unifying) + (0x046d, 0x406b, KeyboardType::NonAlphabetic, true), + // Logitech MX Anywhere 2 (Unifying) + (0x046d, 0x4072, KeyboardType::NonAlphabetic, true), + // Logitech Pebble M350 + (0x046d, 0x4080, KeyboardType::NonAlphabetic, true), + // Logitech T630 Ultrathin + (0x046d, 0xb00d, KeyboardType::NonAlphabetic, true), + // Logitech M558 + (0x046d, 0xb011, KeyboardType::NonAlphabetic, true), + // Logitech MX Master (Bluetooth) + (0x046d, 0xb012, KeyboardType::NonAlphabetic, true), + // Logitech MX Anywhere 2 (Bluetooth) + (0x046d, 0xb013, KeyboardType::NonAlphabetic, true), + // Logitech M720 Triathlon (Bluetooth) + (0x046d, 0xb015, KeyboardType::NonAlphabetic, true), + // Logitech M535 + (0x046d, 0xb016, KeyboardType::NonAlphabetic, true), + // Logitech MX Master / Anywhere 2 (Bluetooth) + (0x046d, 0xb017, KeyboardType::NonAlphabetic, true), + // Logitech MX Master 2S (Bluetooth) + (0x046d, 0xb019, KeyboardType::NonAlphabetic, true), + // Logitech MX Anywhere 2S (Bluetooth) + (0x046d, 0xb01a, KeyboardType::NonAlphabetic, true), + // Logitech M585/M590 (Bluetooth) + (0x046d, 0xb01b, KeyboardType::NonAlphabetic, true), + // Logitech G603 Lightspeed Gaming Mouse (Bluetooth) + (0x046d, 0xb01c, KeyboardType::NonAlphabetic, true), + // Logitech MX Master (Bluetooth) + (0x046d, 0xb01e, KeyboardType::NonAlphabetic, true), + // Logitech MX Anywhere 2 (Bluetooth) + (0x046d, 0xb01f, KeyboardType::NonAlphabetic, true), + // Logitech MX Master 3 (Bluetooth) + (0x046d, 0xb023, KeyboardType::NonAlphabetic, true), + // Logitech G604 Lightspeed Gaming Mouse (Bluetooth) + (0x046d, 0xb024, KeyboardType::NonAlphabetic, true), + // Logitech Spotlight Presentation Remote (Bluetooth) + (0x046d, 0xb503, KeyboardType::NonAlphabetic, true), + // Logitech R500 (Bluetooth) + (0x046d, 0xb505, KeyboardType::NonAlphabetic, true), + // Logitech M500s + (0x046d, 0xc093, KeyboardType::NonAlphabetic, true), + // Logitech Spotlight Presentation Remote (USB dongle) + (0x046d, 0xc53e, KeyboardType::NonAlphabetic, true), + // Elecom Enelo IR LED Mouse 350 + (0x056e, 0x0134, KeyboardType::NonAlphabetic, true), + // Elecom EPRIM Blue LED 5 button mouse 228 + (0x056e, 0x0141, KeyboardType::NonAlphabetic, true), + // Elecom Blue LED Mouse 203 + (0x056e, 0x0159, KeyboardType::NonAlphabetic, true), + // Zebra LS2208 barcode scanner + (0x05e0, 0x1200, KeyboardType::NonAlphabetic, true), + // RDing FootSwitch1F1 + (0x0c45, 0x7403, KeyboardType::NonAlphabetic, true), + // SteelSeries Sensei RAW Frost Blue + (0x1038, 0x1369, KeyboardType::NonAlphabetic, true), + // SteelSeries Rival 3 Wired + (0x1038, 0x1824, KeyboardType::NonAlphabetic, true), + // SteelSeries Rival 3 Wireless (USB dongle) + (0x1038, 0x1830, KeyboardType::NonAlphabetic, true), + // Yubico.com Yubikey + (0x1050, 0x0010, KeyboardType::NonAlphabetic, true), + // Yubico.com Yubikey 4 OTP+U2F+CCID + (0x1050, 0x0407, KeyboardType::NonAlphabetic, true), + // Lenovo USB-C Wired Compact Mouse + (0x17ef, 0x6123, KeyboardType::NonAlphabetic, true), + // Corsair Katar Pro Wireless (USB dongle) + (0x1b1c, 0x1b94, KeyboardType::NonAlphabetic, true), + // Corsair Katar Pro Wireless (Bluetooth) + (0x1bae, 0x1b1c, KeyboardType::NonAlphabetic, true), + // Kensington Pro Fit Full-size + (0x1bcf, 0x08a0, KeyboardType::NonAlphabetic, true), + // Huion HS64 + (0x256c, 0x006d, KeyboardType::NonAlphabetic, true), + // XP-Pen Star G640 + (0x28bd, 0x0914, KeyboardType::NonAlphabetic, true), + // XP-Pen Artist 12 Pro + (0x28bd, 0x091f, KeyboardType::NonAlphabetic, true), + // XP-Pen Deco mini7W + (0x28bd, 0x0928, KeyboardType::NonAlphabetic, true), +]; diff --git a/libs/input/rust/keyboard_classifier.rs b/libs/input/rust/keyboard_classifier.rs index 1063fac664..3c789b41e2 100644 --- a/libs/input/rust/keyboard_classifier.rs +++ b/libs/input/rust/keyboard_classifier.rs @@ -31,39 +31,37 @@ //! across multiple device connections in a time period, then change type to //! KeyboardType::NonAlphabetic. Once changed, it can still change back to Alphabetic //! (i.e. verified = false). -//! -//! TODO(b/263559234): Data store implementation to store information about past classification +use crate::data_store::DataStore; use crate::input::{DeviceId, InputDevice, KeyboardType}; +use crate::keyboard_classification_config::CLASSIFIED_DEVICES; use crate::{DeviceClass, ModifierState}; use std::collections::HashMap; /// The KeyboardClassifier is used to classify a keyboard device into non-keyboard, alphabetic /// keyboard or non-alphabetic keyboard -#[derive(Default)] pub struct KeyboardClassifier { device_map: HashMap<DeviceId, KeyboardInfo>, + data_store: DataStore, } struct KeyboardInfo { - _device: InputDevice, + device: InputDevice, keyboard_type: KeyboardType, is_finalized: bool, } impl KeyboardClassifier { /// Create a new KeyboardClassifier - pub fn new() -> Self { - Default::default() + pub fn new(data_store: DataStore) -> Self { + Self { device_map: HashMap::new(), data_store } } /// Adds keyboard to KeyboardClassifier pub fn notify_keyboard_changed(&mut self, device: InputDevice) { let (keyboard_type, is_finalized) = self.classify_keyboard(&device); - self.device_map.insert( - device.device_id, - KeyboardInfo { _device: device, keyboard_type, is_finalized }, - ); + self.device_map + .insert(device.device_id, KeyboardInfo { device, keyboard_type, is_finalized }); } /// Get keyboard type for a tracked keyboard in KeyboardClassifier @@ -106,11 +104,16 @@ impl KeyboardClassifier { if Self::is_alphabetic_key(&evdev_code) { keyboard.keyboard_type = KeyboardType::Alphabetic; keyboard.is_finalized = true; + self.data_store.set_keyboard_type( + &keyboard.device.identifier.descriptor, + keyboard.keyboard_type, + keyboard.is_finalized, + ); } } } - fn classify_keyboard(&self, device: &InputDevice) -> (KeyboardType, bool) { + fn classify_keyboard(&mut self, device: &InputDevice) -> (KeyboardType, bool) { // This should never happen but having keyboard device class is necessary to be classified // as any type of keyboard. if !device.classes.contains(DeviceClass::Keyboard) { @@ -126,6 +129,21 @@ impl KeyboardClassifier { (KeyboardType::NonAlphabetic, true) }; } + + // Check in data store + if let Some((keyboard_type, is_finalized)) = + self.data_store.get_keyboard_type(&device.identifier.descriptor) + { + return (keyboard_type, is_finalized); + } + + // Check in known device list for classification + for (vendor, product, keyboard_type, is_finalized) in CLASSIFIED_DEVICES.iter() { + if device.identifier.vendor == *vendor && device.identifier.product == *product { + return (*keyboard_type, *is_finalized); + } + } + // Any composite device with multiple device classes should be categorized as non-alphabetic // keyboard initially if device.classes.contains(DeviceClass::Touch) @@ -168,17 +186,20 @@ impl KeyboardClassifier { #[cfg(test)] mod tests { + use crate::data_store::{test_file_reader_writer::TestFileReaderWriter, DataStore}; use crate::input::{DeviceId, InputDevice, KeyboardType}; + use crate::keyboard_classification_config::CLASSIFIED_DEVICES; use crate::keyboard_classifier::KeyboardClassifier; use crate::{DeviceClass, ModifierState, RustInputDeviceIdentifier}; static DEVICE_ID: DeviceId = DeviceId(1); + static SECOND_DEVICE_ID: DeviceId = DeviceId(2); static KEY_A: i32 = 30; static KEY_1: i32 = 2; #[test] fn classify_external_alphabetic_keyboard() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::AlphabeticKey | DeviceClass::External, )); @@ -188,7 +209,7 @@ mod tests { #[test] fn classify_external_non_alphabetic_keyboard() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier .notify_keyboard_changed(create_device(DeviceClass::Keyboard | DeviceClass::External)); assert_eq!(classifier.get_keyboard_type(DEVICE_ID), KeyboardType::NonAlphabetic); @@ -197,7 +218,7 @@ mod tests { #[test] fn classify_mouse_pretending_as_keyboard() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::Cursor @@ -210,7 +231,7 @@ mod tests { #[test] fn classify_touchpad_pretending_as_keyboard() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::Touchpad @@ -223,7 +244,7 @@ mod tests { #[test] fn classify_stylus_pretending_as_keyboard() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::ExternalStylus @@ -236,7 +257,7 @@ mod tests { #[test] fn classify_dpad_pretending_as_keyboard() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::Dpad @@ -249,7 +270,7 @@ mod tests { #[test] fn classify_joystick_pretending_as_keyboard() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::Joystick @@ -262,7 +283,7 @@ mod tests { #[test] fn classify_gamepad_pretending_as_keyboard() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::Gamepad @@ -275,7 +296,7 @@ mod tests { #[test] fn reclassify_keyboard_on_alphabetic_key_event() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::Dpad @@ -293,7 +314,7 @@ mod tests { #[test] fn dont_reclassify_keyboard_on_non_alphabetic_key_event() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::Dpad @@ -311,7 +332,7 @@ mod tests { #[test] fn dont_reclassify_keyboard_on_alphabetic_key_event_with_modifiers() { - let mut classifier = KeyboardClassifier::new(); + let mut classifier = create_classifier(); classifier.notify_keyboard_changed(create_device( DeviceClass::Keyboard | DeviceClass::Dpad @@ -326,20 +347,82 @@ mod tests { assert!(!classifier.is_finalized(DEVICE_ID)); } + #[test] + fn classify_known_devices() { + let mut classifier = create_classifier(); + for (vendor, product, keyboard_type, is_finalized) in CLASSIFIED_DEVICES.iter() { + classifier + .notify_keyboard_changed(create_device_with_vendor_product_ids(*vendor, *product)); + assert_eq!(classifier.get_keyboard_type(DEVICE_ID), *keyboard_type); + assert_eq!(classifier.is_finalized(DEVICE_ID), *is_finalized); + } + } + + #[test] + fn classify_previously_reclassified_devices() { + let test_reader_writer = TestFileReaderWriter::new(); + { + let mut classifier = + KeyboardClassifier::new(DataStore::new(Box::new(test_reader_writer.clone()))); + let device = create_device( + DeviceClass::Keyboard + | DeviceClass::Dpad + | DeviceClass::AlphabeticKey + | DeviceClass::External, + ); + classifier.notify_keyboard_changed(device); + classifier.process_key(DEVICE_ID, KEY_A, ModifierState::None); + } + + // Re-create classifier and data store to mimic a reboot (but use the same file system + // reader writer) + { + let mut classifier = + KeyboardClassifier::new(DataStore::new(Box::new(test_reader_writer.clone()))); + let device = InputDevice { + device_id: SECOND_DEVICE_ID, + identifier: create_identifier(/* vendor= */ 234, /* product= */ 345), + classes: DeviceClass::Keyboard + | DeviceClass::Dpad + | DeviceClass::AlphabeticKey + | DeviceClass::External, + }; + classifier.notify_keyboard_changed(device); + assert_eq!(classifier.get_keyboard_type(SECOND_DEVICE_ID), KeyboardType::Alphabetic); + assert!(classifier.is_finalized(SECOND_DEVICE_ID)); + } + } + + fn create_classifier() -> KeyboardClassifier { + KeyboardClassifier::new(DataStore::new(Box::new(TestFileReaderWriter::new()))) + } + + fn create_identifier(vendor: u16, product: u16) -> RustInputDeviceIdentifier { + RustInputDeviceIdentifier { + name: "test_device".to_string(), + location: "location".to_string(), + unique_id: "unique_id".to_string(), + bus: 123, + vendor, + product, + version: 567, + descriptor: "descriptor".to_string(), + } + } + fn create_device(classes: DeviceClass) -> InputDevice { InputDevice { device_id: DEVICE_ID, - identifier: RustInputDeviceIdentifier { - name: "test_device".to_string(), - location: "location".to_string(), - unique_id: "unique_id".to_string(), - bus: 123, - vendor: 234, - product: 345, - version: 567, - descriptor: "descriptor".to_string(), - }, + identifier: create_identifier(/* vendor= */ 234, /* product= */ 345), classes, } } + + fn create_device_with_vendor_product_ids(vendor: u16, product: u16) -> InputDevice { + InputDevice { + device_id: DEVICE_ID, + identifier: create_identifier(vendor, product), + classes: DeviceClass::Keyboard | DeviceClass::AlphabeticKey | DeviceClass::External, + } + } } diff --git a/libs/input/rust/lib.rs b/libs/input/rust/lib.rs index 88374695f1..4f4ea8568b 100644 --- a/libs/input/rust/lib.rs +++ b/libs/input/rust/lib.rs @@ -16,12 +16,16 @@ //! The rust component of libinput. +mod data_store; mod input; mod input_verifier; +mod keyboard_classification_config; mod keyboard_classifier; +pub use data_store::{DataStore, DefaultFileReaderWriter}; pub use input::{ - DeviceClass, DeviceId, InputDevice, ModifierState, MotionAction, MotionFlags, Source, + DeviceClass, DeviceId, InputDevice, KeyboardType, ModifierState, MotionAction, MotionFlags, + Source, }; pub use input_verifier::InputVerifier; pub use keyboard_classifier::KeyboardClassifier; @@ -149,7 +153,14 @@ fn reset_device(verifier: &mut InputVerifier, device_id: i32) { } fn create_keyboard_classifier() -> Box<KeyboardClassifier> { - Box::new(KeyboardClassifier::new()) + // Future design: Make this data store singleton by passing it to C++ side and making it global + // and pass by reference to components that need to store persistent data. + // + // Currently only used by rust keyboard classifier so keeping it here. + let data_store = DataStore::new(Box::new(DefaultFileReaderWriter::new( + "/data/system/inputflinger-data.json".to_string(), + ))); + Box::new(KeyboardClassifier::new(data_store)) } fn notify_keyboard_changed( diff --git a/libs/input/tests/Android.bp b/libs/input/tests/Android.bp index 9c0a41eafe..81c6175805 100644 --- a/libs/input/tests/Android.bp +++ b/libs/input/tests/Android.bp @@ -16,6 +16,7 @@ cc_test { "BlockingQueue_test.cpp", "IdGenerator_test.cpp", "InputChannel_test.cpp", + "InputConsumer_test.cpp", "InputDevice_test.cpp", "InputEvent_test.cpp", "InputPublisherAndConsumer_test.cpp", @@ -23,7 +24,9 @@ cc_test { "InputVerifier_test.cpp", "MotionPredictor_test.cpp", "MotionPredictorMetricsManager_test.cpp", + "Resampler_test.cpp", "RingBuffer_test.cpp", + "TestInputChannel.cpp", "TfLiteMotionPredictor_test.cpp", "TouchResampling_test.cpp", "TouchVideoFrame_test.cpp", diff --git a/libs/input/tests/InputChannel_test.cpp b/libs/input/tests/InputChannel_test.cpp index 02d4c07bfa..25356cfcf0 100644 --- a/libs/input/tests/InputChannel_test.cpp +++ b/libs/input/tests/InputChannel_test.cpp @@ -65,11 +65,7 @@ TEST_F(InputChannelTest, OpenInputChannelPair_ReturnsAPairOfConnectedChannels) { ASSERT_EQ(OK, result) << "should have successfully opened a channel pair"; - // Name - EXPECT_STREQ("channel name (server)", serverChannel->getName().c_str()) - << "server channel should have suffixed name"; - EXPECT_STREQ("channel name (client)", clientChannel->getName().c_str()) - << "client channel should have suffixed name"; + EXPECT_EQ(serverChannel->getName(), clientChannel->getName()); // Server->Client communication InputMessage serverMsg = {}; @@ -78,9 +74,10 @@ TEST_F(InputChannelTest, OpenInputChannelPair_ReturnsAPairOfConnectedChannels) { EXPECT_EQ(OK, serverChannel->sendMessage(&serverMsg)) << "server channel should be able to send message to client channel"; - InputMessage clientMsg; - EXPECT_EQ(OK, clientChannel->receiveMessage(&clientMsg)) + android::base::Result<InputMessage> clientMsgResult = clientChannel->receiveMessage(); + ASSERT_TRUE(clientMsgResult.ok()) << "client channel should be able to receive message from server channel"; + const InputMessage& clientMsg = *clientMsgResult; EXPECT_EQ(serverMsg.header.type, clientMsg.header.type) << "client channel should receive the correct message from server channel"; EXPECT_EQ(serverMsg.body.key.action, clientMsg.body.key.action) @@ -94,9 +91,10 @@ TEST_F(InputChannelTest, OpenInputChannelPair_ReturnsAPairOfConnectedChannels) { EXPECT_EQ(OK, clientChannel->sendMessage(&clientReply)) << "client channel should be able to send message to server channel"; - InputMessage serverReply; - EXPECT_EQ(OK, serverChannel->receiveMessage(&serverReply)) + android::base::Result<InputMessage> serverReplyResult = serverChannel->receiveMessage(); + ASSERT_TRUE(serverReplyResult.ok()) << "server channel should be able to receive message from client channel"; + const InputMessage& serverReply = *serverReplyResult; EXPECT_EQ(clientReply.header.type, serverReply.header.type) << "server channel should receive the correct message from client channel"; EXPECT_EQ(clientReply.header.seq, serverReply.header.seq) @@ -134,9 +132,10 @@ TEST_F(InputChannelTest, ProbablyHasInput) { << "client channel should observe that message is available before receiving it"; // Receive (consume) the message. - InputMessage clientMsg; - EXPECT_EQ(OK, receiverChannel->receiveMessage(&clientMsg)) + android::base::Result<InputMessage> clientMsgResult = receiverChannel->receiveMessage(); + ASSERT_TRUE(clientMsgResult.ok()) << "client channel should be able to receive message from server channel"; + const InputMessage& clientMsg = *clientMsgResult; EXPECT_EQ(serverMsg.header.type, clientMsg.header.type) << "client channel should receive the correct message from server channel"; EXPECT_EQ(serverMsg.body.key.action, clientMsg.body.key.action) @@ -156,8 +155,8 @@ TEST_F(InputChannelTest, ReceiveSignal_WhenNoSignalPresent_ReturnsAnError) { ASSERT_EQ(OK, result) << "should have successfully opened a channel pair"; - InputMessage msg; - EXPECT_EQ(WOULD_BLOCK, clientChannel->receiveMessage(&msg)) + android::base::Result<InputMessage> msgResult = clientChannel->receiveMessage(); + EXPECT_EQ(WOULD_BLOCK, msgResult.error().code()) << "receiveMessage should have returned WOULD_BLOCK"; } @@ -172,8 +171,8 @@ TEST_F(InputChannelTest, ReceiveSignal_WhenPeerClosed_ReturnsAnError) { serverChannel.reset(); // close server channel - InputMessage msg; - EXPECT_EQ(DEAD_OBJECT, clientChannel->receiveMessage(&msg)) + android::base::Result<InputMessage> msgResult = clientChannel->receiveMessage(); + EXPECT_EQ(DEAD_OBJECT, msgResult.error().code()) << "receiveMessage should have returned DEAD_OBJECT"; } @@ -207,7 +206,7 @@ TEST_F(InputChannelTest, SendAndReceive_MotionClassification) { MotionClassification::DEEP_PRESS, }; - InputMessage serverMsg = {}, clientMsg; + InputMessage serverMsg = {}; serverMsg.header.type = InputMessage::Type::MOTION; serverMsg.header.seq = 1; serverMsg.body.motion.pointerCount = 1; @@ -218,11 +217,13 @@ TEST_F(InputChannelTest, SendAndReceive_MotionClassification) { EXPECT_EQ(OK, serverChannel->sendMessage(&serverMsg)) << "server channel should be able to send message to client channel"; - EXPECT_EQ(OK, clientChannel->receiveMessage(&clientMsg)) + android::base::Result<InputMessage> clientMsgResult = clientChannel->receiveMessage(); + ASSERT_TRUE(clientMsgResult.ok()) << "client channel should be able to receive message from server channel"; + const InputMessage& clientMsg = *clientMsgResult; EXPECT_EQ(serverMsg.header.type, clientMsg.header.type); - EXPECT_EQ(classification, clientMsg.body.motion.classification) << - "Expected to receive " << motionClassificationToString(classification); + EXPECT_EQ(classification, clientMsg.body.motion.classification) + << "Expected to receive " << motionClassificationToString(classification); } } diff --git a/libs/input/tests/InputConsumer_test.cpp b/libs/input/tests/InputConsumer_test.cpp new file mode 100644 index 0000000000..d708316236 --- /dev/null +++ b/libs/input/tests/InputConsumer_test.cpp @@ -0,0 +1,247 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <input/InputConsumerNoResampling.h> + +#include <memory> +#include <optional> + +#include <TestEventMatchers.h> +#include <TestInputChannel.h> +#include <android-base/logging.h> +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <input/BlockingQueue.h> +#include <input/InputEventBuilders.h> +#include <utils/Looper.h> +#include <utils/StrongPointer.h> + +namespace android { + +namespace { + +using std::chrono::nanoseconds; + +using ::testing::AllOf; +using ::testing::Matcher; +using ::testing::Not; + +} // namespace + +class InputConsumerTest : public testing::Test, public InputConsumerCallbacks { +protected: + InputConsumerTest() + : mClientTestChannel{std::make_shared<TestInputChannel>("TestChannel")}, + mLooper{sp<Looper>::make(/*allowNonCallbacks=*/false)} { + Looper::setForThread(mLooper); + mConsumer = + std::make_unique<InputConsumerNoResampling>(mClientTestChannel, mLooper, *this, + std::make_unique<LegacyResampler>()); + } + + void invokeLooperCallback() const { + sp<LooperCallback> callback; + ASSERT_TRUE(mLooper->getFdStateDebug(mClientTestChannel->getFd(), /*ident=*/nullptr, + /*events=*/nullptr, &callback, /*data=*/nullptr)); + callback->handleEvent(mClientTestChannel->getFd(), ALOOPER_EVENT_INPUT, /*data=*/nullptr); + } + + void assertOnBatchedInputEventPendingWasCalled() { + ASSERT_GT(mOnBatchedInputEventPendingInvocationCount, 0UL) + << "onBatchedInputEventPending has not been called."; + --mOnBatchedInputEventPendingInvocationCount; + } + + void assertReceivedMotionEvent(const Matcher<MotionEvent>& matcher) { + std::unique_ptr<MotionEvent> motionEvent = mMotionEvents.pop(); + ASSERT_NE(motionEvent, nullptr); + EXPECT_THAT(*motionEvent, matcher); + } + + std::shared_ptr<TestInputChannel> mClientTestChannel; + sp<Looper> mLooper; + std::unique_ptr<InputConsumerNoResampling> mConsumer; + + BlockingQueue<std::unique_ptr<KeyEvent>> mKeyEvents; + BlockingQueue<std::unique_ptr<MotionEvent>> mMotionEvents; + BlockingQueue<std::unique_ptr<FocusEvent>> mFocusEvents; + BlockingQueue<std::unique_ptr<CaptureEvent>> mCaptureEvents; + BlockingQueue<std::unique_ptr<DragEvent>> mDragEvents; + BlockingQueue<std::unique_ptr<TouchModeEvent>> mTouchModeEvents; + +private: + size_t mOnBatchedInputEventPendingInvocationCount{0}; + + // InputConsumerCallbacks interface + void onKeyEvent(std::unique_ptr<KeyEvent> event, uint32_t seq) override { + mKeyEvents.push(std::move(event)); + mConsumer->finishInputEvent(seq, true); + } + void onMotionEvent(std::unique_ptr<MotionEvent> event, uint32_t seq) override { + mMotionEvents.push(std::move(event)); + mConsumer->finishInputEvent(seq, true); + } + void onBatchedInputEventPending(int32_t pendingBatchSource) override { + if (!mConsumer->probablyHasInput()) { + ADD_FAILURE() << "should deterministically have input because there is a batch"; + } + ++mOnBatchedInputEventPendingInvocationCount; + }; + void onFocusEvent(std::unique_ptr<FocusEvent> event, uint32_t seq) override { + mFocusEvents.push(std::move(event)); + mConsumer->finishInputEvent(seq, true); + }; + void onCaptureEvent(std::unique_ptr<CaptureEvent> event, uint32_t seq) override { + mCaptureEvents.push(std::move(event)); + mConsumer->finishInputEvent(seq, true); + }; + void onDragEvent(std::unique_ptr<DragEvent> event, uint32_t seq) override { + mDragEvents.push(std::move(event)); + mConsumer->finishInputEvent(seq, true); + } + void onTouchModeEvent(std::unique_ptr<TouchModeEvent> event, uint32_t seq) override { + mTouchModeEvents.push(std::move(event)); + mConsumer->finishInputEvent(seq, true); + }; +}; + +TEST_F(InputConsumerTest, MessageStreamBatchedInMotionEvent) { + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/0} + .eventTime(nanoseconds{0ms}.count()) + .action(AMOTION_EVENT_ACTION_DOWN) + .build()); + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/1} + .eventTime(nanoseconds{5ms}.count()) + .action(AMOTION_EVENT_ACTION_MOVE) + .build()); + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/2} + .eventTime(nanoseconds{10ms}.count()) + .action(AMOTION_EVENT_ACTION_MOVE) + .build()); + + mClientTestChannel->assertNoSentMessages(); + + invokeLooperCallback(); + + assertOnBatchedInputEventPendingWasCalled(); + + mConsumer->consumeBatchedInputEvents(/*frameTime=*/std::nullopt); + + std::unique_ptr<MotionEvent> downMotionEvent = mMotionEvents.pop(); + ASSERT_NE(downMotionEvent, nullptr); + + std::unique_ptr<MotionEvent> moveMotionEvent = mMotionEvents.pop(); + ASSERT_NE(moveMotionEvent, nullptr); + EXPECT_EQ(moveMotionEvent->getHistorySize() + 1, 3UL); + + mClientTestChannel->assertFinishMessage(/*seq=*/0, /*handled=*/true); + mClientTestChannel->assertFinishMessage(/*seq=*/1, /*handled=*/true); + mClientTestChannel->assertFinishMessage(/*seq=*/2, /*handled=*/true); +} + +TEST_F(InputConsumerTest, LastBatchedSampleIsLessThanResampleTime) { + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/0} + .eventTime(nanoseconds{0ms}.count()) + .action(AMOTION_EVENT_ACTION_DOWN) + .build()); + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/1} + .eventTime(nanoseconds{5ms}.count()) + .action(AMOTION_EVENT_ACTION_MOVE) + .build()); + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/2} + .eventTime(nanoseconds{10ms}.count()) + .action(AMOTION_EVENT_ACTION_MOVE) + .build()); + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/3} + .eventTime(nanoseconds{15ms}.count()) + .action(AMOTION_EVENT_ACTION_MOVE) + .build()); + + mClientTestChannel->assertNoSentMessages(); + + invokeLooperCallback(); + + assertOnBatchedInputEventPendingWasCalled(); + + mConsumer->consumeBatchedInputEvents(16'000'000 /*ns*/); + + std::unique_ptr<MotionEvent> downMotionEvent = mMotionEvents.pop(); + ASSERT_NE(downMotionEvent, nullptr); + + std::unique_ptr<MotionEvent> moveMotionEvent = mMotionEvents.pop(); + ASSERT_NE(moveMotionEvent, nullptr); + const size_t numSamples = moveMotionEvent->getHistorySize() + 1; + EXPECT_LT(moveMotionEvent->getHistoricalEventTime(numSamples - 2), + moveMotionEvent->getEventTime()); + + // Consume all remaining events before ending the test. Otherwise, the smart pointer that owns + // consumer is set to null before destroying consumer. This leads to a member function call on a + // null object. + // TODO(b/332613662): Remove this workaround. + mConsumer->consumeBatchedInputEvents(std::nullopt); + + mClientTestChannel->assertFinishMessage(/*seq=*/0, true); + mClientTestChannel->assertFinishMessage(/*seq=*/1, true); + mClientTestChannel->assertFinishMessage(/*seq=*/2, true); + mClientTestChannel->assertFinishMessage(/*seq=*/3, true); +} + +TEST_F(InputConsumerTest, BatchedEventsMultiDeviceConsumption) { + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/0} + .deviceId(0) + .action(AMOTION_EVENT_ACTION_DOWN) + .build()); + + invokeLooperCallback(); + assertReceivedMotionEvent(AllOf(WithDeviceId(0), WithMotionAction(AMOTION_EVENT_ACTION_DOWN))); + + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/1} + .deviceId(0) + .action(AMOTION_EVENT_ACTION_MOVE) + .build()); + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/2} + .deviceId(0) + .action(AMOTION_EVENT_ACTION_MOVE) + .build()); + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/3} + .deviceId(0) + .action(AMOTION_EVENT_ACTION_MOVE) + .build()); + + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/4} + .deviceId(1) + .action(AMOTION_EVENT_ACTION_DOWN) + .build()); + + invokeLooperCallback(); + assertReceivedMotionEvent(AllOf(WithDeviceId(1), WithMotionAction(AMOTION_EVENT_ACTION_DOWN))); + + mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/5} + .deviceId(0) + .action(AMOTION_EVENT_ACTION_UP) + .build()); + + invokeLooperCallback(); + assertReceivedMotionEvent(AllOf(WithDeviceId(0), WithMotionAction(AMOTION_EVENT_ACTION_MOVE), + Not(MotionEventIsResampled()))); + + mClientTestChannel->assertFinishMessage(/*seq=*/0, /*handled=*/true); + mClientTestChannel->assertFinishMessage(/*seq=*/4, /*handled=*/true); + mClientTestChannel->assertFinishMessage(/*seq=*/1, /*handled=*/true); + mClientTestChannel->assertFinishMessage(/*seq=*/2, /*handled=*/true); + mClientTestChannel->assertFinishMessage(/*seq=*/3, /*handled=*/true); +} +} // namespace android diff --git a/libs/input/tests/InputEvent_test.cpp b/libs/input/tests/InputEvent_test.cpp index 3717f49fef..a67e1ef472 100644 --- a/libs/input/tests/InputEvent_test.cpp +++ b/libs/input/tests/InputEvent_test.cpp @@ -371,8 +371,8 @@ void MotionEventTest::initializeEventWithHistory(MotionEvent* event) { mTransform, 2.0f, 2.1f, AMOTION_EVENT_INVALID_CURSOR_POSITION, AMOTION_EVENT_INVALID_CURSOR_POSITION, mRawTransform, ARBITRARY_DOWN_TIME, ARBITRARY_EVENT_TIME, 2, mPointerProperties, mSamples[0].pointerCoords); - event->addSample(ARBITRARY_EVENT_TIME + 1, mSamples[1].pointerCoords); - event->addSample(ARBITRARY_EVENT_TIME + 2, mSamples[2].pointerCoords); + event->addSample(ARBITRARY_EVENT_TIME + 1, mSamples[1].pointerCoords, event->getId()); + event->addSample(ARBITRARY_EVENT_TIME + 2, mSamples[2].pointerCoords, event->getId()); } void MotionEventTest::assertEqualsEventWithHistory(const MotionEvent* event) { @@ -591,6 +591,22 @@ TEST_F(MotionEventTest, CopyFrom_DoNotKeepHistory) { ASSERT_EQ(event.getX(0), copy.getX(0)); } +TEST_F(MotionEventTest, CheckEventIdWithHistoryIsIncremented) { + MotionEvent event; + constexpr int32_t ARBITRARY_ID = 42; + event.initialize(ARBITRARY_ID, 2, AINPUT_SOURCE_TOUCHSCREEN, DISPLAY_ID, INVALID_HMAC, + AMOTION_EVENT_ACTION_MOVE, 0, 0, AMOTION_EVENT_EDGE_FLAG_NONE, AMETA_NONE, + AMOTION_EVENT_BUTTON_PRIMARY, MotionClassification::NONE, mTransform, 0, 0, + AMOTION_EVENT_INVALID_CURSOR_POSITION, AMOTION_EVENT_INVALID_CURSOR_POSITION, + mRawTransform, ARBITRARY_DOWN_TIME, ARBITRARY_EVENT_TIME, 2, + mPointerProperties, mSamples[0].pointerCoords); + ASSERT_EQ(event.getId(), ARBITRARY_ID); + event.addSample(ARBITRARY_EVENT_TIME + 1, mSamples[1].pointerCoords, ARBITRARY_ID + 1); + ASSERT_EQ(event.getId(), ARBITRARY_ID + 1); + event.addSample(ARBITRARY_EVENT_TIME + 2, mSamples[2].pointerCoords, ARBITRARY_ID + 2); + ASSERT_EQ(event.getId(), ARBITRARY_ID + 2); +} + TEST_F(MotionEventTest, SplitPointerDown) { MotionEvent event = MotionEventBuilder(POINTER_1_DOWN, AINPUT_SOURCE_TOUCHSCREEN) .downTime(ARBITRARY_DOWN_TIME) diff --git a/libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp b/libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp index f49469ccca..1210f711de 100644 --- a/libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp +++ b/libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp @@ -52,7 +52,7 @@ struct PublishMotionArgs { const int32_t action; const nsecs_t downTime; const uint32_t seq; - const int32_t eventId; + int32_t eventId; const int32_t deviceId = 1; const uint32_t source = AINPUT_SOURCE_TOUCHSCREEN; const ui::LogicalDisplayId displayId = ui::LogicalDisplayId::DEFAULT; @@ -291,6 +291,7 @@ protected: void publishAndConsumeKeyEvent(); void publishAndConsumeMotionStream(); void publishAndConsumeMotionDown(nsecs_t downTime); + void publishAndConsumeSinglePointerMultipleSamples(const size_t nSamples); void publishAndConsumeBatchedMotionMove(nsecs_t downTime); void publishAndConsumeFocusEvent(); void publishAndConsumeCaptureEvent(); @@ -298,6 +299,7 @@ protected: void publishAndConsumeTouchModeEvent(); void publishAndConsumeMotionEvent(int32_t action, nsecs_t downTime, const std::vector<Pointer>& pointers); + void TearDown() override { // Destroy the consumer, flushing any of the pending ack's. sendMessage(LooperMessage::DESTROY_CONSUMER); @@ -362,7 +364,7 @@ private: if (!mConsumer->probablyHasInput()) { ADD_FAILURE() << "should deterministically have input because there is a batch"; } - mConsumer->consumeBatchedInputEvents(std::nullopt); + mConsumer->consumeBatchedInputEvents(/*frameTime=*/std::nullopt); }; void onFocusEvent(std::unique_ptr<FocusEvent> event, uint32_t seq) override { mFocusEvents.push(std::move(event)); @@ -394,8 +396,9 @@ void InputPublisherAndConsumerNoResamplingTest::handleMessage(const Message& mes break; } case LooperMessage::CREATE_CONSUMER: { - mConsumer = std::make_unique<InputConsumerNoResampling>(std::move(mClientChannel), - mLooper, *this); + mConsumer = + std::make_unique<InputConsumerNoResampling>(std::move(mClientChannel), mLooper, + *this, /*resampler=*/nullptr); break; } case LooperMessage::DESTROY_CONSUMER: { @@ -519,6 +522,123 @@ void InputPublisherAndConsumerNoResamplingTest::publishAndConsumeMotionDown(nsec {Pointer{.id = 0, .x = 20, .y = 30}}); } +/* + * Decompose a potential multi-sampled MotionEvent into multiple MotionEvents + * with a single sample. + */ +std::vector<MotionEvent> splitBatchedMotionEvent(const MotionEvent& batchedMotionEvent) { + std::vector<MotionEvent> singleMotionEvents; + const size_t batchSize = batchedMotionEvent.getHistorySize() + 1; + for (size_t i = 0; i < batchSize; ++i) { + MotionEvent singleMotionEvent; + singleMotionEvent + .initialize(batchedMotionEvent.getId(), batchedMotionEvent.getDeviceId(), + batchedMotionEvent.getSource(), batchedMotionEvent.getDisplayId(), + batchedMotionEvent.getHmac(), batchedMotionEvent.getAction(), + batchedMotionEvent.getActionButton(), batchedMotionEvent.getFlags(), + batchedMotionEvent.getEdgeFlags(), batchedMotionEvent.getMetaState(), + batchedMotionEvent.getButtonState(), + batchedMotionEvent.getClassification(), + batchedMotionEvent.getTransform(), batchedMotionEvent.getXPrecision(), + batchedMotionEvent.getYPrecision(), + batchedMotionEvent.getRawXCursorPosition(), + batchedMotionEvent.getRawYCursorPosition(), + batchedMotionEvent.getRawTransform(), batchedMotionEvent.getDownTime(), + batchedMotionEvent.getHistoricalEventTime(/*historicalIndex=*/i), + batchedMotionEvent.getPointerCount(), + batchedMotionEvent.getPointerProperties(), + (batchedMotionEvent.getSamplePointerCoords() + i)); + singleMotionEvents.push_back(singleMotionEvent); + } + return singleMotionEvents; +} + +/* + * Simulates a single pointer touching the screen and leaving it there for a period of time. + * Publishes a DOWN event and consumes it right away. Then, publishes a sequence of MOVE + * samples for the same pointer, and waits until it has been consumed. Splits batched MotionEvents + * into individual samples. Checks the consumed MotionEvents against the published ones. + * This test is non-deterministic because it depends on the timing of arrival of events to the + * socket. + * + * @param nSamples The number of MOVE samples to publish before attempting consumption. + */ +void InputPublisherAndConsumerNoResamplingTest::publishAndConsumeSinglePointerMultipleSamples( + const size_t nSamples) { + const nsecs_t downTime = systemTime(SYSTEM_TIME_MONOTONIC); + const Pointer pointer(0, 20, 30); + + const PublishMotionArgs argsDown(AMOTION_EVENT_ACTION_DOWN, downTime, {pointer}, mSeq); + const nsecs_t publishTimeOfDown = systemTime(SYSTEM_TIME_MONOTONIC); + publishMotionEvent(*mPublisher, argsDown); + + // Consume the DOWN event. + ASSERT_TRUE(mMotionEvents.popWithTimeout(TIMEOUT).has_value()); + + verifyFinishedSignal(*mPublisher, mSeq, publishTimeOfDown); + + std::vector<nsecs_t> publishTimes; + std::vector<PublishMotionArgs> argsMoves; + std::queue<uint32_t> publishedSequenceNumbers; + + // Block Looper to increase the chance of batching events + { + std::scoped_lock l(mLock); + mLooperMayProceed = false; + } + sendMessage(LooperMessage::BLOCK_LOOPER); + { + std::unique_lock l(mLock); + mNotifyLooperWaiting.wait(l, [this] { return mLooperIsBlocked; }); + } + + uint32_t firstSampleId; + for (size_t i = 0; i < nSamples; ++i) { + publishedSequenceNumbers.push(++mSeq); + PublishMotionArgs argsMove(AMOTION_EVENT_ACTION_MOVE, downTime, {pointer}, mSeq); + // A batched MotionEvent only has a single event id, currently determined when the + // MotionEvent is initialized. Therefore, to pass the eventId comparisons inside + // verifyArgsEqualToEvent, we need to override the event id of the published args to match + // the event id of the first sample inside the MotionEvent. + if (i == 0) { + firstSampleId = argsMove.eventId; + } + argsMove.eventId = firstSampleId; + publishTimes.push_back(systemTime(SYSTEM_TIME_MONOTONIC)); + argsMoves.push_back(argsMove); + publishMotionEvent(*mPublisher, argsMove); + } + + std::vector<MotionEvent> singleSampledMotionEvents; + + // Unblock Looper + { + std::scoped_lock l(mLock); + mLooperMayProceed = true; + } + mNotifyLooperMayProceed.notify_all(); + + // We have no control over the socket behavior, so the consumer can receive + // the motion as a batched event, or as a sequence of multiple single-sample MotionEvents (or a + // mix of those) + while (singleSampledMotionEvents.size() != nSamples) { + const std::optional<std::unique_ptr<MotionEvent>> batchedMotionEvent = + mMotionEvents.popWithTimeout(TIMEOUT); + // The events received by these calls are never null + std::vector<MotionEvent> splitMotionEvents = splitBatchedMotionEvent(**batchedMotionEvent); + singleSampledMotionEvents.insert(singleSampledMotionEvents.end(), splitMotionEvents.begin(), + splitMotionEvents.end()); + } + + // Consumer can choose to finish events in any order. For simplicity, + // we verify the events in sequence (since that is how the test is implemented). + for (size_t i = 0; i < nSamples; ++i) { + verifyArgsEqualToEvent(argsMoves[i], singleSampledMotionEvents[i]); + verifyFinishedSignal(*mPublisher, publishedSequenceNumbers.front(), publishTimes[i]); + publishedSequenceNumbers.pop(); + } +} + void InputPublisherAndConsumerNoResamplingTest::publishAndConsumeBatchedMotionMove( nsecs_t downTime) { uint32_t seq = mSeq++; @@ -814,4 +934,8 @@ TEST_F(InputPublisherAndConsumerNoResamplingTest, PublishMultipleEvents_EndToEnd ASSERT_NO_FATAL_FAILURE(publishAndConsumeTouchModeEvent()); } +TEST_F(InputPublisherAndConsumerNoResamplingTest, PublishAndConsumeSinglePointer) { + publishAndConsumeSinglePointerMultipleSamples(3); +} + } // namespace android diff --git a/libs/input/tests/MotionPredictorMetricsManager_test.cpp b/libs/input/tests/MotionPredictorMetricsManager_test.cpp index cc41eeb5e7..0542f39f6d 100644 --- a/libs/input/tests/MotionPredictorMetricsManager_test.cpp +++ b/libs/input/tests/MotionPredictorMetricsManager_test.cpp @@ -167,7 +167,8 @@ MotionEvent makeMotionEvent(const std::vector<PredictionPoint>& predictionPoints .y(predictionPoints[i].position[0]) .axis(AMOTION_EVENT_AXIS_PRESSURE, predictionPoints[i].pressure) .buildCoords(); - predictionEvent.addSample(predictionPoints[i].targetTimestamp, &coords); + predictionEvent.addSample(predictionPoints[i].targetTimestamp, &coords, + predictionEvent.getId()); } return predictionEvent; } diff --git a/libs/input/tests/MotionPredictor_test.cpp b/libs/input/tests/MotionPredictor_test.cpp index d077760757..106e686a81 100644 --- a/libs/input/tests/MotionPredictor_test.cpp +++ b/libs/input/tests/MotionPredictor_test.cpp @@ -70,7 +70,7 @@ static MotionEvent getMotionEvent(int32_t action, float x, float y, } TEST(JerkTrackerTest, JerkReadiness) { - JerkTracker jerkTracker(true); + JerkTracker jerkTracker(/*normalizedDt=*/true, /*alpha=*/1); EXPECT_FALSE(jerkTracker.jerkMagnitude()); jerkTracker.pushSample(/*timestamp=*/0, 20, 50); EXPECT_FALSE(jerkTracker.jerkMagnitude()); @@ -87,7 +87,8 @@ TEST(JerkTrackerTest, JerkReadiness) { } TEST(JerkTrackerTest, JerkCalculationNormalizedDtTrue) { - JerkTracker jerkTracker(true); + const float alpha = .5; + JerkTracker jerkTracker(/*normalizedDt=*/true, alpha); jerkTracker.pushSample(/*timestamp=*/0, 20, 50); jerkTracker.pushSample(/*timestamp=*/1, 25, 53); jerkTracker.pushSample(/*timestamp=*/2, 30, 60); @@ -118,11 +119,13 @@ TEST(JerkTrackerTest, JerkCalculationNormalizedDtTrue) { * y'': 3 -> -15 * y''': -18 */ - EXPECT_FLOAT_EQ(jerkTracker.jerkMagnitude().value(), std::hypot(-50, -18)); + const float newJerk = (1 - alpha) * std::hypot(10, -1) + alpha * std::hypot(-50, -18); + EXPECT_FLOAT_EQ(jerkTracker.jerkMagnitude().value(), newJerk); } TEST(JerkTrackerTest, JerkCalculationNormalizedDtFalse) { - JerkTracker jerkTracker(false); + const float alpha = .5; + JerkTracker jerkTracker(/*normalizedDt=*/false, alpha); jerkTracker.pushSample(/*timestamp=*/0, 20, 50); jerkTracker.pushSample(/*timestamp=*/10, 25, 53); jerkTracker.pushSample(/*timestamp=*/20, 30, 60); @@ -153,11 +156,12 @@ TEST(JerkTrackerTest, JerkCalculationNormalizedDtFalse) { * y'': .03 -> -.125 (delta above, divide by 10) * y''': -.0155 (delta above, divide by 10) */ - EXPECT_FLOAT_EQ(jerkTracker.jerkMagnitude().value(), std::hypot(-.0375, -.0155)); + const float newJerk = (1 - alpha) * std::hypot(.01, -.001) + alpha * std::hypot(-.0375, -.0155); + EXPECT_FLOAT_EQ(jerkTracker.jerkMagnitude().value(), newJerk); } TEST(JerkTrackerTest, JerkCalculationAfterReset) { - JerkTracker jerkTracker(true); + JerkTracker jerkTracker(/*normalizedDt=*/true, /*alpha=*/1); jerkTracker.pushSample(/*timestamp=*/0, 20, 50); jerkTracker.pushSample(/*timestamp=*/1, 25, 53); jerkTracker.pushSample(/*timestamp=*/2, 30, 60); @@ -291,15 +295,22 @@ TEST_WITH_FLAGS( MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/0, []() { return true /*enable prediction*/; }); - // Jerk is medium (1.05 normalized, which is halfway between LOW_JANK and HIGH_JANK) - predictor.record(getMotionEvent(DOWN, 0, 5.2, 20ms)); - predictor.record(getMotionEvent(MOVE, 0, 11.5, 30ms)); - predictor.record(getMotionEvent(MOVE, 0, 22, 40ms)); - predictor.record(getMotionEvent(MOVE, 0, 37.75, 50ms)); - predictor.record(getMotionEvent(MOVE, 0, 59.8, 60ms)); + // Create another instance of TfLiteMotionPredictorModel to read config details. + std::unique_ptr<TfLiteMotionPredictorModel> testTfLiteModel = + TfLiteMotionPredictorModel::create(); + const float mediumJerk = + (testTfLiteModel->config().lowJerk + testTfLiteModel->config().highJerk) / 2; + const float a = 3; // initial acceleration + const float b = 4; // initial velocity + const float c = 5; // initial position + predictor.record(getMotionEvent(DOWN, 0, c, 20ms)); + predictor.record(getMotionEvent(MOVE, 0, c + b, 30ms)); + predictor.record(getMotionEvent(MOVE, 0, c + 2 * b + a, 40ms)); + predictor.record(getMotionEvent(MOVE, 0, c + 3 * b + 3 * a + mediumJerk, 50ms)); + predictor.record(getMotionEvent(MOVE, 0, c + 4 * b + 6 * a + 4 * mediumJerk, 60ms)); std::unique_ptr<MotionEvent> predicted = predictor.predict(82 * NSEC_PER_MSEC); EXPECT_NE(nullptr, predicted); - // Halfway between LOW_JANK and HIGH_JANK means that half of the predictions + // Halfway between LOW_JERK and HIGH_JERK means that half of the predictions // will be pruned. If model prediction window is close enough to predict() // call time window, then half of the model predictions (5/2 -> 2) will be // ouputted. diff --git a/libs/input/tests/Resampler_test.cpp b/libs/input/tests/Resampler_test.cpp new file mode 100644 index 0000000000..26dee393c1 --- /dev/null +++ b/libs/input/tests/Resampler_test.cpp @@ -0,0 +1,873 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <input/Resampler.h> + +#include <gtest/gtest.h> + +#include <chrono> +#include <memory> +#include <vector> + +#include <input/Input.h> +#include <input/InputEventBuilders.h> +#include <input/InputTransport.h> +#include <utils/Timers.h> + +namespace android { + +namespace { + +using namespace std::literals::chrono_literals; + +constexpr float EPSILON = MotionEvent::ROUNDING_PRECISION; + +struct Pointer { + int32_t id{0}; + ToolType toolType{ToolType::FINGER}; + float x{0.0f}; + float y{0.0f}; + bool isResampled{false}; + /** + * Converts from Pointer to PointerCoords. Enables calling LegacyResampler methods and + * assertions only with the relevant data for tests. + */ + operator PointerCoords() const; +}; + +Pointer::operator PointerCoords() const { + PointerCoords pointerCoords; + pointerCoords.setAxisValue(AMOTION_EVENT_AXIS_X, x); + pointerCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, y); + pointerCoords.isResampled = isResampled; + return pointerCoords; +} + +struct InputSample { + std::chrono::milliseconds eventTime{0}; + std::vector<Pointer> pointers{}; + + explicit InputSample(std::chrono::milliseconds eventTime, const std::vector<Pointer>& pointers) + : eventTime{eventTime}, pointers{pointers} {} + /** + * Converts from InputSample to InputMessage. Enables calling LegacyResampler methods only with + * the relevant data for tests. + */ + operator InputMessage() const; +}; + +InputSample::operator InputMessage() const { + InputMessageBuilder messageBuilder = + InputMessageBuilder{InputMessage::Type::MOTION, /*seq=*/0} + .eventTime(std::chrono::nanoseconds{eventTime}.count()) + .source(AINPUT_SOURCE_TOUCHSCREEN) + .downTime(0); + + for (const Pointer& pointer : pointers) { + messageBuilder.pointer( + PointerBuilder{pointer.id, pointer.toolType}.x(pointer.x).y(pointer.y).isResampled( + pointer.isResampled)); + } + return messageBuilder.build(); +} + +struct InputStream { + std::vector<InputSample> samples{}; + int32_t action{0}; + DeviceId deviceId{0}; + /** + * Converts from InputStream to MotionEvent. Enables calling LegacyResampler methods only with + * the relevant data for tests. + */ + operator MotionEvent() const; +}; + +InputStream::operator MotionEvent() const { + const InputSample& firstSample{*samples.begin()}; + MotionEventBuilder motionEventBuilder = + MotionEventBuilder(action, AINPUT_SOURCE_CLASS_POINTER) + .downTime(0) + .eventTime(static_cast<std::chrono::nanoseconds>(firstSample.eventTime).count()) + .deviceId(deviceId); + for (const Pointer& pointer : firstSample.pointers) { + const PointerBuilder pointerBuilder = + PointerBuilder(pointer.id, pointer.toolType).x(pointer.x).y(pointer.y); + motionEventBuilder.pointer(pointerBuilder); + } + MotionEvent motionEvent = motionEventBuilder.build(); + const size_t numSamples = samples.size(); + for (size_t i = 1; i < numSamples; ++i) { + std::vector<PointerCoords> pointersCoords{samples[i].pointers.begin(), + samples[i].pointers.end()}; + motionEvent.addSample(static_cast<std::chrono::nanoseconds>(samples[i].eventTime).count(), + pointersCoords.data(), motionEvent.getId()); + } + return motionEvent; +} + +} // namespace + +/** + * The testing setup assumes an input rate of 200 Hz and a display rate of 60 Hz. This implies that + * input events are received every 5 milliseconds, while the display consumes batched events every + * ~16 milliseconds. The resampler's RESAMPLE_LATENCY constant determines the resample time, which + * is calculated as frameTime - RESAMPLE_LATENCY. resampleTime specifies the time used for + * resampling. For example, if the desired frame time consumption is ~16 milliseconds, the resample + * time would be ~11 milliseconds. Consequenly, the last added sample to the motion event has an + * event time of ~11 milliseconds. Note that there are specific scenarios where resampleMotionEvent + * is not called with a multiple of ~16 milliseconds. These cases are primarily for data addition + * or to test other functionalities of the resampler. + * + * Coordinates are calculated using linear interpolation (lerp) based on the last two available + * samples. Linear interpolation is defined as (a + alpha*(b - a)). Let t_b and t_a be the + * timestamps of samples a and b, respectively. The interpolation factor alpha is calculated as + * (resampleTime - t_a) / (t_b - t_a). The value of alpha determines whether the resampled + * coordinates are interpolated or extrapolated. If alpha falls within the semi-closed interval [0, + * 1), the coordinates are interpolated. If alpha is greater than or equal to 1, the coordinates are + * extrapolated. + * + * The timeline below depics an interpolation scenario + * -----------------------------------|---------|---------|---------|---------- + * 10ms 11ms 15ms 16ms + * MOVE | MOVE | + * resampleTime frameTime + * Based on the timeline alpha is (11 - 10)/(15 - 10) = 1/5. Thus, coordinates are interpolated. + * + * The following timeline portrays an extrapolation scenario + * -------------------------|---------|---------|-------------------|---------- + * 5ms 10ms 11ms 16ms + * MOVE MOVE | | + * resampleTime frameTime + * Likewise, alpha = (11 - 5)/(10 - 5) = 6/5. Hence, coordinates are extrapolated. + * + * If a motion event was resampled, the tests will check that the following conditions are satisfied + * to guarantee resampling correctness: + * - The motion event metadata must not change. + * - The number of samples in the motion event must only increment by 1. + * - The resampled values must be at the end of motion event coordinates. + * - The rasamples values must be near the hand calculations. + * - The resampled time must be the most recent one in motion event. + */ +class ResamplerTest : public testing::Test { +protected: + ResamplerTest() : mResampler(std::make_unique<LegacyResampler>()) {} + + ~ResamplerTest() override {} + + void SetUp() override {} + + void TearDown() override {} + + std::unique_ptr<Resampler> mResampler; + + /** + * Checks that beforeCall and afterCall are equal except for the mutated attributes by addSample + * member function. + * @param beforeCall MotionEvent before passing it to resampleMotionEvent + * @param afterCall MotionEvent after passing it to resampleMotionEvent + */ + void assertMotionEventMetaDataDidNotMutate(const MotionEvent& beforeCall, + const MotionEvent& afterCall); + + /** + * Asserts the MotionEvent is resampled by checking an increment in history size and that the + * resampled coordinates are near the expected ones. + */ + void assertMotionEventIsResampledAndCoordsNear( + const MotionEvent& original, const MotionEvent& resampled, + const std::vector<PointerCoords>& expectedCoords); + + void assertMotionEventIsNotResampled(const MotionEvent& original, + const MotionEvent& notResampled); +}; + +void ResamplerTest::assertMotionEventMetaDataDidNotMutate(const MotionEvent& beforeCall, + const MotionEvent& afterCall) { + EXPECT_EQ(beforeCall.getDeviceId(), afterCall.getDeviceId()); + EXPECT_EQ(beforeCall.getAction(), afterCall.getAction()); + EXPECT_EQ(beforeCall.getActionButton(), afterCall.getActionButton()); + EXPECT_EQ(beforeCall.getButtonState(), afterCall.getButtonState()); + EXPECT_EQ(beforeCall.getFlags(), afterCall.getFlags()); + EXPECT_EQ(beforeCall.getEdgeFlags(), afterCall.getEdgeFlags()); + EXPECT_EQ(beforeCall.getClassification(), afterCall.getClassification()); + EXPECT_EQ(beforeCall.getPointerCount(), afterCall.getPointerCount()); + EXPECT_EQ(beforeCall.getMetaState(), afterCall.getMetaState()); + EXPECT_EQ(beforeCall.getSource(), afterCall.getSource()); + EXPECT_EQ(beforeCall.getXPrecision(), afterCall.getXPrecision()); + EXPECT_EQ(beforeCall.getYPrecision(), afterCall.getYPrecision()); + EXPECT_EQ(beforeCall.getDownTime(), afterCall.getDownTime()); + EXPECT_EQ(beforeCall.getDisplayId(), afterCall.getDisplayId()); +} + +void ResamplerTest::assertMotionEventIsResampledAndCoordsNear( + const MotionEvent& original, const MotionEvent& resampled, + const std::vector<PointerCoords>& expectedCoords) { + assertMotionEventMetaDataDidNotMutate(original, resampled); + + const size_t originalSampleSize = original.getHistorySize() + 1; + const size_t resampledSampleSize = resampled.getHistorySize() + 1; + EXPECT_EQ(originalSampleSize + 1, resampledSampleSize); + + const size_t numPointers = resampled.getPointerCount(); + const size_t beginLatestSample = resampledSampleSize - 1; + for (size_t i = 0; i < numPointers; ++i) { + SCOPED_TRACE(i); + EXPECT_EQ(original.getPointerId(i), resampled.getPointerId(i)); + EXPECT_EQ(original.getToolType(i), resampled.getToolType(i)); + + const PointerCoords& resampledCoords = + resampled.getSamplePointerCoords()[beginLatestSample * numPointers + i]; + + EXPECT_TRUE(resampledCoords.isResampled); + EXPECT_NEAR(expectedCoords[i].getX(), resampledCoords.getX(), EPSILON); + EXPECT_NEAR(expectedCoords[i].getY(), resampledCoords.getY(), EPSILON); + } +} + +void ResamplerTest::assertMotionEventIsNotResampled(const MotionEvent& original, + const MotionEvent& notResampled) { + assertMotionEventMetaDataDidNotMutate(original, notResampled); + const size_t originalSampleSize = original.getHistorySize() + 1; + const size_t notResampledSampleSize = notResampled.getHistorySize() + 1; + EXPECT_EQ(originalSampleSize, notResampledSampleSize); +} + +TEST_F(ResamplerTest, NonResampledAxesArePreserved) { + constexpr float TOUCH_MAJOR_VALUE = 1.0f; + + MotionEvent motionEvent = + InputStream{{InputSample{5ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + constexpr std::chrono::nanoseconds eventTime{10ms}; + PointerCoords pointerCoords{}; + pointerCoords.isResampled = false; + pointerCoords.setAxisValue(AMOTION_EVENT_AXIS_X, 2.0f); + pointerCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, 2.0f); + pointerCoords.setAxisValue(AMOTION_EVENT_AXIS_TOUCH_MAJOR, TOUCH_MAJOR_VALUE); + + motionEvent.addSample(eventTime.count(), &pointerCoords, motionEvent.getId()); + + const InputMessage futureSample = + InputSample{15ms, {{.id = 0, .x = 3.0f, .y = 4.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + EXPECT_EQ(motionEvent.getTouchMajor(0), TOUCH_MAJOR_VALUE); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.id = 0, + .x = 2.2f, + .y = 2.4f, + .isResampled = true}}); +} + +TEST_F(ResamplerTest, SinglePointerNotEnoughDataToResample) { + MotionEvent motionEvent = + InputStream{{InputSample{5ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, SinglePointerDifferentDeviceIdBetweenMotionEvents) { + MotionEvent motionFromFirstDevice = + InputStream{{InputSample{4ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}, + InputSample{8ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE, + .deviceId = 0}; + + mResampler->resampleMotionEvent(10ms, motionFromFirstDevice, nullptr); + + MotionEvent motionFromSecondDevice = + InputStream{{InputSample{11ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE, + .deviceId = 1}; + const MotionEvent originalMotionEvent = motionFromSecondDevice; + + mResampler->resampleMotionEvent(12ms, motionFromSecondDevice, nullptr); + // The MotionEvent should not be resampled because the second event came from a different device + // than the previous event. + assertMotionEventIsNotResampled(originalMotionEvent, motionFromSecondDevice); +} + +TEST_F(ResamplerTest, SinglePointerSingleSampleInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 1.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + const InputMessage futureSample = + InputSample{15ms, {{.id = 0, .x = 2.0f, .y = 4.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.id = 0, + .x = 1.2f, + .y = 2.4f, + .isResampled = true}}); +} + +TEST_F(ResamplerTest, SinglePointerDeltaTooSmallInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 1.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + const InputMessage futureSample = + InputSample{11ms, {{.id = 0, .x = 2.0f, .y = 4.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(10'500'000ns, motionEvent, &futureSample); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +/** + * Tests extrapolation given two MotionEvents with a single sample. + */ +TEST_F(ResamplerTest, SinglePointerSingleSampleExtrapolation) { + MotionEvent firstMotionEvent = + InputStream{{InputSample{5ms, {{.id = 0, .x = 1.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + mResampler->resampleMotionEvent(9ms, firstMotionEvent, nullptr); + + MotionEvent secondMotionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 2.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = secondMotionEvent; + + mResampler->resampleMotionEvent(16ms, secondMotionEvent, nullptr); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, secondMotionEvent, + {Pointer{.id = 0, + .x = 2.2f, + .y = 4.4f, + .isResampled = true}}); +} + +TEST_F(ResamplerTest, SinglePointerMultipleSampleInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{5ms, {{.id = 0, .x = 1.0f, .y = 2.0f, .isResampled = false}}}, + InputSample{10ms, + {{.id = 0, .x = 2.0f, .y = 3.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage futureSample = + InputSample{15ms, {{.id = 0, .x = 3.0f, .y = 5.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.id = 0, + .x = 2.2f, + .y = 3.4f, + .isResampled = true}}); +} + +TEST_F(ResamplerTest, SinglePointerMultipleSampleExtrapolation) { + MotionEvent motionEvent = + InputStream{{InputSample{5ms, {{.id = 0, .x = 1.0f, .y = 2.0f, .isResampled = false}}}, + InputSample{10ms, + {{.id = 0, .x = 2.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, nullptr); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.id = 0, + .x = 2.2f, + .y = 4.4f, + .isResampled = true}}); +} + +TEST_F(ResamplerTest, SinglePointerDeltaTooSmallExtrapolation) { + MotionEvent motionEvent = + InputStream{{InputSample{9ms, {{.id = 0, .x = 1.0f, .y = 2.0f, .isResampled = false}}}, + InputSample{10ms, + {{.id = 0, .x = 2.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, nullptr); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, SinglePointerDeltaTooLargeExtrapolation) { + MotionEvent motionEvent = + InputStream{{InputSample{5ms, {{.id = 0, .x = 1.0f, .y = 2.0f, .isResampled = false}}}, + InputSample{26ms, + {{.id = 0, .x = 2.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(32ms, motionEvent, nullptr); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, SinglePointerResampleTimeTooFarExtrapolation) { + MotionEvent motionEvent = + InputStream{{InputSample{5ms, {{.id = 0, .x = 1.0f, .y = 2.0f, .isResampled = false}}}, + InputSample{25ms, + {{.id = 0, .x = 2.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(48ms, motionEvent, nullptr); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.id = 0, + .x = 2.4f, + .y = 4.8f, + .isResampled = true}}); +} + +TEST_F(ResamplerTest, MultiplePointerSingleSampleInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{5ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage futureSample = + InputSample{15ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.x = 2.2f, .y = 2.2f, .isResampled = true}, + Pointer{.x = 3.2f, .y = 3.2f, .isResampled = true}}); +} + +TEST_F(ResamplerTest, MultiplePointerSingleSampleExtrapolation) { + MotionEvent firstMotionEvent = + InputStream{{InputSample{5ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + mResampler->resampleMotionEvent(9ms, firstMotionEvent, /*futureSample=*/nullptr); + + MotionEvent secondMotionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = secondMotionEvent; + + mResampler->resampleMotionEvent(16ms, secondMotionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, secondMotionEvent, + {Pointer{.x = 3.4f, .y = 3.4f, .isResampled = true}, + Pointer{.x = 4.4f, .y = 4.4f, .isResampled = true}}); +} + +TEST_F(ResamplerTest, MultiplePointerMultipleSampleInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{5ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}, + InputSample{10ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + const InputMessage futureSample = + InputSample{15ms, + {{.id = 0, .x = 5.0f, .y = 5.0f, .isResampled = false}, + {.id = 1, .x = 6.0f, .y = 6.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.x = 3.4f, .y = 3.4f, .isResampled = true}, + Pointer{.x = 4.4f, .y = 4.4f, .isResampled = true}}); +} + +TEST_F(ResamplerTest, MultiplePointerMultipleSampleExtrapolation) { + MotionEvent motionEvent = + InputStream{{InputSample{5ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}, + InputSample{10ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.x = 3.4f, .y = 3.4f, .isResampled = true}, + Pointer{.x = 4.4f, .y = 4.4f, .isResampled = true}}); +} + +TEST_F(ResamplerTest, MultiplePointerIncreaseNumPointersInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage futureSample = + InputSample{15ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 2, .x = 5.0f, .y = 5.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + {Pointer{.x = 1.4f, .y = 1.4f, .isResampled = true}, + Pointer{.x = 2.4f, .y = 2.4f, .isResampled = true}}); + + MotionEvent secondMotionEvent = + InputStream{{InputSample{25ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 2, .x = 5.0f, .y = 5.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage secondFutureSample = + InputSample{30ms, + {{.id = 0, .x = 5.0f, .y = 5.0f, .isResampled = false}, + {.id = 1, .x = 6.0f, .y = 6.0f, .isResampled = false}, + {.id = 2, .x = 7.0f, .y = 7.0f, .isResampled = false}}}; + + const MotionEvent originalSecondMotionEvent = secondMotionEvent; + + mResampler->resampleMotionEvent(32ms, secondMotionEvent, &secondFutureSample); + + assertMotionEventIsResampledAndCoordsNear(originalSecondMotionEvent, secondMotionEvent, + {Pointer{.x = 3.8f, .y = 3.8f, .isResampled = true}, + Pointer{.x = 4.8f, .y = 4.8f, .isResampled = true}, + Pointer{.x = 5.8f, .y = 5.8f, .isResampled = true}}); +} + +TEST_F(ResamplerTest, MultiplePointerIncreaseNumPointersExtrapolation) { + MotionEvent firstMotionEvent = + InputStream{{InputSample{5ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + mResampler->resampleMotionEvent(9ms, firstMotionEvent, /*futureSample=*/nullptr); + + MotionEvent secondMotionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 2, .x = 5.0f, .y = 5.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent secondOriginalMotionEvent = secondMotionEvent; + + mResampler->resampleMotionEvent(16ms, secondMotionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsNotResampled(secondOriginalMotionEvent, secondMotionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerDecreaseNumPointersInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 2, .x = 5.0f, .y = 5.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage futureSample = + InputSample{15ms, + {{.id = 0, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 1, .x = 5.0f, .y = 5.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerDecreaseNumPointersExtrapolation) { + MotionEvent firstMotionEvent = + InputStream{{InputSample{5ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}, + {.id = 2, .x = 3.0f, .y = 3.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + mResampler->resampleMotionEvent(9ms, firstMotionEvent, /*futureSample=*/nullptr); + + MotionEvent secondMotionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}, + {.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent secondOriginalMotionEvent = secondMotionEvent; + + mResampler->resampleMotionEvent(16ms, secondMotionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsResampledAndCoordsNear(secondOriginalMotionEvent, secondMotionEvent, + {Pointer{.x = 3.4f, .y = 3.4f, .isResampled = true}, + Pointer{.x = 4.4f, .y = 4.4f, .isResampled = true}}); +} + +TEST_F(ResamplerTest, MultiplePointerDifferentIdOrderInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage futureSample = + InputSample{15ms, + {{.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerDifferentIdOrderExtrapolation) { + MotionEvent firstMotionEvent = + InputStream{{InputSample{5ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + mResampler->resampleMotionEvent(9ms, firstMotionEvent, /*futureSample=*/nullptr); + + MotionEvent secondMotionEvent = + InputStream{{InputSample{10ms, + {{.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent secondOriginalMotionEvent = secondMotionEvent; + + mResampler->resampleMotionEvent(16ms, secondMotionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsNotResampled(secondOriginalMotionEvent, secondMotionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerDifferentIdsInterpolation) { + MotionEvent motionEvent = + InputStream{{InputSample{10ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage futureSample = + InputSample{15ms, + {{.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 2, .x = 3.0f, .y = 3.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerDifferentIdsExtrapolation) { + MotionEvent firstMotionEvent = + InputStream{{InputSample{5ms, + {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}, + {.id = 1, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + mResampler->resampleMotionEvent(9ms, firstMotionEvent, /*futureSample=*/nullptr); + + MotionEvent secondMotionEvent = + InputStream{{InputSample{10ms, + {{.id = 1, .x = 4.0f, .y = 4.0f, .isResampled = false}, + {.id = 2, .x = 3.0f, .y = 3.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent secondOriginalMotionEvent = secondMotionEvent; + + mResampler->resampleMotionEvent(16ms, secondMotionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsNotResampled(secondOriginalMotionEvent, secondMotionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerDifferentToolTypeInterpolation) { + MotionEvent motionEvent = InputStream{{InputSample{10ms, + {{.id = 0, + .toolType = ToolType::FINGER, + .x = 1.0f, + .y = 1.0f, + .isResampled = false}, + {.id = 1, + .toolType = ToolType::FINGER, + .x = 2.0f, + .y = 2.0f, + .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage futureSample = InputSample{15ms, + {{.id = 0, + .toolType = ToolType::FINGER, + .x = 3.0, + .y = 3.0, + .isResampled = false}, + {.id = 1, + .toolType = ToolType::STYLUS, + .x = 4.0, + .y = 4.0, + .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, &futureSample); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerDifferentToolTypeExtrapolation) { + MotionEvent firstMotionEvent = InputStream{{InputSample{5ms, + {{.id = 0, + .toolType = ToolType::FINGER, + .x = 1.0f, + .y = 1.0f, + .isResampled = false}, + {.id = 1, + .toolType = ToolType::FINGER, + .x = 2.0f, + .y = 2.0f, + .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + mResampler->resampleMotionEvent(9ms, firstMotionEvent, /*futureSample=*/nullptr); + + MotionEvent secondMotionEvent = InputStream{{InputSample{10ms, + {{.id = 0, + .toolType = ToolType::FINGER, + .x = 1.0f, + .y = 1.0f, + .isResampled = false}, + {.id = 1, + .toolType = ToolType::STYLUS, + .x = 2.0f, + .y = 2.0f, + .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent secondOriginalMotionEvent = secondMotionEvent; + + mResampler->resampleMotionEvent(16ms, secondMotionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsNotResampled(secondOriginalMotionEvent, secondMotionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerShouldNotResampleToolTypeInterpolation) { + MotionEvent motionEvent = InputStream{{InputSample{10ms, + {{.id = 0, + .toolType = ToolType::PALM, + .x = 1.0f, + .y = 1.0f, + .isResampled = false}, + {.id = 1, + .toolType = ToolType::PALM, + .x = 2.0f, + .y = 2.0f, + .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const InputMessage futureSample = InputSample{15ms, + {{.id = 0, + .toolType = ToolType::PALM, + .x = 3.0, + .y = 3.0, + .isResampled = false}, + {.id = 1, + .toolType = ToolType::PALM, + .x = 4.0, + .y = 4.0, + .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, MultiplePointerShouldNotResampleToolTypeExtrapolation) { + MotionEvent motionEvent = InputStream{{InputSample{5ms, + {{.id = 0, + .toolType = ToolType::PALM, + .x = 1.0f, + .y = 1.0f, + .isResampled = false}, + {.id = 1, + .toolType = ToolType::PALM, + .x = 2.0f, + .y = 2.0f, + .isResampled = false}}}, + InputSample{10ms, + {{.id = 0, + .toolType = ToolType::PALM, + .x = 3.0f, + .y = 3.0f, + .isResampled = false}, + {.id = 1, + .toolType = ToolType::PALM, + .x = 4.0f, + .y = 4.0f, + .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(16ms, motionEvent, /*futureSample=*/nullptr); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} +} // namespace android diff --git a/libs/input/tests/TestEventMatchers.h b/libs/input/tests/TestEventMatchers.h new file mode 100644 index 0000000000..dd2e40c025 --- /dev/null +++ b/libs/input/tests/TestEventMatchers.h @@ -0,0 +1,110 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <ostream> + +#include <input/Input.h> + +namespace android { + +/** + * This file contains a copy of Matchers from .../inputflinger/tests/TestEventMatchers.h. Ideally, + * implementations must not be duplicated. + * TODO(b/365606513): Find a way to share TestEventMatchers.h between inputflinger and libinput. + */ + +class WithDeviceIdMatcher { +public: + using is_gtest_matcher = void; + explicit WithDeviceIdMatcher(DeviceId deviceId) : mDeviceId(deviceId) {} + + bool MatchAndExplain(const InputEvent& event, std::ostream*) const { + return mDeviceId == event.getDeviceId(); + } + + void DescribeTo(std::ostream* os) const { *os << "with device id " << mDeviceId; } + + void DescribeNegationTo(std::ostream* os) const { *os << "wrong device id"; } + +private: + const DeviceId mDeviceId; +}; + +inline WithDeviceIdMatcher WithDeviceId(int32_t deviceId) { + return WithDeviceIdMatcher(deviceId); +} + +class WithMotionActionMatcher { +public: + using is_gtest_matcher = void; + explicit WithMotionActionMatcher(int32_t action) : mAction(action) {} + + bool MatchAndExplain(const MotionEvent& event, std::ostream*) const { + bool matches = mAction == event.getAction(); + if (event.getAction() == AMOTION_EVENT_ACTION_CANCEL) { + matches &= (event.getFlags() & AMOTION_EVENT_FLAG_CANCELED) != 0; + } + return matches; + } + + void DescribeTo(std::ostream* os) const { + *os << "with motion action " << MotionEvent::actionToString(mAction); + if (mAction == AMOTION_EVENT_ACTION_CANCEL) { + *os << " and FLAG_CANCELED"; + } + } + + void DescribeNegationTo(std::ostream* os) const { *os << "wrong action"; } + +private: + const int32_t mAction; +}; + +inline WithMotionActionMatcher WithMotionAction(int32_t action) { + return WithMotionActionMatcher(action); +} + +class MotionEventIsResampledMatcher { +public: + using is_gtest_matcher = void; + + bool MatchAndExplain(const MotionEvent& motionEvent, std::ostream*) const { + const size_t numSamples = motionEvent.getHistorySize() + 1; + const size_t numPointers = motionEvent.getPointerCount(); + if (numPointers <= 0 || numSamples <= 0) { + return false; + } + for (size_t i = 0; i < numPointers; ++i) { + const PointerCoords& pointerCoords = + motionEvent.getSamplePointerCoords()[numSamples * numPointers + i]; + if (!pointerCoords.isResampled) { + return false; + } + } + return true; + } + + void DescribeTo(std::ostream* os) const { *os << "MotionEvent is resampled."; } + + void DescribeNegationTo(std::ostream* os) const { *os << "MotionEvent is not resampled."; } +}; + +inline MotionEventIsResampledMatcher MotionEventIsResampled() { + return MotionEventIsResampledMatcher(); +} +} // namespace android diff --git a/libs/input/tests/TestInputChannel.cpp b/libs/input/tests/TestInputChannel.cpp new file mode 100644 index 0000000000..26a0ca2106 --- /dev/null +++ b/libs/input/tests/TestInputChannel.cpp @@ -0,0 +1,102 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "TestInputChannel" +#define ATRACE_TAG ATRACE_TAG_INPUT + +#include <TestInputChannel.h> + +#include <sys/socket.h> +#include <unistd.h> + +#include <array> + +#include <android-base/logging.h> +#include <android-base/unique_fd.h> +#include <binder/IBinder.h> +#include <utils/StrongPointer.h> + +namespace android { + +namespace { + +/** + * Returns a stub file descriptor by opening a socket pair and closing one of the fds. The returned + * fd can be used to construct an InputChannel. + */ +base::unique_fd generateFileDescriptor() { + std::array<int, 2> kFileDescriptors; + LOG_IF(FATAL, ::socketpair(AF_UNIX, SOCK_SEQPACKET, 0, kFileDescriptors.data()) != 0) + << "TestInputChannel. Failed to create socket pair."; + LOG_IF(FATAL, ::close(kFileDescriptors[1]) != 0) + << "TestInputChannel. Failed to close file descriptor."; + return base::unique_fd{kFileDescriptors[0]}; +} +} // namespace + +// --- TestInputChannel --- + +TestInputChannel::TestInputChannel(const std::string& name) + : InputChannel{name, generateFileDescriptor(), sp<BBinder>::make()} {} + +void TestInputChannel::enqueueMessage(const InputMessage& message) { + mReceivedMessages.push(message); +} + +status_t TestInputChannel::sendMessage(const InputMessage* message) { + LOG_IF(FATAL, message == nullptr) + << "TestInputChannel " << getName() << ". No message was passed to sendMessage."; + + mSentMessages.push(*message); + return OK; +} + +base::Result<InputMessage> TestInputChannel::receiveMessage() { + if (mReceivedMessages.empty()) { + return base::Error(WOULD_BLOCK); + } + InputMessage message = mReceivedMessages.front(); + mReceivedMessages.pop(); + return message; +} + +bool TestInputChannel::probablyHasInput() const { + return !mReceivedMessages.empty(); +} + +void TestInputChannel::assertFinishMessage(uint32_t seq, bool handled) { + ASSERT_FALSE(mSentMessages.empty()) + << "TestInputChannel " << getName() << ". Cannot assert. mSentMessages is empty."; + + const InputMessage& finishMessage = mSentMessages.front(); + + EXPECT_EQ(finishMessage.header.seq, seq) + << "TestInputChannel " << getName() + << ". Sequence mismatch. Message seq: " << finishMessage.header.seq + << " Expected seq: " << seq; + + EXPECT_EQ(finishMessage.body.finished.handled, handled) + << "TestInputChannel " << getName() + << ". Handled value mismatch. Message val: " << std::boolalpha + << finishMessage.body.finished.handled << "Expected val: " << handled + << std::noboolalpha; + mSentMessages.pop(); +} + +void TestInputChannel::assertNoSentMessages() const { + ASSERT_TRUE(mSentMessages.empty()); +} +} // namespace android
\ No newline at end of file diff --git a/libs/input/tests/TestInputChannel.h b/libs/input/tests/TestInputChannel.h new file mode 100644 index 0000000000..43253ec0ef --- /dev/null +++ b/libs/input/tests/TestInputChannel.h @@ -0,0 +1,66 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <queue> +#include <string> + +#include <android-base/result.h> +#include <gtest/gtest.h> +#include <input/InputTransport.h> +#include <utils/Errors.h> + +namespace android { + +class TestInputChannel final : public InputChannel { +public: + explicit TestInputChannel(const std::string& name); + + /** + * Enqueues a message in mReceivedMessages. + */ + void enqueueMessage(const InputMessage& message); + + /** + * Pushes message to mSentMessages. In the default implementation, InputChannel sends messages + * through a file descriptor. TestInputChannel, on the contrary, stores sent messages in + * mSentMessages for assertion reasons. + */ + status_t sendMessage(const InputMessage* message) override; + + /** + * Returns an InputMessage from mReceivedMessages. This is done instead of retrieving data + * directly from fd. + */ + base::Result<InputMessage> receiveMessage() override; + + /** + * Returns if mReceivedMessages is not empty. + */ + bool probablyHasInput() const override; + + void assertFinishMessage(uint32_t seq, bool handled); + + void assertNoSentMessages() const; + +private: + // InputMessages received by the endpoint. + std::queue<InputMessage> mReceivedMessages; + // InputMessages sent by the endpoint. + std::queue<InputMessage> mSentMessages; +}; +} // namespace android diff --git a/libs/nativedisplay/ADisplay.cpp b/libs/nativedisplay/ADisplay.cpp index e3be3bc8f8..d0ca78e658 100644 --- a/libs/nativedisplay/ADisplay.cpp +++ b/libs/nativedisplay/ADisplay.cpp @@ -129,7 +129,7 @@ int ADisplay_acquirePhysicalDisplays(ADisplay*** outDisplays) { std::vector<DisplayConfigImpl> modesPerDisplay[size]; ui::DisplayConnectionType displayConnectionTypes[size]; int numModes = 0; - for (int i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { ui::StaticDisplayInfo staticInfo; if (const status_t status = SurfaceComposerClient::getStaticDisplayInfo(ids[i].value, &staticInfo); @@ -151,7 +151,7 @@ int ADisplay_acquirePhysicalDisplays(ADisplay*** outDisplays) { numModes += modes.size(); modesPerDisplay[i].reserve(modes.size()); - for (int j = 0; j < modes.size(); ++j) { + for (size_t j = 0; j < modes.size(); ++j) { const ui::DisplayMode& mode = modes[j]; modesPerDisplay[i].emplace_back( DisplayConfigImpl{static_cast<size_t>(mode.id), mode.resolution.getWidth(), @@ -224,7 +224,7 @@ float ADisplay_getMaxSupportedFps(ADisplay* display) { CHECK_NOT_NULL(display); DisplayImpl* impl = reinterpret_cast<DisplayImpl*>(display); float maxFps = 0.0; - for (int i = 0; i < impl->numConfigs; ++i) { + for (size_t i = 0; i < impl->numConfigs; ++i) { maxFps = std::max(maxFps, impl->configs[i].fps); } return maxFps; @@ -261,7 +261,7 @@ int ADisplay_getCurrentConfig(ADisplay* display, ADisplayConfig** outConfig) { for (size_t i = 0; i < impl->numConfigs; i++) { auto* config = impl->configs + i; - if (config->id == info.activeDisplayModeId) { + if (info.activeDisplayModeId >= 0 && config->id == (size_t)info.activeDisplayModeId) { *outConfig = reinterpret_cast<ADisplayConfig*>(config); return OK; } diff --git a/libs/nativedisplay/include/surfacetexture/SurfaceTexture.h b/libs/nativedisplay/include/surfacetexture/SurfaceTexture.h index 099f47dbe1..f1453bd64d 100644 --- a/libs/nativedisplay/include/surfacetexture/SurfaceTexture.h +++ b/libs/nativedisplay/include/surfacetexture/SurfaceTexture.h @@ -98,11 +98,25 @@ public: * is created in a detached state, and attachToContext must be called before * calls to updateTexImage. */ +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + SurfaceTexture(uint32_t tex, uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); + + SurfaceTexture(uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); + + SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, + bool useFenceSync, bool isControlledByApp) + __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); + + SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, + bool isControlledByApp) + __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); +#else SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) /** * updateTexImage acquires the most recently queued buffer, and sets the @@ -499,6 +513,8 @@ protected: friend class EGLConsumer; private: + void initialize(); + // Proxy listener to avoid having SurfaceTexture directly implement FrameAvailableListener as it // is extending ConsumerBase which also implements FrameAvailableListener. class FrameAvailableListenerProxy : public ConsumerBase::FrameAvailableListener { diff --git a/libs/nativedisplay/surfacetexture/SurfaceTexture.cpp b/libs/nativedisplay/surfacetexture/SurfaceTexture.cpp index 3a09204878..ce232cc4c7 100644 --- a/libs/nativedisplay/surfacetexture/SurfaceTexture.cpp +++ b/libs/nativedisplay/surfacetexture/SurfaceTexture.cpp @@ -35,6 +35,49 @@ namespace android { static const mat4 mtxIdentity; +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) +SurfaceTexture::SurfaceTexture(uint32_t tex, uint32_t texTarget, bool useFenceSync, + bool isControlledByApp) + : ConsumerBase(isControlledByApp), + mCurrentCrop(Rect::EMPTY_RECT), + mCurrentTransform(0), + mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE), + mCurrentFence(Fence::NO_FENCE), + mCurrentTimestamp(0), + mCurrentDataSpace(HAL_DATASPACE_UNKNOWN), + mCurrentFrameNumber(0), + mDefaultWidth(1), + mDefaultHeight(1), + mFilteringEnabled(true), + mTexName(tex), + mUseFenceSync(useFenceSync), + mTexTarget(texTarget), + mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), + mOpMode(OpMode::attachedToGL) { + initialize(); +} + +SurfaceTexture::SurfaceTexture(uint32_t texTarget, bool useFenceSync, bool isControlledByApp) + : ConsumerBase(isControlledByApp), + mCurrentCrop(Rect::EMPTY_RECT), + mCurrentTransform(0), + mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE), + mCurrentFence(Fence::NO_FENCE), + mCurrentTimestamp(0), + mCurrentDataSpace(HAL_DATASPACE_UNKNOWN), + mCurrentFrameNumber(0), + mDefaultWidth(1), + mDefaultHeight(1), + mFilteringEnabled(true), + mTexName(0), + mUseFenceSync(useFenceSync), + mTexTarget(texTarget), + mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), + mOpMode(OpMode::detached) { + initialize(); +} +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + SurfaceTexture::SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t texTarget, bool useFenceSync, bool isControlledByApp) : ConsumerBase(bq, isControlledByApp), @@ -53,11 +96,7 @@ SurfaceTexture::SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t te mTexTarget(texTarget), mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), mOpMode(OpMode::attachedToGL) { - SFT_LOGV("SurfaceTexture"); - - memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix)); - - mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS); + initialize(); } SurfaceTexture::SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t texTarget, @@ -78,11 +117,7 @@ SurfaceTexture::SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t te mTexTarget(texTarget), mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), mOpMode(OpMode::detached) { - SFT_LOGV("SurfaceTexture"); - - memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix)); - - mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS); + initialize(); } status_t SurfaceTexture::setDefaultBufferSize(uint32_t w, uint32_t h) { @@ -531,4 +566,12 @@ void SurfaceTexture::onSetFrameRate(float frameRate, int8_t compatibility, } #endif +void SurfaceTexture::initialize() { + SFT_LOGV("SurfaceTexture"); + + memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix)); + + mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS); +} + } // namespace android diff --git a/libs/nativewindow/AHardwareBuffer.cpp b/libs/nativewindow/AHardwareBuffer.cpp index dd78049b16..ca41346d46 100644 --- a/libs/nativewindow/AHardwareBuffer.cpp +++ b/libs/nativewindow/AHardwareBuffer.cpp @@ -196,10 +196,10 @@ int AHardwareBuffer_lockAndGetInfo(AHardwareBuffer* buffer, uint64_t usage, return BAD_VALUE; } - if (usage & ~(AHARDWAREBUFFER_USAGE_CPU_READ_MASK | - AHARDWAREBUFFER_USAGE_CPU_WRITE_MASK)) { + if (usage & ~(AHARDWAREBUFFER_USAGE_CPU_READ_MASK | AHARDWAREBUFFER_USAGE_CPU_WRITE_MASK) || + usage == 0) { ALOGE("Invalid usage flags passed to AHardwareBuffer_lock; only " - "AHARDWAREBUFFER_USAGE_CPU_* flags are allowed"); + "AHARDWAREBUFFER_USAGE_CPU_* flags are allowed"); return BAD_VALUE; } @@ -248,10 +248,10 @@ int AHardwareBuffer_lock(AHardwareBuffer* buffer, uint64_t usage, if (!buffer) return BAD_VALUE; - if (usage & ~(AHARDWAREBUFFER_USAGE_CPU_READ_MASK | - AHARDWAREBUFFER_USAGE_CPU_WRITE_MASK)) { + if (usage & ~(AHARDWAREBUFFER_USAGE_CPU_READ_MASK | AHARDWAREBUFFER_USAGE_CPU_WRITE_MASK) || + usage == 0) { ALOGE("Invalid usage flags passed to AHardwareBuffer_lock; only " - "AHARDWAREBUFFER_USAGE_CPU_* flags are allowed"); + "AHARDWAREBUFFER_USAGE_CPU_* flags are allowed"); return BAD_VALUE; } @@ -277,10 +277,10 @@ int AHardwareBuffer_lockPlanes(AHardwareBuffer* buffer, uint64_t usage, int32_t fence, const ARect* rect, AHardwareBuffer_Planes* outPlanes) { if (!buffer || !outPlanes) return BAD_VALUE; - if (usage & ~(AHARDWAREBUFFER_USAGE_CPU_READ_MASK | - AHARDWAREBUFFER_USAGE_CPU_WRITE_MASK)) { + if (usage & ~(AHARDWAREBUFFER_USAGE_CPU_READ_MASK | AHARDWAREBUFFER_USAGE_CPU_WRITE_MASK) || + usage == 0) { ALOGE("Invalid usage flags passed to AHardwareBuffer_lock; only " - " AHARDWAREBUFFER_USAGE_CPU_* flags are allowed"); + " AHARDWAREBUFFER_USAGE_CPU_* flags are allowed"); return BAD_VALUE; } diff --git a/libs/nativewindow/tests/ANativeWindowTest.cpp b/libs/nativewindow/tests/ANativeWindowTest.cpp index 6cf8291da2..937ff02241 100644 --- a/libs/nativewindow/tests/ANativeWindowTest.cpp +++ b/libs/nativewindow/tests/ANativeWindowTest.cpp @@ -50,9 +50,14 @@ protected: const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); ALOGV("**** Setting up for %s.%s\n", test_info->test_case_name(), test_info->name()); +#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) + mItemConsumer = new BufferItemConsumer(GRALLOC_USAGE_SW_READ_OFTEN); + mWindow = new TestableSurface(mItemConsumer->getSurface()->getIGraphicBufferProducer()); +#else BufferQueue::createBufferQueue(&mProducer, &mConsumer); mItemConsumer = new BufferItemConsumer(mConsumer, GRALLOC_USAGE_SW_READ_OFTEN); mWindow = new TestableSurface(mProducer); +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) const int success = native_window_api_connect(mWindow.get(), NATIVE_WINDOW_API_CPU); EXPECT_EQ(0, success); } @@ -64,10 +69,12 @@ protected: const int success = native_window_api_disconnect(mWindow.get(), NATIVE_WINDOW_API_CPU); EXPECT_EQ(0, success); } + +#if !COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) sp<IGraphicBufferProducer> mProducer; sp<IGraphicBufferConsumer> mConsumer; +#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) sp<BufferItemConsumer> mItemConsumer; - sp<TestableSurface> mWindow; }; diff --git a/libs/renderengine/Android.bp b/libs/renderengine/Android.bp index 4a04467308..d248ea0b84 100644 --- a/libs/renderengine/Android.bp +++ b/libs/renderengine/Android.bp @@ -100,11 +100,14 @@ filegroup { "skia/debug/SkiaCapture.cpp", "skia/debug/SkiaMemoryReporter.cpp", "skia/filters/BlurFilter.cpp", + "skia/filters/GainmapFactory.cpp", "skia/filters/GaussianBlurFilter.cpp", + "skia/filters/KawaseBlurDualFilter.cpp", "skia/filters/KawaseBlurFilter.cpp", "skia/filters/LinearEffect.cpp", "skia/filters/MouriMap.cpp", "skia/filters/StretchShaderFactory.cpp", + "skia/filters/EdgeExtensionShaderFactory.cpp", ], } diff --git a/libs/renderengine/ExternalTexture.cpp b/libs/renderengine/ExternalTexture.cpp index 6f2a96a87b..8d0fbba2e0 100644 --- a/libs/renderengine/ExternalTexture.cpp +++ b/libs/renderengine/ExternalTexture.cpp @@ -14,11 +14,11 @@ * limitations under the License. */ +#include <common/trace.h> #include <log/log.h> #include <renderengine/RenderEngine.h> #include <renderengine/impl/ExternalTexture.h> #include <ui/GraphicBuffer.h> -#include <utils/Trace.h> namespace android::renderengine::impl { @@ -35,7 +35,7 @@ ExternalTexture::~ExternalTexture() { } void ExternalTexture::remapBuffer() { - ATRACE_CALL(); + SFTRACE_CALL(); { auto buf = mBuffer; mRenderEngine.unmapExternalTextureBuffer(std::move(buf)); diff --git a/libs/renderengine/RenderEngine.cpp b/libs/renderengine/RenderEngine.cpp index bc3976d9f1..907590a236 100644 --- a/libs/renderengine/RenderEngine.cpp +++ b/libs/renderengine/RenderEngine.cpp @@ -21,6 +21,7 @@ #include "skia/GraphiteVkRenderEngine.h" #include "skia/SkiaGLRenderEngine.h" #include "threaded/RenderEngineThreaded.h" +#include "ui/GraphicTypes.h" #include <com_android_graphics_surfaceflinger_flags.h> #include <cutils/properties.h> @@ -101,17 +102,34 @@ ftl::Future<FenceResult> RenderEngine::drawLayers(const DisplaySettings& display base::unique_fd&& bufferFence) { const auto resultPromise = std::make_shared<std::promise<FenceResult>>(); std::future<FenceResult> resultFuture = resultPromise->get_future(); - updateProtectedContext(layers, buffer); + updateProtectedContext(layers, {buffer.get()}); drawLayersInternal(std::move(resultPromise), display, layers, buffer, std::move(bufferFence)); return resultFuture; } +ftl::Future<FenceResult> RenderEngine::drawGainmap( + const std::shared_ptr<ExternalTexture>& sdr, base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, base::borrowed_fd&& hdrFence, + float hdrSdrRatio, ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap) { + const auto resultPromise = std::make_shared<std::promise<FenceResult>>(); + std::future<FenceResult> resultFuture = resultPromise->get_future(); + updateProtectedContext({}, {sdr.get(), hdr.get(), gainmap.get()}); + drawGainmapInternal(std::move(resultPromise), sdr, std::move(sdrFence), hdr, + std::move(hdrFence), hdrSdrRatio, dataspace, gainmap); + return resultFuture; +} + void RenderEngine::updateProtectedContext(const std::vector<LayerSettings>& layers, - const std::shared_ptr<ExternalTexture>& buffer) { + vector<const ExternalTexture*> buffers) { const bool needsProtectedContext = - (buffer && (buffer->getUsage() & GRALLOC_USAGE_PROTECTED)) || - std::any_of(layers.begin(), layers.end(), [](const LayerSettings& layer) { - const std::shared_ptr<ExternalTexture>& buffer = layer.source.buffer.buffer; + std::any_of(layers.begin(), layers.end(), + [](const LayerSettings& layer) { + const std::shared_ptr<ExternalTexture>& buffer = + layer.source.buffer.buffer; + return buffer && (buffer->getUsage() & GRALLOC_USAGE_PROTECTED); + }) || + std::any_of(buffers.begin(), buffers.end(), [](const ExternalTexture* buffer) { return buffer && (buffer->getUsage() & GRALLOC_USAGE_PROTECTED); }); useProtectedContext(needsProtectedContext); diff --git a/libs/renderengine/benchmark/Android.bp b/libs/renderengine/benchmark/Android.bp index e1a6f6af0d..f84db0b04c 100644 --- a/libs/renderengine/benchmark/Android.bp +++ b/libs/renderengine/benchmark/Android.bp @@ -57,6 +57,7 @@ cc_benchmark { "libui", "libutils", "server_configurable_flags", + "libtracing_perfetto", ], data: ["resources/*"], diff --git a/libs/renderengine/benchmark/AndroidTest.xml b/libs/renderengine/benchmark/AndroidTest.xml new file mode 100644 index 0000000000..3b923cb3a8 --- /dev/null +++ b/libs/renderengine/benchmark/AndroidTest.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright 2024 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Config for librenderengine_bench."> + <option name="test-suite-tag" value="apct" /> + <option name="test-suite-tag" value="apct-native-metric" /> + + <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer"> + <option name="cleanup" value="true" /> + <option name="push" value="librenderengine_bench->/data/local/tmp/librenderengine_bench" /> + </target_preparer> + <test class="com.android.tradefed.testtype.GoogleBenchmarkTest" > + <option name="native-benchmark-device-path" value="/data/local/tmp" /> + <option name="benchmark-module-name" value="librenderengine_bench" /> + <option name="file-exclusion-filter-regex" value=".*\.config$" /> + <option name="file-exclusion-filter-regex" value=".*/resources/.*" /> + </test> +</configuration> diff --git a/libs/renderengine/benchmark/RenderEngineBench.cpp b/libs/renderengine/benchmark/RenderEngineBench.cpp index 05a2063423..a9264b3914 100644 --- a/libs/renderengine/benchmark/RenderEngineBench.cpp +++ b/libs/renderengine/benchmark/RenderEngineBench.cpp @@ -29,6 +29,16 @@ using namespace android; using namespace android::renderengine; +// To run tests: +/** + * mmm frameworks/native/libs/renderengine/benchmark;\ + * adb push $OUT/data/benchmarktest/librenderengine_bench/librenderengine_bench + * /data/benchmarktest/librenderengine_bench/librenderengine_bench;\ + * adb shell /data/benchmarktest/librenderengine_bench/librenderengine_bench + * + * (64-bit devices: out directory contains benchmarktest64 instead of benchmarktest) + */ + /////////////////////////////////////////////////////////////////////////////// // Helpers for calling drawLayers /////////////////////////////////////////////////////////////////////////////// @@ -64,14 +74,15 @@ std::pair<uint32_t, uint32_t> getDisplaySize() { return std::pair<uint32_t, uint32_t>(width, height); } -static std::unique_ptr<RenderEngine> createRenderEngine(RenderEngine::Threaded threaded, - RenderEngine::GraphicsApi graphicsApi) { +static std::unique_ptr<RenderEngine> createRenderEngine( + RenderEngine::Threaded threaded, RenderEngine::GraphicsApi graphicsApi, + RenderEngine::BlurAlgorithm blurAlgorithm = RenderEngine::BlurAlgorithm::KAWASE) { auto args = RenderEngineCreationArgs::Builder() .setPixelFormat(static_cast<int>(ui::PixelFormat::RGBA_8888)) .setImageCacheSize(1) .setEnableProtectedContext(true) .setPrecacheToneMapperShaderOnly(false) - .setBlurAlgorithm(renderengine::RenderEngine::BlurAlgorithm::KAWASE) + .setBlurAlgorithm(blurAlgorithm) .setContextPriority(RenderEngine::ContextPriority::REALTIME) .setThreaded(threaded) .setGraphicsApi(graphicsApi) @@ -172,28 +183,67 @@ static void benchDrawLayers(RenderEngine& re, const std::vector<LayerSettings>& } } +/** + * Return a buffer with the image in the provided path, relative to the executable directory + */ +static std::shared_ptr<ExternalTexture> createTexture(RenderEngine& re, const char* relPathImg) { + // Initially use cpu access so we can decode into it with AImageDecoder. + auto [width, height] = getDisplaySize(); + auto srcBuffer = + allocateBuffer(re, width, height, GRALLOC_USAGE_SW_WRITE_OFTEN, "decoded_source"); + std::string fileName = base::GetExecutableDirectory().append(relPathImg); + renderenginebench::decode(fileName.c_str(), srcBuffer->getBuffer()); + // Now copy into GPU-only buffer for more realistic timing. + srcBuffer = copyBuffer(re, srcBuffer, 0, "source"); + return srcBuffer; +} + /////////////////////////////////////////////////////////////////////////////// // Benchmarks /////////////////////////////////////////////////////////////////////////////// +constexpr char kHomescreenPath[] = "/resources/homescreen.png"; + +/** + * Draw a layer with texture and no additional shaders as a baseline to evaluate a shader's impact + * on performance + */ template <class... Args> -void BM_blur(benchmark::State& benchState, Args&&... args) { +void BM_homescreen(benchmark::State& benchState, Args&&... args) { auto args_tuple = std::make_tuple(std::move(args)...); auto re = createRenderEngine(static_cast<RenderEngine::Threaded>(std::get<0>(args_tuple)), static_cast<RenderEngine::GraphicsApi>(std::get<1>(args_tuple))); - // Initially use cpu access so we can decode into it with AImageDecoder. auto [width, height] = getDisplaySize(); - auto srcBuffer = - allocateBuffer(*re, width, height, GRALLOC_USAGE_SW_WRITE_OFTEN, "decoded_source"); - { - std::string srcImage = base::GetExecutableDirectory(); - srcImage.append("/resources/homescreen.png"); - renderenginebench::decode(srcImage.c_str(), srcBuffer->getBuffer()); - - // Now copy into GPU-only buffer for more realistic timing. - srcBuffer = copyBuffer(*re, srcBuffer, 0, "source"); - } + auto srcBuffer = createTexture(*re, kHomescreenPath); + + const FloatRect layerRect(0, 0, width, height); + LayerSettings layer{ + .geometry = + Geometry{ + .boundaries = layerRect, + }, + .source = + PixelSource{ + .buffer = + Buffer{ + .buffer = srcBuffer, + }, + }, + .alpha = half(1.0f), + }; + auto layers = std::vector<LayerSettings>{layer}; + benchDrawLayers(*re, layers, benchState, "homescreen"); +} + +template <class... Args> +void BM_homescreen_blur(benchmark::State& benchState, Args&&... args) { + auto args_tuple = std::make_tuple(std::move(args)...); + auto re = createRenderEngine(static_cast<RenderEngine::Threaded>(std::get<0>(args_tuple)), + static_cast<RenderEngine::GraphicsApi>(std::get<1>(args_tuple))); + + auto [width, height] = getDisplaySize(); + auto srcBuffer = createTexture(*re, kHomescreenPath); const FloatRect layerRect(0, 0, width, height); LayerSettings layer{ @@ -221,8 +271,55 @@ void BM_blur(benchmark::State& benchState, Args&&... args) { }; auto layers = std::vector<LayerSettings>{layer, blurLayer}; - benchDrawLayers(*re, layers, benchState, "blurred"); + benchDrawLayers(*re, layers, benchState, "homescreen_blurred"); } -BENCHMARK_CAPTURE(BM_blur, SkiaGLThreaded, RenderEngine::Threaded::YES, +template <class... Args> +void BM_homescreen_edgeExtension(benchmark::State& benchState, Args&&... args) { + auto args_tuple = std::make_tuple(std::move(args)...); + auto re = createRenderEngine(static_cast<RenderEngine::Threaded>(std::get<0>(args_tuple)), + static_cast<RenderEngine::GraphicsApi>(std::get<1>(args_tuple))); + + auto [width, height] = getDisplaySize(); + auto srcBuffer = createTexture(*re, kHomescreenPath); + + LayerSettings layer{ + .geometry = + Geometry{ + .boundaries = FloatRect(0, 0, width, height), + }, + .source = + PixelSource{ + .buffer = + Buffer{ + .buffer = srcBuffer, + // Part of the screen is not covered by the texture but + // will be filled in by the shader + .textureTransform = + mat4(mat3(), + vec3(width * 0.3f, height * 0.3f, 0.0f)), + }, + }, + .alpha = half(1.0f), + .edgeExtensionEffect = + EdgeExtensionEffect(/* left */ true, + /* right */ false, /* top */ true, /* bottom */ false), + }; + auto layers = std::vector<LayerSettings>{layer}; + benchDrawLayers(*re, layers, benchState, "homescreen_edge_extension"); +} + +BENCHMARK_CAPTURE(BM_homescreen_blur, gaussian, RenderEngine::Threaded::YES, + RenderEngine::GraphicsApi::GL, RenderEngine::BlurAlgorithm::GAUSSIAN); + +BENCHMARK_CAPTURE(BM_homescreen_blur, kawase, RenderEngine::Threaded::YES, + RenderEngine::GraphicsApi::GL, RenderEngine::BlurAlgorithm::KAWASE); + +BENCHMARK_CAPTURE(BM_homescreen_blur, kawase_dual_filter, RenderEngine::Threaded::YES, + RenderEngine::GraphicsApi::GL, RenderEngine::BlurAlgorithm::KAWASE_DUAL_FILTER); + +BENCHMARK_CAPTURE(BM_homescreen, SkiaGLThreaded, RenderEngine::Threaded::YES, + RenderEngine::GraphicsApi::GL); + +BENCHMARK_CAPTURE(BM_homescreen_edgeExtension, SkiaGLThreaded, RenderEngine::Threaded::YES, RenderEngine::GraphicsApi::GL); diff --git a/libs/renderengine/include/renderengine/DisplaySettings.h b/libs/renderengine/include/renderengine/DisplaySettings.h index b640983a55..280ec19a4c 100644 --- a/libs/renderengine/include/renderengine/DisplaySettings.h +++ b/libs/renderengine/include/renderengine/DisplaySettings.h @@ -102,6 +102,9 @@ struct DisplaySettings { Local, }; TonemapStrategy tonemapStrategy = TonemapStrategy::Libtonemap; + + // For now, meaningful primarily when the TonemappingStrategy is Local + float targetHdrSdrRatio = 1.f; }; static inline bool operator==(const DisplaySettings& lhs, const DisplaySettings& rhs) { diff --git a/libs/renderengine/include/renderengine/LayerSettings.h b/libs/renderengine/include/renderengine/LayerSettings.h index 8ac0af47c6..859ae8b6e2 100644 --- a/libs/renderengine/include/renderengine/LayerSettings.h +++ b/libs/renderengine/include/renderengine/LayerSettings.h @@ -31,6 +31,7 @@ #include <ui/ShadowSettings.h> #include <ui/StretchEffect.h> #include <ui/Transform.h> +#include "ui/EdgeExtensionEffect.h" #include <iosfwd> @@ -134,6 +135,7 @@ struct LayerSettings { mat4 blurRegionTransform = mat4(); StretchEffect stretchEffect; + EdgeExtensionEffect edgeExtensionEffect; // Name associated with the layer for debugging purposes. std::string name; @@ -183,7 +185,9 @@ static inline bool operator==(const LayerSettings& lhs, const LayerSettings& rhs lhs.skipContentDraw == rhs.skipContentDraw && lhs.shadow == rhs.shadow && lhs.backgroundBlurRadius == rhs.backgroundBlurRadius && lhs.blurRegionTransform == rhs.blurRegionTransform && - lhs.stretchEffect == rhs.stretchEffect && lhs.whitePointNits == rhs.whitePointNits; + lhs.stretchEffect == rhs.stretchEffect && + lhs.edgeExtensionEffect == rhs.edgeExtensionEffect && + lhs.whitePointNits == rhs.whitePointNits; } static inline void PrintTo(const Buffer& settings, ::std::ostream* os) { @@ -254,6 +258,10 @@ static inline void PrintTo(const StretchEffect& effect, ::std::ostream* os) { *os << "\n}"; } +static inline void PrintTo(const EdgeExtensionEffect& effect, ::std::ostream* os) { + *os << effect; +} + static inline void PrintTo(const LayerSettings& settings, ::std::ostream* os) { *os << "LayerSettings for '" << settings.name.c_str() << "' {"; *os << "\n .geometry = "; @@ -285,6 +293,10 @@ static inline void PrintTo(const LayerSettings& settings, ::std::ostream* os) { *os << "\n .stretchEffect = "; PrintTo(settings.stretchEffect, os); } + + if (settings.edgeExtensionEffect.hasEffect()) { + *os << "\n .edgeExtensionEffect = " << settings.edgeExtensionEffect; + } *os << "\n .whitePointNits = " << settings.whitePointNits; *os << "\n}"; } diff --git a/libs/renderengine/include/renderengine/RenderEngine.h b/libs/renderengine/include/renderengine/RenderEngine.h index 7207394356..0fd982e812 100644 --- a/libs/renderengine/include/renderengine/RenderEngine.h +++ b/libs/renderengine/include/renderengine/RenderEngine.h @@ -97,6 +97,7 @@ struct PrimeCacheConfig { bool cacheImageDimmedLayers = true; bool cacheClippedLayers = true; bool cacheShadowLayers = true; + bool cacheEdgeExtension = true; bool cachePIPImageLayers = true; bool cacheTransparentImageDimmedLayers = true; bool cacheClippedDimmedImageLayers = true; @@ -131,6 +132,7 @@ public: NONE, GAUSSIAN, KAWASE, + KAWASE_DUAL_FILTER, }; static std::unique_ptr<RenderEngine> create(const RenderEngineCreationArgs& args); @@ -207,6 +209,13 @@ public: const std::shared_ptr<ExternalTexture>& buffer, base::unique_fd&& bufferFence); + virtual ftl::Future<FenceResult> drawGainmap(const std::shared_ptr<ExternalTexture>& sdr, + base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, + base::borrowed_fd&& hdrFence, float hdrSdrRatio, + ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap); + // Clean-up method that should be called on the main thread after the // drawFence returned by drawLayers fires. This method will free up // resources used by the most recently drawn frame. If the frame is still @@ -284,8 +293,7 @@ protected: // Update protectedContext mode depending on whether or not any layer has a protected buffer. void updateProtectedContext(const std::vector<LayerSettings>&, - const std::shared_ptr<ExternalTexture>&); - + std::vector<const ExternalTexture*>); // Attempt to switch RenderEngine into and out of protectedContext mode virtual void useProtectedContext(bool useProtectedContext) = 0; @@ -293,6 +301,13 @@ protected: const std::shared_ptr<std::promise<FenceResult>>&& resultPromise, const DisplaySettings& display, const std::vector<LayerSettings>& layers, const std::shared_ptr<ExternalTexture>& buffer, base::unique_fd&& bufferFence) = 0; + + virtual void drawGainmapInternal( + const std::shared_ptr<std::promise<FenceResult>>&& resultPromise, + const std::shared_ptr<ExternalTexture>& sdr, base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, base::borrowed_fd&& hdrFence, + float hdrSdrRatio, ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap) = 0; }; struct RenderEngineCreationArgs { diff --git a/libs/renderengine/include/renderengine/mock/RenderEngine.h b/libs/renderengine/include/renderengine/mock/RenderEngine.h index a8c242a86f..fb8331d870 100644 --- a/libs/renderengine/include/renderengine/mock/RenderEngine.h +++ b/libs/renderengine/include/renderengine/mock/RenderEngine.h @@ -46,6 +46,17 @@ public: ftl::Future<FenceResult>(const DisplaySettings&, const std::vector<LayerSettings>&, const std::shared_ptr<ExternalTexture>&, base::unique_fd&&)); + MOCK_METHOD7(drawGainmap, + ftl::Future<FenceResult>(const std::shared_ptr<ExternalTexture>&, + base::borrowed_fd&&, + const std::shared_ptr<ExternalTexture>&, + base::borrowed_fd&&, float, ui::Dataspace, + const std::shared_ptr<ExternalTexture>&)); + MOCK_METHOD8(drawGainmapInternal, + void(const std::shared_ptr<std::promise<FenceResult>>&&, + const std::shared_ptr<ExternalTexture>&, base::borrowed_fd&&, + const std::shared_ptr<ExternalTexture>&, base::borrowed_fd&&, float, + ui::Dataspace, const std::shared_ptr<ExternalTexture>&)); MOCK_METHOD5(drawLayersInternal, void(const std::shared_ptr<std::promise<FenceResult>>&&, const DisplaySettings&, const std::vector<LayerSettings>&, const std::shared_ptr<ExternalTexture>&, diff --git a/libs/renderengine/skia/AutoBackendTexture.cpp b/libs/renderengine/skia/AutoBackendTexture.cpp index 8aeef9f4bc..b7b7a4d086 100644 --- a/libs/renderengine/skia/AutoBackendTexture.cpp +++ b/libs/renderengine/skia/AutoBackendTexture.cpp @@ -25,8 +25,8 @@ #include "compat/SkiaBackendTexture.h" +#include <common/trace.h> #include <log/log_main.h> -#include <utils/Trace.h> namespace android { namespace renderengine { @@ -63,7 +63,7 @@ void AutoBackendTexture::releaseImageProc(SkImages::ReleaseContext releaseContex } sk_sp<SkImage> AutoBackendTexture::makeImage(ui::Dataspace dataspace, SkAlphaType alphaType) { - ATRACE_CALL(); + SFTRACE_CALL(); sk_sp<SkImage> image = mBackendTexture->makeImage(alphaType, dataspace, releaseImageProc, this); // The following ref will be counteracted by releaseProc, when SkImage is discarded. @@ -75,7 +75,7 @@ sk_sp<SkImage> AutoBackendTexture::makeImage(ui::Dataspace dataspace, SkAlphaTyp } sk_sp<SkSurface> AutoBackendTexture::getOrCreateSurface(ui::Dataspace dataspace) { - ATRACE_CALL(); + SFTRACE_CALL(); LOG_ALWAYS_FATAL_IF(!mBackendTexture->isOutputBuffer(), "You can't generate an SkSurface for a read-only texture"); if (!mSurface.get() || mDataspace != dataspace) { diff --git a/libs/renderengine/skia/AutoBackendTexture.h b/libs/renderengine/skia/AutoBackendTexture.h index 74daf471fa..a570ad01a2 100644 --- a/libs/renderengine/skia/AutoBackendTexture.h +++ b/libs/renderengine/skia/AutoBackendTexture.h @@ -16,9 +16,9 @@ #pragma once -#include <GrDirectContext.h> #include <SkImage.h> #include <SkSurface.h> +#include <include/gpu/ganesh/GrDirectContext.h> #include <sys/types.h> #include <ui/GraphicTypes.h> diff --git a/libs/renderengine/skia/Cache.cpp b/libs/renderengine/skia/Cache.cpp index 59b06568a0..57041ee6a1 100644 --- a/libs/renderengine/skia/Cache.cpp +++ b/libs/renderengine/skia/Cache.cpp @@ -27,6 +27,8 @@ #include "ui/Rect.h" #include "utils/Timers.h" +#include <com_android_graphics_libgui_flags.h> + namespace android::renderengine::skia { namespace { @@ -619,6 +621,32 @@ static void drawP3ImageLayers(SkiaRenderEngine* renderengine, const DisplaySetti } } +static void drawEdgeExtensionLayers(SkiaRenderEngine* renderengine, const DisplaySettings& display, + const std::shared_ptr<ExternalTexture>& dstTexture, + const std::shared_ptr<ExternalTexture>& srcTexture) { + const Rect& displayRect = display.physicalDisplay; + // Make the layer + LayerSettings layer{ + // Make the layer bigger than the texture + .geometry = Geometry{.boundaries = FloatRect(0, 0, displayRect.width(), + displayRect.height())}, + .source = PixelSource{.buffer = + Buffer{ + .buffer = srcTexture, + .isOpaque = 1, + }}, + // The type of effect does not affect the shader's uniforms, but the layer must have a + // valid EdgeExtensionEffect to apply the shader + .edgeExtensionEffect = + EdgeExtensionEffect(true /* left */, false, false, true /* bottom */), + }; + for (float alpha : {0.5, 0.0, 1.0}) { + layer.alpha = alpha; + auto layers = std::vector<LayerSettings>{layer}; + renderengine->drawLayers(display, layers, dstTexture, base::unique_fd()); + } +} + // // The collection of shaders cached here were found by using perfetto to record shader compiles // during actions that involve RenderEngine, logging the layer settings, and the shader code @@ -761,6 +789,12 @@ void Cache::primeShaderCache(SkiaRenderEngine* renderengine, PrimeCacheConfig co // Draw layers for b/185569240. drawClippedLayers(renderengine, display, dstTexture, texture); } + + if (com::android::graphics::libgui::flags::edge_extension_shader() && + config.cacheEdgeExtension) { + drawEdgeExtensionLayers(renderengine, display, dstTexture, texture); + drawEdgeExtensionLayers(renderengine, p3Display, dstTexture, texture); + } } if (config.cachePIPImageLayers) { diff --git a/libs/renderengine/skia/GaneshVkRenderEngine.cpp b/libs/renderengine/skia/GaneshVkRenderEngine.cpp index 68798bf8b4..a3a43e20be 100644 --- a/libs/renderengine/skia/GaneshVkRenderEngine.cpp +++ b/libs/renderengine/skia/GaneshVkRenderEngine.cpp @@ -21,9 +21,9 @@ #include <include/gpu/ganesh/vk/GrVkBackendSemaphore.h> +#include <common/trace.h> #include <log/log_main.h> #include <sync/sync.h> -#include <utils/Trace.h> namespace android::renderengine::skia { @@ -78,7 +78,7 @@ base::unique_fd GaneshVkRenderEngine::flushAndSubmit(SkiaGpuContext* context, sk_sp<SkSurface> dstSurface) { sk_sp<GrDirectContext> grContext = context->grDirectContext(); { - ATRACE_NAME("flush surface"); + SFTRACE_NAME("flush surface"); // TODO: Investigate feasibility of combining this "surface flush" into the "context flush" // below. context->grDirectContext()->flush(dstSurface.get()); diff --git a/libs/renderengine/skia/GraphiteVkRenderEngine.cpp b/libs/renderengine/skia/GraphiteVkRenderEngine.cpp index b5cb21b35d..390ad6efd1 100644 --- a/libs/renderengine/skia/GraphiteVkRenderEngine.cpp +++ b/libs/renderengine/skia/GraphiteVkRenderEngine.cpp @@ -23,6 +23,7 @@ #include <include/gpu/graphite/BackendSemaphore.h> #include <include/gpu/graphite/Context.h> #include <include/gpu/graphite/Recording.h> +#include <include/gpu/graphite/vk/VulkanGraphiteTypes.h> #include <log/log_main.h> #include <sync/sync.h> @@ -77,7 +78,7 @@ void GraphiteVkRenderEngine::waitFence(SkiaGpuContext*, base::borrowed_fd fenceF base::unique_fd fenceDup(dupedFd); VkSemaphore waitSemaphore = getVulkanInterface(isProtected()).importSemaphoreFromSyncFd(fenceDup.release()); - graphite::BackendSemaphore beSemaphore(waitSemaphore); + auto beSemaphore = graphite::BackendSemaphores::MakeVulkan(waitSemaphore); mStagedWaitSemaphores.push_back(beSemaphore); } @@ -92,7 +93,7 @@ base::unique_fd GraphiteVkRenderEngine::flushAndSubmit(SkiaGpuContext* context, // This "signal" semaphore is called after rendering, but it is cleaned up in the same mechanism // as "wait" semaphores from waitFence. VkSemaphore vkSignalSemaphore = vulkanInterface.createExportableSemaphore(); - graphite::BackendSemaphore backendSignalSemaphore(vkSignalSemaphore); + auto backendSignalSemaphore = graphite::BackendSemaphores::MakeVulkan(vkSignalSemaphore); // Collect all Vk semaphores that DestroySemaphoreInfo needs to own and delete after GPU work. std::vector<VkSemaphore> vkSemaphoresToCleanUp; @@ -100,7 +101,8 @@ base::unique_fd GraphiteVkRenderEngine::flushAndSubmit(SkiaGpuContext* context, vkSemaphoresToCleanUp.push_back(vkSignalSemaphore); } for (auto backendWaitSemaphore : mStagedWaitSemaphores) { - vkSemaphoresToCleanUp.push_back(backendWaitSemaphore.getVkSemaphore()); + vkSemaphoresToCleanUp.push_back( + graphite::BackendSemaphores::GetVkSemaphore(backendWaitSemaphore)); } DestroySemaphoreInfo* destroySemaphoreInfo = nullptr; diff --git a/libs/renderengine/skia/SkiaGLRenderEngine.cpp b/libs/renderengine/skia/SkiaGLRenderEngine.cpp index 48270e1d2b..4ef7d5bccb 100644 --- a/libs/renderengine/skia/SkiaGLRenderEngine.cpp +++ b/libs/renderengine/skia/SkiaGLRenderEngine.cpp @@ -21,19 +21,17 @@ #include "SkiaGLRenderEngine.h" -#include "compat/SkiaGpuContext.h" - #include <EGL/egl.h> #include <EGL/eglext.h> -#include <GrContextOptions.h> -#include <GrTypes.h> #include <android-base/stringprintf.h> -#include <gl/GrGLInterface.h> +#include <common/trace.h> +#include <include/gpu/ganesh/GrContextOptions.h> +#include <include/gpu/ganesh/GrTypes.h> #include <include/gpu/ganesh/gl/GrGLDirectContext.h> -#include <gui/TraceUtils.h> +#include <include/gpu/ganesh/gl/GrGLInterface.h> +#include <log/log_main.h> #include <sync/sync.h> #include <ui/DebugUtils.h> -#include <utils/Trace.h> #include <cmath> #include <cstdint> @@ -41,7 +39,7 @@ #include <numeric> #include "GLExtensions.h" -#include "log/log_main.h" +#include "compat/SkiaGpuContext.h" namespace android { namespace renderengine { @@ -332,7 +330,7 @@ bool SkiaGLRenderEngine::useProtectedContextImpl(GrProtected isProtected) { void SkiaGLRenderEngine::waitFence(SkiaGpuContext*, base::borrowed_fd fenceFd) { if (fenceFd.get() >= 0 && !waitGpuFence(fenceFd)) { - ATRACE_NAME("SkiaGLRenderEngine::waitFence"); + SFTRACE_NAME("SkiaGLRenderEngine::waitFence"); sync_wait(fenceFd.get(), -1); } } @@ -341,19 +339,19 @@ base::unique_fd SkiaGLRenderEngine::flushAndSubmit(SkiaGpuContext* context, sk_sp<SkSurface> dstSurface) { sk_sp<GrDirectContext> grContext = context->grDirectContext(); { - ATRACE_NAME("flush surface"); + SFTRACE_NAME("flush surface"); grContext->flush(dstSurface.get()); } base::unique_fd drawFence = flushGL(); bool requireSync = drawFence.get() < 0; if (requireSync) { - ATRACE_BEGIN("Submit(sync=true)"); + SFTRACE_BEGIN("Submit(sync=true)"); } else { - ATRACE_BEGIN("Submit(sync=false)"); + SFTRACE_BEGIN("Submit(sync=false)"); } bool success = grContext->submit(requireSync ? GrSyncCpu::kYes : GrSyncCpu::kNo); - ATRACE_END(); + SFTRACE_END(); if (!success) { ALOGE("Failed to flush RenderEngine commands"); // Chances are, something illegal happened (Skia's internal GPU object @@ -400,7 +398,7 @@ bool SkiaGLRenderEngine::waitGpuFence(base::borrowed_fd fenceFd) { } base::unique_fd SkiaGLRenderEngine::flushGL() { - ATRACE_CALL(); + SFTRACE_CALL(); if (!GLExtensions::getInstance().hasNativeFenceSync()) { return base::unique_fd(); } diff --git a/libs/renderengine/skia/SkiaGLRenderEngine.h b/libs/renderengine/skia/SkiaGLRenderEngine.h index bd177e60ba..765103889e 100644 --- a/libs/renderengine/skia/SkiaGLRenderEngine.h +++ b/libs/renderengine/skia/SkiaGLRenderEngine.h @@ -20,9 +20,10 @@ #include <EGL/egl.h> #include <EGL/eglext.h> #include <GLES2/gl2.h> -#include <GrDirectContext.h> #include <SkSurface.h> #include <android-base/thread_annotations.h> +#include <include/gpu/ganesh/GrContextOptions.h> +#include <include/gpu/ganesh/GrDirectContext.h> #include <renderengine/ExternalTexture.h> #include <renderengine/RenderEngine.h> #include <sys/types.h> @@ -32,7 +33,6 @@ #include "AutoBackendTexture.h" #include "EGL/egl.h" -#include "GrContextOptions.h" #include "SkImageInfo.h" #include "SkiaRenderEngine.h" #include "android-base/macros.h" diff --git a/libs/renderengine/skia/SkiaRenderEngine.cpp b/libs/renderengine/skia/SkiaRenderEngine.cpp index e62640eb85..ec9d3efb88 100644 --- a/libs/renderengine/skia/SkiaRenderEngine.cpp +++ b/libs/renderengine/skia/SkiaRenderEngine.cpp @@ -20,9 +20,6 @@ #include "SkiaRenderEngine.h" -#include <GrBackendSemaphore.h> -#include <GrContextOptions.h> -#include <GrTypes.h> #include <SkBlendMode.h> #include <SkCanvas.h> #include <SkColor.h> @@ -54,8 +51,11 @@ #include <SkTileMode.h> #include <android-base/stringprintf.h> #include <common/FlagManager.h> +#include <common/trace.h> #include <gui/FenceMonitor.h> -#include <gui/TraceUtils.h> +#include <include/gpu/ganesh/GrBackendSemaphore.h> +#include <include/gpu/ganesh/GrContextOptions.h> +#include <include/gpu/ganesh/GrTypes.h> #include <include/gpu/ganesh/SkSurfaceGanesh.h> #include <pthread.h> #include <src/core/SkTraceEventCommon.h> @@ -64,7 +64,6 @@ #include <ui/DebugUtils.h> #include <ui/GraphicBuffer.h> #include <ui/HdrRenderTypeUtils.h> -#include <utils/Trace.h> #include <cmath> #include <cstdint> @@ -76,7 +75,9 @@ #include "ColorSpaces.h" #include "compat/SkiaGpuContext.h" #include "filters/BlurFilter.h" +#include "filters/GainmapFactory.h" #include "filters/GaussianBlurFilter.h" +#include "filters/KawaseBlurDualFilter.h" #include "filters/KawaseBlurFilter.h" #include "filters/LinearEffect.h" #include "filters/MouriMap.h" @@ -238,12 +239,22 @@ static inline SkM44 getSkM44(const android::mat4& matrix) { static inline SkPoint3 getSkPoint3(const android::vec3& vector) { return SkPoint3::Make(vector.x, vector.y, vector.z); } + } // namespace namespace android { namespace renderengine { namespace skia { +namespace { +void trace(sp<Fence> fence) { + if (SFTRACE_ENABLED()) { + static gui::FenceMonitor sMonitor("RE Completion"); + sMonitor.queueFence(std::move(fence)); + } +} +} // namespace + using base::StringAppendF; std::future<void> SkiaRenderEngine::primeCache(PrimeCacheConfig config) { @@ -261,7 +272,7 @@ void SkiaRenderEngine::SkSLCacheMonitor::store(const SkData& key, const SkData& const SkString& description) { mShadersCachedSinceLastCall++; mTotalShadersCompiled++; - ATRACE_FORMAT("SF cache: %i shaders", mTotalShadersCompiled); + SFTRACE_FORMAT("SF cache: %i shaders", mTotalShadersCompiled); } int SkiaRenderEngine::reportShadersCompiled() { @@ -286,6 +297,11 @@ SkiaRenderEngine::SkiaRenderEngine(Threaded threaded, PixelFormat pixelFormat, mBlurFilter = new KawaseBlurFilter(); break; } + case BlurAlgorithm::KAWASE_DUAL_FILTER: { + ALOGD("Background Blurs Enabled (Kawase dual-filtering algorithm)"); + mBlurFilter = new KawaseBlurDualFilter(); + break; + } default: { mBlurFilter = nullptr; break; @@ -416,7 +432,7 @@ void SkiaRenderEngine::mapExternalTextureBuffer(const sp<GraphicBuffer>& buffer, if (isProtectedBuffer || isProtected() || !isGpuSampleable) { return; } - ATRACE_CALL(); + SFTRACE_CALL(); // If we were to support caching protected buffers then we will need to switch the // currently bound context if we are not already using the protected context (and subsequently @@ -441,7 +457,7 @@ void SkiaRenderEngine::mapExternalTextureBuffer(const sp<GraphicBuffer>& buffer, } void SkiaRenderEngine::unmapExternalTextureBuffer(sp<GraphicBuffer>&& buffer) { - ATRACE_CALL(); + SFTRACE_CALL(); std::lock_guard<std::mutex> lock(mRenderingMutex); if (const auto& iter = mGraphicBufferExternalRefs.find(buffer->getId()); iter != mGraphicBufferExternalRefs.end()) { @@ -498,7 +514,7 @@ bool SkiaRenderEngine::canSkipPostRenderCleanup() const { } void SkiaRenderEngine::cleanupPostRender() { - ATRACE_CALL(); + SFTRACE_CALL(); std::lock_guard<std::mutex> lock(mRenderingMutex); mTextureCleanupMgr.cleanup(); } @@ -507,16 +523,24 @@ sk_sp<SkShader> SkiaRenderEngine::createRuntimeEffectShader( const RuntimeEffectShaderParameters& parameters) { // The given surface will be stretched by HWUI via matrix transformation // which gets similar results for most surfaces - // Determine later on if we need to leverage the stertch shader within + // Determine later on if we need to leverage the stretch shader within // surface flinger const auto& stretchEffect = parameters.layer.stretchEffect; const auto& targetBuffer = parameters.layer.source.buffer.buffer; + const auto graphicBuffer = targetBuffer ? targetBuffer->getBuffer() : nullptr; + auto shader = parameters.shader; - if (stretchEffect.hasEffect()) { - const auto graphicBuffer = targetBuffer ? targetBuffer->getBuffer() : nullptr; - if (graphicBuffer && parameters.shader) { + if (graphicBuffer && parameters.shader) { + if (stretchEffect.hasEffect()) { shader = mStretchShaderFactory.createSkShader(shader, stretchEffect); } + // The given surface requires to be filled outside of its buffer bounds if the edge + // extension is required + const auto& edgeExtensionEffect = parameters.layer.edgeExtensionEffect; + if (edgeExtensionEffect.hasEffect()) { + shader = mEdgeExtensionShaderFactory.createSkShader(shader, parameters.layer, + parameters.imageBounds); + } } if (parameters.requiresLinearEffect) { @@ -525,21 +549,28 @@ sk_sp<SkShader> SkiaRenderEngine::createRuntimeEffectShader( static_cast<ui::PixelFormat>(targetBuffer->getPixelFormat())) : std::nullopt; - if (parameters.display.tonemapStrategy == DisplaySettings::TonemapStrategy::Local) { - // TODO: Handle color matrix transforms in linear space. - SkImage* image = parameters.shader->isAImage((SkMatrix*)nullptr, (SkTileMode*)nullptr); - if (image) { - static MouriMap kMapper; - const float ratio = getHdrRenderType(parameters.layer.sourceDataspace, format) == - HdrRenderType::GENERIC_HDR - ? 1.0f - : parameters.layerDimmingRatio; - return kMapper.mouriMap(getActiveContext(), parameters.shader, ratio); - } + const auto hdrType = getHdrRenderType(parameters.layer.sourceDataspace, format, + parameters.layerDimmingRatio); + + const auto usingLocalTonemap = + parameters.display.tonemapStrategy == DisplaySettings::TonemapStrategy::Local && + hdrType != HdrRenderType::SDR && + shader->isAImage((SkMatrix*)nullptr, (SkTileMode*)nullptr) && + (hdrType != HdrRenderType::DISPLAY_HDR || + parameters.display.targetHdrSdrRatio < parameters.layerDimmingRatio); + if (usingLocalTonemap) { + const float inputRatio = + hdrType == HdrRenderType::GENERIC_HDR ? 1.0f : parameters.layerDimmingRatio; + static MouriMap kMapper; + shader = kMapper.mouriMap(getActiveContext(), shader, inputRatio, + parameters.display.targetHdrSdrRatio); } + // disable tonemapping if we already locally tonemapped + auto inputDataspace = + usingLocalTonemap ? parameters.outputDataSpace : parameters.layer.sourceDataspace; auto effect = - shaders::LinearEffect{.inputDataspace = parameters.layer.sourceDataspace, + shaders::LinearEffect{.inputDataspace = inputDataspace, .outputDataspace = parameters.outputDataSpace, .undoPremultipliedAlpha = parameters.undoPremultipliedAlpha, .fakeOutputDataspace = parameters.fakeOutputDataspace}; @@ -555,20 +586,20 @@ sk_sp<SkShader> SkiaRenderEngine::createRuntimeEffectShader( mat4 colorTransform = parameters.layer.colorTransform; - colorTransform *= - mat4::scale(vec4(parameters.layerDimmingRatio, parameters.layerDimmingRatio, - parameters.layerDimmingRatio, 1.f)); + if (!usingLocalTonemap) { + colorTransform *= + mat4::scale(vec4(parameters.layerDimmingRatio, parameters.layerDimmingRatio, + parameters.layerDimmingRatio, 1.f)); + } - const auto targetBuffer = parameters.layer.source.buffer.buffer; - const auto graphicBuffer = targetBuffer ? targetBuffer->getBuffer() : nullptr; const auto hardwareBuffer = graphicBuffer ? graphicBuffer->toAHardwareBuffer() : nullptr; - return createLinearEffectShader(parameters.shader, effect, runtimeEffect, - std::move(colorTransform), parameters.display.maxLuminance, + return createLinearEffectShader(shader, effect, runtimeEffect, std::move(colorTransform), + parameters.display.maxLuminance, parameters.display.currentLuminanceNits, parameters.layer.source.buffer.maxLuminanceNits, hardwareBuffer, parameters.display.renderIntent); } - return parameters.shader; + return shader; } void SkiaRenderEngine::initCanvas(SkCanvas* canvas, const DisplaySettings& display) { @@ -683,7 +714,7 @@ void SkiaRenderEngine::drawLayersInternal( const std::shared_ptr<std::promise<FenceResult>>&& resultPromise, const DisplaySettings& display, const std::vector<LayerSettings>& layers, const std::shared_ptr<ExternalTexture>& buffer, base::unique_fd&& bufferFence) { - ATRACE_FORMAT("%s for %s", __func__, display.namePlusId.c_str()); + SFTRACE_FORMAT("%s for %s", __func__, display.namePlusId.c_str()); std::lock_guard<std::mutex> lock(mRenderingMutex); @@ -775,7 +806,7 @@ void SkiaRenderEngine::drawLayersInternal( logSettings(display); } for (const auto& layer : layers) { - ATRACE_FORMAT("DrawLayer: %s", layer.name.c_str()); + SFTRACE_FORMAT("DrawLayer: %s", layer.name.c_str()); if (kPrintLayerSettings) { logSettings(layer); @@ -869,7 +900,7 @@ void SkiaRenderEngine::drawLayersInternal( // TODO(b/182216890): Filter out empty layers earlier if (blurRect.width() > 0 && blurRect.height() > 0) { if (layer.backgroundBlurRadius > 0) { - ATRACE_NAME("BackgroundBlur"); + SFTRACE_NAME("BackgroundBlur"); auto blurredImage = mBlurFilter->generate(context, layer.backgroundBlurRadius, blurInput, blurRect); @@ -882,7 +913,7 @@ void SkiaRenderEngine::drawLayersInternal( canvas->concat(getSkM44(layer.blurRegionTransform).asM33()); for (auto region : layer.blurRegions) { if (cachedBlurs[region.blurRadius] == nullptr) { - ATRACE_NAME("BlurRegion"); + SFTRACE_NAME("BlurRegion"); cachedBlurs[region.blurRadius] = mBlurFilter->generate(context, region.blurRadius, blurInput, blurRect); @@ -965,7 +996,7 @@ void SkiaRenderEngine::drawLayersInternal( SkPaint paint; if (layer.source.buffer.buffer) { - ATRACE_NAME("DrawImage"); + SFTRACE_NAME("DrawImage"); validateInputBufferUsage(layer.source.buffer.buffer->getBuffer()); const auto& item = layer.source.buffer; auto imageTextureRef = getOrCreateBackendTexture(item.buffer->getBuffer(), false); @@ -1032,18 +1063,20 @@ void SkiaRenderEngine::drawLayersInternal( toSkColorSpace(layerDataspace))); } - paint.setShader(createRuntimeEffectShader( - RuntimeEffectShaderParameters{.shader = shader, - .layer = layer, - .display = display, - .undoPremultipliedAlpha = !item.isOpaque && - item.usePremultipliedAlpha, - .requiresLinearEffect = requiresLinearEffect, - .layerDimmingRatio = dimInLinearSpace - ? layerDimmingRatio - : 1.f, - .outputDataSpace = display.outputDataspace, - .fakeOutputDataspace = fakeDataspace})); + SkRect imageBounds; + matrix.mapRect(&imageBounds, SkRect::Make(image->bounds())); + + paint.setShader(createRuntimeEffectShader(RuntimeEffectShaderParameters{ + .shader = shader, + .layer = layer, + .display = display, + .undoPremultipliedAlpha = !item.isOpaque && item.usePremultipliedAlpha, + .requiresLinearEffect = requiresLinearEffect, + .layerDimmingRatio = dimInLinearSpace ? layerDimmingRatio : 1.f, + .outputDataSpace = display.outputDataspace, + .fakeOutputDataspace = fakeDataspace, + .imageBounds = imageBounds, + })); // Turn on dithering when dimming beyond this (arbitrary) threshold... static constexpr float kDimmingThreshold = 0.9f; @@ -1096,7 +1129,7 @@ void SkiaRenderEngine::drawLayersInternal( paint.setColorFilter(SkColorFilters::Matrix(colorMatrix)); } } else { - ATRACE_NAME("DrawColor"); + SFTRACE_NAME("DrawColor"); const auto color = layer.source.solidColor; sk_sp<SkShader> shader = SkShaders::Color(SkColor4f{.fR = color.r, .fG = color.g, @@ -1111,7 +1144,8 @@ void SkiaRenderEngine::drawLayersInternal( .requiresLinearEffect = requiresLinearEffect, .layerDimmingRatio = layerDimmingRatio, .outputDataSpace = display.outputDataspace, - .fakeOutputDataspace = fakeDataspace})); + .fakeOutputDataspace = fakeDataspace, + .imageBounds = SkRect::MakeEmpty()})); } if (layer.disableBlending) { @@ -1152,7 +1186,7 @@ void SkiaRenderEngine::drawLayersInternal( canvas->drawRect(bounds.rect(), paint); } if (kGaneshFlushAfterEveryLayer) { - ATRACE_NAME("flush surface"); + SFTRACE_NAME("flush surface"); // No-op in Graphite. If "flushing" Skia's drawing commands after each layer is desired // in Graphite, then a graphite::Recording would need to be snapped and tracked for each // layer, which is likely possible but adds non-trivial complexity (in both bookkeeping @@ -1166,11 +1200,48 @@ void SkiaRenderEngine::drawLayersInternal( LOG_ALWAYS_FATAL_IF(activeSurface != dstSurface); auto drawFence = sp<Fence>::make(flushAndSubmit(context, dstSurface)); + trace(drawFence); + resultPromise->set_value(std::move(drawFence)); +} - if (ATRACE_ENABLED()) { - static gui::FenceMonitor sMonitor("RE Completion"); - sMonitor.queueFence(drawFence); - } +void SkiaRenderEngine::drawGainmapInternal( + const std::shared_ptr<std::promise<FenceResult>>&& resultPromise, + const std::shared_ptr<ExternalTexture>& sdr, base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, base::borrowed_fd&& hdrFence, + float hdrSdrRatio, ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap) { + std::lock_guard<std::mutex> lock(mRenderingMutex); + auto context = getActiveContext(); + auto surfaceTextureRef = getOrCreateBackendTexture(gainmap->getBuffer(), true); + sk_sp<SkSurface> dstSurface = + surfaceTextureRef->getOrCreateSurface(ui::Dataspace::V0_SRGB_LINEAR); + + waitFence(context, sdrFence); + const auto sdrTextureRef = getOrCreateBackendTexture(sdr->getBuffer(), false); + const auto sdrImage = sdrTextureRef->makeImage(dataspace, kPremul_SkAlphaType); + const auto sdrShader = + sdrImage->makeShader(SkTileMode::kClamp, SkTileMode::kClamp, + SkSamplingOptions({SkFilterMode::kLinear, SkMipmapMode::kNone}), + nullptr); + waitFence(context, hdrFence); + const auto hdrTextureRef = getOrCreateBackendTexture(hdr->getBuffer(), false); + const auto hdrImage = hdrTextureRef->makeImage(dataspace, kPremul_SkAlphaType); + const auto hdrShader = + hdrImage->makeShader(SkTileMode::kClamp, SkTileMode::kClamp, + SkSamplingOptions({SkFilterMode::kLinear, SkMipmapMode::kNone}), + nullptr); + + static GainmapFactory kGainmapFactory; + const auto gainmapShader = kGainmapFactory.createSkShader(sdrShader, hdrShader, hdrSdrRatio); + + const auto canvas = dstSurface->getCanvas(); + SkPaint paint; + paint.setShader(gainmapShader); + paint.setBlendMode(SkBlendMode::kSrc); + canvas->drawPaint(paint); + + auto drawFence = sp<Fence>::make(flushAndSubmit(context, dstSurface)); + trace(drawFence); resultPromise->set_value(std::move(drawFence)); } @@ -1185,7 +1256,7 @@ size_t SkiaRenderEngine::getMaxViewportDims() const { void SkiaRenderEngine::drawShadow(SkCanvas* canvas, const SkRRect& casterRRect, const ShadowSettings& settings) { - ATRACE_CALL(); + SFTRACE_CALL(); const float casterZ = settings.length / 2.0f; const auto flags = settings.casterIsTranslucent ? kTransparentOccluder_ShadowFlag : kNone_ShadowFlag; diff --git a/libs/renderengine/skia/SkiaRenderEngine.h b/libs/renderengine/skia/SkiaRenderEngine.h index c8f9241257..b5f8898263 100644 --- a/libs/renderengine/skia/SkiaRenderEngine.h +++ b/libs/renderengine/skia/SkiaRenderEngine.h @@ -18,11 +18,12 @@ #define SF_SKIARENDERENGINE_H_ #include <renderengine/RenderEngine.h> -#include <sys/types.h> -#include <GrBackendSemaphore.h> -#include <SkSurface.h> #include <android-base/thread_annotations.h> +#include <include/core/SkImageInfo.h> +#include <include/core/SkSurface.h> +#include <include/gpu/ganesh/GrBackendSemaphore.h> +#include <include/gpu/ganesh/GrContextOptions.h> #include <renderengine/ExternalTexture.h> #include <renderengine/RenderEngine.h> #include <sys/types.h> @@ -32,12 +33,11 @@ #include <unordered_map> #include "AutoBackendTexture.h" -#include "GrContextOptions.h" -#include "SkImageInfo.h" #include "android-base/macros.h" #include "compat/SkiaGpuContext.h" #include "debug/SkiaCapture.h" #include "filters/BlurFilter.h" +#include "filters/EdgeExtensionShaderFactory.h" #include "filters/LinearEffect.h" #include "filters/StretchShaderFactory.h" @@ -142,6 +142,13 @@ private: const std::vector<LayerSettings>& layers, const std::shared_ptr<ExternalTexture>& buffer, base::unique_fd&& bufferFence) override final; + void drawGainmapInternal(const std::shared_ptr<std::promise<FenceResult>>&& resultPromise, + const std::shared_ptr<ExternalTexture>& sdr, + base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, + base::borrowed_fd&& hdrFence, float hdrSdrRatio, + ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap) override final; void dump(std::string& result) override final; @@ -156,6 +163,7 @@ private: float layerDimmingRatio; const ui::Dataspace outputDataSpace; const ui::Dataspace fakeOutputDataspace; + const SkRect& imageBounds; }; sk_sp<SkShader> createRuntimeEffectShader(const RuntimeEffectShaderParameters&); @@ -175,6 +183,7 @@ private: AutoBackendTexture::CleanupManager mTextureCleanupMgr GUARDED_BY(mRenderingMutex); StretchShaderFactory mStretchShaderFactory; + EdgeExtensionShaderFactory mEdgeExtensionShaderFactory; sp<Fence> mLastDrawFence; BlurFilter* mBlurFilter = nullptr; diff --git a/libs/renderengine/skia/SkiaVkRenderEngine.cpp b/libs/renderengine/skia/SkiaVkRenderEngine.cpp index bd501073d7..677a2b63b2 100644 --- a/libs/renderengine/skia/SkiaVkRenderEngine.cpp +++ b/libs/renderengine/skia/SkiaVkRenderEngine.cpp @@ -24,18 +24,16 @@ #include "GaneshVkRenderEngine.h" #include "compat/SkiaGpuContext.h" -#include <GrBackendSemaphore.h> -#include <GrContextOptions.h> -#include <GrDirectContext.h> +#include <include/gpu/ganesh/GrBackendSemaphore.h> +#include <include/gpu/ganesh/GrContextOptions.h> +#include <include/gpu/ganesh/GrDirectContext.h> #include <include/gpu/ganesh/vk/GrVkBackendSemaphore.h> #include <include/gpu/ganesh/vk/GrVkDirectContext.h> -#include <vk/GrVkExtensions.h> -#include <vk/GrVkTypes.h> +#include <include/gpu/ganesh/vk/GrVkTypes.h> #include <android-base/stringprintf.h> -#include <gui/TraceUtils.h> +#include <common/trace.h> #include <sync/sync.h> -#include <utils/Trace.h> #include <memory> #include <string> diff --git a/libs/renderengine/skia/SkiaVkRenderEngine.h b/libs/renderengine/skia/SkiaVkRenderEngine.h index 0a2f9b2228..d2bb3d53cf 100644 --- a/libs/renderengine/skia/SkiaVkRenderEngine.h +++ b/libs/renderengine/skia/SkiaVkRenderEngine.h @@ -17,8 +17,6 @@ #ifndef SF_SKIAVKRENDERENGINE_H_ #define SF_SKIAVKRENDERENGINE_H_ -#include <vk/GrVkBackendContext.h> - #include "SkiaRenderEngine.h" #include "VulkanInterface.h" #include "compat/SkiaGpuContext.h" diff --git a/libs/renderengine/skia/VulkanInterface.cpp b/libs/renderengine/skia/VulkanInterface.cpp index 5e756b03ed..37b69f6590 100644 --- a/libs/renderengine/skia/VulkanInterface.cpp +++ b/libs/renderengine/skia/VulkanInterface.cpp @@ -32,21 +32,8 @@ namespace android { namespace renderengine { namespace skia { -GrVkBackendContext VulkanInterface::getGaneshBackendContext() { - GrVkBackendContext backendContext; - backendContext.fInstance = mInstance; - backendContext.fPhysicalDevice = mPhysicalDevice; - backendContext.fDevice = mDevice; - backendContext.fQueue = mQueue; - backendContext.fGraphicsQueueIndex = mQueueIndex; - backendContext.fMaxAPIVersion = mApiVersion; - backendContext.fVkExtensions = &mGrExtensions; - backendContext.fDeviceFeatures2 = mPhysicalDeviceFeatures2; - backendContext.fGetProc = mGrGetProc; - backendContext.fProtectedContext = mIsProtected ? Protected::kYes : Protected::kNo; - backendContext.fDeviceLostContext = this; // VulkanInterface is long-lived - backendContext.fDeviceLostProc = onVkDeviceFault; - return backendContext; +VulkanBackendContext VulkanInterface::getGaneshBackendContext() { + return this->getGraphiteBackendContext(); }; VulkanBackendContext VulkanInterface::getGraphiteBackendContext() { @@ -57,7 +44,7 @@ VulkanBackendContext VulkanInterface::getGraphiteBackendContext() { backendContext.fQueue = mQueue; backendContext.fGraphicsQueueIndex = mQueueIndex; backendContext.fMaxAPIVersion = mApiVersion; - backendContext.fVkExtensions = &mGrExtensions; + backendContext.fVkExtensions = &mVulkanExtensions; backendContext.fDeviceFeatures2 = mPhysicalDeviceFeatures2; backendContext.fGetProc = mGrGetProc; backendContext.fProtectedContext = mIsProtected ? Protected::kYes : Protected::kNo; @@ -429,11 +416,11 @@ void VulkanInterface::init(bool protectedContent) { mDeviceExtensionNames.push_back(devExt.extensionName); } - mGrExtensions.init(sGetProc, instance, physicalDevice, enabledInstanceExtensionNames.size(), - enabledInstanceExtensionNames.data(), enabledDeviceExtensionNames.size(), - enabledDeviceExtensionNames.data()); + mVulkanExtensions.init(sGetProc, instance, physicalDevice, enabledInstanceExtensionNames.size(), + enabledInstanceExtensionNames.data(), enabledDeviceExtensionNames.size(), + enabledDeviceExtensionNames.data()); - if (!mGrExtensions.hasExtension(VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, 1)) { + if (!mVulkanExtensions.hasExtension(VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, 1)) { BAIL("Vulkan driver doesn't support external semaphore fd"); } @@ -458,7 +445,7 @@ void VulkanInterface::init(bool protectedContent) { tailPnext = &mProtectedMemoryFeatures->pNext; } - if (mGrExtensions.hasExtension(VK_EXT_DEVICE_FAULT_EXTENSION_NAME, 1)) { + if (mVulkanExtensions.hasExtension(VK_EXT_DEVICE_FAULT_EXTENSION_NAME, 1)) { mDeviceFaultFeatures = new VkPhysicalDeviceFaultFeaturesEXT; mDeviceFaultFeatures->sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT; mDeviceFaultFeatures->pNext = nullptr; @@ -484,7 +471,7 @@ void VulkanInterface::init(bool protectedContent) { queuePriority, }; - if (mGrExtensions.hasExtension(VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME, 2)) { + if (mVulkanExtensions.hasExtension(VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME, 2)) { queueNextPtr = &queuePriorityCreateInfo; } @@ -606,7 +593,7 @@ void VulkanInterface::teardown() { mQueue = VK_NULL_HANDLE; // Implicitly destroyed by destroying mDevice. mQueueIndex = 0; mApiVersion = 0; - mGrExtensions = skgpu::VulkanExtensions(); + mVulkanExtensions = skgpu::VulkanExtensions(); mGrGetProc = nullptr; mIsProtected = false; mIsRealtimePriority = false; diff --git a/libs/renderengine/skia/VulkanInterface.h b/libs/renderengine/skia/VulkanInterface.h index f20b00251b..d0fe4d1544 100644 --- a/libs/renderengine/skia/VulkanInterface.h +++ b/libs/renderengine/skia/VulkanInterface.h @@ -16,7 +16,7 @@ #pragma once -#include <include/gpu/vk/GrVkBackendContext.h> +#include <include/gpu/vk/VulkanBackendContext.h> #include <include/gpu/vk/VulkanExtensions.h> #include <include/gpu/vk/VulkanTypes.h> @@ -24,10 +24,6 @@ using namespace skgpu; -namespace skgpu { -struct VulkanBackendContext; -} // namespace skgpu - namespace android { namespace renderengine { namespace skia { @@ -48,7 +44,8 @@ public: bool takeOwnership(); void teardown(); - GrVkBackendContext getGaneshBackendContext(); + // TODO(b/309785258) Combine these into one now that they are the same implementation. + VulkanBackendContext getGaneshBackendContext(); VulkanBackendContext getGraphiteBackendContext(); VkSemaphore createExportableSemaphore(); VkSemaphore importSemaphoreFromSyncFd(int syncFd); @@ -86,7 +83,7 @@ private: VkQueue mQueue = VK_NULL_HANDLE; int mQueueIndex = 0; uint32_t mApiVersion = 0; - skgpu::VulkanExtensions mGrExtensions; + skgpu::VulkanExtensions mVulkanExtensions; VkPhysicalDeviceFeatures2* mPhysicalDeviceFeatures2 = nullptr; VkPhysicalDeviceSamplerYcbcrConversionFeatures* mSamplerYcbcrConversionFeatures = nullptr; VkPhysicalDeviceProtectedMemoryFeatures* mProtectedMemoryFeatures = nullptr; diff --git a/libs/renderengine/skia/compat/GaneshBackendTexture.cpp b/libs/renderengine/skia/compat/GaneshBackendTexture.cpp index d246466965..88282e7f5a 100644 --- a/libs/renderengine/skia/compat/GaneshBackendTexture.cpp +++ b/libs/renderengine/skia/compat/GaneshBackendTexture.cpp @@ -21,26 +21,26 @@ #define ATRACE_TAG ATRACE_TAG_GRAPHICS #include <include/core/SkImage.h> -#include <include/gpu/GrDirectContext.h> +#include <include/gpu/ganesh/GrDirectContext.h> #include <include/gpu/ganesh/SkImageGanesh.h> #include <include/gpu/ganesh/SkSurfaceGanesh.h> #include <include/gpu/ganesh/gl/GrGLBackendSurface.h> #include <include/gpu/ganesh/vk/GrVkBackendSurface.h> -#include <include/gpu/vk/GrVkTypes.h> +#include <include/gpu/ganesh/vk/GrVkTypes.h> #include "skia/ColorSpaces.h" #include "skia/compat/SkiaBackendTexture.h" #include <android/hardware_buffer.h> +#include <common/trace.h> #include <log/log_main.h> -#include <utils/Trace.h> namespace android::renderengine::skia { GaneshBackendTexture::GaneshBackendTexture(sk_sp<GrDirectContext> grContext, AHardwareBuffer* buffer, bool isOutputBuffer) : SkiaBackendTexture(buffer, isOutputBuffer), mGrContext(grContext) { - ATRACE_CALL(); + SFTRACE_CALL(); AHardwareBuffer_Desc desc; AHardwareBuffer_describe(buffer, &desc); const bool createProtectedImage = 0 != (desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT); diff --git a/libs/renderengine/skia/compat/GaneshBackendTexture.h b/libs/renderengine/skia/compat/GaneshBackendTexture.h index 5cf8647801..4337df18d4 100644 --- a/libs/renderengine/skia/compat/GaneshBackendTexture.h +++ b/libs/renderengine/skia/compat/GaneshBackendTexture.h @@ -21,7 +21,7 @@ #include <include/android/GrAHardwareBufferUtils.h> #include <include/core/SkColorSpace.h> -#include <include/gpu/GrDirectContext.h> +#include <include/gpu/ganesh/GrDirectContext.h> #include <android-base/macros.h> diff --git a/libs/renderengine/skia/compat/GaneshGpuContext.cpp b/libs/renderengine/skia/compat/GaneshGpuContext.cpp index b2eae009ed..931f8433da 100644 --- a/libs/renderengine/skia/compat/GaneshGpuContext.cpp +++ b/libs/renderengine/skia/compat/GaneshGpuContext.cpp @@ -19,13 +19,13 @@ #include <include/core/SkImageInfo.h> #include <include/core/SkSurface.h> #include <include/core/SkTraceMemoryDump.h> -#include <include/gpu/GrDirectContext.h> -#include <include/gpu/GrTypes.h> +#include <include/gpu/ganesh/GrDirectContext.h> +#include <include/gpu/ganesh/GrTypes.h> #include <include/gpu/ganesh/SkSurfaceGanesh.h> #include <include/gpu/ganesh/gl/GrGLDirectContext.h> +#include <include/gpu/ganesh/gl/GrGLInterface.h> #include <include/gpu/ganesh/vk/GrVkDirectContext.h> -#include <include/gpu/gl/GrGLInterface.h> -#include <include/gpu/vk/GrVkBackendContext.h> +#include <include/gpu/vk/VulkanBackendContext.h> #include "../AutoBackendTexture.h" #include "GaneshBackendTexture.h" @@ -56,10 +56,10 @@ std::unique_ptr<SkiaGpuContext> SkiaGpuContext::MakeGL_Ganesh( } std::unique_ptr<SkiaGpuContext> SkiaGpuContext::MakeVulkan_Ganesh( - const GrVkBackendContext& grVkBackendContext, + const skgpu::VulkanBackendContext& vkBackendContext, GrContextOptions::PersistentCache& skSLCacheMonitor) { return std::make_unique<GaneshGpuContext>( - GrDirectContexts::MakeVulkan(grVkBackendContext, ganeshOptions(skSLCacheMonitor))); + GrDirectContexts::MakeVulkan(vkBackendContext, ganeshOptions(skSLCacheMonitor))); } GaneshGpuContext::GaneshGpuContext(sk_sp<GrDirectContext> grContext) : mGrContext(grContext) { diff --git a/libs/renderengine/skia/compat/GraphiteBackendTexture.cpp b/libs/renderengine/skia/compat/GraphiteBackendTexture.cpp index 3dd9ed242e..a6e93ba7e0 100644 --- a/libs/renderengine/skia/compat/GraphiteBackendTexture.cpp +++ b/libs/renderengine/skia/compat/GraphiteBackendTexture.cpp @@ -28,16 +28,16 @@ #include "skia/ColorSpaces.h" #include <android/hardware_buffer.h> +#include <common/trace.h> #include <inttypes.h> #include <log/log_main.h> -#include <utils/Trace.h> namespace android::renderengine::skia { GraphiteBackendTexture::GraphiteBackendTexture(std::shared_ptr<skgpu::graphite::Recorder> recorder, AHardwareBuffer* buffer, bool isOutputBuffer) : SkiaBackendTexture(buffer, isOutputBuffer), mRecorder(std::move(recorder)) { - ATRACE_CALL(); + SFTRACE_CALL(); AHardwareBuffer_Desc desc; AHardwareBuffer_describe(buffer, &desc); const bool createProtectedImage = 0 != (desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT); diff --git a/libs/renderengine/skia/compat/SkiaBackendTexture.h b/libs/renderengine/skia/compat/SkiaBackendTexture.h index 09877a5ede..fa12624ff7 100644 --- a/libs/renderengine/skia/compat/SkiaBackendTexture.h +++ b/libs/renderengine/skia/compat/SkiaBackendTexture.h @@ -18,7 +18,7 @@ #include <include/android/GrAHardwareBufferUtils.h> #include <include/core/SkColorSpace.h> -#include <include/gpu/GrDirectContext.h> +#include <include/gpu/ganesh/GrDirectContext.h> #include <android/hardware_buffer.h> #include <ui/GraphicTypes.h> diff --git a/libs/renderengine/skia/compat/SkiaGpuContext.h b/libs/renderengine/skia/compat/SkiaGpuContext.h index 282dfe7abe..0bd8283987 100644 --- a/libs/renderengine/skia/compat/SkiaGpuContext.h +++ b/libs/renderengine/skia/compat/SkiaGpuContext.h @@ -20,11 +20,10 @@ #define LOG_TAG "RenderEngine" #include <include/core/SkSurface.h> -#include <include/gpu/GrDirectContext.h> -#include <include/gpu/gl/GrGLInterface.h> +#include <include/gpu/ganesh/GrDirectContext.h> +#include <include/gpu/ganesh/gl/GrGLInterface.h> #include <include/gpu/graphite/Context.h> -#include <include/gpu/vk/GrVkBackendContext.h> -#include "include/gpu/vk/VulkanBackendContext.h" +#include <include/gpu/vk/VulkanBackendContext.h> #include "SkiaBackendTexture.h" @@ -52,10 +51,10 @@ public: GrContextOptions::PersistentCache& skSLCacheMonitor); /** - * grVkBackendContext must remain valid until after SkiaGpuContext is destroyed. + * vkBackendContext must remain valid until after SkiaGpuContext is destroyed. */ static std::unique_ptr<SkiaGpuContext> MakeVulkan_Ganesh( - const GrVkBackendContext& grVkBackendContext, + const skgpu::VulkanBackendContext& vkBackendContext, GrContextOptions::PersistentCache& skSLCacheMonitor); // TODO: b/293371537 - Need shader / pipeline monitoring support in Graphite. diff --git a/libs/renderengine/skia/debug/CommonPool.cpp b/libs/renderengine/skia/debug/CommonPool.cpp index bf15300227..9d7c69b019 100644 --- a/libs/renderengine/skia/debug/CommonPool.cpp +++ b/libs/renderengine/skia/debug/CommonPool.cpp @@ -20,8 +20,8 @@ #define LOG_TAG "RenderEngine" #define ATRACE_TAG ATRACE_TAG_GRAPHICS +#include <common/trace.h> #include <sys/resource.h> -#include <utils/Trace.h> #include <system/thread_defs.h> #include <array> @@ -31,7 +31,7 @@ namespace renderengine { namespace skia { CommonPool::CommonPool() { - ATRACE_CALL(); + SFTRACE_CALL(); CommonPool* pool = this; // Create 2 workers diff --git a/libs/renderengine/skia/debug/SkiaCapture.cpp b/libs/renderengine/skia/debug/SkiaCapture.cpp index e778884629..e6a0e22dcf 100644 --- a/libs/renderengine/skia/debug/SkiaCapture.cpp +++ b/libs/renderengine/skia/debug/SkiaCapture.cpp @@ -22,9 +22,9 @@ #include <android-base/properties.h> #include <android-base/stringprintf.h> +#include <common/trace.h> #include <log/log.h> #include <renderengine/RenderEngine.h> -#include <utils/Trace.h> #include "CommonPool.h" #include "SkCanvas.h" @@ -48,7 +48,7 @@ SkiaCapture::~SkiaCapture() { } SkCanvas* SkiaCapture::tryCapture(SkSurface* surface) NO_THREAD_SAFETY_ANALYSIS { - ATRACE_CALL(); + SFTRACE_CALL(); // If we are not running yet, set up. if (CC_LIKELY(!mCaptureRunning)) { @@ -86,7 +86,7 @@ SkCanvas* SkiaCapture::tryCapture(SkSurface* surface) NO_THREAD_SAFETY_ANALYSIS } void SkiaCapture::endCapture() NO_THREAD_SAFETY_ANALYSIS { - ATRACE_CALL(); + SFTRACE_CALL(); // Don't end anything if we are not running. if (CC_LIKELY(!mCaptureRunning)) { return; @@ -102,7 +102,7 @@ void SkiaCapture::endCapture() NO_THREAD_SAFETY_ANALYSIS { } SkCanvas* SkiaCapture::tryOffscreenCapture(SkSurface* surface, OffscreenState* state) { - ATRACE_CALL(); + SFTRACE_CALL(); // Don't start anything if we are not running. if (CC_LIKELY(!mCaptureRunning)) { return surface->getCanvas(); @@ -122,7 +122,7 @@ SkCanvas* SkiaCapture::tryOffscreenCapture(SkSurface* surface, OffscreenState* s } uint64_t SkiaCapture::endOffscreenCapture(OffscreenState* state) { - ATRACE_CALL(); + SFTRACE_CALL(); // Don't end anything if we are not running. if (CC_LIKELY(!mCaptureRunning)) { return 0; @@ -151,7 +151,7 @@ uint64_t SkiaCapture::endOffscreenCapture(OffscreenState* state) { } void SkiaCapture::writeToFile() { - ATRACE_CALL(); + SFTRACE_CALL(); // Pass mMultiPic and mOpenMultiPicStream to a background thread, which will // handle the heavyweight serialization work and destroy them. // mOpenMultiPicStream is released to a bare pointer because keeping it in @@ -169,7 +169,7 @@ void SkiaCapture::writeToFile() { } bool SkiaCapture::setupMultiFrameCapture() { - ATRACE_CALL(); + SFTRACE_CALL(); ALOGD("Set up multi-frame capture, ms = %llu", mTimerInterval.count()); base::SetProperty(PROPERTY_DEBUG_RENDERENGINE_CAPTURE_FILENAME, ""); diff --git a/libs/renderengine/skia/filters/BlurFilter.cpp b/libs/renderengine/skia/filters/BlurFilter.cpp index 1e0c4cf9d0..cd1bd71807 100644 --- a/libs/renderengine/skia/filters/BlurFilter.cpp +++ b/libs/renderengine/skia/filters/BlurFilter.cpp @@ -25,8 +25,8 @@ #include <SkString.h> #include <SkSurface.h> #include <SkTileMode.h> +#include <common/trace.h> #include <log/log.h> -#include <utils/Trace.h> namespace android { namespace renderengine { @@ -79,7 +79,7 @@ void BlurFilter::drawBlurRegion(SkCanvas* canvas, const SkRRect& effectRegion, const uint32_t blurRadius, const float blurAlpha, const SkRect& blurRect, sk_sp<SkImage> blurredImage, sk_sp<SkImage> input) { - ATRACE_CALL(); + SFTRACE_CALL(); SkPaint paint; paint.setAlphaf(blurAlpha); diff --git a/libs/renderengine/skia/filters/EdgeExtensionShaderFactory.cpp b/libs/renderengine/skia/filters/EdgeExtensionShaderFactory.cpp new file mode 100644 index 0000000000..4164c4b4c9 --- /dev/null +++ b/libs/renderengine/skia/filters/EdgeExtensionShaderFactory.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "EdgeExtensionShaderFactory.h" +#include <SkPoint.h> +#include <SkRuntimeEffect.h> +#include <SkStream.h> +#include <SkString.h> +#include <com_android_graphics_libgui_flags.h> +#include "log/log_main.h" + +namespace android::renderengine::skia { + +static const SkString edgeShader = SkString(R"( + uniform shader uContentTexture; + uniform vec2 uImgSize; + + // TODO(b/214232209) oobTolerance is temporary and will be removed when the scrollbar will be + // hidden during the animation + const float oobTolerance = 15; + const int blurRadius = 3; + const float blurArea = float((2 * blurRadius + 1) * (2 * blurRadius + 1)); + + vec4 boxBlur(vec2 p) { + vec4 sumColors = vec4(0); + + for (int i = -blurRadius; i <= blurRadius; i++) { + for (int j = -blurRadius; j <= blurRadius; j++) { + sumColors += uContentTexture.eval(p + vec2(i, j)); + } + } + return sumColors / blurArea; + } + + vec4 main(vec2 coord) { + vec2 nearestTexturePoint = clamp(coord, vec2(0, 0), uImgSize); + if (coord == nearestTexturePoint) { + return uContentTexture.eval(coord); + } else { + vec2 samplePoint = nearestTexturePoint + oobTolerance * normalize( + nearestTexturePoint - coord); + return boxBlur(samplePoint); + } + } +)"); + +EdgeExtensionShaderFactory::EdgeExtensionShaderFactory() { + if (!com::android::graphics::libgui::flags::edge_extension_shader()) { + return; + } + mResult = std::make_unique<SkRuntimeEffect::Result>(SkRuntimeEffect::MakeForShader(edgeShader)); + LOG_ALWAYS_FATAL_IF(!mResult->errorText.isEmpty(), + "EdgeExtensionShaderFactory compilation " + "failed with an unexpected error: %s", + mResult->errorText.c_str()); +} + +sk_sp<SkShader> EdgeExtensionShaderFactory::createSkShader(const sk_sp<SkShader>& inputShader, + const LayerSettings& layer, + const SkRect& imageBounds) const { + LOG_ALWAYS_FATAL_IF(mResult == nullptr, + "EdgeExtensionShaderFactory did not initialize mResult. " + "This means that we unexpectedly applied the edge extension shader"); + + SkRuntimeShaderBuilder builder = SkRuntimeShaderBuilder(mResult->effect); + + builder.child("uContentTexture") = inputShader; + if (imageBounds.isEmpty()) { + builder.uniform("uImgSize") = SkPoint{layer.geometry.boundaries.getWidth(), + layer.geometry.boundaries.getHeight()}; + } else { + builder.uniform("uImgSize") = SkPoint{imageBounds.width(), imageBounds.height()}; + } + return builder.makeShader(); +} +} // namespace android::renderengine::skia
\ No newline at end of file diff --git a/libs/renderengine/skia/filters/EdgeExtensionShaderFactory.h b/libs/renderengine/skia/filters/EdgeExtensionShaderFactory.h new file mode 100644 index 0000000000..17c6b9139f --- /dev/null +++ b/libs/renderengine/skia/filters/EdgeExtensionShaderFactory.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <SkImage.h> +#include <SkRect.h> +#include <SkRuntimeEffect.h> +#include <SkShader.h> +#include <renderengine/LayerSettings.h> +#include <ui/EdgeExtensionEffect.h> + +namespace android::renderengine::skia { + +/** + * This shader is designed to prolong the texture of a surface whose bounds have been extended over + * the size of the texture. This shader is similar to the default clamp, but adds a blur effect and + * samples from close to the edge (compared to on the edge) to avoid weird artifacts when elements + * (in particular, scrollbars) touch the edge. + */ +class EdgeExtensionShaderFactory { +public: + EdgeExtensionShaderFactory(); + + sk_sp<SkShader> createSkShader(const sk_sp<SkShader>& inputShader, const LayerSettings& layer, + const SkRect& imageBounds) const; + +private: + std::unique_ptr<const SkRuntimeEffect::Result> mResult; +}; +} // namespace android::renderengine::skia diff --git a/libs/renderengine/skia/filters/GainmapFactory.cpp b/libs/renderengine/skia/filters/GainmapFactory.cpp new file mode 100644 index 0000000000..e4d4fe96f8 --- /dev/null +++ b/libs/renderengine/skia/filters/GainmapFactory.cpp @@ -0,0 +1,72 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "GainmapFactory.h" + +#include <log/log.h> + +namespace android { +namespace renderengine { +namespace skia { +namespace { + +sk_sp<SkRuntimeEffect> makeEffect(const SkString& sksl) { + auto [effect, error] = SkRuntimeEffect::MakeForShader(sksl); + LOG_ALWAYS_FATAL_IF(!effect, "RuntimeShader error: %s", error.c_str()); + return effect; +} + +// Please refer to https://developer.android.com/media/platform/hdr-image-format#gain_map-generation +static const SkString kGainmapShader = SkString(R"( + uniform shader sdr; + uniform shader hdr; + uniform float mapMaxLog2; + + const float mapMinLog2 = 0.0; + const float mapGamma = 1.0; + const float offsetSdr = 0.015625; + const float offsetHdr = 0.015625; + + float luminance(vec3 linearColor) { + return 0.2126 * linearColor.r + 0.7152 * linearColor.g + 0.0722 * linearColor.b; + } + + vec4 main(vec2 xy) { + float sdrY = luminance(toLinearSrgb(sdr.eval(xy).rgb)); + float hdrY = luminance(toLinearSrgb(hdr.eval(xy).rgb)); + float pixelGain = (hdrY + offsetHdr) / (sdrY + offsetSdr); + float logRecovery = (log2(pixelGain) - mapMinLog2) / (mapMaxLog2 - mapMinLog2); + return vec4(pow(clamp(logRecovery, 0.0, 1.0), mapGamma)); + } +)"); +} // namespace + +const float INTERPOLATION_STRENGTH_VALUE = 0.7f; + +GainmapFactory::GainmapFactory() : mEffect(makeEffect(kGainmapShader)) {} + +sk_sp<SkShader> GainmapFactory::createSkShader(const sk_sp<SkShader>& sdr, + const sk_sp<SkShader>& hdr, float hdrSdrRatio) { + SkRuntimeShaderBuilder shaderBuilder(mEffect); + shaderBuilder.child("sdr") = sdr; + shaderBuilder.child("hdr") = hdr; + shaderBuilder.uniform("mapMaxLog2") = std::log2(hdrSdrRatio); + return shaderBuilder.makeShader(); +} + +} // namespace skia +} // namespace renderengine +} // namespace android diff --git a/libs/renderengine/skia/filters/GainmapFactory.h b/libs/renderengine/skia/filters/GainmapFactory.h new file mode 100644 index 0000000000..7aea5e2195 --- /dev/null +++ b/libs/renderengine/skia/filters/GainmapFactory.h @@ -0,0 +1,44 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include <SkRuntimeEffect.h> +#include <SkShader.h> + +namespace android { +namespace renderengine { +namespace skia { + +/** + * Generates a shader for computing a gainmap, given an SDR base image and its idealized HDR + * rendition. The shader follows the procedure in the UltraHDR spec: + * https://developer.android.com/media/platform/hdr-image-format#gain_map-generation, but makes some + * simplifying assumptions about metadata typical for RenderEngine's usage. + */ +class GainmapFactory { +public: + GainmapFactory(); + // Generates the gainmap shader. The hdrSdrRatio is the max_content_boost in the UltraHDR + // specification. + sk_sp<SkShader> createSkShader(const sk_sp<SkShader>& sdr, const sk_sp<SkShader>& hdr, + float hdrSdrRatio); + +private: + sk_sp<SkRuntimeEffect> mEffect; +}; +} // namespace skia +} // namespace renderengine +} // namespace android diff --git a/libs/renderengine/skia/filters/GaussianBlurFilter.cpp b/libs/renderengine/skia/filters/GaussianBlurFilter.cpp index c9499cbc24..8c52c571a9 100644 --- a/libs/renderengine/skia/filters/GaussianBlurFilter.cpp +++ b/libs/renderengine/skia/filters/GaussianBlurFilter.cpp @@ -19,18 +19,18 @@ #include "GaussianBlurFilter.h" #include <SkBlendMode.h> #include <SkCanvas.h> +#include <SkImageFilters.h> #include <SkPaint.h> #include <SkRRect.h> #include <SkRuntimeEffect.h> -#include <SkImageFilters.h> #include <SkSize.h> #include <SkString.h> #include <SkSurface.h> #include <SkTileMode.h> +#include <common/trace.h> #include <include/gpu/ganesh/SkSurfaceGanesh.h> -#include "include/gpu/GpuTypes.h" // from Skia #include <log/log.h> -#include <utils/Trace.h> +#include "include/gpu/GpuTypes.h" // from Skia namespace android { namespace renderengine { diff --git a/libs/renderengine/skia/filters/KawaseBlurDualFilter.cpp b/libs/renderengine/skia/filters/KawaseBlurDualFilter.cpp new file mode 100644 index 0000000000..db0b133a26 --- /dev/null +++ b/libs/renderengine/skia/filters/KawaseBlurDualFilter.cpp @@ -0,0 +1,173 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define ATRACE_TAG ATRACE_TAG_GRAPHICS + +#include "KawaseBlurDualFilter.h" +#include <SkAlphaType.h> +#include <SkBlendMode.h> +#include <SkCanvas.h> +#include <SkData.h> +#include <SkPaint.h> +#include <SkRRect.h> +#include <SkRuntimeEffect.h> +#include <SkShader.h> +#include <SkSize.h> +#include <SkString.h> +#include <SkSurface.h> +#include <SkTileMode.h> +#include <include/gpu/GpuTypes.h> +#include <include/gpu/ganesh/SkSurfaceGanesh.h> +#include <log/log.h> +#include <utils/Trace.h> + +namespace android { +namespace renderengine { +namespace skia { + +KawaseBlurDualFilter::KawaseBlurDualFilter() : BlurFilter() { + // A shader to sample each vertex of a unit regular heptagon + // plus the original fragment coordinate. + SkString blurString(R"( + uniform shader child; + uniform float in_blurOffset; + uniform float in_crossFade; + + const float2 STEP_0 = float2( 1.0, 0.0); + const float2 STEP_1 = float2( 0.623489802, 0.781831482); + const float2 STEP_2 = float2(-0.222520934, 0.974927912); + const float2 STEP_3 = float2(-0.900968868, 0.433883739); + const float2 STEP_4 = float2( 0.900968868, -0.433883739); + const float2 STEP_5 = float2(-0.222520934, -0.974927912); + const float2 STEP_6 = float2(-0.623489802, -0.781831482); + + half4 main(float2 xy) { + half3 c = child.eval(xy).rgb; + + c += child.eval(xy + STEP_0 * in_blurOffset).rgb; + c += child.eval(xy + STEP_1 * in_blurOffset).rgb; + c += child.eval(xy + STEP_2 * in_blurOffset).rgb; + c += child.eval(xy + STEP_3 * in_blurOffset).rgb; + c += child.eval(xy + STEP_4 * in_blurOffset).rgb; + c += child.eval(xy + STEP_5 * in_blurOffset).rgb; + c += child.eval(xy + STEP_6 * in_blurOffset).rgb; + + return half4(c * 0.125 * in_crossFade, in_crossFade); + } + )"); + + auto [blurEffect, error] = SkRuntimeEffect::MakeForShader(blurString); + LOG_ALWAYS_FATAL_IF(!blurEffect, "RuntimeShader error: %s", error.c_str()); + mBlurEffect = std::move(blurEffect); +} + +static sk_sp<SkSurface> makeSurface(SkiaGpuContext* context, const SkRect& origRect, int scale) { + SkImageInfo scaledInfo = + SkImageInfo::MakeN32Premul(ceil(static_cast<float>(origRect.width()) / scale), + ceil(static_cast<float>(origRect.height()) / scale)); + return context->createRenderTarget(scaledInfo); +} + +void KawaseBlurDualFilter::blurInto(const sk_sp<SkSurface>& drawSurface, + const sk_sp<SkImage>& readImage, const float radius, + const float alpha) const { + const float scale = static_cast<float>(drawSurface->width()) / readImage->width(); + SkMatrix blurMatrix = SkMatrix::Scale(scale, scale); + blurInto(drawSurface, + readImage->makeShader(SkTileMode::kClamp, SkTileMode::kClamp, + SkSamplingOptions(SkFilterMode::kLinear, SkMipmapMode::kNone), + blurMatrix), + readImage->width() / static_cast<float>(drawSurface->width()), radius, alpha); +} + +void KawaseBlurDualFilter::blurInto(const sk_sp<SkSurface>& drawSurface, sk_sp<SkShader> input, + const float inverseScale, const float radius, + const float alpha) const { + SkRuntimeShaderBuilder blurBuilder(mBlurEffect); + blurBuilder.child("child") = std::move(input); + blurBuilder.uniform("in_inverseScale") = inverseScale; + blurBuilder.uniform("in_blurOffset") = radius; + blurBuilder.uniform("in_crossFade") = alpha; + SkPaint paint; + paint.setShader(blurBuilder.makeShader(nullptr)); + paint.setBlendMode(alpha == 1.0f ? SkBlendMode::kSrc : SkBlendMode::kSrcOver); + drawSurface->getCanvas()->drawPaint(paint); +} + +sk_sp<SkImage> KawaseBlurDualFilter::generate(SkiaGpuContext* context, const uint32_t blurRadius, + const sk_sp<SkImage> input, + const SkRect& blurRect) const { + // Apply a conversion factor of (1 / sqrt(3)) to match Skia's built-in blur as used by + // RenderEffect. See the comment in SkBlurMask.cpp for reasoning behind this. + const float radius = blurRadius * 0.57735f; + + // Use a variable number of blur passes depending on the radius. The non-integer part of this + // calculation is used to mix the final pass into the second-last with an alpha blend. + constexpr int kMaxSurfaces = 4; + const float filterDepth = + std::min(kMaxSurfaces - 1.0f, 1.0f + std::max(0.0f, log2f(radius * kInputScale))); + const int filterPasses = std::min(kMaxSurfaces - 1, static_cast<int>(ceil(filterDepth))); + + // Render into surfaces downscaled by 1x, 1x, 2x, and 4x from the initial downscale. + sk_sp<SkSurface> surfaces[kMaxSurfaces] = + {filterPasses >= 0 ? makeSurface(context, blurRect, 1 * kInverseInputScale) : nullptr, + filterPasses >= 1 ? makeSurface(context, blurRect, 1 * kInverseInputScale) : nullptr, + filterPasses >= 2 ? makeSurface(context, blurRect, 2 * kInverseInputScale) : nullptr, + filterPasses >= 3 ? makeSurface(context, blurRect, 4 * kInverseInputScale) : nullptr}; + + // These weights for scaling offsets per-pass are handpicked to look good at 1 <= radius <= 600. + static const float kWeights[7] = {1.0f, 2.0f, 3.5f, 1.0f, 2.0f, 2.0f, 2.0f}; + + // Kawase is an approximation of Gaussian, but behaves differently because it is made up of many + // simpler blurs. A transformation is required to approximate the same effect as Gaussian. + float sumSquaredR = powf(kWeights[0] * powf(2.0f, 1), 2.0f); + for (int i = 0; i < filterPasses; i++) { + const float alpha = std::min(1.0f, filterDepth - i); + sumSquaredR += powf(powf(2.0f, i + 1) * alpha * kWeights[1 + i], 2.0f); + sumSquaredR += powf(powf(2.0f, i + 1) * alpha * kWeights[6 - i], 2.0f); + } + // Solve for R = sqrt(sum(r_i^2)). Divide R by hypot(1,1) to find some (x,y) offsets. + const float step = M_SQRT1_2 * + sqrtf(max(0.0f, (powf(radius, 2.0f) - powf(kInverseInputScale, 2.0f)) / sumSquaredR)); + + // Start by downscaling and doing the first blur pass. + { + // For sampling Skia's API expects the inverse of what logically seems appropriate. In this + // case one may expect Translate(blurRect.fLeft, blurRect.fTop) * Scale(kInverseInputScale) + // but instead we must do the inverse. + SkMatrix blurMatrix = SkMatrix::Translate(-blurRect.fLeft, -blurRect.fTop); + blurMatrix.postScale(kInputScale, kInputScale); + const auto sourceShader = + input->makeShader(SkTileMode::kClamp, SkTileMode::kClamp, + SkSamplingOptions(SkFilterMode::kLinear, SkMipmapMode::kNone), + blurMatrix); + blurInto(surfaces[0], std::move(sourceShader), kInputScale, kWeights[0] * step, 1.0f); + } + // Next the remaining downscale blur passes. + for (int i = 0; i < filterPasses; i++) { + blurInto(surfaces[i + 1], surfaces[i]->makeImageSnapshot(), kWeights[1 + i] * step, 1.0f); + } + // Finally blur+upscale back to our original size. + for (int i = filterPasses - 1; i >= 0; i--) { + blurInto(surfaces[i], surfaces[i + 1]->makeImageSnapshot(), kWeights[6 - i] * step, + std::min(1.0f, filterDepth - i)); + } + return surfaces[0]->makeImageSnapshot(); +} + +} // namespace skia +} // namespace renderengine +} // namespace android diff --git a/libs/renderengine/skia/filters/KawaseBlurDualFilter.h b/libs/renderengine/skia/filters/KawaseBlurDualFilter.h new file mode 100644 index 0000000000..6f4adbf34c --- /dev/null +++ b/libs/renderengine/skia/filters/KawaseBlurDualFilter.h @@ -0,0 +1,55 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <SkCanvas.h> +#include <SkImage.h> +#include <SkRuntimeEffect.h> +#include <SkSurface.h> +#include "BlurFilter.h" + +namespace android { +namespace renderengine { +namespace skia { + +/** + * This is an implementation of a Kawase blur with dual-filtering passes, as described in here: + * https://community.arm.com/cfs-file/__key/communityserver-blogs-components-weblogfiles/00-00-00-20-66/siggraph2015_2D00_mmg_2D00_marius_2D00_slides.pdf + * https://community.arm.com/cfs-file/__key/communityserver-blogs-components-weblogfiles/00-00-00-20-66/siggraph2015_2D00_mmg_2D00_marius_2D00_notes.pdf + */ +class KawaseBlurDualFilter : public BlurFilter { +public: + explicit KawaseBlurDualFilter(); + virtual ~KawaseBlurDualFilter() {} + + // Execute blur, saving it to a texture + sk_sp<SkImage> generate(SkiaGpuContext* context, const uint32_t radius, + const sk_sp<SkImage> blurInput, const SkRect& blurRect) const override; + +private: + sk_sp<SkRuntimeEffect> mBlurEffect; + + void blurInto(const sk_sp<SkSurface>& drawSurface, const sk_sp<SkImage>& readImage, + const float radius, const float alpha) const; + + void blurInto(const sk_sp<SkSurface>& drawSurface, const sk_sp<SkShader> input, + const float inverseScale, const float radius, const float alpha) const; +}; + +} // namespace skia +} // namespace renderengine +} // namespace android diff --git a/libs/renderengine/skia/filters/KawaseBlurFilter.cpp b/libs/renderengine/skia/filters/KawaseBlurFilter.cpp index 7a070d7024..defaf6e476 100644 --- a/libs/renderengine/skia/filters/KawaseBlurFilter.cpp +++ b/libs/renderengine/skia/filters/KawaseBlurFilter.cpp @@ -29,10 +29,10 @@ #include <SkString.h> #include <SkSurface.h> #include <SkTileMode.h> +#include <common/trace.h> #include <include/gpu/GpuTypes.h> #include <include/gpu/ganesh/SkSurfaceGanesh.h> #include <log/log.h> -#include <utils/Trace.h> namespace android { namespace renderengine { diff --git a/libs/renderengine/skia/filters/LinearEffect.cpp b/libs/renderengine/skia/filters/LinearEffect.cpp index f7dcd3a6e1..3bc3564f4c 100644 --- a/libs/renderengine/skia/filters/LinearEffect.cpp +++ b/libs/renderengine/skia/filters/LinearEffect.cpp @@ -19,9 +19,9 @@ #define ATRACE_TAG ATRACE_TAG_GRAPHICS #include <SkString.h> +#include <common/trace.h> #include <log/log.h> #include <shaders/shaders.h> -#include <utils/Trace.h> #include <math/mat4.h> @@ -30,7 +30,7 @@ namespace renderengine { namespace skia { sk_sp<SkRuntimeEffect> buildRuntimeEffect(const shaders::LinearEffect& linearEffect) { - ATRACE_CALL(); + SFTRACE_CALL(); SkString shaderString = SkString(shaders::buildLinearEffectSkSL(linearEffect)); auto [shader, error] = SkRuntimeEffect::MakeForShader(shaderString); @@ -45,7 +45,7 @@ sk_sp<SkShader> createLinearEffectShader( sk_sp<SkRuntimeEffect> runtimeEffect, const mat4& colorTransform, float maxDisplayLuminance, float currentDisplayLuminanceNits, float maxLuminance, AHardwareBuffer* buffer, aidl::android::hardware::graphics::composer3::RenderIntent renderIntent) { - ATRACE_CALL(); + SFTRACE_CALL(); SkRuntimeShaderBuilder effectBuilder(runtimeEffect); effectBuilder.child("child") = shader; diff --git a/libs/renderengine/skia/filters/MouriMap.cpp b/libs/renderengine/skia/filters/MouriMap.cpp index 7d8b8a5116..b099bcf3d7 100644 --- a/libs/renderengine/skia/filters/MouriMap.cpp +++ b/libs/renderengine/skia/filters/MouriMap.cpp @@ -35,7 +35,7 @@ const SkString kCrosstalkAndChunk16x16(R"( float maximum = 0.0; for (int y = 0; y < 16; y++) { for (int x = 0; x < 16; x++) { - float3 linear = toLinearSrgb(bitmap.eval(xy * 16 + vec2(x, y)).rgb) * hdrSdrRatio; + float3 linear = toLinearSrgb(bitmap.eval((xy - 0.5) * 16 + 0.5 + vec2(x, y)).rgb) * hdrSdrRatio; float maxRGB = max(linear.r, max(linear.g, linear.b)); maximum = max(maximum, log2(max(maxRGB, 1.0))); } @@ -49,7 +49,7 @@ const SkString kChunk8x8(R"( float maximum = 0.0; for (int y = 0; y < 8; y++) { for (int x = 0; x < 8; x++) { - maximum = max(maximum, bitmap.eval(xy * 8 + vec2(x, y)).r); + maximum = max(maximum, bitmap.eval((xy - 0.5) * 8 + 0.5 + vec2(x, y)).r); } } return float4(float3(maximum), 1.0); @@ -67,7 +67,7 @@ const SkString kBlur(R"( float result = 0.0; for (int y = -2; y <= 2; y++) { for (int x = -2; x <= 2; x++) { - result += C[y + 2] * C[x + 2] * bitmap.eval(xy + vec2(x, y)).r; + result += C[y + 2] * C[x + 2] * bitmap.eval(xy + vec2(x, y)).r; } } return float4(float3(exp2(result)), 1.0); @@ -78,19 +78,21 @@ const SkString kTonemap(R"( uniform shader lux; uniform float scaleFactor; uniform float hdrSdrRatio; + uniform float targetHdrSdrRatio; vec4 main(vec2 xy) { float localMax = lux.eval(xy * scaleFactor).r; float4 rgba = image.eval(xy); float3 linear = toLinearSrgb(rgba.rgb) * hdrSdrRatio; - if (localMax <= 1.0) { - return float4(fromLinearSrgb(linear), 1.0); + if (localMax <= targetHdrSdrRatio) { + return float4(fromLinearSrgb(linear), rgba.a); } float maxRGB = max(linear.r, max(linear.g, linear.b)); localMax = max(localMax, maxRGB); - float gain = (1 + maxRGB / (localMax * localMax)) / (1 + maxRGB); - return float4(fromLinearSrgb(linear * gain), 1.0); + float gain = (1 + maxRGB * (targetHdrSdrRatio / (localMax * localMax))) + / (1 + maxRGB / targetHdrSdrRatio); + return float4(fromLinearSrgb(linear * gain), rgba.a); } )"); @@ -114,10 +116,10 @@ MouriMap::MouriMap() mTonemap(makeEffect(kTonemap)) {} sk_sp<SkShader> MouriMap::mouriMap(SkiaGpuContext* context, sk_sp<SkShader> input, - float hdrSdrRatio) { + float hdrSdrRatio, float targetHdrSdrRatio) { auto downchunked = downchunk(context, input, hdrSdrRatio); auto localLux = blur(context, downchunked.get()); - return tonemap(input, localLux.get(), hdrSdrRatio); + return tonemap(input, localLux.get(), hdrSdrRatio, targetHdrSdrRatio); } sk_sp<SkImage> MouriMap::downchunk(SkiaGpuContext* context, sk_sp<SkShader> input, @@ -166,8 +168,8 @@ sk_sp<SkImage> MouriMap::blur(SkiaGpuContext* context, SkImage* input) const { LOG_ALWAYS_FATAL_IF(!blurSurface, "%s: Failed to create surface!", __func__); return makeImage(blurSurface.get(), blurBuilder); } -sk_sp<SkShader> MouriMap::tonemap(sk_sp<SkShader> input, SkImage* localLux, - float hdrSdrRatio) const { +sk_sp<SkShader> MouriMap::tonemap(sk_sp<SkShader> input, SkImage* localLux, float hdrSdrRatio, + float targetHdrSdrRatio) const { static constexpr float kScaleFactor = 1.0f / 128.0f; SkRuntimeShaderBuilder tonemapBuilder(mTonemap); tonemapBuilder.child("image") = input; @@ -176,8 +178,9 @@ sk_sp<SkShader> MouriMap::tonemap(sk_sp<SkShader> input, SkImage* localLux, SkSamplingOptions(SkFilterMode::kLinear, SkMipmapMode::kNone)); tonemapBuilder.uniform("scaleFactor") = kScaleFactor; tonemapBuilder.uniform("hdrSdrRatio") = hdrSdrRatio; + tonemapBuilder.uniform("targetHdrSdrRatio") = targetHdrSdrRatio; return tonemapBuilder.makeShader(); } } // namespace skia } // namespace renderengine -} // namespace android
\ No newline at end of file +} // namespace android diff --git a/libs/renderengine/skia/filters/MouriMap.h b/libs/renderengine/skia/filters/MouriMap.h index 3c0df8abf0..9ba2b6ff9d 100644 --- a/libs/renderengine/skia/filters/MouriMap.h +++ b/libs/renderengine/skia/filters/MouriMap.h @@ -64,13 +64,16 @@ public: // Apply the MouriMap tonemmaping operator to the input. // The HDR/SDR ratio describes the luminace range of the input. 1.0 means SDR. Anything larger // then 1.0 means that there is headroom above the SDR region. - sk_sp<SkShader> mouriMap(SkiaGpuContext* context, sk_sp<SkShader> input, float hdrSdrRatio); + // Similarly, the target HDR/SDR ratio describes the luminance range of the output. + sk_sp<SkShader> mouriMap(SkiaGpuContext* context, sk_sp<SkShader> input, float inputHdrSdrRatio, + float targetHdrSdrRatio); private: sk_sp<SkImage> downchunk(SkiaGpuContext* context, sk_sp<SkShader> input, float hdrSdrRatio) const; sk_sp<SkImage> blur(SkiaGpuContext* context, SkImage* input) const; - sk_sp<SkShader> tonemap(sk_sp<SkShader> input, SkImage* localLux, float hdrSdrRatio) const; + sk_sp<SkShader> tonemap(sk_sp<SkShader> input, SkImage* localLux, float hdrSdrRatio, + float targetHdrSdrRatio) const; const sk_sp<SkRuntimeEffect> mCrosstalkAndChunk16x16; const sk_sp<SkRuntimeEffect> mChunk8x8; const sk_sp<SkRuntimeEffect> mBlur; @@ -78,4 +81,4 @@ private: }; } // namespace skia } // namespace renderengine -} // namespace android
\ No newline at end of file +} // namespace android diff --git a/libs/renderengine/tests/Android.bp b/libs/renderengine/tests/Android.bp index 0783714eb9..7fbbf49586 100644 --- a/libs/renderengine/tests/Android.bp +++ b/libs/renderengine/tests/Android.bp @@ -66,5 +66,6 @@ cc_test { "libutils", "server_configurable_flags", "libaconfig_storage_read_api_cc", + "libtracing_perfetto", ], } diff --git a/libs/renderengine/tests/RenderEngineTest.cpp b/libs/renderengine/tests/RenderEngineTest.cpp index a8a98236e2..b5cc65f27d 100644 --- a/libs/renderengine/tests/RenderEngineTest.cpp +++ b/libs/renderengine/tests/RenderEngineTest.cpp @@ -34,9 +34,12 @@ #include <ui/ColorSpace.h> #include <ui/PixelFormat.h> +#include <algorithm> #include <chrono> #include <condition_variable> +#include <filesystem> #include <fstream> +#include <system_error> #include "../skia/SkiaGLRenderEngine.h" #include "../skia/SkiaVkRenderEngine.h" @@ -259,22 +262,51 @@ public: ~RenderEngineTest() { if (WRITE_BUFFER_TO_FILE_ON_FAILURE && ::testing::Test::HasFailure()) { - writeBufferToFile("/data/texture_out_"); + writeBufferToFile("/data/local/tmp/RenderEngineTest/"); } const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); ALOGI("**** Tearing down after %s.%s\n", test_info->test_case_name(), test_info->name()); } - void writeBufferToFile(const char* basename) { - std::string filename(basename); - filename.append(::testing::UnitTest::GetInstance()->current_test_info()->name()); - filename.append(".ppm"); - std::ofstream file(filename.c_str(), std::ios::binary); + // If called during e.g. + // `PerRenderEngineType/RenderEngineTest#drawLayers_fillBufferCheckersRotate90_colorSource/0` + // with a directory of `/data/local/tmp/RenderEngineTest`, then mBuffer will be dumped to + // `/data/local/tmp/RenderEngineTest/drawLayers_fillBufferCheckersRotate90_colorSource-0.ppm` + // + // Note: if `directory` does not exist, then its full path will be recursively created with 777 + // permissions. If `directory` already exists but does not grant the executing user write + // permissions, then saving the buffer will fail. + // + // Since this is test-only code, no security considerations are made. + void writeBufferToFile(const filesystem::path& directory) { + const auto currentTestInfo = ::testing::UnitTest::GetInstance()->current_test_info(); + LOG_ALWAYS_FATAL_IF(!currentTestInfo, + "writeBufferToFile must be called during execution of a test"); + + std::string fileName(currentTestInfo->name()); + // Test names may include the RenderEngine variant separated by '/', which would separate + // the file name into a subdirectory if not corrected. + std::replace(fileName.begin(), fileName.end(), '/', '-'); + fileName.append(".ppm"); + + std::error_code err; + filesystem::create_directories(directory, err); + if (err.value()) { + ALOGE("Unable to create directory %s for writing %s (%d: %s)", directory.c_str(), + fileName.c_str(), err.value(), err.message().c_str()); + return; + } + + // Append operator ("/") ensures exactly one "/" directly before the argument. + const filesystem::path filePath = directory / fileName; + std::ofstream file(filePath.c_str(), std::ios::binary); if (!file.is_open()) { - ALOGE("Unable to open file: %s", filename.c_str()); - ALOGE("You may need to do: \"adb shell setenforce 0\" to enable " - "surfaceflinger to write debug images"); + ALOGE("Unable to open file: %s", filePath.c_str()); + ALOGE("You may need to do: \"adb shell setenforce 0\" to enable surfaceflinger to " + "write debug images, or the %s directory might not give the executing user write " + "permission", + directory.c_str()); return; } @@ -304,6 +336,7 @@ public: } } file.write(reinterpret_cast<char*>(outBuffer.data()), outBuffer.size()); + ALOGI("Image of incorrect output written to %s", filePath.c_str()); mBuffer->getBuffer()->unlock(); } @@ -3147,6 +3180,214 @@ TEST_P(RenderEngineTest, r8_respects_color_transform_when_device_handles) { expectBufferColor(Rect(0, 0, 1, 1), 0, 70, 0, 255); } +TEST_P(RenderEngineTest, localTonemap_preservesFullscreenSdr) { + if (!GetParam()->apiSupported()) { + GTEST_SKIP(); + } + + initializeRenderEngine(); + + mBuffer = std::make_shared< + renderengine::impl:: + ExternalTexture>(sp<GraphicBuffer>::make(1, 1, HAL_PIXEL_FORMAT_RGBA_8888, 1, + GRALLOC_USAGE_SW_READ_OFTEN | + GRALLOC_USAGE_SW_WRITE_OFTEN | + GRALLOC_USAGE_HW_RENDER | + GRALLOC_USAGE_HW_TEXTURE, + "output"), + *mRE, + renderengine::impl::ExternalTexture::Usage::READABLE | + renderengine::impl::ExternalTexture::Usage::WRITEABLE); + ASSERT_EQ(0, mBuffer->getBuffer()->initCheck()); + + const auto whiteBuffer = allocateAndFillSourceBuffer(1, 1, ubyte4(51, 51, 51, 255)); + + const auto rect = Rect(0, 0, 1, 1); + const renderengine::DisplaySettings display{ + .physicalDisplay = rect, + .clip = rect, + .outputDataspace = ui::Dataspace::SRGB, + .targetLuminanceNits = 40, + .tonemapStrategy = renderengine::DisplaySettings::TonemapStrategy::Local, + }; + + const renderengine::LayerSettings whiteLayer{ + .geometry.boundaries = rect.toFloatRect(), + .source = + renderengine::PixelSource{ + .buffer = + renderengine::Buffer{ + .buffer = whiteBuffer, + }, + }, + .alpha = 1.0f, + .sourceDataspace = ui::Dataspace::V0_SCRGB_LINEAR, + .whitePointNits = 200, + }; + + std::vector<renderengine::LayerSettings> layers{whiteLayer}; + invokeDraw(display, layers); + + expectBufferColor(Rect(0, 0, 1, 1), 255, 255, 255, 255); +} + +TEST_P(RenderEngineTest, localTonemap_preservesFarawaySdrRegions) { + if (!GetParam()->apiSupported()) { + GTEST_SKIP(); + } + + initializeRenderEngine(); + + const auto blockWidth = 256; + const auto width = blockWidth * 4; + + const auto buffer = allocateSourceBuffer(width, 1); + + mBuffer = std::make_shared< + renderengine::impl:: + ExternalTexture>(sp<GraphicBuffer>::make(width, 1, HAL_PIXEL_FORMAT_RGBA_8888, + 1, + GRALLOC_USAGE_SW_READ_OFTEN | + GRALLOC_USAGE_SW_WRITE_OFTEN | + GRALLOC_USAGE_HW_RENDER | + GRALLOC_USAGE_HW_TEXTURE, + "output"), + *mRE, + renderengine::impl::ExternalTexture::Usage::READABLE | + renderengine::impl::ExternalTexture::Usage::WRITEABLE); + + { + uint8_t* pixels; + buffer->getBuffer()->lock(GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN, + reinterpret_cast<void**>(&pixels)); + uint8_t* dst = pixels; + for (uint32_t i = 0; i < width; i++) { + uint8_t value = 0; + if (i < blockWidth) { + value = 51; + } else if (i >= blockWidth * 3) { + value = 255; + } + dst[0] = value; + dst[1] = value; + dst[2] = value; + dst[3] = 255; + dst += 4; + } + buffer->getBuffer()->unlock(); + } + + const auto rect = Rect(0, 0, width, 1); + const renderengine::DisplaySettings display{ + .physicalDisplay = rect, + .clip = rect, + .outputDataspace = ui::Dataspace::V0_SRGB_LINEAR, + .targetLuminanceNits = 40, + .tonemapStrategy = renderengine::DisplaySettings::TonemapStrategy::Local, + }; + + const renderengine::LayerSettings whiteLayer{ + .geometry.boundaries = rect.toFloatRect(), + .source = + renderengine::PixelSource{ + .buffer = + renderengine::Buffer{ + .buffer = buffer, + }, + }, + .alpha = 1.0f, + .sourceDataspace = ui::Dataspace::V0_SCRGB_LINEAR, + .whitePointNits = 200, + }; + + std::vector<renderengine::LayerSettings> layers{whiteLayer}; + invokeDraw(display, layers); + + // SDR regions are boosted to preserve SDR detail. + expectBufferColor(Rect(0, 0, blockWidth, 1), 255, 255, 255, 255); + expectBufferColor(Rect(blockWidth, 0, blockWidth * 2, 1), 0, 0, 0, 255); + expectBufferColor(Rect(blockWidth * 2, 0, blockWidth * 3, 1), 0, 0, 0, 255); + expectBufferColor(Rect(blockWidth * 3, 0, blockWidth * 4, 1), 255, 255, 255, 255); +} + +TEST_P(RenderEngineTest, localTonemap_tonemapsNearbySdrRegions) { + if (!GetParam()->apiSupported()) { + GTEST_SKIP(); + } + + initializeRenderEngine(); + + const auto blockWidth = 2; + const auto width = blockWidth * 2; + + mBuffer = std::make_shared< + renderengine::impl:: + ExternalTexture>(sp<GraphicBuffer>::make(width, 1, HAL_PIXEL_FORMAT_RGBA_8888, + 1, + GRALLOC_USAGE_SW_READ_OFTEN | + GRALLOC_USAGE_SW_WRITE_OFTEN | + GRALLOC_USAGE_HW_RENDER | + GRALLOC_USAGE_HW_TEXTURE, + "output"), + *mRE, + renderengine::impl::ExternalTexture::Usage::READABLE | + renderengine::impl::ExternalTexture::Usage::WRITEABLE); + + const auto buffer = allocateSourceBuffer(width, 1); + + { + uint8_t* pixels; + buffer->getBuffer()->lock(GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN, + reinterpret_cast<void**>(&pixels)); + uint8_t* dst = pixels; + for (uint32_t i = 0; i < width; i++) { + uint8_t value = 0; + if (i < blockWidth) { + value = 51; + } else if (i >= blockWidth) { + value = 255; + } + dst[0] = value; + dst[1] = value; + dst[2] = value; + dst[3] = 255; + dst += 4; + } + buffer->getBuffer()->unlock(); + } + + const auto rect = Rect(0, 0, width, 1); + const renderengine::DisplaySettings display{ + .physicalDisplay = rect, + .clip = rect, + .outputDataspace = ui::Dataspace::V0_SRGB_LINEAR, + .targetLuminanceNits = 40, + .tonemapStrategy = renderengine::DisplaySettings::TonemapStrategy::Local, + }; + + const renderengine::LayerSettings whiteLayer{ + .geometry.boundaries = rect.toFloatRect(), + .source = + renderengine::PixelSource{ + .buffer = + renderengine::Buffer{ + .buffer = buffer, + }, + }, + .alpha = 1.0f, + .sourceDataspace = ui::Dataspace::V0_SCRGB_LINEAR, + .whitePointNits = 200, + }; + + std::vector<renderengine::LayerSettings> layers{whiteLayer}; + invokeDraw(display, layers); + + // SDR regions remain "dimmed", but preserve detail with a roll-off curve. + expectBufferColor(Rect(0, 0, blockWidth, 1), 132, 132, 132, 255, 2); + // HDR regions are not dimmed. + expectBufferColor(Rect(blockWidth, 0, blockWidth * 2, 1), 255, 255, 255, 255); +} + TEST_P(RenderEngineTest, primeShaderCache) { // TODO: b/331447071 - Fix in Graphite and re-enable. if (GetParam()->skiaBackend() == renderengine::RenderEngine::SkiaBackend::GRAPHITE) { diff --git a/libs/renderengine/threaded/RenderEngineThreaded.cpp b/libs/renderengine/threaded/RenderEngineThreaded.cpp index d27c151e72..c187f93089 100644 --- a/libs/renderengine/threaded/RenderEngineThreaded.cpp +++ b/libs/renderengine/threaded/RenderEngineThreaded.cpp @@ -23,9 +23,9 @@ #include <future> #include <android-base/stringprintf.h> +#include <common/trace.h> #include <private/gui/SyncFeatures.h> #include <processgroup/processgroup.h> -#include <utils/Trace.h> using namespace std::chrono_literals; @@ -39,7 +39,7 @@ std::unique_ptr<RenderEngineThreaded> RenderEngineThreaded::create(CreateInstanc RenderEngineThreaded::RenderEngineThreaded(CreateInstanceFactory factory) : RenderEngine(Threaded::YES) { - ATRACE_CALL(); + SFTRACE_CALL(); std::lock_guard lockThread(mThreadMutex); mThread = std::thread(&RenderEngineThreaded::threadMain, this, factory); @@ -76,7 +76,7 @@ status_t RenderEngineThreaded::setSchedFifo(bool enabled) { // NO_THREAD_SAFETY_ANALYSIS is because std::unique_lock presently lacks thread safety annotations. void RenderEngineThreaded::threadMain(CreateInstanceFactory factory) NO_THREAD_SAFETY_ANALYSIS { - ATRACE_CALL(); + SFTRACE_CALL(); if (!SetTaskProfiles(0, {"SFRenderEnginePolicy"})) { ALOGW("Failed to set render-engine task profile!"); @@ -133,13 +133,13 @@ void RenderEngineThreaded::waitUntilInitialized() const { std::future<void> RenderEngineThreaded::primeCache(PrimeCacheConfig config) { const auto resultPromise = std::make_shared<std::promise<void>>(); std::future<void> resultFuture = resultPromise->get_future(); - ATRACE_CALL(); + SFTRACE_CALL(); // This function is designed so it can run asynchronously, so we do not need to wait // for the futures. { std::lock_guard lock(mThreadMutex); mFunctionCalls.push([resultPromise, config](renderengine::RenderEngine& instance) { - ATRACE_NAME("REThreaded::primeCache"); + SFTRACE_NAME("REThreaded::primeCache"); if (setSchedFifo(false) != NO_ERROR) { ALOGW("Couldn't set SCHED_OTHER for primeCache"); } @@ -163,7 +163,7 @@ void RenderEngineThreaded::dump(std::string& result) { { std::lock_guard lock(mThreadMutex); mFunctionCalls.push([&resultPromise, &result](renderengine::RenderEngine& instance) { - ATRACE_NAME("REThreaded::dump"); + SFTRACE_NAME("REThreaded::dump"); std::string localResult = result; instance.dump(localResult); resultPromise.set_value(std::move(localResult)); @@ -176,13 +176,13 @@ void RenderEngineThreaded::dump(std::string& result) { void RenderEngineThreaded::mapExternalTextureBuffer(const sp<GraphicBuffer>& buffer, bool isRenderable) { - ATRACE_CALL(); + SFTRACE_CALL(); // This function is designed so it can run asynchronously, so we do not need to wait // for the futures. { std::lock_guard lock(mThreadMutex); mFunctionCalls.push([=](renderengine::RenderEngine& instance) { - ATRACE_NAME("REThreaded::mapExternalTextureBuffer"); + SFTRACE_NAME("REThreaded::mapExternalTextureBuffer"); instance.mapExternalTextureBuffer(buffer, isRenderable); }); } @@ -190,14 +190,14 @@ void RenderEngineThreaded::mapExternalTextureBuffer(const sp<GraphicBuffer>& buf } void RenderEngineThreaded::unmapExternalTextureBuffer(sp<GraphicBuffer>&& buffer) { - ATRACE_CALL(); + SFTRACE_CALL(); // This function is designed so it can run asynchronously, so we do not need to wait // for the futures. { std::lock_guard lock(mThreadMutex); mFunctionCalls.push( [=, buffer = std::move(buffer)](renderengine::RenderEngine& instance) mutable { - ATRACE_NAME("REThreaded::unmapExternalTextureBuffer"); + SFTRACE_NAME("REThreaded::unmapExternalTextureBuffer"); instance.unmapExternalTextureBuffer(std::move(buffer)); }); } @@ -229,7 +229,7 @@ void RenderEngineThreaded::cleanupPostRender() { { std::lock_guard lock(mThreadMutex); mFunctionCalls.push([=](renderengine::RenderEngine& instance) { - ATRACE_NAME("REThreaded::cleanupPostRender"); + SFTRACE_NAME("REThreaded::cleanupPostRender"); instance.cleanupPostRender(); }); mNeedsPostRenderCleanup = false; @@ -249,10 +249,20 @@ void RenderEngineThreaded::drawLayersInternal( return; } +void RenderEngineThreaded::drawGainmapInternal( + const std::shared_ptr<std::promise<FenceResult>>&& resultPromise, + const std::shared_ptr<ExternalTexture>& sdr, base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, base::borrowed_fd&& hdrFence, + float hdrSdrRatio, ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap) { + resultPromise->set_value(Fence::NO_FENCE); + return; +} + ftl::Future<FenceResult> RenderEngineThreaded::drawLayers( const DisplaySettings& display, const std::vector<LayerSettings>& layers, const std::shared_ptr<ExternalTexture>& buffer, base::unique_fd&& bufferFence) { - ATRACE_CALL(); + SFTRACE_CALL(); const auto resultPromise = std::make_shared<std::promise<FenceResult>>(); std::future<FenceResult> resultFuture = resultPromise->get_future(); int fd = bufferFence.release(); @@ -261,8 +271,8 @@ ftl::Future<FenceResult> RenderEngineThreaded::drawLayers( mNeedsPostRenderCleanup = true; mFunctionCalls.push( [resultPromise, display, layers, buffer, fd](renderengine::RenderEngine& instance) { - ATRACE_NAME("REThreaded::drawLayers"); - instance.updateProtectedContext(layers, buffer); + SFTRACE_NAME("REThreaded::drawLayers"); + instance.updateProtectedContext(layers, {buffer.get()}); instance.drawLayersInternal(std::move(resultPromise), display, layers, buffer, base::unique_fd(fd)); }); @@ -271,13 +281,37 @@ ftl::Future<FenceResult> RenderEngineThreaded::drawLayers( return resultFuture; } +ftl::Future<FenceResult> RenderEngineThreaded::drawGainmap( + const std::shared_ptr<ExternalTexture>& sdr, base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, base::borrowed_fd&& hdrFence, + float hdrSdrRatio, ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap) { + SFTRACE_CALL(); + const auto resultPromise = std::make_shared<std::promise<FenceResult>>(); + std::future<FenceResult> resultFuture = resultPromise->get_future(); + { + std::lock_guard lock(mThreadMutex); + mNeedsPostRenderCleanup = true; + mFunctionCalls.push([resultPromise, sdr, sdrFence = std::move(sdrFence), hdr, + hdrFence = std::move(hdrFence), hdrSdrRatio, dataspace, + gainmap](renderengine::RenderEngine& instance) mutable { + SFTRACE_NAME("REThreaded::drawGainmap"); + instance.updateProtectedContext({}, {sdr.get(), hdr.get(), gainmap.get()}); + instance.drawGainmapInternal(std::move(resultPromise), sdr, std::move(sdrFence), hdr, + std::move(hdrFence), hdrSdrRatio, dataspace, gainmap); + }); + } + mCondition.notify_one(); + return resultFuture; +} + int RenderEngineThreaded::getContextPriority() { std::promise<int> resultPromise; std::future<int> resultFuture = resultPromise.get_future(); { std::lock_guard lock(mThreadMutex); mFunctionCalls.push([&resultPromise](renderengine::RenderEngine& instance) { - ATRACE_NAME("REThreaded::getContextPriority"); + SFTRACE_NAME("REThreaded::getContextPriority"); int priority = instance.getContextPriority(); resultPromise.set_value(priority); }); @@ -297,7 +331,7 @@ void RenderEngineThreaded::onActiveDisplaySizeChanged(ui::Size size) { { std::lock_guard lock(mThreadMutex); mFunctionCalls.push([size](renderengine::RenderEngine& instance) { - ATRACE_NAME("REThreaded::onActiveDisplaySizeChanged"); + SFTRACE_NAME("REThreaded::onActiveDisplaySizeChanged"); instance.onActiveDisplaySizeChanged(size); }); } @@ -324,7 +358,7 @@ void RenderEngineThreaded::setEnableTracing(bool tracingEnabled) { { std::lock_guard lock(mThreadMutex); mFunctionCalls.push([tracingEnabled](renderengine::RenderEngine& instance) { - ATRACE_NAME("REThreaded::setEnableTracing"); + SFTRACE_NAME("REThreaded::setEnableTracing"); instance.setEnableTracing(tracingEnabled); }); } diff --git a/libs/renderengine/threaded/RenderEngineThreaded.h b/libs/renderengine/threaded/RenderEngineThreaded.h index d4997d6c93..cb6e924d81 100644 --- a/libs/renderengine/threaded/RenderEngineThreaded.h +++ b/libs/renderengine/threaded/RenderEngineThreaded.h @@ -55,6 +55,12 @@ public: const std::vector<LayerSettings>& layers, const std::shared_ptr<ExternalTexture>& buffer, base::unique_fd&& bufferFence) override; + ftl::Future<FenceResult> drawGainmap(const std::shared_ptr<ExternalTexture>& sdr, + base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, + base::borrowed_fd&& hdrFence, float hdrSdrRatio, + ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap) override; int getContextPriority() override; bool supportsBackgroundBlur() override; @@ -71,6 +77,13 @@ protected: const std::vector<LayerSettings>& layers, const std::shared_ptr<ExternalTexture>& buffer, base::unique_fd&& bufferFence) override; + void drawGainmapInternal(const std::shared_ptr<std::promise<FenceResult>>&& resultPromise, + const std::shared_ptr<ExternalTexture>& sdr, + base::borrowed_fd&& sdrFence, + const std::shared_ptr<ExternalTexture>& hdr, + base::borrowed_fd&& hdrFence, float hdrSdrRatio, + ui::Dataspace dataspace, + const std::shared_ptr<ExternalTexture>& gainmap) override; private: void threadMain(CreateInstanceFactory factory); diff --git a/libs/sensor/Android.bp b/libs/sensor/Android.bp index 7fa47b45f0..659666d6b6 100644 --- a/libs/sensor/Android.bp +++ b/libs/sensor/Android.bp @@ -63,6 +63,8 @@ cc_library { "libhardware", "libpermission", "android.companion.virtual.virtualdevice_aidl-cpp", + "libaconfig_storage_read_api_cc", + "server_configurable_flags", ], static_libs: [ diff --git a/libs/sensor/SensorEventQueue.cpp b/libs/sensor/SensorEventQueue.cpp index 4438d454c0..bec9255fd7 100644 --- a/libs/sensor/SensorEventQueue.cpp +++ b/libs/sensor/SensorEventQueue.cpp @@ -15,31 +15,41 @@ */ #define LOG_TAG "Sensors" +#define ATRACE_TAG ATRACE_TAG_SYSTEM_SERVER +#include <android/sensor.h> +#include <com_android_hardware_libsensor_flags.h> +#include <cutils/trace.h> +#include <hardware/sensors-base.h> +#include <sensor/BitTube.h> +#include <sensor/ISensorEventConnection.h> +#include <sensor/Sensor.h> #include <sensor/SensorEventQueue.h> - -#include <algorithm> +#include <sensor/SensorManager.h> #include <sys/socket.h> - -#include <utils/RefBase.h> #include <utils/Looper.h> +#include <utils/RefBase.h> -#include <sensor/Sensor.h> -#include <sensor/BitTube.h> -#include <sensor/ISensorEventConnection.h> - -#include <android/sensor.h> -#include <hardware/sensors-base.h> +#include <algorithm> +#include <cinttypes> +#include <string> using std::min; +namespace libsensor_flags = com::android::hardware::libsensor::flags; // ---------------------------------------------------------------------------- namespace android { // ---------------------------------------------------------------------------- -SensorEventQueue::SensorEventQueue(const sp<ISensorEventConnection>& connection) - : mSensorEventConnection(connection), mRecBuffer(nullptr), mAvailable(0), mConsumed(0), - mNumAcksToSend(0) { +SensorEventQueue::SensorEventQueue(const sp<ISensorEventConnection>& connection, + SensorManager& sensorManager, String8 packageName) + : mSensorEventConnection(connection), + mRecBuffer(nullptr), + mSensorManager(sensorManager), + mPackageName(packageName), + mAvailable(0), + mConsumed(0), + mNumAcksToSend(0) { mRecBuffer = new ASensorEvent[MAX_RECEIVE_BUFFER_EVENT_COUNT]; } @@ -65,8 +75,8 @@ ssize_t SensorEventQueue::write(const sp<BitTube>& tube, ssize_t SensorEventQueue::read(ASensorEvent* events, size_t numEvents) { if (mAvailable == 0) { - ssize_t err = BitTube::recvObjects(mSensorChannel, - mRecBuffer, MAX_RECEIVE_BUFFER_EVENT_COUNT); + ssize_t err = + BitTube::recvObjects(mSensorChannel, mRecBuffer, MAX_RECEIVE_BUFFER_EVENT_COUNT); if (err < 0) { return err; } @@ -75,6 +85,23 @@ ssize_t SensorEventQueue::read(ASensorEvent* events, size_t numEvents) { } size_t count = min(numEvents, mAvailable); memcpy(events, mRecBuffer + mConsumed, count * sizeof(ASensorEvent)); + + if (CC_UNLIKELY(ATRACE_ENABLED()) && + libsensor_flags::sensor_event_queue_report_sensor_usage_in_tracing()) { + for (size_t i = 0; i < count; i++) { + std::optional<std::string_view> sensorName = + mSensorManager.getSensorNameByHandle(events->sensor); + if (sensorName.has_value()) { + char buffer[UINT8_MAX]; + IPCThreadState* thread = IPCThreadState::self(); + pid_t pid = (thread != nullptr) ? thread->getCallingPid() : -1; + std::snprintf(buffer, sizeof(buffer), + "Sensor event from %s to %s PID: %d (%zu/%zu)", + sensorName.value().data(), mPackageName.c_str(), pid, i, count); + ATRACE_INSTANT_FOR_TRACK(LOG_TAG, buffer); + } + } + } mAvailable -= count; mConsumed += count; return static_cast<ssize_t>(count); diff --git a/libs/sensor/SensorManager.cpp b/libs/sensor/SensorManager.cpp index 9411e204e9..7b4a86c215 100644 --- a/libs/sensor/SensorManager.cpp +++ b/libs/sensor/SensorManager.cpp @@ -38,6 +38,7 @@ #include <sensor/SensorEventQueue.h> #include <com_android_hardware_libsensor_flags.h> +namespace libsensor_flags = com::android::hardware::libsensor::flags; // ---------------------------------------------------------------------------- namespace android { @@ -72,12 +73,25 @@ int getDeviceIdForUid(uid_t uid) { return deviceId; } } - } else { - ALOGW("Cannot get virtualdevice_native service"); } return DEVICE_ID_DEFAULT; } +bool findSensorNameInList(int32_t handle, const Vector<Sensor>& sensorList, + std::string* outString) { + for (auto& sensor : sensorList) { + if (sensor.getHandle() == handle) { + std::ostringstream oss; + oss << sensor.getStringType() << ":" << sensor.getName(); + if (outString) { + *outString = oss.str(); + } + return true; + } + } + return false; +} + } // namespace Mutex SensorManager::sLock; @@ -355,6 +369,25 @@ Sensor const* SensorManager::getDefaultSensor(int type) return nullptr; } +std::optional<std::string_view> SensorManager::getSensorNameByHandle(int32_t handle) { + std::lock_guard<std::mutex> lock(mSensorHandleToNameMutex); + auto iterator = mSensorHandleToName.find(handle); + if (iterator != mSensorHandleToName.end()) { + return iterator->second; + } + + std::string sensorName; + if (!findSensorNameInList(handle, mSensors, &sensorName) && + !findSensorNameInList(handle, mDynamicSensors, &sensorName)) { + ALOGW("Cannot find sensor with handle %d", handle); + return std::nullopt; + } + + mSensorHandleToName[handle] = std::move(sensorName); + + return mSensorHandleToName[handle]; +} + sp<SensorEventQueue> SensorManager::createEventQueue( String8 packageName, int mode, String16 attributionTag) { sp<SensorEventQueue> queue; @@ -368,7 +401,7 @@ sp<SensorEventQueue> SensorManager::createEventQueue( ALOGE("createEventQueue: connection is NULL."); return nullptr; } - queue = new SensorEventQueue(connection); + queue = new SensorEventQueue(connection, *this, packageName); break; } return queue; diff --git a/libs/sensor/include/sensor/SensorEventQueue.h b/libs/sensor/include/sensor/SensorEventQueue.h index 8c3fde0fa1..d31def7425 100644 --- a/libs/sensor/include/sensor/SensorEventQueue.h +++ b/libs/sensor/include/sensor/SensorEventQueue.h @@ -20,9 +20,10 @@ #include <sys/types.h> #include <utils/Errors.h> +#include <utils/Mutex.h> #include <utils/RefBase.h> +#include <utils/String8.h> #include <utils/Timers.h> -#include <utils/Mutex.h> #include <sensor/BitTube.h> @@ -42,6 +43,7 @@ namespace android { // ---------------------------------------------------------------------------- class ISensorEventConnection; +class SensorManager; class Sensor; class Looper; @@ -65,7 +67,8 @@ public: // Default sensor sample period static constexpr int32_t SENSOR_DELAY_NORMAL = 200000; - explicit SensorEventQueue(const sp<ISensorEventConnection>& connection); + explicit SensorEventQueue(const sp<ISensorEventConnection>& connection, + SensorManager& sensorManager, String8 packageName); virtual ~SensorEventQueue(); virtual void onFirstRef(); @@ -107,6 +110,8 @@ private: mutable Mutex mLock; mutable sp<Looper> mLooper; ASensorEvent* mRecBuffer; + SensorManager& mSensorManager; + String8 mPackageName; size_t mAvailable; size_t mConsumed; uint32_t mNumAcksToSend; diff --git a/libs/sensor/include/sensor/SensorManager.h b/libs/sensor/include/sensor/SensorManager.h index 49f050a0ac..8d7237d548 100644 --- a/libs/sensor/include/sensor/SensorManager.h +++ b/libs/sensor/include/sensor/SensorManager.h @@ -17,22 +17,20 @@ #ifndef ANDROID_GUI_SENSOR_MANAGER_H #define ANDROID_GUI_SENSOR_MANAGER_H -#include <map> -#include <unordered_map> - -#include <stdint.h> -#include <sys/types.h> - #include <binder/IBinder.h> #include <binder/IPCThreadState.h> #include <binder/IServiceManager.h> - +#include <sensor/SensorEventQueue.h> +#include <stdint.h> +#include <sys/types.h> #include <utils/Errors.h> +#include <utils/String8.h> #include <utils/StrongPointer.h> #include <utils/Vector.h> -#include <utils/String8.h> -#include <sensor/SensorEventQueue.h> +#include <map> +#include <string> +#include <unordered_map> // ---------------------------------------------------------------------------- // Concrete types for the NDK @@ -66,6 +64,7 @@ public: sp<SensorEventQueue> createEventQueue( String8 packageName = String8(""), int mode = 0, String16 attributionTag = String16("")); bool isDataInjectionEnabled(); + std::optional<std::string_view> getSensorNameByHandle(int32_t handle); bool isReplayDataInjectionEnabled(); bool isHalBypassReplayDataInjectionEnabled(); int createDirectChannel(size_t size, int channelType, const native_handle_t *channelData); @@ -97,6 +96,9 @@ private: const String16 mOpPackageName; const int mDeviceId; std::unordered_map<int, sp<ISensorEventConnection>> mDirectConnection; + + std::mutex mSensorHandleToNameMutex; + std::unordered_map<int32_t, std::string> mSensorHandleToName; int32_t mDirectConnectionHandle; }; diff --git a/libs/tracing_perfetto/Android.bp b/libs/tracing_perfetto/Android.bp index 3a4c869e46..b5c56c5c52 100644 --- a/libs/tracing_perfetto/Android.bp +++ b/libs/tracing_perfetto/Android.bp @@ -40,6 +40,7 @@ cc_library_shared { ], shared_libs: [ + "libbase", "libcutils", "libperfetto_c", "android.os.flags-aconfig-cc-host", diff --git a/libs/tracing_perfetto/include/tracing_perfetto.h b/libs/tracing_perfetto/include/tracing_perfetto.h index 2c1c2a49e7..59c43d6dcc 100644 --- a/libs/tracing_perfetto/include/tracing_perfetto.h +++ b/libs/tracing_perfetto/include/tracing_perfetto.h @@ -14,40 +14,40 @@ * limitations under the License. */ -#ifndef TRACING_PERFETTO_H -#define TRACING_PERFETTO_H +#pragma once #include <stdint.h> -#include "trace_result.h" - namespace tracing_perfetto { void registerWithPerfetto(bool test = false); -Result traceBegin(uint64_t category, const char* name); +void traceBegin(uint64_t category, const char* name); + +void traceEnd(uint64_t category); -Result traceEnd(uint64_t category); +void traceAsyncBegin(uint64_t category, const char* name, int32_t cookie); -Result traceAsyncBegin(uint64_t category, const char* name, int32_t cookie); +void traceFormatBegin(uint64_t category, const char* fmt, ...); -Result traceAsyncEnd(uint64_t category, const char* name, int32_t cookie); +void traceAsyncEnd(uint64_t category, const char* name, int32_t cookie); -Result traceAsyncBeginForTrack(uint64_t category, const char* name, +void traceAsyncBeginForTrack(uint64_t category, const char* name, const char* trackName, int32_t cookie); -Result traceAsyncEndForTrack(uint64_t category, const char* trackName, +void traceAsyncEndForTrack(uint64_t category, const char* trackName, int32_t cookie); -Result traceInstant(uint64_t category, const char* name); +void traceInstant(uint64_t category, const char* name); + +void traceFormatInstant(uint64_t category, const char* fmt, ...); -Result traceInstantForTrack(uint64_t category, const char* trackName, +void traceInstantForTrack(uint64_t category, const char* trackName, const char* name); -Result traceCounter(uint64_t category, const char* name, int64_t value); +void traceCounter(uint64_t category, const char* name, int64_t value); -bool isTagEnabled(uint64_t category); +void traceCounter32(uint64_t category, const char* name, int32_t value); +bool isTagEnabled(uint64_t category); } // namespace tracing_perfetto - -#endif // TRACING_PERFETTO_H diff --git a/libs/tracing_perfetto/tests/Android.bp b/libs/tracing_perfetto/tests/Android.bp index a35b0e0c83..d203467783 100644 --- a/libs/tracing_perfetto/tests/Android.bp +++ b/libs/tracing_perfetto/tests/Android.bp @@ -26,6 +26,7 @@ cc_test { static_libs: [ "libflagtest", "libgmock", + "perfetto_trace_protos", ], cflags: [ "-Wall", @@ -35,6 +36,8 @@ cc_test { "android.os.flags-aconfig-cc-host", "libbase", "libperfetto_c", + "liblog", + "libprotobuf-cpp-lite", "libtracing_perfetto", ], srcs: [ diff --git a/libs/tracing_perfetto/tests/tracing_perfetto_test.cpp b/libs/tracing_perfetto/tests/tracing_perfetto_test.cpp index 7716b9a316..e9fee2e6cf 100644 --- a/libs/tracing_perfetto/tests/tracing_perfetto_test.cpp +++ b/libs/tracing_perfetto/tests/tracing_perfetto_test.cpp @@ -16,10 +16,10 @@ #include "tracing_perfetto.h" -#include <thread> - #include <android_os.h> #include <flag_macros.h> +#include <thread> +#include <unistd.h> #include "gtest/gtest.h" #include "perfetto/public/abi/data_source_abi.h" @@ -45,67 +45,182 @@ #include "trace_categories.h" #include "utils.h" +#include "protos/perfetto/trace/trace.pb.h" +#include "protos/perfetto/trace/trace_packet.pb.h" +#include "protos/perfetto/trace/interned_data/interned_data.pb.h" + +#include <fstream> +#include <iterator> namespace tracing_perfetto { -using ::perfetto::shlib::test_utils::AllFieldsWithId; -using ::perfetto::shlib::test_utils::FieldView; -using ::perfetto::shlib::test_utils::IdFieldView; -using ::perfetto::shlib::test_utils::MsgField; -using ::perfetto::shlib::test_utils::PbField; -using ::perfetto::shlib::test_utils::StringField; +using ::perfetto::protos::Trace; +using ::perfetto::protos::TracePacket; +using ::perfetto::protos::EventCategory; +using ::perfetto::protos::EventName; +using ::perfetto::protos::FtraceEvent; +using ::perfetto::protos::FtraceEventBundle; +using ::perfetto::protos::InternedData; + using ::perfetto::shlib::test_utils::TracingSession; -using ::perfetto::shlib::test_utils::VarIntField; -using ::testing::_; -using ::testing::ElementsAre; -using ::testing::UnorderedElementsAre; const auto PERFETTO_SDK_TRACING = ACONFIG_FLAG(android::os, perfetto_sdk_tracing); +// TODO(b/303199244): Add tests for all the library functions. class TracingPerfettoTest : public testing::Test { protected: void SetUp() override { - tracing_perfetto::registerWithPerfetto(true /* test */); + tracing_perfetto::registerWithPerfetto(false /* test */); } }; -// TODO(b/303199244): Add tests for all the library functions. - -TEST_F_WITH_FLAGS(TracingPerfettoTest, traceInstant, - REQUIRES_FLAGS_ENABLED(PERFETTO_SDK_TRACING)) { - TracingSession tracing_session = - TracingSession::Builder().set_data_source_name("track_event").Build(); - tracing_perfetto::traceInstant(TRACE_CATEGORY_INPUT, ""); - +Trace stopSession(TracingSession& tracing_session) { + tracing_session.FlushBlocking(5000); tracing_session.StopBlocking(); std::vector<uint8_t> data = tracing_session.ReadBlocking(); + std::string data_string(data.begin(), data.end()); + + perfetto::protos::Trace trace; + trace.ParseFromString(data_string); + + return trace; +} + +void verifyTrackEvent(const Trace& trace, const std::string expected_category, + const std::string& expected_name) { bool found = false; - for (struct PerfettoPbDecoderField trace_field : FieldView(data)) { - ASSERT_THAT(trace_field, PbField(perfetto_protos_Trace_packet_field_number, - MsgField(_))); - IdFieldView track_event( - trace_field, perfetto_protos_TracePacket_track_event_field_number); - if (track_event.size() == 0) { - continue; + for (const TracePacket& packet: trace.packet()) { + if (packet.has_track_event() && packet.has_interned_data()) { + + const InternedData& interned_data = packet.interned_data(); + if (interned_data.event_categories_size() > 0) { + const EventCategory& event_category = packet.interned_data().event_categories(0); + if (event_category.name() == expected_category) { + found = true; + } + } + + if (interned_data.event_names_size() > 0) { + const EventName& event_name = packet.interned_data().event_names(0); + if (event_name.name() == expected_name) { + found &= true; + } + } + + if (found) { + break; + } + } + } + EXPECT_TRUE(found); +} + +void verifyAtraceEvent(const Trace& trace, const std::string& expected_name) { + std::string expected_print_buf = "I|" + std::to_string(gettid()) + "|" + expected_name + "\n"; + + bool found = false; + for (const TracePacket& packet: trace.packet()) { + if (packet.has_ftrace_events()) { + const FtraceEventBundle& ftrace_events_bundle = packet.ftrace_events(); + + if (ftrace_events_bundle.event_size() > 0) { + const FtraceEvent& ftrace_event = ftrace_events_bundle.event(0); + if (ftrace_event.has_print() && (ftrace_event.print().buf() == expected_print_buf)) { + found = true; + break; + } + } } - found = true; - IdFieldView cat_iid_fields( - track_event.front(), - perfetto_protos_TrackEvent_category_iids_field_number); - ASSERT_THAT(cat_iid_fields, ElementsAre(VarIntField(_))); - uint64_t cat_iid = cat_iid_fields.front().value.integer64; - EXPECT_THAT( - trace_field, - AllFieldsWithId( - perfetto_protos_TracePacket_interned_data_field_number, - ElementsAre(AllFieldsWithId( - perfetto_protos_InternedData_event_categories_field_number, - ElementsAre(MsgField(UnorderedElementsAre( - PbField(perfetto_protos_EventCategory_iid_field_number, - VarIntField(cat_iid)), - PbField(perfetto_protos_EventCategory_name_field_number, - StringField("input"))))))))); } EXPECT_TRUE(found); } -} // namespace tracing_perfetto
\ No newline at end of file +TEST_F_WITH_FLAGS(TracingPerfettoTest, traceInstantWithPerfetto, + REQUIRES_FLAGS_ENABLED(PERFETTO_SDK_TRACING)) { + std::string event_category = "input"; + std::string event_name = "traceInstantWithPerfetto"; + + TracingSession tracing_session = + TracingSession::Builder().add_enabled_category(event_category).Build(); + + tracing_perfetto::traceInstant(TRACE_CATEGORY_INPUT, event_name.c_str()); + + Trace trace = stopSession(tracing_session); + + verifyTrackEvent(trace, event_category, event_name); +} + +TEST_F_WITH_FLAGS(TracingPerfettoTest, traceInstantWithAtrace, + REQUIRES_FLAGS_ENABLED(PERFETTO_SDK_TRACING)) { + std::string event_category = "input"; + std::string event_name = "traceInstantWithAtrace"; + + TracingSession tracing_session = + TracingSession::Builder().add_atrace_category(event_category).Build(); + + tracing_perfetto::traceInstant(TRACE_CATEGORY_INPUT, event_name.c_str()); + + Trace trace = stopSession(tracing_session); + + verifyAtraceEvent(trace, event_name); +} + +TEST_F_WITH_FLAGS(TracingPerfettoTest, traceInstantWithPerfettoAndAtrace, + REQUIRES_FLAGS_ENABLED(PERFETTO_SDK_TRACING)) { + std::string event_category = "input"; + std::string event_name = "traceInstantWithPerfettoAndAtrace"; + + TracingSession tracing_session = + TracingSession::Builder() + .add_atrace_category(event_category) + .add_enabled_category(event_category).Build(); + + tracing_perfetto::traceInstant(TRACE_CATEGORY_INPUT, event_name.c_str()); + + Trace trace = stopSession(tracing_session); + + verifyAtraceEvent(trace, event_name); +} + +TEST_F_WITH_FLAGS(TracingPerfettoTest, traceInstantWithPerfettoAndAtraceAndPreferTrackEvent, + REQUIRES_FLAGS_ENABLED(PERFETTO_SDK_TRACING)) { + std::string event_category = "input"; + std::string event_name = "traceInstantWithPerfettoAndAtraceAndPreferTrackEvent"; + + TracingSession tracing_session = + TracingSession::Builder() + .add_atrace_category(event_category) + .add_atrace_category_prefer_sdk(event_category) + .add_enabled_category(event_category).Build(); + + tracing_perfetto::traceInstant(TRACE_CATEGORY_INPUT, event_name.c_str()); + + Trace trace = stopSession(tracing_session); + + verifyTrackEvent(trace, event_category, event_name); +} + +TEST_F_WITH_FLAGS(TracingPerfettoTest, traceInstantWithPerfettoAndAtraceConcurrently, + REQUIRES_FLAGS_ENABLED(PERFETTO_SDK_TRACING)) { + std::string event_category = "input"; + std::string event_name = "traceInstantWithPerfettoAndAtraceConcurrently"; + + TracingSession perfetto_tracing_session = + TracingSession::Builder() + .add_atrace_category(event_category) + .add_atrace_category_prefer_sdk(event_category) + .add_enabled_category(event_category).Build(); + + TracingSession atrace_tracing_session = + TracingSession::Builder() + .add_atrace_category(event_category) + .add_enabled_category(event_category).Build(); + + tracing_perfetto::traceInstant(TRACE_CATEGORY_INPUT, event_name.c_str()); + + Trace atrace_trace = stopSession(atrace_tracing_session); + Trace perfetto_trace = stopSession(perfetto_tracing_session); + + verifyAtraceEvent(atrace_trace, event_name); + verifyAtraceEvent(perfetto_trace, event_name); +} +} // namespace tracing_perfetto diff --git a/libs/tracing_perfetto/tests/utils.cpp b/libs/tracing_perfetto/tests/utils.cpp index 9c4202808a..8c4d4a8925 100644 --- a/libs/tracing_perfetto/tests/utils.cpp +++ b/libs/tracing_perfetto/tests/utils.cpp @@ -26,6 +26,11 @@ #include "perfetto/public/protos/config/track_event/track_event_config.pzc.h" #include "perfetto/public/tracing_session.h" +#include "protos/perfetto/config/ftrace/ftrace_config.pb.h" +#include "protos/perfetto/config/track_event/track_event_config.pb.h" +#include "protos/perfetto/config/data_source_config.pb.h" +#include "protos/perfetto/config/trace_config.pb.h" + namespace perfetto { namespace shlib { namespace test_utils { @@ -44,63 +49,54 @@ std::string ToHexChars(uint8_t val) { } // namespace TracingSession TracingSession::Builder::Build() { - struct PerfettoPbMsgWriter writer; - struct PerfettoHeapBuffer* hb = PerfettoHeapBufferCreate(&writer.writer); + perfetto::protos::TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(1024); - struct perfetto_protos_TraceConfig cfg; - PerfettoPbMsgInit(&cfg.msg, &writer); + auto* track_event_ds_config = trace_config.add_data_sources()->mutable_config(); + auto* ftrace_ds_config = trace_config.add_data_sources()->mutable_config(); - { - struct perfetto_protos_TraceConfig_BufferConfig buffers; - perfetto_protos_TraceConfig_begin_buffers(&cfg, &buffers); + track_event_ds_config->set_name("track_event"); + track_event_ds_config->set_target_buffer(0); + + ftrace_ds_config->set_name("linux.ftrace"); + ftrace_ds_config->set_target_buffer(0); - perfetto_protos_TraceConfig_BufferConfig_set_size_kb(&buffers, 1024); + { + auto* ftrace_config = ftrace_ds_config->mutable_ftrace_config(); + if (!atrace_categories_.empty()) { + ftrace_config->add_ftrace_events("ftrace/print"); + for (const std::string& cat : atrace_categories_) { + ftrace_config->add_atrace_categories(cat); + } - perfetto_protos_TraceConfig_end_buffers(&cfg, &buffers); + for (const std::string& cat : atrace_categories_prefer_sdk_) { + ftrace_config->add_atrace_categories_prefer_sdk(cat); + } + } } { - struct perfetto_protos_TraceConfig_DataSource data_sources; - perfetto_protos_TraceConfig_begin_data_sources(&cfg, &data_sources); - - { - struct perfetto_protos_DataSourceConfig ds_cfg; - perfetto_protos_TraceConfig_DataSource_begin_config(&data_sources, - &ds_cfg); - - perfetto_protos_DataSourceConfig_set_cstr_name(&ds_cfg, - data_source_name_.c_str()); - if (!enabled_categories_.empty() && !disabled_categories_.empty()) { - perfetto_protos_TrackEventConfig te_cfg; - perfetto_protos_DataSourceConfig_begin_track_event_config(&ds_cfg, - &te_cfg); - for (const std::string& cat : enabled_categories_) { - perfetto_protos_TrackEventConfig_set_enabled_categories( - &te_cfg, cat.data(), cat.size()); - } - for (const std::string& cat : disabled_categories_) { - perfetto_protos_TrackEventConfig_set_disabled_categories( - &te_cfg, cat.data(), cat.size()); - } - perfetto_protos_DataSourceConfig_end_track_event_config(&ds_cfg, - &te_cfg); + auto* track_event_config = track_event_ds_config->mutable_track_event_config(); + if (!enabled_categories_.empty() || !disabled_categories_.empty()) { + for (const std::string& cat : enabled_categories_) { + track_event_config->add_enabled_categories(cat); } - perfetto_protos_TraceConfig_DataSource_end_config(&data_sources, &ds_cfg); + for (const std::string& cat : disabled_categories_) { + track_event_config->add_disabled_categories(cat); + } } - - perfetto_protos_TraceConfig_end_data_sources(&cfg, &data_sources); } - size_t cfg_size = PerfettoStreamWriterGetWrittenSize(&writer.writer); - std::unique_ptr<uint8_t[]> ser(new uint8_t[cfg_size]); - PerfettoHeapBufferCopyInto(hb, &writer.writer, ser.get(), cfg_size); - PerfettoHeapBufferDestroy(hb, &writer.writer); struct PerfettoTracingSessionImpl* ts = - PerfettoTracingSessionCreate(PERFETTO_BACKEND_IN_PROCESS); + PerfettoTracingSessionCreate(PERFETTO_BACKEND_SYSTEM); + + std::string trace_config_string; + trace_config.SerializeToString(&trace_config_string); - PerfettoTracingSessionSetup(ts, ser.get(), cfg_size); + PerfettoTracingSessionSetup(ts, trace_config_string.data(), trace_config_string.length()); + // Fails to start here PerfettoTracingSessionStartBlocking(ts); return TracingSession::Adopt(ts); diff --git a/libs/tracing_perfetto/tests/utils.h b/libs/tracing_perfetto/tests/utils.h index 4353554963..8edb4143ee 100644 --- a/libs/tracing_perfetto/tests/utils.h +++ b/libs/tracing_perfetto/tests/utils.h @@ -74,10 +74,6 @@ class TracingSession { class Builder { public: Builder() = default; - Builder& set_data_source_name(std::string data_source_name) { - data_source_name_ = std::move(data_source_name); - return *this; - } Builder& add_enabled_category(std::string category) { enabled_categories_.push_back(std::move(category)); return *this; @@ -86,12 +82,21 @@ class TracingSession { disabled_categories_.push_back(std::move(category)); return *this; } + Builder& add_atrace_category(std::string category) { + atrace_categories_.push_back(std::move(category)); + return *this; + } + Builder& add_atrace_category_prefer_sdk(std::string category) { + atrace_categories_prefer_sdk_.push_back(std::move(category)); + return *this; + } TracingSession Build(); private: - std::string data_source_name_; std::vector<std::string> enabled_categories_; std::vector<std::string> disabled_categories_; + std::vector<std::string> atrace_categories_; + std::vector<std::string> atrace_categories_prefer_sdk_; }; static TracingSession Adopt(struct PerfettoTracingSessionImpl*); diff --git a/libs/tracing_perfetto/tracing_perfetto.cpp b/libs/tracing_perfetto/tracing_perfetto.cpp index 6f716eea9a..c35e078b02 100644 --- a/libs/tracing_perfetto/tracing_perfetto.cpp +++ b/libs/tracing_perfetto/tracing_perfetto.cpp @@ -17,6 +17,7 @@ #include "tracing_perfetto.h" #include <cutils/trace.h> +#include <cstdarg> #include "perfetto/public/te_category_macros.h" #include "trace_categories.h" @@ -28,116 +29,172 @@ void registerWithPerfetto(bool test) { internal::registerWithPerfetto(test); } -Result traceBegin(uint64_t category, const char* name) { +void traceBegin(uint64_t category, const char* name) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceBegin(*perfettoTeCategory, name); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_begin(category, name); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceBegin(*perfettoTeCategory, name); + } +} + +void traceFormatBegin(uint64_t category, const char* fmt, ...) { + struct PerfettoTeCategory* perfettoTeCategory = + internal::toPerfettoCategory(category); + const bool preferAtrace = internal::shouldPreferAtrace(perfettoTeCategory, category); + const bool preferPerfetto = internal::isPerfettoCategoryEnabled(perfettoTeCategory); + if (CC_LIKELY(!(preferAtrace || preferPerfetto))) { + return; + } + + const int BUFFER_SIZE = 256; + va_list ap; + char buf[BUFFER_SIZE]; + + va_start(ap, fmt); + vsnprintf(buf, BUFFER_SIZE, fmt, ap); + va_end(ap); + + + if (preferAtrace) { + atrace_begin(category, buf); + } else if (preferPerfetto) { + internal::perfettoTraceBegin(*perfettoTeCategory, buf); } } -Result traceEnd(uint64_t category) { +void traceEnd(uint64_t category) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceEnd(*perfettoTeCategory); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_end(category); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceEnd(*perfettoTeCategory); } } -Result traceAsyncBegin(uint64_t category, const char* name, int32_t cookie) { +void traceAsyncBegin(uint64_t category, const char* name, int32_t cookie) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceAsyncBegin(*perfettoTeCategory, name, cookie); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_async_begin(category, name, cookie); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceAsyncBegin(*perfettoTeCategory, name, cookie); } } -Result traceAsyncEnd(uint64_t category, const char* name, int32_t cookie) { +void traceAsyncEnd(uint64_t category, const char* name, int32_t cookie) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceAsyncEnd(*perfettoTeCategory, name, cookie); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_async_end(category, name, cookie); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceAsyncEnd(*perfettoTeCategory, name, cookie); } } -Result traceAsyncBeginForTrack(uint64_t category, const char* name, +void traceAsyncBeginForTrack(uint64_t category, const char* name, const char* trackName, int32_t cookie) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceAsyncBeginForTrack(*perfettoTeCategory, name, trackName, cookie); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_async_for_track_begin(category, trackName, name, cookie); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceAsyncBeginForTrack(*perfettoTeCategory, name, trackName, cookie); } } -Result traceAsyncEndForTrack(uint64_t category, const char* trackName, +void traceAsyncEndForTrack(uint64_t category, const char* trackName, int32_t cookie) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceAsyncEndForTrack(*perfettoTeCategory, trackName, cookie); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_async_for_track_end(category, trackName, cookie); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceAsyncEndForTrack(*perfettoTeCategory, trackName, cookie); } } -Result traceInstant(uint64_t category, const char* name) { +void traceInstant(uint64_t category, const char* name) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceInstant(*perfettoTeCategory, name); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_instant(category, name); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceInstant(*perfettoTeCategory, name); + } +} + +void traceFormatInstant(uint64_t category, const char* fmt, ...) { + struct PerfettoTeCategory* perfettoTeCategory = + internal::toPerfettoCategory(category); + const bool preferAtrace = internal::shouldPreferAtrace(perfettoTeCategory, category); + const bool preferPerfetto = internal::isPerfettoCategoryEnabled(perfettoTeCategory); + if (CC_LIKELY(!(preferAtrace || preferPerfetto))) { + return; + } + + const int BUFFER_SIZE = 256; + va_list ap; + char buf[BUFFER_SIZE]; + + va_start(ap, fmt); + vsnprintf(buf, BUFFER_SIZE, fmt, ap); + va_end(ap); + + if (preferAtrace) { + atrace_instant(category, buf); + } else if (preferPerfetto) { + internal::perfettoTraceInstant(*perfettoTeCategory, buf); } } -Result traceInstantForTrack(uint64_t category, const char* trackName, +void traceInstantForTrack(uint64_t category, const char* trackName, const char* name) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceInstantForTrack(*perfettoTeCategory, trackName, name); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_instant_for_track(category, trackName, name); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceInstantForTrack(*perfettoTeCategory, trackName, name); } } -Result traceCounter(uint64_t category, const char* name, int64_t value) { +void traceCounter(uint64_t category, const char* name, int64_t value) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return internal::perfettoTraceCounter(*perfettoTeCategory, name, value); - } else { + + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { atrace_int64(category, name, value); - return Result::SUCCESS; + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceCounter(*perfettoTeCategory, name, value); + } +} + +void traceCounter32(uint64_t category, const char* name, int32_t value) { + struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); + if (internal::shouldPreferAtrace(perfettoTeCategory, category)) { + atrace_int(category, name, value); + } else if (internal::isPerfettoCategoryEnabled(perfettoTeCategory)) { + internal::perfettoTraceCounter(*perfettoTeCategory, name, + static_cast<int64_t>(value)); } } bool isTagEnabled(uint64_t category) { struct PerfettoTeCategory* perfettoTeCategory = internal::toPerfettoCategory(category); - if (perfettoTeCategory != nullptr) { - return true; - } else { - return (atrace_get_enabled_tags() & category) != 0; - } + return internal::isPerfettoCategoryEnabled(perfettoTeCategory) + || atrace_is_tag_enabled(category); } } // namespace tracing_perfetto diff --git a/libs/tracing_perfetto/tracing_perfetto_internal.cpp b/libs/tracing_perfetto/tracing_perfetto_internal.cpp index 758ace63ab..9a0042aee5 100644 --- a/libs/tracing_perfetto/tracing_perfetto_internal.cpp +++ b/libs/tracing_perfetto/tracing_perfetto_internal.cpp @@ -44,13 +44,13 @@ C(rro, "rro", "RRO category") \ C(thermal, "thermal", "Thermal category") -#include "tracing_perfetto_internal.h" - -#include <inttypes.h> - +#include <atomic> #include <mutex> #include <android_os.h> +#include <android-base/properties.h> +#include <cutils/trace.h> +#include <inttypes.h> #include "perfetto/public/compiler.h" #include "perfetto/public/producer.h" @@ -58,19 +58,42 @@ #include "perfetto/public/te_macros.h" #include "perfetto/public/track_event.h" #include "trace_categories.h" -#include "trace_result.h" +#include "tracing_perfetto_internal.h" + +#ifdef __BIONIC__ +#define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ +#include <sys/_system_properties.h> +#endif namespace tracing_perfetto { namespace internal { namespace { - PERFETTO_TE_CATEGORIES_DECLARE(FRAMEWORK_CATEGORIES); PERFETTO_TE_CATEGORIES_DEFINE(FRAMEWORK_CATEGORIES); -std::atomic_bool is_perfetto_registered = false; +static constexpr char kPreferFlagProperty[] = "debug.atrace.prefer_sdk"; +static std::atomic<const prop_info*> prefer_property_info = nullptr; +static std::atomic_uint32_t last_prefer_seq_num = 0; +static std::atomic_uint64_t prefer_flags = 0; + +static const prop_info* system_property_find(const char* name [[maybe_unused]]) { + #ifdef __BIONIC__ + return __system_property_find(name); + #endif + + return nullptr; +} + +static uint32_t system_property_serial(const prop_info* pi [[maybe_unused]]) { + #ifdef __BIONIC__ + return __system_property_serial(pi); + #endif + + return last_prefer_seq_num; +} struct PerfettoTeCategory* toCategory(uint64_t inCategory) { switch (inCategory) { @@ -137,8 +160,60 @@ struct PerfettoTeCategory* toCategory(uint64_t inCategory) { } // namespace -bool isPerfettoRegistered() { - return is_perfetto_registered; +bool isPerfettoCategoryEnabled(PerfettoTeCategory* category) { + return category != nullptr; +} + +/** + * Updates the cached |prefer_flags|. + * + * We cache the prefer_flags because reading it on every trace event is expensive. + * The cache is invalidated when a sys_prop sequence number changes. + */ +void updatePreferFlags() { + if (!prefer_property_info.load(std::memory_order_acquire)) { + auto* new_prefer_property_info = system_property_find(kPreferFlagProperty); + prefer_flags.store(android::base::GetIntProperty(kPreferFlagProperty, 0), + std::memory_order_relaxed); + + if (!new_prefer_property_info) { + // This should never happen. If it does, we fail gracefully and end up reading the property + // traced event. + return; + } + + last_prefer_seq_num = system_property_serial(new_prefer_property_info); + prefer_property_info.store(new_prefer_property_info, std::memory_order_release); + } + + uint32_t prefer_seq_num = system_property_serial(prefer_property_info); + if (prefer_seq_num != last_prefer_seq_num.load(std::memory_order_acquire)) { + prefer_flags.store(android::base::GetIntProperty(kPreferFlagProperty, 0), + std::memory_order_relaxed); + last_prefer_seq_num.store(prefer_seq_num, std::memory_order_release); + } +} + +bool shouldPreferAtrace(PerfettoTeCategory *perfettoCategory, uint64_t atraceCategory) { + // There are 3 cases: + // 1. Atrace is not enabled. + if (!atrace_is_tag_enabled(atraceCategory)) { + return false; + } + + // 2. Atrace is enabled but perfetto is not enabled. + if (!isPerfettoCategoryEnabled(perfettoCategory)) { + return true; + } + + // Update prefer_flags before checking it below + updatePreferFlags(); + + // 3. Atrace and perfetto are enabled. + // Even though this category is enabled for track events, the config mandates that we downgrade + // it to atrace if the same atrace category is currently enabled. This prevents missing the + // event from a concurrent session that needs the same category in atrace. + return (atraceCategory & prefer_flags.load(std::memory_order_relaxed)) == 0; } struct PerfettoTeCategory* toPerfettoCategory(uint64_t category) { @@ -148,7 +223,7 @@ struct PerfettoTeCategory* toPerfettoCategory(uint64_t category) { } bool enabled = PERFETTO_UNLIKELY(PERFETTO_ATOMIC_LOAD_EXPLICIT( - (*perfettoCategory).enabled, PERFETTO_MEMORY_ORDER_RELAXED)); + (*perfettoCategory).enabled, PERFETTO_MEMORY_ORDER_RELAXED)); return enabled ? perfettoCategory : nullptr; } @@ -164,70 +239,57 @@ void registerWithPerfetto(bool test) { PerfettoProducerInit(args); PerfettoTeInit(); PERFETTO_TE_REGISTER_CATEGORIES(FRAMEWORK_CATEGORIES); - is_perfetto_registered = true; }); } -Result perfettoTraceBegin(const struct PerfettoTeCategory& category, const char* name) { +void perfettoTraceBegin(const struct PerfettoTeCategory& category, const char* name) { PERFETTO_TE(category, PERFETTO_TE_SLICE_BEGIN(name)); - return Result::SUCCESS; } -Result perfettoTraceEnd(const struct PerfettoTeCategory& category) { +void perfettoTraceEnd(const struct PerfettoTeCategory& category) { PERFETTO_TE(category, PERFETTO_TE_SLICE_END()); - return Result::SUCCESS; } -Result perfettoTraceAsyncBeginForTrack(const struct PerfettoTeCategory& category, const char* name, +void perfettoTraceAsyncBeginForTrack(const struct PerfettoTeCategory& category, const char* name, const char* trackName, uint64_t cookie) { PERFETTO_TE( category, PERFETTO_TE_SLICE_BEGIN(name), PERFETTO_TE_NAMED_TRACK(trackName, cookie, PerfettoTeProcessTrackUuid())); - return Result::SUCCESS; } -Result perfettoTraceAsyncEndForTrack(const struct PerfettoTeCategory& category, +void perfettoTraceAsyncEndForTrack(const struct PerfettoTeCategory& category, const char* trackName, uint64_t cookie) { PERFETTO_TE( category, PERFETTO_TE_SLICE_END(), PERFETTO_TE_NAMED_TRACK(trackName, cookie, PerfettoTeProcessTrackUuid())); - return Result::SUCCESS; } -Result perfettoTraceAsyncBegin(const struct PerfettoTeCategory& category, const char* name, +void perfettoTraceAsyncBegin(const struct PerfettoTeCategory& category, const char* name, uint64_t cookie) { - return perfettoTraceAsyncBeginForTrack(category, name, name, cookie); + perfettoTraceAsyncBeginForTrack(category, name, name, cookie); } -Result perfettoTraceAsyncEnd(const struct PerfettoTeCategory& category, const char* name, +void perfettoTraceAsyncEnd(const struct PerfettoTeCategory& category, const char* name, uint64_t cookie) { - return perfettoTraceAsyncEndForTrack(category, name, cookie); + perfettoTraceAsyncEndForTrack(category, name, cookie); } -Result perfettoTraceInstant(const struct PerfettoTeCategory& category, const char* name) { +void perfettoTraceInstant(const struct PerfettoTeCategory& category, const char* name) { PERFETTO_TE(category, PERFETTO_TE_INSTANT(name)); - return Result::SUCCESS; } -Result perfettoTraceInstantForTrack(const struct PerfettoTeCategory& category, +void perfettoTraceInstantForTrack(const struct PerfettoTeCategory& category, const char* trackName, const char* name) { PERFETTO_TE( category, PERFETTO_TE_INSTANT(name), PERFETTO_TE_NAMED_TRACK(trackName, 1, PerfettoTeProcessTrackUuid())); - return Result::SUCCESS; } -Result perfettoTraceCounter(const struct PerfettoTeCategory& category, +void perfettoTraceCounter(const struct PerfettoTeCategory& category, [[maybe_unused]] const char* name, int64_t value) { PERFETTO_TE(category, PERFETTO_TE_COUNTER(), PERFETTO_TE_INT_COUNTER(value)); - return Result::SUCCESS; -} - -uint64_t getDefaultCategories() { - return TRACE_CATEGORIES; } - } // namespace internal } // namespace tracing_perfetto diff --git a/libs/tracing_perfetto/tracing_perfetto_internal.h b/libs/tracing_perfetto/tracing_perfetto_internal.h index 79e4b8f1b4..3e1ac2a112 100644 --- a/libs/tracing_perfetto/tracing_perfetto_internal.h +++ b/libs/tracing_perfetto/tracing_perfetto_internal.h @@ -19,7 +19,6 @@ #include <stdint.h> -#include "include/trace_result.h" #include "perfetto/public/te_category_macros.h" namespace tracing_perfetto { @@ -32,31 +31,33 @@ struct PerfettoTeCategory* toPerfettoCategory(uint64_t category); void registerWithPerfetto(bool test = false); -Result perfettoTraceBegin(const struct PerfettoTeCategory& category, const char* name); +void perfettoTraceBegin(const struct PerfettoTeCategory& category, const char* name); -Result perfettoTraceEnd(const struct PerfettoTeCategory& category); +void perfettoTraceEnd(const struct PerfettoTeCategory& category); -Result perfettoTraceAsyncBegin(const struct PerfettoTeCategory& category, const char* name, +void perfettoTraceAsyncBegin(const struct PerfettoTeCategory& category, const char* name, uint64_t cookie); -Result perfettoTraceAsyncEnd(const struct PerfettoTeCategory& category, const char* name, +void perfettoTraceAsyncEnd(const struct PerfettoTeCategory& category, const char* name, uint64_t cookie); -Result perfettoTraceAsyncBeginForTrack(const struct PerfettoTeCategory& category, const char* name, +void perfettoTraceAsyncBeginForTrack(const struct PerfettoTeCategory& category, const char* name, const char* trackName, uint64_t cookie); -Result perfettoTraceAsyncEndForTrack(const struct PerfettoTeCategory& category, +void perfettoTraceAsyncEndForTrack(const struct PerfettoTeCategory& category, const char* trackName, uint64_t cookie); -Result perfettoTraceInstant(const struct PerfettoTeCategory& category, const char* name); +void perfettoTraceInstant(const struct PerfettoTeCategory& category, const char* name); -Result perfettoTraceInstantForTrack(const struct PerfettoTeCategory& category, +void perfettoTraceInstantForTrack(const struct PerfettoTeCategory& category, const char* trackName, const char* name); -Result perfettoTraceCounter(const struct PerfettoTeCategory& category, const char* name, +void perfettoTraceCounter(const struct PerfettoTeCategory& category, const char* name, int64_t value); -uint64_t getDefaultCategories(); +bool isPerfettoCategoryEnabled(PerfettoTeCategory *perfettoTeCategory); + +bool shouldPreferAtrace(PerfettoTeCategory *perfettoTeCategory, uint64_t category); } // namespace internal diff --git a/libs/ui/DisplayIdentification.cpp b/libs/ui/DisplayIdentification.cpp index e5af7406ed..8b13d78840 100644 --- a/libs/ui/DisplayIdentification.cpp +++ b/libs/ui/DisplayIdentification.cpp @@ -26,6 +26,7 @@ #include <ftl/hash.h> #include <log/log.h> #include <ui/DisplayIdentification.h> +#include <ui/Size.h> namespace android { namespace { @@ -46,6 +47,10 @@ std::optional<uint8_t> getEdidDescriptorType(const byte_view& view) { return view[3]; } +bool isDetailedTimingDescriptor(const byte_view& view) { + return view[0] != 0 && view[1] != 0; +} + std::string_view parseEdidText(const byte_view& view) { std::string_view text(reinterpret_cast<const char*>(view.data()), view.size()); text = text.substr(0, text.find('\n')); @@ -219,6 +224,8 @@ std::optional<Edid> parseEdid(const DisplayIdentificationData& edid) { std::string_view displayName; std::string_view serialNumber; std::string_view asciiText; + ui::Size preferredDTDPixelSize; + ui::Size preferredDTDPhysicalSize; constexpr size_t kDescriptorCount = 4; constexpr size_t kDescriptorLength = 18; @@ -243,6 +250,35 @@ std::optional<Edid> parseEdid(const DisplayIdentificationData& edid) { serialNumber = parseEdidText(descriptor); break; } + } else if (isDetailedTimingDescriptor(view)) { + static constexpr size_t kHorizontalPhysicalLsbOffset = 12; + static constexpr size_t kHorizontalPhysicalMsbOffset = 14; + static constexpr size_t kVerticalPhysicalLsbOffset = 13; + static constexpr size_t kVerticalPhysicalMsbOffset = 14; + const uint32_t hSize = + static_cast<uint32_t>(view[kHorizontalPhysicalLsbOffset] | + ((view[kHorizontalPhysicalMsbOffset] >> 4) << 8)); + const uint32_t vSize = + static_cast<uint32_t>(view[kVerticalPhysicalLsbOffset] | + ((view[kVerticalPhysicalMsbOffset] & 0b1111) << 8)); + + static constexpr size_t kHorizontalPixelLsbOffset = 2; + static constexpr size_t kHorizontalPixelMsbOffset = 4; + static constexpr size_t kVerticalPixelLsbOffset = 5; + static constexpr size_t kVerticalPixelMsbOffset = 7; + + const uint8_t hLsb = view[kHorizontalPixelLsbOffset]; + const uint8_t hMsb = view[kHorizontalPixelMsbOffset]; + const int32_t hPixel = hLsb + ((hMsb & 0xF0) << 4); + + const uint8_t vLsb = view[kVerticalPixelLsbOffset]; + const uint8_t vMsb = view[kVerticalPixelMsbOffset]; + const int32_t vPixel = vLsb + ((vMsb & 0xF0) << 4); + + preferredDTDPixelSize.setWidth(hPixel); + preferredDTDPixelSize.setHeight(vPixel); + preferredDTDPhysicalSize.setWidth(hSize); + preferredDTDPhysicalSize.setHeight(vSize); } view = view.subspan(kDescriptorLength); @@ -297,14 +333,22 @@ std::optional<Edid> parseEdid(const DisplayIdentificationData& edid) { } } - return Edid{.manufacturerId = manufacturerId, - .productId = productId, - .pnpId = *pnpId, - .modelHash = modelHash, - .displayName = displayName, - .manufactureOrModelYear = manufactureOrModelYear, - .manufactureWeek = manufactureWeek, - .cea861Block = cea861Block}; + DetailedTimingDescriptor preferredDetailedTimingDescriptor{ + .pixelSizeCount = preferredDTDPixelSize, + .physicalSizeInMm = preferredDTDPhysicalSize, + }; + + return Edid{ + .manufacturerId = manufacturerId, + .productId = productId, + .pnpId = *pnpId, + .modelHash = modelHash, + .displayName = displayName, + .manufactureOrModelYear = manufactureOrModelYear, + .manufactureWeek = manufactureWeek, + .cea861Block = cea861Block, + .preferredDetailedTimingDescriptor = preferredDetailedTimingDescriptor, + }; } std::optional<PnpId> getPnpId(uint16_t manufacturerId) { @@ -336,9 +380,12 @@ std::optional<DisplayIdentificationInfo> parseDisplayIdentificationData( } const auto displayId = PhysicalDisplayId::fromEdid(port, edid->manufacturerId, edid->modelHash); - return DisplayIdentificationInfo{.id = displayId, - .name = std::string(edid->displayName), - .deviceProductInfo = buildDeviceProductInfo(*edid)}; + return DisplayIdentificationInfo{ + .id = displayId, + .name = std::string(edid->displayName), + .deviceProductInfo = buildDeviceProductInfo(*edid), + .preferredDetailedTimingDescriptor = edid->preferredDetailedTimingDescriptor, + }; } PhysicalDisplayId getVirtualDisplayId(uint32_t id) { diff --git a/libs/ui/Transform.cpp b/libs/ui/Transform.cpp index 42dd85e959..23249fa4a9 100644 --- a/libs/ui/Transform.cpp +++ b/libs/ui/Transform.cpp @@ -110,10 +110,12 @@ const vec3& Transform::operator [] (size_t i) const { return mMatrix[i]; } +// x translate float Transform::tx() const { return mMatrix[2][0]; } +// y translate float Transform::ty() const { return mMatrix[2][1]; } @@ -167,11 +169,15 @@ void Transform::set(float tx, float ty) { } } -void Transform::set(float a, float b, float c, float d) { +// x and y are the coordinates in the destination (i.e. the screen) +// s and t are the coordinates in the source (i.e. the texture) +// d means derivative +// dsdx means ds/dx derivative of s with respect to x, etc. +void Transform::set(float dsdx, float dtdy, float dtdx, float dsdy) { mat33& M(mMatrix); - M[0][0] = a; M[1][0] = b; - M[0][1] = c; M[1][1] = d; - M[0][2] = 0; M[1][2] = 0; + M[0][0] = dsdx; M[1][0] = dtdy; + M[0][1] = dtdx; M[1][1] = dsdy; + M[0][2] = 0; M[1][2] = 0; mType = UNKNOWN_TYPE; } diff --git a/libs/ui/include/ui/DisplayIdentification.h b/libs/ui/include/ui/DisplayIdentification.h index 8bc2017b55..648e024d86 100644 --- a/libs/ui/include/ui/DisplayIdentification.h +++ b/libs/ui/include/ui/DisplayIdentification.h @@ -25,6 +25,7 @@ #include <ui/DeviceProductInfo.h> #include <ui/DisplayId.h> +#include <ui/Size.h> #define LEGACY_DISPLAY_TYPE_PRIMARY 0 #define LEGACY_DISPLAY_TYPE_EXTERNAL 1 @@ -33,10 +34,16 @@ namespace android { using DisplayIdentificationData = std::vector<uint8_t>; +struct DetailedTimingDescriptor { + ui::Size pixelSizeCount; + ui::Size physicalSizeInMm; +}; + struct DisplayIdentificationInfo { PhysicalDisplayId id; std::string name; std::optional<DeviceProductInfo> deviceProductInfo; + std::optional<DetailedTimingDescriptor> preferredDetailedTimingDescriptor; }; struct ExtensionBlock { @@ -68,6 +75,7 @@ struct Edid { uint8_t manufactureOrModelYear; uint8_t manufactureWeek; std::optional<Cea861ExtensionBlock> cea861Block; + std::optional<DetailedTimingDescriptor> preferredDetailedTimingDescriptor; }; bool isEdid(const DisplayIdentificationData&); diff --git a/libs/ui/include/ui/EdgeExtensionEffect.h b/libs/ui/include/ui/EdgeExtensionEffect.h new file mode 100644 index 0000000000..02126b12be --- /dev/null +++ b/libs/ui/include/ui/EdgeExtensionEffect.h @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/** + * Stores the edge that will be extended + */ +namespace android { + +enum CanonicalDirections { NONE = 0, LEFT = 1, RIGHT = 2, TOP = 4, BOTTOM = 8 }; + +inline std::string to_string(CanonicalDirections direction) { + switch (direction) { + case LEFT: + return "LEFT"; + case RIGHT: + return "RIGHT"; + case TOP: + return "TOP"; + case BOTTOM: + return "BOTTOM"; + case NONE: + return "NONE"; + } +} + +struct EdgeExtensionEffect { + EdgeExtensionEffect(bool left, bool right, bool top, bool bottom) { + extensionEdges = NONE; + if (left) { + extensionEdges |= LEFT; + } + if (right) { + extensionEdges |= RIGHT; + } + if (top) { + extensionEdges |= TOP; + } + if (bottom) { + extensionEdges |= BOTTOM; + } + } + + EdgeExtensionEffect() { EdgeExtensionEffect(false, false, false, false); } + + bool extendsEdge(CanonicalDirections edge) const { return extensionEdges & edge; } + + bool hasEffect() const { return extensionEdges != NONE; }; + + void reset() { extensionEdges = NONE; } + + bool operator==(const EdgeExtensionEffect& other) const { + return extensionEdges == other.extensionEdges; + } + + bool operator!=(const EdgeExtensionEffect& other) const { return !(*this == other); } + +private: + int extensionEdges; +}; + +inline std::string to_string(const EdgeExtensionEffect& effect) { + std::string s = "EdgeExtensionEffect={edges=["; + if (effect.hasEffect()) { + for (CanonicalDirections edge : {LEFT, RIGHT, TOP, BOTTOM}) { + if (effect.extendsEdge(edge)) { + s += to_string(edge) + ", "; + } + } + } else { + s += to_string(NONE); + } + s += "]}"; + return s; +} + +inline std::ostream& operator<<(std::ostream& os, const EdgeExtensionEffect effect) { + os << to_string(effect); + return os; +} +} // namespace android diff --git a/libs/ui/include/ui/Fence.h b/libs/ui/include/ui/Fence.h index 9aae145c04..a75ba37d2c 100644 --- a/libs/ui/include/ui/Fence.h +++ b/libs/ui/include/ui/Fence.h @@ -156,6 +156,6 @@ private: base::unique_fd mFenceFd; }; -}; // namespace android +} // namespace android #endif // ANDROID_FENCE_H diff --git a/libs/ui/include/ui/GraphicBuffer.h b/libs/ui/include/ui/GraphicBuffer.h index 652d8ba709..936bf8f862 100644 --- a/libs/ui/include/ui/GraphicBuffer.h +++ b/libs/ui/include/ui/GraphicBuffer.h @@ -297,6 +297,6 @@ private: mDeathCallbacks; }; -}; // namespace android +} // namespace android #endif // ANDROID_GRAPHIC_BUFFER_H diff --git a/libs/ui/include/ui/GraphicBufferAllocator.h b/libs/ui/include/ui/GraphicBufferAllocator.h index bbb2d77058..97ed05af6d 100644 --- a/libs/ui/include/ui/GraphicBufferAllocator.h +++ b/libs/ui/include/ui/GraphicBufferAllocator.h @@ -137,6 +137,6 @@ protected: }; // --------------------------------------------------------------------------- -}; // namespace android +} // namespace android #endif // ANDROID_BUFFER_ALLOCATOR_H diff --git a/libs/ui/include/ui/GraphicBufferMapper.h b/libs/ui/include/ui/GraphicBufferMapper.h index 9da1447aed..91aabe9f12 100644 --- a/libs/ui/include/ui/GraphicBufferMapper.h +++ b/libs/ui/include/ui/GraphicBufferMapper.h @@ -188,7 +188,7 @@ private: // --------------------------------------------------------------------------- -}; // namespace android +} // namespace android #endif // ANDROID_UI_BUFFER_MAPPER_H diff --git a/libs/ui/include/ui/PixelFormat.h b/libs/ui/include/ui/PixelFormat.h index cf5c2e8c12..1f20787bb0 100644 --- a/libs/ui/include/ui/PixelFormat.h +++ b/libs/ui/include/ui/PixelFormat.h @@ -72,6 +72,6 @@ typedef int32_t PixelFormat; uint32_t bytesPerPixel(PixelFormat format); -}; // namespace android +} // namespace android #endif // UI_PIXELFORMAT_H diff --git a/libs/ui/include/ui/Point.h b/libs/ui/include/ui/Point.h index d050ede02d..97a54beca7 100644 --- a/libs/ui/include/ui/Point.h +++ b/libs/ui/include/ui/Point.h @@ -83,6 +83,6 @@ public: ANDROID_BASIC_TYPES_TRAITS(Point) -}; // namespace android +} // namespace android #endif // ANDROID_UI_POINT diff --git a/libs/ui/include/ui/Rect.h b/libs/ui/include/ui/Rect.h index 9e24a077ff..2eb9330cc9 100644 --- a/libs/ui/include/ui/Rect.h +++ b/libs/ui/include/ui/Rect.h @@ -233,7 +233,7 @@ void PrintTo(const Rect& rect, ::std::ostream* os); ANDROID_BASIC_TYPES_TRAITS(Rect) -}; // namespace android +} // namespace android namespace std { template <> diff --git a/libs/ui/include/ui/Region.h b/libs/ui/include/ui/Region.h index 927c334c85..d1b38f3bd3 100644 --- a/libs/ui/include/ui/Region.h +++ b/libs/ui/include/ui/Region.h @@ -233,7 +233,7 @@ static inline void PrintTo(const Region& region, ::std::ostream* os) { } // --------------------------------------------------------------------------- -}; // namespace android +} // namespace android namespace std { template <> diff --git a/libs/ui/tests/DisplayIdentification_test.cpp b/libs/ui/tests/DisplayIdentification_test.cpp index 721b46688e..76e3f66e1b 100644 --- a/libs/ui/tests/DisplayIdentification_test.cpp +++ b/libs/ui/tests/DisplayIdentification_test.cpp @@ -194,6 +194,10 @@ TEST(DisplayIdentificationTest, parseEdid) { EXPECT_EQ(21, edid->manufactureOrModelYear); EXPECT_EQ(0, edid->manufactureWeek); EXPECT_FALSE(edid->cea861Block); + EXPECT_EQ(1280, edid->preferredDetailedTimingDescriptor->pixelSizeCount.width); + EXPECT_EQ(800, edid->preferredDetailedTimingDescriptor->pixelSizeCount.height); + EXPECT_EQ(261, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.width); + EXPECT_EQ(163, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.height); edid = parseEdid(getExternalEdid()); ASSERT_TRUE(edid); @@ -206,6 +210,10 @@ TEST(DisplayIdentificationTest, parseEdid) { EXPECT_EQ(22, edid->manufactureOrModelYear); EXPECT_EQ(2, edid->manufactureWeek); EXPECT_FALSE(edid->cea861Block); + EXPECT_EQ(1280, edid->preferredDetailedTimingDescriptor->pixelSizeCount.width); + EXPECT_EQ(800, edid->preferredDetailedTimingDescriptor->pixelSizeCount.height); + EXPECT_EQ(641, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.width); + EXPECT_EQ(400, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.height); edid = parseEdid(getExternalEedid()); ASSERT_TRUE(edid); @@ -224,6 +232,10 @@ TEST(DisplayIdentificationTest, parseEdid) { EXPECT_EQ(0, physicalAddress.b); EXPECT_EQ(0, physicalAddress.c); EXPECT_EQ(0, physicalAddress.d); + EXPECT_EQ(1366, edid->preferredDetailedTimingDescriptor->pixelSizeCount.width); + EXPECT_EQ(768, edid->preferredDetailedTimingDescriptor->pixelSizeCount.height); + EXPECT_EQ(160, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.width); + EXPECT_EQ(90, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.height); edid = parseEdid(getPanasonicTvEdid()); ASSERT_TRUE(edid); @@ -242,6 +254,10 @@ TEST(DisplayIdentificationTest, parseEdid) { EXPECT_EQ(0, physicalAddress.b); EXPECT_EQ(0, physicalAddress.c); EXPECT_EQ(0, physicalAddress.d); + EXPECT_EQ(1920, edid->preferredDetailedTimingDescriptor->pixelSizeCount.width); + EXPECT_EQ(1080, edid->preferredDetailedTimingDescriptor->pixelSizeCount.height); + EXPECT_EQ(698, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.width); + EXPECT_EQ(392, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.height); edid = parseEdid(getHisenseTvEdid()); ASSERT_TRUE(edid); @@ -260,6 +276,10 @@ TEST(DisplayIdentificationTest, parseEdid) { EXPECT_EQ(2, physicalAddress.b); EXPECT_EQ(3, physicalAddress.c); EXPECT_EQ(4, physicalAddress.d); + EXPECT_EQ(1920, edid->preferredDetailedTimingDescriptor->pixelSizeCount.width); + EXPECT_EQ(1080, edid->preferredDetailedTimingDescriptor->pixelSizeCount.height); + EXPECT_EQ(575, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.width); + EXPECT_EQ(323, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.height); edid = parseEdid(getCtlDisplayEdid()); ASSERT_TRUE(edid); @@ -273,6 +293,10 @@ TEST(DisplayIdentificationTest, parseEdid) { EXPECT_EQ(0xff, edid->manufactureWeek); ASSERT_TRUE(edid->cea861Block); EXPECT_FALSE(edid->cea861Block->hdmiVendorDataBlock); + EXPECT_EQ(1360, edid->preferredDetailedTimingDescriptor->pixelSizeCount.width); + EXPECT_EQ(768, edid->preferredDetailedTimingDescriptor->pixelSizeCount.height); + EXPECT_EQ(521, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.width); + EXPECT_EQ(293, edid->preferredDetailedTimingDescriptor->physicalSizeInMm.height); } TEST(DisplayIdentificationTest, parseInvalidEdid) { diff --git a/libs/vibrator/ExternalVibration.cpp b/libs/vibrator/ExternalVibration.cpp index c97e496083..cae2de273e 100644 --- a/libs/vibrator/ExternalVibration.cpp +++ b/libs/vibrator/ExternalVibration.cpp @@ -93,8 +93,8 @@ os::HapticScale ExternalVibration::externalVibrationScaleToHapticScale( externalVibrationScale.scaleLevel); } - return {/*level=*/scaleLevel, /*adaptiveScaleFactor=*/ - externalVibrationScale.adaptiveHapticsScale}; + return os::HapticScale(scaleLevel, externalVibrationScale.scaleFactor, + externalVibrationScale.adaptiveHapticsScale); } } // namespace os diff --git a/libs/vibrator/ExternalVibrationUtils.cpp b/libs/vibrator/ExternalVibrationUtils.cpp index 706f3d7a56..ca13afcbaf 100644 --- a/libs/vibrator/ExternalVibrationUtils.cpp +++ b/libs/vibrator/ExternalVibrationUtils.cpp @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#define LOG_TAG "ExternalVibrationUtils" + #include <cstring> #include <android_os_vibrator.h> @@ -20,6 +22,7 @@ #include <algorithm> #include <math.h> +#include <log/log.h> #include <vibrator/ExternalVibrationUtils.h> namespace android::os { @@ -29,6 +32,7 @@ static constexpr float HAPTIC_SCALE_VERY_LOW_RATIO = 2.0f / 3.0f; static constexpr float HAPTIC_SCALE_LOW_RATIO = 3.0f / 4.0f; static constexpr float HAPTIC_MAX_AMPLITUDE_FLOAT = 1.0f; static constexpr float SCALE_GAMMA = 0.65f; // Same as VibrationEffect.SCALE_GAMMA +static constexpr float SCALE_LEVEL_GAIN = 1.4f; // Same as VibrationConfig.DEFAULT_SCALE_LEVEL_GAIN float getOldHapticScaleGamma(HapticLevel level) { switch (level) { @@ -60,9 +64,34 @@ float getOldHapticMaxAmplitudeRatio(HapticLevel level) { } } -/* Same as VibrationScaler.SCALE_LEVEL_* */ -float getHapticScaleFactor(HapticLevel level) { - switch (level) { +/* Same as VibrationScaler.getScaleFactor */ +float getHapticScaleFactor(HapticScale scale) { + if (android_os_vibrator_haptics_scale_v2_enabled()) { + if (scale.getScaleFactor() >= 0) { + // ExternalVibratorService provided the scale factor, use it. + return scale.getScaleFactor(); + } + + HapticLevel level = scale.getLevel(); + switch (level) { + case HapticLevel::MUTE: + return 0.0f; + case HapticLevel::NONE: + return 1.0f; + default: + float scaleFactor = powf(SCALE_LEVEL_GAIN, static_cast<int32_t>(level)); + if (scaleFactor <= 0) { + ALOGE("Invalid scale factor %.2f for level %d, using fallback to 1.0", + scaleFactor, static_cast<int32_t>(level)); + scaleFactor = 1.0f; + } + return scaleFactor; + } + } + // Same as VibrationScaler.SCALE_FACTOR_* + switch (scale.getLevel()) { + case HapticLevel::MUTE: + return 0.0f; case HapticLevel::VERY_LOW: return 0.6f; case HapticLevel::LOW: @@ -83,6 +112,14 @@ float applyOldHapticScale(float value, float gamma, float maxAmplitudeRatio) { } float applyNewHapticScale(float value, float scaleFactor) { + if (android_os_vibrator_haptics_scale_v2_enabled()) { + if (scaleFactor <= 1 || value == 0) { + return value * scaleFactor; + } else { + // Using S * x / (1 + (S - 1) * x^2) as the scale up function to converge to 1.0. + return (value * scaleFactor) / (1 + (scaleFactor - 1) * value * value); + } + } float scale = powf(scaleFactor, 1.0f / SCALE_GAMMA); if (scaleFactor <= 1) { // Scale down is simply a gamma corrected application of scaleFactor to the intensity. @@ -115,21 +152,22 @@ void applyHapticScale(float* buffer, size_t length, HapticScale scale) { return; } HapticLevel hapticLevel = scale.getLevel(); - float scaleFactor = getHapticScaleFactor(hapticLevel); + float scaleFactor = getHapticScaleFactor(scale); float adaptiveScaleFactor = scale.getAdaptiveScaleFactor(); float oldGamma = getOldHapticScaleGamma(hapticLevel); float oldMaxAmplitudeRatio = getOldHapticMaxAmplitudeRatio(hapticLevel); for (size_t i = 0; i < length; i++) { if (hapticLevel != HapticLevel::NONE) { - if (android_os_vibrator_fix_audio_coupled_haptics_scaling()) { + if (android_os_vibrator_fix_audio_coupled_haptics_scaling() || + android_os_vibrator_haptics_scale_v2_enabled()) { buffer[i] = applyNewHapticScale(buffer[i], scaleFactor); } else { buffer[i] = applyOldHapticScale(buffer[i], oldGamma, oldMaxAmplitudeRatio); } } - if (adaptiveScaleFactor != 1.0f) { + if (adaptiveScaleFactor >= 0 && adaptiveScaleFactor != 1.0f) { buffer[i] *= adaptiveScaleFactor; } } diff --git a/libs/vibrator/include/vibrator/ExternalVibrationUtils.h b/libs/vibrator/include/vibrator/ExternalVibrationUtils.h index d9a2b81ddf..f0760fda1b 100644 --- a/libs/vibrator/include/vibrator/ExternalVibrationUtils.h +++ b/libs/vibrator/include/vibrator/ExternalVibrationUtils.h @@ -17,9 +17,13 @@ #ifndef ANDROID_EXTERNAL_VIBRATION_UTILS_H #define ANDROID_EXTERNAL_VIBRATION_UTILS_H +#include <cstring> +#include <sstream> +#include <string> + namespace android::os { -enum class HapticLevel { +enum class HapticLevel : int32_t { MUTE = -100, VERY_LOW = -2, LOW = -1, @@ -31,32 +35,44 @@ enum class HapticLevel { class HapticScale { private: HapticLevel mLevel = HapticLevel::NONE; +float mScaleFactor = -1.0f; // undefined, use haptic level to define scale factor float mAdaptiveScaleFactor = 1.0f; public: -constexpr HapticScale(HapticLevel level, float adaptiveScaleFactor) - : mLevel(level), mAdaptiveScaleFactor(adaptiveScaleFactor) {} -constexpr HapticScale(HapticLevel level) : mLevel(level) {} -constexpr HapticScale() {} + explicit HapticScale(HapticLevel level, float scaleFactor, float adaptiveScaleFactor) + : mLevel(level), mScaleFactor(scaleFactor), mAdaptiveScaleFactor(adaptiveScaleFactor) {} + explicit HapticScale(HapticLevel level) : mLevel(level) {} + constexpr HapticScale() {} -HapticLevel getLevel() const { return mLevel; } -float getAdaptiveScaleFactor() const { return mAdaptiveScaleFactor; } + HapticLevel getLevel() const { return mLevel; } + float getScaleFactor() const { return mScaleFactor; } + float getAdaptiveScaleFactor() const { return mAdaptiveScaleFactor; } -bool operator==(const HapticScale& other) const { - return mLevel == other.mLevel && mAdaptiveScaleFactor == other.mAdaptiveScaleFactor; -} + bool operator==(const HapticScale& other) const { + return mLevel == other.mLevel && mScaleFactor == other.mScaleFactor && + mAdaptiveScaleFactor == other.mAdaptiveScaleFactor; + } bool isScaleNone() const { - return mLevel == HapticLevel::NONE && mAdaptiveScaleFactor == 1.0f; + return (mLevel == HapticLevel::NONE || mScaleFactor == 1.0f) && mAdaptiveScaleFactor == 1.0f; } bool isScaleMute() const { - return mLevel == HapticLevel::MUTE; + return mLevel == HapticLevel::MUTE || mScaleFactor == 0 || mAdaptiveScaleFactor == 0; } -static HapticScale mute() { - return {/*level=*/os::HapticLevel::MUTE}; +std::string toString() const { + std::ostringstream os; + os << "HapticScale { level: " << static_cast<int>(mLevel); + os << ", scaleFactor: " << mScaleFactor; + os << ", adaptiveScaleFactor: " << mAdaptiveScaleFactor; + os << "}"; + return os.str(); } + +static HapticScale mute() { return os::HapticScale(os::HapticLevel::MUTE); } + +static HapticScale none() { return os::HapticScale(os::HapticLevel::NONE); } }; bool isValidHapticScale(HapticScale scale); diff --git a/libs/vibrator/tests/ExternalVibrationTest.cpp b/libs/vibrator/tests/ExternalVibrationTest.cpp index 3141380219..4133836785 100644 --- a/libs/vibrator/tests/ExternalVibrationTest.cpp +++ b/libs/vibrator/tests/ExternalVibrationTest.cpp @@ -57,11 +57,14 @@ TEST_F(ExternalVibrationTest, TestReadAndWriteToParcel) { originalAttrs.usage = AUDIO_USAGE_ASSISTANCE_SONIFICATION; originalAttrs.source = AUDIO_SOURCE_VOICE_COMMUNICATION; originalAttrs.flags = AUDIO_FLAG_BYPASS_MUTE; + sp<TestVibrationController> vibrationController = new TestVibrationController(); ASSERT_NE(vibrationController, nullptr); + sp<os::ExternalVibration> original = new os::ExternalVibration(uid, pkg, originalAttrs, vibrationController); ASSERT_NE(original, nullptr); + EXPECT_EQ(original->getUid(), uid); EXPECT_EQ(original->getPackage(), pkg); EXPECT_EQ(original->getAudioAttributes().content_type, originalAttrs.content_type); @@ -69,18 +72,22 @@ TEST_F(ExternalVibrationTest, TestReadAndWriteToParcel) { EXPECT_EQ(original->getAudioAttributes().source, originalAttrs.source); EXPECT_EQ(original->getAudioAttributes().flags, originalAttrs.flags); EXPECT_EQ(original->getController(), vibrationController); + audio_attributes_t defaultAttrs; defaultAttrs.content_type = AUDIO_CONTENT_TYPE_UNKNOWN; defaultAttrs.usage = AUDIO_USAGE_UNKNOWN; defaultAttrs.source = AUDIO_SOURCE_DEFAULT; defaultAttrs.flags = AUDIO_FLAG_NONE; + sp<os::ExternalVibration> parceled = new os::ExternalVibration(0, std::string(""), defaultAttrs, nullptr); ASSERT_NE(parceled, nullptr); + Parcel parcel; original->writeToParcel(&parcel); parcel.setDataPosition(0); parceled->readFromParcel(&parcel); + EXPECT_EQ(parceled->getUid(), uid); EXPECT_EQ(parceled->getPackage(), pkg); EXPECT_EQ(parceled->getAudioAttributes().content_type, originalAttrs.content_type); @@ -93,12 +100,17 @@ TEST_F(ExternalVibrationTest, TestReadAndWriteToParcel) { TEST_F(ExternalVibrationTest, TestExternalVibrationScaleToHapticScale) { os::ExternalVibrationScale externalVibrationScale; externalVibrationScale.scaleLevel = ScaleLevel::SCALE_HIGH; + externalVibrationScale.scaleFactor = 0.5f; externalVibrationScale.adaptiveHapticsScale = 0.8f; + os::HapticScale hapticScale = os::ExternalVibration::externalVibrationScaleToHapticScale(externalVibrationScale); - // Check scale factor is forwarded. + + // Check scale factors are forwarded. EXPECT_EQ(hapticScale.getLevel(), HapticLevel::HIGH); + EXPECT_EQ(hapticScale.getScaleFactor(), 0.5f); EXPECT_EQ(hapticScale.getAdaptiveScaleFactor(), 0.8f); + // Check conversion for all levels. EXPECT_EQ(toHapticLevel(ScaleLevel::SCALE_MUTE), HapticLevel::MUTE); EXPECT_EQ(toHapticLevel(ScaleLevel::SCALE_VERY_LOW), HapticLevel::VERY_LOW); diff --git a/libs/vibrator/tests/ExternalVibrationUtilsTest.cpp b/libs/vibrator/tests/ExternalVibrationUtilsTest.cpp index 3d8dd9cb5b..9369f80da3 100644 --- a/libs/vibrator/tests/ExternalVibrationUtilsTest.cpp +++ b/libs/vibrator/tests/ExternalVibrationUtilsTest.cpp @@ -41,7 +41,7 @@ public: protected: void scaleBuffer(HapticLevel hapticLevel) { - scaleBuffer(HapticScale(hapticLevel), 0 /* limit */); + scaleBuffer(HapticScale(hapticLevel)); } void scaleBuffer(HapticLevel hapticLevel, float adaptiveScaleFactor) { @@ -49,7 +49,11 @@ protected: } void scaleBuffer(HapticLevel hapticLevel, float adaptiveScaleFactor, float limit) { - scaleBuffer(HapticScale(hapticLevel, adaptiveScaleFactor), limit); + scaleBuffer(HapticScale(hapticLevel, -1 /* scaleFactor */, adaptiveScaleFactor), limit); + } + + void scaleBuffer(HapticScale hapticScale) { + scaleBuffer(hapticScale, 0 /* limit */); } void scaleBuffer(HapticScale hapticScale, float limit) { @@ -60,11 +64,9 @@ protected: float mBuffer[TEST_BUFFER_LENGTH]; }; -TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestLegacyScaleMute, - REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestLegacyScaleMute, + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling), + ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { float expected[TEST_BUFFER_LENGTH]; std::fill(std::begin(expected), std::end(expected), 0); @@ -72,11 +74,9 @@ TEST_F_WITH_FLAGS( EXPECT_FLOATS_NEARLY_EQ(expected, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); } -TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestFixedScaleMute, - REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestFixedScaleMute, + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)), + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { float expected[TEST_BUFFER_LENGTH]; std::fill(std::begin(expected), std::end(expected), 0); @@ -85,10 +85,19 @@ TEST_F_WITH_FLAGS( } TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestLegacyScaleNone, - REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { + ExternalVibrationUtilsTest, TestScaleV2Mute, + // Value of fix_audio_coupled_haptics_scaling is not important, should work with either + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + float expected[TEST_BUFFER_LENGTH]; + std::fill(std::begin(expected), std::end(expected), 0); + + scaleBuffer(HapticLevel::MUTE); + EXPECT_FLOATS_NEARLY_EQ(expected, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestLegacyScaleNone, + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling), + ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { float expected[TEST_BUFFER_LENGTH]; std::copy(std::begin(TEST_BUFFER), std::end(TEST_BUFFER), std::begin(expected)); @@ -96,11 +105,9 @@ TEST_F_WITH_FLAGS( EXPECT_FLOATS_NEARLY_EQ(expected, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); } -TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestFixedScaleNone, - REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestFixedScaleNone, + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)), + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { float expected[TEST_BUFFER_LENGTH]; std::copy(std::begin(TEST_BUFFER), std::end(TEST_BUFFER), std::begin(expected)); @@ -109,10 +116,19 @@ TEST_F_WITH_FLAGS( } TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestLegacyScaleToHapticLevel, - REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { + ExternalVibrationUtilsTest, TestScaleV2None, + // Value of fix_audio_coupled_haptics_scaling is not important, should work with either + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + float expected[TEST_BUFFER_LENGTH]; + std::copy(std::begin(TEST_BUFFER), std::end(TEST_BUFFER), std::begin(expected)); + + scaleBuffer(HapticLevel::NONE); + EXPECT_FLOATS_NEARLY_EQ(expected, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestLegacyScaleToHapticLevel, + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling), + ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { float expectedVeryHigh[TEST_BUFFER_LENGTH] = { 1, -1, 0.84f, -0.66f }; scaleBuffer(HapticLevel::VERY_HIGH); EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); @@ -130,11 +146,9 @@ TEST_F_WITH_FLAGS( EXPECT_FLOATS_NEARLY_EQ(expectedVeryLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); } -TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestFixedScaleToHapticLevel, - REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestFixedScaleToHapticLevel, + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)), + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { float expectedVeryHigh[TEST_BUFFER_LENGTH] = { 1, -1, 0.79f, -0.39f }; scaleBuffer(HapticLevel::VERY_HIGH); EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); @@ -153,10 +167,76 @@ TEST_F_WITH_FLAGS( } TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestAdaptiveScaleFactorAppliedAfterLegacyScale, - REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { + ExternalVibrationUtilsTest, TestScaleV2ToHapticLevel, + // Value of fix_audio_coupled_haptics_scaling is not important, should work with either + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + float expectedVeryHigh[TEST_BUFFER_LENGTH] = { 1, -1, 0.8f, -0.38f }; + scaleBuffer(HapticLevel::VERY_HIGH); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + float expectedHigh[TEST_BUFFER_LENGTH] = { 1, -1, 0.63f, -0.27f }; + scaleBuffer(HapticLevel::HIGH); + EXPECT_FLOATS_NEARLY_EQ(expectedHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + float expectedLow[TEST_BUFFER_LENGTH] = { 0.71f, -0.71f, 0.35f, -0.14f }; + scaleBuffer(HapticLevel::LOW); + EXPECT_FLOATS_NEARLY_EQ(expectedLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + float expectedVeryLow[TEST_BUFFER_LENGTH] = { 0.51f, -0.51f, 0.25f, -0.1f }; + scaleBuffer(HapticLevel::VERY_LOW); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS( + ExternalVibrationUtilsTest, TestScaleV2ToScaleFactorUndefinedUsesHapticLevel, + // Value of fix_audio_coupled_haptics_scaling is not important, should work with either + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + constexpr float adaptiveScaleNone = 1.0f; + float expectedVeryHigh[TEST_BUFFER_LENGTH] = {1, -1, 0.8f, -0.38f}; + scaleBuffer(HapticScale(HapticLevel::VERY_HIGH, -1.0f /* scaleFactor */, adaptiveScaleNone)); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS( + ExternalVibrationUtilsTest, TestScaleV2ToScaleFactorIgnoresLevel, + // Value of fix_audio_coupled_haptics_scaling is not important, should work with either + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + constexpr float adaptiveScaleNone = 1.0f; + + float expectedVeryHigh[TEST_BUFFER_LENGTH] = { 1, -1, 1, -0.55f }; + scaleBuffer(HapticScale(HapticLevel::LOW, 3.0f /* scaleFactor */, adaptiveScaleNone)); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + float expectedHigh[TEST_BUFFER_LENGTH] = { 1, -1, 0.66f, -0.29f }; + scaleBuffer(HapticScale(HapticLevel::LOW, 1.5f /* scaleFactor */, adaptiveScaleNone)); + EXPECT_FLOATS_NEARLY_EQ(expectedHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + float expectedLow[TEST_BUFFER_LENGTH] = { 0.8f, -0.8f, 0.4f, -0.16f }; + scaleBuffer(HapticScale(HapticLevel::HIGH, 0.8f /* scaleFactor */, adaptiveScaleNone)); + EXPECT_FLOATS_NEARLY_EQ(expectedLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + float expectedVeryLow[TEST_BUFFER_LENGTH] = { 0.4f, -0.4f, 0.2f, -0.08f }; + scaleBuffer(HapticScale(HapticLevel::HIGH, 0.4f /* scaleFactor */, adaptiveScaleNone)); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestAdaptiveScaleFactorUndefinedIsIgnoredLegacyScale, + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling), + ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + float expectedVeryHigh[TEST_BUFFER_LENGTH] = {1, -1, 0.79f, -0.39f}; + scaleBuffer(HapticLevel::VERY_HIGH, -1.0f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestAdaptiveScaleFactorAppliedAfterLegacyScale, + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling), + ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + // Adaptive scale mutes vibration + float expectedMuted[TEST_BUFFER_LENGTH]; + std::fill(std::begin(expectedMuted), std::end(expectedMuted), 0); + scaleBuffer(HapticLevel::VERY_HIGH, 0.0f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedMuted, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + // Haptic level scale up then adaptive scale down float expectedVeryHigh[TEST_BUFFER_LENGTH] = { 0.2, -0.2, 0.16f, -0.13f }; scaleBuffer(HapticLevel::VERY_HIGH, 0.2f /* adaptiveScaleFactor */); @@ -178,11 +258,23 @@ TEST_F_WITH_FLAGS( EXPECT_FLOATS_NEARLY_EQ(expectedVeryLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); } -TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestAdaptiveScaleFactorAppliedAfterFixedScale, - REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestAdaptiveScaleFactorUndefinedIgnoredFixedScale, + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)), + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + float expectedVeryHigh[TEST_BUFFER_LENGTH] = {1, -1, 0.79f, -0.39f}; + scaleBuffer(HapticLevel::VERY_HIGH, -1.0f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestAdaptiveScaleFactorAppliedAfterFixedScale, + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)), + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + // Adaptive scale mutes vibration + float expectedMuted[TEST_BUFFER_LENGTH]; + std::fill(std::begin(expectedMuted), std::end(expectedMuted), 0); + scaleBuffer(HapticLevel::VERY_HIGH, 0.0f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedMuted, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + // Haptic level scale up then adaptive scale down float expectedVeryHigh[TEST_BUFFER_LENGTH] = { 0.2, -0.2, 0.16f, -0.07f }; scaleBuffer(HapticLevel::VERY_HIGH, 0.2f /* adaptiveScaleFactor */); @@ -205,10 +297,48 @@ TEST_F_WITH_FLAGS( } TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestLimitAppliedAfterLegacyScale, - REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { + ExternalVibrationUtilsTest, TestAdaptiveScaleFactorUndefinedIgnoredScaleV2, + // Value of fix_audio_coupled_haptics_scaling is not important, should work with either + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + float expectedVeryHigh[TEST_BUFFER_LENGTH] = {1, -1, 0.8f, -0.38f}; + scaleBuffer(HapticLevel::VERY_HIGH, -1.0f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS( + ExternalVibrationUtilsTest, TestAdaptiveScaleFactorAppliedAfterScaleV2, + // Value of fix_audio_coupled_haptics_scaling is not important, should work with either + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + // Adaptive scale mutes vibration + float expectedMuted[TEST_BUFFER_LENGTH]; + std::fill(std::begin(expectedMuted), std::end(expectedMuted), 0); + scaleBuffer(HapticLevel::VERY_HIGH, 0.0f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedMuted, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + // Haptic level scale up then adaptive scale down + float expectedVeryHigh[TEST_BUFFER_LENGTH] = { 0.2, -0.2, 0.15f, -0.07f }; + scaleBuffer(HapticLevel::VERY_HIGH, 0.2f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + // Haptic level scale up then adaptive scale up + float expectedHigh[TEST_BUFFER_LENGTH] = { 1.5f, -1.5f, 0.95f, -0.41f }; + scaleBuffer(HapticLevel::HIGH, 1.5f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + // Haptic level scale down then adaptive scale down + float expectedLow[TEST_BUFFER_LENGTH] = { 0.42f, -0.42f, 0.21f, -0.08f }; + scaleBuffer(HapticLevel::LOW, 0.6f /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + // Haptic level scale down then adaptive scale up + float expectedVeryLow[TEST_BUFFER_LENGTH] = { 1.02f, -1.02f, 0.51f, -0.2f }; + scaleBuffer(HapticLevel::VERY_LOW, 2 /* adaptiveScaleFactor */); + EXPECT_FLOATS_NEARLY_EQ(expectedVeryLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} + +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestLimitAppliedAfterLegacyScale, + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling), + ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { // Scaled = { 0.2, -0.2, 0.16f, -0.13f }; float expectedClippedVeryHigh[TEST_BUFFER_LENGTH] = { 0.15f, -0.15f, 0.15f, -0.13f }; scaleBuffer(HapticLevel::VERY_HIGH, 0.2f /* adaptiveScaleFactor */, 0.15f /* limit */); @@ -220,11 +350,9 @@ TEST_F_WITH_FLAGS( EXPECT_FLOATS_NEARLY_EQ(expectedClippedVeryLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); } -TEST_F_WITH_FLAGS( - ExternalVibrationUtilsTest, - TestLimitAppliedAfterFixedScale, - REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)) -) { +TEST_F_WITH_FLAGS(ExternalVibrationUtilsTest, TestLimitAppliedAfterFixedScale, + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, fix_audio_coupled_haptics_scaling)), + REQUIRES_FLAGS_DISABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { // Scaled = { 0.2, -0.2, 0.16f, -0.13f }; float expectedClippedVeryHigh[TEST_BUFFER_LENGTH] = { 0.15f, -0.15f, 0.15f, -0.07f }; scaleBuffer(HapticLevel::VERY_HIGH, 0.2f /* adaptiveScaleFactor */, 0.15f /* limit */); @@ -235,3 +363,18 @@ TEST_F_WITH_FLAGS( scaleBuffer(HapticLevel::VERY_LOW, 2 /* adaptiveScaleFactor */, 0.7f /* limit */); EXPECT_FLOATS_NEARLY_EQ(expectedClippedVeryLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); } + +TEST_F_WITH_FLAGS( + ExternalVibrationUtilsTest, TestLimitAppliedAfterScaleV2, + // Value of fix_audio_coupled_haptics_scaling is not important, should work with either + REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(FLAG_NS, haptics_scale_v2_enabled))) { + // Scaled = { 0.2, -0.2, 0.15f, -0.07f }; + float expectedClippedVeryHigh[TEST_BUFFER_LENGTH] = { 0.15f, -0.15f, 0.15f, -0.07f }; + scaleBuffer(HapticLevel::VERY_HIGH, 0.2f /* adaptiveScaleFactor */, 0.15f /* limit */); + EXPECT_FLOATS_NEARLY_EQ(expectedClippedVeryHigh, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); + + // Scaled = { 1.02f, -1.02f, 0.51f, -0.2f } + float expectedClippedVeryLow[TEST_BUFFER_LENGTH] = { 0.7f, -0.7f, 0.51f, -0.2f }; + scaleBuffer(HapticLevel::VERY_LOW, 2 /* adaptiveScaleFactor */, 0.7f /* limit */); + EXPECT_FLOATS_NEARLY_EQ(expectedClippedVeryLow, mBuffer, TEST_BUFFER_LENGTH, TEST_TOLERANCE); +} |