diff options
Diffstat (limited to 'libs')
69 files changed, 3154 insertions, 649 deletions
diff --git a/libs/binder/IActivityManager.cpp b/libs/binder/IActivityManager.cpp index 28975618e1..152c815ec3 100644 --- a/libs/binder/IActivityManager.cpp +++ b/libs/binder/IActivityManager.cpp @@ -193,8 +193,7 @@ public: status_t err = remote()->transact(LOG_FGS_API_BEGIN_TRANSACTION, data, &reply, IBinder::FLAG_ONEWAY); if (err != NO_ERROR || ((err = reply.readExceptionCode()) != NO_ERROR)) { - ALOGD("FGS Logger Transaction failed"); - ALOGD("%d", err); + ALOGD("%s: FGS Logger Transaction failed, %d", __func__, err); return err; } return NO_ERROR; @@ -209,8 +208,7 @@ public: status_t err = remote()->transact(LOG_FGS_API_END_TRANSACTION, data, &reply, IBinder::FLAG_ONEWAY); if (err != NO_ERROR || ((err = reply.readExceptionCode()) != NO_ERROR)) { - ALOGD("FGS Logger Transaction failed"); - ALOGD("%d", err); + ALOGD("%s: FGS Logger Transaction failed, %d", __func__, err); return err; } return NO_ERROR; @@ -224,11 +222,10 @@ public: data.writeInt32(state); data.writeInt32(appUid); data.writeInt32(appPid); - status_t err = remote()->transact(LOG_FGS_API_BEGIN_TRANSACTION, data, &reply, + status_t err = remote()->transact(LOG_FGS_API_STATE_CHANGED_TRANSACTION, data, &reply, IBinder::FLAG_ONEWAY); if (err != NO_ERROR || ((err = reply.readExceptionCode()) != NO_ERROR)) { - ALOGD("FGS Logger Transaction failed"); - ALOGD("%d", err); + ALOGD("%s: FGS Logger Transaction failed, %d", __func__, err); return err; } return NO_ERROR; diff --git a/libs/gui/Android.bp b/libs/gui/Android.bp index d7e7eb8ea1..298838d816 100644 --- a/libs/gui/Android.bp +++ b/libs/gui/Android.bp @@ -62,6 +62,7 @@ filegroup { name: "guiconstants_aidl", srcs: [ "android/gui/DropInputMode.aidl", + "android/gui/StalledTransactionInfo.aidl", "android/**/TouchOcclusionMode.aidl", ], } @@ -140,6 +141,7 @@ aidl_library { "android/gui/IWindowInfosListener.aidl", "android/gui/IWindowInfosPublisher.aidl", "android/gui/IWindowInfosReportedListener.aidl", + "android/gui/StalledTransactionInfo.aidl", "android/gui/WindowInfo.aidl", "android/gui/WindowInfosUpdate.aidl", ], diff --git a/libs/gui/BLASTBufferQueue.cpp b/libs/gui/BLASTBufferQueue.cpp index 5c324b29cd..207fa4fd31 100644 --- a/libs/gui/BLASTBufferQueue.cpp +++ b/libs/gui/BLASTBufferQueue.cpp @@ -303,13 +303,8 @@ void BLASTBufferQueue::transactionCommittedCallback(nsecs_t /*latchTime*/, // frame numbers that were in a sync. We remove the frame from mSyncedFrameNumbers // set and then check if it's empty. If there are no more pending syncs, we can // proceed with flushing the shadow queue. - // We also want to check if mSyncTransaction is null because it's possible another - // sync request came in while waiting, but it hasn't started processing yet. In that - // case, we don't actually want to flush the frames in between since they will get - // processed and merged with the sync transaction and released earlier than if they - // were sent to SF mSyncedFrameNumbers.erase(currFrameNumber); - if (mSyncedFrameNumbers.empty() && mSyncTransaction == nullptr) { + if (mSyncedFrameNumbers.empty()) { flushShadowQueue(); } } else { diff --git a/libs/gui/BufferQueueProducer.cpp b/libs/gui/BufferQueueProducer.cpp index ce5d5d382e..920b83dba9 100644 --- a/libs/gui/BufferQueueProducer.cpp +++ b/libs/gui/BufferQueueProducer.cpp @@ -418,6 +418,9 @@ status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* ou EGLSyncKHR eglFence = EGL_NO_SYNC_KHR; bool attachedByConsumer = false; + sp<IConsumerListener> listener; + bool callOnFrameDequeued = false; + uint64_t bufferId = 0; // Only used if callOnFrameDequeued == true { // Autolock scope std::unique_lock<std::mutex> lock(mCore->mMutex); @@ -561,10 +564,11 @@ status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* ou } if (!(returnFlags & BUFFER_NEEDS_REALLOCATION)) { - if (mCore->mConsumerListener != nullptr) { - mCore->mConsumerListener->onFrameDequeued(mSlots[*outSlot].mGraphicBuffer->getId()); - } + callOnFrameDequeued = true; + bufferId = mSlots[*outSlot].mGraphicBuffer->getId(); } + + listener = mCore->mConsumerListener; } // Autolock scope if (returnFlags & BUFFER_NEEDS_REALLOCATION) { @@ -581,10 +585,8 @@ status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* ou if (error == NO_ERROR && !mCore->mIsAbandoned) { graphicBuffer->setGenerationNumber(mCore->mGenerationNumber); mSlots[*outSlot].mGraphicBuffer = graphicBuffer; - if (mCore->mConsumerListener != nullptr) { - mCore->mConsumerListener->onFrameDequeued( - mSlots[*outSlot].mGraphicBuffer->getId()); - } + callOnFrameDequeued = true; + bufferId = mSlots[*outSlot].mGraphicBuffer->getId(); } mCore->mIsAllocating = false; @@ -608,6 +610,10 @@ status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* ou } // Autolock scope } + if (listener != nullptr && callOnFrameDequeued) { + listener->onFrameDequeued(bufferId); + } + if (attachedByConsumer) { returnFlags |= BUFFER_NEEDS_REALLOCATION; } @@ -647,6 +653,8 @@ status_t BufferQueueProducer::detachBuffer(int slot) { BQ_LOGV("detachBuffer: slot %d", slot); sp<IConsumerListener> listener; + bool callOnFrameDetached = false; + uint64_t bufferId = 0; // Only used if callOnFrameDetached is true { std::lock_guard<std::mutex> lock(mCore->mMutex); @@ -684,8 +692,9 @@ status_t BufferQueueProducer::detachBuffer(int slot) { listener = mCore->mConsumerListener; auto gb = mSlots[slot].mGraphicBuffer; - if (listener != nullptr && gb != nullptr) { - listener->onFrameDetached(gb->getId()); + if (gb != nullptr) { + callOnFrameDetached = true; + bufferId = gb->getId(); } mSlots[slot].mBufferState.detachProducer(); mCore->mActiveBuffers.erase(slot); @@ -695,6 +704,10 @@ status_t BufferQueueProducer::detachBuffer(int slot) { VALIDATE_CONSISTENCY(); } + if (listener != nullptr && callOnFrameDetached) { + listener->onFrameDetached(bufferId); + } + if (listener != nullptr) { listener->onBuffersReleased(); } @@ -1104,57 +1117,70 @@ status_t BufferQueueProducer::queueBuffer(int slot, status_t BufferQueueProducer::cancelBuffer(int slot, const sp<Fence>& fence) { ATRACE_CALL(); BQ_LOGV("cancelBuffer: slot %d", slot); - std::lock_guard<std::mutex> lock(mCore->mMutex); - if (mCore->mIsAbandoned) { - BQ_LOGE("cancelBuffer: BufferQueue has been abandoned"); - return NO_INIT; - } + sp<IConsumerListener> listener; + bool callOnFrameCancelled = false; + uint64_t bufferId = 0; // Only used if callOnFrameCancelled == true + { + std::lock_guard<std::mutex> lock(mCore->mMutex); - if (mCore->mConnectedApi == BufferQueueCore::NO_CONNECTED_API) { - BQ_LOGE("cancelBuffer: BufferQueue has no connected producer"); - return NO_INIT; - } + if (mCore->mIsAbandoned) { + BQ_LOGE("cancelBuffer: BufferQueue has been abandoned"); + return NO_INIT; + } - if (mCore->mSharedBufferMode) { - BQ_LOGE("cancelBuffer: cannot cancel a buffer in shared buffer mode"); - return BAD_VALUE; - } + if (mCore->mConnectedApi == BufferQueueCore::NO_CONNECTED_API) { + BQ_LOGE("cancelBuffer: BufferQueue has no connected producer"); + return NO_INIT; + } - if (slot < 0 || slot >= BufferQueueDefs::NUM_BUFFER_SLOTS) { - BQ_LOGE("cancelBuffer: slot index %d out of range [0, %d)", - slot, BufferQueueDefs::NUM_BUFFER_SLOTS); - return BAD_VALUE; - } else if (!mSlots[slot].mBufferState.isDequeued()) { - BQ_LOGE("cancelBuffer: slot %d is not owned by the producer " - "(state = %s)", slot, mSlots[slot].mBufferState.string()); - return BAD_VALUE; - } else if (fence == nullptr) { - BQ_LOGE("cancelBuffer: fence is NULL"); - return BAD_VALUE; - } + if (mCore->mSharedBufferMode) { + BQ_LOGE("cancelBuffer: cannot cancel a buffer in shared buffer mode"); + return BAD_VALUE; + } + + if (slot < 0 || slot >= BufferQueueDefs::NUM_BUFFER_SLOTS) { + BQ_LOGE("cancelBuffer: slot index %d out of range [0, %d)", slot, + BufferQueueDefs::NUM_BUFFER_SLOTS); + return BAD_VALUE; + } else if (!mSlots[slot].mBufferState.isDequeued()) { + BQ_LOGE("cancelBuffer: slot %d is not owned by the producer " + "(state = %s)", + slot, mSlots[slot].mBufferState.string()); + return BAD_VALUE; + } else if (fence == nullptr) { + BQ_LOGE("cancelBuffer: fence is NULL"); + return BAD_VALUE; + } - mSlots[slot].mBufferState.cancel(); + mSlots[slot].mBufferState.cancel(); - // After leaving shared buffer mode, the shared buffer will still be around. - // Mark it as no longer shared if this operation causes it to be free. - if (!mCore->mSharedBufferMode && mSlots[slot].mBufferState.isFree()) { - mSlots[slot].mBufferState.mShared = false; - } + // After leaving shared buffer mode, the shared buffer will still be around. + // Mark it as no longer shared if this operation causes it to be free. + if (!mCore->mSharedBufferMode && mSlots[slot].mBufferState.isFree()) { + mSlots[slot].mBufferState.mShared = false; + } - // Don't put the shared buffer on the free list. - if (!mSlots[slot].mBufferState.isShared()) { - mCore->mActiveBuffers.erase(slot); - mCore->mFreeBuffers.push_back(slot); + // Don't put the shared buffer on the free list. + if (!mSlots[slot].mBufferState.isShared()) { + mCore->mActiveBuffers.erase(slot); + mCore->mFreeBuffers.push_back(slot); + } + + auto gb = mSlots[slot].mGraphicBuffer; + if (gb != nullptr) { + callOnFrameCancelled = true; + bufferId = gb->getId(); + } + mSlots[slot].mFence = fence; + mCore->mDequeueCondition.notify_all(); + listener = mCore->mConsumerListener; + VALIDATE_CONSISTENCY(); } - auto gb = mSlots[slot].mGraphicBuffer; - if (mCore->mConsumerListener != nullptr && gb != nullptr) { - mCore->mConsumerListener->onFrameCancelled(gb->getId()); + if (listener != nullptr && callOnFrameCancelled) { + listener->onFrameCancelled(bufferId); } - mSlots[slot].mFence = fence; - mCore->mDequeueCondition.notify_all(); - VALIDATE_CONSISTENCY(); return NO_ERROR; } diff --git a/libs/gui/Surface.cpp b/libs/gui/Surface.cpp index ed691006e9..53a2f64d11 100644 --- a/libs/gui/Surface.cpp +++ b/libs/gui/Surface.cpp @@ -1792,19 +1792,20 @@ int Surface::dispatchGetLastQueuedBuffer2(va_list args) { int Surface::dispatchSetFrameTimelineInfo(va_list args) { ATRACE_CALL(); - auto frameNumber = static_cast<uint64_t>(va_arg(args, uint64_t)); - auto frameTimelineVsyncId = static_cast<int64_t>(va_arg(args, int64_t)); - auto inputEventId = static_cast<int32_t>(va_arg(args, int32_t)); - auto startTimeNanos = static_cast<int64_t>(va_arg(args, int64_t)); - auto useForRefreshRateSelection = static_cast<bool>(va_arg(args, int32_t)); - ALOGV("Surface::%s", __func__); + + const auto nativeWindowFtlInfo = static_cast<ANativeWindowFrameTimelineInfo>( + va_arg(args, ANativeWindowFrameTimelineInfo)); + FrameTimelineInfo ftlInfo; - ftlInfo.vsyncId = frameTimelineVsyncId; - ftlInfo.inputEventId = inputEventId; - ftlInfo.startTimeNanos = startTimeNanos; - ftlInfo.useForRefreshRateSelection = useForRefreshRateSelection; - return setFrameTimelineInfo(frameNumber, ftlInfo); + ftlInfo.vsyncId = nativeWindowFtlInfo.frameTimelineVsyncId; + ftlInfo.inputEventId = nativeWindowFtlInfo.inputEventId; + ftlInfo.startTimeNanos = nativeWindowFtlInfo.startTimeNanos; + ftlInfo.useForRefreshRateSelection = nativeWindowFtlInfo.useForRefreshRateSelection; + ftlInfo.skippedFrameVsyncId = nativeWindowFtlInfo.skippedFrameVsyncId; + ftlInfo.skippedFrameStartTimeNanos = nativeWindowFtlInfo.skippedFrameStartTimeNanos; + + return setFrameTimelineInfo(nativeWindowFtlInfo.frameNumber, ftlInfo); } bool Surface::transformToDisplayInverse() const { diff --git a/libs/gui/SurfaceComposerClient.cpp b/libs/gui/SurfaceComposerClient.cpp index 8a1f7c6238..00495ee5f6 100644 --- a/libs/gui/SurfaceComposerClient.cpp +++ b/libs/gui/SurfaceComposerClient.cpp @@ -59,7 +59,7 @@ #include <private/gui/ComposerServiceAIDL.h> // This server size should always be smaller than the server cache size -#define BUFFER_CACHE_MAX_SIZE 64 +#define BUFFER_CACHE_MAX_SIZE 4096 namespace android { @@ -1027,7 +1027,7 @@ void SurfaceComposerClient::Transaction::clear() { mEarlyWakeupEnd = false; mDesiredPresentTime = 0; mIsAutoTimestamp = true; - clearFrameTimelineInfo(mFrameTimelineInfo); + mFrameTimelineInfo = {}; mApplyToken = nullptr; mMergedTransactionIds.clear(); } @@ -1302,6 +1302,13 @@ sp<IBinder> SurfaceComposerClient::getPhysicalDisplayToken(PhysicalDisplayId dis return status.isOk() ? display : nullptr; } +std::optional<gui::StalledTransactionInfo> SurfaceComposerClient::getStalledTransactionInfo( + pid_t pid) { + std::optional<gui::StalledTransactionInfo> result; + ComposerServiceAIDL::getComposerService()->getStalledTransactionInfo(pid, &result); + return result; +} + void SurfaceComposerClient::Transaction::setAnimationTransaction() { mAnimation = true; } @@ -2279,27 +2286,13 @@ void SurfaceComposerClient::Transaction::mergeFrameTimelineInfo(FrameTimelineInf if (t.vsyncId != FrameTimelineInfo::INVALID_VSYNC_ID && other.vsyncId != FrameTimelineInfo::INVALID_VSYNC_ID) { if (other.vsyncId > t.vsyncId) { - t.vsyncId = other.vsyncId; - t.inputEventId = other.inputEventId; - t.startTimeNanos = other.startTimeNanos; - t.useForRefreshRateSelection = other.useForRefreshRateSelection; + t = other; } } else if (t.vsyncId == FrameTimelineInfo::INVALID_VSYNC_ID) { - t.vsyncId = other.vsyncId; - t.inputEventId = other.inputEventId; - t.startTimeNanos = other.startTimeNanos; - t.useForRefreshRateSelection = other.useForRefreshRateSelection; + t = other; } } -// copied from FrameTimelineInfo::clear() -void SurfaceComposerClient::Transaction::clearFrameTimelineInfo(FrameTimelineInfo& t) { - t.vsyncId = FrameTimelineInfo::INVALID_VSYNC_ID; - t.inputEventId = os::IInputConstants::INVALID_INPUT_EVENT_ID; - t.startTimeNanos = 0; - t.useForRefreshRateSelection = false; -} - SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setTrustedPresentationCallback( const sp<SurfaceControl>& sc, TrustedPresentationCallback cb, @@ -2523,38 +2516,41 @@ status_t SurfaceComposerClient::getStaticDisplayInfo(int64_t displayId, outInfo->secure = ginfo.secure; outInfo->installOrientation = static_cast<ui::Rotation>(ginfo.installOrientation); - DeviceProductInfo info; - std::optional<gui::DeviceProductInfo> dpi = ginfo.deviceProductInfo; - gui::DeviceProductInfo::ManufactureOrModelDate& date = dpi->manufactureOrModelDate; - info.name = dpi->name; - if (dpi->manufacturerPnpId.size() > 0) { - // copid from PnpId = std::array<char, 4> in ui/DeviceProductInfo.h - constexpr int kMaxPnpIdSize = 4; - size_t count = std::max<size_t>(kMaxPnpIdSize, dpi->manufacturerPnpId.size()); - std::copy_n(dpi->manufacturerPnpId.begin(), count, info.manufacturerPnpId.begin()); - } - if (dpi->relativeAddress.size() > 0) { - std::copy(dpi->relativeAddress.begin(), dpi->relativeAddress.end(), - std::back_inserter(info.relativeAddress)); - } - info.productId = dpi->productId; - if (date.getTag() == Tag::modelYear) { - DeviceProductInfo::ModelYear modelYear; - modelYear.year = static_cast<uint32_t>(date.get<Tag::modelYear>().year); - info.manufactureOrModelDate = modelYear; - } else if (date.getTag() == Tag::manufactureYear) { - DeviceProductInfo::ManufactureYear manufactureYear; - manufactureYear.year = date.get<Tag::manufactureYear>().modelYear.year; - info.manufactureOrModelDate = manufactureYear; - } else if (date.getTag() == Tag::manufactureWeekAndYear) { - DeviceProductInfo::ManufactureWeekAndYear weekAndYear; - weekAndYear.year = - date.get<Tag::manufactureWeekAndYear>().manufactureYear.modelYear.year; - weekAndYear.week = date.get<Tag::manufactureWeekAndYear>().week; - info.manufactureOrModelDate = weekAndYear; - } + if (const std::optional<gui::DeviceProductInfo> dpi = ginfo.deviceProductInfo) { + DeviceProductInfo info; + info.name = dpi->name; + if (dpi->manufacturerPnpId.size() > 0) { + // copid from PnpId = std::array<char, 4> in ui/DeviceProductInfo.h + constexpr int kMaxPnpIdSize = 4; + size_t count = std::max<size_t>(kMaxPnpIdSize, dpi->manufacturerPnpId.size()); + std::copy_n(dpi->manufacturerPnpId.begin(), count, info.manufacturerPnpId.begin()); + } + if (dpi->relativeAddress.size() > 0) { + std::copy(dpi->relativeAddress.begin(), dpi->relativeAddress.end(), + std::back_inserter(info.relativeAddress)); + } + info.productId = dpi->productId; + + const gui::DeviceProductInfo::ManufactureOrModelDate& date = + dpi->manufactureOrModelDate; + if (date.getTag() == Tag::modelYear) { + DeviceProductInfo::ModelYear modelYear; + modelYear.year = static_cast<uint32_t>(date.get<Tag::modelYear>().year); + info.manufactureOrModelDate = modelYear; + } else if (date.getTag() == Tag::manufactureYear) { + DeviceProductInfo::ManufactureYear manufactureYear; + manufactureYear.year = date.get<Tag::manufactureYear>().modelYear.year; + info.manufactureOrModelDate = manufactureYear; + } else if (date.getTag() == Tag::manufactureWeekAndYear) { + DeviceProductInfo::ManufactureWeekAndYear weekAndYear; + weekAndYear.year = + date.get<Tag::manufactureWeekAndYear>().manufactureYear.modelYear.year; + weekAndYear.week = date.get<Tag::manufactureWeekAndYear>().week; + info.manufactureOrModelDate = weekAndYear; + } - outInfo->deviceProductInfo = info; + outInfo->deviceProductInfo = info; + } } return statusTFromBinderStatus(status); } @@ -2754,6 +2750,20 @@ status_t SurfaceComposerClient::setOverrideFrameRate(uid_t uid, float frameRate) return statusTFromBinderStatus(status); } +status_t SurfaceComposerClient::updateSmallAreaDetection(std::vector<int32_t>& uids, + std::vector<float>& thresholds) { + binder::Status status = + ComposerServiceAIDL::getComposerService()->updateSmallAreaDetection(uids, thresholds); + return statusTFromBinderStatus(status); +} + +status_t SurfaceComposerClient::setSmallAreaDetectionThreshold(uid_t uid, float threshold) { + binder::Status status = + ComposerServiceAIDL::getComposerService()->setSmallAreaDetectionThreshold(uid, + threshold); + return statusTFromBinderStatus(status); +} + void SurfaceComposerClient::setAutoLowLatencyMode(const sp<IBinder>& display, bool on) { ComposerServiceAIDL::getComposerService()->setAutoLowLatencyMode(display, on); } diff --git a/libs/gui/TEST_MAPPING b/libs/gui/TEST_MAPPING index 941503548d..a4d9e77351 100644 --- a/libs/gui/TEST_MAPPING +++ b/libs/gui/TEST_MAPPING @@ -4,10 +4,58 @@ "path": "frameworks/native/libs/nativewindow" } ], - "postsubmit": [ + "presubmit": [ { - // TODO(257123981): move this to presubmit after dealing with existing breakages. - "name": "libgui_test" + "name": "libgui_test", + "options": [ + // TODO(b/277604286): Failing on Cuttlefish. + { + "exclude-filter": "MultiTextureConsumerTest#EGLImageTargetWorks" + }, + + // TODO(b/285011590): Failing on Cuttlefish. + { + "exclude-filter": "SurfaceTest#GetHdrSupport" + }, + { + "exclude-filter": "SurfaceTest#GetWideColorSupport" + }, + + // TODO(b/285006554): Failing on Cuttlefish. + { + "exclude-filter": "SurfaceTextureGLTest#InvalidWidthOrHeightFails" + }, + + // TODO(b/277347351): Known test data issues, failing across devices. + { + "exclude-filter": "SurfaceTextureGLTest#TexturingFromCpuFilledYV12BufferNpot" + }, + { + "exclude-filter": "SurfaceTextureGLTest#TexturingFromCpuFilledYV12BufferPow2" + }, + { + "exclude-filter": "SurfaceTextureGLTest#TexturingFromCpuFilledYV12BufferWithCrop" + }, + { + "exclude-filter": "SurfaceTextureGLTest#TexturingFromCpuFilledYV12BuffersRepeatedly" + }, + + // TODO(b/285041169): Hanging on Cuttlefish. + { + "exclude-filter": "SurfaceTextureGLThreadToGLTest#UpdateTexImageBeforeFrameFinishedCompletes" + }, + { + "exclude-filter": "SurfaceTextureGLThreadToGLTest#RepeatedUpdateTexImageBeforeFrameFinishedCompletes" + }, + { + "exclude-filter": "SurfaceTextureGLThreadToGLTest#RepeatedUpdateTexImageAfterFrameFinishedCompletes" + }, + + // TODO(b/285041070): Failing on Cuttlefish. + { + "exclude-filter": "SurfaceTextureGLToGLTest#EglDestroySurfaceUnrefsBuffers" + } + ] } ] } diff --git a/libs/gui/WindowInfo.cpp b/libs/gui/WindowInfo.cpp index 6df9ff1664..52af9d5114 100644 --- a/libs/gui/WindowInfo.cpp +++ b/libs/gui/WindowInfo.cpp @@ -90,8 +90,10 @@ status_t WindowInfo::writeToParcel(android::Parcel* parcel) const { } parcel->writeInt32(1); - // Ensure that the size of the flags that we use is 32 bits for writing into the parcel. + // Ensure that the size of custom types are what we expect for writing into the parcel. static_assert(sizeof(inputConfig) == 4u); + static_assert(sizeof(ownerPid.val()) == 4u); + static_assert(sizeof(ownerUid.val()) == 4u); // clang-format off status_t status = parcel->writeStrongBinder(token) ?: @@ -115,8 +117,8 @@ status_t WindowInfo::writeToParcel(android::Parcel* parcel) const { parcel->writeFloat(transform.dsdy()) ?: parcel->writeFloat(transform.ty()) ?: parcel->writeInt32(static_cast<int32_t>(touchOcclusionMode)) ?: - parcel->writeInt32(ownerPid) ?: - parcel->writeInt32(ownerUid) ?: + parcel->writeInt32(ownerPid.val()) ?: + parcel->writeInt32(ownerUid.val()) ?: parcel->writeUtf8AsUtf16(packageName) ?: parcel->writeInt32(inputConfig.get()) ?: parcel->writeInt32(displayId) ?: @@ -147,7 +149,7 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) { } float dsdx, dtdx, tx, dtdy, dsdy, ty; - int32_t lpFlags, lpType, touchOcclusionModeInt, inputConfigInt; + int32_t lpFlags, lpType, touchOcclusionModeInt, inputConfigInt, ownerPidInt, ownerUidInt; sp<IBinder> touchableRegionCropHandleSp; // clang-format off @@ -167,8 +169,8 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) { parcel->readFloat(&dsdy) ?: parcel->readFloat(&ty) ?: parcel->readInt32(&touchOcclusionModeInt) ?: - parcel->readInt32(&ownerPid) ?: - parcel->readInt32(&ownerUid) ?: + parcel->readInt32(&ownerPidInt) ?: + parcel->readInt32(&ownerUidInt) ?: parcel->readUtf8FromUtf16(&packageName) ?: parcel->readInt32(&inputConfigInt) ?: parcel->readInt32(&displayId) ?: @@ -190,6 +192,8 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) { transform.set({dsdx, dtdx, tx, dtdy, dsdy, ty, 0, 0, 1}); touchOcclusionMode = static_cast<TouchOcclusionMode>(touchOcclusionModeInt); inputConfig = ftl::Flags<InputConfig>(inputConfigInt); + ownerPid = Pid{ownerPidInt}; + ownerUid = Uid{static_cast<uid_t>(ownerUidInt)}; touchableRegionCropHandle = touchableRegionCropHandleSp; return OK; diff --git a/libs/gui/aidl/android/gui/FrameTimelineInfo.aidl b/libs/gui/aidl/android/gui/FrameTimelineInfo.aidl index 6a86c6a5cd..4b647a4ad2 100644 --- a/libs/gui/aidl/android/gui/FrameTimelineInfo.aidl +++ b/libs/gui/aidl/android/gui/FrameTimelineInfo.aidl @@ -37,4 +37,10 @@ parcelable FrameTimelineInfo { // Whether this vsyncId should be used to heuristically select the display refresh rate // TODO(b/281695725): Clean this up once TextureView use setFrameRate API boolean useForRefreshRateSelection = false; + + // The VsyncId of a frame that was not drawn and squashed into this frame. + long skippedFrameVsyncId = INVALID_VSYNC_ID; + + // The start time of a frame that was not drawn and squashed into this frame. + long skippedFrameStartTimeNanos = 0; } diff --git a/libs/gui/aidl/android/gui/ISurfaceComposer.aidl b/libs/gui/aidl/android/gui/ISurfaceComposer.aidl index 539a1c140e..516d159cc5 100644 --- a/libs/gui/aidl/android/gui/ISurfaceComposer.aidl +++ b/libs/gui/aidl/android/gui/ISurfaceComposer.aidl @@ -46,6 +46,7 @@ import android.gui.LayerDebugInfo; import android.gui.OverlayProperties; import android.gui.PullAtomData; import android.gui.ARect; +import android.gui.StalledTransactionInfo; import android.gui.StaticDisplayInfo; import android.gui.WindowInfosListenerInfo; @@ -480,6 +481,15 @@ interface ISurfaceComposer { */ void setOverrideFrameRate(int uid, float frameRate); + oneway void updateSmallAreaDetection(in int[] uids, in float[] thresholds); + + /** + * Set the small area detection threshold for a specified uid by SmallAreaDetectionController. + * Passing the threshold and uid to SurfaceFlinger to update the uid-threshold mapping + * in the scheduler. + */ + oneway void setSmallAreaDetectionThreshold(int uid, float threshold); + /** * Gets priority of the RenderEngine in SurfaceFlinger. */ @@ -507,4 +517,10 @@ interface ISurfaceComposer { void removeWindowInfosListener(IWindowInfosListener windowInfosListener); OverlayProperties getOverlaySupport(); + + /** + * Returns an instance of StalledTransaction if a transaction from the passed pid has not been + * applied in SurfaceFlinger due to an unsignaled fence. Otherwise, null is returned. + */ + @nullable StalledTransactionInfo getStalledTransactionInfo(int pid); } diff --git a/libs/gui/android/gui/StalledTransactionInfo.aidl b/libs/gui/android/gui/StalledTransactionInfo.aidl new file mode 100644 index 0000000000..e6aa9bd1c7 --- /dev/null +++ b/libs/gui/android/gui/StalledTransactionInfo.aidl @@ -0,0 +1,24 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.gui; + +/** @hide */ +parcelable StalledTransactionInfo { + String layerName; + long bufferId; + long frameNumber; +}
\ No newline at end of file diff --git a/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp b/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp index f684874aec..fd8ffe1f01 100644 --- a/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp +++ b/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp @@ -1172,9 +1172,12 @@ status_t H2BGraphicBufferProducer::setGenerationNumber(uint32_t generationNumber String8 H2BGraphicBufferProducer::getConsumerName() const { String8 lName; - mBase->getConsumerName([&lName] (hidl_string const& name) { - lName = name.c_str(); - }); + status_t transStatus = toStatusT( + mBase->getConsumerName([&lName](hidl_string const& name) { lName = name.c_str(); })); + if (transStatus != NO_ERROR) { + ALOGE("getConsumerName failed to transact: %d", transStatus); + return String8("TransactFailed"); + } return lName; } diff --git a/libs/gui/bufferqueue/2.0/H2BGraphicBufferProducer.cpp b/libs/gui/bufferqueue/2.0/H2BGraphicBufferProducer.cpp index 2f5b73ccbb..ae00a2642e 100644 --- a/libs/gui/bufferqueue/2.0/H2BGraphicBufferProducer.cpp +++ b/libs/gui/bufferqueue/2.0/H2BGraphicBufferProducer.cpp @@ -437,6 +437,10 @@ String8 H2BGraphicBufferProducer::getConsumerName() const { [&bName](hidl_string const& name) { bName = name.c_str(); }); + if (!transResult.isOk()) { + LOG(ERROR) << "getConsumerName: corrupted transaction."; + return String8("TransactFailed"); + } return bName; } diff --git a/libs/gui/fuzzer/libgui_fuzzer_utils.h b/libs/gui/fuzzer/libgui_fuzzer_utils.h index 4c7d0562af..67915f905e 100644 --- a/libs/gui/fuzzer/libgui_fuzzer_utils.h +++ b/libs/gui/fuzzer/libgui_fuzzer_utils.h @@ -151,6 +151,9 @@ public: MOCK_METHOD(binder::Status, getDisplayDecorationSupport, (const sp<IBinder>&, std::optional<gui::DisplayDecorationSupport>*), (override)); MOCK_METHOD(binder::Status, setOverrideFrameRate, (int32_t, float), (override)); + MOCK_METHOD(binder::Status, updateSmallAreaDetection, + (const std::vector<int32_t>&, const std::vector<float>&), (override)); + MOCK_METHOD(binder::Status, setSmallAreaDetectionThreshold, (int32_t, float), (override)); MOCK_METHOD(binder::Status, getGpuContextPriority, (int32_t*), (override)); MOCK_METHOD(binder::Status, getMaxAcquiredBufferCount, (int32_t*), (override)); MOCK_METHOD(binder::Status, addWindowInfosListener, @@ -158,6 +161,8 @@ public: MOCK_METHOD(binder::Status, removeWindowInfosListener, (const sp<gui::IWindowInfosListener>&), (override)); MOCK_METHOD(binder::Status, getOverlaySupport, (gui::OverlayProperties*), (override)); + MOCK_METHOD(binder::Status, getStalledTransactionInfo, + (int32_t, std::optional<gui::StalledTransactionInfo>*), (override)); }; class FakeBnSurfaceComposerClient : public gui::BnSurfaceComposerClient { diff --git a/libs/gui/fuzzer/libgui_surfaceComposerClient_fuzzer.cpp b/libs/gui/fuzzer/libgui_surfaceComposerClient_fuzzer.cpp index 57720dd513..95b7f39c11 100644 --- a/libs/gui/fuzzer/libgui_surfaceComposerClient_fuzzer.cpp +++ b/libs/gui/fuzzer/libgui_surfaceComposerClient_fuzzer.cpp @@ -186,8 +186,8 @@ void SurfaceComposerClientFuzzer::getWindowInfo(gui::WindowInfo* windowInfo) { windowInfo->touchableRegion = Region(getRect(&mFdp)); windowInfo->replaceTouchableRegionWithCrop = mFdp.ConsumeBool(); windowInfo->touchOcclusionMode = mFdp.PickValueInArray(kMode); - windowInfo->ownerPid = mFdp.ConsumeIntegral<int32_t>(); - windowInfo->ownerUid = mFdp.ConsumeIntegral<int32_t>(); + windowInfo->ownerPid = gui::Pid{mFdp.ConsumeIntegral<pid_t>()}; + windowInfo->ownerUid = gui::Uid{mFdp.ConsumeIntegral<uid_t>()}; windowInfo->packageName = mFdp.ConsumeRandomLengthString(kRandomStringMaxBytes); windowInfo->inputConfig = mFdp.PickValueInArray(kFeatures); } diff --git a/libs/gui/include/gui/LayerState.h b/libs/gui/include/gui/LayerState.h index a6f503ef55..62e5f89d21 100644 --- a/libs/gui/include/gui/LayerState.h +++ b/libs/gui/include/gui/LayerState.h @@ -270,9 +270,9 @@ struct layer_state_t { layer_state_t::eFrameRateChanged | layer_state_t::eFixedTransformHintChanged; // Changes affecting data sent to input. - static constexpr uint64_t INPUT_CHANGES = layer_state_t::GEOMETRY_CHANGES | - layer_state_t::HIERARCHY_CHANGES | layer_state_t::eInputInfoChanged | - layer_state_t::eDropInputModeChanged | layer_state_t::eTrustedOverlayChanged; + static constexpr uint64_t INPUT_CHANGES = layer_state_t::eInputInfoChanged | + layer_state_t::eDropInputModeChanged | layer_state_t::eTrustedOverlayChanged | + layer_state_t::eLayerStackChanged; // Changes that affect the visible region on a display. static constexpr uint64_t VISIBLE_REGION_CHANGES = diff --git a/libs/gui/include/gui/PidUid.h b/libs/gui/include/gui/PidUid.h new file mode 100644 index 0000000000..7930942882 --- /dev/null +++ b/libs/gui/include/gui/PidUid.h @@ -0,0 +1,56 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <ftl/mixins.h> +#include <sys/types.h> +#include <string> + +namespace android::gui { + +// Type-safe wrapper for a PID. +struct Pid : ftl::Constructible<Pid, pid_t>, ftl::Equatable<Pid>, ftl::Orderable<Pid> { + using Constructible::Constructible; + + const static Pid INVALID; + + constexpr auto val() const { return ftl::to_underlying(*this); } + + constexpr bool isValid() const { return val() >= 0; } + + std::string toString() const { return std::to_string(val()); } +}; + +const inline Pid Pid::INVALID{-1}; + +// Type-safe wrapper for a UID. +// We treat the unsigned equivalent of -1 as a singular invalid value. +struct Uid : ftl::Constructible<Uid, uid_t>, ftl::Equatable<Uid>, ftl::Orderable<Uid> { + using Constructible::Constructible; + + const static Uid INVALID; + + constexpr auto val() const { return ftl::to_underlying(*this); } + + constexpr bool isValid() const { return val() != static_cast<uid_t>(-1); } + + std::string toString() const { return std::to_string(val()); } +}; + +const inline Uid Uid::INVALID{static_cast<uid_t>(-1)}; + +} // namespace android::gui diff --git a/libs/gui/include/gui/SurfaceComposerClient.h b/libs/gui/include/gui/SurfaceComposerClient.h index fb57f63dad..7c55100784 100644 --- a/libs/gui/include/gui/SurfaceComposerClient.h +++ b/libs/gui/include/gui/SurfaceComposerClient.h @@ -203,6 +203,16 @@ public: // by GameManager. static status_t setOverrideFrameRate(uid_t uid, float frameRate); + // Update the small area detection whole uid-threshold mappings by same size uid and threshold + // vector. + // Ref:setSmallAreaDetectionThreshold. + static status_t updateSmallAreaDetection(std::vector<int32_t>& uids, + std::vector<float>& thresholds); + + // Sets the small area detection threshold to particular apps (uid). Passing value 0 means + // to disable small area detection to the app. + static status_t setSmallAreaDetectionThreshold(uid_t uid, float threshold); + // Switches on/off Auto Low Latency Mode on the connected display. This should only be // called if the connected display supports Auto Low Latency Mode as reported by // #getAutoLowLatencyModeSupport @@ -371,6 +381,10 @@ public: //! Get token for a physical display given its stable ID static sp<IBinder> getPhysicalDisplayToken(PhysicalDisplayId displayId); + // Returns StalledTransactionInfo if a transaction from the provided pid has not been applied + // due to an unsignaled fence. + static std::optional<gui::StalledTransactionInfo> getStalledTransactionInfo(pid_t pid); + struct SCHash { std::size_t operator()(const sp<SurfaceControl>& sc) const { return std::hash<SurfaceControl *>{}(sc.get()); @@ -410,7 +424,6 @@ public: static sp<IBinder> sApplyToken; void releaseBufferIfOverwriting(const layer_state_t& state); static void mergeFrameTimelineInfo(FrameTimelineInfo& t, const FrameTimelineInfo& other); - static void clearFrameTimelineInfo(FrameTimelineInfo& t); protected: std::unordered_map<sp<IBinder>, ComposerState, IBinderHash> mComposerStates; diff --git a/libs/gui/include/gui/WindowInfo.h b/libs/gui/include/gui/WindowInfo.h index 70b2ee8e32..7ff73874ae 100644 --- a/libs/gui/include/gui/WindowInfo.h +++ b/libs/gui/include/gui/WindowInfo.h @@ -21,6 +21,8 @@ #include <binder/Parcel.h> #include <binder/Parcelable.h> #include <ftl/flags.h> +#include <ftl/mixins.h> +#include <gui/PidUid.h> #include <gui/constants.h> #include <ui/Rect.h> #include <ui/Region.h> @@ -223,8 +225,8 @@ struct WindowInfo : public Parcelable { Region touchableRegion; TouchOcclusionMode touchOcclusionMode = TouchOcclusionMode::BLOCK_UNTRUSTED; - int32_t ownerPid = -1; - int32_t ownerUid = -1; + Pid ownerPid = Pid::INVALID; + Uid ownerUid = Uid::INVALID; std::string packageName; ftl::Flags<InputConfig> inputConfig; int32_t displayId = ADISPLAY_ID_NONE; diff --git a/libs/gui/tests/Android.bp b/libs/gui/tests/Android.bp index cd35d2fe3c..462ce6e14f 100644 --- a/libs/gui/tests/Android.bp +++ b/libs/gui/tests/Android.bp @@ -21,6 +21,7 @@ cc_test { ], srcs: [ + "LibGuiMain.cpp", // Custom gtest entrypoint "BLASTBufferQueue_test.cpp", "BufferItemConsumer_test.cpp", "BufferQueue_test.cpp", diff --git a/libs/gui/tests/AndroidTest.xml b/libs/gui/tests/AndroidTest.xml index 5e09fff6bb..31b10d7f54 100644 --- a/libs/gui/tests/AndroidTest.xml +++ b/libs/gui/tests/AndroidTest.xml @@ -23,6 +23,7 @@ <option name="screen-always-on" value="on" /> </target_preparer> <option name="test-suite-tag" value="apct" /> + <option name="not-shardable" value="true" /> <test class="com.android.tradefed.testtype.GTest" > <option name="native-test-device-path" value="/data/local/tmp" /> <option name="module-name" value="libgui_test" /> diff --git a/libs/gui/tests/BLASTBufferQueue_test.cpp b/libs/gui/tests/BLASTBufferQueue_test.cpp index a3ad6807c5..cd90168784 100644 --- a/libs/gui/tests/BLASTBufferQueue_test.cpp +++ b/libs/gui/tests/BLASTBufferQueue_test.cpp @@ -176,18 +176,6 @@ private: class BLASTBufferQueueTest : public ::testing::Test { public: protected: - BLASTBufferQueueTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), testInfo->name()); - } - - ~BLASTBufferQueueTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), testInfo->name()); - } - void SetUp() { mComposer = ComposerService::getComposerService(); mClient = new SurfaceComposerClient(); diff --git a/libs/gui/tests/BufferQueue_test.cpp b/libs/gui/tests/BufferQueue_test.cpp index 2f1fd3e78f..d585881582 100644 --- a/libs/gui/tests/BufferQueue_test.cpp +++ b/libs/gui/tests/BufferQueue_test.cpp @@ -17,6 +17,7 @@ #define LOG_TAG "BufferQueue_test" //#define LOG_NDEBUG 0 +#include "Constants.h" #include "MockConsumer.h" #include <gui/BufferItem.h> @@ -46,20 +47,6 @@ class BufferQueueTest : public ::testing::Test { public: protected: - BufferQueueTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } - - ~BufferQueueTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } - void GetMinUndequeuedBufferCount(int* bufferCount) { ASSERT_TRUE(bufferCount != nullptr); ASSERT_EQ(OK, mProducer->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, @@ -535,7 +522,8 @@ TEST_F(BufferQueueTest, TestGenerationNumbers) { int slot; sp<Fence> fence; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); sp<GraphicBuffer> buffer; ASSERT_EQ(OK, mProducer->requestBuffer(slot, &buffer)); @@ -578,7 +566,8 @@ TEST_F(BufferQueueTest, TestSharedBufferModeWithoutAutoRefresh) { sp<Fence> fence; sp<GraphicBuffer> buffer; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->requestBuffer(sharedSlot, &buffer)); // Queue the buffer @@ -592,7 +581,9 @@ TEST_F(BufferQueueTest, TestSharedBufferModeWithoutAutoRefresh) { // always the same one and because async mode gets enabled. int slot; for (int i = 0; i < 5; i++) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(sharedSlot, slot); ASSERT_EQ(OK, mProducer->queueBuffer(sharedSlot, input, &output)); } @@ -629,7 +620,8 @@ TEST_F(BufferQueueTest, TestSharedBufferModeWithAutoRefresh) { sp<Fence> fence; sp<GraphicBuffer> buffer; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->requestBuffer(sharedSlot, &buffer)); // Queue the buffer @@ -656,7 +648,9 @@ TEST_F(BufferQueueTest, TestSharedBufferModeWithAutoRefresh) { // always return the same one. int slot; for (int i = 0; i < 5; i++) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(sharedSlot, slot); ASSERT_EQ(OK, mProducer->queueBuffer(sharedSlot, input, &output)); } @@ -695,7 +689,8 @@ TEST_F(BufferQueueTest, TestSharedBufferModeUsingAlreadyDequeuedBuffer) { sp<Fence> fence; sp<GraphicBuffer> buffer; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->requestBuffer(sharedSlot, &buffer)); // Enable shared buffer mode @@ -712,7 +707,9 @@ TEST_F(BufferQueueTest, TestSharedBufferModeUsingAlreadyDequeuedBuffer) { // always the same one and because async mode gets enabled. int slot; for (int i = 0; i < 5; i++) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(sharedSlot, slot); ASSERT_EQ(OK, mProducer->queueBuffer(sharedSlot, input, &output)); } @@ -747,7 +744,8 @@ TEST_F(BufferQueueTest, TestTimeouts) { for (int i = 0; i < 5; ++i) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> fence = Fence::NO_FENCE; - auto result = mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr); + auto result = mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr); if (i < 2) { ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); @@ -774,7 +772,9 @@ TEST_F(BufferQueueTest, TestTimeouts) { for (int i = 0; i < 2; ++i) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> fence = Fence::NO_FENCE; - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->requestBuffer(slot, &buffer)); IGraphicBufferProducer::QueueBufferInput input(0ull, true, HAL_DATASPACE_UNKNOWN, Rect::INVALID_RECT, @@ -785,7 +785,9 @@ TEST_F(BufferQueueTest, TestTimeouts) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> fence = Fence::NO_FENCE; auto startTime = systemTime(); - ASSERT_EQ(TIMED_OUT, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(TIMED_OUT, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_GE(systemTime() - startTime, TIMEOUT); // We're technically attaching the same buffer multiple times (since we @@ -806,7 +808,8 @@ TEST_F(BufferQueueTest, CanAttachWhileDisallowingAllocation) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> sourceFence; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&slot, &sourceFence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&slot, &sourceFence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); sp<GraphicBuffer> buffer; ASSERT_EQ(OK, mProducer->requestBuffer(slot, &buffer)); ASSERT_EQ(OK, mProducer->detachBuffer(slot)); @@ -829,7 +832,8 @@ TEST_F(BufferQueueTest, CanRetrieveLastQueuedBuffer) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> fence; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); sp<GraphicBuffer> firstBuffer; ASSERT_EQ(OK, mProducer->requestBuffer(slot, &firstBuffer)); @@ -841,7 +845,8 @@ TEST_F(BufferQueueTest, CanRetrieveLastQueuedBuffer) { // Dequeue a second buffer slot = BufferQueue::INVALID_BUFFER_SLOT; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); sp<GraphicBuffer> secondBuffer; ASSERT_EQ(OK, mProducer->requestBuffer(slot, &secondBuffer)); @@ -892,8 +897,8 @@ TEST_F(BufferQueueTest, TestOccupancyHistory) { int slots[3] = {}; mProducer->setMaxDequeuedBufferCount(3); for (size_t i = 0; i < 3; ++i) { - status_t result = - mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, 0, nullptr, nullptr); + status_t result = mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, + TEST_PRODUCER_USAGE_BITS, nullptr, nullptr); ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); ASSERT_EQ(OK, mProducer->requestBuffer(slots[i], &buffer)); } @@ -906,7 +911,9 @@ TEST_F(BufferQueueTest, TestOccupancyHistory) { // The first segment is a two-buffer segment, so we only put one buffer into // the queue at a time for (size_t i = 0; i < 5; ++i) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); ASSERT_EQ(OK, mConsumer->releaseBuffer(item.mSlot, item.mFrameNumber, @@ -921,16 +928,22 @@ TEST_F(BufferQueueTest, TestOccupancyHistory) { // two-buffer segment, but then at the end, we put two buffers in the queue // at the same time before draining it. for (size_t i = 0; i < 5; ++i) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); ASSERT_EQ(OK, mConsumer->releaseBuffer(item.mSlot, item.mFrameNumber, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE)); std::this_thread::sleep_for(16ms); } - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); ASSERT_EQ(OK, mConsumer->releaseBuffer(item.mSlot, item.mFrameNumber, @@ -945,10 +958,14 @@ TEST_F(BufferQueueTest, TestOccupancyHistory) { // The third segment is a triple-buffer segment, so the queue is switching // between one buffer and two buffers deep. - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); for (size_t i = 0; i < 5; ++i) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); ASSERT_EQ(OK, mConsumer->releaseBuffer(item.mSlot, item.mFrameNumber, @@ -1047,8 +1064,8 @@ TEST_F(BufferQueueTest, TestDiscardFreeBuffers) { int slots[4] = {}; mProducer->setMaxDequeuedBufferCount(4); for (size_t i = 0; i < 4; ++i) { - status_t result = - mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, 0, nullptr, nullptr); + status_t result = mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, + TEST_PRODUCER_USAGE_BITS, nullptr, nullptr); ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); ASSERT_EQ(OK, mProducer->requestBuffer(slots[i], &buffer)); } @@ -1059,14 +1076,22 @@ TEST_F(BufferQueueTest, TestDiscardFreeBuffers) { // Get buffers in all states: dequeued, filled, acquired, free // Fill 3 buffers - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); // Dequeue 1 buffer - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); // Acquire and free 1 buffer ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); @@ -1132,8 +1157,8 @@ TEST_F(BufferQueueTest, TestBufferReplacedInQueueBuffer) { int slots[2] = {}; ASSERT_EQ(OK, mProducer->setMaxDequeuedBufferCount(2)); for (size_t i = 0; i < 2; ++i) { - status_t result = - mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, 0, nullptr, nullptr); + status_t result = mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, + TEST_PRODUCER_USAGE_BITS, nullptr, nullptr); ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); ASSERT_EQ(OK, mProducer->requestBuffer(slots[i], &buffer)); } @@ -1143,10 +1168,14 @@ TEST_F(BufferQueueTest, TestBufferReplacedInQueueBuffer) { // Fill 2 buffers without consumer consuming them. Verify that all // queued buffer returns proper bufferReplaced flag - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(false, output.bufferReplaced); - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(true, output.bufferReplaced); } @@ -1167,7 +1196,8 @@ TEST_F(BufferQueueTest, TestStaleBufferHandleSentAfterDisconnect) { NATIVE_WINDOW_SCALING_MODE_FREEZE, 0, Fence::NO_FENCE); // Dequeue, request, and queue one buffer - status_t result = mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr); + status_t result = mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr); ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); ASSERT_EQ(OK, mProducer->requestBuffer(slot, &buffer)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); @@ -1182,7 +1212,9 @@ TEST_F(BufferQueueTest, TestStaleBufferHandleSentAfterDisconnect) { EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE)); // Dequeue and queue the buffer again - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); // Acquire and release the buffer again. Upon acquiring, the buffer handle @@ -1194,7 +1226,9 @@ TEST_F(BufferQueueTest, TestStaleBufferHandleSentAfterDisconnect) { EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE)); // Dequeue and queue the buffer again - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); // Disconnect the producer end. This should clear all of the slots and mark diff --git a/libs/gui/tests/Constants.h b/libs/gui/tests/Constants.h new file mode 100644 index 0000000000..85c0f9faab --- /dev/null +++ b/libs/gui/tests/Constants.h @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <hardware/gralloc.h> + +// Arbitrary non-zero usage flag. +constexpr uint64_t TEST_PRODUCER_USAGE_BITS = GRALLOC_USAGE_SW_READ_RARELY; diff --git a/libs/gui/tests/CpuConsumer_test.cpp b/libs/gui/tests/CpuConsumer_test.cpp index 0a14afac55..d80bd9c62a 100644 --- a/libs/gui/tests/CpuConsumer_test.cpp +++ b/libs/gui/tests/CpuConsumer_test.cpp @@ -62,7 +62,7 @@ protected: const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); CpuConsumerTestParams params = GetParam(); - ALOGD("** Starting test %s (%d x %d, %d, 0x%x)", + ALOGD("** Starting parameterized test %s (%d x %d, %d, 0x%x)", test_info->name(), params.width, params.height, params.maxLockedBuffers, params.format); diff --git a/libs/gui/tests/EndToEndNativeInputTest.cpp b/libs/gui/tests/EndToEndNativeInputTest.cpp index 4ec7a06cb8..4d5bd5b3fa 100644 --- a/libs/gui/tests/EndToEndNativeInputTest.cpp +++ b/libs/gui/tests/EndToEndNativeInputTest.cpp @@ -821,7 +821,7 @@ TEST_F(InputSurfacesTest, touch_flag_obscured) { // with flag AMOTION_EVENT_FLAG_WINDOW_IS_OBSCURED std::unique_ptr<InputSurface> nonTouchableSurface = makeSurface(100, 100); nonTouchableSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - nonTouchableSurface->mInputInfo.ownerUid = 22222; + nonTouchableSurface->mInputInfo.ownerUid = gui::Uid{22222}; // Overriding occlusion mode otherwise the touch would be discarded at InputDispatcher by // the default obscured/untrusted touch filter introduced in S. nonTouchableSurface->mInputInfo.touchOcclusionMode = TouchOcclusionMode::ALLOW; @@ -842,8 +842,8 @@ TEST_F(InputSurfacesTest, touch_flag_partially_obscured_with_crop) { std::unique_ptr<InputSurface> nonTouchableSurface = makeSurface(100, 100); nonTouchableSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); parentSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - nonTouchableSurface->mInputInfo.ownerUid = 22222; - parentSurface->mInputInfo.ownerUid = 22222; + nonTouchableSurface->mInputInfo.ownerUid = gui::Uid{22222}; + parentSurface->mInputInfo.ownerUid = gui::Uid{22222}; nonTouchableSurface->showAt(0, 0); parentSurface->showAt(100, 100); @@ -866,8 +866,8 @@ TEST_F(InputSurfacesTest, touch_not_obscured_with_crop) { std::unique_ptr<InputSurface> nonTouchableSurface = makeSurface(100, 100); nonTouchableSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); parentSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - nonTouchableSurface->mInputInfo.ownerUid = 22222; - parentSurface->mInputInfo.ownerUid = 22222; + nonTouchableSurface->mInputInfo.ownerUid = gui::Uid{22222}; + parentSurface->mInputInfo.ownerUid = gui::Uid{22222}; nonTouchableSurface->showAt(0, 0); parentSurface->showAt(50, 50); @@ -886,7 +886,7 @@ TEST_F(InputSurfacesTest, touch_not_obscured_with_zero_sized_bql) { std::unique_ptr<InputSurface> bufferSurface = InputSurface::makeBufferInputSurface(mComposerClient, 0, 0); bufferSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - bufferSurface->mInputInfo.ownerUid = 22222; + bufferSurface->mInputInfo.ownerUid = gui::Uid{22222}; surface->showAt(10, 10); bufferSurface->showAt(50, 50, Rect::EMPTY_RECT); @@ -901,7 +901,7 @@ TEST_F(InputSurfacesTest, touch_not_obscured_with_zero_sized_blast) { std::unique_ptr<BlastInputSurface> bufferSurface = BlastInputSurface::makeBlastInputSurface(mComposerClient, 0, 0); bufferSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - bufferSurface->mInputInfo.ownerUid = 22222; + bufferSurface->mInputInfo.ownerUid = gui::Uid{22222}; surface->showAt(10, 10); bufferSurface->showAt(50, 50, Rect::EMPTY_RECT); @@ -948,13 +948,13 @@ TEST_F(InputSurfacesTest, strict_unobscured_input_scaled_without_crop_window) { TEST_F(InputSurfacesTest, strict_unobscured_input_obscured_window) { std::unique_ptr<InputSurface> surface = makeSurface(100, 100); - surface->mInputInfo.ownerUid = 11111; + surface->mInputInfo.ownerUid = gui::Uid{11111}; surface->doTransaction( [&](auto &t, auto &sc) { t.setDropInputMode(sc, gui::DropInputMode::OBSCURED); }); surface->showAt(100, 100); std::unique_ptr<InputSurface> obscuringSurface = makeSurface(100, 100); obscuringSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - obscuringSurface->mInputInfo.ownerUid = 22222; + obscuringSurface->mInputInfo.ownerUid = gui::Uid{22222}; obscuringSurface->showAt(100, 100); injectTap(101, 101); EXPECT_EQ(surface->consumeEvent(100), nullptr); @@ -967,13 +967,13 @@ TEST_F(InputSurfacesTest, strict_unobscured_input_obscured_window) { TEST_F(InputSurfacesTest, strict_unobscured_input_partially_obscured_window) { std::unique_ptr<InputSurface> surface = makeSurface(100, 100); - surface->mInputInfo.ownerUid = 11111; + surface->mInputInfo.ownerUid = gui::Uid{11111}; surface->doTransaction( [&](auto &t, auto &sc) { t.setDropInputMode(sc, gui::DropInputMode::OBSCURED); }); surface->showAt(100, 100); std::unique_ptr<InputSurface> obscuringSurface = makeSurface(100, 100); obscuringSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - obscuringSurface->mInputInfo.ownerUid = 22222; + obscuringSurface->mInputInfo.ownerUid = gui::Uid{22222}; obscuringSurface->showAt(190, 190); injectTap(101, 101); diff --git a/libs/gui/tests/GLTest.cpp b/libs/gui/tests/GLTest.cpp index e5f4aaa999..40af8e845a 100644 --- a/libs/gui/tests/GLTest.cpp +++ b/libs/gui/tests/GLTest.cpp @@ -29,10 +29,6 @@ static int abs(int value) { } void GLTest::SetUp() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), testInfo->name()); - mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); ASSERT_EQ(EGL_SUCCESS, eglGetError()); ASSERT_NE(EGL_NO_DISPLAY, mEglDisplay); @@ -132,10 +128,6 @@ void GLTest::TearDown() { eglTerminate(mEglDisplay); } ASSERT_EQ(EGL_SUCCESS, eglGetError()); - - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), testInfo->name()); } EGLint const* GLTest::getConfigAttribs() { diff --git a/libs/gui/tests/IGraphicBufferProducer_test.cpp b/libs/gui/tests/IGraphicBufferProducer_test.cpp index e6cb89cb83..b1f5d083c7 100644 --- a/libs/gui/tests/IGraphicBufferProducer_test.cpp +++ b/libs/gui/tests/IGraphicBufferProducer_test.cpp @@ -17,6 +17,7 @@ #define LOG_TAG "IGraphicBufferProducer_test" //#define LOG_NDEBUG 0 +#include "Constants.h" #include "MockConsumer.h" #include <gtest/gtest.h> @@ -40,7 +41,6 @@ #define TEST_API NATIVE_WINDOW_API_CPU #define TEST_API_OTHER NATIVE_WINDOW_API_EGL // valid API that's not TEST_API #define TEST_CONTROLLED_BY_APP false -#define TEST_PRODUCER_USAGE_BITS (0) namespace android { @@ -82,11 +82,6 @@ protected: IGraphicBufferProducerTest() {} virtual void SetUp() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - mMC = new MockConsumer; switch (GetParam()) { @@ -111,13 +106,6 @@ protected: ASSERT_OK(mConsumer->consumerConnect(mMC, /*controlledByApp*/ false)); } - virtual void TearDown() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } - status_t TryConnectProducer() { IGraphicBufferProducer::QueueBufferOutput output; return mProducer->connect(TEST_TOKEN, diff --git a/libs/gui/tests/LibGuiMain.cpp b/libs/gui/tests/LibGuiMain.cpp new file mode 100644 index 0000000000..10f7207588 --- /dev/null +++ b/libs/gui/tests/LibGuiMain.cpp @@ -0,0 +1,38 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gtest/gtest.h" +#include "log/log.h" + +namespace { + +class TestCaseLogger : public ::testing::EmptyTestEventListener { + void OnTestStart(const ::testing::TestInfo& testInfo) override { + ALOGD("Begin test: %s#%s", testInfo.test_suite_name(), testInfo.name()); + } + + void OnTestEnd(const testing::TestInfo& testInfo) override { + ALOGD("End test: %s#%s", testInfo.test_suite_name(), testInfo.name()); + } +}; + +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + testing::UnitTest::GetInstance()->listeners().Append(new TestCaseLogger()); + return RUN_ALL_TESTS(); +}
\ No newline at end of file diff --git a/libs/gui/tests/Malicious.cpp b/libs/gui/tests/Malicious.cpp index 58d7cc6f35..376420c42b 100644 --- a/libs/gui/tests/Malicious.cpp +++ b/libs/gui/tests/Malicious.cpp @@ -151,7 +151,6 @@ TEST(Malicious, Bug36991414Max) { sp<MaliciousBQP> malicious = getMaliciousBQP(); sp<Surface> surface = new Surface(malicious); - ASSERT_EQ(NO_ERROR, surface->connect(NATIVE_WINDOW_API_CPU, nullptr, false)); ANativeWindow_Buffer buffer; ASSERT_EQ(NO_ERROR, surface->lock(&buffer, nullptr)); ASSERT_EQ(NO_ERROR, surface->unlockAndPost()); @@ -165,7 +164,6 @@ TEST(Malicious, Bug36991414Min) { sp<MaliciousBQP> malicious = getMaliciousBQP(); sp<Surface> surface = new Surface(malicious); - ASSERT_EQ(NO_ERROR, surface->connect(NATIVE_WINDOW_API_CPU, nullptr, false)); ANativeWindow_Buffer buffer; ASSERT_EQ(NO_ERROR, surface->lock(&buffer, nullptr)); ASSERT_EQ(NO_ERROR, surface->unlockAndPost()); @@ -179,7 +177,6 @@ TEST(Malicious, Bug36991414NegativeOne) { sp<MaliciousBQP> malicious = getMaliciousBQP(); sp<Surface> surface = new Surface(malicious); - ASSERT_EQ(NO_ERROR, surface->connect(NATIVE_WINDOW_API_CPU, nullptr, false)); ANativeWindow_Buffer buffer; ASSERT_EQ(NO_ERROR, surface->lock(&buffer, nullptr)); ASSERT_EQ(NO_ERROR, surface->unlockAndPost()); @@ -193,7 +190,6 @@ TEST(Malicious, Bug36991414NumSlots) { sp<MaliciousBQP> malicious = getMaliciousBQP(); sp<Surface> surface = new Surface(malicious); - ASSERT_EQ(NO_ERROR, surface->connect(NATIVE_WINDOW_API_CPU, nullptr, false)); ANativeWindow_Buffer buffer; ASSERT_EQ(NO_ERROR, surface->lock(&buffer, nullptr)); ASSERT_EQ(NO_ERROR, surface->unlockAndPost()); diff --git a/libs/gui/tests/StreamSplitter_test.cpp b/libs/gui/tests/StreamSplitter_test.cpp index 2f14924a15..f34b03eade 100644 --- a/libs/gui/tests/StreamSplitter_test.cpp +++ b/libs/gui/tests/StreamSplitter_test.cpp @@ -30,23 +30,7 @@ namespace android { -class StreamSplitterTest : public ::testing::Test { - -protected: - StreamSplitterTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } - - ~StreamSplitterTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } -}; +class StreamSplitterTest : public ::testing::Test {}; struct FakeListener : public BnConsumerListener { virtual void onFrameAvailable(const BufferItem& /* item */) {} diff --git a/libs/gui/tests/SurfaceTextureClient_test.cpp b/libs/gui/tests/SurfaceTextureClient_test.cpp index 82b66972d9..b28dca8ab4 100644 --- a/libs/gui/tests/SurfaceTextureClient_test.cpp +++ b/libs/gui/tests/SurfaceTextureClient_test.cpp @@ -40,11 +40,6 @@ protected: } virtual void SetUp() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - sp<IGraphicBufferProducer> producer; sp<IGraphicBufferConsumer> consumer; BufferQueue::createBufferQueue(&producer, &consumer); @@ -96,11 +91,6 @@ protected: eglDestroyContext(mEglDisplay, mEglContext); eglDestroySurface(mEglDisplay, mEglSurface); eglTerminate(mEglDisplay); - - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), - testInfo->name()); } virtual EGLint const* getConfigAttribs() { diff --git a/libs/gui/tests/SurfaceTextureGL.h b/libs/gui/tests/SurfaceTextureGL.h index 53eb68cad8..9d8af5d0f5 100644 --- a/libs/gui/tests/SurfaceTextureGL.h +++ b/libs/gui/tests/SurfaceTextureGL.h @@ -17,6 +17,7 @@ #ifndef ANDROID_SURFACE_TEXTURE_GL_H #define ANDROID_SURFACE_TEXTURE_GL_H +#include "Constants.h" #include "GLTest.h" #include "FrameWaiter.h" @@ -43,6 +44,7 @@ protected: true, false); mSTC = new Surface(producer); mANW = mSTC; + ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(), TEST_PRODUCER_USAGE_BITS)); mTextureRenderer = new TextureRenderer(TEX_ID, mST); ASSERT_NO_FATAL_FAILURE(mTextureRenderer->SetUp()); mFW = new FrameWaiter; diff --git a/libs/gui/tests/Surface_test.cpp b/libs/gui/tests/Surface_test.cpp index c1b67b4396..6adcd966c5 100644 --- a/libs/gui/tests/Surface_test.cpp +++ b/libs/gui/tests/Surface_test.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "Constants.h" #include "MockConsumer.h" #include <gtest/gtest.h> @@ -148,6 +149,7 @@ protected: /*listener*/listener)); const int BUFFER_COUNT = 4 + extraDiscardedBuffers; ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(window.get(), BUFFER_COUNT)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); ANativeWindowBuffer* buffers[BUFFER_COUNT]; // Dequeue first to allocate a number of buffers @@ -530,7 +532,8 @@ TEST_F(SurfaceTest, DynamicSetBufferCount) { ASSERT_EQ(NO_ERROR, native_window_api_connect(window.get(), NATIVE_WINDOW_API_CPU)); - native_window_set_buffer_count(window.get(), 4); + ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(window.get(), 4)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); int fence; ANativeWindowBuffer* buffer; @@ -560,6 +563,7 @@ TEST_F(SurfaceTest, GetAndFlushRemovedBuffers) { /*reportBufferRemoval*/true)); const int BUFFER_COUNT = 4; ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(window.get(), BUFFER_COUNT)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); sp<GraphicBuffer> detachedBuffer; sp<Fence> outFence; @@ -989,6 +993,15 @@ public: return binder::Status::ok(); } + binder::Status updateSmallAreaDetection(const std::vector<int32_t>& /*uids*/, + const std::vector<float>& /*thresholds*/) { + return binder::Status::ok(); + } + + binder::Status setSmallAreaDetectionThreshold(int32_t /*uid*/, float /*threshold*/) { + return binder::Status::ok(); + } + binder::Status getGpuContextPriority(int32_t* /*outPriority*/) override { return binder::Status::ok(); } @@ -1012,6 +1025,11 @@ public: return binder::Status::ok(); } + binder::Status getStalledTransactionInfo( + int32_t /*pid*/, std::optional<gui::StalledTransactionInfo>* /*result*/) override { + return binder::Status::ok(); + } + protected: IBinder* onAsBinder() override { return nullptr; } @@ -1203,7 +1221,8 @@ protected: ASSERT_EQ(NO_ERROR, native_window_api_connect(mWindow.get(), NATIVE_WINDOW_API_CPU)); - native_window_set_buffer_count(mWindow.get(), 4); + ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(mWindow.get(), 4)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(mWindow.get(), TEST_PRODUCER_USAGE_BITS)); } void disableFrameTimestamps() { @@ -2069,8 +2088,9 @@ TEST_F(SurfaceTest, DequeueWithConsumerDrivenSize) { sp<Surface> surface = new Surface(producer); sp<ANativeWindow> window(surface); - native_window_api_connect(window.get(), NATIVE_WINDOW_API_CPU); - native_window_set_buffers_dimensions(window.get(), 0, 0); + ASSERT_EQ(NO_ERROR, native_window_api_connect(window.get(), NATIVE_WINDOW_API_CPU)); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_dimensions(window.get(), 0, 0)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); int fence; ANativeWindowBuffer* buffer; @@ -2122,6 +2142,7 @@ TEST_F(SurfaceTest, DequeueWithConsumerDrivenSize) { native_window_api_connect(window.get(), NATIVE_WINDOW_API_CPU); consumer->setTransformHint(NATIVE_WINDOW_TRANSFORM_ROT_270); native_window_set_buffers_dimensions(window.get(), 0, 0); + native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS); ASSERT_EQ(NO_ERROR, window->dequeueBuffer(window.get(), &buffer, &fence)); EXPECT_EQ(10, buffer->width); EXPECT_EQ(20, buffer->height); diff --git a/libs/gui/tests/WindowInfo_test.cpp b/libs/gui/tests/WindowInfo_test.cpp index 11b87efda7..461fe4a4ce 100644 --- a/libs/gui/tests/WindowInfo_test.cpp +++ b/libs/gui/tests/WindowInfo_test.cpp @@ -61,8 +61,8 @@ TEST(WindowInfo, Parcelling) { i.alpha = 0.7; i.transform.set({0.4, -1, 100, 0.5, 0, 40, 0, 0, 1}); i.touchOcclusionMode = TouchOcclusionMode::ALLOW; - i.ownerPid = 19; - i.ownerUid = 24; + i.ownerPid = gui::Pid{19}; + i.ownerUid = gui::Uid{24}; i.packageName = "com.example.package"; i.inputConfig = WindowInfo::InputConfig::NOT_FOCUSABLE; i.displayId = 34; diff --git a/libs/input/Android.bp b/libs/input/Android.bp index 869458c407..022dfaddc1 100644 --- a/libs/input/Android.bp +++ b/libs/input/Android.bp @@ -33,6 +33,138 @@ filegroup { ], } +aidl_interface { + name: "inputconstants", + host_supported: true, + vendor_available: true, + unstable: true, + srcs: [ + ":inputconstants_aidl", + ], + + backend: { + rust: { + enabled: true, + }, + }, +} + +rust_bindgen { + name: "libinput_bindgen", + host_supported: true, + crate_name: "input_bindgen", + visibility: ["//frameworks/native/services/inputflinger"], + wrapper_src: "InputWrapper.hpp", + + include_dirs: [ + "frameworks/native/include", + ], + + source_stem: "bindings", + + bindgen_flags: [ + "--verbose", + "--allowlist-var=AMOTION_EVENT_FLAG_CANCELED", + "--allowlist-var=AMOTION_EVENT_ACTION_CANCEL", + "--allowlist-var=AMOTION_EVENT_ACTION_UP", + "--allowlist-var=AMOTION_EVENT_ACTION_POINTER_DOWN", + "--allowlist-var=AMOTION_EVENT_ACTION_DOWN", + "--allowlist-var=AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT", + "--allowlist-var=MAX_POINTER_ID", + ], + + static_libs: [ + "inputconstants-cpp", + "libui-types", + ], + shared_libs: ["libc++"], + header_libs: [ + "native_headers", + "jni_headers", + "flatbuffer_headers", + ], +} + +// Contains methods to help access C++ code from rust +cc_library_static { + name: "libinput_from_rust_to_cpp", + cpp_std: "c++20", + host_supported: true, + cflags: [ + "-Wall", + "-Wextra", + "-Werror", + ], + srcs: [ + "FromRustToCpp.cpp", + ], + + generated_headers: [ + "cxx-bridge-header", + ], + generated_sources: ["libinput_cxx_bridge_code"], + + shared_libs: [ + "libbase", + ], +} + +genrule { + name: "libinput_cxx_bridge_code", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) >> $(out)", + srcs: ["input_verifier.rs"], + out: ["inputverifier_generated.cpp"], +} + +genrule { + name: "libinput_cxx_bridge_header", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) --header >> $(out)", + srcs: ["input_verifier.rs"], + out: ["input_verifier.rs.h"], +} + +rust_defaults { + name: "libinput_rust_defaults", + srcs: ["input_verifier.rs"], + host_supported: true, + rustlibs: [ + "libbitflags", + "libcxx", + "libinput_bindgen", + "liblogger", + "liblog_rust", + "inputconstants-rust", + ], + + shared_libs: [ + "libbase", + "liblog", + ], +} + +rust_ffi_static { + name: "libinput_rust", + crate_name: "input", + defaults: ["libinput_rust_defaults"], +} + +rust_test { + name: "libinput_rust_test", + defaults: ["libinput_rust_defaults"], + whole_static_libs: [ + "libinput_from_rust_to_cpp", + ], + test_options: { + unit_test: true, + }, + test_suites: ["device_tests"], + sanitize: { + hwaddress: true, + }, +} + cc_library { name: "libinput", cpp_std: "c++20", @@ -44,6 +176,7 @@ cc_library { "-Wno-unused-parameter", ], srcs: [ + "FromRustToCpp.cpp", "Input.cpp", "InputDevice.cpp", "InputEventLabels.cpp", @@ -52,6 +185,7 @@ cc_library { "KeyCharacterMap.cpp", "KeyLayoutMap.cpp", "MotionPredictor.cpp", + "MotionPredictorMetricsManager.cpp", "PrintTools.cpp", "PropertyMap.cpp", "TfLiteMotionPredictor.cpp", @@ -65,19 +199,28 @@ cc_library { header_libs: [ "flatbuffer_headers", "jni_headers", + "libeigen", "tensorflow_headers", ], - export_header_lib_headers: ["jni_headers"], + export_header_lib_headers: [ + "jni_headers", + "libeigen", + ], generated_headers: [ + "cxx-bridge-header", + "libinput_cxx_bridge_header", "toolbox_input_labels", ], + generated_sources: ["libinput_cxx_bridge_code"], + shared_libs: [ "libbase", "libcutils", "liblog", "libPlatformProperties", + "libtinyxml2", "libvintf", ], @@ -85,21 +228,36 @@ cc_library { "-Wl,--exclude-libs=libtflite_static.a", ], + sanitize: { + undefined: true, + all_undefined: true, + misc_undefined: ["integer"], + }, + static_libs: [ + "inputconstants-cpp", "libui-types", "libtflite_static", ], + whole_static_libs: [ + "libinput_rust", + ], + export_static_lib_headers: [ "libui-types", ], + export_generated_headers: [ + "cxx-bridge-header", + "libinput_cxx_bridge_header", + ], + target: { android: { srcs: [ "InputTransport.cpp", "android/os/IInputFlinger.aidl", - ":inputconstants_aidl", ], export_shared_lib_headers: ["libbinder"], @@ -107,6 +265,10 @@ cc_library { shared_libs: [ "libutils", "libbinder", + // Stats logging library and its dependencies. + "libstatslog_libinput", + "libstatsbootstrap", + "android.os.statsbootstrap_aidl-cpp", ], static_libs: [ @@ -117,12 +279,9 @@ cc_library { "libgui_window_info_static", ], - sanitize: { - misc_undefined: ["integer"], - }, - required: [ "motion_predictor_model_prebuilt", + "motion_predictor_model_config", ], }, host: { @@ -138,12 +297,8 @@ cc_library { host_linux: { srcs: [ "InputTransport.cpp", - "android/os/IInputConstants.aidl", - "android/os/IInputFlinger.aidl", - "android/os/InputConfig.aidl", ], static_libs: [ - "libhostgraphics", "libgui_window_info_static", ], shared_libs: [ @@ -165,6 +320,43 @@ cc_library { }, } +// Use bootstrap version of stats logging library. +// libinput is a bootstrap process (starts early in the boot process), and thus can't use the normal +// `libstatslog` because that requires `libstatssocket`, which is only available later in the boot. +cc_library { + name: "libstatslog_libinput", + generated_sources: ["statslog_libinput.cpp"], + generated_headers: ["statslog_libinput.h"], + export_generated_headers: ["statslog_libinput.h"], + shared_libs: [ + "libbinder", + "libstatsbootstrap", + "libutils", + "android.os.statsbootstrap_aidl-cpp", + ], +} + +genrule { + name: "statslog_libinput.h", + tools: ["stats-log-api-gen"], + cmd: "$(location stats-log-api-gen) --header $(genDir)/statslog_libinput.h --module libinput" + + " --namespace android,stats,libinput --bootstrap", + out: [ + "statslog_libinput.h", + ], +} + +genrule { + name: "statslog_libinput.cpp", + tools: ["stats-log-api-gen"], + cmd: "$(location stats-log-api-gen) --cpp $(genDir)/statslog_libinput.cpp --module libinput" + + " --namespace android,stats,libinput --importHeader statslog_libinput.h" + + " --bootstrap", + out: [ + "statslog_libinput.cpp", + ], +} + cc_defaults { name: "libinput_fuzz_defaults", cpp_std: "c++20", diff --git a/libs/ui/include_types/ui/DataspaceUtils.h b/libs/input/FromRustToCpp.cpp index a461cb4e68..e4ce62e734 100644 --- a/libs/ui/include_types/ui/DataspaceUtils.h +++ b/libs/input/FromRustToCpp.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Android Open Source Project + * Copyright 2023 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,13 @@ * limitations under the License. */ -#pragma once - -#include <ui/GraphicTypes.h> +#include <android-base/logging.h> +#include <ffi/FromRustToCpp.h> namespace android { -inline bool isHdrDataspace(ui::Dataspace dataspace) { - const auto transfer = dataspace & HAL_DATASPACE_TRANSFER_MASK; - - return transfer == HAL_DATASPACE_TRANSFER_ST2084 || transfer == HAL_DATASPACE_TRANSFER_HLG; +bool shouldLog(rust::Str tag) { + return android::base::ShouldLog(android::base::LogSeverity::DEBUG, tag.data()); } -} // namespace android
\ No newline at end of file +} // namespace android diff --git a/libs/input/InputEventLabels.cpp b/libs/input/InputEventLabels.cpp index f99a7d640e..bade68629d 100644 --- a/libs/input/InputEventLabels.cpp +++ b/libs/input/InputEventLabels.cpp @@ -404,7 +404,8 @@ namespace android { DEFINE_AXIS(GESTURE_Y_OFFSET), \ DEFINE_AXIS(GESTURE_SCROLL_X_DISTANCE), \ DEFINE_AXIS(GESTURE_SCROLL_Y_DISTANCE), \ - DEFINE_AXIS(GESTURE_PINCH_SCALE_FACTOR) + DEFINE_AXIS(GESTURE_PINCH_SCALE_FACTOR), \ + DEFINE_AXIS(GESTURE_SWIPE_FINGER_COUNT) // NOTE: If you add new LEDs here, you must also add them to Input.h #define LEDS_SEQUENCE \ @@ -433,17 +434,14 @@ namespace android { // clang-format on // --- InputEventLookup --- -const std::unordered_map<std::string, int> InputEventLookup::KEYCODES = {KEYCODES_SEQUENCE}; -const std::vector<InputEventLabel> InputEventLookup::KEY_NAMES = {KEYCODES_SEQUENCE}; - -const std::unordered_map<std::string, int> InputEventLookup::AXES = {AXES_SEQUENCE}; - -const std::vector<InputEventLabel> InputEventLookup::AXES_NAMES = {AXES_SEQUENCE}; - -const std::unordered_map<std::string, int> InputEventLookup::LEDS = {LEDS_SEQUENCE}; - -const std::unordered_map<std::string, int> InputEventLookup::FLAGS = {FLAGS_SEQUENCE}; +InputEventLookup::InputEventLookup() + : KEYCODES({KEYCODES_SEQUENCE}), + KEY_NAMES({KEYCODES_SEQUENCE}), + AXES({AXES_SEQUENCE}), + AXES_NAMES({AXES_SEQUENCE}), + LEDS({LEDS_SEQUENCE}), + FLAGS({FLAGS_SEQUENCE}) {} std::optional<int> InputEventLookup::lookupValueByLabel( const std::unordered_map<std::string, int>& map, const char* literal) { @@ -461,30 +459,36 @@ const char* InputEventLookup::lookupLabelByValue(const std::vector<InputEventLab } std::optional<int> InputEventLookup::getKeyCodeByLabel(const char* label) { - return lookupValueByLabel(KEYCODES, label); + const auto& self = get(); + return self.lookupValueByLabel(self.KEYCODES, label); } const char* InputEventLookup::getLabelByKeyCode(int32_t keyCode) { - if (keyCode >= 0 && static_cast<size_t>(keyCode) < KEYCODES.size()) { - return lookupLabelByValue(KEY_NAMES, keyCode); + const auto& self = get(); + if (keyCode >= 0 && static_cast<size_t>(keyCode) < self.KEYCODES.size()) { + return get().lookupLabelByValue(self.KEY_NAMES, keyCode); } return nullptr; } std::optional<int> InputEventLookup::getKeyFlagByLabel(const char* label) { - return lookupValueByLabel(FLAGS, label); + const auto& self = get(); + return lookupValueByLabel(self.FLAGS, label); } std::optional<int> InputEventLookup::getAxisByLabel(const char* label) { - return lookupValueByLabel(AXES, label); + const auto& self = get(); + return lookupValueByLabel(self.AXES, label); } const char* InputEventLookup::getAxisLabel(int32_t axisId) { - return lookupLabelByValue(AXES_NAMES, axisId); + const auto& self = get(); + return lookupLabelByValue(self.AXES_NAMES, axisId); } std::optional<int> InputEventLookup::getLedByLabel(const char* label) { - return lookupValueByLabel(LEDS, label); + const auto& self = get(); + return lookupValueByLabel(self.LEDS, label); } namespace { diff --git a/libs/input/InputTransport.cpp b/libs/input/InputTransport.cpp index f6b4648d67..4d3d8bc31c 100644 --- a/libs/input/InputTransport.cpp +++ b/libs/input/InputTransport.cpp @@ -4,6 +4,7 @@ // Provides a shared memory transport for input events. // #define LOG_TAG "InputTransport" +#define ATRACE_TAG ATRACE_TAG_INPUT #include <errno.h> #include <fcntl.h> @@ -13,6 +14,7 @@ #include <sys/types.h> #include <unistd.h> +#include <android-base/logging.h> #include <android-base/properties.h> #include <android-base/stringprintf.h> #include <binder/Parcel.h> @@ -80,6 +82,7 @@ const bool DEBUG_RESAMPLING = } // namespace +using android::base::Result; using android::base::StringPrintf; namespace android { @@ -449,6 +452,13 @@ status_t InputChannel::sendMessage(const InputMessage* msg) { ALOGD_IF(DEBUG_CHANNEL_MESSAGES, "channel '%s' ~ sent message of type %s", mName.c_str(), ftl::enum_string(msg->header.type).c_str()); + + if (ATRACE_ENABLED()) { + std::string message = + StringPrintf("sendMessage(inputChannel=%s, seq=0x%" PRIx32 ", type=0x%" PRIx32 ")", + mName.c_str(), msg->header.seq, msg->header.type); + ATRACE_NAME(message.c_str()); + } return OK; } @@ -484,6 +494,13 @@ status_t InputChannel::receiveMessage(InputMessage* msg) { ALOGD_IF(DEBUG_CHANNEL_MESSAGES, "channel '%s' ~ received message of type %s", mName.c_str(), ftl::enum_string(msg->header.type).c_str()); + + if (ATRACE_ENABLED()) { + std::string message = StringPrintf("receiveMessage(inputChannel=%s, seq=0x%" PRIx32 + ", type=0x%" PRIx32 ")", + mName.c_str(), msg->header.seq, msg->header.type); + ATRACE_NAME(message.c_str()); + } return OK; } @@ -606,8 +623,12 @@ status_t InputPublisher::publishMotionEvent( ATRACE_NAME(message.c_str()); } if (verifyEvents()) { - mInputVerifier.processMovement(deviceId, action, pointerCount, pointerProperties, - pointerCoords, flags); + Result<void> result = + mInputVerifier.processMovement(deviceId, action, pointerCount, pointerProperties, + pointerCoords, flags); + if (!result.ok()) { + LOG(FATAL) << "Bad stream: " << result.error(); + } } if (debugTransportPublisher()) { std::string transformString; diff --git a/libs/input/InputVerifier.cpp b/libs/input/InputVerifier.cpp index eb758045cc..9745e89234 100644 --- a/libs/input/InputVerifier.cpp +++ b/libs/input/InputVerifier.cpp @@ -18,111 +18,35 @@ #include <android-base/logging.h> #include <input/InputVerifier.h> +#include "input_verifier.rs.h" -namespace android { +using android::base::Error; +using android::base::Result; +using android::input::RustPointerProperties; -/** - * Log all of the movements that are sent to this verifier. Helps to identify the streams that lead - * to inconsistent events. - * Enable this via "adb shell setprop log.tag.InputVerifierLogEvents DEBUG" - */ -static bool logEvents() { - return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "LogEvents", ANDROID_LOG_INFO); -} +namespace android { // --- InputVerifier --- -InputVerifier::InputVerifier(const std::string& name) : mName(name){}; +InputVerifier::InputVerifier(const std::string& name) + : mVerifier(android::input::verifier::create(rust::String::lossy(name))){}; -void InputVerifier::processMovement(int32_t deviceId, int32_t action, uint32_t pointerCount, - const PointerProperties* pointerProperties, - const PointerCoords* pointerCoords, int32_t flags) { - if (logEvents()) { - LOG(ERROR) << "Processing " << MotionEvent::actionToString(action) << " for device " - << deviceId << " (" << pointerCount << " pointer" - << (pointerCount == 1 ? "" : "s") << ") on " << mName; +Result<void> InputVerifier::processMovement(int32_t deviceId, int32_t action, uint32_t pointerCount, + const PointerProperties* pointerProperties, + const PointerCoords* pointerCoords, int32_t flags) { + std::vector<RustPointerProperties> rpp; + for (size_t i = 0; i < pointerCount; i++) { + rpp.emplace_back(RustPointerProperties{.id = pointerProperties[i].id}); } - - switch (MotionEvent::getActionMasked(action)) { - case AMOTION_EVENT_ACTION_DOWN: { - auto [it, inserted] = mTouchingPointerIdsByDevice.insert({deviceId, {}}); - if (!inserted) { - LOG(FATAL) << "Got ACTION_DOWN, but already have touching pointers " << it->second - << " for device " << deviceId << " on " << mName; - } - it->second.set(pointerProperties[0].id); - break; - } - case AMOTION_EVENT_ACTION_POINTER_DOWN: { - auto it = mTouchingPointerIdsByDevice.find(deviceId); - if (it == mTouchingPointerIdsByDevice.end()) { - LOG(FATAL) << "Got POINTER_DOWN, but no touching pointers for device " << deviceId - << " on " << mName; - } - it->second.set(pointerProperties[MotionEvent::getActionIndex(action)].id); - break; - } - case AMOTION_EVENT_ACTION_MOVE: { - ensureTouchingPointersMatch(deviceId, pointerCount, pointerProperties, "MOVE"); - break; - } - case AMOTION_EVENT_ACTION_POINTER_UP: { - auto it = mTouchingPointerIdsByDevice.find(deviceId); - if (it == mTouchingPointerIdsByDevice.end()) { - LOG(FATAL) << "Got POINTER_UP, but no touching pointers for device " << deviceId - << " on " << mName; - } - it->second.reset(pointerProperties[MotionEvent::getActionIndex(action)].id); - break; - } - case AMOTION_EVENT_ACTION_UP: { - auto it = mTouchingPointerIdsByDevice.find(deviceId); - if (it == mTouchingPointerIdsByDevice.end()) { - LOG(FATAL) << "Got ACTION_UP, but no record for deviceId " << deviceId << " on " - << mName; - } - const auto& [_, touchingPointerIds] = *it; - if (touchingPointerIds.count() != 1) { - LOG(FATAL) << "Got ACTION_UP, but we have pointers: " << touchingPointerIds - << " for deviceId " << deviceId << " on " << mName; - } - const int32_t pointerId = pointerProperties[0].id; - if (!touchingPointerIds.test(pointerId)) { - LOG(FATAL) << "Got ACTION_UP, but pointerId " << pointerId - << " is not touching. Touching pointers: " << touchingPointerIds - << " for deviceId " << deviceId << " on " << mName; - } - mTouchingPointerIdsByDevice.erase(it); - break; - } - case AMOTION_EVENT_ACTION_CANCEL: { - if ((flags & AMOTION_EVENT_FLAG_CANCELED) != AMOTION_EVENT_FLAG_CANCELED) { - LOG(FATAL) << "For ACTION_CANCEL, must set FLAG_CANCELED"; - } - ensureTouchingPointersMatch(deviceId, pointerCount, pointerProperties, "CANCEL"); - mTouchingPointerIdsByDevice.erase(deviceId); - break; - } + rust::Slice<const RustPointerProperties> properties{rpp.data(), rpp.size()}; + rust::String errorMessage = + android::input::verifier::process_movement(*mVerifier, deviceId, action, properties, + flags); + if (errorMessage.empty()) { + return {}; + } else { + return Error() << errorMessage; } } -void InputVerifier::ensureTouchingPointersMatch(int32_t deviceId, uint32_t pointerCount, - const PointerProperties* pointerProperties, - const char* action) const { - auto it = mTouchingPointerIdsByDevice.find(deviceId); - if (it == mTouchingPointerIdsByDevice.end()) { - LOG(FATAL) << "Got " << action << ", but no touching pointers for device " << deviceId - << " on " << mName; - } - const auto& [_, touchingPointerIds] = *it; - for (size_t i = 0; i < pointerCount; i++) { - const int32_t pointerId = pointerProperties[i].id; - if (!touchingPointerIds.test(pointerId)) { - LOG(FATAL) << "Got " << action << " for pointerId " << pointerId - << " but the touching pointers are " << touchingPointerIds << " on " - << mName; - } - } -}; - } // namespace android diff --git a/libs/input/InputWrapper.hpp b/libs/input/InputWrapper.hpp new file mode 100644 index 0000000000..a01080d319 --- /dev/null +++ b/libs/input/InputWrapper.hpp @@ -0,0 +1,18 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <android/input.h> +#include "input/Input.h" diff --git a/libs/input/MotionPredictor.cpp b/libs/input/MotionPredictor.cpp index abcca345d3..5736ad7eed 100644 --- a/libs/input/MotionPredictor.cpp +++ b/libs/input/MotionPredictor.cpp @@ -36,9 +36,6 @@ namespace android { namespace { -const int64_t PREDICTION_INTERVAL_NANOS = - 12500000 / 3; // TODO(b/266747937): Get this from the model. - /** * Log debug messages about predictions. * Enable this via "adb shell setprop log.tag.MotionPredictor DEBUG" @@ -70,7 +67,7 @@ MotionPredictor::MotionPredictor(nsecs_t predictionTimestampOffsetNanos, android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { if (mLastEvent && mLastEvent->getDeviceId() != event.getDeviceId()) { // We still have an active gesture for another device. The provided MotionEvent is not - // consistent the previous gesture. + // consistent with the previous gesture. LOG(ERROR) << "Inconsistent event stream: last event is " << *mLastEvent << ", but " << __func__ << " is called with " << event; return android::base::Error() @@ -86,9 +83,10 @@ android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { // Initialise the model now that it's likely to be used. if (!mModel) { mModel = TfLiteMotionPredictorModel::create(); + LOG_ALWAYS_FATAL_IF(!mModel); } - if (mBuffers == nullptr) { + if (!mBuffers) { mBuffers = std::make_unique<TfLiteMotionPredictorBuffers>(mModel->inputLength()); } @@ -136,6 +134,13 @@ android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { mLastEvent = MotionEvent(); } mLastEvent->copyFrom(&event, /*keepHistory=*/false); + + // Pass input event to the MetricsManager. + if (!mMetricsManager) { + mMetricsManager.emplace(mModel->config().predictionInterval, mModel->outputLength()); + } + mMetricsManager->onRecord(event); + return {}; } @@ -178,19 +183,30 @@ std::unique_ptr<MotionEvent> MotionPredictor::predict(nsecs_t timestamp) { for (size_t i = 0; i < static_cast<size_t>(predictedR.size()) && predictionTime <= futureTime; ++i) { - const TfLiteMotionPredictorSample::Point point = - convertPrediction(axisFrom, axisTo, predictedR[i], predictedPhi[i]); + if (predictedR[i] < mModel->config().distanceNoiseFloor) { + // Stop predicting when the predicted output is below the model's noise floor. + // + // We assume that all subsequent predictions in the batch are unreliable because later + // predictions are conditional on earlier predictions, and a state of noise is not a + // good basis for prediction. + // + // The UX trade-off is that this potentially sacrifices some predictions when the input + // device starts to speed up, but avoids producing noisy predictions as it slows down. + break; + } // TODO(b/266747654): Stop predictions if confidence is < some threshold. - ALOGD_IF(isDebug(), "prediction %zu: %f, %f", i, point.x, point.y); + const TfLiteMotionPredictorSample::Point predictedPoint = + convertPrediction(axisFrom, axisTo, predictedR[i], predictedPhi[i]); + + ALOGD_IF(isDebug(), "prediction %zu: %f, %f", i, predictedPoint.x, predictedPoint.y); PointerCoords coords; coords.clear(); - coords.setAxisValue(AMOTION_EVENT_AXIS_X, point.x); - coords.setAxisValue(AMOTION_EVENT_AXIS_Y, point.y); - // TODO(b/266747654): Stop predictions if predicted pressure is < some threshold. + coords.setAxisValue(AMOTION_EVENT_AXIS_X, predictedPoint.x); + coords.setAxisValue(AMOTION_EVENT_AXIS_Y, predictedPoint.y); coords.setAxisValue(AMOTION_EVENT_AXIS_PRESSURE, predictedPressure[i]); - predictionTime += PREDICTION_INTERVAL_NANOS; + predictionTime += mModel->config().predictionInterval; if (i == 0) { hasPredictions = true; prediction->initialize(InputEvent::nextId(), event.getDeviceId(), event.getSource(), @@ -207,12 +223,17 @@ std::unique_ptr<MotionEvent> MotionPredictor::predict(nsecs_t timestamp) { } axisFrom = axisTo; - axisTo = point; + axisTo = predictedPoint; } - // TODO(b/266747511): Interpolate to futureTime? + if (!hasPredictions) { return nullptr; } + + // Pass predictions to the MetricsManager. + LOG_ALWAYS_FATAL_IF(!mMetricsManager); + mMetricsManager->onPredict(*prediction); + return prediction; } diff --git a/libs/input/MotionPredictorMetricsManager.cpp b/libs/input/MotionPredictorMetricsManager.cpp new file mode 100644 index 0000000000..67b103290f --- /dev/null +++ b/libs/input/MotionPredictorMetricsManager.cpp @@ -0,0 +1,373 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "MotionPredictorMetricsManager" + +#include <input/MotionPredictorMetricsManager.h> + +#include <algorithm> + +#include <android-base/logging.h> + +#include "Eigen/Core" +#include "Eigen/Geometry" + +#ifdef __ANDROID__ +#include <statslog_libinput.h> +#endif + +namespace android { +namespace { + +inline constexpr int NANOS_PER_SECOND = 1'000'000'000; // nanoseconds per second +inline constexpr int NANOS_PER_MILLIS = 1'000'000; // nanoseconds per millisecond + +// Velocity threshold at which we report "high-velocity" metrics, in pixels per second. +// This value was selected from manual experimentation, as a threshold that separates "fast" +// (semi-sloppy) handwriting from more careful medium to slow handwriting. +inline constexpr float HIGH_VELOCITY_THRESHOLD = 1100.0; + +// Small value to add to the path length when computing scale-invariant error to avoid division by +// zero. +inline constexpr float PATH_LENGTH_EPSILON = 0.001; + +} // namespace + +MotionPredictorMetricsManager::MotionPredictorMetricsManager(nsecs_t predictionInterval, + size_t maxNumPredictions) + : mPredictionInterval(predictionInterval), + mMaxNumPredictions(maxNumPredictions), + mRecentGroundTruthPoints(maxNumPredictions + 1), + mAggregatedMetrics(maxNumPredictions), + mAtomFields(maxNumPredictions) {} + +void MotionPredictorMetricsManager::onRecord(const MotionEvent& inputEvent) { + // Convert MotionEvent to GroundTruthPoint. + const PointerCoords* coords = inputEvent.getRawPointerCoords(/*pointerIndex=*/0); + LOG_ALWAYS_FATAL_IF(coords == nullptr); + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f{coords->getY(), + coords->getX()}, + .pressure = + inputEvent.getPressure(/*pointerIndex=*/0)}, + .timestamp = inputEvent.getEventTime()}; + + // Handle event based on action type. + switch (inputEvent.getActionMasked()) { + case AMOTION_EVENT_ACTION_DOWN: { + clearStrokeData(); + incorporateNewGroundTruth(groundTruthPoint); + break; + } + case AMOTION_EVENT_ACTION_MOVE: { + incorporateNewGroundTruth(groundTruthPoint); + break; + } + case AMOTION_EVENT_ACTION_UP: + case AMOTION_EVENT_ACTION_CANCEL: { + // Only expect meaningful predictions when given at least two input points. + if (mRecentGroundTruthPoints.size() >= 2) { + computeAtomFields(); + reportMetrics(); + break; + } + } + } +} + +// Adds new predictions to mRecentPredictions and maintains the invariant that elements are +// sorted in ascending order of targetTimestamp. +void MotionPredictorMetricsManager::onPredict(const MotionEvent& predictionEvent) { + for (size_t i = 0; i < predictionEvent.getHistorySize() + 1; ++i) { + // Convert MotionEvent to PredictionPoint. + const PointerCoords* coords = + predictionEvent.getHistoricalRawPointerCoords(/*pointerIndex=*/0, i); + LOG_ALWAYS_FATAL_IF(coords == nullptr); + const nsecs_t targetTimestamp = predictionEvent.getHistoricalEventTime(i); + mRecentPredictions.push_back( + PredictionPoint{{.position = Eigen::Vector2f{coords->getY(), coords->getX()}, + .pressure = + predictionEvent.getHistoricalPressure(/*pointerIndex=*/0, + i)}, + .originTimestamp = mRecentGroundTruthPoints.back().timestamp, + .targetTimestamp = targetTimestamp}); + } + + std::sort(mRecentPredictions.begin(), mRecentPredictions.end()); +} + +void MotionPredictorMetricsManager::clearStrokeData() { + mRecentGroundTruthPoints.clear(); + mRecentPredictions.clear(); + std::fill(mAggregatedMetrics.begin(), mAggregatedMetrics.end(), AggregatedStrokeMetrics{}); + std::fill(mAtomFields.begin(), mAtomFields.end(), AtomFields{}); +} + +void MotionPredictorMetricsManager::incorporateNewGroundTruth( + const GroundTruthPoint& groundTruthPoint) { + // Note: this removes the oldest point if `mRecentGroundTruthPoints` is already at capacity. + mRecentGroundTruthPoints.pushBack(groundTruthPoint); + + // Remove outdated predictions – those that can never be matched with the current or any future + // ground truth points. We use fuzzy association for the timestamps here, because ground truth + // and prediction timestamps may not be perfectly synchronized. + const nsecs_t fuzzy_association_time_delta = mPredictionInterval / 4; + const auto firstCurrentIt = + std::find_if(mRecentPredictions.begin(), mRecentPredictions.end(), + [&groundTruthPoint, + fuzzy_association_time_delta](const PredictionPoint& prediction) { + return prediction.targetTimestamp > + groundTruthPoint.timestamp - fuzzy_association_time_delta; + }); + mRecentPredictions.erase(mRecentPredictions.begin(), firstCurrentIt); + + // Fuzzily match the new ground truth's timestamp to recent predictions' targetTimestamp and + // update the corresponding metrics. + for (const PredictionPoint& prediction : mRecentPredictions) { + if ((prediction.targetTimestamp > + groundTruthPoint.timestamp - fuzzy_association_time_delta) && + (prediction.targetTimestamp < + groundTruthPoint.timestamp + fuzzy_association_time_delta)) { + updateAggregatedMetrics(prediction); + } + } +} + +void MotionPredictorMetricsManager::updateAggregatedMetrics( + const PredictionPoint& predictionPoint) { + if (mRecentGroundTruthPoints.size() < 2) { + return; + } + + const GroundTruthPoint& latestGroundTruthPoint = mRecentGroundTruthPoints.back(); + const GroundTruthPoint& previousGroundTruthPoint = + mRecentGroundTruthPoints[mRecentGroundTruthPoints.size() - 2]; + // Calculate prediction error vector. + const Eigen::Vector2f groundTruthTrajectory = + latestGroundTruthPoint.position - previousGroundTruthPoint.position; + const Eigen::Vector2f predictionTrajectory = + predictionPoint.position - previousGroundTruthPoint.position; + const Eigen::Vector2f predictionError = predictionTrajectory - groundTruthTrajectory; + + // By default, prediction error counts fully as both off-trajectory and along-trajectory error. + // This serves as the fallback when the two most recent ground truth points are equal. + const float predictionErrorNorm = predictionError.norm(); + float alongTrajectoryError = predictionErrorNorm; + float offTrajectoryError = predictionErrorNorm; + if (groundTruthTrajectory.squaredNorm() > 0) { + // Rotate the prediction error vector by the angle of the ground truth trajectory vector. + // This yields a vector whose first component is the along-trajectory error and whose + // second component is the off-trajectory error. + const float theta = std::atan2(groundTruthTrajectory[1], groundTruthTrajectory[0]); + const Eigen::Vector2f rotatedPredictionError = Eigen::Rotation2Df(-theta) * predictionError; + alongTrajectoryError = rotatedPredictionError[0]; + offTrajectoryError = rotatedPredictionError[1]; + } + + // Compute the multiple of mPredictionInterval nearest to the amount of time into the + // future being predicted. This serves as the time bucket index into mAggregatedMetrics. + const float timestampDeltaFloat = + static_cast<float>(predictionPoint.targetTimestamp - predictionPoint.originTimestamp); + const size_t tIndex = + static_cast<size_t>(std::round(timestampDeltaFloat / mPredictionInterval - 1)); + + // Aggregate values into "general errors". + mAggregatedMetrics[tIndex].alongTrajectoryErrorSum += alongTrajectoryError; + mAggregatedMetrics[tIndex].alongTrajectorySumSquaredErrors += + alongTrajectoryError * alongTrajectoryError; + mAggregatedMetrics[tIndex].offTrajectorySumSquaredErrors += + offTrajectoryError * offTrajectoryError; + const float pressureError = predictionPoint.pressure - latestGroundTruthPoint.pressure; + mAggregatedMetrics[tIndex].pressureSumSquaredErrors += pressureError * pressureError; + ++mAggregatedMetrics[tIndex].generalErrorsCount; + + // Aggregate values into high-velocity metrics, if we are in one of the last two time buckets + // and the velocity is above the threshold. Velocity here is measured in pixels per second. + const float velocity = groundTruthTrajectory.norm() / + (static_cast<float>(latestGroundTruthPoint.timestamp - + previousGroundTruthPoint.timestamp) / + NANOS_PER_SECOND); + if ((tIndex + 2 >= mMaxNumPredictions) && (velocity > HIGH_VELOCITY_THRESHOLD)) { + mAggregatedMetrics[tIndex].highVelocityAlongTrajectorySse += + alongTrajectoryError * alongTrajectoryError; + mAggregatedMetrics[tIndex].highVelocityOffTrajectorySse += + offTrajectoryError * offTrajectoryError; + ++mAggregatedMetrics[tIndex].highVelocityErrorsCount; + } + + // Compute path length for scale-invariant errors. + float pathLength = 0; + for (size_t i = 1; i < mRecentGroundTruthPoints.size(); ++i) { + pathLength += + (mRecentGroundTruthPoints[i].position - mRecentGroundTruthPoints[i - 1].position) + .norm(); + } + // Avoid overweighting errors at the beginning of a stroke: compute the path length as if there + // were a full ground truth history by filling in missing segments with the average length. + // Note: the "- 1" is needed to translate from number of endpoints to number of segments. + pathLength *= static_cast<float>(mRecentGroundTruthPoints.capacity() - 1) / + (mRecentGroundTruthPoints.size() - 1); + pathLength += PATH_LENGTH_EPSILON; // Ensure path length is nonzero (>= PATH_LENGTH_EPSILON). + + // Compute and aggregate scale-invariant errors. + const float scaleInvariantAlongTrajectoryError = alongTrajectoryError / pathLength; + const float scaleInvariantOffTrajectoryError = offTrajectoryError / pathLength; + mAggregatedMetrics[tIndex].scaleInvariantAlongTrajectorySse += + scaleInvariantAlongTrajectoryError * scaleInvariantAlongTrajectoryError; + mAggregatedMetrics[tIndex].scaleInvariantOffTrajectorySse += + scaleInvariantOffTrajectoryError * scaleInvariantOffTrajectoryError; + ++mAggregatedMetrics[tIndex].scaleInvariantErrorsCount; +} + +void MotionPredictorMetricsManager::computeAtomFields() { + for (size_t i = 0; i < mAggregatedMetrics.size(); ++i) { + if (mAggregatedMetrics[i].generalErrorsCount == 0) { + // We have not received data corresponding to metrics for this time bucket. + continue; + } + + mAtomFields[i].deltaTimeBucketMilliseconds = + static_cast<int>(mPredictionInterval / NANOS_PER_MILLIS * (i + 1)); + + // Note: we need the "* 1000"s below because we report values in integral milli-units. + + { // General errors: reported for every time bucket. + const float alongTrajectoryErrorMean = mAggregatedMetrics[i].alongTrajectoryErrorSum / + mAggregatedMetrics[i].generalErrorsCount; + mAtomFields[i].alongTrajectoryErrorMeanMillipixels = + static_cast<int>(alongTrajectoryErrorMean * 1000); + + const float alongTrajectoryMse = mAggregatedMetrics[i].alongTrajectorySumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount; + // Take the max with 0 to avoid negative values caused by numerical instability. + const float alongTrajectoryErrorVariance = + std::max(0.0f, + alongTrajectoryMse - + alongTrajectoryErrorMean * alongTrajectoryErrorMean); + const float alongTrajectoryErrorStd = std::sqrt(alongTrajectoryErrorVariance); + mAtomFields[i].alongTrajectoryErrorStdMillipixels = + static_cast<int>(alongTrajectoryErrorStd * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].offTrajectorySumSquaredErrors < 0, + "mAggregatedMetrics[%zu].offTrajectorySumSquaredErrors = %f should " + "not be negative", + i, mAggregatedMetrics[i].offTrajectorySumSquaredErrors); + const float offTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].offTrajectorySumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount); + mAtomFields[i].offTrajectoryRmseMillipixels = + static_cast<int>(offTrajectoryRmse * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].pressureSumSquaredErrors < 0, + "mAggregatedMetrics[%zu].pressureSumSquaredErrors = %f should not " + "be negative", + i, mAggregatedMetrics[i].pressureSumSquaredErrors); + const float pressureRmse = std::sqrt(mAggregatedMetrics[i].pressureSumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount); + mAtomFields[i].pressureRmseMilliunits = static_cast<int>(pressureRmse * 1000); + } + + // High-velocity errors: reported only for last two time buckets. + // Check if we are in one of the last two time buckets, and there is high-velocity data. + if ((i + 2 >= mMaxNumPredictions) && (mAggregatedMetrics[i].highVelocityErrorsCount > 0)) { + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].highVelocityAlongTrajectorySse < 0, + "mAggregatedMetrics[%zu].highVelocityAlongTrajectorySse = %f " + "should not be negative", + i, mAggregatedMetrics[i].highVelocityAlongTrajectorySse); + const float alongTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].highVelocityAlongTrajectorySse / + mAggregatedMetrics[i].highVelocityErrorsCount); + mAtomFields[i].highVelocityAlongTrajectoryRmse = + static_cast<int>(alongTrajectoryRmse * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].highVelocityOffTrajectorySse < 0, + "mAggregatedMetrics[%zu].highVelocityOffTrajectorySse = %f should " + "not be negative", + i, mAggregatedMetrics[i].highVelocityOffTrajectorySse); + const float offTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].highVelocityOffTrajectorySse / + mAggregatedMetrics[i].highVelocityErrorsCount); + mAtomFields[i].highVelocityOffTrajectoryRmse = + static_cast<int>(offTrajectoryRmse * 1000); + } + + // Scale-invariant errors: reported only for the last time bucket, where the values + // represent an average across all time buckets. + if (i + 1 == mMaxNumPredictions) { + // Compute error averages. + float alongTrajectoryRmseSum = 0; + float offTrajectoryRmseSum = 0; + for (size_t j = 0; j < mAggregatedMetrics.size(); ++j) { + // If we have general errors (checked above), we should always also have + // scale-invariant errors. + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantErrorsCount == 0, + "mAggregatedMetrics[%zu].scaleInvariantErrorsCount is 0", j); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse < 0, + "mAggregatedMetrics[%zu].scaleInvariantAlongTrajectorySse = %f " + "should not be negative", + j, mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse); + alongTrajectoryRmseSum += + std::sqrt(mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse / + mAggregatedMetrics[j].scaleInvariantErrorsCount); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantOffTrajectorySse < 0, + "mAggregatedMetrics[%zu].scaleInvariantOffTrajectorySse = %f " + "should not be negative", + j, mAggregatedMetrics[j].scaleInvariantOffTrajectorySse); + offTrajectoryRmseSum += + std::sqrt(mAggregatedMetrics[j].scaleInvariantOffTrajectorySse / + mAggregatedMetrics[j].scaleInvariantErrorsCount); + } + + const float averageAlongTrajectoryRmse = + alongTrajectoryRmseSum / mAggregatedMetrics.size(); + mAtomFields.back().scaleInvariantAlongTrajectoryRmse = + static_cast<int>(averageAlongTrajectoryRmse * 1000); + + const float averageOffTrajectoryRmse = offTrajectoryRmseSum / mAggregatedMetrics.size(); + mAtomFields.back().scaleInvariantOffTrajectoryRmse = + static_cast<int>(averageOffTrajectoryRmse * 1000); + } + } +} + +void MotionPredictorMetricsManager::reportMetrics() { + // Report one atom for each time bucket. + for (size_t i = 0; i < mAtomFields.size(); ++i) { + // Call stats_write logging function only on Android targets (not supported on host). +#ifdef __ANDROID__ + android::stats::libinput:: + stats_write(android::stats::libinput::STYLUS_PREDICTION_METRICS_REPORTED, + /*stylus_vendor_id=*/0, + /*stylus_product_id=*/0, mAtomFields[i].deltaTimeBucketMilliseconds, + mAtomFields[i].alongTrajectoryErrorMeanMillipixels, + mAtomFields[i].alongTrajectoryErrorStdMillipixels, + mAtomFields[i].offTrajectoryRmseMillipixels, + mAtomFields[i].pressureRmseMilliunits, + mAtomFields[i].highVelocityAlongTrajectoryRmse, + mAtomFields[i].highVelocityOffTrajectoryRmse, + mAtomFields[i].scaleInvariantAlongTrajectoryRmse, + mAtomFields[i].scaleInvariantOffTrajectoryRmse); +#endif + } + + // Set mock atom fields, if available. + if (mMockLoggedAtomFields != nullptr) { + *mMockLoggedAtomFields = mAtomFields; + } +} + +} // namespace android diff --git a/libs/input/TfLiteMotionPredictor.cpp b/libs/input/TfLiteMotionPredictor.cpp index 8d10ff56b0..d17476e216 100644 --- a/libs/input/TfLiteMotionPredictor.cpp +++ b/libs/input/TfLiteMotionPredictor.cpp @@ -36,6 +36,7 @@ #define ATRACE_TAG ATRACE_TAG_INPUT #include <cutils/trace.h> #include <log/log.h> +#include <utils/Timers.h> #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/core/api/op_resolver.h" @@ -44,6 +45,8 @@ #include "tensorflow/lite/model.h" #include "tensorflow/lite/mutable_op_resolver.h" +#include "tinyxml2.h" + namespace android { namespace { @@ -72,16 +75,41 @@ bool fileExists(const char* filename) { std::string getModelPath() { #if defined(__ANDROID__) - static const char* oemModel = "/vendor/etc/motion_predictor_model.fb"; + static const char* oemModel = "/vendor/etc/motion_predictor_model.tflite"; if (fileExists(oemModel)) { return oemModel; } - return "/system/etc/motion_predictor_model.fb"; + return "/system/etc/motion_predictor_model.tflite"; #else - return base::GetExecutableDirectory() + "/motion_predictor_model.fb"; + return base::GetExecutableDirectory() + "/motion_predictor_model.tflite"; #endif } +std::string getConfigPath() { + // The config file should be alongside the model file. + return base::Dirname(getModelPath()) + "/motion_predictor_config.xml"; +} + +int64_t parseXMLInt64(const tinyxml2::XMLElement& configRoot, const char* elementName) { + const tinyxml2::XMLElement* element = configRoot.FirstChildElement(elementName); + LOG_ALWAYS_FATAL_IF(!element, "Could not find '%s' element", elementName); + + int64_t value = 0; + LOG_ALWAYS_FATAL_IF(element->QueryInt64Text(&value) != tinyxml2::XML_SUCCESS, + "Failed to parse %s: %s", elementName, element->GetText()); + return value; +} + +float parseXMLFloat(const tinyxml2::XMLElement& configRoot, const char* elementName) { + const tinyxml2::XMLElement* element = configRoot.FirstChildElement(elementName); + LOG_ALWAYS_FATAL_IF(!element, "Could not find '%s' element", elementName); + + float value = 0; + LOG_ALWAYS_FATAL_IF(element->QueryFloatText(&value) != tinyxml2::XML_SUCCESS, + "Failed to parse %s: %s", elementName, element->GetText()); + return value; +} + // A TFLite ErrorReporter that logs to logcat. class LoggingErrorReporter : public tflite::ErrorReporter { public: @@ -133,6 +161,7 @@ std::unique_ptr<tflite::OpResolver> createOpResolver() { ::tflite::ops::builtin::Register_CONCATENATION()); resolver->AddBuiltin(::tflite::BuiltinOperator_FULLY_CONNECTED, ::tflite::ops::builtin::Register_FULLY_CONNECTED()); + resolver->AddBuiltin(::tflite::BuiltinOperator_GELU, ::tflite::ops::builtin::Register_GELU()); return resolver; } @@ -189,13 +218,7 @@ void TfLiteMotionPredictorBuffers::pushSample(int64_t timestamp, float phi = 0; float orientation = 0; - // Ignore the sample if there is no movement. These samples can occur when there's change to a - // property other than the coordinates and pollute the input to the model. - if (r == 0) { - return; - } - - if (!mAxisFrom) { // Second point. + if (!mAxisFrom && r > 0) { // Second point. // We can only determine the distance from the first point, and not any // angle. However, if the second point forms an axis, the orientation can // be transformed relative to that axis. @@ -216,8 +239,10 @@ void TfLiteMotionPredictorBuffers::pushSample(int64_t timestamp, } // Update the axis for the next point. - mAxisFrom = mAxisTo; - mAxisTo = sample; + if (r > 0) { + mAxisFrom = mAxisTo; + mAxisTo = sample; + } // Push the current sample onto the end of the input buffers. mInputR.pushBack(r); @@ -245,13 +270,26 @@ std::unique_ptr<TfLiteMotionPredictorModel> TfLiteMotionPredictorModel::create() PLOG(FATAL) << "Failed to mmap model"; } + const std::string configPath = getConfigPath(); + tinyxml2::XMLDocument configDocument; + LOG_ALWAYS_FATAL_IF(configDocument.LoadFile(configPath.c_str()) != tinyxml2::XML_SUCCESS, + "Failed to load config file from %s", configPath.c_str()); + + // Parse configuration file. + const tinyxml2::XMLElement* configRoot = configDocument.FirstChildElement("motion-predictor"); + LOG_ALWAYS_FATAL_IF(!configRoot); + Config config{ + .predictionInterval = parseXMLInt64(*configRoot, "prediction-interval"), + .distanceNoiseFloor = parseXMLFloat(*configRoot, "distance-noise-floor"), + }; + return std::unique_ptr<TfLiteMotionPredictorModel>( - new TfLiteMotionPredictorModel(std::move(modelBuffer))); + new TfLiteMotionPredictorModel(std::move(modelBuffer), std::move(config))); } TfLiteMotionPredictorModel::TfLiteMotionPredictorModel( - std::unique_ptr<android::base::MappedFile> model) - : mFlatBuffer(std::move(model)) { + std::unique_ptr<android::base::MappedFile> model, Config config) + : mFlatBuffer(std::move(model)), mConfig(std::move(config)) { CHECK(mFlatBuffer); mErrorReporter = std::make_unique<LoggingErrorReporter>(); mModel = tflite::FlatBufferModel::VerifyAndBuildFromBuffer(mFlatBuffer->data(), diff --git a/libs/input/VelocityTracker.cpp b/libs/input/VelocityTracker.cpp index 8551e5fa1c..078109a5b6 100644 --- a/libs/input/VelocityTracker.cpp +++ b/libs/input/VelocityTracker.cpp @@ -16,7 +16,9 @@ #define LOG_TAG "VelocityTracker" +#include <android-base/logging.h> #include <array> +#include <ftl/enum.h> #include <inttypes.h> #include <limits.h> #include <math.h> @@ -145,27 +147,19 @@ static std::string matrixToString(const float* a, uint32_t m, uint32_t n, bool r VelocityTracker::VelocityTracker(const Strategy strategy) : mLastEventTime(0), mCurrentPointerIdBits(0), mOverrideStrategy(strategy) {} -VelocityTracker::~VelocityTracker() { -} - bool VelocityTracker::isAxisSupported(int32_t axis) { return DEFAULT_STRATEGY_BY_AXIS.find(axis) != DEFAULT_STRATEGY_BY_AXIS.end(); } void VelocityTracker::configureStrategy(int32_t axis) { const bool isDifferentialAxis = DIFFERENTIAL_AXES.find(axis) != DIFFERENTIAL_AXES.end(); - - std::unique_ptr<VelocityTrackerStrategy> createdStrategy; - if (mOverrideStrategy != VelocityTracker::Strategy::DEFAULT) { - createdStrategy = createStrategy(mOverrideStrategy, /*deltaValues=*/isDifferentialAxis); + if (isDifferentialAxis || mOverrideStrategy == VelocityTracker::Strategy::DEFAULT) { + // Do not allow overrides of strategies for differential axes, for now. + mConfiguredStrategies[axis] = createStrategy(DEFAULT_STRATEGY_BY_AXIS.at(axis), + /*deltaValues=*/isDifferentialAxis); } else { - createdStrategy = createStrategy(DEFAULT_STRATEGY_BY_AXIS.at(axis), - /*deltaValues=*/isDifferentialAxis); + mConfiguredStrategies[axis] = createStrategy(mOverrideStrategy, /*deltaValues=*/false); } - - LOG_ALWAYS_FATAL_IF(createdStrategy == nullptr, - "Could not create velocity tracker strategy for axis '%" PRId32 "'!", axis); - mConfiguredStrategies[axis] = std::move(createdStrategy); } std::unique_ptr<VelocityTrackerStrategy> VelocityTracker::createStrategy( @@ -213,6 +207,9 @@ std::unique_ptr<VelocityTrackerStrategy> VelocityTracker::createStrategy( default: break; } + LOG(FATAL) << "Invalid strategy: " << ftl::enum_string(strategy) + << ", deltaValues = " << deltaValues; + return nullptr; } diff --git a/libs/input/ffi/FromRustToCpp.h b/libs/input/ffi/FromRustToCpp.h new file mode 100644 index 0000000000..889945c32b --- /dev/null +++ b/libs/input/ffi/FromRustToCpp.h @@ -0,0 +1,23 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "rust/cxx.h" + +namespace android { + +bool shouldLog(rust::Str tag); + +} // namespace android diff --git a/libs/input/input_verifier.rs b/libs/input/input_verifier.rs new file mode 100644 index 0000000000..dd2ac4ca91 --- /dev/null +++ b/libs/input/input_verifier.rs @@ -0,0 +1,422 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Validate the incoming motion stream. +//! This class is not thread-safe. +//! State is stored in the "InputVerifier" object +//! that can be created via the 'create' method. +//! Usage: +//! Box<InputVerifier> verifier = create("inputChannel name"); +//! result = process_movement(verifier, ...); +//! if (result) { +//! crash(result.error_message()); +//! } + +use std::collections::HashMap; +use std::collections::HashSet; + +use bitflags::bitflags; +use log::info; + +#[cxx::bridge(namespace = "android::input")] +#[allow(unsafe_op_in_unsafe_fn)] +mod ffi { + #[namespace = "android"] + unsafe extern "C++" { + include!("ffi/FromRustToCpp.h"); + fn shouldLog(tag: &str) -> bool; + } + #[namespace = "android::input::verifier"] + extern "Rust" { + type InputVerifier; + + fn create(name: String) -> Box<InputVerifier>; + fn process_movement( + verifier: &mut InputVerifier, + device_id: i32, + action: u32, + pointer_properties: &[RustPointerProperties], + flags: i32, + ) -> String; + } + + pub struct RustPointerProperties { + id: i32, + } +} + +use crate::ffi::shouldLog; +use crate::ffi::RustPointerProperties; + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +struct DeviceId(i32); + +fn process_movement( + verifier: &mut InputVerifier, + device_id: i32, + action: u32, + pointer_properties: &[RustPointerProperties], + flags: i32, +) -> String { + let result = verifier.process_movement( + DeviceId(device_id), + action, + pointer_properties, + Flags::from_bits(flags).unwrap(), + ); + match result { + Ok(()) => "".to_string(), + Err(e) => e, + } +} + +fn create(name: String) -> Box<InputVerifier> { + Box::new(InputVerifier::new(&name)) +} + +#[repr(u32)] +enum MotionAction { + Down = input_bindgen::AMOTION_EVENT_ACTION_DOWN, + Up = input_bindgen::AMOTION_EVENT_ACTION_UP, + Move = input_bindgen::AMOTION_EVENT_ACTION_MOVE, + Cancel = input_bindgen::AMOTION_EVENT_ACTION_CANCEL, + Outside = input_bindgen::AMOTION_EVENT_ACTION_OUTSIDE, + PointerDown { action_index: usize } = input_bindgen::AMOTION_EVENT_ACTION_POINTER_DOWN, + PointerUp { action_index: usize } = input_bindgen::AMOTION_EVENT_ACTION_POINTER_UP, + HoverEnter = input_bindgen::AMOTION_EVENT_ACTION_HOVER_ENTER, + HoverMove = input_bindgen::AMOTION_EVENT_ACTION_HOVER_MOVE, + HoverExit = input_bindgen::AMOTION_EVENT_ACTION_HOVER_EXIT, + Scroll = input_bindgen::AMOTION_EVENT_ACTION_SCROLL, + ButtonPress = input_bindgen::AMOTION_EVENT_ACTION_BUTTON_PRESS, + ButtonRelease = input_bindgen::AMOTION_EVENT_ACTION_BUTTON_RELEASE, +} + +fn get_action_index(action: u32) -> usize { + let index = (action & input_bindgen::AMOTION_EVENT_ACTION_POINTER_INDEX_MASK) + >> input_bindgen::AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT; + index.try_into().unwrap() +} + +impl From<u32> for MotionAction { + fn from(action: u32) -> Self { + let action_masked = action & input_bindgen::AMOTION_EVENT_ACTION_MASK; + let action_index = get_action_index(action); + match action_masked { + input_bindgen::AMOTION_EVENT_ACTION_DOWN => MotionAction::Down, + input_bindgen::AMOTION_EVENT_ACTION_UP => MotionAction::Up, + input_bindgen::AMOTION_EVENT_ACTION_MOVE => MotionAction::Move, + input_bindgen::AMOTION_EVENT_ACTION_CANCEL => MotionAction::Cancel, + input_bindgen::AMOTION_EVENT_ACTION_OUTSIDE => MotionAction::Outside, + input_bindgen::AMOTION_EVENT_ACTION_POINTER_DOWN => { + MotionAction::PointerDown { action_index } + } + input_bindgen::AMOTION_EVENT_ACTION_POINTER_UP => { + MotionAction::PointerUp { action_index } + } + input_bindgen::AMOTION_EVENT_ACTION_HOVER_ENTER => MotionAction::HoverEnter, + input_bindgen::AMOTION_EVENT_ACTION_HOVER_MOVE => MotionAction::HoverMove, + input_bindgen::AMOTION_EVENT_ACTION_HOVER_EXIT => MotionAction::HoverExit, + input_bindgen::AMOTION_EVENT_ACTION_SCROLL => MotionAction::Scroll, + input_bindgen::AMOTION_EVENT_ACTION_BUTTON_PRESS => MotionAction::ButtonPress, + input_bindgen::AMOTION_EVENT_ACTION_BUTTON_RELEASE => MotionAction::ButtonRelease, + _ => panic!("Unknown action: {}", action), + } + } +} + +bitflags! { + struct Flags: i32 { + const CANCELED = input_bindgen::AMOTION_EVENT_FLAG_CANCELED; + } +} + +fn motion_action_to_string(action: u32) -> String { + match action.into() { + MotionAction::Down => "DOWN".to_string(), + MotionAction::Up => "UP".to_string(), + MotionAction::Move => "MOVE".to_string(), + MotionAction::Cancel => "CANCEL".to_string(), + MotionAction::Outside => "OUTSIDE".to_string(), + MotionAction::PointerDown { action_index } => { + format!("POINTER_DOWN({})", action_index) + } + MotionAction::PointerUp { action_index } => { + format!("POINTER_UP({})", action_index) + } + MotionAction::HoverMove => "HOVER_MOVE".to_string(), + MotionAction::Scroll => "SCROLL".to_string(), + MotionAction::HoverEnter => "HOVER_ENTER".to_string(), + MotionAction::HoverExit => "HOVER_EXIT".to_string(), + MotionAction::ButtonPress => "BUTTON_PRESS".to_string(), + MotionAction::ButtonRelease => "BUTTON_RELEASE".to_string(), + } +} + +/** + * Log all of the movements that are sent to this verifier. Helps to identify the streams that lead + * to inconsistent events. + * Enable this via "adb shell setprop log.tag.InputVerifierLogEvents DEBUG" + */ +fn log_events() -> bool { + shouldLog("InputVerifierLogEvents") +} + +struct InputVerifier { + name: String, + touching_pointer_ids_by_device: HashMap<DeviceId, HashSet<i32>>, +} + +impl InputVerifier { + fn new(name: &str) -> Self { + logger::init( + logger::Config::default() + .with_tag_on_device("InputVerifier") + .with_min_level(log::Level::Trace), + ); + Self { name: name.to_owned(), touching_pointer_ids_by_device: HashMap::new() } + } + + fn process_movement( + &mut self, + device_id: DeviceId, + action: u32, + pointer_properties: &[RustPointerProperties], + flags: Flags, + ) -> Result<(), String> { + if log_events() { + info!( + "Processing {} for device {:?} ({} pointer{}) on {}", + motion_action_to_string(action), + device_id, + pointer_properties.len(), + if pointer_properties.len() == 1 { "" } else { "s" }, + self.name + ); + } + + match action.into() { + MotionAction::Down => { + let it = self + .touching_pointer_ids_by_device + .entry(device_id) + .or_insert_with(HashSet::new); + let pointer_id = pointer_properties[0].id; + if it.contains(&pointer_id) { + return Err(format!( + "{}: Invalid DOWN event - pointers already down for device {:?}: {:?}", + self.name, device_id, it + )); + } + it.insert(pointer_id); + } + MotionAction::PointerDown { action_index } => { + if !self.touching_pointer_ids_by_device.contains_key(&device_id) { + return Err(format!( + "{}: Received POINTER_DOWN but no pointers are currently down \ + for device {:?}", + self.name, device_id + )); + } + let it = self.touching_pointer_ids_by_device.get_mut(&device_id).unwrap(); + let pointer_id = pointer_properties[action_index].id; + if it.contains(&pointer_id) { + return Err(format!( + "{}: Pointer with id={} not found in the properties", + self.name, pointer_id + )); + } + it.insert(pointer_id); + } + MotionAction::Move => { + if !self.ensure_touching_pointers_match(device_id, pointer_properties) { + return Err(format!( + "{}: ACTION_MOVE touching pointers don't match", + self.name + )); + } + } + MotionAction::PointerUp { action_index } => { + if !self.touching_pointer_ids_by_device.contains_key(&device_id) { + return Err(format!( + "{}: Received POINTER_UP but no pointers are currently down for device \ + {:?}", + self.name, device_id + )); + } + let it = self.touching_pointer_ids_by_device.get_mut(&device_id).unwrap(); + let pointer_id = pointer_properties[action_index].id; + it.remove(&pointer_id); + } + MotionAction::Up => { + if !self.touching_pointer_ids_by_device.contains_key(&device_id) { + return Err(format!( + "{} Received ACTION_UP but no pointers are currently down for device {:?}", + self.name, device_id + )); + } + let it = self.touching_pointer_ids_by_device.get_mut(&device_id).unwrap(); + if it.len() != 1 { + return Err(format!( + "{}: Got ACTION_UP, but we have pointers: {:?} for device {:?}", + self.name, it, device_id + )); + } + let pointer_id = pointer_properties[0].id; + if !it.contains(&pointer_id) { + return Err(format!( + "{}: Got ACTION_UP, but pointerId {} is not touching. Touching pointers:\ + {:?} for device {:?}", + self.name, pointer_id, it, device_id + )); + } + it.clear(); + } + MotionAction::Cancel => { + if flags.contains(Flags::CANCELED) { + return Err(format!( + "{}: For ACTION_CANCEL, must set FLAG_CANCELED", + self.name + )); + } + if !self.ensure_touching_pointers_match(device_id, pointer_properties) { + return Err(format!( + "{}: Got ACTION_CANCEL, but the pointers don't match. \ + Existing pointers: {:?}", + self.name, self.touching_pointer_ids_by_device + )); + } + self.touching_pointer_ids_by_device.remove(&device_id); + } + _ => return Ok(()), + } + Ok(()) + } + + fn ensure_touching_pointers_match( + &self, + device_id: DeviceId, + pointer_properties: &[RustPointerProperties], + ) -> bool { + let Some(pointers) = self.touching_pointer_ids_by_device.get(&device_id) else { + return false; + }; + + for pointer_property in pointer_properties.iter() { + let pointer_id = pointer_property.id; + if !pointers.contains(&pointer_id) { + return false; + } + } + true + } +} + +#[cfg(test)] +mod tests { + use crate::DeviceId; + use crate::Flags; + use crate::InputVerifier; + use crate::RustPointerProperties; + #[test] + fn single_pointer_stream() { + let mut verifier = InputVerifier::new("Test"); + let pointer_properties = Vec::from([RustPointerProperties { id: 0 }]); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_DOWN, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_MOVE, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_UP, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + } + + #[test] + fn multi_device_stream() { + let mut verifier = InputVerifier::new("Test"); + let pointer_properties = Vec::from([RustPointerProperties { id: 0 }]); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_DOWN, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_MOVE, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(2), + input_bindgen::AMOTION_EVENT_ACTION_DOWN, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(2), + input_bindgen::AMOTION_EVENT_ACTION_MOVE, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_UP, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + } + + #[test] + fn test_invalid_up() { + let mut verifier = InputVerifier::new("Test"); + let pointer_properties = Vec::from([RustPointerProperties { id: 0 }]); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_UP, + &pointer_properties, + Flags::empty(), + ) + .is_err()); + } +} diff --git a/libs/input/tests/Android.bp b/libs/input/tests/Android.bp index 42bdf57514..e7224ff752 100644 --- a/libs/input/tests/Android.bp +++ b/libs/input/tests/Android.bp @@ -18,7 +18,9 @@ cc_test { "InputDevice_test.cpp", "InputEvent_test.cpp", "InputPublisherAndConsumer_test.cpp", + "InputVerifier_test.cpp", "MotionPredictor_test.cpp", + "MotionPredictorMetricsManager_test.cpp", "RingBuffer_test.cpp", "TfLiteMotionPredictor_test.cpp", "TouchResampling_test.cpp", @@ -44,6 +46,7 @@ cc_test { "-Wno-unused-parameter", ], sanitize: { + hwaddress: true, undefined: true, all_undefined: true, diag: { @@ -56,17 +59,33 @@ cc_test { "libcutils", "liblog", "libPlatformProperties", + "libtinyxml2", "libutils", "libvintf", ], data: [ "data/*", - ":motion_predictor_model.fb", + ":motion_predictor_model", ], test_options: { unit_test: true, }, test_suites: ["device-tests"], + target: { + host: { + sanitize: { + address: true, + }, + }, + android: { + static_libs: [ + // Stats logging library and its dependencies. + "libstatslog_libinput", + "libstatsbootstrap", + "android.os.statsbootstrap_aidl-cpp", + ], + }, + }, } // NOTE: This is a compile time test, and does not need to be diff --git a/libs/input/tests/InputVerifier_test.cpp b/libs/input/tests/InputVerifier_test.cpp new file mode 100644 index 0000000000..e24fa6ed0b --- /dev/null +++ b/libs/input/tests/InputVerifier_test.cpp @@ -0,0 +1,29 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> +#include <input/InputVerifier.h> +#include <string> + +namespace android { + +TEST(InputVerifierTest, CreationWithInvalidUtfStringDoesNotCrash) { + constexpr char bytes[] = {static_cast<char>(0xC0), static_cast<char>(0x80)}; + const std::string name(bytes, sizeof(bytes)); + InputVerifier verifier(name); +} + +} // namespace android diff --git a/libs/input/tests/MotionPredictorMetricsManager_test.cpp b/libs/input/tests/MotionPredictorMetricsManager_test.cpp new file mode 100644 index 0000000000..b420a5a4e7 --- /dev/null +++ b/libs/input/tests/MotionPredictorMetricsManager_test.cpp @@ -0,0 +1,972 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <input/MotionPredictor.h> + +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <numeric> +#include <vector> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <input/InputEventBuilders.h> +#include <utils/Timers.h> // for nsecs_t + +#include "Eigen/Core" +#include "Eigen/Geometry" + +namespace android { +namespace { + +using ::testing::FloatNear; +using ::testing::Matches; + +using GroundTruthPoint = MotionPredictorMetricsManager::GroundTruthPoint; +using PredictionPoint = MotionPredictorMetricsManager::PredictionPoint; +using AtomFields = MotionPredictorMetricsManager::AtomFields; + +inline constexpr int NANOS_PER_MILLIS = 1'000'000; + +inline constexpr nsecs_t TEST_INITIAL_TIMESTAMP = 1'000'000'000; +inline constexpr size_t TEST_MAX_NUM_PREDICTIONS = 5; +inline constexpr nsecs_t TEST_PREDICTION_INTERVAL_NANOS = 12'500'000 / 3; // 1 / (240 hz) +inline constexpr int NO_DATA_SENTINEL = MotionPredictorMetricsManager::NO_DATA_SENTINEL; + +// Parameters: +// • arg: Eigen::Vector2f +// • target: Eigen::Vector2f +// • epsilon: float +MATCHER_P2(Vector2fNear, target, epsilon, "") { + return Matches(FloatNear(target[0], epsilon))(arg[0]) && + Matches(FloatNear(target[1], epsilon))(arg[1]); +} + +// Parameters: +// • arg: PredictionPoint +// • target: PredictionPoint +// • epsilon: float +MATCHER_P2(PredictionPointNear, target, epsilon, "") { + if (!Matches(Vector2fNear(target.position, epsilon))(arg.position)) { + *result_listener << "Position mismatch. Actual: (" << arg.position[0] << ", " + << arg.position[1] << "), expected: (" << target.position[0] << ", " + << target.position[1] << ")"; + return false; + } + if (!Matches(FloatNear(target.pressure, epsilon))(arg.pressure)) { + *result_listener << "Pressure mismatch. Actual: " << arg.pressure + << ", expected: " << target.pressure; + return false; + } + if (arg.originTimestamp != target.originTimestamp) { + *result_listener << "Origin timestamp mismatch. Actual: " << arg.originTimestamp + << ", expected: " << target.originTimestamp; + return false; + } + if (arg.targetTimestamp != target.targetTimestamp) { + *result_listener << "Target timestamp mismatch. Actual: " << arg.targetTimestamp + << ", expected: " << target.targetTimestamp; + return false; + } + return true; +} + +// --- Mathematical helper functions. --- + +template <typename T> +T average(std::vector<T> values) { + return std::accumulate(values.begin(), values.end(), T{}) / static_cast<T>(values.size()); +} + +template <typename T> +T standardDeviation(std::vector<T> values) { + T mean = average(values); + T accumulator = {}; + for (const T value : values) { + accumulator += value * value - mean * mean; + } + // Take the max with 0 to avoid negative values caused by numerical instability. + return std::sqrt(std::max(T{}, accumulator) / static_cast<T>(values.size())); +} + +template <typename T> +T rmse(std::vector<T> errors) { + T sse = {}; + for (const T error : errors) { + sse += error * error; + } + return std::sqrt(sse / static_cast<T>(errors.size())); +} + +TEST(MathematicalHelperFunctionTest, Average) { + std::vector<float> values{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + EXPECT_EQ(5.5f, average(values)); +} + +TEST(MathematicalHelperFunctionTest, StandardDeviation) { + // https://www.calculator.net/standard-deviation-calculator.html?numberinputs=10%2C+12%2C+23%2C+23%2C+16%2C+23%2C+21%2C+16 + std::vector<float> values{10, 12, 23, 23, 16, 23, 21, 16}; + EXPECT_FLOAT_EQ(4.8989794855664f, standardDeviation(values)); +} + +TEST(MathematicalHelperFunctionTest, Rmse) { + std::vector<float> errors{1, 5, 7, 7, 8, 20}; + EXPECT_FLOAT_EQ(9.899494937f, rmse(errors)); +} + +// --- MotionEvent-related helper functions. --- + +// Creates a MotionEvent corresponding to the given GroundTruthPoint. +MotionEvent makeMotionEvent(const GroundTruthPoint& groundTruthPoint) { + // Build single pointer of type STYLUS, with coordinates from groundTruthPoint. + PointerBuilder pointerBuilder = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(groundTruthPoint.position[1]) + .y(groundTruthPoint.position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, groundTruthPoint.pressure); + return MotionEventBuilder(/*action=*/AMOTION_EVENT_ACTION_MOVE, + /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .eventTime(groundTruthPoint.timestamp) + .pointer(pointerBuilder) + .build(); +} + +// Creates a MotionEvent corresponding to the given sequence of PredictionPoints. +MotionEvent makeMotionEvent(const std::vector<PredictionPoint>& predictionPoints) { + // Build single pointer of type STYLUS, with coordinates from first prediction point. + PointerBuilder pointerBuilder = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(predictionPoints[0].position[1]) + .y(predictionPoints[0].position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, predictionPoints[0].pressure); + MotionEvent predictionEvent = + MotionEventBuilder( + /*action=*/AMOTION_EVENT_ACTION_MOVE, /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .eventTime(predictionPoints[0].targetTimestamp) + .pointer(pointerBuilder) + .build(); + for (size_t i = 1; i < predictionPoints.size(); ++i) { + PointerCoords coords = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(predictionPoints[i].position[1]) + .y(predictionPoints[i].position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, predictionPoints[i].pressure) + .buildCoords(); + predictionEvent.addSample(predictionPoints[i].targetTimestamp, &coords); + } + return predictionEvent; +} + +// Creates a MotionEvent corresponding to a stylus lift (UP) ground truth event. +MotionEvent makeLiftMotionEvent() { + return MotionEventBuilder(/*action=*/AMOTION_EVENT_ACTION_UP, + /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .pointer(PointerBuilder(/*id=*/0, ToolType::STYLUS)) + .build(); +} + +TEST(MakeMotionEventTest, MakeGroundTruthMotionEvent) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10.0f, 20.0f), + .pressure = 0.6f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const MotionEvent groundTruthMotionEvent = makeMotionEvent(groundTruthPoint); + + ASSERT_EQ(1u, groundTruthMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(0u, groundTruthMotionEvent.getHistorySize()); + EXPECT_EQ(groundTruthPoint.position[0], groundTruthMotionEvent.getRawPointerCoords(0)->getY()); + EXPECT_EQ(groundTruthPoint.position[1], groundTruthMotionEvent.getRawPointerCoords(0)->getX()); + EXPECT_EQ(groundTruthPoint.pressure, + groundTruthMotionEvent.getRawPointerCoords(0)->getAxisValue( + AMOTION_EVENT_AXIS_PRESSURE)); + EXPECT_EQ(AMOTION_EVENT_ACTION_MOVE, groundTruthMotionEvent.getAction()); +} + +TEST(MakeMotionEventTest, MakePredictionMotionEvent) { + const nsecs_t originTimestamp = TEST_INITIAL_TIMESTAMP; + const std::vector<PredictionPoint> + predictionPoints{{{.position = Eigen::Vector2f(10.0f, 20.0f), .pressure = 0.6f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 5 * NANOS_PER_MILLIS}, + {{.position = Eigen::Vector2f(11.0f, 22.0f), .pressure = 0.5f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 10 * NANOS_PER_MILLIS}, + {{.position = Eigen::Vector2f(12.0f, 24.0f), .pressure = 0.4f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 15 * NANOS_PER_MILLIS}}; + const MotionEvent predictionMotionEvent = makeMotionEvent(predictionPoints); + + ASSERT_EQ(1u, predictionMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(predictionPoints.size(), predictionMotionEvent.getHistorySize() + 1); + for (size_t i = 0; i < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const PointerCoords coords = *predictionMotionEvent.getHistoricalRawPointerCoords( + /*pointerIndex=*/0, /*historicalIndex=*/i); + EXPECT_EQ(predictionPoints[i].position[0], coords.getY()); + EXPECT_EQ(predictionPoints[i].position[1], coords.getX()); + EXPECT_EQ(predictionPoints[i].pressure, coords.getAxisValue(AMOTION_EVENT_AXIS_PRESSURE)); + // Note: originTimestamp is discarded when converting PredictionPoint to MotionEvent. + EXPECT_EQ(predictionPoints[i].targetTimestamp, + predictionMotionEvent.getHistoricalEventTime(i)); + EXPECT_EQ(AMOTION_EVENT_ACTION_MOVE, predictionMotionEvent.getAction()); + } +} + +TEST(MakeMotionEventTest, MakeLiftMotionEvent) { + const MotionEvent liftMotionEvent = makeLiftMotionEvent(); + ASSERT_EQ(1u, liftMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(0u, liftMotionEvent.getHistorySize()); + EXPECT_EQ(AMOTION_EVENT_ACTION_UP, liftMotionEvent.getAction()); +} + +// --- Ground-truth-generation helper functions. --- + +std::vector<GroundTruthPoint> generateConstantGroundTruthPoints( + const GroundTruthPoint& groundTruthPoint, size_t numPoints) { + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = groundTruthPoint.timestamp; + for (size_t i = 0; i < numPoints; ++i) { + groundTruthPoints.emplace_back(groundTruthPoint); + groundTruthPoints.back().timestamp = timestamp; + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + return groundTruthPoints; +} + +// This function uses the coordinate system (y, x), with +y pointing downwards and +x pointing +// rightwards. Angles are measured counterclockwise from down (+y). +std::vector<GroundTruthPoint> generateCircularArcGroundTruthPoints(Eigen::Vector2f initialPosition, + float initialAngle, + float velocity, + float turningAngle, + size_t numPoints) { + std::vector<GroundTruthPoint> groundTruthPoints; + // Create first point. + if (numPoints > 0) { + groundTruthPoints.push_back({{.position = initialPosition, .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}); + } + float trajectoryAngle = initialAngle; // measured counterclockwise from +y axis. + for (size_t i = 1; i < numPoints; ++i) { + const Eigen::Vector2f trajectory = + Eigen::Rotation2D(trajectoryAngle) * Eigen::Vector2f(1, 0); + groundTruthPoints.push_back( + {{.position = groundTruthPoints.back().position + velocity * trajectory, + .pressure = 0.0f}, + .timestamp = groundTruthPoints.back().timestamp + TEST_PREDICTION_INTERVAL_NANOS}); + trajectoryAngle += turningAngle; + } + return groundTruthPoints; +} + +TEST(GenerateConstantGroundTruthPointsTest, BasicTest) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10, 20), .pressure = 0.3f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(groundTruthPoint, /*numPoints=*/3); + + ASSERT_EQ(3u, groundTruthPoints.size()); + // First point. + EXPECT_EQ(groundTruthPoints[0].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[0].pressure, groundTruthPoint.pressure); + EXPECT_EQ(groundTruthPoints[0].timestamp, groundTruthPoint.timestamp); + // Second point. + EXPECT_EQ(groundTruthPoints[1].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[1].pressure, groundTruthPoint.pressure); + EXPECT_GT(groundTruthPoints[1].timestamp, groundTruthPoints[0].timestamp); + // Third point. + EXPECT_EQ(groundTruthPoints[2].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[2].pressure, groundTruthPoint.pressure); + EXPECT_GT(groundTruthPoints[2].timestamp, groundTruthPoints[1].timestamp); +} + +TEST(GenerateCircularArcGroundTruthTest, StraightLineUpwards) { + const std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(0, 0), + /*initialAngle=*/M_PI, + /*velocity=*/1.0f, + /*turningAngle=*/0.0f, + /*numPoints=*/3); + + ASSERT_EQ(3u, groundTruthPoints.size()); + EXPECT_THAT(groundTruthPoints[0].position, Vector2fNear(Eigen::Vector2f(0, 0), 1e-6)); + EXPECT_THAT(groundTruthPoints[1].position, Vector2fNear(Eigen::Vector2f(-1, 0), 1e-6)); + EXPECT_THAT(groundTruthPoints[2].position, Vector2fNear(Eigen::Vector2f(-2, 0), 1e-6)); + // Check that timestamps are increasing between consecutive ground truth points. + EXPECT_GT(groundTruthPoints[1].timestamp, groundTruthPoints[0].timestamp); + EXPECT_GT(groundTruthPoints[2].timestamp, groundTruthPoints[1].timestamp); +} + +TEST(GenerateCircularArcGroundTruthTest, CounterclockwiseSquare) { + // Generate points in a counterclockwise unit square starting pointing right. + const std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(10, 100), + /*initialAngle=*/M_PI_2, + /*velocity=*/1.0f, + /*turningAngle=*/M_PI_2, + /*numPoints=*/5); + + ASSERT_EQ(5u, groundTruthPoints.size()); + EXPECT_THAT(groundTruthPoints[0].position, Vector2fNear(Eigen::Vector2f(10, 100), 1e-6)); + EXPECT_THAT(groundTruthPoints[1].position, Vector2fNear(Eigen::Vector2f(10, 101), 1e-6)); + EXPECT_THAT(groundTruthPoints[2].position, Vector2fNear(Eigen::Vector2f(9, 101), 1e-6)); + EXPECT_THAT(groundTruthPoints[3].position, Vector2fNear(Eigen::Vector2f(9, 100), 1e-6)); + EXPECT_THAT(groundTruthPoints[4].position, Vector2fNear(Eigen::Vector2f(10, 100), 1e-6)); +} + +// --- Prediction-generation helper functions. --- + +// Creates a sequence of predictions with values equal to those of the given GroundTruthPoint. +std::vector<PredictionPoint> generateConstantPredictions(const GroundTruthPoint& groundTruthPoint) { + std::vector<PredictionPoint> predictions; + nsecs_t predictionTimestamp = groundTruthPoint.timestamp + TEST_PREDICTION_INTERVAL_NANOS; + for (size_t j = 0; j < TEST_MAX_NUM_PREDICTIONS; ++j) { + predictions.push_back(PredictionPoint{{.position = groundTruthPoint.position, + .pressure = groundTruthPoint.pressure}, + .originTimestamp = groundTruthPoint.timestamp, + .targetTimestamp = predictionTimestamp}); + predictionTimestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + return predictions; +} + +// Generates TEST_MAX_NUM_PREDICTIONS predictions from the given most recent two ground truth points +// by linear extrapolation of position and pressure. The interval between consecutive predictions' +// timestamps is TEST_PREDICTION_INTERVAL_NANOS. +std::vector<PredictionPoint> generatePredictionsByLinearExtrapolation( + const GroundTruthPoint& firstGroundTruth, const GroundTruthPoint& secondGroundTruth) { + // Precompute deltas. + const Eigen::Vector2f trajectory = secondGroundTruth.position - firstGroundTruth.position; + const float deltaPressure = secondGroundTruth.pressure - firstGroundTruth.pressure; + // Compute predictions. + std::vector<PredictionPoint> predictions; + Eigen::Vector2f predictionPosition = secondGroundTruth.position; + float predictionPressure = secondGroundTruth.pressure; + nsecs_t predictionTargetTimestamp = secondGroundTruth.timestamp; + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS; ++i) { + predictionPosition += trajectory; + predictionPressure += deltaPressure; + predictionTargetTimestamp += TEST_PREDICTION_INTERVAL_NANOS; + predictions.push_back( + PredictionPoint{{.position = predictionPosition, .pressure = predictionPressure}, + .originTimestamp = secondGroundTruth.timestamp, + .targetTimestamp = predictionTargetTimestamp}); + } + return predictions; +} + +TEST(GeneratePredictionsTest, GenerateConstantPredictions) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10, 20), .pressure = 0.3f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const std::vector<PredictionPoint> predictionPoints = + generateConstantPredictions(groundTruthPoint); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, predictionPoints.size()); + for (size_t i = 0; i < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + EXPECT_THAT(predictionPoints[i].position, Vector2fNear(groundTruthPoint.position, 1e-6)); + EXPECT_THAT(predictionPoints[i].pressure, FloatNear(groundTruthPoint.pressure, 1e-6)); + EXPECT_EQ(predictionPoints[i].originTimestamp, groundTruthPoint.timestamp); + EXPECT_EQ(predictionPoints[i].targetTimestamp, + groundTruthPoint.timestamp + + static_cast<nsecs_t>(i + 1) * TEST_PREDICTION_INTERVAL_NANOS); + } +} + +TEST(GeneratePredictionsTest, LinearExtrapolationFromTwoPoints) { + const nsecs_t initialTimestamp = TEST_INITIAL_TIMESTAMP; + const std::vector<PredictionPoint> predictionPoints = generatePredictionsByLinearExtrapolation( + GroundTruthPoint{{.position = Eigen::Vector2f(100, 200), .pressure = 0.9f}, + .timestamp = initialTimestamp}, + GroundTruthPoint{{.position = Eigen::Vector2f(105, 190), .pressure = 0.8f}, + .timestamp = initialTimestamp + TEST_PREDICTION_INTERVAL_NANOS}); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, predictionPoints.size()); + const nsecs_t originTimestamp = initialTimestamp + TEST_PREDICTION_INTERVAL_NANOS; + EXPECT_THAT(predictionPoints[0], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(110, 180), + .pressure = 0.7f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[1], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(115, 170), + .pressure = 0.6f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 2 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[2], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(120, 160), + .pressure = 0.5f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 3 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[3], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(125, 150), + .pressure = 0.4f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 4 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[4], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(130, 140), + .pressure = 0.3f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 5 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); +} + +// Generates predictions by linear extrapolation for each consecutive pair of ground truth points +// (see the comment for the above function for further explanation). Returns a vector of vectors of +// prediction points, where the first index is the source ground truth index, and the second is the +// prediction target index. +// +// The returned vector has size equal to the input vector, and the first element of the returned +// vector is always empty. +std::vector<std::vector<PredictionPoint>> generateAllPredictionsByLinearExtrapolation( + const std::vector<GroundTruthPoint>& groundTruthPoints) { + std::vector<std::vector<PredictionPoint>> allPredictions; + allPredictions.emplace_back(); + for (size_t i = 1; i < groundTruthPoints.size(); ++i) { + allPredictions.push_back(generatePredictionsByLinearExtrapolation(groundTruthPoints[i - 1], + groundTruthPoints[i])); + } + return allPredictions; +} + +TEST(GeneratePredictionsTest, GenerateAllPredictions) { + const nsecs_t initialTimestamp = TEST_INITIAL_TIMESTAMP; + std::vector<GroundTruthPoint> + groundTruthPoints{GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.5f}, + .timestamp = initialTimestamp}, + GroundTruthPoint{{.position = Eigen::Vector2f(1, -1), + .pressure = 0.51f}, + .timestamp = initialTimestamp + + 2 * TEST_PREDICTION_INTERVAL_NANOS}, + GroundTruthPoint{{.position = Eigen::Vector2f(2, -2), + .pressure = 0.52f}, + .timestamp = initialTimestamp + + 3 * TEST_PREDICTION_INTERVAL_NANOS}}; + + const std::vector<std::vector<PredictionPoint>> allPredictions = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + // Check format of allPredictions data. + ASSERT_EQ(groundTruthPoints.size(), allPredictions.size()); + EXPECT_TRUE(allPredictions[0].empty()); + EXPECT_EQ(TEST_MAX_NUM_PREDICTIONS, allPredictions[1].size()); + EXPECT_EQ(TEST_MAX_NUM_PREDICTIONS, allPredictions[2].size()); + + // Check positions of predictions generated from first pair of ground truth points. + EXPECT_THAT(allPredictions[1][0].position, Vector2fNear(Eigen::Vector2f(2, -2), 1e-9)); + EXPECT_THAT(allPredictions[1][1].position, Vector2fNear(Eigen::Vector2f(3, -3), 1e-9)); + EXPECT_THAT(allPredictions[1][2].position, Vector2fNear(Eigen::Vector2f(4, -4), 1e-9)); + EXPECT_THAT(allPredictions[1][3].position, Vector2fNear(Eigen::Vector2f(5, -5), 1e-9)); + EXPECT_THAT(allPredictions[1][4].position, Vector2fNear(Eigen::Vector2f(6, -6), 1e-9)); + + // Check pressures of predictions generated from first pair of ground truth points. + EXPECT_FLOAT_EQ(0.52f, allPredictions[1][0].pressure); + EXPECT_FLOAT_EQ(0.53f, allPredictions[1][1].pressure); + EXPECT_FLOAT_EQ(0.54f, allPredictions[1][2].pressure); + EXPECT_FLOAT_EQ(0.55f, allPredictions[1][3].pressure); + EXPECT_FLOAT_EQ(0.56f, allPredictions[1][4].pressure); +} + +// --- Prediction error helper functions. --- + +struct GeneralPositionErrors { + float alongTrajectoryErrorMean; + float alongTrajectoryErrorStd; + float offTrajectoryRmse; +}; + +// Inputs: +// • Vector of ground truth points +// • Vector of vectors of prediction points, where the first index is the source ground truth +// index, and the second is the prediction target index. +// +// Returns a vector of GeneralPositionErrors, indexed by prediction time delta bucket. +std::vector<GeneralPositionErrors> computeGeneralPositionErrors( + const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints) { + // Aggregate errors by time bucket (prediction target index). + std::vector<GeneralPositionErrors> generalPostitionErrors; + for (size_t predictionTargetIndex = 0; predictionTargetIndex < TEST_MAX_NUM_PREDICTIONS; + ++predictionTargetIndex) { + std::vector<float> alongTrajectoryErrors; + std::vector<float> alongTrajectorySquaredErrors; + std::vector<float> offTrajectoryErrors; + for (size_t sourceGroundTruthIndex = 1; sourceGroundTruthIndex < groundTruthPoints.size(); + ++sourceGroundTruthIndex) { + const size_t targetGroundTruthIndex = + sourceGroundTruthIndex + predictionTargetIndex + 1; + // Only include errors for points with a ground truth value. + if (targetGroundTruthIndex < groundTruthPoints.size()) { + const Eigen::Vector2f trajectory = + (groundTruthPoints[targetGroundTruthIndex].position - + groundTruthPoints[targetGroundTruthIndex - 1].position) + .normalized(); + const Eigen::Vector2f orthogonalTrajectory = + Eigen::Rotation2Df(M_PI_2) * trajectory; + const Eigen::Vector2f positionError = + predictionPoints[sourceGroundTruthIndex][predictionTargetIndex].position - + groundTruthPoints[targetGroundTruthIndex].position; + alongTrajectoryErrors.push_back(positionError.dot(trajectory)); + alongTrajectorySquaredErrors.push_back(alongTrajectoryErrors.back() * + alongTrajectoryErrors.back()); + offTrajectoryErrors.push_back(positionError.dot(orthogonalTrajectory)); + } + } + generalPostitionErrors.push_back( + {.alongTrajectoryErrorMean = average(alongTrajectoryErrors), + .alongTrajectoryErrorStd = standardDeviation(alongTrajectoryErrors), + .offTrajectoryRmse = rmse(offTrajectoryErrors)}); + } + return generalPostitionErrors; +} + +// Inputs: +// • Vector of ground truth points +// • Vector of vectors of prediction points, where the first index is the source ground truth +// index, and the second is the prediction target index. +// +// Returns a vector of pressure RMSEs, indexed by prediction time delta bucket. +std::vector<float> computePressureRmses( + const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints) { + // Aggregate errors by time bucket (prediction target index). + std::vector<float> pressureRmses; + for (size_t predictionTargetIndex = 0; predictionTargetIndex < TEST_MAX_NUM_PREDICTIONS; + ++predictionTargetIndex) { + std::vector<float> pressureErrors; + for (size_t sourceGroundTruthIndex = 1; sourceGroundTruthIndex < groundTruthPoints.size(); + ++sourceGroundTruthIndex) { + const size_t targetGroundTruthIndex = + sourceGroundTruthIndex + predictionTargetIndex + 1; + // Only include errors for points with a ground truth value. + if (targetGroundTruthIndex < groundTruthPoints.size()) { + pressureErrors.push_back( + predictionPoints[sourceGroundTruthIndex][predictionTargetIndex].pressure - + groundTruthPoints[targetGroundTruthIndex].pressure); + } + } + pressureRmses.push_back(rmse(pressureErrors)); + } + return pressureRmses; +} + +TEST(ErrorComputationHelperTest, ComputeGeneralPositionErrorsSimpleTest) { + std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + groundTruthPoints[3].position = Eigen::Vector2f(1, 0); + groundTruthPoints[4].position = Eigen::Vector2f(1, 1); + groundTruthPoints[5].position = Eigen::Vector2f(1, 3); + groundTruthPoints[6].position = Eigen::Vector2f(2, 3); + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + // The generated predictions look like: + // + // | Source | Target Ground Truth Index | + // | Index | 2 | 3 | 4 | 5 | 6 | + // |------------|--------|--------|--------|--------|--------| + // | 1 | (0, 0) | (0, 0) | (0, 0) | (0, 0) | (0, 0) | + // | 2 | | (0, 0) | (0, 0) | (0, 0) | (0, 0) | + // | 3 | | | (2, 0) | (3, 0) | (4, 0) | + // | 4 | | | | (1, 2) | (1, 3) | + // | 5 | | | | | (1, 5) | + // |---------------------------------------------------------| + // | Actual Ground Truth Values | + // | Position | (0, 0) | (1, 0) | (1, 1) | (1, 3) | (2, 3) | + // | Previous | (0, 0) | (0, 0) | (1, 0) | (1, 1) | (1, 3) | + // + // Note: this table organizes prediction targets by target ground truth index. Metrics are + // aggregated across points with the same prediction time bucket index, which is different. + // Each down-right diagonal from this table gives us points from a unique time bucket. + + // Initialize expected prediction errors from the table above. The first time bucket corresponds + // to the long diagonal of the table, and subsequent time buckets step up-right from there. + const std::vector<std::vector<float>> expectedAlongTrajectoryErrors{{0, -1, -1, -1, -1}, + {-1, -1, -3, -1}, + {-1, -3, 2}, + {-3, -2}, + {-2}}; + const std::vector<std::vector<float>> expectedOffTrajectoryErrors{{0, 0, 1, 0, 2}, + {0, 1, 2, 0}, + {1, 1, 3}, + {1, 3}, + {3}}; + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, generalPositionErrors.size()); + for (size_t i = 0; i < generalPositionErrors.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + EXPECT_FLOAT_EQ(average(expectedAlongTrajectoryErrors[i]), + generalPositionErrors[i].alongTrajectoryErrorMean); + EXPECT_FLOAT_EQ(standardDeviation(expectedAlongTrajectoryErrors[i]), + generalPositionErrors[i].alongTrajectoryErrorStd); + EXPECT_FLOAT_EQ(rmse(expectedOffTrajectoryErrors[i]), + generalPositionErrors[i].offTrajectoryRmse); + } +} + +TEST(ErrorComputationHelperTest, ComputePressureRmsesSimpleTest) { + // Generate ground truth points with pressures {0.0, 0.0, 0.0, 0.0, 0.5, 0.5, 0.5}. + // (We need TEST_MAX_NUM_PREDICTIONS + 2 to test all prediction time buckets.) + std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + for (size_t i = 4; i < groundTruthPoints.size(); ++i) { + groundTruthPoints[i].pressure = 0.5f; + } + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<float> pressureRmses = computePressureRmses(groundTruthPoints, predictionPoints); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, pressureRmses.size()); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{0.0f, 0.0f, -0.5f, 0.5f, 0.0f}), pressureRmses[0]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{0.0f, -0.5f, -0.5f, 1.0f}), pressureRmses[1]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f, -0.5f, -0.5f}), pressureRmses[2]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f, -0.5f}), pressureRmses[3]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f}), pressureRmses[4]); +} + +// --- MotionPredictorMetricsManager tests. --- + +// Helper function that instantiates a MetricsManager with the given mock logged AtomFields. Takes +// vectors of ground truth and prediction points of the same length, and passes these points to the +// MetricsManager. The format of these vectors is expected to be: +// • groundTruthPoints: chronologically-ordered ground truth points, with at least 2 elements. +// • predictionPoints: the first index points to a vector of predictions corresponding to the +// source ground truth point with the same index. +// - The first element should be empty, because there are not expected to be predictions until +// we have received 2 ground truth points. +// - The last element may be empty, because there will be no future ground truth points to +// associate with those predictions (if not empty, it will be ignored). +// - To test all prediction buckets, there should be at least TEST_MAX_NUM_PREDICTIONS non-empty +// prediction sets (that is, excluding the first and last). Thus, groundTruthPoints and +// predictionPoints should have size at least TEST_MAX_NUM_PREDICTIONS + 2. +// +// The passed-in outAtomFields will contain the logged AtomFields when the function returns. +// +// This function returns void so that it can use test assertions. +void runMetricsManager(const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints, + std::vector<AtomFields>& outAtomFields) { + MotionPredictorMetricsManager metricsManager(TEST_PREDICTION_INTERVAL_NANOS, + TEST_MAX_NUM_PREDICTIONS); + metricsManager.setMockLoggedAtomFields(&outAtomFields); + + // Validate structure of groundTruthPoints and predictionPoints. + ASSERT_EQ(predictionPoints.size(), groundTruthPoints.size()); + ASSERT_GE(groundTruthPoints.size(), 2u); + ASSERT_EQ(predictionPoints[0].size(), 0u); + for (size_t i = 1; i + 1 < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + ASSERT_EQ(predictionPoints[i].size(), TEST_MAX_NUM_PREDICTIONS); + } + + // Pass ground truth points and predictions (for all except first and last ground truth). + for (size_t i = 0; i < groundTruthPoints.size(); ++i) { + metricsManager.onRecord(makeMotionEvent(groundTruthPoints[i])); + if ((i > 0) && (i + 1 < predictionPoints.size())) { + metricsManager.onPredict(makeMotionEvent(predictionPoints[i])); + } + } + // Send a stroke-end event to trigger the logging call. + metricsManager.onRecord(makeLiftMotionEvent()); +} + +// Vacuous test: +// • Input: no prediction data. +// • Expectation: no metrics should be logged. +TEST(MotionPredictorMetricsManagerTest, NoPredictions) { + std::vector<AtomFields> mockLoggedAtomFields; + MotionPredictorMetricsManager metricsManager(TEST_PREDICTION_INTERVAL_NANOS, + TEST_MAX_NUM_PREDICTIONS); + metricsManager.setMockLoggedAtomFields(&mockLoggedAtomFields); + + metricsManager.onRecord(makeMotionEvent( + GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), .pressure = 0}, .timestamp = 0})); + metricsManager.onRecord(makeLiftMotionEvent()); + + // Check that mockLoggedAtomFields is still empty (as it was initialized empty), ensuring that + // no metrics were logged. + EXPECT_EQ(0u, mockLoggedAtomFields.size()); +} + +// Perfect predictions test: +// • Input: constant input events, perfect predictions matching the input events. +// • Expectation: all error metrics should be zero, or NO_DATA_SENTINEL for "unreported" metrics. +// (For example, scale-invariant errors are only reported for the final time bucket.) +TEST(MotionPredictorMetricsManagerTest, ConstantGroundTruthPerfectPredictions) { + GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10.0f, 20.0f), .pressure = 0.6f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + + // Generate ground truth and prediction points as described by the runMetricsManager comment. + std::vector<GroundTruthPoint> groundTruthPoints; + std::vector<std::vector<PredictionPoint>> predictionPoints; + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + groundTruthPoints.push_back(groundTruthPoint); + predictionPoints.push_back(i > 0 ? generateConstantPredictions(groundTruthPoint) + : std::vector<PredictionPoint>{}); + groundTruthPoint.timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + // Check that errors are all zero, or NO_DATA_SENTINEL for unreported metrics. + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // General errors: reported for every time bucket. + EXPECT_EQ(0, atom.alongTrajectoryErrorMeanMillipixels); + EXPECT_EQ(0, atom.alongTrajectoryErrorStdMillipixels); + EXPECT_EQ(0, atom.offTrajectoryRmseMillipixels); + EXPECT_EQ(0, atom.pressureRmseMilliunits); + // High-velocity errors: reported only for the last two time buckets. + // However, this data has zero velocity, so these metrics should all be NO_DATA_SENTINEL. + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityOffTrajectoryRmse); + // Scale-invariant errors: reported only for the last time bucket. + if (i + 1 == atomFields.size()) { + EXPECT_EQ(0, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(0, atom.scaleInvariantOffTrajectoryRmse); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantOffTrajectoryRmse); + } + } +} + +TEST(MotionPredictorMetricsManagerTest, QuadraticPressureLinearPredictions) { + // Generate ground truth points. + // + // Ground truth pressures are a quadratically increasing function from some initial value. + const float initialPressure = 0.5f; + const float quadraticCoefficient = 0.01f; + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = TEST_INITIAL_TIMESTAMP; + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + const float pressure = initialPressure + quadraticCoefficient * static_cast<float>(i * i); + groundTruthPoints.push_back( + GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), .pressure = pressure}, + .timestamp = timestamp}); + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + // Note: the first index is the source ground truth index, and the second is the prediction + // target index. + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + const std::vector<float> pressureErrors = + computePressureRmses(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + // Check time bucket delta matches expectation based on index and prediction interval. + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // Check pressure error matches expectation. + EXPECT_NEAR(static_cast<int>(1000 * pressureErrors[i]), atom.pressureRmseMilliunits, 1); + } +} + +TEST(MotionPredictorMetricsManagerTest, QuadraticPositionLinearPredictionsGeneralErrors) { + // Generate ground truth points. + // + // Each component of the ground truth positions are an independent quadratically increasing + // function from some initial value. + const Eigen::Vector2f initialPosition(200, 300); + const Eigen::Vector2f quadraticCoefficients(-2, 3); + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = TEST_INITIAL_TIMESTAMP; + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + const Eigen::Vector2f position = + initialPosition + quadraticCoefficients * static_cast<float>(i * i); + groundTruthPoints.push_back( + GroundTruthPoint{{.position = position, .pressure = 0.5}, .timestamp = timestamp}); + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + // Note: the first index is the source ground truth index, and the second is the prediction + // target index. + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + // Check time bucket delta matches expectation based on index and prediction interval. + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // Check general position errors match expectation. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorMean), + atom.alongTrajectoryErrorMeanMillipixels, 1); + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorStd), + atom.alongTrajectoryErrorStdMillipixels, 1); + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].offTrajectoryRmse), + atom.offTrajectoryRmseMillipixels, 1); + } +} + +// Counterclockwise regular octagonal section test: +// • Input – ground truth: constantly-spaced input events starting at a trajectory pointing exactly +// rightwards, and rotating by 45° counterclockwise after each input. +// • Input – predictions: simple linear extrapolations of previous two ground truth points. +// +// The code below uses the following terminology to distinguish references to ground truth events: +// • Source ground truth: the most recent ground truth point received at the time the prediction +// was made. +// • Target ground truth: the ground truth event that the prediction was attempting to match. +TEST(MotionPredictorMetricsManagerTest, CounterclockwiseOctagonGroundTruthLinearPredictions) { + // Select a stroke velocity that exceeds the high-velocity threshold of 1100 px/sec. + // For an input rate of 240 hz, 1100 px/sec * (1/240) sec/input ≈ 4.58 pixels per input. + const float strokeVelocity = 10; // pixels per input + + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(100, 100), + /*initialAngle=*/M_PI_2, + /*velocity=*/strokeVelocity, + /*turningAngle=*/-M_PI_4, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + + // General errors: reported for every time bucket. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorMean), + atom.alongTrajectoryErrorMeanMillipixels, 1); + // We allow for some floating point error in standard deviation (0.02 pixels). + EXPECT_NEAR(1000 * generalPositionErrors[i].alongTrajectoryErrorStd, + atom.alongTrajectoryErrorStdMillipixels, 20); + // All position errors are equal, so the standard deviation should be approximately zero. + EXPECT_NEAR(0, atom.alongTrajectoryErrorStdMillipixels, 20); + // Absolute value for RMSE, since it must be non-negative. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].offTrajectoryRmse), + atom.offTrajectoryRmseMillipixels, 1); + + // High-velocity errors: reported only for the last two time buckets. + // + // Since our input stroke velocity is chosen to be above the high-velocity threshold, all + // data contributes to high-velocity errors, and thus high-velocity errors should be equal + // to general errors (where reported). + // + // As above, use absolute value for RMSE, since it must be non-negative. + if (i + 2 >= atomFields.size()) { + EXPECT_NEAR(static_cast<int>( + 1000 * std::abs(generalPositionErrors[i].alongTrajectoryErrorMean)), + atom.highVelocityAlongTrajectoryRmse, 1); + EXPECT_NEAR(static_cast<int>(1000 * + std::abs(generalPositionErrors[i].offTrajectoryRmse)), + atom.highVelocityOffTrajectoryRmse, 1); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityOffTrajectoryRmse); + } + + // Scale-invariant errors: reported only for the last time bucket, where the reported value + // is the aggregation across all time buckets. + // + // The MetricsManager stores mMaxNumPredictions recent ground truth segments. Our ground + // truth segments here all have a length of strokeVelocity, so we can convert general errors + // to scale-invariant errors by dividing by `strokeVelocty * TEST_MAX_NUM_PREDICTIONS`. + // + // As above, use absolute value for RMSE, since it must be non-negative. + if (i + 1 == atomFields.size()) { + const float pathLength = strokeVelocity * TEST_MAX_NUM_PREDICTIONS; + std::vector<float> alongTrajectoryAbsoluteErrors; + std::vector<float> offTrajectoryAbsoluteErrors; + for (size_t j = 0; j < TEST_MAX_NUM_PREDICTIONS; ++j) { + alongTrajectoryAbsoluteErrors.push_back( + std::abs(generalPositionErrors[j].alongTrajectoryErrorMean)); + offTrajectoryAbsoluteErrors.push_back( + std::abs(generalPositionErrors[j].offTrajectoryRmse)); + } + EXPECT_NEAR(static_cast<int>(1000 * average(alongTrajectoryAbsoluteErrors) / + pathLength), + atom.scaleInvariantAlongTrajectoryRmse, 1); + EXPECT_NEAR(static_cast<int>(1000 * average(offTrajectoryAbsoluteErrors) / pathLength), + atom.scaleInvariantOffTrajectoryRmse, 1); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantOffTrajectoryRmse); + } + } +} + +} // namespace +} // namespace android diff --git a/libs/input/tests/MotionPredictor_test.cpp b/libs/input/tests/MotionPredictor_test.cpp index 7a62f5ec58..4ac7ae920e 100644 --- a/libs/input/tests/MotionPredictor_test.cpp +++ b/libs/input/tests/MotionPredictor_test.cpp @@ -72,11 +72,20 @@ TEST(MotionPredictorTest, IsPredictionAvailable) { ASSERT_FALSE(predictor.isPredictionAvailable(/*deviceId=*/1, AINPUT_SOURCE_TOUCHSCREEN)); } +TEST(MotionPredictorTest, StationaryNoiseFloor) { + MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/1, + []() { return true /*enable prediction*/; }); + predictor.record(getMotionEvent(DOWN, 0, 1, 30ms)); + predictor.record(getMotionEvent(MOVE, 0, 1, 35ms)); // No movement. + std::unique_ptr<MotionEvent> predicted = predictor.predict(40 * NSEC_PER_MSEC); + ASSERT_EQ(nullptr, predicted); +} + TEST(MotionPredictorTest, Offset) { MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/1, []() { return true /*enable prediction*/; }); predictor.record(getMotionEvent(DOWN, 0, 1, 30ms)); - predictor.record(getMotionEvent(MOVE, 0, 2, 35ms)); + predictor.record(getMotionEvent(MOVE, 0, 5, 35ms)); // Move enough to overcome the noise floor. std::unique_ptr<MotionEvent> predicted = predictor.predict(40 * NSEC_PER_MSEC); ASSERT_NE(nullptr, predicted); ASSERT_GE(predicted->getEventTime(), 41); diff --git a/libs/input/tests/VelocityTracker_test.cpp b/libs/input/tests/VelocityTracker_test.cpp index ae721093a0..73f25cc615 100644 --- a/libs/input/tests/VelocityTracker_test.cpp +++ b/libs/input/tests/VelocityTracker_test.cpp @@ -282,6 +282,11 @@ static void computeAndCheckAxisScrollVelocity( const std::vector<std::pair<std::chrono::nanoseconds, float>>& motions, std::optional<float> targetVelocity) { checkVelocity(computeVelocity(strategy, motions, AMOTION_EVENT_AXIS_SCROLL), targetVelocity); + // The strategy LSQ2 is not compatible with AXIS_SCROLL. In those situations, we should fall + // back to a strategy that supports differential axes. + checkVelocity(computeVelocity(VelocityTracker::Strategy::LSQ2, motions, + AMOTION_EVENT_AXIS_SCROLL), + targetVelocity); } static void computeAndCheckQuadraticEstimate(const std::vector<PlanarMotionEventEntry>& motions, diff --git a/libs/nativewindow/include/system/window.h b/libs/nativewindow/include/system/window.h index 0fee3c112e..edaa422e55 100644 --- a/libs/nativewindow/include/system/window.h +++ b/libs/nativewindow/include/system/window.h @@ -1066,12 +1066,33 @@ static inline int native_window_set_frame_rate(struct ANativeWindow* window, flo (int)compatibility, (int)changeFrameRateStrategy); } +struct ANativeWindowFrameTimelineInfo { + // Frame Id received from ANativeWindow_getNextFrameId. + uint64_t frameNumber; + + // VsyncId received from the Choreographer callback that started this frame. + int64_t frameTimelineVsyncId; + + // Input Event ID received from the input event that started this frame. + int32_t inputEventId; + + // The time which this frame rendering started (i.e. when Choreographer callback actually run) + int64_t startTimeNanos; + + // Whether or not to use the vsyncId to determine the refresh rate. Used for TextureView only. + int32_t useForRefreshRateSelection; + + // The VsyncId of a frame that was not drawn and squashed into this frame. + // Used for UI thread updates that were not picked up by RenderThread on time. + int64_t skippedFrameVsyncId; + + // The start time of a frame that was not drawn and squashed into this frame. + int64_t skippedFrameStartTimeNanos; +}; + static inline int native_window_set_frame_timeline_info( - struct ANativeWindow* window, uint64_t frameNumber, int64_t frameTimelineVsyncId, - int32_t inputEventId, int64_t startTimeNanos, int32_t useForRefreshRateSelection) { - return window->perform(window, NATIVE_WINDOW_SET_FRAME_TIMELINE_INFO, frameNumber, - frameTimelineVsyncId, inputEventId, startTimeNanos, - useForRefreshRateSelection); + struct ANativeWindow* window, struct ANativeWindowFrameTimelineInfo frameTimelineInfo) { + return window->perform(window, NATIVE_WINDOW_SET_FRAME_TIMELINE_INFO, frameTimelineInfo); } // ------------------------------------------------------------------------------------------------ diff --git a/libs/renderengine/skia/AutoBackendTexture.cpp b/libs/renderengine/skia/AutoBackendTexture.cpp index c412c9cff7..23c99b0796 100644 --- a/libs/renderengine/skia/AutoBackendTexture.cpp +++ b/libs/renderengine/skia/AutoBackendTexture.cpp @@ -86,14 +86,38 @@ void AutoBackendTexture::releaseImageProc(SkImage::ReleaseContext releaseContext void logFatalTexture(const char* msg, const GrBackendTexture& tex, ui::Dataspace dataspace, SkColorType colorType) { - GrGLTextureInfo textureInfo; - bool retrievedTextureInfo = tex.getGLTextureInfo(&textureInfo); - LOG_ALWAYS_FATAL("%s isTextureValid:%d dataspace:%d" - "\n\tGrBackendTexture: (%i x %i) hasMipmaps: %i isProtected: %i texType: %i" - "\n\t\tGrGLTextureInfo: success: %i fTarget: %u fFormat: %u colorType %i", - msg, tex.isValid(), dataspace, tex.width(), tex.height(), tex.hasMipmaps(), - tex.isProtected(), static_cast<int>(tex.textureType()), retrievedTextureInfo, - textureInfo.fTarget, textureInfo.fFormat, colorType); + switch (tex.backend()) { + case GrBackendApi::kOpenGL: { + GrGLTextureInfo textureInfo; + bool retrievedTextureInfo = tex.getGLTextureInfo(&textureInfo); + LOG_ALWAYS_FATAL("%s isTextureValid:%d dataspace:%d" + "\n\tGrBackendTexture: (%i x %i) hasMipmaps: %i isProtected: %i " + "texType: %i\n\t\tGrGLTextureInfo: success: %i fTarget: %u fFormat: %u" + " colorType %i", + msg, tex.isValid(), dataspace, tex.width(), tex.height(), + tex.hasMipmaps(), tex.isProtected(), + static_cast<int>(tex.textureType()), retrievedTextureInfo, + textureInfo.fTarget, textureInfo.fFormat, colorType); + break; + } + case GrBackendApi::kVulkan: { + GrVkImageInfo imageInfo; + bool retrievedImageInfo = tex.getVkImageInfo(&imageInfo); + LOG_ALWAYS_FATAL("%s isTextureValid:%d dataspace:%d" + "\n\tGrBackendTexture: (%i x %i) hasMipmaps: %i isProtected: %i " + "texType: %i\n\t\tVkImageInfo: success: %i fFormat: %i " + "fSampleCount: %u fLevelCount: %u colorType %i", + msg, tex.isValid(), dataspace, tex.width(), tex.height(), + tex.hasMipmaps(), tex.isProtected(), + static_cast<int>(tex.textureType()), retrievedImageInfo, + imageInfo.fFormat, imageInfo.fSampleCount, imageInfo.fLevelCount, + colorType); + break; + } + default: + LOG_ALWAYS_FATAL("%s Unexpected backend %u", msg, static_cast<unsigned>(tex.backend())); + break; + } } sk_sp<SkImage> AutoBackendTexture::makeImage(ui::Dataspace dataspace, SkAlphaType alphaType, diff --git a/libs/renderengine/skia/SkiaRenderEngine.cpp b/libs/renderengine/skia/SkiaRenderEngine.cpp index 76ebf9d0c2..d71e55f64c 100644 --- a/libs/renderengine/skia/SkiaRenderEngine.cpp +++ b/libs/renderengine/skia/SkiaRenderEngine.cpp @@ -58,9 +58,9 @@ #include <src/core/SkTraceEventCommon.h> #include <sync/sync.h> #include <ui/BlurRegion.h> -#include <ui/DataspaceUtils.h> #include <ui/DebugUtils.h> #include <ui/GraphicBuffer.h> +#include <ui/HdrRenderTypeUtils.h> #include <utils/Trace.h> #include <cmath> @@ -397,12 +397,10 @@ void SkiaRenderEngine::mapExternalTextureBuffer(const sp<GraphicBuffer>& buffer, } // We don't attempt to map a buffer if the buffer contains protected content. In GL this is // important because GPU resources for protected buffers are much more limited. (In Vk we - // simply match the existing behavior for protected buffers.) In Vk, we never cache any - // buffers while in a protected context, since Vk cannot share across contexts, and protected - // is less common. + // simply match the existing behavior for protected buffers.) We also never cache any + // buffers while in a protected context. const bool isProtectedBuffer = buffer->getUsage() & GRALLOC_USAGE_PROTECTED; - if (isProtectedBuffer || - (mRenderEngineType == RenderEngineType::SKIA_VK_THREADED && isProtected())) { + if (isProtectedBuffer || isProtected()) { return; } ATRACE_CALL(); @@ -467,9 +465,8 @@ void SkiaRenderEngine::unmapExternalTextureBuffer(sp<GraphicBuffer>&& buffer) { std::shared_ptr<AutoBackendTexture::LocalRef> SkiaRenderEngine::getOrCreateBackendTexture( const sp<GraphicBuffer>& buffer, bool isOutputBuffer) { - // Do not lookup the buffer in the cache for protected contexts with the SkiaVk back-end - if (mRenderEngineType == RenderEngineType::SKIA_GL_THREADED || - (mRenderEngineType == RenderEngineType::SKIA_VK_THREADED && !isProtected())) { + // Do not lookup the buffer in the cache for protected contexts + if (!isProtected()) { if (const auto& it = mTextureCache.find(buffer->getId()); it != mTextureCache.end()) { return it->second; } @@ -510,7 +507,8 @@ sk_sp<SkShader> SkiaRenderEngine::createRuntimeEffectShader( auto effect = shaders::LinearEffect{.inputDataspace = parameters.layer.sourceDataspace, .outputDataspace = parameters.outputDataSpace, - .undoPremultipliedAlpha = parameters.undoPremultipliedAlpha}; + .undoPremultipliedAlpha = parameters.undoPremultipliedAlpha, + .fakeOutputDataspace = parameters.fakeOutputDataspace}; auto effectIter = mRuntimeEffects.find(effect); sk_sp<SkRuntimeEffect> runtimeEffect = nullptr; @@ -665,6 +663,8 @@ void SkiaRenderEngine::drawLayersInternal( validateOutputBufferUsage(buffer->getBuffer()); auto grContext = getActiveGrContext(); + LOG_ALWAYS_FATAL_IF(grContext->abandoned(), "GrContext is abandoned/device lost at start of %s", + __func__); // any AutoBackendTexture deletions will now be deferred until cleanupPostRender is called DeferTextureCleanup dtc(mTextureCleanupMgr); @@ -711,7 +711,9 @@ void SkiaRenderEngine::drawLayersInternal( SkCanvas* canvas = dstCanvas; SkiaCapture::OffscreenState offscreenCaptureState; const LayerSettings* blurCompositionLayer = nullptr; - if (mBlurFilter) { + + // TODO (b/270314344): Enable blurs in protected context. + if (mBlurFilter && !mInProtectedContext) { bool requiresCompositionLayer = false; for (const auto& layer : layers) { // if the layer doesn't have blur or it is not visible then continue @@ -805,7 +807,8 @@ void SkiaRenderEngine::drawLayersInternal( const auto [bounds, roundRectClip] = getBoundsAndClip(layer.geometry.boundaries, layer.geometry.roundedCornersCrop, layer.geometry.roundedCornersRadius); - if (mBlurFilter && layerHasBlur(layer, ctModifiesAlpha)) { + // TODO (b/270314344): Enable blurs in protected context. + if (mBlurFilter && layerHasBlur(layer, ctModifiesAlpha) && !mInProtectedContext) { std::unordered_map<uint32_t, sk_sp<SkImage>> cachedBlurs; // if multiple layers have blur, then we need to take a snapshot now because @@ -901,12 +904,14 @@ void SkiaRenderEngine::drawLayersInternal( (display.outputDataspace & ui::Dataspace::TRANSFER_MASK) == static_cast<int32_t>(ui::Dataspace::TRANSFER_SRGB); - const ui::Dataspace runtimeEffectDataspace = !dimInLinearSpace && isExtendedHdr + const bool useFakeOutputDataspaceForRuntimeEffect = !dimInLinearSpace && isExtendedHdr; + + const ui::Dataspace fakeDataspace = useFakeOutputDataspaceForRuntimeEffect ? static_cast<ui::Dataspace>( (display.outputDataspace & ui::Dataspace::STANDARD_MASK) | ui::Dataspace::TRANSFER_GAMMA2_2 | (display.outputDataspace & ui::Dataspace::RANGE_MASK)) - : display.outputDataspace; + : ui::Dataspace::UNKNOWN; // If the input dataspace is range extended, the output dataspace transfer is sRGB // and dimmingStage is GAMMA_OETF, dim in linear space instead, and @@ -1013,7 +1018,8 @@ void SkiaRenderEngine::drawLayersInternal( .layerDimmingRatio = dimInLinearSpace ? layerDimmingRatio : 1.f, - .outputDataSpace = runtimeEffectDataspace})); + .outputDataSpace = display.outputDataspace, + .fakeOutputDataspace = fakeDataspace})); // Turn on dithering when dimming beyond this (arbitrary) threshold... static constexpr float kDimmingThreshold = 0.2f; @@ -1021,7 +1027,10 @@ void SkiaRenderEngine::drawLayersInternal( // Most HDR standards require at least 10-bits of color depth for source content, so we // can just extract the transfer function rather than dig into precise gralloc layout. // Furthermore, we can assume that the only 8-bit target we support is RGBA8888. - const bool requiresDownsample = isHdrDataspace(layer.sourceDataspace) && + const bool requiresDownsample = + getHdrRenderType(layer.sourceDataspace, + std::optional<ui::PixelFormat>(static_cast<ui::PixelFormat>( + buffer->getPixelFormat()))) != HdrRenderType::SDR && buffer->getPixelFormat() == PIXEL_FORMAT_RGBA_8888; if (layerDimmingRatio <= kDimmingThreshold || requiresDownsample) { paint.setDither(true); @@ -1077,7 +1086,8 @@ void SkiaRenderEngine::drawLayersInternal( .undoPremultipliedAlpha = false, .requiresLinearEffect = requiresLinearEffect, .layerDimmingRatio = layerDimmingRatio, - .outputDataSpace = runtimeEffectDataspace})); + .outputDataSpace = display.outputDataspace, + .fakeOutputDataspace = fakeDataspace})); } if (layer.disableBlending) { diff --git a/libs/renderengine/skia/SkiaRenderEngine.h b/libs/renderengine/skia/SkiaRenderEngine.h index 6457bfa9eb..723e73c29e 100644 --- a/libs/renderengine/skia/SkiaRenderEngine.h +++ b/libs/renderengine/skia/SkiaRenderEngine.h @@ -157,6 +157,7 @@ private: bool requiresLinearEffect; float layerDimmingRatio; const ui::Dataspace outputDataSpace; + const ui::Dataspace fakeOutputDataspace; }; sk_sp<SkShader> createRuntimeEffectShader(const RuntimeEffectShaderParameters&); diff --git a/libs/renderengine/skia/SkiaVkRenderEngine.cpp b/libs/renderengine/skia/SkiaVkRenderEngine.cpp index b99e3853ee..c16586bb6b 100644 --- a/libs/renderengine/skia/SkiaVkRenderEngine.cpp +++ b/libs/renderengine/skia/SkiaVkRenderEngine.cpp @@ -263,7 +263,7 @@ VulkanInterface initVulkanInterface(bool protectedContent = false) { VK_GET_INST_PROC(instance, EnumerateDeviceExtensionProperties); VK_GET_INST_PROC(instance, GetPhysicalDeviceProperties2); VK_GET_INST_PROC(instance, GetPhysicalDeviceExternalSemaphoreProperties); - VK_GET_INST_PROC(instance, GetPhysicalDeviceQueueFamilyProperties); + VK_GET_INST_PROC(instance, GetPhysicalDeviceQueueFamilyProperties2); VK_GET_INST_PROC(instance, GetPhysicalDeviceFeatures2); VK_GET_INST_PROC(instance, CreateDevice); @@ -342,17 +342,37 @@ VulkanInterface initVulkanInterface(bool protectedContent = false) { } uint32_t queueCount; - vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, &queueCount, nullptr); + vkGetPhysicalDeviceQueueFamilyProperties2(physicalDevice, &queueCount, nullptr); if (queueCount == 0) { BAIL("Could not find queues for physical device"); } - std::vector<VkQueueFamilyProperties> queueProps(queueCount); - vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, &queueCount, queueProps.data()); + std::vector<VkQueueFamilyProperties2> queueProps(queueCount); + std::vector<VkQueueFamilyGlobalPriorityPropertiesEXT> queuePriorityProps(queueCount); + VkQueueGlobalPriorityKHR queuePriority = VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_KHR; + // Even though we don't yet know if the VK_EXT_global_priority extension is available, + // we can safely add the request to the pNext chain, and if the extension is not + // available, it will be ignored. + for (uint32_t i = 0; i < queueCount; ++i) { + queuePriorityProps[i].sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_GLOBAL_PRIORITY_PROPERTIES_EXT; + queuePriorityProps[i].pNext = nullptr; + queueProps[i].pNext = &queuePriorityProps[i]; + } + vkGetPhysicalDeviceQueueFamilyProperties2(physicalDevice, &queueCount, queueProps.data()); int graphicsQueueIndex = -1; for (uint32_t i = 0; i < queueCount; ++i) { - if (queueProps[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) { + // Look at potential answers to the VK_EXT_global_priority query. If answers were + // provided, we may adjust the queuePriority. + if (queueProps[i].queueFamilyProperties.queueFlags & VK_QUEUE_GRAPHICS_BIT) { + for (uint32_t j = 0; j < queuePriorityProps[i].priorityCount; j++) { + if (queuePriorityProps[i].priorities[j] > queuePriority) { + queuePriority = queuePriorityProps[i].priorities[j]; + } + } + if (queuePriority == VK_QUEUE_GLOBAL_PRIORITY_REALTIME_KHR) { + interface.isRealtimePriority = true; + } graphicsQueueIndex = i; break; } @@ -419,12 +439,11 @@ VulkanInterface initVulkanInterface(bool protectedContent = false) { VK_STRUCTURE_TYPE_DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_EXT, nullptr, // If queue priority is supported, RE should always have realtime priority. - VK_QUEUE_GLOBAL_PRIORITY_REALTIME_EXT, + queuePriority, }; if (interface.grExtensions.hasExtension(VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME, 2)) { queueNextPtr = &queuePriorityCreateInfo; - interface.isRealtimePriority = true; } VkDeviceQueueCreateFlags deviceQueueCreateFlags = diff --git a/libs/shaders/shaders.cpp b/libs/shaders/shaders.cpp index c85517a976..ef039e5c36 100644 --- a/libs/shaders/shaders.cpp +++ b/libs/shaders/shaders.cpp @@ -168,8 +168,8 @@ void generateOOTF(ui::Dataspace inputDataspace, ui::Dataspace outputDataspace, void generateOETF(std::string& shader) { // Only support gamma 2.2 for now shader.append(R"( - float OETF(float3 linear) { - return sign(linear) * pow(abs(linear), (1.0 / 2.2)); + float3 OETF(float3 linear) { + return sign(linear) * pow(abs(linear), float3(1.0 / 2.2)); } )"); } diff --git a/libs/shaders/tests/Android.bp b/libs/shaders/tests/Android.bp index 1e4f45ac45..5639d744df 100644 --- a/libs/shaders/tests/Android.bp +++ b/libs/shaders/tests/Android.bp @@ -37,6 +37,7 @@ cc_test { shared_libs: [ "android.hardware.graphics.common@1.2", "libnativewindow", + "libbase", ], static_libs: [ "libarect", diff --git a/libs/tonemap/tests/Android.bp b/libs/tonemap/tests/Android.bp index 2abf51563c..5c5fc6c8b3 100644 --- a/libs/tonemap/tests/Android.bp +++ b/libs/tonemap/tests/Android.bp @@ -36,6 +36,7 @@ cc_test { ], shared_libs: [ "libnativewindow", + "libbase", ], static_libs: [ "libmath", diff --git a/libs/ui/include/ui/DisplayMap.h b/libs/ui/include/ui/DisplayMap.h new file mode 100644 index 0000000000..7eacb0a7f0 --- /dev/null +++ b/libs/ui/include/ui/DisplayMap.h @@ -0,0 +1,35 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <ftl/small_map.h> +#include <ftl/small_vector.h> + +namespace android::ui { + +// The static capacities were chosen to exceed a typical number of physical and/or virtual displays. + +template <typename Key, typename Value> +using DisplayMap = ftl::SmallMap<Key, Value, 5>; + +template <typename Key, typename Value> +using PhysicalDisplayMap = ftl::SmallMap<Key, Value, 3>; + +template <typename T> +using PhysicalDisplayVector = ftl::SmallVector<T, 3>; + +} // namespace android::ui diff --git a/libs/ui/include/ui/FenceTime.h b/libs/ui/include/ui/FenceTime.h index ac75f431a0..334106f0cf 100644 --- a/libs/ui/include/ui/FenceTime.h +++ b/libs/ui/include/ui/FenceTime.h @@ -142,6 +142,8 @@ private: std::atomic<nsecs_t> mSignalTime{Fence::SIGNAL_TIME_INVALID}; }; +using FenceTimePtr = std::shared_ptr<FenceTime>; + // A queue of FenceTimes that are expected to signal in FIFO order. // Only maintains a queue of weak pointers so it doesn't keep references // to Fences on its own. @@ -190,8 +192,15 @@ private: // before the new one is added. class FenceToFenceTimeMap { public: - // Create a new FenceTime with that wraps the provided Fence. - std::shared_ptr<FenceTime> createFenceTimeForTest(const sp<Fence>& fence); + using FencePair = std::pair<sp<Fence>, FenceTimePtr>; + + FencePair makePendingFenceForTest() { + const auto fence = sp<Fence>::make(); + return {fence, createFenceTimeForTest(fence)}; + } + + // Create a new FenceTime that wraps the provided Fence. + FenceTimePtr createFenceTimeForTest(const sp<Fence>&); // Signals all FenceTimes created through this class that are wrappers // around |fence|. @@ -205,7 +214,6 @@ private: std::unordered_map<Fence*, std::vector<std::weak_ptr<FenceTime>>> mMap; }; - -}; // namespace android +} // namespace android #endif // ANDROID_FENCE_TIME_H diff --git a/libs/ui/include_types/ui/HdrRenderTypeUtils.h b/libs/ui/include_types/ui/HdrRenderTypeUtils.h new file mode 100644 index 0000000000..b0af878cdb --- /dev/null +++ b/libs/ui/include_types/ui/HdrRenderTypeUtils.h @@ -0,0 +1,64 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <ui/GraphicTypes.h> + +namespace android { + +enum class HdrRenderType { + SDR, // just render to SDR + DISPLAY_HDR, // HDR by extended brightness + GENERIC_HDR // tonemapped HDR +}; + +/*** + * A helper function to classify how we treat the result based on params. + * + * @param dataspace the dataspace + * @param pixelFormat optional, in case there is no source buffer. + * @param hdrSdrRatio default is 1.f, render engine side doesn't take care of it. + * @return HdrRenderType + */ +inline HdrRenderType getHdrRenderType(ui::Dataspace dataspace, + std::optional<ui::PixelFormat> pixelFormat, + float hdrSdrRatio = 1.f) { + const auto transfer = dataspace & HAL_DATASPACE_TRANSFER_MASK; + const auto range = dataspace & HAL_DATASPACE_RANGE_MASK; + + if (transfer == HAL_DATASPACE_TRANSFER_ST2084 || transfer == HAL_DATASPACE_TRANSFER_HLG) { + return HdrRenderType::GENERIC_HDR; + } + + static const auto BT2020_LINEAR_EXT = static_cast<ui::Dataspace>(HAL_DATASPACE_STANDARD_BT2020 | + HAL_DATASPACE_TRANSFER_LINEAR | + HAL_DATASPACE_RANGE_EXTENDED); + + if ((dataspace == BT2020_LINEAR_EXT || dataspace == ui::Dataspace::V0_SCRGB) && + pixelFormat.has_value() && pixelFormat.value() == ui::PixelFormat::RGBA_FP16) { + return HdrRenderType::GENERIC_HDR; + } + + // Extended range layer with an hdr/sdr ratio of > 1.01f can "self-promote" to HDR. + if (range == HAL_DATASPACE_RANGE_EXTENDED && hdrSdrRatio > 1.01f) { + return HdrRenderType::DISPLAY_HDR; + } + + return HdrRenderType::SDR; +} + +} // namespace android
\ No newline at end of file diff --git a/libs/ui/tests/Android.bp b/libs/ui/tests/Android.bp index 831b64d877..8ce017d7a3 100644 --- a/libs/ui/tests/Android.bp +++ b/libs/ui/tests/Android.bp @@ -164,9 +164,9 @@ cc_test { } cc_test { - name: "DataspaceUtils_test", + name: "HdrRenderTypeUtils_test", shared_libs: ["libui"], - srcs: ["DataspaceUtils_test.cpp"], + srcs: ["HdrRenderTypeUtils_test.cpp"], cflags: [ "-Wall", "-Werror", diff --git a/libs/ui/tests/DataspaceUtils_test.cpp b/libs/ui/tests/DataspaceUtils_test.cpp deleted file mode 100644 index 3e0967182b..0000000000 --- a/libs/ui/tests/DataspaceUtils_test.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef LOG_TAG -#define LOG_TAG "DataspaceUtilsTest" - -#include <gtest/gtest.h> -#include <ui/DataspaceUtils.h> - -namespace android { - -class DataspaceUtilsTest : public testing::Test {}; - -TEST_F(DataspaceUtilsTest, isHdrDataspace) { - EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_ITU_HLG)); - EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_ITU_PQ)); - EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_PQ)); - EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_HLG)); - - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SRGB_LINEAR)); - // scRGB defines a very wide gamut but not an expanded luminance range - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SCRGB_LINEAR)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SRGB)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SCRGB)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_JFIF)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT601_625)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT601_525)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT709)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DCI_P3_LINEAR)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DCI_P3)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_P3_LINEAR)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_P3)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::ADOBE_RGB)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020_LINEAR)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020_ITU)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_BT2020)); -} - -} // namespace android diff --git a/libs/ui/tests/HdrRenderTypeUtils_test.cpp b/libs/ui/tests/HdrRenderTypeUtils_test.cpp new file mode 100644 index 0000000000..efe819db76 --- /dev/null +++ b/libs/ui/tests/HdrRenderTypeUtils_test.cpp @@ -0,0 +1,65 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#undef LOG_TAG +#define LOG_TAG "HdrRenderTypeUtilsTest" + +#include <gtest/gtest.h> +#include <ui/HdrRenderTypeUtils.h> + +namespace android { + +class HdrRenderTypeUtilsTest : public testing::Test {}; + +TEST_F(HdrRenderTypeUtilsTest, getHdrRenderType) { + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_ITU_HLG, std::nullopt), + HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_ITU_PQ, std::nullopt), + HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_PQ, std::nullopt), HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_HLG, std::nullopt), + HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB, + std::optional<ui::PixelFormat>(ui::PixelFormat::RGBA_FP16)), + HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB, + std::optional<ui::PixelFormat>(ui::PixelFormat::RGBA_8888), 2.f), + HdrRenderType::DISPLAY_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB_LINEAR, + std::optional<ui::PixelFormat>(ui::PixelFormat::RGBA_8888), 2.f), + HdrRenderType::DISPLAY_HDR); + + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SRGB_LINEAR, std::nullopt), HdrRenderType::SDR); + // scRGB defines a very wide gamut but not an expanded luminance range + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB_LINEAR, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SRGB, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_JFIF, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_BT601_625, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_BT601_525, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_BT709, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DCI_P3_LINEAR, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DCI_P3, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DISPLAY_P3_LINEAR, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DISPLAY_P3, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::ADOBE_RGB, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_LINEAR, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_ITU, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DISPLAY_BT2020, std::nullopt), HdrRenderType::SDR); +} + +} // namespace android diff --git a/libs/ultrahdr/include/ultrahdr/jpegr.h b/libs/ultrahdr/include/ultrahdr/jpegr.h index a35fd30634..f80496a758 100644 --- a/libs/ultrahdr/include/ultrahdr/jpegr.h +++ b/libs/ultrahdr/include/ultrahdr/jpegr.h @@ -348,16 +348,6 @@ private: status_t extractPrimaryImageAndGainMap(jr_compressed_ptr compressed_jpegr_image, jr_compressed_ptr primary_image, jr_compressed_ptr gain_map); - /* - * This method is called in the decoding pipeline. It will read XMP metadata to find the start - * position of the compressed gain map, and will extract the compressed gain map. - * - * @param compressed_jpegr_image compressed JPEGR image - * @param dest destination of compressed gain map - * @return NO_ERROR if calculation succeeds, error code if error occurs. - */ - status_t extractGainMap(jr_compressed_ptr compressed_jpegr_image, - jr_compressed_ptr dest); /* * This method is called in the encoding pipeline. It will take the standard 8-bit JPEG image, diff --git a/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h b/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h index 064123210f..5420e1c9cf 100644 --- a/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h +++ b/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h @@ -44,6 +44,7 @@ enum { ERROR_JPEGR_INVALID_TRANS_FUNC = JPEGR_IO_ERROR_BASE - 6, ERROR_JPEGR_INVALID_METADATA = JPEGR_IO_ERROR_BASE - 7, ERROR_JPEGR_UNSUPPORTED_METADATA = JPEGR_IO_ERROR_BASE - 8, + ERROR_JPEGR_GAIN_MAP_IMAGE_NOT_FOUND = JPEGR_IO_ERROR_BASE - 9, JPEGR_RUNTIME_ERROR_BASE = -20000, ERROR_JPEGR_ENCODE_ERROR = JPEGR_RUNTIME_ERROR_BASE - 1, diff --git a/libs/ultrahdr/jpegr.cpp b/libs/ultrahdr/jpegr.cpp index 9c57f34c2a..fb24c9d206 100644 --- a/libs/ultrahdr/jpegr.cpp +++ b/libs/ultrahdr/jpegr.cpp @@ -539,9 +539,12 @@ status_t JpegR::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, jr_info_p return ERROR_JPEGR_INVALID_NULL_PTR; } - jpegr_compressed_struct primary_image, gain_map; - JPEGR_CHECK(extractPrimaryImageAndGainMap(compressed_jpegr_image, - &primary_image, &gain_map)); + jpegr_compressed_struct primary_image, gainmap_image; + status_t status = + extractPrimaryImageAndGainMap(compressed_jpegr_image, &primary_image, &gainmap_image); + if (status != NO_ERROR && status != ERROR_JPEGR_GAIN_MAP_IMAGE_NOT_FOUND) { + return status; + } JpegDecoderHelper jpeg_decoder; if (!jpeg_decoder.getCompressedImageParameters(primary_image.data, primary_image.length, @@ -550,7 +553,7 @@ status_t JpegR::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, jr_info_p return ERROR_JPEGR_DECODE_ERROR; } - return NO_ERROR; + return status; } /* Decode API */ @@ -586,45 +589,56 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, return ERROR_JPEGR_INVALID_INPUT_TYPE; } + jpegr_compressed_struct primary_image, gainmap_image; + status_t status = + extractPrimaryImageAndGainMap(compressed_jpegr_image, &primary_image, &gainmap_image); + if (status != NO_ERROR) { + if (output_format != ULTRAHDR_OUTPUT_SDR || status != ERROR_JPEGR_GAIN_MAP_IMAGE_NOT_FOUND) { + ALOGE("received invalid compressed jpegr image"); + return status; + } + } + + JpegDecoderHelper jpeg_decoder; + if (!jpeg_decoder.decompressImage(primary_image.data, primary_image.length, + (output_format == ULTRAHDR_OUTPUT_SDR))) { + return ERROR_JPEGR_DECODE_ERROR; + } + if (output_format == ULTRAHDR_OUTPUT_SDR) { - JpegDecoderHelper jpeg_decoder; - if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length, - true)) { - return ERROR_JPEGR_DECODE_ERROR; + if ((jpeg_decoder.getDecompressedImageWidth() * + jpeg_decoder.getDecompressedImageHeight() * 4) > + jpeg_decoder.getDecompressedImageSize()) { + return ERROR_JPEGR_CALCULATION_ERROR; } - jpegr_uncompressed_struct uncompressed_rgba_image; - uncompressed_rgba_image.data = jpeg_decoder.getDecompressedImagePtr(); - uncompressed_rgba_image.width = jpeg_decoder.getDecompressedImageWidth(); - uncompressed_rgba_image.height = jpeg_decoder.getDecompressedImageHeight(); - memcpy(dest->data, uncompressed_rgba_image.data, - uncompressed_rgba_image.width * uncompressed_rgba_image.height * 4); - dest->width = uncompressed_rgba_image.width; - dest->height = uncompressed_rgba_image.height; - - if (gain_map == nullptr && exif == nullptr) { - return NO_ERROR; + } else { + if ((jpeg_decoder.getDecompressedImageWidth() * + jpeg_decoder.getDecompressedImageHeight() * 3 / 2) > + jpeg_decoder.getDecompressedImageSize()) { + return ERROR_JPEGR_CALCULATION_ERROR; } + } - if (exif != nullptr) { - if (exif->data == nullptr) { - return ERROR_JPEGR_INVALID_NULL_PTR; - } - if (exif->length < jpeg_decoder.getEXIFSize()) { - return ERROR_JPEGR_BUFFER_TOO_SMALL; - } - memcpy(exif->data, jpeg_decoder.getEXIFPtr(), jpeg_decoder.getEXIFSize()); - exif->length = jpeg_decoder.getEXIFSize(); + if (exif != nullptr) { + if (exif->data == nullptr) { + return ERROR_JPEGR_INVALID_NULL_PTR; } - if (gain_map == nullptr) { - return NO_ERROR; + if (exif->length < jpeg_decoder.getEXIFSize()) { + return ERROR_JPEGR_BUFFER_TOO_SMALL; } + memcpy(exif->data, jpeg_decoder.getEXIFPtr(), jpeg_decoder.getEXIFSize()); + exif->length = jpeg_decoder.getEXIFSize(); } - jpegr_compressed_struct compressed_map; - JPEGR_CHECK(extractGainMap(compressed_jpegr_image, &compressed_map)); + if (output_format == ULTRAHDR_OUTPUT_SDR) { + dest->width = jpeg_decoder.getDecompressedImageWidth(); + dest->height = jpeg_decoder.getDecompressedImageHeight(); + memcpy(dest->data, jpeg_decoder.getDecompressedImagePtr(), dest->width * dest->height * 4); + return NO_ERROR; + } JpegDecoderHelper gain_map_decoder; - if (!gain_map_decoder.decompressImage(compressed_map.data, compressed_map.length)) { + if (!gain_map_decoder.decompressImage(gainmap_image.data, gainmap_image.length)) { return ERROR_JPEGR_DECODE_ERROR; } if ((gain_map_decoder.getDecompressedImageWidth() * @@ -633,12 +647,17 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, return ERROR_JPEGR_CALCULATION_ERROR; } + jpegr_uncompressed_struct map; + map.data = gain_map_decoder.getDecompressedImagePtr(); + map.width = gain_map_decoder.getDecompressedImageWidth(); + map.height = gain_map_decoder.getDecompressedImageHeight(); + if (gain_map != nullptr) { - gain_map->width = gain_map_decoder.getDecompressedImageWidth(); - gain_map->height = gain_map_decoder.getDecompressedImageHeight(); + gain_map->width = map.width; + gain_map->height = map.height; int size = gain_map->width * gain_map->height; gain_map->data = malloc(size); - memcpy(gain_map->data, gain_map_decoder.getDecompressedImagePtr(), size); + memcpy(gain_map->data, map.data, size); } ultrahdr_metadata_struct uhdr_metadata; @@ -648,46 +667,16 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, } if (metadata != nullptr) { - metadata->version = uhdr_metadata.version; - metadata->minContentBoost = uhdr_metadata.minContentBoost; - metadata->maxContentBoost = uhdr_metadata.maxContentBoost; - metadata->gamma = uhdr_metadata.gamma; - metadata->offsetSdr = uhdr_metadata.offsetSdr; - metadata->offsetHdr = uhdr_metadata.offsetHdr; - metadata->hdrCapacityMin = uhdr_metadata.hdrCapacityMin; - metadata->hdrCapacityMax = uhdr_metadata.hdrCapacityMax; - } - - if (output_format == ULTRAHDR_OUTPUT_SDR) { - return NO_ERROR; + metadata->version = uhdr_metadata.version; + metadata->minContentBoost = uhdr_metadata.minContentBoost; + metadata->maxContentBoost = uhdr_metadata.maxContentBoost; + metadata->gamma = uhdr_metadata.gamma; + metadata->offsetSdr = uhdr_metadata.offsetSdr; + metadata->offsetHdr = uhdr_metadata.offsetHdr; + metadata->hdrCapacityMin = uhdr_metadata.hdrCapacityMin; + metadata->hdrCapacityMax = uhdr_metadata.hdrCapacityMax; } - JpegDecoderHelper jpeg_decoder; - if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length)) { - return ERROR_JPEGR_DECODE_ERROR; - } - if ((jpeg_decoder.getDecompressedImageWidth() * - jpeg_decoder.getDecompressedImageHeight() * 3 / 2) > - jpeg_decoder.getDecompressedImageSize()) { - return ERROR_JPEGR_CALCULATION_ERROR; - } - - if (exif != nullptr) { - if (exif->data == nullptr) { - return ERROR_JPEGR_INVALID_NULL_PTR; - } - if (exif->length < jpeg_decoder.getEXIFSize()) { - return ERROR_JPEGR_BUFFER_TOO_SMALL; - } - memcpy(exif->data, jpeg_decoder.getEXIFPtr(), jpeg_decoder.getEXIFSize()); - exif->length = jpeg_decoder.getEXIFSize(); - } - - jpegr_uncompressed_struct map; - map.data = gain_map_decoder.getDecompressedImagePtr(); - map.width = gain_map_decoder.getDecompressedImageWidth(); - map.height = gain_map_decoder.getDecompressedImageHeight(); - jpegr_uncompressed_struct uncompressed_yuv_420_image; uncompressed_yuv_420_image.data = jpeg_decoder.getDecompressedImagePtr(); uncompressed_yuv_420_image.width = jpeg_decoder.getDecompressedImageWidth(); @@ -1131,12 +1120,8 @@ status_t JpegR::extractPrimaryImageAndGainMap(jr_compressed_ptr compressed_jpegr const auto& jpeg_info = jpeg_info_builder.GetInfo(); const auto& image_ranges = jpeg_info.GetImageRanges(); - if (image_ranges.empty()) { - return ERROR_JPEGR_INVALID_INPUT_TYPE; - } - if (image_ranges.size() != 2) { - // Must be 2 JPEG Images + if (image_ranges.empty()) { return ERROR_JPEGR_INVALID_INPUT_TYPE; } @@ -1146,23 +1131,23 @@ status_t JpegR::extractPrimaryImageAndGainMap(jr_compressed_ptr compressed_jpegr primary_image->length = image_ranges[0].GetLength(); } + if (image_ranges.size() == 1) { + return ERROR_JPEGR_GAIN_MAP_IMAGE_NOT_FOUND; + } + if (gain_map != nullptr) { gain_map->data = static_cast<uint8_t*>(compressed_jpegr_image->data) + image_ranges[1].GetBegin(); gain_map->length = image_ranges[1].GetLength(); } - return NO_ERROR; -} - - -status_t JpegR::extractGainMap(jr_compressed_ptr compressed_jpegr_image, - jr_compressed_ptr dest) { - if (compressed_jpegr_image == nullptr || dest == nullptr) { - return ERROR_JPEGR_INVALID_NULL_PTR; + // TODO: choose primary image and gain map image carefully + if (image_ranges.size() > 2) { + ALOGW("Number of jpeg images present %d, primary, gain map images may not be correctly chosen", + (int)image_ranges.size()); } - return extractPrimaryImageAndGainMap(compressed_jpegr_image, nullptr, dest); + return NO_ERROR; } // JPEG/R structure: |