diff options
author | 2024-07-10 00:12:41 +0000 | |
---|---|---|
committer | 2024-08-05 18:56:00 +0000 | |
commit | be9c544d82db96bd6cb76f6a97e647c52d186faa (patch) | |
tree | 0d74294c9f7cb2dbbebbacb331462006597d1643 | |
parent | 892e766d2aa2c961e506d27f045a75492320d64e (diff) |
Add resampling to InputConsumerNoResampling
Moved resampling into its own class and provided test coverage for it.
Bug: 297226446
Flag: EXEMPT refactor
Test: TEST=libinput_tests; m $TEST && $ANDROID_HOST_OUT/nativetest64/$TEST/$TEST --gtest_filter="*ResamplingTest*"
Change-Id: Ic6227b05120395c96643ab05e1cda373dba59e19
-rw-r--r-- | include/input/InputConsumerNoResampling.h | 24 | ||||
-rw-r--r-- | include/input/Resampler.h | 115 | ||||
-rw-r--r-- | libs/input/Android.bp | 1 | ||||
-rw-r--r-- | libs/input/InputConsumerNoResampling.cpp | 24 | ||||
-rw-r--r-- | libs/input/Resampler.cpp | 151 | ||||
-rw-r--r-- | libs/input/tests/Android.bp | 1 | ||||
-rw-r--r-- | libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp | 5 | ||||
-rw-r--r-- | libs/input/tests/Resampler_test.cpp | 417 |
8 files changed, 728 insertions, 10 deletions
diff --git a/include/input/InputConsumerNoResampling.h b/include/input/InputConsumerNoResampling.h index ae8de5f411..65c2914b3c 100644 --- a/include/input/InputConsumerNoResampling.h +++ b/include/input/InputConsumerNoResampling.h @@ -17,6 +17,7 @@ #pragma once #include <input/InputTransport.h> +#include <input/Resampler.h> #include <utils/Looper.h> namespace android { @@ -47,13 +48,13 @@ public: /** * Consumes input events from an input channel. * - * This is a re-implementation of InputConsumer that does not have resampling at the current moment. - * A lot of the higher-level logic has been folded into this class, to make it easier to use. - * In the legacy class, InputConsumer, the consumption logic was partially handled in the jni layer, - * as well as various actions like adding the fd to the Choreographer. + * This is a re-implementation of InputConsumer. At the moment it only supports resampling for + * single pointer events. A lot of the higher-level logic has been folded into this class, to make + * it easier to use. In the legacy class, InputConsumer, the consumption logic was partially handled + * in the jni layer, as well as various actions like adding the fd to the Choreographer. * * TODO(b/297226446): use this instead of "InputConsumer": - * - Add resampling to this class + * - Add resampling for multiple pointer events. * - Allow various resampling strategies to be specified * - Delete the old "InputConsumer" and use this class instead, renaming it to "InputConsumer". * - Add tracing @@ -64,8 +65,18 @@ public: */ class InputConsumerNoResampling final { public: + /** + * @param callbacks are used to interact with InputConsumerNoResampling. They're called whenever + * the event is ready to consume. + * @param looper needs to be sp and not shared_ptr because it inherits from + * RefBase + * @param resampler the resampling strategy to use. If null, no resampling will be + * performed. + */ explicit InputConsumerNoResampling(const std::shared_ptr<InputChannel>& channel, - sp<Looper> looper, InputConsumerCallbacks& callbacks); + sp<Looper> looper, InputConsumerCallbacks& callbacks, + std::unique_ptr<Resampler> resampler); + ~InputConsumerNoResampling(); /** @@ -99,6 +110,7 @@ private: std::shared_ptr<InputChannel> mChannel; sp<Looper> mLooper; InputConsumerCallbacks& mCallbacks; + std::unique_ptr<Resampler> mResampler; // Looper-related infrastructure /** diff --git a/include/input/Resampler.h b/include/input/Resampler.h new file mode 100644 index 0000000000..ff9c4b0868 --- /dev/null +++ b/include/input/Resampler.h @@ -0,0 +1,115 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <chrono> +#include <optional> + +#include <input/Input.h> +#include <input/InputTransport.h> +#include <input/RingBuffer.h> +#include <utils/Timers.h> + +namespace android { + +/** + * Resampler is an interface for resampling MotionEvents. Every resampling implementation + * must use this interface to enable resampling inside InputConsumer's logic. + */ +struct Resampler { + virtual ~Resampler() = default; + + /** + * Tries to resample motionEvent at resampleTime. The provided resampleTime must be greater than + * the latest sample time of motionEvent. It is not guaranteed that resampling occurs at + * resampleTime. Interpolation may occur is futureSample is available. Otherwise, motionEvent + * may be resampled by another method, or not resampled at all. Furthermore, it is the + * implementer's responsibility to guarantee the following: + * - If resampling occurs, a single additional sample should be added to motionEvent. That is, + * if motionEvent had N samples before being passed to Resampler, then it will have N + 1 + * samples by the end of the resampling. No other field of motionEvent should be modified. + * - If resampling does not occur, then motionEvent must not be modified in any way. + */ + virtual void resampleMotionEvent(const std::chrono::nanoseconds resampleTime, + MotionEvent& motionEvent, + const InputMessage* futureSample) = 0; +}; + +class LegacyResampler final : public Resampler { +public: + /** + * Tries to resample `motionEvent` at `resampleTime` by adding a resampled sample at the end of + * `motionEvent` with eventTime equal to `resampleTime` and pointer coordinates determined by + * linear interpolation or linear extrapolation. An earlier `resampleTime` will be used if + * extrapolation takes place and `resampleTime` is too far in the future. If `futureSample` is + * not null, interpolation will occur. If `futureSample` is null and there is enough historical + * data, LegacyResampler will extrapolate. Otherwise, no resampling takes place and + * `motionEvent` is unmodified. + */ + void resampleMotionEvent(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent, + const InputMessage* futureSample) override; + +private: + struct Pointer { + PointerProperties properties; + PointerCoords coords; + }; + + struct Sample { + std::chrono::nanoseconds eventTime; + Pointer pointer; + + Sample(const std::chrono::nanoseconds eventTime, const PointerProperties& properties, + const PointerCoords& coords) + : eventTime{eventTime}, pointer{properties, coords} {} + }; + + /** + * Keeps track of the previous MotionEvent deviceId to enable comparison between the previous + * and the current deviceId. + */ + std::optional<DeviceId> mPreviousDeviceId; + + /** + * Up to two latest samples from MotionEvent. Updated every time resampleMotionEvent is called. + * Note: We store up to two samples in order to simplify the implementation. Although, + * calculations are possible with only one previous sample. + */ + RingBuffer<Sample> mLatestSamples{/*capacity=*/2}; + + /** + * Adds up to mLatestSamples.capacity() of motionEvent's latest samples to mLatestSamples. (If + * motionEvent has fewer samples than mLatestSamples.capacity(), then the available samples are + * added to mLatestSamples.) + */ + void updateLatestSamples(const MotionEvent& motionEvent); + + /** + * May add a sample at the end of motionEvent with eventTime equal to resampleTime, and + * interpolated coordinates between the latest motionEvent sample and futureSample. + */ + void interpolate(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent, + const InputMessage& futureSample) const; + + /** + * May add a sample at the end of motionEvent by extrapolating from the latest two samples. The + * added sample either has eventTime equal to resampleTime, or an earlier time if resampleTime + * is too far in the future. + */ + void extrapolate(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent) const; +}; +} // namespace android
\ No newline at end of file diff --git a/libs/input/Android.bp b/libs/input/Android.bp index 8fbf5c6a2e..e4e81adf58 100644 --- a/libs/input/Android.bp +++ b/libs/input/Android.bp @@ -232,6 +232,7 @@ cc_library { "MotionPredictorMetricsManager.cpp", "PrintTools.cpp", "PropertyMap.cpp", + "Resampler.cpp", "TfLiteMotionPredictor.cpp", "TouchVideoFrame.cpp", "VelocityControl.cpp", diff --git a/libs/input/InputConsumerNoResampling.cpp b/libs/input/InputConsumerNoResampling.cpp index c145d5c286..99ffa683dd 100644 --- a/libs/input/InputConsumerNoResampling.cpp +++ b/libs/input/InputConsumerNoResampling.cpp @@ -17,6 +17,8 @@ #define LOG_TAG "InputTransport" #define ATRACE_TAG ATRACE_TAG_INPUT +#include <chrono> + #include <inttypes.h> #include <android-base/logging.h> @@ -168,6 +170,10 @@ InputMessage createTimelineMessage(int32_t inputEventId, nsecs_t gpuCompletedTim return msg; } +bool isPointerEvent(const MotionEvent& motionEvent) { + return (motionEvent.getSource() & AINPUT_SOURCE_CLASS_POINTER) == AINPUT_SOURCE_CLASS_POINTER; +} + } // namespace using android::base::Result; @@ -177,8 +183,13 @@ using android::base::StringPrintf; InputConsumerNoResampling::InputConsumerNoResampling(const std::shared_ptr<InputChannel>& channel, sp<Looper> looper, - InputConsumerCallbacks& callbacks) - : mChannel(channel), mLooper(looper), mCallbacks(callbacks), mFdEvents(0) { + InputConsumerCallbacks& callbacks, + std::unique_ptr<Resampler> resampler) + : mChannel(channel), + mLooper(looper), + mCallbacks(callbacks), + mResampler(std::move(resampler)), + mFdEvents(0) { LOG_ALWAYS_FATAL_IF(mLooper == nullptr); mCallback = sp<LooperEventCallback>::make( std::bind(&InputConsumerNoResampling::handleReceiveCallback, this, @@ -463,6 +474,15 @@ InputConsumerNoResampling::createBatchedMotionEvent(const nsecs_t frameTime, } messages.pop(); } + // Check if resampling should be performed. + if (motionEvent != nullptr && isPointerEvent(*motionEvent) && mResampler != nullptr) { + InputMessage* futureSample = nullptr; + if (!messages.empty()) { + futureSample = &messages.front(); + } + mResampler->resampleMotionEvent(static_cast<std::chrono::nanoseconds>(frameTime), + *motionEvent, futureSample); + } return std::make_pair(std::move(motionEvent), firstSeqForBatch); } diff --git a/libs/input/Resampler.cpp b/libs/input/Resampler.cpp new file mode 100644 index 0000000000..af8354c70c --- /dev/null +++ b/libs/input/Resampler.cpp @@ -0,0 +1,151 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "LegacyResampler" + +#include <algorithm> +#include <chrono> + +#include <android-base/logging.h> +#include <android-base/properties.h> + +#include <input/Resampler.h> +#include <utils/Timers.h> + +using std::chrono::nanoseconds; + +namespace android { + +namespace { + +const bool IS_DEBUGGABLE_BUILD = +#if defined(__ANDROID__) + android::base::GetBoolProperty("ro.debuggable", false); +#else + true; +#endif + +bool debugResampling() { + if (!IS_DEBUGGABLE_BUILD) { + static const bool DEBUG_TRANSPORT_RESAMPLING = + __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", + ANDROID_LOG_INFO); + return DEBUG_TRANSPORT_RESAMPLING; + } + return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", ANDROID_LOG_INFO); +} + +constexpr std::chrono::milliseconds RESAMPLE_LATENCY{5}; + +constexpr std::chrono::milliseconds RESAMPLE_MIN_DELTA{2}; + +constexpr std::chrono::milliseconds RESAMPLE_MAX_DELTA{20}; + +constexpr std::chrono::milliseconds RESAMPLE_MAX_PREDICTION{8}; + +inline float lerp(float a, float b, float alpha) { + return a + alpha * (b - a); +} + +const PointerCoords calculateResampledCoords(const PointerCoords& a, const PointerCoords& b, + const float alpha) { + // Ensure the struct PointerCoords is initialized. + PointerCoords resampledCoords{}; + resampledCoords.isResampled = true; + resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_X, lerp(a.getX(), b.getX(), alpha)); + resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, lerp(a.getY(), b.getY(), alpha)); + return resampledCoords; +} +} // namespace + +void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) { + const size_t motionEventSampleSize = motionEvent.getHistorySize() + 1; + for (size_t i = 0; i < motionEventSampleSize; ++i) { + Sample sample{static_cast<nanoseconds>(motionEvent.getHistoricalEventTime(i)), + *motionEvent.getPointerProperties(0), + motionEvent.getSamplePointerCoords()[i]}; + mLatestSamples.pushBack(sample); + } +} + +void LegacyResampler::interpolate(const nanoseconds resampleTime, MotionEvent& motionEvent, + const InputMessage& futureSample) const { + const Sample pastSample = mLatestSamples.back(); + const nanoseconds delta = + static_cast<nanoseconds>(futureSample.body.motion.eventTime) - pastSample.eventTime; + if (delta < RESAMPLE_MIN_DELTA) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns."; + return; + } + const float alpha = + std::chrono::duration<float, std::milli>(resampleTime - pastSample.eventTime) / delta; + + const PointerCoords resampledCoords = + calculateResampledCoords(pastSample.pointer.coords, + futureSample.body.motion.pointers[0].coords, alpha); + motionEvent.addSample(resampleTime.count(), &resampledCoords, motionEvent.getId()); +} + +void LegacyResampler::extrapolate(const nanoseconds resampleTime, MotionEvent& motionEvent) const { + if (mLatestSamples.size() < 2) { + return; + } + const Sample pastSample = *(mLatestSamples.end() - 2); + const Sample presentSample = *(mLatestSamples.end() - 1); + const nanoseconds delta = + static_cast<nanoseconds>(presentSample.eventTime - pastSample.eventTime); + if (delta < RESAMPLE_MIN_DELTA) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns."; + return; + } else if (delta > RESAMPLE_MAX_DELTA) { + LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too large: " << delta << "ns."; + return; + } + // The farthest future time to which we can extrapolate. If the given resampleTime exceeds this, + // we use this value as the resample time target. + const nanoseconds farthestPrediction = static_cast<nanoseconds>(presentSample.eventTime) + + std::min<nanoseconds>(delta / 2, RESAMPLE_MAX_PREDICTION); + const nanoseconds newResampleTime = + (resampleTime > farthestPrediction) ? (farthestPrediction) : (resampleTime); + LOG_IF(INFO, debugResampling() && newResampleTime == farthestPrediction) + << "Resample time is too far in the future. Adjusting prediction from " + << (resampleTime - presentSample.eventTime) << " to " + << (farthestPrediction - presentSample.eventTime) << "ns."; + const float alpha = + std::chrono::duration<float, std::milli>(newResampleTime - pastSample.eventTime) / + delta; + + const PointerCoords resampledCoords = + calculateResampledCoords(pastSample.pointer.coords, presentSample.pointer.coords, + alpha); + motionEvent.addSample(newResampleTime.count(), &resampledCoords, motionEvent.getId()); +} + +void LegacyResampler::resampleMotionEvent(const nanoseconds resampleTime, MotionEvent& motionEvent, + const InputMessage* futureSample) { + if (mPreviousDeviceId && *mPreviousDeviceId != motionEvent.getDeviceId()) { + mLatestSamples.clear(); + } + mPreviousDeviceId = motionEvent.getDeviceId(); + updateLatestSamples(motionEvent); + if (futureSample) { + interpolate(resampleTime, motionEvent, *futureSample); + } else { + extrapolate(resampleTime, motionEvent); + } + LOG_IF(INFO, debugResampling()) << "Not resampled. Not enough data."; +} +} // namespace android diff --git a/libs/input/tests/Android.bp b/libs/input/tests/Android.bp index e9d799ed3f..132866bd99 100644 --- a/libs/input/tests/Android.bp +++ b/libs/input/tests/Android.bp @@ -23,6 +23,7 @@ cc_test { "InputVerifier_test.cpp", "MotionPredictor_test.cpp", "MotionPredictorMetricsManager_test.cpp", + "Resampler_test.cpp", "RingBuffer_test.cpp", "TfLiteMotionPredictor_test.cpp", "TouchResampling_test.cpp", diff --git a/libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp b/libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp index e7106135fd..467c3b46ce 100644 --- a/libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp +++ b/libs/input/tests/InputPublisherAndConsumerNoResampling_test.cpp @@ -396,8 +396,9 @@ void InputPublisherAndConsumerNoResamplingTest::handleMessage(const Message& mes break; } case LooperMessage::CREATE_CONSUMER: { - mConsumer = std::make_unique<InputConsumerNoResampling>(std::move(mClientChannel), - mLooper, *this); + mConsumer = + std::make_unique<InputConsumerNoResampling>(std::move(mClientChannel), mLooper, + *this, /*resampler=*/nullptr); break; } case LooperMessage::DESTROY_CONSUMER: { diff --git a/libs/input/tests/Resampler_test.cpp b/libs/input/tests/Resampler_test.cpp new file mode 100644 index 0000000000..e160ca06d0 --- /dev/null +++ b/libs/input/tests/Resampler_test.cpp @@ -0,0 +1,417 @@ +/** + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <input/Resampler.h> + +#include <gtest/gtest.h> + +#include <chrono> +#include <memory> +#include <vector> + +#include <input/Input.h> +#include <input/InputEventBuilders.h> +#include <input/InputTransport.h> +#include <utils/Timers.h> + +namespace android { + +namespace { + +using namespace std::literals::chrono_literals; + +constexpr float EPSILON = MotionEvent::ROUNDING_PRECISION; + +struct Pointer { + int32_t id{0}; + ToolType toolType{ToolType::FINGER}; + float x{0.0f}; + float y{0.0f}; + bool isResampled{false}; + /** + * Converts from Pointer to PointerCoords. Enables calling LegacyResampler methods and + * assertions only with the relevant data for tests. + */ + operator PointerCoords() const; +}; + +Pointer::operator PointerCoords() const { + PointerCoords pointerCoords; + pointerCoords.setAxisValue(AMOTION_EVENT_AXIS_X, x); + pointerCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, y); + pointerCoords.isResampled = isResampled; + return pointerCoords; +} + +struct InputSample { + std::chrono::milliseconds eventTime{0}; + std::vector<Pointer> pointers{}; + /** + * Converts from InputSample to InputMessage. Enables calling LegacyResampler methods only with + * the relevant data for tests. + */ + operator InputMessage() const; +}; + +InputSample::operator InputMessage() const { + InputMessage message; + message.header.type = InputMessage::Type::MOTION; + message.body.motion.pointerCount = pointers.size(); + message.body.motion.eventTime = static_cast<std::chrono::nanoseconds>(eventTime).count(); + message.body.motion.source = AINPUT_SOURCE_CLASS_POINTER; + message.body.motion.downTime = 0; + const uint32_t pointerCount = message.body.motion.pointerCount; + for (uint32_t i = 0; i < pointerCount; ++i) { + message.body.motion.pointers[i].properties.id = pointers[i].id; + message.body.motion.pointers[i].properties.toolType = pointers[i].toolType; + message.body.motion.pointers[i].coords.setAxisValue(AMOTION_EVENT_AXIS_X, pointers[i].x); + message.body.motion.pointers[i].coords.setAxisValue(AMOTION_EVENT_AXIS_Y, pointers[i].y); + message.body.motion.pointers[i].coords.isResampled = pointers[i].isResampled; + } + return message; +} + +struct InputStream { + std::vector<InputSample> samples{}; + int32_t action{0}; + DeviceId deviceId{0}; + /** + * Converts from InputStream to MotionEvent. Enables calling LegacyResampler methods only with + * the relevant data for tests. + */ + operator MotionEvent() const; +}; + +InputStream::operator MotionEvent() const { + const InputSample& firstSample{*samples.begin()}; + MotionEventBuilder motionEventBuilder = + MotionEventBuilder(action, AINPUT_SOURCE_CLASS_POINTER) + .downTime(0) + .eventTime(static_cast<std::chrono::nanoseconds>(firstSample.eventTime).count()) + .deviceId(deviceId); + for (const Pointer& pointer : firstSample.pointers) { + const PointerBuilder pointerBuilder = + PointerBuilder(pointer.id, pointer.toolType).x(pointer.x).y(pointer.y); + motionEventBuilder.pointer(pointerBuilder); + } + MotionEvent motionEvent = motionEventBuilder.build(); + const size_t numSamples = samples.size(); + for (size_t i = 1; i < numSamples; ++i) { + std::vector<PointerCoords> pointersCoords{samples[i].pointers.begin(), + samples[i].pointers.end()}; + motionEvent.addSample(static_cast<std::chrono::nanoseconds>(samples[i].eventTime).count(), + pointersCoords.data(), motionEvent.getId()); + } + return motionEvent; +} + +} // namespace + +class ResamplerTest : public testing::Test { +protected: + ResamplerTest() : mResampler(std::make_unique<LegacyResampler>()) {} + + ~ResamplerTest() override {} + + void SetUp() override {} + + void TearDown() override {} + + std::unique_ptr<Resampler> mResampler; + + MotionEvent buildMotionEvent(const int32_t action, const nsecs_t eventTime, + const std::vector<PointerBuilder>& pointers); + + InputMessage createMessage(const uint32_t pointerCount, const nsecs_t eventTime, + const int32_t action, + const std::vector<PointerProperties>& properties, + const std::vector<PointerCoords>& coords); + + /** + * Checks that beforeCall and afterCall are equal except for the mutated attributes by addSample + * member function. + * @param beforeCall MotionEvent before passing it to resampleMotionEvent + * @param afterCall MotionEvent after passing it to resampleMotionEvent + */ + void assertMotionEventMetaDataDidNotMutate(const MotionEvent& beforeCall, + const MotionEvent& afterCall); + + /** + * Asserts the MotionEvent is resampled by checking an increment in history size and that the + * resampled coordinates are near the expected ones. + */ + void assertMotionEventIsResampledAndCoordsNear(const MotionEvent& original, + const MotionEvent& resampled, + const PointerCoords& expectedCoords); + + void assertMotionEventIsNotResampled(const MotionEvent& original, + const MotionEvent& notResampled); +}; + +MotionEvent ResamplerTest::buildMotionEvent(const int32_t action, const nsecs_t eventTime, + const std::vector<PointerBuilder>& pointerBuilders) { + MotionEventBuilder motionEventBuilder = MotionEventBuilder(action, AINPUT_SOURCE_CLASS_POINTER) + .downTime(0) + .eventTime(eventTime); + for (const PointerBuilder& pointerBuilder : pointerBuilders) { + motionEventBuilder.pointer(pointerBuilder); + } + return motionEventBuilder.build(); +} + +InputMessage ResamplerTest::createMessage(const uint32_t pointerCount, const nsecs_t eventTime, + const int32_t action, + const std::vector<PointerProperties>& properties, + const std::vector<PointerCoords>& coords) { + InputMessage message; + message.header.type = InputMessage::Type::MOTION; + message.body.motion.pointerCount = pointerCount; + message.body.motion.eventTime = eventTime; + message.body.motion.source = AINPUT_SOURCE_CLASS_POINTER; + message.body.motion.downTime = 0; + for (uint32_t i = 0; i < pointerCount; ++i) { + message.body.motion.pointers[i].properties = properties[i]; + message.body.motion.pointers[i].coords = coords[i]; + } + return message; +} + +void ResamplerTest::assertMotionEventMetaDataDidNotMutate(const MotionEvent& beforeCall, + const MotionEvent& afterCall) { + EXPECT_EQ(beforeCall.getDeviceId(), afterCall.getDeviceId()); + EXPECT_EQ(beforeCall.getAction(), afterCall.getAction()); + EXPECT_EQ(beforeCall.getActionButton(), afterCall.getActionButton()); + EXPECT_EQ(beforeCall.getButtonState(), afterCall.getButtonState()); + EXPECT_EQ(beforeCall.getFlags(), afterCall.getFlags()); + EXPECT_EQ(beforeCall.getEdgeFlags(), afterCall.getEdgeFlags()); + EXPECT_EQ(beforeCall.getClassification(), afterCall.getClassification()); + EXPECT_EQ(beforeCall.getPointerCount(), afterCall.getPointerCount()); + EXPECT_EQ(beforeCall.getMetaState(), afterCall.getMetaState()); + EXPECT_EQ(beforeCall.getSource(), afterCall.getSource()); + EXPECT_EQ(beforeCall.getXPrecision(), afterCall.getXPrecision()); + EXPECT_EQ(beforeCall.getYPrecision(), afterCall.getYPrecision()); + EXPECT_EQ(beforeCall.getDownTime(), afterCall.getDownTime()); + EXPECT_EQ(beforeCall.getDisplayId(), afterCall.getDisplayId()); +} + +void ResamplerTest::assertMotionEventIsResampledAndCoordsNear(const MotionEvent& original, + const MotionEvent& resampled, + const PointerCoords& expectedCoords) { + assertMotionEventMetaDataDidNotMutate(original, resampled); + const size_t originalSampleSize = original.getHistorySize() + 1; + const size_t resampledSampleSize = resampled.getHistorySize() + 1; + EXPECT_EQ(originalSampleSize + 1, resampledSampleSize); + const PointerCoords& resampledCoords = + resampled.getSamplePointerCoords()[resampled.getHistorySize()]; + EXPECT_TRUE(resampledCoords.isResampled); + EXPECT_NEAR(expectedCoords.getX(), resampledCoords.getX(), EPSILON); + EXPECT_NEAR(expectedCoords.getY(), resampledCoords.getY(), EPSILON); +} + +void ResamplerTest::assertMotionEventIsNotResampled(const MotionEvent& original, + const MotionEvent& notResampled) { + assertMotionEventMetaDataDidNotMutate(original, notResampled); + const size_t originalSampleSize = original.getHistorySize() + 1; + const size_t notResampledSampleSize = notResampled.getHistorySize() + 1; + EXPECT_EQ(originalSampleSize, notResampledSampleSize); +} + +TEST_F(ResamplerTest, SinglePointerNotEnoughDataToResample) { + MotionEvent motionEvent = + InputStream{{{5ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE, + .deviceId = 0}; + const MotionEvent originalMotionEvent = motionEvent; + mResampler->resampleMotionEvent(11ms, motionEvent, nullptr); + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, SinglePointerDifferentDeviceIdBetweenMotionEvents) { + MotionEvent motionFromFirstDevice = + InputStream{{{4ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}, + {8ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE, + .deviceId = 0}; + mResampler->resampleMotionEvent(10ms, motionFromFirstDevice, nullptr); + MotionEvent motionFromSecondDevice = + InputStream{{{11ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE, + .deviceId = 1}; + const MotionEvent originalMotionEvent = motionFromSecondDevice; + mResampler->resampleMotionEvent(12ms, motionFromSecondDevice, nullptr); + // The MotionEvent should not be resampled because the second event came from a different device + // than the previous event. + assertMotionEventIsNotResampled(originalMotionEvent, motionFromSecondDevice); +} + +// Increments of 16 ms for display refresh rate +// Increments of 6 ms for input frequency +// Resampling latency is known to be 5 ms +// Therefore, first resampling time will be 11 ms + +/** + * Timeline + * ----+----------------------+---------+---------+---------+---------- + * 0ms 10ms 11ms 15ms 16ms + * DOWN MOVE | MSG | + * resample frame + * Resampling occurs at 11ms. It is possible to interpolate because there is a sample available + * after the resample time. It is assumed that the InputMessage frequency is 100Hz, and the frame + * frequency is 60Hz. This means the time between InputMessage samples is 10ms, and the time between + * frames is ~16ms. Resample time is frameTime - RESAMPLE_LATENCY. The resampled sample must be the + * last one in the batch to consume. + */ +TEST_F(ResamplerTest, SinglePointerSingleSampleInterpolation) { + MotionEvent motionEvent = + InputStream{{{10ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + const InputMessage futureSample = + InputSample{15ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(11ms, motionEvent, &futureSample); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + Pointer{.id = 0, + .x = 1.2f, + .y = 1.2f, + .isResampled = true}); +} + +TEST_F(ResamplerTest, SinglePointerDeltaTooSmallInterpolation) { + MotionEvent motionEvent = + InputStream{{{10ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + const InputMessage futureSample = + InputSample{11ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(10'500'000ns, motionEvent, &futureSample); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +/** + * Tests extrapolation given two MotionEvents with a single sample. + */ +TEST_F(ResamplerTest, SinglePointerSingleSampleExtrapolation) { + MotionEvent previousMotionEvent = + InputStream{{{5ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + mResampler->resampleMotionEvent(10ms, previousMotionEvent, nullptr); + + MotionEvent motionEvent = + InputStream{{{10ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(11ms, motionEvent, nullptr); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + Pointer{.id = 0, + .x = 1.0f, + .y = 1.0f, + .isResampled = true}); + // Integrity of the whole motionEvent + // History size should increment by 1 + // Check if the resampled value is the last one + // Check if the resampleTime is correct + // Check if the PointerCoords are consistent with the other computations +} + +TEST_F(ResamplerTest, SinglePointerMultipleSampleInterpolation) { + MotionEvent motionEvent = + InputStream{{{5ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}, + {10ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + const InputMessage futureSample = + InputSample{15ms, {{.id = 0, .x = 3.0f, .y = 3.0f, .isResampled = false}}}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(11ms, motionEvent, &futureSample); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + Pointer{.id = 0, + .x = 2.2f, + .y = 2.2f, + .isResampled = true}); +} + +TEST_F(ResamplerTest, SinglePointerMultipleSampleExtrapolation) { + MotionEvent motionEvent = + InputStream{{{5ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}, + {10ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(11ms, motionEvent, nullptr); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + Pointer{.id = 0, + .x = 2.2f, + .y = 2.2f, + .isResampled = true}); +} + +TEST_F(ResamplerTest, SinglePointerDeltaTooSmallExtrapolation) { + MotionEvent motionEvent = + InputStream{{{9ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}, + {10ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(11ms, motionEvent, nullptr); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, SinglePointerDeltaTooLargeExtrapolation) { + MotionEvent motionEvent = + InputStream{{{5ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}, + {26ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(27ms, motionEvent, nullptr); + + assertMotionEventIsNotResampled(originalMotionEvent, motionEvent); +} + +TEST_F(ResamplerTest, SinglePointerResampleTimeTooFarExtrapolation) { + MotionEvent motionEvent = + InputStream{{{5ms, {{.id = 0, .x = 1.0f, .y = 1.0f, .isResampled = false}}}, + {25ms, {{.id = 0, .x = 2.0f, .y = 2.0f, .isResampled = false}}}}, + AMOTION_EVENT_ACTION_MOVE}; + + const MotionEvent originalMotionEvent = motionEvent; + + mResampler->resampleMotionEvent(43ms, motionEvent, nullptr); + + assertMotionEventIsResampledAndCoordsNear(originalMotionEvent, motionEvent, + Pointer{.id = 0, + .x = 2.4f, + .y = 2.4f, + .isResampled = true}); +} +} // namespace android |