diff options
author | 2023-06-29 13:19:01 -0700 | |
---|---|---|
committer | 2023-08-21 12:51:55 -0700 | |
commit | 52db47413680d078a678e0985bbb83edda6a915d (patch) | |
tree | f91d54b971f5d08b4c7b7716728c846e537038fa | |
parent | 55637826b9ac7931590396c8822eb3ff108c9ff2 (diff) |
Implement Stylus Prediction Metrics
Fills out the implementation and tests for MotionPredictorMetricsManager.
(Cherry pick of ag/23861881 from udc-qpr-dev)
Test: atest frameworks/native/libs/input/tests/MotionPredictorMetricsManager_test.cpp
Test: Manual testing on-device, computed metric values seem reasonable.
Bug: 268245099
Change-Id: Iec18415de9c3070f2b285c5c82f5a5e0ceaaf471
-rw-r--r-- | include/input/MotionPredictorMetricsManager.h | 182 | ||||
-rw-r--r-- | libs/input/Android.bp | 55 | ||||
-rw-r--r-- | libs/input/MotionPredictor.cpp | 5 | ||||
-rw-r--r-- | libs/input/MotionPredictorMetricsManager.cpp | 373 | ||||
-rw-r--r-- | libs/input/tests/Android.bp | 23 | ||||
-rw-r--r-- | libs/input/tests/MotionPredictorMetricsManager_test.cpp | 972 |
6 files changed, 1592 insertions, 18 deletions
diff --git a/include/input/MotionPredictorMetricsManager.h b/include/input/MotionPredictorMetricsManager.h index 6284f07117..12e50ba3b4 100644 --- a/include/input/MotionPredictorMetricsManager.h +++ b/include/input/MotionPredictorMetricsManager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 The Android Open Source Project + * Copyright 2023 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,23 +14,193 @@ * limitations under the License. */ -#include <utils/Timers.h> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <limits> +#include <optional> +#include <vector> + +#include <input/Input.h> // for MotionEvent +#include <input/RingBuffer.h> +#include <utils/Timers.h> // for nsecs_t + +#include "Eigen/Core" namespace android { /** * Class to handle computing and reporting metrics for MotionPredictor. * - * Currently an empty implementation, containing only the API. + * The public API provides two methods: `onRecord` and `onPredict`, which expect to receive the + * MotionEvents from the corresponding methods in MotionPredictor. + * + * This class stores AggregatedStrokeMetrics, updating them as new MotionEvents are passed in. When + * onRecord receives an UP or CANCEL event, this indicates the end of the stroke, and the final + * AtomFields are computed and reported to the stats library. + * + * If mMockLoggedAtomFields is set, the batch of AtomFields that are reported to the stats library + * for one stroke are also stored in mMockLoggedAtomFields at the time they're reported. */ class MotionPredictorMetricsManager { public: // Note: the MetricsManager assumes that the input interval equals the prediction interval. - MotionPredictorMetricsManager(nsecs_t /*predictionInterval*/, size_t /*maxNumPredictions*/) {} + MotionPredictorMetricsManager(nsecs_t predictionInterval, size_t maxNumPredictions); + + // This method should be called once for each call to MotionPredictor::record, receiving the + // forwarded MotionEvent argument. + void onRecord(const MotionEvent& inputEvent); + + // This method should be called once for each call to MotionPredictor::predict, receiving the + // MotionEvent that will be returned by MotionPredictor::predict. + void onPredict(const MotionEvent& predictionEvent); + + // Simple structs to hold relevant touch input information. Public so they can be used in tests. + + struct TouchPoint { + Eigen::Vector2f position; // (y, x) in pixels + float pressure; + }; + + struct GroundTruthPoint : TouchPoint { + nsecs_t timestamp; + }; + + struct PredictionPoint : TouchPoint { + // The timestamp of the last ground truth point when the prediction was made. + nsecs_t originTimestamp; + + nsecs_t targetTimestamp; + + // Order by targetTimestamp when sorting. + bool operator<(const PredictionPoint& other) const { + return this->targetTimestamp < other.targetTimestamp; + } + }; + + // Metrics aggregated so far for the current stroke. These are not the final fields to be + // reported in the atom (see AtomFields below), but rather an intermediate representation of the + // data that can be conveniently aggregated and from which the atom fields can be derived later. + // + // Displacement units are in pixels. + // + // "Along-trajectory error" is the dot product of the prediction error with the unit vector + // pointing towards the ground truth point whose timestamp corresponds to the prediction + // target timestamp, originating from the preceding ground truth point. + // + // "Off-trajectory error" is the component of the prediction error orthogonal to the + // "along-trajectory" unit vector described above. + // + // "High-velocity" errors are errors that are only accumulated when the velocity between the + // most recent two input events exceeds a certain threshold. + // + // "Scale-invariant errors" are the errors produced when the path length of the stroke is + // scaled to 1. (In other words, the error distances are normalized by the path length.) + struct AggregatedStrokeMetrics { + // General errors + float alongTrajectoryErrorSum = 0; + float alongTrajectorySumSquaredErrors = 0; + float offTrajectorySumSquaredErrors = 0; + float pressureSumSquaredErrors = 0; + size_t generalErrorsCount = 0; + + // High-velocity errors + float highVelocityAlongTrajectorySse = 0; + float highVelocityOffTrajectorySse = 0; + size_t highVelocityErrorsCount = 0; + + // Scale-invariant errors + float scaleInvariantAlongTrajectorySse = 0; + float scaleInvariantOffTrajectorySse = 0; + size_t scaleInvariantErrorsCount = 0; + }; + + // In order to explicitly indicate "no relevant data" for a metric, we report this + // large-magnitude negative sentinel value. (Most metrics are non-negative, so this value is + // completely unobtainable. For along-trajectory error mean, which can be negative, the + // magnitude makes it unobtainable in practice.) + static const int NO_DATA_SENTINEL = std::numeric_limits<int32_t>::min(); + + // Final metrics reported in the atom. + struct AtomFields { + int deltaTimeBucketMilliseconds = 0; + + // General errors + int alongTrajectoryErrorMeanMillipixels = NO_DATA_SENTINEL; + int alongTrajectoryErrorStdMillipixels = NO_DATA_SENTINEL; + int offTrajectoryRmseMillipixels = NO_DATA_SENTINEL; + int pressureRmseMilliunits = NO_DATA_SENTINEL; + + // High-velocity errors + int highVelocityAlongTrajectoryRmse = NO_DATA_SENTINEL; // millipixels + int highVelocityOffTrajectoryRmse = NO_DATA_SENTINEL; // millipixels + + // Scale-invariant errors + int scaleInvariantAlongTrajectoryRmse = NO_DATA_SENTINEL; // millipixels + int scaleInvariantOffTrajectoryRmse = NO_DATA_SENTINEL; // millipixels + }; + + // Allow tests to pass in a mock AtomFields pointer. + // + // When metrics are reported to the stats library on stroke end, they will also be written to + // mockLoggedAtomFields, overwriting existing data. The size of mockLoggedAtomFields will equal + // the number of calls to stats_write for that stroke. + void setMockLoggedAtomFields(std::vector<AtomFields>* mockLoggedAtomFields) { + mMockLoggedAtomFields = mockLoggedAtomFields; + } + +private: + // The interval between consecutive predictions' target timestamps. We assume that the input + // interval also equals this value. + const nsecs_t mPredictionInterval; + + // The maximum number of input frames into the future the model can predict. + // Used to perform time-bucketing of metrics. + const size_t mMaxNumPredictions; + + // History of mMaxNumPredictions + 1 ground truth points, used to compute scale-invariant + // error. (Also, the last two points are used to compute the ground truth trajectory.) + RingBuffer<GroundTruthPoint> mRecentGroundTruthPoints; + + // Predictions having a targetTimestamp after the most recent ground truth point's timestamp. + // Invariant: sorted in ascending order of targetTimestamp. + std::vector<PredictionPoint> mRecentPredictions; + + // Containers for the intermediate representation of stroke metrics and the final atom fields. + // These are indexed by the number of input frames into the future being predicted minus one, + // and always have size mMaxNumPredictions. + std::vector<AggregatedStrokeMetrics> mAggregatedMetrics; + std::vector<AtomFields> mAtomFields; + + // Non-owning pointer to the location of mock AtomFields. If present, will be filled with the + // values reported to stats_write on each batch of reported metrics. + // + // This pointer must remain valid as long as the MotionPredictorMetricsManager exists. + std::vector<AtomFields>* mMockLoggedAtomFields = nullptr; + + // Helper methods for the implementation of onRecord and onPredict. + + // Clears stored ground truth and prediction points, as well as all stored metrics for the + // current stroke. + void clearStrokeData(); + + // Adds the new ground truth point to mRecentGroundTruths, removes outdated predictions from + // mRecentPredictions, and updates the aggregated metrics to include the recent predictions that + // fuzzily match with the new ground truth point. + void incorporateNewGroundTruth(const GroundTruthPoint& groundTruthPoint); + + // Given a new prediction with targetTimestamp matching the latest ground truth point's + // timestamp, computes the corresponding metrics and updates mAggregatedMetrics. + void updateAggregatedMetrics(const PredictionPoint& predictionPoint); - void onRecord(const MotionEvent& /*inputEvent*/) {} + // Computes the atom fields to mAtomFields from the values in mAggregatedMetrics. + void computeAtomFields(); - void onPredict(const MotionEvent& /*predictionEvent*/) {} + // Reports the metrics given by the current data in mAtomFields: + // • If on an Android device, reports the metrics to stats_write. + // • If mMockLoggedAtomFields is present, it will be overwritten with logged metrics, with one + // AtomFields element per call to stats_write. + void reportMetrics(); }; } // namespace android diff --git a/libs/input/Android.bp b/libs/input/Android.bp index 757cde2074..8656b26837 100644 --- a/libs/input/Android.bp +++ b/libs/input/Android.bp @@ -139,6 +139,7 @@ cc_library { "KeyCharacterMap.cpp", "KeyLayoutMap.cpp", "MotionPredictor.cpp", + "MotionPredictorMetricsManager.cpp", "PrintTools.cpp", "PropertyMap.cpp", "TfLiteMotionPredictor.cpp", @@ -152,9 +153,13 @@ cc_library { header_libs: [ "flatbuffer_headers", "jni_headers", + "libeigen", "tensorflow_headers", ], - export_header_lib_headers: ["jni_headers"], + export_header_lib_headers: [ + "jni_headers", + "libeigen", + ], generated_headers: [ "cxx-bridge-header", @@ -206,6 +211,17 @@ cc_library { target: { android: { + export_shared_lib_headers: ["libbinder"], + + shared_libs: [ + "libutils", + "libbinder", + // Stats logging library and its dependencies. + "libstatslog_libinput", + "libstatsbootstrap", + "android.os.statsbootstrap_aidl-cpp", + ], + required: [ "motion_predictor_model_prebuilt", "motion_predictor_model_config", @@ -228,6 +244,43 @@ cc_library { }, } +// Use bootstrap version of stats logging library. +// libinput is a bootstrap process (starts early in the boot process), and thus can't use the normal +// `libstatslog` because that requires `libstatssocket`, which is only available later in the boot. +cc_library { + name: "libstatslog_libinput", + generated_sources: ["statslog_libinput.cpp"], + generated_headers: ["statslog_libinput.h"], + export_generated_headers: ["statslog_libinput.h"], + shared_libs: [ + "libbinder", + "libstatsbootstrap", + "libutils", + "android.os.statsbootstrap_aidl-cpp", + ], +} + +genrule { + name: "statslog_libinput.h", + tools: ["stats-log-api-gen"], + cmd: "$(location stats-log-api-gen) --header $(genDir)/statslog_libinput.h --module libinput" + + " --namespace android,stats,libinput --bootstrap", + out: [ + "statslog_libinput.h", + ], +} + +genrule { + name: "statslog_libinput.cpp", + tools: ["stats-log-api-gen"], + cmd: "$(location stats-log-api-gen) --cpp $(genDir)/statslog_libinput.cpp --module libinput" + + " --namespace android,stats,libinput --importHeader statslog_libinput.h" + + " --bootstrap", + out: [ + "statslog_libinput.cpp", + ], +} + cc_defaults { name: "libinput_fuzz_defaults", cpp_std: "c++20", diff --git a/libs/input/MotionPredictor.cpp b/libs/input/MotionPredictor.cpp index 0961a9d975..b5a5e720e9 100644 --- a/libs/input/MotionPredictor.cpp +++ b/libs/input/MotionPredictor.cpp @@ -137,10 +137,7 @@ android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { // Pass input event to the MetricsManager. if (!mMetricsManager) { - mMetricsManager = - std::make_optional<MotionPredictorMetricsManager>(mModel->config() - .predictionInterval, - mModel->outputLength()); + mMetricsManager.emplace(mModel->config().predictionInterval, mModel->outputLength()); } mMetricsManager->onRecord(event); diff --git a/libs/input/MotionPredictorMetricsManager.cpp b/libs/input/MotionPredictorMetricsManager.cpp new file mode 100644 index 0000000000..67b103290f --- /dev/null +++ b/libs/input/MotionPredictorMetricsManager.cpp @@ -0,0 +1,373 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "MotionPredictorMetricsManager" + +#include <input/MotionPredictorMetricsManager.h> + +#include <algorithm> + +#include <android-base/logging.h> + +#include "Eigen/Core" +#include "Eigen/Geometry" + +#ifdef __ANDROID__ +#include <statslog_libinput.h> +#endif + +namespace android { +namespace { + +inline constexpr int NANOS_PER_SECOND = 1'000'000'000; // nanoseconds per second +inline constexpr int NANOS_PER_MILLIS = 1'000'000; // nanoseconds per millisecond + +// Velocity threshold at which we report "high-velocity" metrics, in pixels per second. +// This value was selected from manual experimentation, as a threshold that separates "fast" +// (semi-sloppy) handwriting from more careful medium to slow handwriting. +inline constexpr float HIGH_VELOCITY_THRESHOLD = 1100.0; + +// Small value to add to the path length when computing scale-invariant error to avoid division by +// zero. +inline constexpr float PATH_LENGTH_EPSILON = 0.001; + +} // namespace + +MotionPredictorMetricsManager::MotionPredictorMetricsManager(nsecs_t predictionInterval, + size_t maxNumPredictions) + : mPredictionInterval(predictionInterval), + mMaxNumPredictions(maxNumPredictions), + mRecentGroundTruthPoints(maxNumPredictions + 1), + mAggregatedMetrics(maxNumPredictions), + mAtomFields(maxNumPredictions) {} + +void MotionPredictorMetricsManager::onRecord(const MotionEvent& inputEvent) { + // Convert MotionEvent to GroundTruthPoint. + const PointerCoords* coords = inputEvent.getRawPointerCoords(/*pointerIndex=*/0); + LOG_ALWAYS_FATAL_IF(coords == nullptr); + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f{coords->getY(), + coords->getX()}, + .pressure = + inputEvent.getPressure(/*pointerIndex=*/0)}, + .timestamp = inputEvent.getEventTime()}; + + // Handle event based on action type. + switch (inputEvent.getActionMasked()) { + case AMOTION_EVENT_ACTION_DOWN: { + clearStrokeData(); + incorporateNewGroundTruth(groundTruthPoint); + break; + } + case AMOTION_EVENT_ACTION_MOVE: { + incorporateNewGroundTruth(groundTruthPoint); + break; + } + case AMOTION_EVENT_ACTION_UP: + case AMOTION_EVENT_ACTION_CANCEL: { + // Only expect meaningful predictions when given at least two input points. + if (mRecentGroundTruthPoints.size() >= 2) { + computeAtomFields(); + reportMetrics(); + break; + } + } + } +} + +// Adds new predictions to mRecentPredictions and maintains the invariant that elements are +// sorted in ascending order of targetTimestamp. +void MotionPredictorMetricsManager::onPredict(const MotionEvent& predictionEvent) { + for (size_t i = 0; i < predictionEvent.getHistorySize() + 1; ++i) { + // Convert MotionEvent to PredictionPoint. + const PointerCoords* coords = + predictionEvent.getHistoricalRawPointerCoords(/*pointerIndex=*/0, i); + LOG_ALWAYS_FATAL_IF(coords == nullptr); + const nsecs_t targetTimestamp = predictionEvent.getHistoricalEventTime(i); + mRecentPredictions.push_back( + PredictionPoint{{.position = Eigen::Vector2f{coords->getY(), coords->getX()}, + .pressure = + predictionEvent.getHistoricalPressure(/*pointerIndex=*/0, + i)}, + .originTimestamp = mRecentGroundTruthPoints.back().timestamp, + .targetTimestamp = targetTimestamp}); + } + + std::sort(mRecentPredictions.begin(), mRecentPredictions.end()); +} + +void MotionPredictorMetricsManager::clearStrokeData() { + mRecentGroundTruthPoints.clear(); + mRecentPredictions.clear(); + std::fill(mAggregatedMetrics.begin(), mAggregatedMetrics.end(), AggregatedStrokeMetrics{}); + std::fill(mAtomFields.begin(), mAtomFields.end(), AtomFields{}); +} + +void MotionPredictorMetricsManager::incorporateNewGroundTruth( + const GroundTruthPoint& groundTruthPoint) { + // Note: this removes the oldest point if `mRecentGroundTruthPoints` is already at capacity. + mRecentGroundTruthPoints.pushBack(groundTruthPoint); + + // Remove outdated predictions – those that can never be matched with the current or any future + // ground truth points. We use fuzzy association for the timestamps here, because ground truth + // and prediction timestamps may not be perfectly synchronized. + const nsecs_t fuzzy_association_time_delta = mPredictionInterval / 4; + const auto firstCurrentIt = + std::find_if(mRecentPredictions.begin(), mRecentPredictions.end(), + [&groundTruthPoint, + fuzzy_association_time_delta](const PredictionPoint& prediction) { + return prediction.targetTimestamp > + groundTruthPoint.timestamp - fuzzy_association_time_delta; + }); + mRecentPredictions.erase(mRecentPredictions.begin(), firstCurrentIt); + + // Fuzzily match the new ground truth's timestamp to recent predictions' targetTimestamp and + // update the corresponding metrics. + for (const PredictionPoint& prediction : mRecentPredictions) { + if ((prediction.targetTimestamp > + groundTruthPoint.timestamp - fuzzy_association_time_delta) && + (prediction.targetTimestamp < + groundTruthPoint.timestamp + fuzzy_association_time_delta)) { + updateAggregatedMetrics(prediction); + } + } +} + +void MotionPredictorMetricsManager::updateAggregatedMetrics( + const PredictionPoint& predictionPoint) { + if (mRecentGroundTruthPoints.size() < 2) { + return; + } + + const GroundTruthPoint& latestGroundTruthPoint = mRecentGroundTruthPoints.back(); + const GroundTruthPoint& previousGroundTruthPoint = + mRecentGroundTruthPoints[mRecentGroundTruthPoints.size() - 2]; + // Calculate prediction error vector. + const Eigen::Vector2f groundTruthTrajectory = + latestGroundTruthPoint.position - previousGroundTruthPoint.position; + const Eigen::Vector2f predictionTrajectory = + predictionPoint.position - previousGroundTruthPoint.position; + const Eigen::Vector2f predictionError = predictionTrajectory - groundTruthTrajectory; + + // By default, prediction error counts fully as both off-trajectory and along-trajectory error. + // This serves as the fallback when the two most recent ground truth points are equal. + const float predictionErrorNorm = predictionError.norm(); + float alongTrajectoryError = predictionErrorNorm; + float offTrajectoryError = predictionErrorNorm; + if (groundTruthTrajectory.squaredNorm() > 0) { + // Rotate the prediction error vector by the angle of the ground truth trajectory vector. + // This yields a vector whose first component is the along-trajectory error and whose + // second component is the off-trajectory error. + const float theta = std::atan2(groundTruthTrajectory[1], groundTruthTrajectory[0]); + const Eigen::Vector2f rotatedPredictionError = Eigen::Rotation2Df(-theta) * predictionError; + alongTrajectoryError = rotatedPredictionError[0]; + offTrajectoryError = rotatedPredictionError[1]; + } + + // Compute the multiple of mPredictionInterval nearest to the amount of time into the + // future being predicted. This serves as the time bucket index into mAggregatedMetrics. + const float timestampDeltaFloat = + static_cast<float>(predictionPoint.targetTimestamp - predictionPoint.originTimestamp); + const size_t tIndex = + static_cast<size_t>(std::round(timestampDeltaFloat / mPredictionInterval - 1)); + + // Aggregate values into "general errors". + mAggregatedMetrics[tIndex].alongTrajectoryErrorSum += alongTrajectoryError; + mAggregatedMetrics[tIndex].alongTrajectorySumSquaredErrors += + alongTrajectoryError * alongTrajectoryError; + mAggregatedMetrics[tIndex].offTrajectorySumSquaredErrors += + offTrajectoryError * offTrajectoryError; + const float pressureError = predictionPoint.pressure - latestGroundTruthPoint.pressure; + mAggregatedMetrics[tIndex].pressureSumSquaredErrors += pressureError * pressureError; + ++mAggregatedMetrics[tIndex].generalErrorsCount; + + // Aggregate values into high-velocity metrics, if we are in one of the last two time buckets + // and the velocity is above the threshold. Velocity here is measured in pixels per second. + const float velocity = groundTruthTrajectory.norm() / + (static_cast<float>(latestGroundTruthPoint.timestamp - + previousGroundTruthPoint.timestamp) / + NANOS_PER_SECOND); + if ((tIndex + 2 >= mMaxNumPredictions) && (velocity > HIGH_VELOCITY_THRESHOLD)) { + mAggregatedMetrics[tIndex].highVelocityAlongTrajectorySse += + alongTrajectoryError * alongTrajectoryError; + mAggregatedMetrics[tIndex].highVelocityOffTrajectorySse += + offTrajectoryError * offTrajectoryError; + ++mAggregatedMetrics[tIndex].highVelocityErrorsCount; + } + + // Compute path length for scale-invariant errors. + float pathLength = 0; + for (size_t i = 1; i < mRecentGroundTruthPoints.size(); ++i) { + pathLength += + (mRecentGroundTruthPoints[i].position - mRecentGroundTruthPoints[i - 1].position) + .norm(); + } + // Avoid overweighting errors at the beginning of a stroke: compute the path length as if there + // were a full ground truth history by filling in missing segments with the average length. + // Note: the "- 1" is needed to translate from number of endpoints to number of segments. + pathLength *= static_cast<float>(mRecentGroundTruthPoints.capacity() - 1) / + (mRecentGroundTruthPoints.size() - 1); + pathLength += PATH_LENGTH_EPSILON; // Ensure path length is nonzero (>= PATH_LENGTH_EPSILON). + + // Compute and aggregate scale-invariant errors. + const float scaleInvariantAlongTrajectoryError = alongTrajectoryError / pathLength; + const float scaleInvariantOffTrajectoryError = offTrajectoryError / pathLength; + mAggregatedMetrics[tIndex].scaleInvariantAlongTrajectorySse += + scaleInvariantAlongTrajectoryError * scaleInvariantAlongTrajectoryError; + mAggregatedMetrics[tIndex].scaleInvariantOffTrajectorySse += + scaleInvariantOffTrajectoryError * scaleInvariantOffTrajectoryError; + ++mAggregatedMetrics[tIndex].scaleInvariantErrorsCount; +} + +void MotionPredictorMetricsManager::computeAtomFields() { + for (size_t i = 0; i < mAggregatedMetrics.size(); ++i) { + if (mAggregatedMetrics[i].generalErrorsCount == 0) { + // We have not received data corresponding to metrics for this time bucket. + continue; + } + + mAtomFields[i].deltaTimeBucketMilliseconds = + static_cast<int>(mPredictionInterval / NANOS_PER_MILLIS * (i + 1)); + + // Note: we need the "* 1000"s below because we report values in integral milli-units. + + { // General errors: reported for every time bucket. + const float alongTrajectoryErrorMean = mAggregatedMetrics[i].alongTrajectoryErrorSum / + mAggregatedMetrics[i].generalErrorsCount; + mAtomFields[i].alongTrajectoryErrorMeanMillipixels = + static_cast<int>(alongTrajectoryErrorMean * 1000); + + const float alongTrajectoryMse = mAggregatedMetrics[i].alongTrajectorySumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount; + // Take the max with 0 to avoid negative values caused by numerical instability. + const float alongTrajectoryErrorVariance = + std::max(0.0f, + alongTrajectoryMse - + alongTrajectoryErrorMean * alongTrajectoryErrorMean); + const float alongTrajectoryErrorStd = std::sqrt(alongTrajectoryErrorVariance); + mAtomFields[i].alongTrajectoryErrorStdMillipixels = + static_cast<int>(alongTrajectoryErrorStd * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].offTrajectorySumSquaredErrors < 0, + "mAggregatedMetrics[%zu].offTrajectorySumSquaredErrors = %f should " + "not be negative", + i, mAggregatedMetrics[i].offTrajectorySumSquaredErrors); + const float offTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].offTrajectorySumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount); + mAtomFields[i].offTrajectoryRmseMillipixels = + static_cast<int>(offTrajectoryRmse * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].pressureSumSquaredErrors < 0, + "mAggregatedMetrics[%zu].pressureSumSquaredErrors = %f should not " + "be negative", + i, mAggregatedMetrics[i].pressureSumSquaredErrors); + const float pressureRmse = std::sqrt(mAggregatedMetrics[i].pressureSumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount); + mAtomFields[i].pressureRmseMilliunits = static_cast<int>(pressureRmse * 1000); + } + + // High-velocity errors: reported only for last two time buckets. + // Check if we are in one of the last two time buckets, and there is high-velocity data. + if ((i + 2 >= mMaxNumPredictions) && (mAggregatedMetrics[i].highVelocityErrorsCount > 0)) { + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].highVelocityAlongTrajectorySse < 0, + "mAggregatedMetrics[%zu].highVelocityAlongTrajectorySse = %f " + "should not be negative", + i, mAggregatedMetrics[i].highVelocityAlongTrajectorySse); + const float alongTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].highVelocityAlongTrajectorySse / + mAggregatedMetrics[i].highVelocityErrorsCount); + mAtomFields[i].highVelocityAlongTrajectoryRmse = + static_cast<int>(alongTrajectoryRmse * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].highVelocityOffTrajectorySse < 0, + "mAggregatedMetrics[%zu].highVelocityOffTrajectorySse = %f should " + "not be negative", + i, mAggregatedMetrics[i].highVelocityOffTrajectorySse); + const float offTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].highVelocityOffTrajectorySse / + mAggregatedMetrics[i].highVelocityErrorsCount); + mAtomFields[i].highVelocityOffTrajectoryRmse = + static_cast<int>(offTrajectoryRmse * 1000); + } + + // Scale-invariant errors: reported only for the last time bucket, where the values + // represent an average across all time buckets. + if (i + 1 == mMaxNumPredictions) { + // Compute error averages. + float alongTrajectoryRmseSum = 0; + float offTrajectoryRmseSum = 0; + for (size_t j = 0; j < mAggregatedMetrics.size(); ++j) { + // If we have general errors (checked above), we should always also have + // scale-invariant errors. + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantErrorsCount == 0, + "mAggregatedMetrics[%zu].scaleInvariantErrorsCount is 0", j); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse < 0, + "mAggregatedMetrics[%zu].scaleInvariantAlongTrajectorySse = %f " + "should not be negative", + j, mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse); + alongTrajectoryRmseSum += + std::sqrt(mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse / + mAggregatedMetrics[j].scaleInvariantErrorsCount); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantOffTrajectorySse < 0, + "mAggregatedMetrics[%zu].scaleInvariantOffTrajectorySse = %f " + "should not be negative", + j, mAggregatedMetrics[j].scaleInvariantOffTrajectorySse); + offTrajectoryRmseSum += + std::sqrt(mAggregatedMetrics[j].scaleInvariantOffTrajectorySse / + mAggregatedMetrics[j].scaleInvariantErrorsCount); + } + + const float averageAlongTrajectoryRmse = + alongTrajectoryRmseSum / mAggregatedMetrics.size(); + mAtomFields.back().scaleInvariantAlongTrajectoryRmse = + static_cast<int>(averageAlongTrajectoryRmse * 1000); + + const float averageOffTrajectoryRmse = offTrajectoryRmseSum / mAggregatedMetrics.size(); + mAtomFields.back().scaleInvariantOffTrajectoryRmse = + static_cast<int>(averageOffTrajectoryRmse * 1000); + } + } +} + +void MotionPredictorMetricsManager::reportMetrics() { + // Report one atom for each time bucket. + for (size_t i = 0; i < mAtomFields.size(); ++i) { + // Call stats_write logging function only on Android targets (not supported on host). +#ifdef __ANDROID__ + android::stats::libinput:: + stats_write(android::stats::libinput::STYLUS_PREDICTION_METRICS_REPORTED, + /*stylus_vendor_id=*/0, + /*stylus_product_id=*/0, mAtomFields[i].deltaTimeBucketMilliseconds, + mAtomFields[i].alongTrajectoryErrorMeanMillipixels, + mAtomFields[i].alongTrajectoryErrorStdMillipixels, + mAtomFields[i].offTrajectoryRmseMillipixels, + mAtomFields[i].pressureRmseMilliunits, + mAtomFields[i].highVelocityAlongTrajectoryRmse, + mAtomFields[i].highVelocityOffTrajectoryRmse, + mAtomFields[i].scaleInvariantAlongTrajectoryRmse, + mAtomFields[i].scaleInvariantOffTrajectoryRmse); +#endif + } + + // Set mock atom fields, if available. + if (mMockLoggedAtomFields != nullptr) { + *mMockLoggedAtomFields = mAtomFields; + } +} + +} // namespace android diff --git a/libs/input/tests/Android.bp b/libs/input/tests/Android.bp index 86b996b3b6..e7224ff752 100644 --- a/libs/input/tests/Android.bp +++ b/libs/input/tests/Android.bp @@ -20,6 +20,7 @@ cc_test { "InputPublisherAndConsumer_test.cpp", "InputVerifier_test.cpp", "MotionPredictor_test.cpp", + "MotionPredictorMetricsManager_test.cpp", "RingBuffer_test.cpp", "TfLiteMotionPredictor_test.cpp", "TouchResampling_test.cpp", @@ -52,13 +53,6 @@ cc_test { undefined: true, }, }, - target: { - host: { - sanitize: { - address: true, - }, - }, - }, shared_libs: [ "libbase", "libbinder", @@ -77,6 +71,21 @@ cc_test { unit_test: true, }, test_suites: ["device-tests"], + target: { + host: { + sanitize: { + address: true, + }, + }, + android: { + static_libs: [ + // Stats logging library and its dependencies. + "libstatslog_libinput", + "libstatsbootstrap", + "android.os.statsbootstrap_aidl-cpp", + ], + }, + }, } // NOTE: This is a compile time test, and does not need to be diff --git a/libs/input/tests/MotionPredictorMetricsManager_test.cpp b/libs/input/tests/MotionPredictorMetricsManager_test.cpp new file mode 100644 index 0000000000..b420a5a4e7 --- /dev/null +++ b/libs/input/tests/MotionPredictorMetricsManager_test.cpp @@ -0,0 +1,972 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <input/MotionPredictor.h> + +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <numeric> +#include <vector> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <input/InputEventBuilders.h> +#include <utils/Timers.h> // for nsecs_t + +#include "Eigen/Core" +#include "Eigen/Geometry" + +namespace android { +namespace { + +using ::testing::FloatNear; +using ::testing::Matches; + +using GroundTruthPoint = MotionPredictorMetricsManager::GroundTruthPoint; +using PredictionPoint = MotionPredictorMetricsManager::PredictionPoint; +using AtomFields = MotionPredictorMetricsManager::AtomFields; + +inline constexpr int NANOS_PER_MILLIS = 1'000'000; + +inline constexpr nsecs_t TEST_INITIAL_TIMESTAMP = 1'000'000'000; +inline constexpr size_t TEST_MAX_NUM_PREDICTIONS = 5; +inline constexpr nsecs_t TEST_PREDICTION_INTERVAL_NANOS = 12'500'000 / 3; // 1 / (240 hz) +inline constexpr int NO_DATA_SENTINEL = MotionPredictorMetricsManager::NO_DATA_SENTINEL; + +// Parameters: +// • arg: Eigen::Vector2f +// • target: Eigen::Vector2f +// • epsilon: float +MATCHER_P2(Vector2fNear, target, epsilon, "") { + return Matches(FloatNear(target[0], epsilon))(arg[0]) && + Matches(FloatNear(target[1], epsilon))(arg[1]); +} + +// Parameters: +// • arg: PredictionPoint +// • target: PredictionPoint +// • epsilon: float +MATCHER_P2(PredictionPointNear, target, epsilon, "") { + if (!Matches(Vector2fNear(target.position, epsilon))(arg.position)) { + *result_listener << "Position mismatch. Actual: (" << arg.position[0] << ", " + << arg.position[1] << "), expected: (" << target.position[0] << ", " + << target.position[1] << ")"; + return false; + } + if (!Matches(FloatNear(target.pressure, epsilon))(arg.pressure)) { + *result_listener << "Pressure mismatch. Actual: " << arg.pressure + << ", expected: " << target.pressure; + return false; + } + if (arg.originTimestamp != target.originTimestamp) { + *result_listener << "Origin timestamp mismatch. Actual: " << arg.originTimestamp + << ", expected: " << target.originTimestamp; + return false; + } + if (arg.targetTimestamp != target.targetTimestamp) { + *result_listener << "Target timestamp mismatch. Actual: " << arg.targetTimestamp + << ", expected: " << target.targetTimestamp; + return false; + } + return true; +} + +// --- Mathematical helper functions. --- + +template <typename T> +T average(std::vector<T> values) { + return std::accumulate(values.begin(), values.end(), T{}) / static_cast<T>(values.size()); +} + +template <typename T> +T standardDeviation(std::vector<T> values) { + T mean = average(values); + T accumulator = {}; + for (const T value : values) { + accumulator += value * value - mean * mean; + } + // Take the max with 0 to avoid negative values caused by numerical instability. + return std::sqrt(std::max(T{}, accumulator) / static_cast<T>(values.size())); +} + +template <typename T> +T rmse(std::vector<T> errors) { + T sse = {}; + for (const T error : errors) { + sse += error * error; + } + return std::sqrt(sse / static_cast<T>(errors.size())); +} + +TEST(MathematicalHelperFunctionTest, Average) { + std::vector<float> values{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + EXPECT_EQ(5.5f, average(values)); +} + +TEST(MathematicalHelperFunctionTest, StandardDeviation) { + // https://www.calculator.net/standard-deviation-calculator.html?numberinputs=10%2C+12%2C+23%2C+23%2C+16%2C+23%2C+21%2C+16 + std::vector<float> values{10, 12, 23, 23, 16, 23, 21, 16}; + EXPECT_FLOAT_EQ(4.8989794855664f, standardDeviation(values)); +} + +TEST(MathematicalHelperFunctionTest, Rmse) { + std::vector<float> errors{1, 5, 7, 7, 8, 20}; + EXPECT_FLOAT_EQ(9.899494937f, rmse(errors)); +} + +// --- MotionEvent-related helper functions. --- + +// Creates a MotionEvent corresponding to the given GroundTruthPoint. +MotionEvent makeMotionEvent(const GroundTruthPoint& groundTruthPoint) { + // Build single pointer of type STYLUS, with coordinates from groundTruthPoint. + PointerBuilder pointerBuilder = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(groundTruthPoint.position[1]) + .y(groundTruthPoint.position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, groundTruthPoint.pressure); + return MotionEventBuilder(/*action=*/AMOTION_EVENT_ACTION_MOVE, + /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .eventTime(groundTruthPoint.timestamp) + .pointer(pointerBuilder) + .build(); +} + +// Creates a MotionEvent corresponding to the given sequence of PredictionPoints. +MotionEvent makeMotionEvent(const std::vector<PredictionPoint>& predictionPoints) { + // Build single pointer of type STYLUS, with coordinates from first prediction point. + PointerBuilder pointerBuilder = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(predictionPoints[0].position[1]) + .y(predictionPoints[0].position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, predictionPoints[0].pressure); + MotionEvent predictionEvent = + MotionEventBuilder( + /*action=*/AMOTION_EVENT_ACTION_MOVE, /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .eventTime(predictionPoints[0].targetTimestamp) + .pointer(pointerBuilder) + .build(); + for (size_t i = 1; i < predictionPoints.size(); ++i) { + PointerCoords coords = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(predictionPoints[i].position[1]) + .y(predictionPoints[i].position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, predictionPoints[i].pressure) + .buildCoords(); + predictionEvent.addSample(predictionPoints[i].targetTimestamp, &coords); + } + return predictionEvent; +} + +// Creates a MotionEvent corresponding to a stylus lift (UP) ground truth event. +MotionEvent makeLiftMotionEvent() { + return MotionEventBuilder(/*action=*/AMOTION_EVENT_ACTION_UP, + /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .pointer(PointerBuilder(/*id=*/0, ToolType::STYLUS)) + .build(); +} + +TEST(MakeMotionEventTest, MakeGroundTruthMotionEvent) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10.0f, 20.0f), + .pressure = 0.6f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const MotionEvent groundTruthMotionEvent = makeMotionEvent(groundTruthPoint); + + ASSERT_EQ(1u, groundTruthMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(0u, groundTruthMotionEvent.getHistorySize()); + EXPECT_EQ(groundTruthPoint.position[0], groundTruthMotionEvent.getRawPointerCoords(0)->getY()); + EXPECT_EQ(groundTruthPoint.position[1], groundTruthMotionEvent.getRawPointerCoords(0)->getX()); + EXPECT_EQ(groundTruthPoint.pressure, + groundTruthMotionEvent.getRawPointerCoords(0)->getAxisValue( + AMOTION_EVENT_AXIS_PRESSURE)); + EXPECT_EQ(AMOTION_EVENT_ACTION_MOVE, groundTruthMotionEvent.getAction()); +} + +TEST(MakeMotionEventTest, MakePredictionMotionEvent) { + const nsecs_t originTimestamp = TEST_INITIAL_TIMESTAMP; + const std::vector<PredictionPoint> + predictionPoints{{{.position = Eigen::Vector2f(10.0f, 20.0f), .pressure = 0.6f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 5 * NANOS_PER_MILLIS}, + {{.position = Eigen::Vector2f(11.0f, 22.0f), .pressure = 0.5f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 10 * NANOS_PER_MILLIS}, + {{.position = Eigen::Vector2f(12.0f, 24.0f), .pressure = 0.4f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 15 * NANOS_PER_MILLIS}}; + const MotionEvent predictionMotionEvent = makeMotionEvent(predictionPoints); + + ASSERT_EQ(1u, predictionMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(predictionPoints.size(), predictionMotionEvent.getHistorySize() + 1); + for (size_t i = 0; i < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const PointerCoords coords = *predictionMotionEvent.getHistoricalRawPointerCoords( + /*pointerIndex=*/0, /*historicalIndex=*/i); + EXPECT_EQ(predictionPoints[i].position[0], coords.getY()); + EXPECT_EQ(predictionPoints[i].position[1], coords.getX()); + EXPECT_EQ(predictionPoints[i].pressure, coords.getAxisValue(AMOTION_EVENT_AXIS_PRESSURE)); + // Note: originTimestamp is discarded when converting PredictionPoint to MotionEvent. + EXPECT_EQ(predictionPoints[i].targetTimestamp, + predictionMotionEvent.getHistoricalEventTime(i)); + EXPECT_EQ(AMOTION_EVENT_ACTION_MOVE, predictionMotionEvent.getAction()); + } +} + +TEST(MakeMotionEventTest, MakeLiftMotionEvent) { + const MotionEvent liftMotionEvent = makeLiftMotionEvent(); + ASSERT_EQ(1u, liftMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(0u, liftMotionEvent.getHistorySize()); + EXPECT_EQ(AMOTION_EVENT_ACTION_UP, liftMotionEvent.getAction()); +} + +// --- Ground-truth-generation helper functions. --- + +std::vector<GroundTruthPoint> generateConstantGroundTruthPoints( + const GroundTruthPoint& groundTruthPoint, size_t numPoints) { + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = groundTruthPoint.timestamp; + for (size_t i = 0; i < numPoints; ++i) { + groundTruthPoints.emplace_back(groundTruthPoint); + groundTruthPoints.back().timestamp = timestamp; + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + return groundTruthPoints; +} + +// This function uses the coordinate system (y, x), with +y pointing downwards and +x pointing +// rightwards. Angles are measured counterclockwise from down (+y). +std::vector<GroundTruthPoint> generateCircularArcGroundTruthPoints(Eigen::Vector2f initialPosition, + float initialAngle, + float velocity, + float turningAngle, + size_t numPoints) { + std::vector<GroundTruthPoint> groundTruthPoints; + // Create first point. + if (numPoints > 0) { + groundTruthPoints.push_back({{.position = initialPosition, .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}); + } + float trajectoryAngle = initialAngle; // measured counterclockwise from +y axis. + for (size_t i = 1; i < numPoints; ++i) { + const Eigen::Vector2f trajectory = + Eigen::Rotation2D(trajectoryAngle) * Eigen::Vector2f(1, 0); + groundTruthPoints.push_back( + {{.position = groundTruthPoints.back().position + velocity * trajectory, + .pressure = 0.0f}, + .timestamp = groundTruthPoints.back().timestamp + TEST_PREDICTION_INTERVAL_NANOS}); + trajectoryAngle += turningAngle; + } + return groundTruthPoints; +} + +TEST(GenerateConstantGroundTruthPointsTest, BasicTest) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10, 20), .pressure = 0.3f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(groundTruthPoint, /*numPoints=*/3); + + ASSERT_EQ(3u, groundTruthPoints.size()); + // First point. + EXPECT_EQ(groundTruthPoints[0].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[0].pressure, groundTruthPoint.pressure); + EXPECT_EQ(groundTruthPoints[0].timestamp, groundTruthPoint.timestamp); + // Second point. + EXPECT_EQ(groundTruthPoints[1].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[1].pressure, groundTruthPoint.pressure); + EXPECT_GT(groundTruthPoints[1].timestamp, groundTruthPoints[0].timestamp); + // Third point. + EXPECT_EQ(groundTruthPoints[2].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[2].pressure, groundTruthPoint.pressure); + EXPECT_GT(groundTruthPoints[2].timestamp, groundTruthPoints[1].timestamp); +} + +TEST(GenerateCircularArcGroundTruthTest, StraightLineUpwards) { + const std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(0, 0), + /*initialAngle=*/M_PI, + /*velocity=*/1.0f, + /*turningAngle=*/0.0f, + /*numPoints=*/3); + + ASSERT_EQ(3u, groundTruthPoints.size()); + EXPECT_THAT(groundTruthPoints[0].position, Vector2fNear(Eigen::Vector2f(0, 0), 1e-6)); + EXPECT_THAT(groundTruthPoints[1].position, Vector2fNear(Eigen::Vector2f(-1, 0), 1e-6)); + EXPECT_THAT(groundTruthPoints[2].position, Vector2fNear(Eigen::Vector2f(-2, 0), 1e-6)); + // Check that timestamps are increasing between consecutive ground truth points. + EXPECT_GT(groundTruthPoints[1].timestamp, groundTruthPoints[0].timestamp); + EXPECT_GT(groundTruthPoints[2].timestamp, groundTruthPoints[1].timestamp); +} + +TEST(GenerateCircularArcGroundTruthTest, CounterclockwiseSquare) { + // Generate points in a counterclockwise unit square starting pointing right. + const std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(10, 100), + /*initialAngle=*/M_PI_2, + /*velocity=*/1.0f, + /*turningAngle=*/M_PI_2, + /*numPoints=*/5); + + ASSERT_EQ(5u, groundTruthPoints.size()); + EXPECT_THAT(groundTruthPoints[0].position, Vector2fNear(Eigen::Vector2f(10, 100), 1e-6)); + EXPECT_THAT(groundTruthPoints[1].position, Vector2fNear(Eigen::Vector2f(10, 101), 1e-6)); + EXPECT_THAT(groundTruthPoints[2].position, Vector2fNear(Eigen::Vector2f(9, 101), 1e-6)); + EXPECT_THAT(groundTruthPoints[3].position, Vector2fNear(Eigen::Vector2f(9, 100), 1e-6)); + EXPECT_THAT(groundTruthPoints[4].position, Vector2fNear(Eigen::Vector2f(10, 100), 1e-6)); +} + +// --- Prediction-generation helper functions. --- + +// Creates a sequence of predictions with values equal to those of the given GroundTruthPoint. +std::vector<PredictionPoint> generateConstantPredictions(const GroundTruthPoint& groundTruthPoint) { + std::vector<PredictionPoint> predictions; + nsecs_t predictionTimestamp = groundTruthPoint.timestamp + TEST_PREDICTION_INTERVAL_NANOS; + for (size_t j = 0; j < TEST_MAX_NUM_PREDICTIONS; ++j) { + predictions.push_back(PredictionPoint{{.position = groundTruthPoint.position, + .pressure = groundTruthPoint.pressure}, + .originTimestamp = groundTruthPoint.timestamp, + .targetTimestamp = predictionTimestamp}); + predictionTimestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + return predictions; +} + +// Generates TEST_MAX_NUM_PREDICTIONS predictions from the given most recent two ground truth points +// by linear extrapolation of position and pressure. The interval between consecutive predictions' +// timestamps is TEST_PREDICTION_INTERVAL_NANOS. +std::vector<PredictionPoint> generatePredictionsByLinearExtrapolation( + const GroundTruthPoint& firstGroundTruth, const GroundTruthPoint& secondGroundTruth) { + // Precompute deltas. + const Eigen::Vector2f trajectory = secondGroundTruth.position - firstGroundTruth.position; + const float deltaPressure = secondGroundTruth.pressure - firstGroundTruth.pressure; + // Compute predictions. + std::vector<PredictionPoint> predictions; + Eigen::Vector2f predictionPosition = secondGroundTruth.position; + float predictionPressure = secondGroundTruth.pressure; + nsecs_t predictionTargetTimestamp = secondGroundTruth.timestamp; + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS; ++i) { + predictionPosition += trajectory; + predictionPressure += deltaPressure; + predictionTargetTimestamp += TEST_PREDICTION_INTERVAL_NANOS; + predictions.push_back( + PredictionPoint{{.position = predictionPosition, .pressure = predictionPressure}, + .originTimestamp = secondGroundTruth.timestamp, + .targetTimestamp = predictionTargetTimestamp}); + } + return predictions; +} + +TEST(GeneratePredictionsTest, GenerateConstantPredictions) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10, 20), .pressure = 0.3f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const std::vector<PredictionPoint> predictionPoints = + generateConstantPredictions(groundTruthPoint); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, predictionPoints.size()); + for (size_t i = 0; i < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + EXPECT_THAT(predictionPoints[i].position, Vector2fNear(groundTruthPoint.position, 1e-6)); + EXPECT_THAT(predictionPoints[i].pressure, FloatNear(groundTruthPoint.pressure, 1e-6)); + EXPECT_EQ(predictionPoints[i].originTimestamp, groundTruthPoint.timestamp); + EXPECT_EQ(predictionPoints[i].targetTimestamp, + groundTruthPoint.timestamp + + static_cast<nsecs_t>(i + 1) * TEST_PREDICTION_INTERVAL_NANOS); + } +} + +TEST(GeneratePredictionsTest, LinearExtrapolationFromTwoPoints) { + const nsecs_t initialTimestamp = TEST_INITIAL_TIMESTAMP; + const std::vector<PredictionPoint> predictionPoints = generatePredictionsByLinearExtrapolation( + GroundTruthPoint{{.position = Eigen::Vector2f(100, 200), .pressure = 0.9f}, + .timestamp = initialTimestamp}, + GroundTruthPoint{{.position = Eigen::Vector2f(105, 190), .pressure = 0.8f}, + .timestamp = initialTimestamp + TEST_PREDICTION_INTERVAL_NANOS}); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, predictionPoints.size()); + const nsecs_t originTimestamp = initialTimestamp + TEST_PREDICTION_INTERVAL_NANOS; + EXPECT_THAT(predictionPoints[0], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(110, 180), + .pressure = 0.7f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[1], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(115, 170), + .pressure = 0.6f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 2 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[2], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(120, 160), + .pressure = 0.5f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 3 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[3], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(125, 150), + .pressure = 0.4f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 4 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[4], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(130, 140), + .pressure = 0.3f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 5 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); +} + +// Generates predictions by linear extrapolation for each consecutive pair of ground truth points +// (see the comment for the above function for further explanation). Returns a vector of vectors of +// prediction points, where the first index is the source ground truth index, and the second is the +// prediction target index. +// +// The returned vector has size equal to the input vector, and the first element of the returned +// vector is always empty. +std::vector<std::vector<PredictionPoint>> generateAllPredictionsByLinearExtrapolation( + const std::vector<GroundTruthPoint>& groundTruthPoints) { + std::vector<std::vector<PredictionPoint>> allPredictions; + allPredictions.emplace_back(); + for (size_t i = 1; i < groundTruthPoints.size(); ++i) { + allPredictions.push_back(generatePredictionsByLinearExtrapolation(groundTruthPoints[i - 1], + groundTruthPoints[i])); + } + return allPredictions; +} + +TEST(GeneratePredictionsTest, GenerateAllPredictions) { + const nsecs_t initialTimestamp = TEST_INITIAL_TIMESTAMP; + std::vector<GroundTruthPoint> + groundTruthPoints{GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.5f}, + .timestamp = initialTimestamp}, + GroundTruthPoint{{.position = Eigen::Vector2f(1, -1), + .pressure = 0.51f}, + .timestamp = initialTimestamp + + 2 * TEST_PREDICTION_INTERVAL_NANOS}, + GroundTruthPoint{{.position = Eigen::Vector2f(2, -2), + .pressure = 0.52f}, + .timestamp = initialTimestamp + + 3 * TEST_PREDICTION_INTERVAL_NANOS}}; + + const std::vector<std::vector<PredictionPoint>> allPredictions = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + // Check format of allPredictions data. + ASSERT_EQ(groundTruthPoints.size(), allPredictions.size()); + EXPECT_TRUE(allPredictions[0].empty()); + EXPECT_EQ(TEST_MAX_NUM_PREDICTIONS, allPredictions[1].size()); + EXPECT_EQ(TEST_MAX_NUM_PREDICTIONS, allPredictions[2].size()); + + // Check positions of predictions generated from first pair of ground truth points. + EXPECT_THAT(allPredictions[1][0].position, Vector2fNear(Eigen::Vector2f(2, -2), 1e-9)); + EXPECT_THAT(allPredictions[1][1].position, Vector2fNear(Eigen::Vector2f(3, -3), 1e-9)); + EXPECT_THAT(allPredictions[1][2].position, Vector2fNear(Eigen::Vector2f(4, -4), 1e-9)); + EXPECT_THAT(allPredictions[1][3].position, Vector2fNear(Eigen::Vector2f(5, -5), 1e-9)); + EXPECT_THAT(allPredictions[1][4].position, Vector2fNear(Eigen::Vector2f(6, -6), 1e-9)); + + // Check pressures of predictions generated from first pair of ground truth points. + EXPECT_FLOAT_EQ(0.52f, allPredictions[1][0].pressure); + EXPECT_FLOAT_EQ(0.53f, allPredictions[1][1].pressure); + EXPECT_FLOAT_EQ(0.54f, allPredictions[1][2].pressure); + EXPECT_FLOAT_EQ(0.55f, allPredictions[1][3].pressure); + EXPECT_FLOAT_EQ(0.56f, allPredictions[1][4].pressure); +} + +// --- Prediction error helper functions. --- + +struct GeneralPositionErrors { + float alongTrajectoryErrorMean; + float alongTrajectoryErrorStd; + float offTrajectoryRmse; +}; + +// Inputs: +// • Vector of ground truth points +// • Vector of vectors of prediction points, where the first index is the source ground truth +// index, and the second is the prediction target index. +// +// Returns a vector of GeneralPositionErrors, indexed by prediction time delta bucket. +std::vector<GeneralPositionErrors> computeGeneralPositionErrors( + const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints) { + // Aggregate errors by time bucket (prediction target index). + std::vector<GeneralPositionErrors> generalPostitionErrors; + for (size_t predictionTargetIndex = 0; predictionTargetIndex < TEST_MAX_NUM_PREDICTIONS; + ++predictionTargetIndex) { + std::vector<float> alongTrajectoryErrors; + std::vector<float> alongTrajectorySquaredErrors; + std::vector<float> offTrajectoryErrors; + for (size_t sourceGroundTruthIndex = 1; sourceGroundTruthIndex < groundTruthPoints.size(); + ++sourceGroundTruthIndex) { + const size_t targetGroundTruthIndex = + sourceGroundTruthIndex + predictionTargetIndex + 1; + // Only include errors for points with a ground truth value. + if (targetGroundTruthIndex < groundTruthPoints.size()) { + const Eigen::Vector2f trajectory = + (groundTruthPoints[targetGroundTruthIndex].position - + groundTruthPoints[targetGroundTruthIndex - 1].position) + .normalized(); + const Eigen::Vector2f orthogonalTrajectory = + Eigen::Rotation2Df(M_PI_2) * trajectory; + const Eigen::Vector2f positionError = + predictionPoints[sourceGroundTruthIndex][predictionTargetIndex].position - + groundTruthPoints[targetGroundTruthIndex].position; + alongTrajectoryErrors.push_back(positionError.dot(trajectory)); + alongTrajectorySquaredErrors.push_back(alongTrajectoryErrors.back() * + alongTrajectoryErrors.back()); + offTrajectoryErrors.push_back(positionError.dot(orthogonalTrajectory)); + } + } + generalPostitionErrors.push_back( + {.alongTrajectoryErrorMean = average(alongTrajectoryErrors), + .alongTrajectoryErrorStd = standardDeviation(alongTrajectoryErrors), + .offTrajectoryRmse = rmse(offTrajectoryErrors)}); + } + return generalPostitionErrors; +} + +// Inputs: +// • Vector of ground truth points +// • Vector of vectors of prediction points, where the first index is the source ground truth +// index, and the second is the prediction target index. +// +// Returns a vector of pressure RMSEs, indexed by prediction time delta bucket. +std::vector<float> computePressureRmses( + const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints) { + // Aggregate errors by time bucket (prediction target index). + std::vector<float> pressureRmses; + for (size_t predictionTargetIndex = 0; predictionTargetIndex < TEST_MAX_NUM_PREDICTIONS; + ++predictionTargetIndex) { + std::vector<float> pressureErrors; + for (size_t sourceGroundTruthIndex = 1; sourceGroundTruthIndex < groundTruthPoints.size(); + ++sourceGroundTruthIndex) { + const size_t targetGroundTruthIndex = + sourceGroundTruthIndex + predictionTargetIndex + 1; + // Only include errors for points with a ground truth value. + if (targetGroundTruthIndex < groundTruthPoints.size()) { + pressureErrors.push_back( + predictionPoints[sourceGroundTruthIndex][predictionTargetIndex].pressure - + groundTruthPoints[targetGroundTruthIndex].pressure); + } + } + pressureRmses.push_back(rmse(pressureErrors)); + } + return pressureRmses; +} + +TEST(ErrorComputationHelperTest, ComputeGeneralPositionErrorsSimpleTest) { + std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + groundTruthPoints[3].position = Eigen::Vector2f(1, 0); + groundTruthPoints[4].position = Eigen::Vector2f(1, 1); + groundTruthPoints[5].position = Eigen::Vector2f(1, 3); + groundTruthPoints[6].position = Eigen::Vector2f(2, 3); + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + // The generated predictions look like: + // + // | Source | Target Ground Truth Index | + // | Index | 2 | 3 | 4 | 5 | 6 | + // |------------|--------|--------|--------|--------|--------| + // | 1 | (0, 0) | (0, 0) | (0, 0) | (0, 0) | (0, 0) | + // | 2 | | (0, 0) | (0, 0) | (0, 0) | (0, 0) | + // | 3 | | | (2, 0) | (3, 0) | (4, 0) | + // | 4 | | | | (1, 2) | (1, 3) | + // | 5 | | | | | (1, 5) | + // |---------------------------------------------------------| + // | Actual Ground Truth Values | + // | Position | (0, 0) | (1, 0) | (1, 1) | (1, 3) | (2, 3) | + // | Previous | (0, 0) | (0, 0) | (1, 0) | (1, 1) | (1, 3) | + // + // Note: this table organizes prediction targets by target ground truth index. Metrics are + // aggregated across points with the same prediction time bucket index, which is different. + // Each down-right diagonal from this table gives us points from a unique time bucket. + + // Initialize expected prediction errors from the table above. The first time bucket corresponds + // to the long diagonal of the table, and subsequent time buckets step up-right from there. + const std::vector<std::vector<float>> expectedAlongTrajectoryErrors{{0, -1, -1, -1, -1}, + {-1, -1, -3, -1}, + {-1, -3, 2}, + {-3, -2}, + {-2}}; + const std::vector<std::vector<float>> expectedOffTrajectoryErrors{{0, 0, 1, 0, 2}, + {0, 1, 2, 0}, + {1, 1, 3}, + {1, 3}, + {3}}; + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, generalPositionErrors.size()); + for (size_t i = 0; i < generalPositionErrors.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + EXPECT_FLOAT_EQ(average(expectedAlongTrajectoryErrors[i]), + generalPositionErrors[i].alongTrajectoryErrorMean); + EXPECT_FLOAT_EQ(standardDeviation(expectedAlongTrajectoryErrors[i]), + generalPositionErrors[i].alongTrajectoryErrorStd); + EXPECT_FLOAT_EQ(rmse(expectedOffTrajectoryErrors[i]), + generalPositionErrors[i].offTrajectoryRmse); + } +} + +TEST(ErrorComputationHelperTest, ComputePressureRmsesSimpleTest) { + // Generate ground truth points with pressures {0.0, 0.0, 0.0, 0.0, 0.5, 0.5, 0.5}. + // (We need TEST_MAX_NUM_PREDICTIONS + 2 to test all prediction time buckets.) + std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + for (size_t i = 4; i < groundTruthPoints.size(); ++i) { + groundTruthPoints[i].pressure = 0.5f; + } + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<float> pressureRmses = computePressureRmses(groundTruthPoints, predictionPoints); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, pressureRmses.size()); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{0.0f, 0.0f, -0.5f, 0.5f, 0.0f}), pressureRmses[0]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{0.0f, -0.5f, -0.5f, 1.0f}), pressureRmses[1]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f, -0.5f, -0.5f}), pressureRmses[2]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f, -0.5f}), pressureRmses[3]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f}), pressureRmses[4]); +} + +// --- MotionPredictorMetricsManager tests. --- + +// Helper function that instantiates a MetricsManager with the given mock logged AtomFields. Takes +// vectors of ground truth and prediction points of the same length, and passes these points to the +// MetricsManager. The format of these vectors is expected to be: +// • groundTruthPoints: chronologically-ordered ground truth points, with at least 2 elements. +// • predictionPoints: the first index points to a vector of predictions corresponding to the +// source ground truth point with the same index. +// - The first element should be empty, because there are not expected to be predictions until +// we have received 2 ground truth points. +// - The last element may be empty, because there will be no future ground truth points to +// associate with those predictions (if not empty, it will be ignored). +// - To test all prediction buckets, there should be at least TEST_MAX_NUM_PREDICTIONS non-empty +// prediction sets (that is, excluding the first and last). Thus, groundTruthPoints and +// predictionPoints should have size at least TEST_MAX_NUM_PREDICTIONS + 2. +// +// The passed-in outAtomFields will contain the logged AtomFields when the function returns. +// +// This function returns void so that it can use test assertions. +void runMetricsManager(const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints, + std::vector<AtomFields>& outAtomFields) { + MotionPredictorMetricsManager metricsManager(TEST_PREDICTION_INTERVAL_NANOS, + TEST_MAX_NUM_PREDICTIONS); + metricsManager.setMockLoggedAtomFields(&outAtomFields); + + // Validate structure of groundTruthPoints and predictionPoints. + ASSERT_EQ(predictionPoints.size(), groundTruthPoints.size()); + ASSERT_GE(groundTruthPoints.size(), 2u); + ASSERT_EQ(predictionPoints[0].size(), 0u); + for (size_t i = 1; i + 1 < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + ASSERT_EQ(predictionPoints[i].size(), TEST_MAX_NUM_PREDICTIONS); + } + + // Pass ground truth points and predictions (for all except first and last ground truth). + for (size_t i = 0; i < groundTruthPoints.size(); ++i) { + metricsManager.onRecord(makeMotionEvent(groundTruthPoints[i])); + if ((i > 0) && (i + 1 < predictionPoints.size())) { + metricsManager.onPredict(makeMotionEvent(predictionPoints[i])); + } + } + // Send a stroke-end event to trigger the logging call. + metricsManager.onRecord(makeLiftMotionEvent()); +} + +// Vacuous test: +// • Input: no prediction data. +// • Expectation: no metrics should be logged. +TEST(MotionPredictorMetricsManagerTest, NoPredictions) { + std::vector<AtomFields> mockLoggedAtomFields; + MotionPredictorMetricsManager metricsManager(TEST_PREDICTION_INTERVAL_NANOS, + TEST_MAX_NUM_PREDICTIONS); + metricsManager.setMockLoggedAtomFields(&mockLoggedAtomFields); + + metricsManager.onRecord(makeMotionEvent( + GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), .pressure = 0}, .timestamp = 0})); + metricsManager.onRecord(makeLiftMotionEvent()); + + // Check that mockLoggedAtomFields is still empty (as it was initialized empty), ensuring that + // no metrics were logged. + EXPECT_EQ(0u, mockLoggedAtomFields.size()); +} + +// Perfect predictions test: +// • Input: constant input events, perfect predictions matching the input events. +// • Expectation: all error metrics should be zero, or NO_DATA_SENTINEL for "unreported" metrics. +// (For example, scale-invariant errors are only reported for the final time bucket.) +TEST(MotionPredictorMetricsManagerTest, ConstantGroundTruthPerfectPredictions) { + GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10.0f, 20.0f), .pressure = 0.6f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + + // Generate ground truth and prediction points as described by the runMetricsManager comment. + std::vector<GroundTruthPoint> groundTruthPoints; + std::vector<std::vector<PredictionPoint>> predictionPoints; + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + groundTruthPoints.push_back(groundTruthPoint); + predictionPoints.push_back(i > 0 ? generateConstantPredictions(groundTruthPoint) + : std::vector<PredictionPoint>{}); + groundTruthPoint.timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + // Check that errors are all zero, or NO_DATA_SENTINEL for unreported metrics. + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // General errors: reported for every time bucket. + EXPECT_EQ(0, atom.alongTrajectoryErrorMeanMillipixels); + EXPECT_EQ(0, atom.alongTrajectoryErrorStdMillipixels); + EXPECT_EQ(0, atom.offTrajectoryRmseMillipixels); + EXPECT_EQ(0, atom.pressureRmseMilliunits); + // High-velocity errors: reported only for the last two time buckets. + // However, this data has zero velocity, so these metrics should all be NO_DATA_SENTINEL. + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityOffTrajectoryRmse); + // Scale-invariant errors: reported only for the last time bucket. + if (i + 1 == atomFields.size()) { + EXPECT_EQ(0, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(0, atom.scaleInvariantOffTrajectoryRmse); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantOffTrajectoryRmse); + } + } +} + +TEST(MotionPredictorMetricsManagerTest, QuadraticPressureLinearPredictions) { + // Generate ground truth points. + // + // Ground truth pressures are a quadratically increasing function from some initial value. + const float initialPressure = 0.5f; + const float quadraticCoefficient = 0.01f; + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = TEST_INITIAL_TIMESTAMP; + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + const float pressure = initialPressure + quadraticCoefficient * static_cast<float>(i * i); + groundTruthPoints.push_back( + GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), .pressure = pressure}, + .timestamp = timestamp}); + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + // Note: the first index is the source ground truth index, and the second is the prediction + // target index. + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + const std::vector<float> pressureErrors = + computePressureRmses(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + // Check time bucket delta matches expectation based on index and prediction interval. + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // Check pressure error matches expectation. + EXPECT_NEAR(static_cast<int>(1000 * pressureErrors[i]), atom.pressureRmseMilliunits, 1); + } +} + +TEST(MotionPredictorMetricsManagerTest, QuadraticPositionLinearPredictionsGeneralErrors) { + // Generate ground truth points. + // + // Each component of the ground truth positions are an independent quadratically increasing + // function from some initial value. + const Eigen::Vector2f initialPosition(200, 300); + const Eigen::Vector2f quadraticCoefficients(-2, 3); + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = TEST_INITIAL_TIMESTAMP; + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + const Eigen::Vector2f position = + initialPosition + quadraticCoefficients * static_cast<float>(i * i); + groundTruthPoints.push_back( + GroundTruthPoint{{.position = position, .pressure = 0.5}, .timestamp = timestamp}); + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + // Note: the first index is the source ground truth index, and the second is the prediction + // target index. + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + // Check time bucket delta matches expectation based on index and prediction interval. + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // Check general position errors match expectation. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorMean), + atom.alongTrajectoryErrorMeanMillipixels, 1); + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorStd), + atom.alongTrajectoryErrorStdMillipixels, 1); + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].offTrajectoryRmse), + atom.offTrajectoryRmseMillipixels, 1); + } +} + +// Counterclockwise regular octagonal section test: +// • Input – ground truth: constantly-spaced input events starting at a trajectory pointing exactly +// rightwards, and rotating by 45° counterclockwise after each input. +// • Input – predictions: simple linear extrapolations of previous two ground truth points. +// +// The code below uses the following terminology to distinguish references to ground truth events: +// • Source ground truth: the most recent ground truth point received at the time the prediction +// was made. +// • Target ground truth: the ground truth event that the prediction was attempting to match. +TEST(MotionPredictorMetricsManagerTest, CounterclockwiseOctagonGroundTruthLinearPredictions) { + // Select a stroke velocity that exceeds the high-velocity threshold of 1100 px/sec. + // For an input rate of 240 hz, 1100 px/sec * (1/240) sec/input ≈ 4.58 pixels per input. + const float strokeVelocity = 10; // pixels per input + + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(100, 100), + /*initialAngle=*/M_PI_2, + /*velocity=*/strokeVelocity, + /*turningAngle=*/-M_PI_4, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + + // General errors: reported for every time bucket. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorMean), + atom.alongTrajectoryErrorMeanMillipixels, 1); + // We allow for some floating point error in standard deviation (0.02 pixels). + EXPECT_NEAR(1000 * generalPositionErrors[i].alongTrajectoryErrorStd, + atom.alongTrajectoryErrorStdMillipixels, 20); + // All position errors are equal, so the standard deviation should be approximately zero. + EXPECT_NEAR(0, atom.alongTrajectoryErrorStdMillipixels, 20); + // Absolute value for RMSE, since it must be non-negative. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].offTrajectoryRmse), + atom.offTrajectoryRmseMillipixels, 1); + + // High-velocity errors: reported only for the last two time buckets. + // + // Since our input stroke velocity is chosen to be above the high-velocity threshold, all + // data contributes to high-velocity errors, and thus high-velocity errors should be equal + // to general errors (where reported). + // + // As above, use absolute value for RMSE, since it must be non-negative. + if (i + 2 >= atomFields.size()) { + EXPECT_NEAR(static_cast<int>( + 1000 * std::abs(generalPositionErrors[i].alongTrajectoryErrorMean)), + atom.highVelocityAlongTrajectoryRmse, 1); + EXPECT_NEAR(static_cast<int>(1000 * + std::abs(generalPositionErrors[i].offTrajectoryRmse)), + atom.highVelocityOffTrajectoryRmse, 1); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityOffTrajectoryRmse); + } + + // Scale-invariant errors: reported only for the last time bucket, where the reported value + // is the aggregation across all time buckets. + // + // The MetricsManager stores mMaxNumPredictions recent ground truth segments. Our ground + // truth segments here all have a length of strokeVelocity, so we can convert general errors + // to scale-invariant errors by dividing by `strokeVelocty * TEST_MAX_NUM_PREDICTIONS`. + // + // As above, use absolute value for RMSE, since it must be non-negative. + if (i + 1 == atomFields.size()) { + const float pathLength = strokeVelocity * TEST_MAX_NUM_PREDICTIONS; + std::vector<float> alongTrajectoryAbsoluteErrors; + std::vector<float> offTrajectoryAbsoluteErrors; + for (size_t j = 0; j < TEST_MAX_NUM_PREDICTIONS; ++j) { + alongTrajectoryAbsoluteErrors.push_back( + std::abs(generalPositionErrors[j].alongTrajectoryErrorMean)); + offTrajectoryAbsoluteErrors.push_back( + std::abs(generalPositionErrors[j].offTrajectoryRmse)); + } + EXPECT_NEAR(static_cast<int>(1000 * average(alongTrajectoryAbsoluteErrors) / + pathLength), + atom.scaleInvariantAlongTrajectoryRmse, 1); + EXPECT_NEAR(static_cast<int>(1000 * average(offTrajectoryAbsoluteErrors) / pathLength), + atom.scaleInvariantOffTrajectoryRmse, 1); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantOffTrajectoryRmse); + } + } +} + +} // namespace +} // namespace android |