diff options
275 files changed, 11967 insertions, 3030 deletions
diff --git a/Android.bp b/Android.bp index 81d0615e64..8c4dfbbea4 100644 --- a/Android.bp +++ b/Android.bp @@ -36,6 +36,14 @@ license { ], } +cc_library_headers { + name: "native_headers", + host_supported: true, + export_include_dirs: [ + "include/", + ], +} + ndk_headers { name: "libandroid_headers", from: "include/android", diff --git a/data/etc/input/Android.bp b/data/etc/input/Android.bp index 90f3c6b49a..b662491272 100644 --- a/data/etc/input/Android.bp +++ b/data/etc/input/Android.bp @@ -3,12 +3,21 @@ package { } filegroup { - name: "motion_predictor_model.fb", - srcs: ["motion_predictor_model.fb"], + name: "motion_predictor_model", + srcs: [ + "motion_predictor_model.tflite", + "motion_predictor_config.xml", + ], } prebuilt_etc { name: "motion_predictor_model_prebuilt", filename_from_src: true, - src: "motion_predictor_model.fb", + src: "motion_predictor_model.tflite", +} + +prebuilt_etc { + name: "motion_predictor_model_config", + filename_from_src: true, + src: "motion_predictor_config.xml", } diff --git a/data/etc/input/motion_predictor_config.xml b/data/etc/input/motion_predictor_config.xml new file mode 100644 index 0000000000..39772aece2 --- /dev/null +++ b/data/etc/input/motion_predictor_config.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<motion-predictor> + <!-- The time interval (ns) between the model's predictions. --> + <prediction-interval>4166666</prediction-interval> <!-- 4.167 ms = ~240 Hz --> + <!-- The noise floor (px) for predicted distances. + + As the model is trained stochastically, there is some expected minimum + variability in its output. This can be a UX issue when the input device + is moving slowly and the variability is large relative to the magnitude + of the motion. In these cases, it is better to inhibit the prediction, + rather than show noisy predictions (and there is little benefit to + prediction anyway). + + The value for this parameter should at least be close to the maximum + predicted distance when the input device is held stationary (i.e. the + expected minimum variability), and perhaps a little larger to capture + the UX issue mentioned above. + --> + <distance-noise-floor>0.2</distance-noise-floor> +</motion-predictor> + diff --git a/data/etc/input/motion_predictor_model.fb b/data/etc/input/motion_predictor_model.fb Binary files differdeleted file mode 100644 index 10b3c8b114..0000000000 --- a/data/etc/input/motion_predictor_model.fb +++ /dev/null diff --git a/data/etc/input/motion_predictor_model.tflite b/data/etc/input/motion_predictor_model.tflite Binary files differnew file mode 100644 index 0000000000..45fc162cd1 --- /dev/null +++ b/data/etc/input/motion_predictor_model.tflite diff --git a/include/android/input.h b/include/android/input.h index a45f065dd0..9a0eb4d838 100644 --- a/include/android/input.h +++ b/include/android/input.h @@ -781,6 +781,8 @@ enum { * * These values are relative to the state from the last event, not accumulated, so developers * should make sure to process this axis value for all batched historical events. + * + * This axis is only set on the first pointer in a motion event. */ AMOTION_EVENT_AXIS_GESTURE_X_OFFSET = 48, /** @@ -797,6 +799,8 @@ enum { * * These values are relative to the state from the last event, not accumulated, so developers * should make sure to process this axis value for all batched historical events. + * + * This axis is only set on the first pointer in a motion event. */ AMOTION_EVENT_AXIS_GESTURE_SCROLL_X_DISTANCE = 50, /** @@ -815,16 +819,29 @@ enum { * * These values are relative to the state from the last event, not accumulated, so developers * should make sure to process this axis value for all batched historical events. + * + * This axis is only set on the first pointer in a motion event. */ AMOTION_EVENT_AXIS_GESTURE_PINCH_SCALE_FACTOR = 52, /** + * Axis constant: the number of fingers being used in a multi-finger swipe gesture. + * + * - For a touch pad, reports the number of fingers being used in a multi-finger swipe gesture + * (with CLASSIFICATION_MULTI_FINGER_SWIPE). + * + * Since CLASSIFICATION_MULTI_FINGER_SWIPE is a hidden API, so is this axis. It is only set on + * the first pointer in a motion event. + */ + AMOTION_EVENT_AXIS_GESTURE_SWIPE_FINGER_COUNT = 53, + + /** * Note: This is not an "Axis constant". It does not represent any axis, nor should it be used * to represent any axis. It is a constant holding the value of the largest defined axis value, * to make some computations (like iterating through all possible axes) cleaner. * Please update the value accordingly if you add a new axis. */ - AMOTION_EVENT_MAXIMUM_VALID_AXIS_VALUE = AMOTION_EVENT_AXIS_GESTURE_PINCH_SCALE_FACTOR, + AMOTION_EVENT_MAXIMUM_VALID_AXIS_VALUE = AMOTION_EVENT_AXIS_GESTURE_SWIPE_FINGER_COUNT, // NOTE: If you add a new axis here you must also add it to several other files. // Refer to frameworks/base/core/java/android/view/MotionEvent.java for the full list. diff --git a/include/input/InputDevice.h b/include/input/InputDevice.h index 1a40fdb90c..b7751f704a 100644 --- a/include/input/InputDevice.h +++ b/include/input/InputDevice.h @@ -18,8 +18,10 @@ #include <android/sensor.h> #include <ftl/flags.h> +#include <ftl/mixins.h> #include <input/Input.h> #include <input/KeyCharacterMap.h> +#include <set> #include <unordered_map> #include <vector> @@ -68,6 +70,9 @@ struct InputDeviceIdentifier { * while conforming to the filename limitations. */ std::string getCanonicalName() const; + + bool operator==(const InputDeviceIdentifier&) const = default; + bool operator!=(const InputDeviceIdentifier&) const = default; }; /* Types of input device sensors. Keep sync with core/java/android/hardware/Sensor.java */ @@ -179,11 +184,24 @@ struct InputDeviceSensorInfo { int32_t id; }; +struct BrightnessLevel : ftl::DefaultConstructible<BrightnessLevel, std::uint8_t>, + ftl::Equatable<BrightnessLevel>, + ftl::Orderable<BrightnessLevel>, + ftl::Addable<BrightnessLevel> { + using DefaultConstructible::DefaultConstructible; +}; + struct InputDeviceLightInfo { explicit InputDeviceLightInfo(std::string name, int32_t id, InputDeviceLightType type, ftl::Flags<InputDeviceLightCapability> capabilityFlags, - int32_t ordinal) - : name(name), id(id), type(type), capabilityFlags(capabilityFlags), ordinal(ordinal) {} + int32_t ordinal, + std::set<BrightnessLevel> preferredBrightnessLevels) + : name(name), + id(id), + type(type), + capabilityFlags(capabilityFlags), + ordinal(ordinal), + preferredBrightnessLevels(std::move(preferredBrightnessLevels)) {} // Name string of the light. std::string name; // Light id @@ -194,6 +212,8 @@ struct InputDeviceLightInfo { ftl::Flags<InputDeviceLightCapability> capabilityFlags; // Ordinal of the light int32_t ordinal; + // Custom brightness levels for the light + std::set<BrightnessLevel> preferredBrightnessLevels; }; struct InputDeviceBatteryInfo { diff --git a/include/input/InputEventBuilders.h b/include/input/InputEventBuilders.h new file mode 100644 index 0000000000..9c0c10e603 --- /dev/null +++ b/include/input/InputEventBuilders.h @@ -0,0 +1,163 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <android/input.h> +#include <attestation/HmacKeyManager.h> +#include <gui/constants.h> +#include <input/Input.h> +#include <utils/Timers.h> // for nsecs_t, systemTime + +#include <vector> + +namespace android { + +// An arbitrary device id. +static constexpr uint32_t DEFAULT_DEVICE_ID = 1; + +// The default policy flags to use for event injection by tests. +static constexpr uint32_t DEFAULT_POLICY_FLAGS = POLICY_FLAG_FILTERED | POLICY_FLAG_PASS_TO_USER; + +class PointerBuilder { +public: + PointerBuilder(int32_t id, ToolType toolType) { + mProperties.clear(); + mProperties.id = id; + mProperties.toolType = toolType; + mCoords.clear(); + } + + PointerBuilder& x(float x) { return axis(AMOTION_EVENT_AXIS_X, x); } + + PointerBuilder& y(float y) { return axis(AMOTION_EVENT_AXIS_Y, y); } + + PointerBuilder& axis(int32_t axis, float value) { + mCoords.setAxisValue(axis, value); + return *this; + } + + PointerProperties buildProperties() const { return mProperties; } + + PointerCoords buildCoords() const { return mCoords; } + +private: + PointerProperties mProperties; + PointerCoords mCoords; +}; + +class MotionEventBuilder { +public: + MotionEventBuilder(int32_t action, int32_t source) { + mAction = action; + mSource = source; + mEventTime = systemTime(SYSTEM_TIME_MONOTONIC); + mDownTime = mEventTime; + } + + MotionEventBuilder& deviceId(int32_t deviceId) { + mDeviceId = deviceId; + return *this; + } + + MotionEventBuilder& downTime(nsecs_t downTime) { + mDownTime = downTime; + return *this; + } + + MotionEventBuilder& eventTime(nsecs_t eventTime) { + mEventTime = eventTime; + return *this; + } + + MotionEventBuilder& displayId(int32_t displayId) { + mDisplayId = displayId; + return *this; + } + + MotionEventBuilder& actionButton(int32_t actionButton) { + mActionButton = actionButton; + return *this; + } + + MotionEventBuilder& buttonState(int32_t buttonState) { + mButtonState = buttonState; + return *this; + } + + MotionEventBuilder& rawXCursorPosition(float rawXCursorPosition) { + mRawXCursorPosition = rawXCursorPosition; + return *this; + } + + MotionEventBuilder& rawYCursorPosition(float rawYCursorPosition) { + mRawYCursorPosition = rawYCursorPosition; + return *this; + } + + MotionEventBuilder& pointer(PointerBuilder pointer) { + mPointers.push_back(pointer); + return *this; + } + + MotionEventBuilder& addFlag(uint32_t flags) { + mFlags |= flags; + return *this; + } + + MotionEvent build() { + std::vector<PointerProperties> pointerProperties; + std::vector<PointerCoords> pointerCoords; + for (const PointerBuilder& pointer : mPointers) { + pointerProperties.push_back(pointer.buildProperties()); + pointerCoords.push_back(pointer.buildCoords()); + } + + // Set mouse cursor position for the most common cases to avoid boilerplate. + if (mSource == AINPUT_SOURCE_MOUSE && + !MotionEvent::isValidCursorPosition(mRawXCursorPosition, mRawYCursorPosition)) { + mRawXCursorPosition = pointerCoords[0].getX(); + mRawYCursorPosition = pointerCoords[0].getY(); + } + + MotionEvent event; + static const ui::Transform kIdentityTransform; + event.initialize(InputEvent::nextId(), mDeviceId, mSource, mDisplayId, INVALID_HMAC, + mAction, mActionButton, mFlags, /*edgeFlags=*/0, AMETA_NONE, mButtonState, + MotionClassification::NONE, kIdentityTransform, + /*xPrecision=*/0, /*yPrecision=*/0, mRawXCursorPosition, + mRawYCursorPosition, kIdentityTransform, mDownTime, mEventTime, + mPointers.size(), pointerProperties.data(), pointerCoords.data()); + return event; + } + +private: + int32_t mAction; + int32_t mDeviceId{DEFAULT_DEVICE_ID}; + int32_t mSource; + nsecs_t mDownTime; + nsecs_t mEventTime; + int32_t mDisplayId{ADISPLAY_ID_DEFAULT}; + int32_t mActionButton{0}; + int32_t mButtonState{0}; + int32_t mFlags{0}; + float mRawXCursorPosition{AMOTION_EVENT_INVALID_CURSOR_POSITION}; + float mRawYCursorPosition{AMOTION_EVENT_INVALID_CURSOR_POSITION}; + + std::vector<PointerBuilder> mPointers; +}; + +} // namespace android diff --git a/include/input/InputEventLabels.h b/include/input/InputEventLabels.h index 9dedd2b2da..909bf087dd 100644 --- a/include/input/InputEventLabels.h +++ b/include/input/InputEventLabels.h @@ -40,7 +40,16 @@ struct EvdevEventLabel { // then you must add it to InputEventLabels.cpp. class InputEventLookup { + /** + * This class is not purely static, but uses a singleton pattern in order to delay the + * initialization of the maps that it contains. If it were purely static, the maps could be + * created early, and would cause sanitizers to report memory leaks. + */ public: + InputEventLookup(InputEventLookup& other) = delete; + + void operator=(const InputEventLookup&) = delete; + static std::optional<int> lookupValueByLabel(const std::unordered_map<std::string, int>& map, const char* literal); @@ -61,17 +70,24 @@ public: static EvdevEventLabel getLinuxEvdevLabel(int32_t type, int32_t code, int32_t value); private: - static const std::unordered_map<std::string, int> KEYCODES; + InputEventLookup(); + + static const InputEventLookup& get() { + static InputEventLookup sLookup; + return sLookup; + } + + const std::unordered_map<std::string, int> KEYCODES; - static const std::vector<InputEventLabel> KEY_NAMES; + const std::vector<InputEventLabel> KEY_NAMES; - static const std::unordered_map<std::string, int> AXES; + const std::unordered_map<std::string, int> AXES; - static const std::vector<InputEventLabel> AXES_NAMES; + const std::vector<InputEventLabel> AXES_NAMES; - static const std::unordered_map<std::string, int> LEDS; + const std::unordered_map<std::string, int> LEDS; - static const std::unordered_map<std::string, int> FLAGS; + const std::unordered_map<std::string, int> FLAGS; }; } // namespace android diff --git a/include/input/InputVerifier.h b/include/input/InputVerifier.h index d4589f53b5..3715408388 100644 --- a/include/input/InputVerifier.h +++ b/include/input/InputVerifier.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 The Android Open Source Project + * Copyright 2023 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,25 @@ #pragma once +#include <android-base/result.h> #include <input/Input.h> -#include <map> +#include "rust/cxx.h" namespace android { +namespace input { +namespace verifier { +struct InputVerifier; +} +} // namespace input + /* * Crash if the provided touch stream is inconsistent. + * This class is a pass-through to the rust implementation of InputVerifier. + * The rust class could also be used directly, but it would be less convenient. + * We can't directly invoke the rust methods on a rust object. So, there's no way to do: + * mVerifier.process_movement(...). + * This C++ class makes it a bit easier to use. * * TODO(b/211379801): Add support for hover events: * - No hover move without enter @@ -34,16 +46,13 @@ class InputVerifier { public: InputVerifier(const std::string& name); - void processMovement(int32_t deviceId, int32_t action, uint32_t pointerCount, - const PointerProperties* pointerProperties, - const PointerCoords* pointerCoords, int32_t flags); + android::base::Result<void> processMovement(int32_t deviceId, int32_t action, + uint32_t pointerCount, + const PointerProperties* pointerProperties, + const PointerCoords* pointerCoords, int32_t flags); private: - const std::string mName; - std::map<int32_t /*deviceId*/, std::bitset<MAX_POINTER_ID + 1>> mTouchingPointerIdsByDevice; - void ensureTouchingPointersMatch(int32_t deviceId, uint32_t pointerCount, - const PointerProperties* pointerProperties, - const char* action) const; + rust::Box<android::input::verifier::InputVerifier> mVerifier; }; } // namespace android diff --git a/include/input/MotionPredictor.h b/include/input/MotionPredictor.h index de8ddcabeb..8797962886 100644 --- a/include/input/MotionPredictor.h +++ b/include/input/MotionPredictor.h @@ -26,7 +26,9 @@ #include <android-base/thread_annotations.h> #include <android/sysprop/InputProperties.sysprop.h> #include <input/Input.h> +#include <input/MotionPredictorMetricsManager.h> #include <input/TfLiteMotionPredictor.h> +#include <utils/Timers.h> // for nsecs_t namespace android { @@ -69,6 +71,7 @@ public: */ MotionPredictor(nsecs_t predictionTimestampOffsetNanos, std::function<bool()> checkEnableMotionPrediction = isMotionPredictionEnabled); + /** * Record the actual motion received by the view. This event will be used for calculating the * predictions. @@ -77,7 +80,9 @@ public: * consistent with the previously recorded events. */ android::base::Result<void> record(const MotionEvent& event); + std::unique_ptr<MotionEvent> predict(nsecs_t timestamp); + bool isPredictionAvailable(int32_t deviceId, int32_t source); private: @@ -88,6 +93,8 @@ private: std::unique_ptr<TfLiteMotionPredictorBuffers> mBuffers; std::optional<MotionEvent> mLastEvent; + + std::optional<MotionPredictorMetricsManager> mMetricsManager; }; } // namespace android diff --git a/include/input/MotionPredictorMetricsManager.h b/include/input/MotionPredictorMetricsManager.h new file mode 100644 index 0000000000..12e50ba3b4 --- /dev/null +++ b/include/input/MotionPredictorMetricsManager.h @@ -0,0 +1,206 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstddef> +#include <cstdint> +#include <functional> +#include <limits> +#include <optional> +#include <vector> + +#include <input/Input.h> // for MotionEvent +#include <input/RingBuffer.h> +#include <utils/Timers.h> // for nsecs_t + +#include "Eigen/Core" + +namespace android { + +/** + * Class to handle computing and reporting metrics for MotionPredictor. + * + * The public API provides two methods: `onRecord` and `onPredict`, which expect to receive the + * MotionEvents from the corresponding methods in MotionPredictor. + * + * This class stores AggregatedStrokeMetrics, updating them as new MotionEvents are passed in. When + * onRecord receives an UP or CANCEL event, this indicates the end of the stroke, and the final + * AtomFields are computed and reported to the stats library. + * + * If mMockLoggedAtomFields is set, the batch of AtomFields that are reported to the stats library + * for one stroke are also stored in mMockLoggedAtomFields at the time they're reported. + */ +class MotionPredictorMetricsManager { +public: + // Note: the MetricsManager assumes that the input interval equals the prediction interval. + MotionPredictorMetricsManager(nsecs_t predictionInterval, size_t maxNumPredictions); + + // This method should be called once for each call to MotionPredictor::record, receiving the + // forwarded MotionEvent argument. + void onRecord(const MotionEvent& inputEvent); + + // This method should be called once for each call to MotionPredictor::predict, receiving the + // MotionEvent that will be returned by MotionPredictor::predict. + void onPredict(const MotionEvent& predictionEvent); + + // Simple structs to hold relevant touch input information. Public so they can be used in tests. + + struct TouchPoint { + Eigen::Vector2f position; // (y, x) in pixels + float pressure; + }; + + struct GroundTruthPoint : TouchPoint { + nsecs_t timestamp; + }; + + struct PredictionPoint : TouchPoint { + // The timestamp of the last ground truth point when the prediction was made. + nsecs_t originTimestamp; + + nsecs_t targetTimestamp; + + // Order by targetTimestamp when sorting. + bool operator<(const PredictionPoint& other) const { + return this->targetTimestamp < other.targetTimestamp; + } + }; + + // Metrics aggregated so far for the current stroke. These are not the final fields to be + // reported in the atom (see AtomFields below), but rather an intermediate representation of the + // data that can be conveniently aggregated and from which the atom fields can be derived later. + // + // Displacement units are in pixels. + // + // "Along-trajectory error" is the dot product of the prediction error with the unit vector + // pointing towards the ground truth point whose timestamp corresponds to the prediction + // target timestamp, originating from the preceding ground truth point. + // + // "Off-trajectory error" is the component of the prediction error orthogonal to the + // "along-trajectory" unit vector described above. + // + // "High-velocity" errors are errors that are only accumulated when the velocity between the + // most recent two input events exceeds a certain threshold. + // + // "Scale-invariant errors" are the errors produced when the path length of the stroke is + // scaled to 1. (In other words, the error distances are normalized by the path length.) + struct AggregatedStrokeMetrics { + // General errors + float alongTrajectoryErrorSum = 0; + float alongTrajectorySumSquaredErrors = 0; + float offTrajectorySumSquaredErrors = 0; + float pressureSumSquaredErrors = 0; + size_t generalErrorsCount = 0; + + // High-velocity errors + float highVelocityAlongTrajectorySse = 0; + float highVelocityOffTrajectorySse = 0; + size_t highVelocityErrorsCount = 0; + + // Scale-invariant errors + float scaleInvariantAlongTrajectorySse = 0; + float scaleInvariantOffTrajectorySse = 0; + size_t scaleInvariantErrorsCount = 0; + }; + + // In order to explicitly indicate "no relevant data" for a metric, we report this + // large-magnitude negative sentinel value. (Most metrics are non-negative, so this value is + // completely unobtainable. For along-trajectory error mean, which can be negative, the + // magnitude makes it unobtainable in practice.) + static const int NO_DATA_SENTINEL = std::numeric_limits<int32_t>::min(); + + // Final metrics reported in the atom. + struct AtomFields { + int deltaTimeBucketMilliseconds = 0; + + // General errors + int alongTrajectoryErrorMeanMillipixels = NO_DATA_SENTINEL; + int alongTrajectoryErrorStdMillipixels = NO_DATA_SENTINEL; + int offTrajectoryRmseMillipixels = NO_DATA_SENTINEL; + int pressureRmseMilliunits = NO_DATA_SENTINEL; + + // High-velocity errors + int highVelocityAlongTrajectoryRmse = NO_DATA_SENTINEL; // millipixels + int highVelocityOffTrajectoryRmse = NO_DATA_SENTINEL; // millipixels + + // Scale-invariant errors + int scaleInvariantAlongTrajectoryRmse = NO_DATA_SENTINEL; // millipixels + int scaleInvariantOffTrajectoryRmse = NO_DATA_SENTINEL; // millipixels + }; + + // Allow tests to pass in a mock AtomFields pointer. + // + // When metrics are reported to the stats library on stroke end, they will also be written to + // mockLoggedAtomFields, overwriting existing data. The size of mockLoggedAtomFields will equal + // the number of calls to stats_write for that stroke. + void setMockLoggedAtomFields(std::vector<AtomFields>* mockLoggedAtomFields) { + mMockLoggedAtomFields = mockLoggedAtomFields; + } + +private: + // The interval between consecutive predictions' target timestamps. We assume that the input + // interval also equals this value. + const nsecs_t mPredictionInterval; + + // The maximum number of input frames into the future the model can predict. + // Used to perform time-bucketing of metrics. + const size_t mMaxNumPredictions; + + // History of mMaxNumPredictions + 1 ground truth points, used to compute scale-invariant + // error. (Also, the last two points are used to compute the ground truth trajectory.) + RingBuffer<GroundTruthPoint> mRecentGroundTruthPoints; + + // Predictions having a targetTimestamp after the most recent ground truth point's timestamp. + // Invariant: sorted in ascending order of targetTimestamp. + std::vector<PredictionPoint> mRecentPredictions; + + // Containers for the intermediate representation of stroke metrics and the final atom fields. + // These are indexed by the number of input frames into the future being predicted minus one, + // and always have size mMaxNumPredictions. + std::vector<AggregatedStrokeMetrics> mAggregatedMetrics; + std::vector<AtomFields> mAtomFields; + + // Non-owning pointer to the location of mock AtomFields. If present, will be filled with the + // values reported to stats_write on each batch of reported metrics. + // + // This pointer must remain valid as long as the MotionPredictorMetricsManager exists. + std::vector<AtomFields>* mMockLoggedAtomFields = nullptr; + + // Helper methods for the implementation of onRecord and onPredict. + + // Clears stored ground truth and prediction points, as well as all stored metrics for the + // current stroke. + void clearStrokeData(); + + // Adds the new ground truth point to mRecentGroundTruths, removes outdated predictions from + // mRecentPredictions, and updates the aggregated metrics to include the recent predictions that + // fuzzily match with the new ground truth point. + void incorporateNewGroundTruth(const GroundTruthPoint& groundTruthPoint); + + // Given a new prediction with targetTimestamp matching the latest ground truth point's + // timestamp, computes the corresponding metrics and updates mAggregatedMetrics. + void updateAggregatedMetrics(const PredictionPoint& predictionPoint); + + // Computes the atom fields to mAtomFields from the values in mAggregatedMetrics. + void computeAtomFields(); + + // Reports the metrics given by the current data in mAtomFields: + // • If on an Android device, reports the metrics to stats_write. + // • If mMockLoggedAtomFields is present, it will be overwritten with logged metrics, with one + // AtomFields element per call to stats_write. + void reportMetrics(); +}; + +} // namespace android diff --git a/include/input/PrintTools.h b/include/input/PrintTools.h index 02bc2010db..0e3fbb1982 100644 --- a/include/input/PrintTools.h +++ b/include/input/PrintTools.h @@ -27,6 +27,9 @@ namespace android { template <size_t N> std::string bitsetToString(const std::bitset<N>& bitset) { + if (bitset.none()) { + return "<none>"; + } return bitset.to_string(); } @@ -88,6 +91,20 @@ std::string dumpMap(const std::map<K, V>& map, std::string (*keyToString)(const } /** + * Convert map keys to string. The keys of the map should be integral type. + */ +template <typename K, typename V> +std::string dumpMapKeys(const std::map<K, V>& map, + std::string (*keyToString)(const K&) = constToString) { + std::string out; + for (const auto& [k, _] : map) { + out += out.empty() ? "{" : ", "; + out += keyToString(k); + } + return out.empty() ? "{}" : (out + "}"); +} + +/** * Convert a vector to a string. The values of the vector should be of a type supported by * constToString. */ diff --git a/include/input/TfLiteMotionPredictor.h b/include/input/TfLiteMotionPredictor.h index a340bd0575..2edc138f67 100644 --- a/include/input/TfLiteMotionPredictor.h +++ b/include/input/TfLiteMotionPredictor.h @@ -25,6 +25,7 @@ #include <android-base/mapped_file.h> #include <input/RingBuffer.h> +#include <utils/Timers.h> #include <tensorflow/lite/core/api/error_reporter.h> #include <tensorflow/lite/interpreter.h> @@ -98,6 +99,14 @@ private: // A TFLite model for generating motion predictions. class TfLiteMotionPredictorModel { public: + struct Config { + // The time between predictions. + nsecs_t predictionInterval = 0; + // The noise floor for predictions. + // Distances (r) less than this should be discarded as noise. + float distanceNoiseFloor = 0; + }; + // Creates a model from an encoded Flatbuffer model. static std::unique_ptr<TfLiteMotionPredictorModel> create(); @@ -109,6 +118,8 @@ public: // Returns the length of the model's output buffers. size_t outputLength() const; + const Config& config() const { return mConfig; } + // Executes the model. // Returns true if the model successfully executed and the output tensors can be read. bool invoke(); @@ -127,7 +138,8 @@ public: std::span<const float> outputPressure() const; private: - explicit TfLiteMotionPredictorModel(std::unique_ptr<android::base::MappedFile> model); + explicit TfLiteMotionPredictorModel(std::unique_ptr<android::base::MappedFile> model, + Config config); void allocateTensors(); void attachInputTensors(); @@ -148,6 +160,8 @@ private: std::unique_ptr<tflite::FlatBufferModel> mModel; std::unique_ptr<tflite::Interpreter> mInterpreter; tflite::SignatureRunner* mRunner = nullptr; + + const Config mConfig = {}; }; } // namespace android diff --git a/include/input/VelocityTracker.h b/include/input/VelocityTracker.h index da97c3e855..4257cb5e05 100644 --- a/include/input/VelocityTracker.h +++ b/include/input/VelocityTracker.h @@ -45,6 +45,7 @@ public: INT2 = 8, LEGACY = 9, MAX = LEGACY, + ftl_last = LEGACY, }; struct Estimator { @@ -95,8 +96,6 @@ public: // TODO(b/32830165): support axis-specific strategies. VelocityTracker(const Strategy strategy = Strategy::DEFAULT); - ~VelocityTracker(); - /** Return true if the axis is supported for velocity tracking, false otherwise. */ static bool isAxisSupported(int32_t axis); diff --git a/include/powermanager/PowerHalController.h b/include/powermanager/PowerHalController.h index 71a36d09e5..82e4551c3c 100644 --- a/include/powermanager/PowerHalController.h +++ b/include/powermanager/PowerHalController.h @@ -53,7 +53,7 @@ public: : mHalConnector(std::move(connector)) {} virtual ~PowerHalController() = default; - void init(); + virtual void init(); virtual HalResult<void> setBoost(hardware::power::Boost boost, int32_t durationMs) override; virtual HalResult<void> setMode(hardware::power::Mode mode, bool enabled) override; diff --git a/libs/binder/IActivityManager.cpp b/libs/binder/IActivityManager.cpp index 28975618e1..152c815ec3 100644 --- a/libs/binder/IActivityManager.cpp +++ b/libs/binder/IActivityManager.cpp @@ -193,8 +193,7 @@ public: status_t err = remote()->transact(LOG_FGS_API_BEGIN_TRANSACTION, data, &reply, IBinder::FLAG_ONEWAY); if (err != NO_ERROR || ((err = reply.readExceptionCode()) != NO_ERROR)) { - ALOGD("FGS Logger Transaction failed"); - ALOGD("%d", err); + ALOGD("%s: FGS Logger Transaction failed, %d", __func__, err); return err; } return NO_ERROR; @@ -209,8 +208,7 @@ public: status_t err = remote()->transact(LOG_FGS_API_END_TRANSACTION, data, &reply, IBinder::FLAG_ONEWAY); if (err != NO_ERROR || ((err = reply.readExceptionCode()) != NO_ERROR)) { - ALOGD("FGS Logger Transaction failed"); - ALOGD("%d", err); + ALOGD("%s: FGS Logger Transaction failed, %d", __func__, err); return err; } return NO_ERROR; @@ -224,11 +222,10 @@ public: data.writeInt32(state); data.writeInt32(appUid); data.writeInt32(appPid); - status_t err = remote()->transact(LOG_FGS_API_BEGIN_TRANSACTION, data, &reply, + status_t err = remote()->transact(LOG_FGS_API_STATE_CHANGED_TRANSACTION, data, &reply, IBinder::FLAG_ONEWAY); if (err != NO_ERROR || ((err = reply.readExceptionCode()) != NO_ERROR)) { - ALOGD("FGS Logger Transaction failed"); - ALOGD("%d", err); + ALOGD("%s: FGS Logger Transaction failed, %d", __func__, err); return err; } return NO_ERROR; diff --git a/libs/gui/Android.bp b/libs/gui/Android.bp index 13fdcd5743..ca493d7a5a 100644 --- a/libs/gui/Android.bp +++ b/libs/gui/Android.bp @@ -62,6 +62,7 @@ filegroup { name: "guiconstants_aidl", srcs: [ "android/gui/DropInputMode.aidl", + "android/gui/StalledTransactionInfo.aidl", "android/**/TouchOcclusionMode.aidl", ], } @@ -140,6 +141,7 @@ aidl_library { "android/gui/IWindowInfosListener.aidl", "android/gui/IWindowInfosPublisher.aidl", "android/gui/IWindowInfosReportedListener.aidl", + "android/gui/StalledTransactionInfo.aidl", "android/gui/WindowInfo.aidl", "android/gui/WindowInfosUpdate.aidl", ], diff --git a/libs/gui/BLASTBufferQueue.cpp b/libs/gui/BLASTBufferQueue.cpp index 5c324b29cd..207fa4fd31 100644 --- a/libs/gui/BLASTBufferQueue.cpp +++ b/libs/gui/BLASTBufferQueue.cpp @@ -303,13 +303,8 @@ void BLASTBufferQueue::transactionCommittedCallback(nsecs_t /*latchTime*/, // frame numbers that were in a sync. We remove the frame from mSyncedFrameNumbers // set and then check if it's empty. If there are no more pending syncs, we can // proceed with flushing the shadow queue. - // We also want to check if mSyncTransaction is null because it's possible another - // sync request came in while waiting, but it hasn't started processing yet. In that - // case, we don't actually want to flush the frames in between since they will get - // processed and merged with the sync transaction and released earlier than if they - // were sent to SF mSyncedFrameNumbers.erase(currFrameNumber); - if (mSyncedFrameNumbers.empty() && mSyncTransaction == nullptr) { + if (mSyncedFrameNumbers.empty()) { flushShadowQueue(); } } else { diff --git a/libs/gui/BufferQueueProducer.cpp b/libs/gui/BufferQueueProducer.cpp index ce5d5d382e..920b83dba9 100644 --- a/libs/gui/BufferQueueProducer.cpp +++ b/libs/gui/BufferQueueProducer.cpp @@ -418,6 +418,9 @@ status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* ou EGLSyncKHR eglFence = EGL_NO_SYNC_KHR; bool attachedByConsumer = false; + sp<IConsumerListener> listener; + bool callOnFrameDequeued = false; + uint64_t bufferId = 0; // Only used if callOnFrameDequeued == true { // Autolock scope std::unique_lock<std::mutex> lock(mCore->mMutex); @@ -561,10 +564,11 @@ status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* ou } if (!(returnFlags & BUFFER_NEEDS_REALLOCATION)) { - if (mCore->mConsumerListener != nullptr) { - mCore->mConsumerListener->onFrameDequeued(mSlots[*outSlot].mGraphicBuffer->getId()); - } + callOnFrameDequeued = true; + bufferId = mSlots[*outSlot].mGraphicBuffer->getId(); } + + listener = mCore->mConsumerListener; } // Autolock scope if (returnFlags & BUFFER_NEEDS_REALLOCATION) { @@ -581,10 +585,8 @@ status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* ou if (error == NO_ERROR && !mCore->mIsAbandoned) { graphicBuffer->setGenerationNumber(mCore->mGenerationNumber); mSlots[*outSlot].mGraphicBuffer = graphicBuffer; - if (mCore->mConsumerListener != nullptr) { - mCore->mConsumerListener->onFrameDequeued( - mSlots[*outSlot].mGraphicBuffer->getId()); - } + callOnFrameDequeued = true; + bufferId = mSlots[*outSlot].mGraphicBuffer->getId(); } mCore->mIsAllocating = false; @@ -608,6 +610,10 @@ status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* ou } // Autolock scope } + if (listener != nullptr && callOnFrameDequeued) { + listener->onFrameDequeued(bufferId); + } + if (attachedByConsumer) { returnFlags |= BUFFER_NEEDS_REALLOCATION; } @@ -647,6 +653,8 @@ status_t BufferQueueProducer::detachBuffer(int slot) { BQ_LOGV("detachBuffer: slot %d", slot); sp<IConsumerListener> listener; + bool callOnFrameDetached = false; + uint64_t bufferId = 0; // Only used if callOnFrameDetached is true { std::lock_guard<std::mutex> lock(mCore->mMutex); @@ -684,8 +692,9 @@ status_t BufferQueueProducer::detachBuffer(int slot) { listener = mCore->mConsumerListener; auto gb = mSlots[slot].mGraphicBuffer; - if (listener != nullptr && gb != nullptr) { - listener->onFrameDetached(gb->getId()); + if (gb != nullptr) { + callOnFrameDetached = true; + bufferId = gb->getId(); } mSlots[slot].mBufferState.detachProducer(); mCore->mActiveBuffers.erase(slot); @@ -695,6 +704,10 @@ status_t BufferQueueProducer::detachBuffer(int slot) { VALIDATE_CONSISTENCY(); } + if (listener != nullptr && callOnFrameDetached) { + listener->onFrameDetached(bufferId); + } + if (listener != nullptr) { listener->onBuffersReleased(); } @@ -1104,57 +1117,70 @@ status_t BufferQueueProducer::queueBuffer(int slot, status_t BufferQueueProducer::cancelBuffer(int slot, const sp<Fence>& fence) { ATRACE_CALL(); BQ_LOGV("cancelBuffer: slot %d", slot); - std::lock_guard<std::mutex> lock(mCore->mMutex); - if (mCore->mIsAbandoned) { - BQ_LOGE("cancelBuffer: BufferQueue has been abandoned"); - return NO_INIT; - } + sp<IConsumerListener> listener; + bool callOnFrameCancelled = false; + uint64_t bufferId = 0; // Only used if callOnFrameCancelled == true + { + std::lock_guard<std::mutex> lock(mCore->mMutex); - if (mCore->mConnectedApi == BufferQueueCore::NO_CONNECTED_API) { - BQ_LOGE("cancelBuffer: BufferQueue has no connected producer"); - return NO_INIT; - } + if (mCore->mIsAbandoned) { + BQ_LOGE("cancelBuffer: BufferQueue has been abandoned"); + return NO_INIT; + } - if (mCore->mSharedBufferMode) { - BQ_LOGE("cancelBuffer: cannot cancel a buffer in shared buffer mode"); - return BAD_VALUE; - } + if (mCore->mConnectedApi == BufferQueueCore::NO_CONNECTED_API) { + BQ_LOGE("cancelBuffer: BufferQueue has no connected producer"); + return NO_INIT; + } - if (slot < 0 || slot >= BufferQueueDefs::NUM_BUFFER_SLOTS) { - BQ_LOGE("cancelBuffer: slot index %d out of range [0, %d)", - slot, BufferQueueDefs::NUM_BUFFER_SLOTS); - return BAD_VALUE; - } else if (!mSlots[slot].mBufferState.isDequeued()) { - BQ_LOGE("cancelBuffer: slot %d is not owned by the producer " - "(state = %s)", slot, mSlots[slot].mBufferState.string()); - return BAD_VALUE; - } else if (fence == nullptr) { - BQ_LOGE("cancelBuffer: fence is NULL"); - return BAD_VALUE; - } + if (mCore->mSharedBufferMode) { + BQ_LOGE("cancelBuffer: cannot cancel a buffer in shared buffer mode"); + return BAD_VALUE; + } + + if (slot < 0 || slot >= BufferQueueDefs::NUM_BUFFER_SLOTS) { + BQ_LOGE("cancelBuffer: slot index %d out of range [0, %d)", slot, + BufferQueueDefs::NUM_BUFFER_SLOTS); + return BAD_VALUE; + } else if (!mSlots[slot].mBufferState.isDequeued()) { + BQ_LOGE("cancelBuffer: slot %d is not owned by the producer " + "(state = %s)", + slot, mSlots[slot].mBufferState.string()); + return BAD_VALUE; + } else if (fence == nullptr) { + BQ_LOGE("cancelBuffer: fence is NULL"); + return BAD_VALUE; + } - mSlots[slot].mBufferState.cancel(); + mSlots[slot].mBufferState.cancel(); - // After leaving shared buffer mode, the shared buffer will still be around. - // Mark it as no longer shared if this operation causes it to be free. - if (!mCore->mSharedBufferMode && mSlots[slot].mBufferState.isFree()) { - mSlots[slot].mBufferState.mShared = false; - } + // After leaving shared buffer mode, the shared buffer will still be around. + // Mark it as no longer shared if this operation causes it to be free. + if (!mCore->mSharedBufferMode && mSlots[slot].mBufferState.isFree()) { + mSlots[slot].mBufferState.mShared = false; + } - // Don't put the shared buffer on the free list. - if (!mSlots[slot].mBufferState.isShared()) { - mCore->mActiveBuffers.erase(slot); - mCore->mFreeBuffers.push_back(slot); + // Don't put the shared buffer on the free list. + if (!mSlots[slot].mBufferState.isShared()) { + mCore->mActiveBuffers.erase(slot); + mCore->mFreeBuffers.push_back(slot); + } + + auto gb = mSlots[slot].mGraphicBuffer; + if (gb != nullptr) { + callOnFrameCancelled = true; + bufferId = gb->getId(); + } + mSlots[slot].mFence = fence; + mCore->mDequeueCondition.notify_all(); + listener = mCore->mConsumerListener; + VALIDATE_CONSISTENCY(); } - auto gb = mSlots[slot].mGraphicBuffer; - if (mCore->mConsumerListener != nullptr && gb != nullptr) { - mCore->mConsumerListener->onFrameCancelled(gb->getId()); + if (listener != nullptr && callOnFrameCancelled) { + listener->onFrameCancelled(bufferId); } - mSlots[slot].mFence = fence; - mCore->mDequeueCondition.notify_all(); - VALIDATE_CONSISTENCY(); return NO_ERROR; } diff --git a/libs/gui/Surface.cpp b/libs/gui/Surface.cpp index ed691006e9..53a2f64d11 100644 --- a/libs/gui/Surface.cpp +++ b/libs/gui/Surface.cpp @@ -1792,19 +1792,20 @@ int Surface::dispatchGetLastQueuedBuffer2(va_list args) { int Surface::dispatchSetFrameTimelineInfo(va_list args) { ATRACE_CALL(); - auto frameNumber = static_cast<uint64_t>(va_arg(args, uint64_t)); - auto frameTimelineVsyncId = static_cast<int64_t>(va_arg(args, int64_t)); - auto inputEventId = static_cast<int32_t>(va_arg(args, int32_t)); - auto startTimeNanos = static_cast<int64_t>(va_arg(args, int64_t)); - auto useForRefreshRateSelection = static_cast<bool>(va_arg(args, int32_t)); - ALOGV("Surface::%s", __func__); + + const auto nativeWindowFtlInfo = static_cast<ANativeWindowFrameTimelineInfo>( + va_arg(args, ANativeWindowFrameTimelineInfo)); + FrameTimelineInfo ftlInfo; - ftlInfo.vsyncId = frameTimelineVsyncId; - ftlInfo.inputEventId = inputEventId; - ftlInfo.startTimeNanos = startTimeNanos; - ftlInfo.useForRefreshRateSelection = useForRefreshRateSelection; - return setFrameTimelineInfo(frameNumber, ftlInfo); + ftlInfo.vsyncId = nativeWindowFtlInfo.frameTimelineVsyncId; + ftlInfo.inputEventId = nativeWindowFtlInfo.inputEventId; + ftlInfo.startTimeNanos = nativeWindowFtlInfo.startTimeNanos; + ftlInfo.useForRefreshRateSelection = nativeWindowFtlInfo.useForRefreshRateSelection; + ftlInfo.skippedFrameVsyncId = nativeWindowFtlInfo.skippedFrameVsyncId; + ftlInfo.skippedFrameStartTimeNanos = nativeWindowFtlInfo.skippedFrameStartTimeNanos; + + return setFrameTimelineInfo(nativeWindowFtlInfo.frameNumber, ftlInfo); } bool Surface::transformToDisplayInverse() const { diff --git a/libs/gui/SurfaceComposerClient.cpp b/libs/gui/SurfaceComposerClient.cpp index 8a1f7c6238..00495ee5f6 100644 --- a/libs/gui/SurfaceComposerClient.cpp +++ b/libs/gui/SurfaceComposerClient.cpp @@ -59,7 +59,7 @@ #include <private/gui/ComposerServiceAIDL.h> // This server size should always be smaller than the server cache size -#define BUFFER_CACHE_MAX_SIZE 64 +#define BUFFER_CACHE_MAX_SIZE 4096 namespace android { @@ -1027,7 +1027,7 @@ void SurfaceComposerClient::Transaction::clear() { mEarlyWakeupEnd = false; mDesiredPresentTime = 0; mIsAutoTimestamp = true; - clearFrameTimelineInfo(mFrameTimelineInfo); + mFrameTimelineInfo = {}; mApplyToken = nullptr; mMergedTransactionIds.clear(); } @@ -1302,6 +1302,13 @@ sp<IBinder> SurfaceComposerClient::getPhysicalDisplayToken(PhysicalDisplayId dis return status.isOk() ? display : nullptr; } +std::optional<gui::StalledTransactionInfo> SurfaceComposerClient::getStalledTransactionInfo( + pid_t pid) { + std::optional<gui::StalledTransactionInfo> result; + ComposerServiceAIDL::getComposerService()->getStalledTransactionInfo(pid, &result); + return result; +} + void SurfaceComposerClient::Transaction::setAnimationTransaction() { mAnimation = true; } @@ -2279,27 +2286,13 @@ void SurfaceComposerClient::Transaction::mergeFrameTimelineInfo(FrameTimelineInf if (t.vsyncId != FrameTimelineInfo::INVALID_VSYNC_ID && other.vsyncId != FrameTimelineInfo::INVALID_VSYNC_ID) { if (other.vsyncId > t.vsyncId) { - t.vsyncId = other.vsyncId; - t.inputEventId = other.inputEventId; - t.startTimeNanos = other.startTimeNanos; - t.useForRefreshRateSelection = other.useForRefreshRateSelection; + t = other; } } else if (t.vsyncId == FrameTimelineInfo::INVALID_VSYNC_ID) { - t.vsyncId = other.vsyncId; - t.inputEventId = other.inputEventId; - t.startTimeNanos = other.startTimeNanos; - t.useForRefreshRateSelection = other.useForRefreshRateSelection; + t = other; } } -// copied from FrameTimelineInfo::clear() -void SurfaceComposerClient::Transaction::clearFrameTimelineInfo(FrameTimelineInfo& t) { - t.vsyncId = FrameTimelineInfo::INVALID_VSYNC_ID; - t.inputEventId = os::IInputConstants::INVALID_INPUT_EVENT_ID; - t.startTimeNanos = 0; - t.useForRefreshRateSelection = false; -} - SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setTrustedPresentationCallback( const sp<SurfaceControl>& sc, TrustedPresentationCallback cb, @@ -2523,38 +2516,41 @@ status_t SurfaceComposerClient::getStaticDisplayInfo(int64_t displayId, outInfo->secure = ginfo.secure; outInfo->installOrientation = static_cast<ui::Rotation>(ginfo.installOrientation); - DeviceProductInfo info; - std::optional<gui::DeviceProductInfo> dpi = ginfo.deviceProductInfo; - gui::DeviceProductInfo::ManufactureOrModelDate& date = dpi->manufactureOrModelDate; - info.name = dpi->name; - if (dpi->manufacturerPnpId.size() > 0) { - // copid from PnpId = std::array<char, 4> in ui/DeviceProductInfo.h - constexpr int kMaxPnpIdSize = 4; - size_t count = std::max<size_t>(kMaxPnpIdSize, dpi->manufacturerPnpId.size()); - std::copy_n(dpi->manufacturerPnpId.begin(), count, info.manufacturerPnpId.begin()); - } - if (dpi->relativeAddress.size() > 0) { - std::copy(dpi->relativeAddress.begin(), dpi->relativeAddress.end(), - std::back_inserter(info.relativeAddress)); - } - info.productId = dpi->productId; - if (date.getTag() == Tag::modelYear) { - DeviceProductInfo::ModelYear modelYear; - modelYear.year = static_cast<uint32_t>(date.get<Tag::modelYear>().year); - info.manufactureOrModelDate = modelYear; - } else if (date.getTag() == Tag::manufactureYear) { - DeviceProductInfo::ManufactureYear manufactureYear; - manufactureYear.year = date.get<Tag::manufactureYear>().modelYear.year; - info.manufactureOrModelDate = manufactureYear; - } else if (date.getTag() == Tag::manufactureWeekAndYear) { - DeviceProductInfo::ManufactureWeekAndYear weekAndYear; - weekAndYear.year = - date.get<Tag::manufactureWeekAndYear>().manufactureYear.modelYear.year; - weekAndYear.week = date.get<Tag::manufactureWeekAndYear>().week; - info.manufactureOrModelDate = weekAndYear; - } + if (const std::optional<gui::DeviceProductInfo> dpi = ginfo.deviceProductInfo) { + DeviceProductInfo info; + info.name = dpi->name; + if (dpi->manufacturerPnpId.size() > 0) { + // copid from PnpId = std::array<char, 4> in ui/DeviceProductInfo.h + constexpr int kMaxPnpIdSize = 4; + size_t count = std::max<size_t>(kMaxPnpIdSize, dpi->manufacturerPnpId.size()); + std::copy_n(dpi->manufacturerPnpId.begin(), count, info.manufacturerPnpId.begin()); + } + if (dpi->relativeAddress.size() > 0) { + std::copy(dpi->relativeAddress.begin(), dpi->relativeAddress.end(), + std::back_inserter(info.relativeAddress)); + } + info.productId = dpi->productId; + + const gui::DeviceProductInfo::ManufactureOrModelDate& date = + dpi->manufactureOrModelDate; + if (date.getTag() == Tag::modelYear) { + DeviceProductInfo::ModelYear modelYear; + modelYear.year = static_cast<uint32_t>(date.get<Tag::modelYear>().year); + info.manufactureOrModelDate = modelYear; + } else if (date.getTag() == Tag::manufactureYear) { + DeviceProductInfo::ManufactureYear manufactureYear; + manufactureYear.year = date.get<Tag::manufactureYear>().modelYear.year; + info.manufactureOrModelDate = manufactureYear; + } else if (date.getTag() == Tag::manufactureWeekAndYear) { + DeviceProductInfo::ManufactureWeekAndYear weekAndYear; + weekAndYear.year = + date.get<Tag::manufactureWeekAndYear>().manufactureYear.modelYear.year; + weekAndYear.week = date.get<Tag::manufactureWeekAndYear>().week; + info.manufactureOrModelDate = weekAndYear; + } - outInfo->deviceProductInfo = info; + outInfo->deviceProductInfo = info; + } } return statusTFromBinderStatus(status); } @@ -2754,6 +2750,20 @@ status_t SurfaceComposerClient::setOverrideFrameRate(uid_t uid, float frameRate) return statusTFromBinderStatus(status); } +status_t SurfaceComposerClient::updateSmallAreaDetection(std::vector<int32_t>& uids, + std::vector<float>& thresholds) { + binder::Status status = + ComposerServiceAIDL::getComposerService()->updateSmallAreaDetection(uids, thresholds); + return statusTFromBinderStatus(status); +} + +status_t SurfaceComposerClient::setSmallAreaDetectionThreshold(uid_t uid, float threshold) { + binder::Status status = + ComposerServiceAIDL::getComposerService()->setSmallAreaDetectionThreshold(uid, + threshold); + return statusTFromBinderStatus(status); +} + void SurfaceComposerClient::setAutoLowLatencyMode(const sp<IBinder>& display, bool on) { ComposerServiceAIDL::getComposerService()->setAutoLowLatencyMode(display, on); } diff --git a/libs/gui/TEST_MAPPING b/libs/gui/TEST_MAPPING index 941503548d..a4d9e77351 100644 --- a/libs/gui/TEST_MAPPING +++ b/libs/gui/TEST_MAPPING @@ -4,10 +4,58 @@ "path": "frameworks/native/libs/nativewindow" } ], - "postsubmit": [ + "presubmit": [ { - // TODO(257123981): move this to presubmit after dealing with existing breakages. - "name": "libgui_test" + "name": "libgui_test", + "options": [ + // TODO(b/277604286): Failing on Cuttlefish. + { + "exclude-filter": "MultiTextureConsumerTest#EGLImageTargetWorks" + }, + + // TODO(b/285011590): Failing on Cuttlefish. + { + "exclude-filter": "SurfaceTest#GetHdrSupport" + }, + { + "exclude-filter": "SurfaceTest#GetWideColorSupport" + }, + + // TODO(b/285006554): Failing on Cuttlefish. + { + "exclude-filter": "SurfaceTextureGLTest#InvalidWidthOrHeightFails" + }, + + // TODO(b/277347351): Known test data issues, failing across devices. + { + "exclude-filter": "SurfaceTextureGLTest#TexturingFromCpuFilledYV12BufferNpot" + }, + { + "exclude-filter": "SurfaceTextureGLTest#TexturingFromCpuFilledYV12BufferPow2" + }, + { + "exclude-filter": "SurfaceTextureGLTest#TexturingFromCpuFilledYV12BufferWithCrop" + }, + { + "exclude-filter": "SurfaceTextureGLTest#TexturingFromCpuFilledYV12BuffersRepeatedly" + }, + + // TODO(b/285041169): Hanging on Cuttlefish. + { + "exclude-filter": "SurfaceTextureGLThreadToGLTest#UpdateTexImageBeforeFrameFinishedCompletes" + }, + { + "exclude-filter": "SurfaceTextureGLThreadToGLTest#RepeatedUpdateTexImageBeforeFrameFinishedCompletes" + }, + { + "exclude-filter": "SurfaceTextureGLThreadToGLTest#RepeatedUpdateTexImageAfterFrameFinishedCompletes" + }, + + // TODO(b/285041070): Failing on Cuttlefish. + { + "exclude-filter": "SurfaceTextureGLToGLTest#EglDestroySurfaceUnrefsBuffers" + } + ] } ] } diff --git a/libs/gui/WindowInfo.cpp b/libs/gui/WindowInfo.cpp index 6df9ff1664..52af9d5114 100644 --- a/libs/gui/WindowInfo.cpp +++ b/libs/gui/WindowInfo.cpp @@ -90,8 +90,10 @@ status_t WindowInfo::writeToParcel(android::Parcel* parcel) const { } parcel->writeInt32(1); - // Ensure that the size of the flags that we use is 32 bits for writing into the parcel. + // Ensure that the size of custom types are what we expect for writing into the parcel. static_assert(sizeof(inputConfig) == 4u); + static_assert(sizeof(ownerPid.val()) == 4u); + static_assert(sizeof(ownerUid.val()) == 4u); // clang-format off status_t status = parcel->writeStrongBinder(token) ?: @@ -115,8 +117,8 @@ status_t WindowInfo::writeToParcel(android::Parcel* parcel) const { parcel->writeFloat(transform.dsdy()) ?: parcel->writeFloat(transform.ty()) ?: parcel->writeInt32(static_cast<int32_t>(touchOcclusionMode)) ?: - parcel->writeInt32(ownerPid) ?: - parcel->writeInt32(ownerUid) ?: + parcel->writeInt32(ownerPid.val()) ?: + parcel->writeInt32(ownerUid.val()) ?: parcel->writeUtf8AsUtf16(packageName) ?: parcel->writeInt32(inputConfig.get()) ?: parcel->writeInt32(displayId) ?: @@ -147,7 +149,7 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) { } float dsdx, dtdx, tx, dtdy, dsdy, ty; - int32_t lpFlags, lpType, touchOcclusionModeInt, inputConfigInt; + int32_t lpFlags, lpType, touchOcclusionModeInt, inputConfigInt, ownerPidInt, ownerUidInt; sp<IBinder> touchableRegionCropHandleSp; // clang-format off @@ -167,8 +169,8 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) { parcel->readFloat(&dsdy) ?: parcel->readFloat(&ty) ?: parcel->readInt32(&touchOcclusionModeInt) ?: - parcel->readInt32(&ownerPid) ?: - parcel->readInt32(&ownerUid) ?: + parcel->readInt32(&ownerPidInt) ?: + parcel->readInt32(&ownerUidInt) ?: parcel->readUtf8FromUtf16(&packageName) ?: parcel->readInt32(&inputConfigInt) ?: parcel->readInt32(&displayId) ?: @@ -190,6 +192,8 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) { transform.set({dsdx, dtdx, tx, dtdy, dsdy, ty, 0, 0, 1}); touchOcclusionMode = static_cast<TouchOcclusionMode>(touchOcclusionModeInt); inputConfig = ftl::Flags<InputConfig>(inputConfigInt); + ownerPid = Pid{ownerPidInt}; + ownerUid = Uid{static_cast<uid_t>(ownerUidInt)}; touchableRegionCropHandle = touchableRegionCropHandleSp; return OK; diff --git a/libs/gui/aidl/android/gui/FrameTimelineInfo.aidl b/libs/gui/aidl/android/gui/FrameTimelineInfo.aidl index 6a86c6a5cd..4b647a4ad2 100644 --- a/libs/gui/aidl/android/gui/FrameTimelineInfo.aidl +++ b/libs/gui/aidl/android/gui/FrameTimelineInfo.aidl @@ -37,4 +37,10 @@ parcelable FrameTimelineInfo { // Whether this vsyncId should be used to heuristically select the display refresh rate // TODO(b/281695725): Clean this up once TextureView use setFrameRate API boolean useForRefreshRateSelection = false; + + // The VsyncId of a frame that was not drawn and squashed into this frame. + long skippedFrameVsyncId = INVALID_VSYNC_ID; + + // The start time of a frame that was not drawn and squashed into this frame. + long skippedFrameStartTimeNanos = 0; } diff --git a/libs/gui/aidl/android/gui/ISurfaceComposer.aidl b/libs/gui/aidl/android/gui/ISurfaceComposer.aidl index 539a1c140e..516d159cc5 100644 --- a/libs/gui/aidl/android/gui/ISurfaceComposer.aidl +++ b/libs/gui/aidl/android/gui/ISurfaceComposer.aidl @@ -46,6 +46,7 @@ import android.gui.LayerDebugInfo; import android.gui.OverlayProperties; import android.gui.PullAtomData; import android.gui.ARect; +import android.gui.StalledTransactionInfo; import android.gui.StaticDisplayInfo; import android.gui.WindowInfosListenerInfo; @@ -480,6 +481,15 @@ interface ISurfaceComposer { */ void setOverrideFrameRate(int uid, float frameRate); + oneway void updateSmallAreaDetection(in int[] uids, in float[] thresholds); + + /** + * Set the small area detection threshold for a specified uid by SmallAreaDetectionController. + * Passing the threshold and uid to SurfaceFlinger to update the uid-threshold mapping + * in the scheduler. + */ + oneway void setSmallAreaDetectionThreshold(int uid, float threshold); + /** * Gets priority of the RenderEngine in SurfaceFlinger. */ @@ -507,4 +517,10 @@ interface ISurfaceComposer { void removeWindowInfosListener(IWindowInfosListener windowInfosListener); OverlayProperties getOverlaySupport(); + + /** + * Returns an instance of StalledTransaction if a transaction from the passed pid has not been + * applied in SurfaceFlinger due to an unsignaled fence. Otherwise, null is returned. + */ + @nullable StalledTransactionInfo getStalledTransactionInfo(int pid); } diff --git a/libs/gui/android/gui/StalledTransactionInfo.aidl b/libs/gui/android/gui/StalledTransactionInfo.aidl new file mode 100644 index 0000000000..e6aa9bd1c7 --- /dev/null +++ b/libs/gui/android/gui/StalledTransactionInfo.aidl @@ -0,0 +1,24 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.gui; + +/** @hide */ +parcelable StalledTransactionInfo { + String layerName; + long bufferId; + long frameNumber; +}
\ No newline at end of file diff --git a/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp b/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp index f684874aec..fd8ffe1f01 100644 --- a/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp +++ b/libs/gui/bufferqueue/1.0/H2BGraphicBufferProducer.cpp @@ -1172,9 +1172,12 @@ status_t H2BGraphicBufferProducer::setGenerationNumber(uint32_t generationNumber String8 H2BGraphicBufferProducer::getConsumerName() const { String8 lName; - mBase->getConsumerName([&lName] (hidl_string const& name) { - lName = name.c_str(); - }); + status_t transStatus = toStatusT( + mBase->getConsumerName([&lName](hidl_string const& name) { lName = name.c_str(); })); + if (transStatus != NO_ERROR) { + ALOGE("getConsumerName failed to transact: %d", transStatus); + return String8("TransactFailed"); + } return lName; } diff --git a/libs/gui/bufferqueue/2.0/H2BGraphicBufferProducer.cpp b/libs/gui/bufferqueue/2.0/H2BGraphicBufferProducer.cpp index 2f5b73ccbb..ae00a2642e 100644 --- a/libs/gui/bufferqueue/2.0/H2BGraphicBufferProducer.cpp +++ b/libs/gui/bufferqueue/2.0/H2BGraphicBufferProducer.cpp @@ -437,6 +437,10 @@ String8 H2BGraphicBufferProducer::getConsumerName() const { [&bName](hidl_string const& name) { bName = name.c_str(); }); + if (!transResult.isOk()) { + LOG(ERROR) << "getConsumerName: corrupted transaction."; + return String8("TransactFailed"); + } return bName; } diff --git a/libs/gui/fuzzer/libgui_fuzzer_utils.h b/libs/gui/fuzzer/libgui_fuzzer_utils.h index 4c7d0562af..67915f905e 100644 --- a/libs/gui/fuzzer/libgui_fuzzer_utils.h +++ b/libs/gui/fuzzer/libgui_fuzzer_utils.h @@ -151,6 +151,9 @@ public: MOCK_METHOD(binder::Status, getDisplayDecorationSupport, (const sp<IBinder>&, std::optional<gui::DisplayDecorationSupport>*), (override)); MOCK_METHOD(binder::Status, setOverrideFrameRate, (int32_t, float), (override)); + MOCK_METHOD(binder::Status, updateSmallAreaDetection, + (const std::vector<int32_t>&, const std::vector<float>&), (override)); + MOCK_METHOD(binder::Status, setSmallAreaDetectionThreshold, (int32_t, float), (override)); MOCK_METHOD(binder::Status, getGpuContextPriority, (int32_t*), (override)); MOCK_METHOD(binder::Status, getMaxAcquiredBufferCount, (int32_t*), (override)); MOCK_METHOD(binder::Status, addWindowInfosListener, @@ -158,6 +161,8 @@ public: MOCK_METHOD(binder::Status, removeWindowInfosListener, (const sp<gui::IWindowInfosListener>&), (override)); MOCK_METHOD(binder::Status, getOverlaySupport, (gui::OverlayProperties*), (override)); + MOCK_METHOD(binder::Status, getStalledTransactionInfo, + (int32_t, std::optional<gui::StalledTransactionInfo>*), (override)); }; class FakeBnSurfaceComposerClient : public gui::BnSurfaceComposerClient { diff --git a/libs/gui/fuzzer/libgui_surfaceComposerClient_fuzzer.cpp b/libs/gui/fuzzer/libgui_surfaceComposerClient_fuzzer.cpp index 57720dd513..95b7f39c11 100644 --- a/libs/gui/fuzzer/libgui_surfaceComposerClient_fuzzer.cpp +++ b/libs/gui/fuzzer/libgui_surfaceComposerClient_fuzzer.cpp @@ -186,8 +186,8 @@ void SurfaceComposerClientFuzzer::getWindowInfo(gui::WindowInfo* windowInfo) { windowInfo->touchableRegion = Region(getRect(&mFdp)); windowInfo->replaceTouchableRegionWithCrop = mFdp.ConsumeBool(); windowInfo->touchOcclusionMode = mFdp.PickValueInArray(kMode); - windowInfo->ownerPid = mFdp.ConsumeIntegral<int32_t>(); - windowInfo->ownerUid = mFdp.ConsumeIntegral<int32_t>(); + windowInfo->ownerPid = gui::Pid{mFdp.ConsumeIntegral<pid_t>()}; + windowInfo->ownerUid = gui::Uid{mFdp.ConsumeIntegral<uid_t>()}; windowInfo->packageName = mFdp.ConsumeRandomLengthString(kRandomStringMaxBytes); windowInfo->inputConfig = mFdp.PickValueInArray(kFeatures); } diff --git a/libs/gui/include/gui/LayerState.h b/libs/gui/include/gui/LayerState.h index a6f503ef55..62e5f89d21 100644 --- a/libs/gui/include/gui/LayerState.h +++ b/libs/gui/include/gui/LayerState.h @@ -270,9 +270,9 @@ struct layer_state_t { layer_state_t::eFrameRateChanged | layer_state_t::eFixedTransformHintChanged; // Changes affecting data sent to input. - static constexpr uint64_t INPUT_CHANGES = layer_state_t::GEOMETRY_CHANGES | - layer_state_t::HIERARCHY_CHANGES | layer_state_t::eInputInfoChanged | - layer_state_t::eDropInputModeChanged | layer_state_t::eTrustedOverlayChanged; + static constexpr uint64_t INPUT_CHANGES = layer_state_t::eInputInfoChanged | + layer_state_t::eDropInputModeChanged | layer_state_t::eTrustedOverlayChanged | + layer_state_t::eLayerStackChanged; // Changes that affect the visible region on a display. static constexpr uint64_t VISIBLE_REGION_CHANGES = diff --git a/libs/gui/include/gui/PidUid.h b/libs/gui/include/gui/PidUid.h new file mode 100644 index 0000000000..7930942882 --- /dev/null +++ b/libs/gui/include/gui/PidUid.h @@ -0,0 +1,56 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <ftl/mixins.h> +#include <sys/types.h> +#include <string> + +namespace android::gui { + +// Type-safe wrapper for a PID. +struct Pid : ftl::Constructible<Pid, pid_t>, ftl::Equatable<Pid>, ftl::Orderable<Pid> { + using Constructible::Constructible; + + const static Pid INVALID; + + constexpr auto val() const { return ftl::to_underlying(*this); } + + constexpr bool isValid() const { return val() >= 0; } + + std::string toString() const { return std::to_string(val()); } +}; + +const inline Pid Pid::INVALID{-1}; + +// Type-safe wrapper for a UID. +// We treat the unsigned equivalent of -1 as a singular invalid value. +struct Uid : ftl::Constructible<Uid, uid_t>, ftl::Equatable<Uid>, ftl::Orderable<Uid> { + using Constructible::Constructible; + + const static Uid INVALID; + + constexpr auto val() const { return ftl::to_underlying(*this); } + + constexpr bool isValid() const { return val() != static_cast<uid_t>(-1); } + + std::string toString() const { return std::to_string(val()); } +}; + +const inline Uid Uid::INVALID{static_cast<uid_t>(-1)}; + +} // namespace android::gui diff --git a/libs/gui/include/gui/SurfaceComposerClient.h b/libs/gui/include/gui/SurfaceComposerClient.h index fb57f63dad..7c55100784 100644 --- a/libs/gui/include/gui/SurfaceComposerClient.h +++ b/libs/gui/include/gui/SurfaceComposerClient.h @@ -203,6 +203,16 @@ public: // by GameManager. static status_t setOverrideFrameRate(uid_t uid, float frameRate); + // Update the small area detection whole uid-threshold mappings by same size uid and threshold + // vector. + // Ref:setSmallAreaDetectionThreshold. + static status_t updateSmallAreaDetection(std::vector<int32_t>& uids, + std::vector<float>& thresholds); + + // Sets the small area detection threshold to particular apps (uid). Passing value 0 means + // to disable small area detection to the app. + static status_t setSmallAreaDetectionThreshold(uid_t uid, float threshold); + // Switches on/off Auto Low Latency Mode on the connected display. This should only be // called if the connected display supports Auto Low Latency Mode as reported by // #getAutoLowLatencyModeSupport @@ -371,6 +381,10 @@ public: //! Get token for a physical display given its stable ID static sp<IBinder> getPhysicalDisplayToken(PhysicalDisplayId displayId); + // Returns StalledTransactionInfo if a transaction from the provided pid has not been applied + // due to an unsignaled fence. + static std::optional<gui::StalledTransactionInfo> getStalledTransactionInfo(pid_t pid); + struct SCHash { std::size_t operator()(const sp<SurfaceControl>& sc) const { return std::hash<SurfaceControl *>{}(sc.get()); @@ -410,7 +424,6 @@ public: static sp<IBinder> sApplyToken; void releaseBufferIfOverwriting(const layer_state_t& state); static void mergeFrameTimelineInfo(FrameTimelineInfo& t, const FrameTimelineInfo& other); - static void clearFrameTimelineInfo(FrameTimelineInfo& t); protected: std::unordered_map<sp<IBinder>, ComposerState, IBinderHash> mComposerStates; diff --git a/libs/gui/include/gui/WindowInfo.h b/libs/gui/include/gui/WindowInfo.h index 70b2ee8e32..7ff73874ae 100644 --- a/libs/gui/include/gui/WindowInfo.h +++ b/libs/gui/include/gui/WindowInfo.h @@ -21,6 +21,8 @@ #include <binder/Parcel.h> #include <binder/Parcelable.h> #include <ftl/flags.h> +#include <ftl/mixins.h> +#include <gui/PidUid.h> #include <gui/constants.h> #include <ui/Rect.h> #include <ui/Region.h> @@ -223,8 +225,8 @@ struct WindowInfo : public Parcelable { Region touchableRegion; TouchOcclusionMode touchOcclusionMode = TouchOcclusionMode::BLOCK_UNTRUSTED; - int32_t ownerPid = -1; - int32_t ownerUid = -1; + Pid ownerPid = Pid::INVALID; + Uid ownerUid = Uid::INVALID; std::string packageName; ftl::Flags<InputConfig> inputConfig; int32_t displayId = ADISPLAY_ID_NONE; diff --git a/libs/gui/tests/Android.bp b/libs/gui/tests/Android.bp index cd35d2fe3c..462ce6e14f 100644 --- a/libs/gui/tests/Android.bp +++ b/libs/gui/tests/Android.bp @@ -21,6 +21,7 @@ cc_test { ], srcs: [ + "LibGuiMain.cpp", // Custom gtest entrypoint "BLASTBufferQueue_test.cpp", "BufferItemConsumer_test.cpp", "BufferQueue_test.cpp", diff --git a/libs/gui/tests/AndroidTest.xml b/libs/gui/tests/AndroidTest.xml index 5e09fff6bb..31b10d7f54 100644 --- a/libs/gui/tests/AndroidTest.xml +++ b/libs/gui/tests/AndroidTest.xml @@ -23,6 +23,7 @@ <option name="screen-always-on" value="on" /> </target_preparer> <option name="test-suite-tag" value="apct" /> + <option name="not-shardable" value="true" /> <test class="com.android.tradefed.testtype.GTest" > <option name="native-test-device-path" value="/data/local/tmp" /> <option name="module-name" value="libgui_test" /> diff --git a/libs/gui/tests/BLASTBufferQueue_test.cpp b/libs/gui/tests/BLASTBufferQueue_test.cpp index a3ad6807c5..cd90168784 100644 --- a/libs/gui/tests/BLASTBufferQueue_test.cpp +++ b/libs/gui/tests/BLASTBufferQueue_test.cpp @@ -176,18 +176,6 @@ private: class BLASTBufferQueueTest : public ::testing::Test { public: protected: - BLASTBufferQueueTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), testInfo->name()); - } - - ~BLASTBufferQueueTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), testInfo->name()); - } - void SetUp() { mComposer = ComposerService::getComposerService(); mClient = new SurfaceComposerClient(); diff --git a/libs/gui/tests/BufferQueue_test.cpp b/libs/gui/tests/BufferQueue_test.cpp index 185ff83a86..2d50b4de74 100644 --- a/libs/gui/tests/BufferQueue_test.cpp +++ b/libs/gui/tests/BufferQueue_test.cpp @@ -17,6 +17,7 @@ #define LOG_TAG "BufferQueue_test" //#define LOG_NDEBUG 0 +#include "Constants.h" #include "MockConsumer.h" #include <gui/BufferItem.h> @@ -46,20 +47,6 @@ class BufferQueueTest : public ::testing::Test { public: protected: - BufferQueueTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } - - ~BufferQueueTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } - void GetMinUndequeuedBufferCount(int* bufferCount) { ASSERT_TRUE(bufferCount != nullptr); ASSERT_EQ(OK, mProducer->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, @@ -535,7 +522,8 @@ TEST_F(BufferQueueTest, TestGenerationNumbers) { int slot; sp<Fence> fence; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); sp<GraphicBuffer> buffer; ASSERT_EQ(OK, mProducer->requestBuffer(slot, &buffer)); @@ -578,7 +566,8 @@ TEST_F(BufferQueueTest, TestSharedBufferModeWithoutAutoRefresh) { sp<Fence> fence; sp<GraphicBuffer> buffer; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->requestBuffer(sharedSlot, &buffer)); // Queue the buffer @@ -592,7 +581,9 @@ TEST_F(BufferQueueTest, TestSharedBufferModeWithoutAutoRefresh) { // always the same one and because async mode gets enabled. int slot; for (int i = 0; i < 5; i++) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(sharedSlot, slot); ASSERT_EQ(OK, mProducer->queueBuffer(sharedSlot, input, &output)); } @@ -629,7 +620,8 @@ TEST_F(BufferQueueTest, TestSharedBufferModeWithAutoRefresh) { sp<Fence> fence; sp<GraphicBuffer> buffer; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->requestBuffer(sharedSlot, &buffer)); // Queue the buffer @@ -656,7 +648,9 @@ TEST_F(BufferQueueTest, TestSharedBufferModeWithAutoRefresh) { // always return the same one. int slot; for (int i = 0; i < 5; i++) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(sharedSlot, slot); ASSERT_EQ(OK, mProducer->queueBuffer(sharedSlot, input, &output)); } @@ -695,7 +689,8 @@ TEST_F(BufferQueueTest, TestSharedBufferModeUsingAlreadyDequeuedBuffer) { sp<Fence> fence; sp<GraphicBuffer> buffer; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&sharedSlot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->requestBuffer(sharedSlot, &buffer)); // Enable shared buffer mode @@ -712,7 +707,9 @@ TEST_F(BufferQueueTest, TestSharedBufferModeUsingAlreadyDequeuedBuffer) { // always the same one and because async mode gets enabled. int slot; for (int i = 0; i < 5; i++) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(sharedSlot, slot); ASSERT_EQ(OK, mProducer->queueBuffer(sharedSlot, input, &output)); } @@ -747,7 +744,8 @@ TEST_F(BufferQueueTest, TestTimeouts) { for (int i = 0; i < 5; ++i) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> fence = Fence::NO_FENCE; - auto result = mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr); + auto result = mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr); if (i < 2) { ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); @@ -774,7 +772,9 @@ TEST_F(BufferQueueTest, TestTimeouts) { for (int i = 0; i < 2; ++i) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> fence = Fence::NO_FENCE; - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->requestBuffer(slot, &buffer)); IGraphicBufferProducer::QueueBufferInput input(0ull, true, HAL_DATASPACE_UNKNOWN, Rect::INVALID_RECT, @@ -785,7 +785,9 @@ TEST_F(BufferQueueTest, TestTimeouts) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> fence = Fence::NO_FENCE; auto startTime = systemTime(); - ASSERT_EQ(TIMED_OUT, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(TIMED_OUT, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_GE(systemTime() - startTime, TIMEOUT); // We're technically attaching the same buffer multiple times (since we @@ -806,7 +808,8 @@ TEST_F(BufferQueueTest, CanAttachWhileDisallowingAllocation) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> sourceFence; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&slot, &sourceFence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&slot, &sourceFence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); sp<GraphicBuffer> buffer; ASSERT_EQ(OK, mProducer->requestBuffer(slot, &buffer)); ASSERT_EQ(OK, mProducer->detachBuffer(slot)); @@ -829,7 +832,8 @@ TEST_F(BufferQueueTest, CanRetrieveLastQueuedBuffer) { int slot = BufferQueue::INVALID_BUFFER_SLOT; sp<Fence> fence; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); sp<GraphicBuffer> firstBuffer; ASSERT_EQ(OK, mProducer->requestBuffer(slot, &firstBuffer)); @@ -841,7 +845,8 @@ TEST_F(BufferQueueTest, CanRetrieveLastQueuedBuffer) { // Dequeue a second buffer slot = BufferQueue::INVALID_BUFFER_SLOT; ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, - mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); sp<GraphicBuffer> secondBuffer; ASSERT_EQ(OK, mProducer->requestBuffer(slot, &secondBuffer)); @@ -892,8 +897,8 @@ TEST_F(BufferQueueTest, TestOccupancyHistory) { int slots[3] = {}; mProducer->setMaxDequeuedBufferCount(3); for (size_t i = 0; i < 3; ++i) { - status_t result = - mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, 0, nullptr, nullptr); + status_t result = mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, + TEST_PRODUCER_USAGE_BITS, nullptr, nullptr); ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); ASSERT_EQ(OK, mProducer->requestBuffer(slots[i], &buffer)); } @@ -906,7 +911,9 @@ TEST_F(BufferQueueTest, TestOccupancyHistory) { // The first segment is a two-buffer segment, so we only put one buffer into // the queue at a time for (size_t i = 0; i < 5; ++i) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); ASSERT_EQ(OK, mConsumer->releaseBuffer(item.mSlot, item.mFrameNumber, @@ -921,16 +928,22 @@ TEST_F(BufferQueueTest, TestOccupancyHistory) { // two-buffer segment, but then at the end, we put two buffers in the queue // at the same time before draining it. for (size_t i = 0; i < 5; ++i) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); ASSERT_EQ(OK, mConsumer->releaseBuffer(item.mSlot, item.mFrameNumber, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE)); std::this_thread::sleep_for(16ms); } - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); ASSERT_EQ(OK, mConsumer->releaseBuffer(item.mSlot, item.mFrameNumber, @@ -945,10 +958,14 @@ TEST_F(BufferQueueTest, TestOccupancyHistory) { // The third segment is a triple-buffer segment, so the queue is switching // between one buffer and two buffers deep. - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); for (size_t i = 0; i < 5; ++i) { - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); ASSERT_EQ(OK, mConsumer->releaseBuffer(item.mSlot, item.mFrameNumber, @@ -1047,8 +1064,8 @@ TEST_F(BufferQueueTest, TestDiscardFreeBuffers) { int slots[4] = {}; mProducer->setMaxDequeuedBufferCount(4); for (size_t i = 0; i < 4; ++i) { - status_t result = - mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, 0, nullptr, nullptr); + status_t result = mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, + TEST_PRODUCER_USAGE_BITS, nullptr, nullptr); ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); ASSERT_EQ(OK, mProducer->requestBuffer(slots[i], &buffer)); } @@ -1059,14 +1076,22 @@ TEST_F(BufferQueueTest, TestDiscardFreeBuffers) { // Get buffers in all states: dequeued, filled, acquired, free // Fill 3 buffers - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); // Dequeue 1 buffer - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); // Acquire and free 1 buffer ASSERT_EQ(OK, mConsumer->acquireBuffer(&item, 0)); @@ -1132,8 +1157,8 @@ TEST_F(BufferQueueTest, TestBufferReplacedInQueueBuffer) { int slots[2] = {}; ASSERT_EQ(OK, mProducer->setMaxDequeuedBufferCount(2)); for (size_t i = 0; i < 2; ++i) { - status_t result = - mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, 0, nullptr, nullptr); + status_t result = mProducer->dequeueBuffer(&slots[i], &fence, 0, 0, 0, + TEST_PRODUCER_USAGE_BITS, nullptr, nullptr); ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); ASSERT_EQ(OK, mProducer->requestBuffer(slots[i], &buffer)); } @@ -1143,10 +1168,14 @@ TEST_F(BufferQueueTest, TestBufferReplacedInQueueBuffer) { // Fill 2 buffers without consumer consuming them. Verify that all // queued buffer returns proper bufferReplaced flag - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(false, output.bufferReplaced); - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); ASSERT_EQ(true, output.bufferReplaced); } @@ -1237,7 +1266,8 @@ TEST_F(BufferQueueTest, TestStaleBufferHandleSentAfterDisconnect) { NATIVE_WINDOW_SCALING_MODE_FREEZE, 0, Fence::NO_FENCE); // Dequeue, request, and queue one buffer - status_t result = mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr); + status_t result = mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, + nullptr, nullptr); ASSERT_EQ(IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION, result); ASSERT_EQ(OK, mProducer->requestBuffer(slot, &buffer)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); @@ -1252,7 +1282,9 @@ TEST_F(BufferQueueTest, TestStaleBufferHandleSentAfterDisconnect) { EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE)); // Dequeue and queue the buffer again - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); // Acquire and release the buffer again. Upon acquiring, the buffer handle @@ -1264,7 +1296,9 @@ TEST_F(BufferQueueTest, TestStaleBufferHandleSentAfterDisconnect) { EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE)); // Dequeue and queue the buffer again - ASSERT_EQ(OK, mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, 0, nullptr, nullptr)); + ASSERT_EQ(OK, + mProducer->dequeueBuffer(&slot, &fence, 0, 0, 0, TEST_PRODUCER_USAGE_BITS, nullptr, + nullptr)); ASSERT_EQ(OK, mProducer->queueBuffer(slot, input, &output)); // Disconnect the producer end. This should clear all of the slots and mark diff --git a/libs/gui/tests/Constants.h b/libs/gui/tests/Constants.h new file mode 100644 index 0000000000..85c0f9faab --- /dev/null +++ b/libs/gui/tests/Constants.h @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <hardware/gralloc.h> + +// Arbitrary non-zero usage flag. +constexpr uint64_t TEST_PRODUCER_USAGE_BITS = GRALLOC_USAGE_SW_READ_RARELY; diff --git a/libs/gui/tests/CpuConsumer_test.cpp b/libs/gui/tests/CpuConsumer_test.cpp index 0a14afac55..d80bd9c62a 100644 --- a/libs/gui/tests/CpuConsumer_test.cpp +++ b/libs/gui/tests/CpuConsumer_test.cpp @@ -62,7 +62,7 @@ protected: const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); CpuConsumerTestParams params = GetParam(); - ALOGD("** Starting test %s (%d x %d, %d, 0x%x)", + ALOGD("** Starting parameterized test %s (%d x %d, %d, 0x%x)", test_info->name(), params.width, params.height, params.maxLockedBuffers, params.format); diff --git a/libs/gui/tests/EndToEndNativeInputTest.cpp b/libs/gui/tests/EndToEndNativeInputTest.cpp index 4ec7a06cb8..4d5bd5b3fa 100644 --- a/libs/gui/tests/EndToEndNativeInputTest.cpp +++ b/libs/gui/tests/EndToEndNativeInputTest.cpp @@ -821,7 +821,7 @@ TEST_F(InputSurfacesTest, touch_flag_obscured) { // with flag AMOTION_EVENT_FLAG_WINDOW_IS_OBSCURED std::unique_ptr<InputSurface> nonTouchableSurface = makeSurface(100, 100); nonTouchableSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - nonTouchableSurface->mInputInfo.ownerUid = 22222; + nonTouchableSurface->mInputInfo.ownerUid = gui::Uid{22222}; // Overriding occlusion mode otherwise the touch would be discarded at InputDispatcher by // the default obscured/untrusted touch filter introduced in S. nonTouchableSurface->mInputInfo.touchOcclusionMode = TouchOcclusionMode::ALLOW; @@ -842,8 +842,8 @@ TEST_F(InputSurfacesTest, touch_flag_partially_obscured_with_crop) { std::unique_ptr<InputSurface> nonTouchableSurface = makeSurface(100, 100); nonTouchableSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); parentSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - nonTouchableSurface->mInputInfo.ownerUid = 22222; - parentSurface->mInputInfo.ownerUid = 22222; + nonTouchableSurface->mInputInfo.ownerUid = gui::Uid{22222}; + parentSurface->mInputInfo.ownerUid = gui::Uid{22222}; nonTouchableSurface->showAt(0, 0); parentSurface->showAt(100, 100); @@ -866,8 +866,8 @@ TEST_F(InputSurfacesTest, touch_not_obscured_with_crop) { std::unique_ptr<InputSurface> nonTouchableSurface = makeSurface(100, 100); nonTouchableSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); parentSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - nonTouchableSurface->mInputInfo.ownerUid = 22222; - parentSurface->mInputInfo.ownerUid = 22222; + nonTouchableSurface->mInputInfo.ownerUid = gui::Uid{22222}; + parentSurface->mInputInfo.ownerUid = gui::Uid{22222}; nonTouchableSurface->showAt(0, 0); parentSurface->showAt(50, 50); @@ -886,7 +886,7 @@ TEST_F(InputSurfacesTest, touch_not_obscured_with_zero_sized_bql) { std::unique_ptr<InputSurface> bufferSurface = InputSurface::makeBufferInputSurface(mComposerClient, 0, 0); bufferSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - bufferSurface->mInputInfo.ownerUid = 22222; + bufferSurface->mInputInfo.ownerUid = gui::Uid{22222}; surface->showAt(10, 10); bufferSurface->showAt(50, 50, Rect::EMPTY_RECT); @@ -901,7 +901,7 @@ TEST_F(InputSurfacesTest, touch_not_obscured_with_zero_sized_blast) { std::unique_ptr<BlastInputSurface> bufferSurface = BlastInputSurface::makeBlastInputSurface(mComposerClient, 0, 0); bufferSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - bufferSurface->mInputInfo.ownerUid = 22222; + bufferSurface->mInputInfo.ownerUid = gui::Uid{22222}; surface->showAt(10, 10); bufferSurface->showAt(50, 50, Rect::EMPTY_RECT); @@ -948,13 +948,13 @@ TEST_F(InputSurfacesTest, strict_unobscured_input_scaled_without_crop_window) { TEST_F(InputSurfacesTest, strict_unobscured_input_obscured_window) { std::unique_ptr<InputSurface> surface = makeSurface(100, 100); - surface->mInputInfo.ownerUid = 11111; + surface->mInputInfo.ownerUid = gui::Uid{11111}; surface->doTransaction( [&](auto &t, auto &sc) { t.setDropInputMode(sc, gui::DropInputMode::OBSCURED); }); surface->showAt(100, 100); std::unique_ptr<InputSurface> obscuringSurface = makeSurface(100, 100); obscuringSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - obscuringSurface->mInputInfo.ownerUid = 22222; + obscuringSurface->mInputInfo.ownerUid = gui::Uid{22222}; obscuringSurface->showAt(100, 100); injectTap(101, 101); EXPECT_EQ(surface->consumeEvent(100), nullptr); @@ -967,13 +967,13 @@ TEST_F(InputSurfacesTest, strict_unobscured_input_obscured_window) { TEST_F(InputSurfacesTest, strict_unobscured_input_partially_obscured_window) { std::unique_ptr<InputSurface> surface = makeSurface(100, 100); - surface->mInputInfo.ownerUid = 11111; + surface->mInputInfo.ownerUid = gui::Uid{11111}; surface->doTransaction( [&](auto &t, auto &sc) { t.setDropInputMode(sc, gui::DropInputMode::OBSCURED); }); surface->showAt(100, 100); std::unique_ptr<InputSurface> obscuringSurface = makeSurface(100, 100); obscuringSurface->mInputInfo.setInputConfig(WindowInfo::InputConfig::NOT_TOUCHABLE, true); - obscuringSurface->mInputInfo.ownerUid = 22222; + obscuringSurface->mInputInfo.ownerUid = gui::Uid{22222}; obscuringSurface->showAt(190, 190); injectTap(101, 101); diff --git a/libs/gui/tests/GLTest.cpp b/libs/gui/tests/GLTest.cpp index e5f4aaa999..40af8e845a 100644 --- a/libs/gui/tests/GLTest.cpp +++ b/libs/gui/tests/GLTest.cpp @@ -29,10 +29,6 @@ static int abs(int value) { } void GLTest::SetUp() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), testInfo->name()); - mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); ASSERT_EQ(EGL_SUCCESS, eglGetError()); ASSERT_NE(EGL_NO_DISPLAY, mEglDisplay); @@ -132,10 +128,6 @@ void GLTest::TearDown() { eglTerminate(mEglDisplay); } ASSERT_EQ(EGL_SUCCESS, eglGetError()); - - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), testInfo->name()); } EGLint const* GLTest::getConfigAttribs() { diff --git a/libs/gui/tests/IGraphicBufferProducer_test.cpp b/libs/gui/tests/IGraphicBufferProducer_test.cpp index e6cb89cb83..b1f5d083c7 100644 --- a/libs/gui/tests/IGraphicBufferProducer_test.cpp +++ b/libs/gui/tests/IGraphicBufferProducer_test.cpp @@ -17,6 +17,7 @@ #define LOG_TAG "IGraphicBufferProducer_test" //#define LOG_NDEBUG 0 +#include "Constants.h" #include "MockConsumer.h" #include <gtest/gtest.h> @@ -40,7 +41,6 @@ #define TEST_API NATIVE_WINDOW_API_CPU #define TEST_API_OTHER NATIVE_WINDOW_API_EGL // valid API that's not TEST_API #define TEST_CONTROLLED_BY_APP false -#define TEST_PRODUCER_USAGE_BITS (0) namespace android { @@ -82,11 +82,6 @@ protected: IGraphicBufferProducerTest() {} virtual void SetUp() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - mMC = new MockConsumer; switch (GetParam()) { @@ -111,13 +106,6 @@ protected: ASSERT_OK(mConsumer->consumerConnect(mMC, /*controlledByApp*/ false)); } - virtual void TearDown() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } - status_t TryConnectProducer() { IGraphicBufferProducer::QueueBufferOutput output; return mProducer->connect(TEST_TOKEN, diff --git a/libs/gui/tests/LibGuiMain.cpp b/libs/gui/tests/LibGuiMain.cpp new file mode 100644 index 0000000000..10f7207588 --- /dev/null +++ b/libs/gui/tests/LibGuiMain.cpp @@ -0,0 +1,38 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gtest/gtest.h" +#include "log/log.h" + +namespace { + +class TestCaseLogger : public ::testing::EmptyTestEventListener { + void OnTestStart(const ::testing::TestInfo& testInfo) override { + ALOGD("Begin test: %s#%s", testInfo.test_suite_name(), testInfo.name()); + } + + void OnTestEnd(const testing::TestInfo& testInfo) override { + ALOGD("End test: %s#%s", testInfo.test_suite_name(), testInfo.name()); + } +}; + +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + testing::UnitTest::GetInstance()->listeners().Append(new TestCaseLogger()); + return RUN_ALL_TESTS(); +}
\ No newline at end of file diff --git a/libs/gui/tests/Malicious.cpp b/libs/gui/tests/Malicious.cpp index 58d7cc6f35..376420c42b 100644 --- a/libs/gui/tests/Malicious.cpp +++ b/libs/gui/tests/Malicious.cpp @@ -151,7 +151,6 @@ TEST(Malicious, Bug36991414Max) { sp<MaliciousBQP> malicious = getMaliciousBQP(); sp<Surface> surface = new Surface(malicious); - ASSERT_EQ(NO_ERROR, surface->connect(NATIVE_WINDOW_API_CPU, nullptr, false)); ANativeWindow_Buffer buffer; ASSERT_EQ(NO_ERROR, surface->lock(&buffer, nullptr)); ASSERT_EQ(NO_ERROR, surface->unlockAndPost()); @@ -165,7 +164,6 @@ TEST(Malicious, Bug36991414Min) { sp<MaliciousBQP> malicious = getMaliciousBQP(); sp<Surface> surface = new Surface(malicious); - ASSERT_EQ(NO_ERROR, surface->connect(NATIVE_WINDOW_API_CPU, nullptr, false)); ANativeWindow_Buffer buffer; ASSERT_EQ(NO_ERROR, surface->lock(&buffer, nullptr)); ASSERT_EQ(NO_ERROR, surface->unlockAndPost()); @@ -179,7 +177,6 @@ TEST(Malicious, Bug36991414NegativeOne) { sp<MaliciousBQP> malicious = getMaliciousBQP(); sp<Surface> surface = new Surface(malicious); - ASSERT_EQ(NO_ERROR, surface->connect(NATIVE_WINDOW_API_CPU, nullptr, false)); ANativeWindow_Buffer buffer; ASSERT_EQ(NO_ERROR, surface->lock(&buffer, nullptr)); ASSERT_EQ(NO_ERROR, surface->unlockAndPost()); @@ -193,7 +190,6 @@ TEST(Malicious, Bug36991414NumSlots) { sp<MaliciousBQP> malicious = getMaliciousBQP(); sp<Surface> surface = new Surface(malicious); - ASSERT_EQ(NO_ERROR, surface->connect(NATIVE_WINDOW_API_CPU, nullptr, false)); ANativeWindow_Buffer buffer; ASSERT_EQ(NO_ERROR, surface->lock(&buffer, nullptr)); ASSERT_EQ(NO_ERROR, surface->unlockAndPost()); diff --git a/libs/gui/tests/StreamSplitter_test.cpp b/libs/gui/tests/StreamSplitter_test.cpp index 2f14924a15..f34b03eade 100644 --- a/libs/gui/tests/StreamSplitter_test.cpp +++ b/libs/gui/tests/StreamSplitter_test.cpp @@ -30,23 +30,7 @@ namespace android { -class StreamSplitterTest : public ::testing::Test { - -protected: - StreamSplitterTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } - - ~StreamSplitterTest() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - } -}; +class StreamSplitterTest : public ::testing::Test {}; struct FakeListener : public BnConsumerListener { virtual void onFrameAvailable(const BufferItem& /* item */) {} diff --git a/libs/gui/tests/SurfaceTextureClient_test.cpp b/libs/gui/tests/SurfaceTextureClient_test.cpp index 82b66972d9..b28dca8ab4 100644 --- a/libs/gui/tests/SurfaceTextureClient_test.cpp +++ b/libs/gui/tests/SurfaceTextureClient_test.cpp @@ -40,11 +40,6 @@ protected: } virtual void SetUp() { - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("Begin test: %s.%s", testInfo->test_case_name(), - testInfo->name()); - sp<IGraphicBufferProducer> producer; sp<IGraphicBufferConsumer> consumer; BufferQueue::createBufferQueue(&producer, &consumer); @@ -96,11 +91,6 @@ protected: eglDestroyContext(mEglDisplay, mEglContext); eglDestroySurface(mEglDisplay, mEglSurface); eglTerminate(mEglDisplay); - - const ::testing::TestInfo* const testInfo = - ::testing::UnitTest::GetInstance()->current_test_info(); - ALOGD("End test: %s.%s", testInfo->test_case_name(), - testInfo->name()); } virtual EGLint const* getConfigAttribs() { diff --git a/libs/gui/tests/SurfaceTextureGL.h b/libs/gui/tests/SurfaceTextureGL.h index 53eb68cad8..9d8af5d0f5 100644 --- a/libs/gui/tests/SurfaceTextureGL.h +++ b/libs/gui/tests/SurfaceTextureGL.h @@ -17,6 +17,7 @@ #ifndef ANDROID_SURFACE_TEXTURE_GL_H #define ANDROID_SURFACE_TEXTURE_GL_H +#include "Constants.h" #include "GLTest.h" #include "FrameWaiter.h" @@ -43,6 +44,7 @@ protected: true, false); mSTC = new Surface(producer); mANW = mSTC; + ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(), TEST_PRODUCER_USAGE_BITS)); mTextureRenderer = new TextureRenderer(TEX_ID, mST); ASSERT_NO_FATAL_FAILURE(mTextureRenderer->SetUp()); mFW = new FrameWaiter; diff --git a/libs/gui/tests/Surface_test.cpp b/libs/gui/tests/Surface_test.cpp index c1b67b4396..6adcd966c5 100644 --- a/libs/gui/tests/Surface_test.cpp +++ b/libs/gui/tests/Surface_test.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "Constants.h" #include "MockConsumer.h" #include <gtest/gtest.h> @@ -148,6 +149,7 @@ protected: /*listener*/listener)); const int BUFFER_COUNT = 4 + extraDiscardedBuffers; ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(window.get(), BUFFER_COUNT)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); ANativeWindowBuffer* buffers[BUFFER_COUNT]; // Dequeue first to allocate a number of buffers @@ -530,7 +532,8 @@ TEST_F(SurfaceTest, DynamicSetBufferCount) { ASSERT_EQ(NO_ERROR, native_window_api_connect(window.get(), NATIVE_WINDOW_API_CPU)); - native_window_set_buffer_count(window.get(), 4); + ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(window.get(), 4)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); int fence; ANativeWindowBuffer* buffer; @@ -560,6 +563,7 @@ TEST_F(SurfaceTest, GetAndFlushRemovedBuffers) { /*reportBufferRemoval*/true)); const int BUFFER_COUNT = 4; ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(window.get(), BUFFER_COUNT)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); sp<GraphicBuffer> detachedBuffer; sp<Fence> outFence; @@ -989,6 +993,15 @@ public: return binder::Status::ok(); } + binder::Status updateSmallAreaDetection(const std::vector<int32_t>& /*uids*/, + const std::vector<float>& /*thresholds*/) { + return binder::Status::ok(); + } + + binder::Status setSmallAreaDetectionThreshold(int32_t /*uid*/, float /*threshold*/) { + return binder::Status::ok(); + } + binder::Status getGpuContextPriority(int32_t* /*outPriority*/) override { return binder::Status::ok(); } @@ -1012,6 +1025,11 @@ public: return binder::Status::ok(); } + binder::Status getStalledTransactionInfo( + int32_t /*pid*/, std::optional<gui::StalledTransactionInfo>* /*result*/) override { + return binder::Status::ok(); + } + protected: IBinder* onAsBinder() override { return nullptr; } @@ -1203,7 +1221,8 @@ protected: ASSERT_EQ(NO_ERROR, native_window_api_connect(mWindow.get(), NATIVE_WINDOW_API_CPU)); - native_window_set_buffer_count(mWindow.get(), 4); + ASSERT_EQ(NO_ERROR, native_window_set_buffer_count(mWindow.get(), 4)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(mWindow.get(), TEST_PRODUCER_USAGE_BITS)); } void disableFrameTimestamps() { @@ -2069,8 +2088,9 @@ TEST_F(SurfaceTest, DequeueWithConsumerDrivenSize) { sp<Surface> surface = new Surface(producer); sp<ANativeWindow> window(surface); - native_window_api_connect(window.get(), NATIVE_WINDOW_API_CPU); - native_window_set_buffers_dimensions(window.get(), 0, 0); + ASSERT_EQ(NO_ERROR, native_window_api_connect(window.get(), NATIVE_WINDOW_API_CPU)); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_dimensions(window.get(), 0, 0)); + ASSERT_EQ(NO_ERROR, native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS)); int fence; ANativeWindowBuffer* buffer; @@ -2122,6 +2142,7 @@ TEST_F(SurfaceTest, DequeueWithConsumerDrivenSize) { native_window_api_connect(window.get(), NATIVE_WINDOW_API_CPU); consumer->setTransformHint(NATIVE_WINDOW_TRANSFORM_ROT_270); native_window_set_buffers_dimensions(window.get(), 0, 0); + native_window_set_usage(window.get(), TEST_PRODUCER_USAGE_BITS); ASSERT_EQ(NO_ERROR, window->dequeueBuffer(window.get(), &buffer, &fence)); EXPECT_EQ(10, buffer->width); EXPECT_EQ(20, buffer->height); diff --git a/libs/gui/tests/WindowInfo_test.cpp b/libs/gui/tests/WindowInfo_test.cpp index 11b87efda7..461fe4a4ce 100644 --- a/libs/gui/tests/WindowInfo_test.cpp +++ b/libs/gui/tests/WindowInfo_test.cpp @@ -61,8 +61,8 @@ TEST(WindowInfo, Parcelling) { i.alpha = 0.7; i.transform.set({0.4, -1, 100, 0.5, 0, 40, 0, 0, 1}); i.touchOcclusionMode = TouchOcclusionMode::ALLOW; - i.ownerPid = 19; - i.ownerUid = 24; + i.ownerPid = gui::Pid{19}; + i.ownerUid = gui::Uid{24}; i.packageName = "com.example.package"; i.inputConfig = WindowInfo::InputConfig::NOT_FOCUSABLE; i.displayId = 34; diff --git a/libs/input/Android.bp b/libs/input/Android.bp index 869458c407..022dfaddc1 100644 --- a/libs/input/Android.bp +++ b/libs/input/Android.bp @@ -33,6 +33,138 @@ filegroup { ], } +aidl_interface { + name: "inputconstants", + host_supported: true, + vendor_available: true, + unstable: true, + srcs: [ + ":inputconstants_aidl", + ], + + backend: { + rust: { + enabled: true, + }, + }, +} + +rust_bindgen { + name: "libinput_bindgen", + host_supported: true, + crate_name: "input_bindgen", + visibility: ["//frameworks/native/services/inputflinger"], + wrapper_src: "InputWrapper.hpp", + + include_dirs: [ + "frameworks/native/include", + ], + + source_stem: "bindings", + + bindgen_flags: [ + "--verbose", + "--allowlist-var=AMOTION_EVENT_FLAG_CANCELED", + "--allowlist-var=AMOTION_EVENT_ACTION_CANCEL", + "--allowlist-var=AMOTION_EVENT_ACTION_UP", + "--allowlist-var=AMOTION_EVENT_ACTION_POINTER_DOWN", + "--allowlist-var=AMOTION_EVENT_ACTION_DOWN", + "--allowlist-var=AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT", + "--allowlist-var=MAX_POINTER_ID", + ], + + static_libs: [ + "inputconstants-cpp", + "libui-types", + ], + shared_libs: ["libc++"], + header_libs: [ + "native_headers", + "jni_headers", + "flatbuffer_headers", + ], +} + +// Contains methods to help access C++ code from rust +cc_library_static { + name: "libinput_from_rust_to_cpp", + cpp_std: "c++20", + host_supported: true, + cflags: [ + "-Wall", + "-Wextra", + "-Werror", + ], + srcs: [ + "FromRustToCpp.cpp", + ], + + generated_headers: [ + "cxx-bridge-header", + ], + generated_sources: ["libinput_cxx_bridge_code"], + + shared_libs: [ + "libbase", + ], +} + +genrule { + name: "libinput_cxx_bridge_code", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) >> $(out)", + srcs: ["input_verifier.rs"], + out: ["inputverifier_generated.cpp"], +} + +genrule { + name: "libinput_cxx_bridge_header", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) --header >> $(out)", + srcs: ["input_verifier.rs"], + out: ["input_verifier.rs.h"], +} + +rust_defaults { + name: "libinput_rust_defaults", + srcs: ["input_verifier.rs"], + host_supported: true, + rustlibs: [ + "libbitflags", + "libcxx", + "libinput_bindgen", + "liblogger", + "liblog_rust", + "inputconstants-rust", + ], + + shared_libs: [ + "libbase", + "liblog", + ], +} + +rust_ffi_static { + name: "libinput_rust", + crate_name: "input", + defaults: ["libinput_rust_defaults"], +} + +rust_test { + name: "libinput_rust_test", + defaults: ["libinput_rust_defaults"], + whole_static_libs: [ + "libinput_from_rust_to_cpp", + ], + test_options: { + unit_test: true, + }, + test_suites: ["device_tests"], + sanitize: { + hwaddress: true, + }, +} + cc_library { name: "libinput", cpp_std: "c++20", @@ -44,6 +176,7 @@ cc_library { "-Wno-unused-parameter", ], srcs: [ + "FromRustToCpp.cpp", "Input.cpp", "InputDevice.cpp", "InputEventLabels.cpp", @@ -52,6 +185,7 @@ cc_library { "KeyCharacterMap.cpp", "KeyLayoutMap.cpp", "MotionPredictor.cpp", + "MotionPredictorMetricsManager.cpp", "PrintTools.cpp", "PropertyMap.cpp", "TfLiteMotionPredictor.cpp", @@ -65,19 +199,28 @@ cc_library { header_libs: [ "flatbuffer_headers", "jni_headers", + "libeigen", "tensorflow_headers", ], - export_header_lib_headers: ["jni_headers"], + export_header_lib_headers: [ + "jni_headers", + "libeigen", + ], generated_headers: [ + "cxx-bridge-header", + "libinput_cxx_bridge_header", "toolbox_input_labels", ], + generated_sources: ["libinput_cxx_bridge_code"], + shared_libs: [ "libbase", "libcutils", "liblog", "libPlatformProperties", + "libtinyxml2", "libvintf", ], @@ -85,21 +228,36 @@ cc_library { "-Wl,--exclude-libs=libtflite_static.a", ], + sanitize: { + undefined: true, + all_undefined: true, + misc_undefined: ["integer"], + }, + static_libs: [ + "inputconstants-cpp", "libui-types", "libtflite_static", ], + whole_static_libs: [ + "libinput_rust", + ], + export_static_lib_headers: [ "libui-types", ], + export_generated_headers: [ + "cxx-bridge-header", + "libinput_cxx_bridge_header", + ], + target: { android: { srcs: [ "InputTransport.cpp", "android/os/IInputFlinger.aidl", - ":inputconstants_aidl", ], export_shared_lib_headers: ["libbinder"], @@ -107,6 +265,10 @@ cc_library { shared_libs: [ "libutils", "libbinder", + // Stats logging library and its dependencies. + "libstatslog_libinput", + "libstatsbootstrap", + "android.os.statsbootstrap_aidl-cpp", ], static_libs: [ @@ -117,12 +279,9 @@ cc_library { "libgui_window_info_static", ], - sanitize: { - misc_undefined: ["integer"], - }, - required: [ "motion_predictor_model_prebuilt", + "motion_predictor_model_config", ], }, host: { @@ -138,12 +297,8 @@ cc_library { host_linux: { srcs: [ "InputTransport.cpp", - "android/os/IInputConstants.aidl", - "android/os/IInputFlinger.aidl", - "android/os/InputConfig.aidl", ], static_libs: [ - "libhostgraphics", "libgui_window_info_static", ], shared_libs: [ @@ -165,6 +320,43 @@ cc_library { }, } +// Use bootstrap version of stats logging library. +// libinput is a bootstrap process (starts early in the boot process), and thus can't use the normal +// `libstatslog` because that requires `libstatssocket`, which is only available later in the boot. +cc_library { + name: "libstatslog_libinput", + generated_sources: ["statslog_libinput.cpp"], + generated_headers: ["statslog_libinput.h"], + export_generated_headers: ["statslog_libinput.h"], + shared_libs: [ + "libbinder", + "libstatsbootstrap", + "libutils", + "android.os.statsbootstrap_aidl-cpp", + ], +} + +genrule { + name: "statslog_libinput.h", + tools: ["stats-log-api-gen"], + cmd: "$(location stats-log-api-gen) --header $(genDir)/statslog_libinput.h --module libinput" + + " --namespace android,stats,libinput --bootstrap", + out: [ + "statslog_libinput.h", + ], +} + +genrule { + name: "statslog_libinput.cpp", + tools: ["stats-log-api-gen"], + cmd: "$(location stats-log-api-gen) --cpp $(genDir)/statslog_libinput.cpp --module libinput" + + " --namespace android,stats,libinput --importHeader statslog_libinput.h" + + " --bootstrap", + out: [ + "statslog_libinput.cpp", + ], +} + cc_defaults { name: "libinput_fuzz_defaults", cpp_std: "c++20", diff --git a/libs/ui/include_types/ui/DataspaceUtils.h b/libs/input/FromRustToCpp.cpp index a461cb4e68..e4ce62e734 100644 --- a/libs/ui/include_types/ui/DataspaceUtils.h +++ b/libs/input/FromRustToCpp.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Android Open Source Project + * Copyright 2023 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,13 @@ * limitations under the License. */ -#pragma once - -#include <ui/GraphicTypes.h> +#include <android-base/logging.h> +#include <ffi/FromRustToCpp.h> namespace android { -inline bool isHdrDataspace(ui::Dataspace dataspace) { - const auto transfer = dataspace & HAL_DATASPACE_TRANSFER_MASK; - - return transfer == HAL_DATASPACE_TRANSFER_ST2084 || transfer == HAL_DATASPACE_TRANSFER_HLG; +bool shouldLog(rust::Str tag) { + return android::base::ShouldLog(android::base::LogSeverity::DEBUG, tag.data()); } -} // namespace android
\ No newline at end of file +} // namespace android diff --git a/libs/input/InputEventLabels.cpp b/libs/input/InputEventLabels.cpp index f99a7d640e..bade68629d 100644 --- a/libs/input/InputEventLabels.cpp +++ b/libs/input/InputEventLabels.cpp @@ -404,7 +404,8 @@ namespace android { DEFINE_AXIS(GESTURE_Y_OFFSET), \ DEFINE_AXIS(GESTURE_SCROLL_X_DISTANCE), \ DEFINE_AXIS(GESTURE_SCROLL_Y_DISTANCE), \ - DEFINE_AXIS(GESTURE_PINCH_SCALE_FACTOR) + DEFINE_AXIS(GESTURE_PINCH_SCALE_FACTOR), \ + DEFINE_AXIS(GESTURE_SWIPE_FINGER_COUNT) // NOTE: If you add new LEDs here, you must also add them to Input.h #define LEDS_SEQUENCE \ @@ -433,17 +434,14 @@ namespace android { // clang-format on // --- InputEventLookup --- -const std::unordered_map<std::string, int> InputEventLookup::KEYCODES = {KEYCODES_SEQUENCE}; -const std::vector<InputEventLabel> InputEventLookup::KEY_NAMES = {KEYCODES_SEQUENCE}; - -const std::unordered_map<std::string, int> InputEventLookup::AXES = {AXES_SEQUENCE}; - -const std::vector<InputEventLabel> InputEventLookup::AXES_NAMES = {AXES_SEQUENCE}; - -const std::unordered_map<std::string, int> InputEventLookup::LEDS = {LEDS_SEQUENCE}; - -const std::unordered_map<std::string, int> InputEventLookup::FLAGS = {FLAGS_SEQUENCE}; +InputEventLookup::InputEventLookup() + : KEYCODES({KEYCODES_SEQUENCE}), + KEY_NAMES({KEYCODES_SEQUENCE}), + AXES({AXES_SEQUENCE}), + AXES_NAMES({AXES_SEQUENCE}), + LEDS({LEDS_SEQUENCE}), + FLAGS({FLAGS_SEQUENCE}) {} std::optional<int> InputEventLookup::lookupValueByLabel( const std::unordered_map<std::string, int>& map, const char* literal) { @@ -461,30 +459,36 @@ const char* InputEventLookup::lookupLabelByValue(const std::vector<InputEventLab } std::optional<int> InputEventLookup::getKeyCodeByLabel(const char* label) { - return lookupValueByLabel(KEYCODES, label); + const auto& self = get(); + return self.lookupValueByLabel(self.KEYCODES, label); } const char* InputEventLookup::getLabelByKeyCode(int32_t keyCode) { - if (keyCode >= 0 && static_cast<size_t>(keyCode) < KEYCODES.size()) { - return lookupLabelByValue(KEY_NAMES, keyCode); + const auto& self = get(); + if (keyCode >= 0 && static_cast<size_t>(keyCode) < self.KEYCODES.size()) { + return get().lookupLabelByValue(self.KEY_NAMES, keyCode); } return nullptr; } std::optional<int> InputEventLookup::getKeyFlagByLabel(const char* label) { - return lookupValueByLabel(FLAGS, label); + const auto& self = get(); + return lookupValueByLabel(self.FLAGS, label); } std::optional<int> InputEventLookup::getAxisByLabel(const char* label) { - return lookupValueByLabel(AXES, label); + const auto& self = get(); + return lookupValueByLabel(self.AXES, label); } const char* InputEventLookup::getAxisLabel(int32_t axisId) { - return lookupLabelByValue(AXES_NAMES, axisId); + const auto& self = get(); + return lookupLabelByValue(self.AXES_NAMES, axisId); } std::optional<int> InputEventLookup::getLedByLabel(const char* label) { - return lookupValueByLabel(LEDS, label); + const auto& self = get(); + return lookupValueByLabel(self.LEDS, label); } namespace { diff --git a/libs/input/InputTransport.cpp b/libs/input/InputTransport.cpp index fd4d5f0f2d..01d9ebbc71 100644 --- a/libs/input/InputTransport.cpp +++ b/libs/input/InputTransport.cpp @@ -4,6 +4,7 @@ // Provides a shared memory transport for input events. // #define LOG_TAG "InputTransport" +#define ATRACE_TAG ATRACE_TAG_INPUT #include <errno.h> #include <fcntl.h> @@ -13,6 +14,7 @@ #include <sys/types.h> #include <unistd.h> +#include <android-base/logging.h> #include <android-base/properties.h> #include <android-base/stringprintf.h> #include <binder/Parcel.h> @@ -80,6 +82,7 @@ const bool DEBUG_RESAMPLING = } // namespace +using android::base::Result; using android::base::StringPrintf; namespace android { @@ -449,6 +452,13 @@ status_t InputChannel::sendMessage(const InputMessage* msg) { ALOGD_IF(DEBUG_CHANNEL_MESSAGES, "channel '%s' ~ sent message of type %s", mName.c_str(), ftl::enum_string(msg->header.type).c_str()); + + if (ATRACE_ENABLED()) { + std::string message = + StringPrintf("sendMessage(inputChannel=%s, seq=0x%" PRIx32 ", type=0x%" PRIx32 ")", + mName.c_str(), msg->header.seq, msg->header.type); + ATRACE_NAME(message.c_str()); + } return OK; } @@ -484,6 +494,13 @@ status_t InputChannel::receiveMessage(InputMessage* msg) { ALOGD_IF(DEBUG_CHANNEL_MESSAGES, "channel '%s' ~ received message of type %s", mName.c_str(), ftl::enum_string(msg->header.type).c_str()); + + if (ATRACE_ENABLED()) { + std::string message = StringPrintf("receiveMessage(inputChannel=%s, seq=0x%" PRIx32 + ", type=0x%" PRIx32 ")", + mName.c_str(), msg->header.seq, msg->header.type); + ATRACE_NAME(message.c_str()); + } return OK; } @@ -606,8 +623,12 @@ status_t InputPublisher::publishMotionEvent( ATRACE_NAME(message.c_str()); } if (verifyEvents()) { - mInputVerifier.processMovement(deviceId, action, pointerCount, pointerProperties, - pointerCoords, flags); + Result<void> result = + mInputVerifier.processMovement(deviceId, action, pointerCount, pointerProperties, + pointerCoords, flags); + if (!result.ok()) { + LOG(FATAL) << "Bad stream: " << result.error(); + } } if (debugTransportPublisher()) { std::string transformString; diff --git a/libs/input/InputVerifier.cpp b/libs/input/InputVerifier.cpp index eb758045cc..9745e89234 100644 --- a/libs/input/InputVerifier.cpp +++ b/libs/input/InputVerifier.cpp @@ -18,111 +18,35 @@ #include <android-base/logging.h> #include <input/InputVerifier.h> +#include "input_verifier.rs.h" -namespace android { +using android::base::Error; +using android::base::Result; +using android::input::RustPointerProperties; -/** - * Log all of the movements that are sent to this verifier. Helps to identify the streams that lead - * to inconsistent events. - * Enable this via "adb shell setprop log.tag.InputVerifierLogEvents DEBUG" - */ -static bool logEvents() { - return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "LogEvents", ANDROID_LOG_INFO); -} +namespace android { // --- InputVerifier --- -InputVerifier::InputVerifier(const std::string& name) : mName(name){}; +InputVerifier::InputVerifier(const std::string& name) + : mVerifier(android::input::verifier::create(rust::String::lossy(name))){}; -void InputVerifier::processMovement(int32_t deviceId, int32_t action, uint32_t pointerCount, - const PointerProperties* pointerProperties, - const PointerCoords* pointerCoords, int32_t flags) { - if (logEvents()) { - LOG(ERROR) << "Processing " << MotionEvent::actionToString(action) << " for device " - << deviceId << " (" << pointerCount << " pointer" - << (pointerCount == 1 ? "" : "s") << ") on " << mName; +Result<void> InputVerifier::processMovement(int32_t deviceId, int32_t action, uint32_t pointerCount, + const PointerProperties* pointerProperties, + const PointerCoords* pointerCoords, int32_t flags) { + std::vector<RustPointerProperties> rpp; + for (size_t i = 0; i < pointerCount; i++) { + rpp.emplace_back(RustPointerProperties{.id = pointerProperties[i].id}); } - - switch (MotionEvent::getActionMasked(action)) { - case AMOTION_EVENT_ACTION_DOWN: { - auto [it, inserted] = mTouchingPointerIdsByDevice.insert({deviceId, {}}); - if (!inserted) { - LOG(FATAL) << "Got ACTION_DOWN, but already have touching pointers " << it->second - << " for device " << deviceId << " on " << mName; - } - it->second.set(pointerProperties[0].id); - break; - } - case AMOTION_EVENT_ACTION_POINTER_DOWN: { - auto it = mTouchingPointerIdsByDevice.find(deviceId); - if (it == mTouchingPointerIdsByDevice.end()) { - LOG(FATAL) << "Got POINTER_DOWN, but no touching pointers for device " << deviceId - << " on " << mName; - } - it->second.set(pointerProperties[MotionEvent::getActionIndex(action)].id); - break; - } - case AMOTION_EVENT_ACTION_MOVE: { - ensureTouchingPointersMatch(deviceId, pointerCount, pointerProperties, "MOVE"); - break; - } - case AMOTION_EVENT_ACTION_POINTER_UP: { - auto it = mTouchingPointerIdsByDevice.find(deviceId); - if (it == mTouchingPointerIdsByDevice.end()) { - LOG(FATAL) << "Got POINTER_UP, but no touching pointers for device " << deviceId - << " on " << mName; - } - it->second.reset(pointerProperties[MotionEvent::getActionIndex(action)].id); - break; - } - case AMOTION_EVENT_ACTION_UP: { - auto it = mTouchingPointerIdsByDevice.find(deviceId); - if (it == mTouchingPointerIdsByDevice.end()) { - LOG(FATAL) << "Got ACTION_UP, but no record for deviceId " << deviceId << " on " - << mName; - } - const auto& [_, touchingPointerIds] = *it; - if (touchingPointerIds.count() != 1) { - LOG(FATAL) << "Got ACTION_UP, but we have pointers: " << touchingPointerIds - << " for deviceId " << deviceId << " on " << mName; - } - const int32_t pointerId = pointerProperties[0].id; - if (!touchingPointerIds.test(pointerId)) { - LOG(FATAL) << "Got ACTION_UP, but pointerId " << pointerId - << " is not touching. Touching pointers: " << touchingPointerIds - << " for deviceId " << deviceId << " on " << mName; - } - mTouchingPointerIdsByDevice.erase(it); - break; - } - case AMOTION_EVENT_ACTION_CANCEL: { - if ((flags & AMOTION_EVENT_FLAG_CANCELED) != AMOTION_EVENT_FLAG_CANCELED) { - LOG(FATAL) << "For ACTION_CANCEL, must set FLAG_CANCELED"; - } - ensureTouchingPointersMatch(deviceId, pointerCount, pointerProperties, "CANCEL"); - mTouchingPointerIdsByDevice.erase(deviceId); - break; - } + rust::Slice<const RustPointerProperties> properties{rpp.data(), rpp.size()}; + rust::String errorMessage = + android::input::verifier::process_movement(*mVerifier, deviceId, action, properties, + flags); + if (errorMessage.empty()) { + return {}; + } else { + return Error() << errorMessage; } } -void InputVerifier::ensureTouchingPointersMatch(int32_t deviceId, uint32_t pointerCount, - const PointerProperties* pointerProperties, - const char* action) const { - auto it = mTouchingPointerIdsByDevice.find(deviceId); - if (it == mTouchingPointerIdsByDevice.end()) { - LOG(FATAL) << "Got " << action << ", but no touching pointers for device " << deviceId - << " on " << mName; - } - const auto& [_, touchingPointerIds] = *it; - for (size_t i = 0; i < pointerCount; i++) { - const int32_t pointerId = pointerProperties[i].id; - if (!touchingPointerIds.test(pointerId)) { - LOG(FATAL) << "Got " << action << " for pointerId " << pointerId - << " but the touching pointers are " << touchingPointerIds << " on " - << mName; - } - } -}; - } // namespace android diff --git a/libs/input/InputWrapper.hpp b/libs/input/InputWrapper.hpp new file mode 100644 index 0000000000..a01080d319 --- /dev/null +++ b/libs/input/InputWrapper.hpp @@ -0,0 +1,18 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <android/input.h> +#include "input/Input.h" diff --git a/libs/input/MotionPredictor.cpp b/libs/input/MotionPredictor.cpp index abcca345d3..5736ad7eed 100644 --- a/libs/input/MotionPredictor.cpp +++ b/libs/input/MotionPredictor.cpp @@ -36,9 +36,6 @@ namespace android { namespace { -const int64_t PREDICTION_INTERVAL_NANOS = - 12500000 / 3; // TODO(b/266747937): Get this from the model. - /** * Log debug messages about predictions. * Enable this via "adb shell setprop log.tag.MotionPredictor DEBUG" @@ -70,7 +67,7 @@ MotionPredictor::MotionPredictor(nsecs_t predictionTimestampOffsetNanos, android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { if (mLastEvent && mLastEvent->getDeviceId() != event.getDeviceId()) { // We still have an active gesture for another device. The provided MotionEvent is not - // consistent the previous gesture. + // consistent with the previous gesture. LOG(ERROR) << "Inconsistent event stream: last event is " << *mLastEvent << ", but " << __func__ << " is called with " << event; return android::base::Error() @@ -86,9 +83,10 @@ android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { // Initialise the model now that it's likely to be used. if (!mModel) { mModel = TfLiteMotionPredictorModel::create(); + LOG_ALWAYS_FATAL_IF(!mModel); } - if (mBuffers == nullptr) { + if (!mBuffers) { mBuffers = std::make_unique<TfLiteMotionPredictorBuffers>(mModel->inputLength()); } @@ -136,6 +134,13 @@ android::base::Result<void> MotionPredictor::record(const MotionEvent& event) { mLastEvent = MotionEvent(); } mLastEvent->copyFrom(&event, /*keepHistory=*/false); + + // Pass input event to the MetricsManager. + if (!mMetricsManager) { + mMetricsManager.emplace(mModel->config().predictionInterval, mModel->outputLength()); + } + mMetricsManager->onRecord(event); + return {}; } @@ -178,19 +183,30 @@ std::unique_ptr<MotionEvent> MotionPredictor::predict(nsecs_t timestamp) { for (size_t i = 0; i < static_cast<size_t>(predictedR.size()) && predictionTime <= futureTime; ++i) { - const TfLiteMotionPredictorSample::Point point = - convertPrediction(axisFrom, axisTo, predictedR[i], predictedPhi[i]); + if (predictedR[i] < mModel->config().distanceNoiseFloor) { + // Stop predicting when the predicted output is below the model's noise floor. + // + // We assume that all subsequent predictions in the batch are unreliable because later + // predictions are conditional on earlier predictions, and a state of noise is not a + // good basis for prediction. + // + // The UX trade-off is that this potentially sacrifices some predictions when the input + // device starts to speed up, but avoids producing noisy predictions as it slows down. + break; + } // TODO(b/266747654): Stop predictions if confidence is < some threshold. - ALOGD_IF(isDebug(), "prediction %zu: %f, %f", i, point.x, point.y); + const TfLiteMotionPredictorSample::Point predictedPoint = + convertPrediction(axisFrom, axisTo, predictedR[i], predictedPhi[i]); + + ALOGD_IF(isDebug(), "prediction %zu: %f, %f", i, predictedPoint.x, predictedPoint.y); PointerCoords coords; coords.clear(); - coords.setAxisValue(AMOTION_EVENT_AXIS_X, point.x); - coords.setAxisValue(AMOTION_EVENT_AXIS_Y, point.y); - // TODO(b/266747654): Stop predictions if predicted pressure is < some threshold. + coords.setAxisValue(AMOTION_EVENT_AXIS_X, predictedPoint.x); + coords.setAxisValue(AMOTION_EVENT_AXIS_Y, predictedPoint.y); coords.setAxisValue(AMOTION_EVENT_AXIS_PRESSURE, predictedPressure[i]); - predictionTime += PREDICTION_INTERVAL_NANOS; + predictionTime += mModel->config().predictionInterval; if (i == 0) { hasPredictions = true; prediction->initialize(InputEvent::nextId(), event.getDeviceId(), event.getSource(), @@ -207,12 +223,17 @@ std::unique_ptr<MotionEvent> MotionPredictor::predict(nsecs_t timestamp) { } axisFrom = axisTo; - axisTo = point; + axisTo = predictedPoint; } - // TODO(b/266747511): Interpolate to futureTime? + if (!hasPredictions) { return nullptr; } + + // Pass predictions to the MetricsManager. + LOG_ALWAYS_FATAL_IF(!mMetricsManager); + mMetricsManager->onPredict(*prediction); + return prediction; } diff --git a/libs/input/MotionPredictorMetricsManager.cpp b/libs/input/MotionPredictorMetricsManager.cpp new file mode 100644 index 0000000000..67b103290f --- /dev/null +++ b/libs/input/MotionPredictorMetricsManager.cpp @@ -0,0 +1,373 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "MotionPredictorMetricsManager" + +#include <input/MotionPredictorMetricsManager.h> + +#include <algorithm> + +#include <android-base/logging.h> + +#include "Eigen/Core" +#include "Eigen/Geometry" + +#ifdef __ANDROID__ +#include <statslog_libinput.h> +#endif + +namespace android { +namespace { + +inline constexpr int NANOS_PER_SECOND = 1'000'000'000; // nanoseconds per second +inline constexpr int NANOS_PER_MILLIS = 1'000'000; // nanoseconds per millisecond + +// Velocity threshold at which we report "high-velocity" metrics, in pixels per second. +// This value was selected from manual experimentation, as a threshold that separates "fast" +// (semi-sloppy) handwriting from more careful medium to slow handwriting. +inline constexpr float HIGH_VELOCITY_THRESHOLD = 1100.0; + +// Small value to add to the path length when computing scale-invariant error to avoid division by +// zero. +inline constexpr float PATH_LENGTH_EPSILON = 0.001; + +} // namespace + +MotionPredictorMetricsManager::MotionPredictorMetricsManager(nsecs_t predictionInterval, + size_t maxNumPredictions) + : mPredictionInterval(predictionInterval), + mMaxNumPredictions(maxNumPredictions), + mRecentGroundTruthPoints(maxNumPredictions + 1), + mAggregatedMetrics(maxNumPredictions), + mAtomFields(maxNumPredictions) {} + +void MotionPredictorMetricsManager::onRecord(const MotionEvent& inputEvent) { + // Convert MotionEvent to GroundTruthPoint. + const PointerCoords* coords = inputEvent.getRawPointerCoords(/*pointerIndex=*/0); + LOG_ALWAYS_FATAL_IF(coords == nullptr); + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f{coords->getY(), + coords->getX()}, + .pressure = + inputEvent.getPressure(/*pointerIndex=*/0)}, + .timestamp = inputEvent.getEventTime()}; + + // Handle event based on action type. + switch (inputEvent.getActionMasked()) { + case AMOTION_EVENT_ACTION_DOWN: { + clearStrokeData(); + incorporateNewGroundTruth(groundTruthPoint); + break; + } + case AMOTION_EVENT_ACTION_MOVE: { + incorporateNewGroundTruth(groundTruthPoint); + break; + } + case AMOTION_EVENT_ACTION_UP: + case AMOTION_EVENT_ACTION_CANCEL: { + // Only expect meaningful predictions when given at least two input points. + if (mRecentGroundTruthPoints.size() >= 2) { + computeAtomFields(); + reportMetrics(); + break; + } + } + } +} + +// Adds new predictions to mRecentPredictions and maintains the invariant that elements are +// sorted in ascending order of targetTimestamp. +void MotionPredictorMetricsManager::onPredict(const MotionEvent& predictionEvent) { + for (size_t i = 0; i < predictionEvent.getHistorySize() + 1; ++i) { + // Convert MotionEvent to PredictionPoint. + const PointerCoords* coords = + predictionEvent.getHistoricalRawPointerCoords(/*pointerIndex=*/0, i); + LOG_ALWAYS_FATAL_IF(coords == nullptr); + const nsecs_t targetTimestamp = predictionEvent.getHistoricalEventTime(i); + mRecentPredictions.push_back( + PredictionPoint{{.position = Eigen::Vector2f{coords->getY(), coords->getX()}, + .pressure = + predictionEvent.getHistoricalPressure(/*pointerIndex=*/0, + i)}, + .originTimestamp = mRecentGroundTruthPoints.back().timestamp, + .targetTimestamp = targetTimestamp}); + } + + std::sort(mRecentPredictions.begin(), mRecentPredictions.end()); +} + +void MotionPredictorMetricsManager::clearStrokeData() { + mRecentGroundTruthPoints.clear(); + mRecentPredictions.clear(); + std::fill(mAggregatedMetrics.begin(), mAggregatedMetrics.end(), AggregatedStrokeMetrics{}); + std::fill(mAtomFields.begin(), mAtomFields.end(), AtomFields{}); +} + +void MotionPredictorMetricsManager::incorporateNewGroundTruth( + const GroundTruthPoint& groundTruthPoint) { + // Note: this removes the oldest point if `mRecentGroundTruthPoints` is already at capacity. + mRecentGroundTruthPoints.pushBack(groundTruthPoint); + + // Remove outdated predictions – those that can never be matched with the current or any future + // ground truth points. We use fuzzy association for the timestamps here, because ground truth + // and prediction timestamps may not be perfectly synchronized. + const nsecs_t fuzzy_association_time_delta = mPredictionInterval / 4; + const auto firstCurrentIt = + std::find_if(mRecentPredictions.begin(), mRecentPredictions.end(), + [&groundTruthPoint, + fuzzy_association_time_delta](const PredictionPoint& prediction) { + return prediction.targetTimestamp > + groundTruthPoint.timestamp - fuzzy_association_time_delta; + }); + mRecentPredictions.erase(mRecentPredictions.begin(), firstCurrentIt); + + // Fuzzily match the new ground truth's timestamp to recent predictions' targetTimestamp and + // update the corresponding metrics. + for (const PredictionPoint& prediction : mRecentPredictions) { + if ((prediction.targetTimestamp > + groundTruthPoint.timestamp - fuzzy_association_time_delta) && + (prediction.targetTimestamp < + groundTruthPoint.timestamp + fuzzy_association_time_delta)) { + updateAggregatedMetrics(prediction); + } + } +} + +void MotionPredictorMetricsManager::updateAggregatedMetrics( + const PredictionPoint& predictionPoint) { + if (mRecentGroundTruthPoints.size() < 2) { + return; + } + + const GroundTruthPoint& latestGroundTruthPoint = mRecentGroundTruthPoints.back(); + const GroundTruthPoint& previousGroundTruthPoint = + mRecentGroundTruthPoints[mRecentGroundTruthPoints.size() - 2]; + // Calculate prediction error vector. + const Eigen::Vector2f groundTruthTrajectory = + latestGroundTruthPoint.position - previousGroundTruthPoint.position; + const Eigen::Vector2f predictionTrajectory = + predictionPoint.position - previousGroundTruthPoint.position; + const Eigen::Vector2f predictionError = predictionTrajectory - groundTruthTrajectory; + + // By default, prediction error counts fully as both off-trajectory and along-trajectory error. + // This serves as the fallback when the two most recent ground truth points are equal. + const float predictionErrorNorm = predictionError.norm(); + float alongTrajectoryError = predictionErrorNorm; + float offTrajectoryError = predictionErrorNorm; + if (groundTruthTrajectory.squaredNorm() > 0) { + // Rotate the prediction error vector by the angle of the ground truth trajectory vector. + // This yields a vector whose first component is the along-trajectory error and whose + // second component is the off-trajectory error. + const float theta = std::atan2(groundTruthTrajectory[1], groundTruthTrajectory[0]); + const Eigen::Vector2f rotatedPredictionError = Eigen::Rotation2Df(-theta) * predictionError; + alongTrajectoryError = rotatedPredictionError[0]; + offTrajectoryError = rotatedPredictionError[1]; + } + + // Compute the multiple of mPredictionInterval nearest to the amount of time into the + // future being predicted. This serves as the time bucket index into mAggregatedMetrics. + const float timestampDeltaFloat = + static_cast<float>(predictionPoint.targetTimestamp - predictionPoint.originTimestamp); + const size_t tIndex = + static_cast<size_t>(std::round(timestampDeltaFloat / mPredictionInterval - 1)); + + // Aggregate values into "general errors". + mAggregatedMetrics[tIndex].alongTrajectoryErrorSum += alongTrajectoryError; + mAggregatedMetrics[tIndex].alongTrajectorySumSquaredErrors += + alongTrajectoryError * alongTrajectoryError; + mAggregatedMetrics[tIndex].offTrajectorySumSquaredErrors += + offTrajectoryError * offTrajectoryError; + const float pressureError = predictionPoint.pressure - latestGroundTruthPoint.pressure; + mAggregatedMetrics[tIndex].pressureSumSquaredErrors += pressureError * pressureError; + ++mAggregatedMetrics[tIndex].generalErrorsCount; + + // Aggregate values into high-velocity metrics, if we are in one of the last two time buckets + // and the velocity is above the threshold. Velocity here is measured in pixels per second. + const float velocity = groundTruthTrajectory.norm() / + (static_cast<float>(latestGroundTruthPoint.timestamp - + previousGroundTruthPoint.timestamp) / + NANOS_PER_SECOND); + if ((tIndex + 2 >= mMaxNumPredictions) && (velocity > HIGH_VELOCITY_THRESHOLD)) { + mAggregatedMetrics[tIndex].highVelocityAlongTrajectorySse += + alongTrajectoryError * alongTrajectoryError; + mAggregatedMetrics[tIndex].highVelocityOffTrajectorySse += + offTrajectoryError * offTrajectoryError; + ++mAggregatedMetrics[tIndex].highVelocityErrorsCount; + } + + // Compute path length for scale-invariant errors. + float pathLength = 0; + for (size_t i = 1; i < mRecentGroundTruthPoints.size(); ++i) { + pathLength += + (mRecentGroundTruthPoints[i].position - mRecentGroundTruthPoints[i - 1].position) + .norm(); + } + // Avoid overweighting errors at the beginning of a stroke: compute the path length as if there + // were a full ground truth history by filling in missing segments with the average length. + // Note: the "- 1" is needed to translate from number of endpoints to number of segments. + pathLength *= static_cast<float>(mRecentGroundTruthPoints.capacity() - 1) / + (mRecentGroundTruthPoints.size() - 1); + pathLength += PATH_LENGTH_EPSILON; // Ensure path length is nonzero (>= PATH_LENGTH_EPSILON). + + // Compute and aggregate scale-invariant errors. + const float scaleInvariantAlongTrajectoryError = alongTrajectoryError / pathLength; + const float scaleInvariantOffTrajectoryError = offTrajectoryError / pathLength; + mAggregatedMetrics[tIndex].scaleInvariantAlongTrajectorySse += + scaleInvariantAlongTrajectoryError * scaleInvariantAlongTrajectoryError; + mAggregatedMetrics[tIndex].scaleInvariantOffTrajectorySse += + scaleInvariantOffTrajectoryError * scaleInvariantOffTrajectoryError; + ++mAggregatedMetrics[tIndex].scaleInvariantErrorsCount; +} + +void MotionPredictorMetricsManager::computeAtomFields() { + for (size_t i = 0; i < mAggregatedMetrics.size(); ++i) { + if (mAggregatedMetrics[i].generalErrorsCount == 0) { + // We have not received data corresponding to metrics for this time bucket. + continue; + } + + mAtomFields[i].deltaTimeBucketMilliseconds = + static_cast<int>(mPredictionInterval / NANOS_PER_MILLIS * (i + 1)); + + // Note: we need the "* 1000"s below because we report values in integral milli-units. + + { // General errors: reported for every time bucket. + const float alongTrajectoryErrorMean = mAggregatedMetrics[i].alongTrajectoryErrorSum / + mAggregatedMetrics[i].generalErrorsCount; + mAtomFields[i].alongTrajectoryErrorMeanMillipixels = + static_cast<int>(alongTrajectoryErrorMean * 1000); + + const float alongTrajectoryMse = mAggregatedMetrics[i].alongTrajectorySumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount; + // Take the max with 0 to avoid negative values caused by numerical instability. + const float alongTrajectoryErrorVariance = + std::max(0.0f, + alongTrajectoryMse - + alongTrajectoryErrorMean * alongTrajectoryErrorMean); + const float alongTrajectoryErrorStd = std::sqrt(alongTrajectoryErrorVariance); + mAtomFields[i].alongTrajectoryErrorStdMillipixels = + static_cast<int>(alongTrajectoryErrorStd * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].offTrajectorySumSquaredErrors < 0, + "mAggregatedMetrics[%zu].offTrajectorySumSquaredErrors = %f should " + "not be negative", + i, mAggregatedMetrics[i].offTrajectorySumSquaredErrors); + const float offTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].offTrajectorySumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount); + mAtomFields[i].offTrajectoryRmseMillipixels = + static_cast<int>(offTrajectoryRmse * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].pressureSumSquaredErrors < 0, + "mAggregatedMetrics[%zu].pressureSumSquaredErrors = %f should not " + "be negative", + i, mAggregatedMetrics[i].pressureSumSquaredErrors); + const float pressureRmse = std::sqrt(mAggregatedMetrics[i].pressureSumSquaredErrors / + mAggregatedMetrics[i].generalErrorsCount); + mAtomFields[i].pressureRmseMilliunits = static_cast<int>(pressureRmse * 1000); + } + + // High-velocity errors: reported only for last two time buckets. + // Check if we are in one of the last two time buckets, and there is high-velocity data. + if ((i + 2 >= mMaxNumPredictions) && (mAggregatedMetrics[i].highVelocityErrorsCount > 0)) { + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].highVelocityAlongTrajectorySse < 0, + "mAggregatedMetrics[%zu].highVelocityAlongTrajectorySse = %f " + "should not be negative", + i, mAggregatedMetrics[i].highVelocityAlongTrajectorySse); + const float alongTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].highVelocityAlongTrajectorySse / + mAggregatedMetrics[i].highVelocityErrorsCount); + mAtomFields[i].highVelocityAlongTrajectoryRmse = + static_cast<int>(alongTrajectoryRmse * 1000); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[i].highVelocityOffTrajectorySse < 0, + "mAggregatedMetrics[%zu].highVelocityOffTrajectorySse = %f should " + "not be negative", + i, mAggregatedMetrics[i].highVelocityOffTrajectorySse); + const float offTrajectoryRmse = + std::sqrt(mAggregatedMetrics[i].highVelocityOffTrajectorySse / + mAggregatedMetrics[i].highVelocityErrorsCount); + mAtomFields[i].highVelocityOffTrajectoryRmse = + static_cast<int>(offTrajectoryRmse * 1000); + } + + // Scale-invariant errors: reported only for the last time bucket, where the values + // represent an average across all time buckets. + if (i + 1 == mMaxNumPredictions) { + // Compute error averages. + float alongTrajectoryRmseSum = 0; + float offTrajectoryRmseSum = 0; + for (size_t j = 0; j < mAggregatedMetrics.size(); ++j) { + // If we have general errors (checked above), we should always also have + // scale-invariant errors. + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantErrorsCount == 0, + "mAggregatedMetrics[%zu].scaleInvariantErrorsCount is 0", j); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse < 0, + "mAggregatedMetrics[%zu].scaleInvariantAlongTrajectorySse = %f " + "should not be negative", + j, mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse); + alongTrajectoryRmseSum += + std::sqrt(mAggregatedMetrics[j].scaleInvariantAlongTrajectorySse / + mAggregatedMetrics[j].scaleInvariantErrorsCount); + + LOG_ALWAYS_FATAL_IF(mAggregatedMetrics[j].scaleInvariantOffTrajectorySse < 0, + "mAggregatedMetrics[%zu].scaleInvariantOffTrajectorySse = %f " + "should not be negative", + j, mAggregatedMetrics[j].scaleInvariantOffTrajectorySse); + offTrajectoryRmseSum += + std::sqrt(mAggregatedMetrics[j].scaleInvariantOffTrajectorySse / + mAggregatedMetrics[j].scaleInvariantErrorsCount); + } + + const float averageAlongTrajectoryRmse = + alongTrajectoryRmseSum / mAggregatedMetrics.size(); + mAtomFields.back().scaleInvariantAlongTrajectoryRmse = + static_cast<int>(averageAlongTrajectoryRmse * 1000); + + const float averageOffTrajectoryRmse = offTrajectoryRmseSum / mAggregatedMetrics.size(); + mAtomFields.back().scaleInvariantOffTrajectoryRmse = + static_cast<int>(averageOffTrajectoryRmse * 1000); + } + } +} + +void MotionPredictorMetricsManager::reportMetrics() { + // Report one atom for each time bucket. + for (size_t i = 0; i < mAtomFields.size(); ++i) { + // Call stats_write logging function only on Android targets (not supported on host). +#ifdef __ANDROID__ + android::stats::libinput:: + stats_write(android::stats::libinput::STYLUS_PREDICTION_METRICS_REPORTED, + /*stylus_vendor_id=*/0, + /*stylus_product_id=*/0, mAtomFields[i].deltaTimeBucketMilliseconds, + mAtomFields[i].alongTrajectoryErrorMeanMillipixels, + mAtomFields[i].alongTrajectoryErrorStdMillipixels, + mAtomFields[i].offTrajectoryRmseMillipixels, + mAtomFields[i].pressureRmseMilliunits, + mAtomFields[i].highVelocityAlongTrajectoryRmse, + mAtomFields[i].highVelocityOffTrajectoryRmse, + mAtomFields[i].scaleInvariantAlongTrajectoryRmse, + mAtomFields[i].scaleInvariantOffTrajectoryRmse); +#endif + } + + // Set mock atom fields, if available. + if (mMockLoggedAtomFields != nullptr) { + *mMockLoggedAtomFields = mAtomFields; + } +} + +} // namespace android diff --git a/libs/input/TfLiteMotionPredictor.cpp b/libs/input/TfLiteMotionPredictor.cpp index 8d10ff56b0..d17476e216 100644 --- a/libs/input/TfLiteMotionPredictor.cpp +++ b/libs/input/TfLiteMotionPredictor.cpp @@ -36,6 +36,7 @@ #define ATRACE_TAG ATRACE_TAG_INPUT #include <cutils/trace.h> #include <log/log.h> +#include <utils/Timers.h> #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/core/api/op_resolver.h" @@ -44,6 +45,8 @@ #include "tensorflow/lite/model.h" #include "tensorflow/lite/mutable_op_resolver.h" +#include "tinyxml2.h" + namespace android { namespace { @@ -72,16 +75,41 @@ bool fileExists(const char* filename) { std::string getModelPath() { #if defined(__ANDROID__) - static const char* oemModel = "/vendor/etc/motion_predictor_model.fb"; + static const char* oemModel = "/vendor/etc/motion_predictor_model.tflite"; if (fileExists(oemModel)) { return oemModel; } - return "/system/etc/motion_predictor_model.fb"; + return "/system/etc/motion_predictor_model.tflite"; #else - return base::GetExecutableDirectory() + "/motion_predictor_model.fb"; + return base::GetExecutableDirectory() + "/motion_predictor_model.tflite"; #endif } +std::string getConfigPath() { + // The config file should be alongside the model file. + return base::Dirname(getModelPath()) + "/motion_predictor_config.xml"; +} + +int64_t parseXMLInt64(const tinyxml2::XMLElement& configRoot, const char* elementName) { + const tinyxml2::XMLElement* element = configRoot.FirstChildElement(elementName); + LOG_ALWAYS_FATAL_IF(!element, "Could not find '%s' element", elementName); + + int64_t value = 0; + LOG_ALWAYS_FATAL_IF(element->QueryInt64Text(&value) != tinyxml2::XML_SUCCESS, + "Failed to parse %s: %s", elementName, element->GetText()); + return value; +} + +float parseXMLFloat(const tinyxml2::XMLElement& configRoot, const char* elementName) { + const tinyxml2::XMLElement* element = configRoot.FirstChildElement(elementName); + LOG_ALWAYS_FATAL_IF(!element, "Could not find '%s' element", elementName); + + float value = 0; + LOG_ALWAYS_FATAL_IF(element->QueryFloatText(&value) != tinyxml2::XML_SUCCESS, + "Failed to parse %s: %s", elementName, element->GetText()); + return value; +} + // A TFLite ErrorReporter that logs to logcat. class LoggingErrorReporter : public tflite::ErrorReporter { public: @@ -133,6 +161,7 @@ std::unique_ptr<tflite::OpResolver> createOpResolver() { ::tflite::ops::builtin::Register_CONCATENATION()); resolver->AddBuiltin(::tflite::BuiltinOperator_FULLY_CONNECTED, ::tflite::ops::builtin::Register_FULLY_CONNECTED()); + resolver->AddBuiltin(::tflite::BuiltinOperator_GELU, ::tflite::ops::builtin::Register_GELU()); return resolver; } @@ -189,13 +218,7 @@ void TfLiteMotionPredictorBuffers::pushSample(int64_t timestamp, float phi = 0; float orientation = 0; - // Ignore the sample if there is no movement. These samples can occur when there's change to a - // property other than the coordinates and pollute the input to the model. - if (r == 0) { - return; - } - - if (!mAxisFrom) { // Second point. + if (!mAxisFrom && r > 0) { // Second point. // We can only determine the distance from the first point, and not any // angle. However, if the second point forms an axis, the orientation can // be transformed relative to that axis. @@ -216,8 +239,10 @@ void TfLiteMotionPredictorBuffers::pushSample(int64_t timestamp, } // Update the axis for the next point. - mAxisFrom = mAxisTo; - mAxisTo = sample; + if (r > 0) { + mAxisFrom = mAxisTo; + mAxisTo = sample; + } // Push the current sample onto the end of the input buffers. mInputR.pushBack(r); @@ -245,13 +270,26 @@ std::unique_ptr<TfLiteMotionPredictorModel> TfLiteMotionPredictorModel::create() PLOG(FATAL) << "Failed to mmap model"; } + const std::string configPath = getConfigPath(); + tinyxml2::XMLDocument configDocument; + LOG_ALWAYS_FATAL_IF(configDocument.LoadFile(configPath.c_str()) != tinyxml2::XML_SUCCESS, + "Failed to load config file from %s", configPath.c_str()); + + // Parse configuration file. + const tinyxml2::XMLElement* configRoot = configDocument.FirstChildElement("motion-predictor"); + LOG_ALWAYS_FATAL_IF(!configRoot); + Config config{ + .predictionInterval = parseXMLInt64(*configRoot, "prediction-interval"), + .distanceNoiseFloor = parseXMLFloat(*configRoot, "distance-noise-floor"), + }; + return std::unique_ptr<TfLiteMotionPredictorModel>( - new TfLiteMotionPredictorModel(std::move(modelBuffer))); + new TfLiteMotionPredictorModel(std::move(modelBuffer), std::move(config))); } TfLiteMotionPredictorModel::TfLiteMotionPredictorModel( - std::unique_ptr<android::base::MappedFile> model) - : mFlatBuffer(std::move(model)) { + std::unique_ptr<android::base::MappedFile> model, Config config) + : mFlatBuffer(std::move(model)), mConfig(std::move(config)) { CHECK(mFlatBuffer); mErrorReporter = std::make_unique<LoggingErrorReporter>(); mModel = tflite::FlatBufferModel::VerifyAndBuildFromBuffer(mFlatBuffer->data(), diff --git a/libs/input/VelocityTracker.cpp b/libs/input/VelocityTracker.cpp index 8551e5fa1c..078109a5b6 100644 --- a/libs/input/VelocityTracker.cpp +++ b/libs/input/VelocityTracker.cpp @@ -16,7 +16,9 @@ #define LOG_TAG "VelocityTracker" +#include <android-base/logging.h> #include <array> +#include <ftl/enum.h> #include <inttypes.h> #include <limits.h> #include <math.h> @@ -145,27 +147,19 @@ static std::string matrixToString(const float* a, uint32_t m, uint32_t n, bool r VelocityTracker::VelocityTracker(const Strategy strategy) : mLastEventTime(0), mCurrentPointerIdBits(0), mOverrideStrategy(strategy) {} -VelocityTracker::~VelocityTracker() { -} - bool VelocityTracker::isAxisSupported(int32_t axis) { return DEFAULT_STRATEGY_BY_AXIS.find(axis) != DEFAULT_STRATEGY_BY_AXIS.end(); } void VelocityTracker::configureStrategy(int32_t axis) { const bool isDifferentialAxis = DIFFERENTIAL_AXES.find(axis) != DIFFERENTIAL_AXES.end(); - - std::unique_ptr<VelocityTrackerStrategy> createdStrategy; - if (mOverrideStrategy != VelocityTracker::Strategy::DEFAULT) { - createdStrategy = createStrategy(mOverrideStrategy, /*deltaValues=*/isDifferentialAxis); + if (isDifferentialAxis || mOverrideStrategy == VelocityTracker::Strategy::DEFAULT) { + // Do not allow overrides of strategies for differential axes, for now. + mConfiguredStrategies[axis] = createStrategy(DEFAULT_STRATEGY_BY_AXIS.at(axis), + /*deltaValues=*/isDifferentialAxis); } else { - createdStrategy = createStrategy(DEFAULT_STRATEGY_BY_AXIS.at(axis), - /*deltaValues=*/isDifferentialAxis); + mConfiguredStrategies[axis] = createStrategy(mOverrideStrategy, /*deltaValues=*/false); } - - LOG_ALWAYS_FATAL_IF(createdStrategy == nullptr, - "Could not create velocity tracker strategy for axis '%" PRId32 "'!", axis); - mConfiguredStrategies[axis] = std::move(createdStrategy); } std::unique_ptr<VelocityTrackerStrategy> VelocityTracker::createStrategy( @@ -213,6 +207,9 @@ std::unique_ptr<VelocityTrackerStrategy> VelocityTracker::createStrategy( default: break; } + LOG(FATAL) << "Invalid strategy: " << ftl::enum_string(strategy) + << ", deltaValues = " << deltaValues; + return nullptr; } diff --git a/libs/input/ffi/FromRustToCpp.h b/libs/input/ffi/FromRustToCpp.h new file mode 100644 index 0000000000..889945c32b --- /dev/null +++ b/libs/input/ffi/FromRustToCpp.h @@ -0,0 +1,23 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "rust/cxx.h" + +namespace android { + +bool shouldLog(rust::Str tag); + +} // namespace android diff --git a/libs/input/input_verifier.rs b/libs/input/input_verifier.rs new file mode 100644 index 0000000000..f8dda3c901 --- /dev/null +++ b/libs/input/input_verifier.rs @@ -0,0 +1,419 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Validate the incoming motion stream. +//! This class is not thread-safe. +//! State is stored in the "InputVerifier" object +//! that can be created via the 'create' method. +//! Usage: +//! Box<InputVerifier> verifier = create("inputChannel name"); +//! result = process_movement(verifier, ...); +//! if (result) { +//! crash(result.error_message()); +//! } + +use std::collections::HashMap; +use std::collections::HashSet; + +use bitflags::bitflags; +use log::info; + +#[cxx::bridge(namespace = "android::input")] +#[allow(unsafe_op_in_unsafe_fn)] +mod ffi { + #[namespace = "android"] + unsafe extern "C++" { + include!("ffi/FromRustToCpp.h"); + fn shouldLog(tag: &str) -> bool; + } + #[namespace = "android::input::verifier"] + extern "Rust" { + type InputVerifier; + + fn create(name: String) -> Box<InputVerifier>; + fn process_movement( + verifier: &mut InputVerifier, + device_id: i32, + action: u32, + pointer_properties: &[RustPointerProperties], + flags: i32, + ) -> String; + } + + pub struct RustPointerProperties { + id: i32, + } +} + +use crate::ffi::shouldLog; +use crate::ffi::RustPointerProperties; + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +struct DeviceId(i32); + +fn process_movement( + verifier: &mut InputVerifier, + device_id: i32, + action: u32, + pointer_properties: &[RustPointerProperties], + flags: i32, +) -> String { + let result = verifier.process_movement( + DeviceId(device_id), + action, + pointer_properties, + Flags::from_bits(flags).unwrap(), + ); + match result { + Ok(()) => "".to_string(), + Err(e) => e, + } +} + +fn create(name: String) -> Box<InputVerifier> { + Box::new(InputVerifier::new(&name)) +} + +#[repr(u32)] +enum MotionAction { + Down = input_bindgen::AMOTION_EVENT_ACTION_DOWN, + Up = input_bindgen::AMOTION_EVENT_ACTION_UP, + Move = input_bindgen::AMOTION_EVENT_ACTION_MOVE, + Cancel = input_bindgen::AMOTION_EVENT_ACTION_CANCEL, + Outside = input_bindgen::AMOTION_EVENT_ACTION_OUTSIDE, + PointerDown { action_index: usize } = input_bindgen::AMOTION_EVENT_ACTION_POINTER_DOWN, + PointerUp { action_index: usize } = input_bindgen::AMOTION_EVENT_ACTION_POINTER_UP, + HoverEnter = input_bindgen::AMOTION_EVENT_ACTION_HOVER_ENTER, + HoverMove = input_bindgen::AMOTION_EVENT_ACTION_HOVER_MOVE, + HoverExit = input_bindgen::AMOTION_EVENT_ACTION_HOVER_EXIT, + Scroll = input_bindgen::AMOTION_EVENT_ACTION_SCROLL, + ButtonPress = input_bindgen::AMOTION_EVENT_ACTION_BUTTON_PRESS, + ButtonRelease = input_bindgen::AMOTION_EVENT_ACTION_BUTTON_RELEASE, +} + +fn get_action_index(action: u32) -> usize { + let index = (action & input_bindgen::AMOTION_EVENT_ACTION_POINTER_INDEX_MASK) + >> input_bindgen::AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT; + index.try_into().unwrap() +} + +impl From<u32> for MotionAction { + fn from(action: u32) -> Self { + let action_masked = action & input_bindgen::AMOTION_EVENT_ACTION_MASK; + let action_index = get_action_index(action); + match action_masked { + input_bindgen::AMOTION_EVENT_ACTION_DOWN => MotionAction::Down, + input_bindgen::AMOTION_EVENT_ACTION_UP => MotionAction::Up, + input_bindgen::AMOTION_EVENT_ACTION_MOVE => MotionAction::Move, + input_bindgen::AMOTION_EVENT_ACTION_CANCEL => MotionAction::Cancel, + input_bindgen::AMOTION_EVENT_ACTION_OUTSIDE => MotionAction::Outside, + input_bindgen::AMOTION_EVENT_ACTION_POINTER_DOWN => { + MotionAction::PointerDown { action_index } + } + input_bindgen::AMOTION_EVENT_ACTION_POINTER_UP => { + MotionAction::PointerUp { action_index } + } + input_bindgen::AMOTION_EVENT_ACTION_HOVER_ENTER => MotionAction::HoverEnter, + input_bindgen::AMOTION_EVENT_ACTION_HOVER_MOVE => MotionAction::HoverMove, + input_bindgen::AMOTION_EVENT_ACTION_HOVER_EXIT => MotionAction::HoverExit, + input_bindgen::AMOTION_EVENT_ACTION_SCROLL => MotionAction::Scroll, + input_bindgen::AMOTION_EVENT_ACTION_BUTTON_PRESS => MotionAction::ButtonPress, + input_bindgen::AMOTION_EVENT_ACTION_BUTTON_RELEASE => MotionAction::ButtonRelease, + _ => panic!("Unknown action: {}", action), + } + } +} + +bitflags! { + struct Flags: i32 { + const CANCELED = input_bindgen::AMOTION_EVENT_FLAG_CANCELED; + } +} + +fn motion_action_to_string(action: u32) -> String { + match action.into() { + MotionAction::Down => "DOWN".to_string(), + MotionAction::Up => "UP".to_string(), + MotionAction::Move => "MOVE".to_string(), + MotionAction::Cancel => "CANCEL".to_string(), + MotionAction::Outside => "OUTSIDE".to_string(), + MotionAction::PointerDown { action_index } => { + format!("POINTER_DOWN({})", action_index) + } + MotionAction::PointerUp { action_index } => { + format!("POINTER_UP({})", action_index) + } + MotionAction::HoverMove => "HOVER_MOVE".to_string(), + MotionAction::Scroll => "SCROLL".to_string(), + MotionAction::HoverEnter => "HOVER_ENTER".to_string(), + MotionAction::HoverExit => "HOVER_EXIT".to_string(), + MotionAction::ButtonPress => "BUTTON_PRESS".to_string(), + MotionAction::ButtonRelease => "BUTTON_RELEASE".to_string(), + } +} + +/** + * Log all of the movements that are sent to this verifier. Helps to identify the streams that lead + * to inconsistent events. + * Enable this via "adb shell setprop log.tag.InputVerifierLogEvents DEBUG" + */ +fn log_events() -> bool { + shouldLog("InputVerifierLogEvents") +} + +struct InputVerifier { + name: String, + touching_pointer_ids_by_device: HashMap<DeviceId, HashSet<i32>>, +} + +impl InputVerifier { + fn new(name: &str) -> Self { + logger::init( + logger::Config::default() + .with_tag_on_device("InputVerifier") + .with_min_level(log::Level::Trace), + ); + Self { name: name.to_owned(), touching_pointer_ids_by_device: HashMap::new() } + } + + fn process_movement( + &mut self, + device_id: DeviceId, + action: u32, + pointer_properties: &[RustPointerProperties], + flags: Flags, + ) -> Result<(), String> { + if log_events() { + info!( + "Processing {} for device {:?} ({} pointer{}) on {}", + motion_action_to_string(action), + device_id, + pointer_properties.len(), + if pointer_properties.len() == 1 { "" } else { "s" }, + self.name + ); + } + + match action.into() { + MotionAction::Down => { + let it = self.touching_pointer_ids_by_device.entry(device_id).or_default(); + let pointer_id = pointer_properties[0].id; + if it.contains(&pointer_id) { + return Err(format!( + "{}: Invalid DOWN event - pointers already down for device {:?}: {:?}", + self.name, device_id, it + )); + } + it.insert(pointer_id); + } + MotionAction::PointerDown { action_index } => { + if !self.touching_pointer_ids_by_device.contains_key(&device_id) { + return Err(format!( + "{}: Received POINTER_DOWN but no pointers are currently down \ + for device {:?}", + self.name, device_id + )); + } + let it = self.touching_pointer_ids_by_device.get_mut(&device_id).unwrap(); + let pointer_id = pointer_properties[action_index].id; + if it.contains(&pointer_id) { + return Err(format!( + "{}: Pointer with id={} not found in the properties", + self.name, pointer_id + )); + } + it.insert(pointer_id); + } + MotionAction::Move => { + if !self.ensure_touching_pointers_match(device_id, pointer_properties) { + return Err(format!( + "{}: ACTION_MOVE touching pointers don't match", + self.name + )); + } + } + MotionAction::PointerUp { action_index } => { + if !self.touching_pointer_ids_by_device.contains_key(&device_id) { + return Err(format!( + "{}: Received POINTER_UP but no pointers are currently down for device \ + {:?}", + self.name, device_id + )); + } + let it = self.touching_pointer_ids_by_device.get_mut(&device_id).unwrap(); + let pointer_id = pointer_properties[action_index].id; + it.remove(&pointer_id); + } + MotionAction::Up => { + if !self.touching_pointer_ids_by_device.contains_key(&device_id) { + return Err(format!( + "{} Received ACTION_UP but no pointers are currently down for device {:?}", + self.name, device_id + )); + } + let it = self.touching_pointer_ids_by_device.get_mut(&device_id).unwrap(); + if it.len() != 1 { + return Err(format!( + "{}: Got ACTION_UP, but we have pointers: {:?} for device {:?}", + self.name, it, device_id + )); + } + let pointer_id = pointer_properties[0].id; + if !it.contains(&pointer_id) { + return Err(format!( + "{}: Got ACTION_UP, but pointerId {} is not touching. Touching pointers:\ + {:?} for device {:?}", + self.name, pointer_id, it, device_id + )); + } + it.clear(); + } + MotionAction::Cancel => { + if flags.contains(Flags::CANCELED) { + return Err(format!( + "{}: For ACTION_CANCEL, must set FLAG_CANCELED", + self.name + )); + } + if !self.ensure_touching_pointers_match(device_id, pointer_properties) { + return Err(format!( + "{}: Got ACTION_CANCEL, but the pointers don't match. \ + Existing pointers: {:?}", + self.name, self.touching_pointer_ids_by_device + )); + } + self.touching_pointer_ids_by_device.remove(&device_id); + } + _ => return Ok(()), + } + Ok(()) + } + + fn ensure_touching_pointers_match( + &self, + device_id: DeviceId, + pointer_properties: &[RustPointerProperties], + ) -> bool { + let Some(pointers) = self.touching_pointer_ids_by_device.get(&device_id) else { + return false; + }; + + for pointer_property in pointer_properties.iter() { + let pointer_id = pointer_property.id; + if !pointers.contains(&pointer_id) { + return false; + } + } + true + } +} + +#[cfg(test)] +mod tests { + use crate::DeviceId; + use crate::Flags; + use crate::InputVerifier; + use crate::RustPointerProperties; + #[test] + fn single_pointer_stream() { + let mut verifier = InputVerifier::new("Test"); + let pointer_properties = Vec::from([RustPointerProperties { id: 0 }]); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_DOWN, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_MOVE, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_UP, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + } + + #[test] + fn multi_device_stream() { + let mut verifier = InputVerifier::new("Test"); + let pointer_properties = Vec::from([RustPointerProperties { id: 0 }]); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_DOWN, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_MOVE, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(2), + input_bindgen::AMOTION_EVENT_ACTION_DOWN, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(2), + input_bindgen::AMOTION_EVENT_ACTION_MOVE, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_UP, + &pointer_properties, + Flags::empty(), + ) + .is_ok()); + } + + #[test] + fn test_invalid_up() { + let mut verifier = InputVerifier::new("Test"); + let pointer_properties = Vec::from([RustPointerProperties { id: 0 }]); + assert!(verifier + .process_movement( + DeviceId(1), + input_bindgen::AMOTION_EVENT_ACTION_UP, + &pointer_properties, + Flags::empty(), + ) + .is_err()); + } +} diff --git a/libs/input/tests/Android.bp b/libs/input/tests/Android.bp index 42bdf57514..e7224ff752 100644 --- a/libs/input/tests/Android.bp +++ b/libs/input/tests/Android.bp @@ -18,7 +18,9 @@ cc_test { "InputDevice_test.cpp", "InputEvent_test.cpp", "InputPublisherAndConsumer_test.cpp", + "InputVerifier_test.cpp", "MotionPredictor_test.cpp", + "MotionPredictorMetricsManager_test.cpp", "RingBuffer_test.cpp", "TfLiteMotionPredictor_test.cpp", "TouchResampling_test.cpp", @@ -44,6 +46,7 @@ cc_test { "-Wno-unused-parameter", ], sanitize: { + hwaddress: true, undefined: true, all_undefined: true, diag: { @@ -56,17 +59,33 @@ cc_test { "libcutils", "liblog", "libPlatformProperties", + "libtinyxml2", "libutils", "libvintf", ], data: [ "data/*", - ":motion_predictor_model.fb", + ":motion_predictor_model", ], test_options: { unit_test: true, }, test_suites: ["device-tests"], + target: { + host: { + sanitize: { + address: true, + }, + }, + android: { + static_libs: [ + // Stats logging library and its dependencies. + "libstatslog_libinput", + "libstatsbootstrap", + "android.os.statsbootstrap_aidl-cpp", + ], + }, + }, } // NOTE: This is a compile time test, and does not need to be diff --git a/libs/input/tests/InputVerifier_test.cpp b/libs/input/tests/InputVerifier_test.cpp new file mode 100644 index 0000000000..e24fa6ed0b --- /dev/null +++ b/libs/input/tests/InputVerifier_test.cpp @@ -0,0 +1,29 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> +#include <input/InputVerifier.h> +#include <string> + +namespace android { + +TEST(InputVerifierTest, CreationWithInvalidUtfStringDoesNotCrash) { + constexpr char bytes[] = {static_cast<char>(0xC0), static_cast<char>(0x80)}; + const std::string name(bytes, sizeof(bytes)); + InputVerifier verifier(name); +} + +} // namespace android diff --git a/libs/input/tests/MotionPredictorMetricsManager_test.cpp b/libs/input/tests/MotionPredictorMetricsManager_test.cpp new file mode 100644 index 0000000000..b420a5a4e7 --- /dev/null +++ b/libs/input/tests/MotionPredictorMetricsManager_test.cpp @@ -0,0 +1,972 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <input/MotionPredictor.h> + +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <numeric> +#include <vector> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <input/InputEventBuilders.h> +#include <utils/Timers.h> // for nsecs_t + +#include "Eigen/Core" +#include "Eigen/Geometry" + +namespace android { +namespace { + +using ::testing::FloatNear; +using ::testing::Matches; + +using GroundTruthPoint = MotionPredictorMetricsManager::GroundTruthPoint; +using PredictionPoint = MotionPredictorMetricsManager::PredictionPoint; +using AtomFields = MotionPredictorMetricsManager::AtomFields; + +inline constexpr int NANOS_PER_MILLIS = 1'000'000; + +inline constexpr nsecs_t TEST_INITIAL_TIMESTAMP = 1'000'000'000; +inline constexpr size_t TEST_MAX_NUM_PREDICTIONS = 5; +inline constexpr nsecs_t TEST_PREDICTION_INTERVAL_NANOS = 12'500'000 / 3; // 1 / (240 hz) +inline constexpr int NO_DATA_SENTINEL = MotionPredictorMetricsManager::NO_DATA_SENTINEL; + +// Parameters: +// • arg: Eigen::Vector2f +// • target: Eigen::Vector2f +// • epsilon: float +MATCHER_P2(Vector2fNear, target, epsilon, "") { + return Matches(FloatNear(target[0], epsilon))(arg[0]) && + Matches(FloatNear(target[1], epsilon))(arg[1]); +} + +// Parameters: +// • arg: PredictionPoint +// • target: PredictionPoint +// • epsilon: float +MATCHER_P2(PredictionPointNear, target, epsilon, "") { + if (!Matches(Vector2fNear(target.position, epsilon))(arg.position)) { + *result_listener << "Position mismatch. Actual: (" << arg.position[0] << ", " + << arg.position[1] << "), expected: (" << target.position[0] << ", " + << target.position[1] << ")"; + return false; + } + if (!Matches(FloatNear(target.pressure, epsilon))(arg.pressure)) { + *result_listener << "Pressure mismatch. Actual: " << arg.pressure + << ", expected: " << target.pressure; + return false; + } + if (arg.originTimestamp != target.originTimestamp) { + *result_listener << "Origin timestamp mismatch. Actual: " << arg.originTimestamp + << ", expected: " << target.originTimestamp; + return false; + } + if (arg.targetTimestamp != target.targetTimestamp) { + *result_listener << "Target timestamp mismatch. Actual: " << arg.targetTimestamp + << ", expected: " << target.targetTimestamp; + return false; + } + return true; +} + +// --- Mathematical helper functions. --- + +template <typename T> +T average(std::vector<T> values) { + return std::accumulate(values.begin(), values.end(), T{}) / static_cast<T>(values.size()); +} + +template <typename T> +T standardDeviation(std::vector<T> values) { + T mean = average(values); + T accumulator = {}; + for (const T value : values) { + accumulator += value * value - mean * mean; + } + // Take the max with 0 to avoid negative values caused by numerical instability. + return std::sqrt(std::max(T{}, accumulator) / static_cast<T>(values.size())); +} + +template <typename T> +T rmse(std::vector<T> errors) { + T sse = {}; + for (const T error : errors) { + sse += error * error; + } + return std::sqrt(sse / static_cast<T>(errors.size())); +} + +TEST(MathematicalHelperFunctionTest, Average) { + std::vector<float> values{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + EXPECT_EQ(5.5f, average(values)); +} + +TEST(MathematicalHelperFunctionTest, StandardDeviation) { + // https://www.calculator.net/standard-deviation-calculator.html?numberinputs=10%2C+12%2C+23%2C+23%2C+16%2C+23%2C+21%2C+16 + std::vector<float> values{10, 12, 23, 23, 16, 23, 21, 16}; + EXPECT_FLOAT_EQ(4.8989794855664f, standardDeviation(values)); +} + +TEST(MathematicalHelperFunctionTest, Rmse) { + std::vector<float> errors{1, 5, 7, 7, 8, 20}; + EXPECT_FLOAT_EQ(9.899494937f, rmse(errors)); +} + +// --- MotionEvent-related helper functions. --- + +// Creates a MotionEvent corresponding to the given GroundTruthPoint. +MotionEvent makeMotionEvent(const GroundTruthPoint& groundTruthPoint) { + // Build single pointer of type STYLUS, with coordinates from groundTruthPoint. + PointerBuilder pointerBuilder = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(groundTruthPoint.position[1]) + .y(groundTruthPoint.position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, groundTruthPoint.pressure); + return MotionEventBuilder(/*action=*/AMOTION_EVENT_ACTION_MOVE, + /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .eventTime(groundTruthPoint.timestamp) + .pointer(pointerBuilder) + .build(); +} + +// Creates a MotionEvent corresponding to the given sequence of PredictionPoints. +MotionEvent makeMotionEvent(const std::vector<PredictionPoint>& predictionPoints) { + // Build single pointer of type STYLUS, with coordinates from first prediction point. + PointerBuilder pointerBuilder = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(predictionPoints[0].position[1]) + .y(predictionPoints[0].position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, predictionPoints[0].pressure); + MotionEvent predictionEvent = + MotionEventBuilder( + /*action=*/AMOTION_EVENT_ACTION_MOVE, /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .eventTime(predictionPoints[0].targetTimestamp) + .pointer(pointerBuilder) + .build(); + for (size_t i = 1; i < predictionPoints.size(); ++i) { + PointerCoords coords = + PointerBuilder(/*id=*/0, ToolType::STYLUS) + .x(predictionPoints[i].position[1]) + .y(predictionPoints[i].position[0]) + .axis(AMOTION_EVENT_AXIS_PRESSURE, predictionPoints[i].pressure) + .buildCoords(); + predictionEvent.addSample(predictionPoints[i].targetTimestamp, &coords); + } + return predictionEvent; +} + +// Creates a MotionEvent corresponding to a stylus lift (UP) ground truth event. +MotionEvent makeLiftMotionEvent() { + return MotionEventBuilder(/*action=*/AMOTION_EVENT_ACTION_UP, + /*source=*/AINPUT_SOURCE_CLASS_POINTER) + .pointer(PointerBuilder(/*id=*/0, ToolType::STYLUS)) + .build(); +} + +TEST(MakeMotionEventTest, MakeGroundTruthMotionEvent) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10.0f, 20.0f), + .pressure = 0.6f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const MotionEvent groundTruthMotionEvent = makeMotionEvent(groundTruthPoint); + + ASSERT_EQ(1u, groundTruthMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(0u, groundTruthMotionEvent.getHistorySize()); + EXPECT_EQ(groundTruthPoint.position[0], groundTruthMotionEvent.getRawPointerCoords(0)->getY()); + EXPECT_EQ(groundTruthPoint.position[1], groundTruthMotionEvent.getRawPointerCoords(0)->getX()); + EXPECT_EQ(groundTruthPoint.pressure, + groundTruthMotionEvent.getRawPointerCoords(0)->getAxisValue( + AMOTION_EVENT_AXIS_PRESSURE)); + EXPECT_EQ(AMOTION_EVENT_ACTION_MOVE, groundTruthMotionEvent.getAction()); +} + +TEST(MakeMotionEventTest, MakePredictionMotionEvent) { + const nsecs_t originTimestamp = TEST_INITIAL_TIMESTAMP; + const std::vector<PredictionPoint> + predictionPoints{{{.position = Eigen::Vector2f(10.0f, 20.0f), .pressure = 0.6f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 5 * NANOS_PER_MILLIS}, + {{.position = Eigen::Vector2f(11.0f, 22.0f), .pressure = 0.5f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 10 * NANOS_PER_MILLIS}, + {{.position = Eigen::Vector2f(12.0f, 24.0f), .pressure = 0.4f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + 15 * NANOS_PER_MILLIS}}; + const MotionEvent predictionMotionEvent = makeMotionEvent(predictionPoints); + + ASSERT_EQ(1u, predictionMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(predictionPoints.size(), predictionMotionEvent.getHistorySize() + 1); + for (size_t i = 0; i < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const PointerCoords coords = *predictionMotionEvent.getHistoricalRawPointerCoords( + /*pointerIndex=*/0, /*historicalIndex=*/i); + EXPECT_EQ(predictionPoints[i].position[0], coords.getY()); + EXPECT_EQ(predictionPoints[i].position[1], coords.getX()); + EXPECT_EQ(predictionPoints[i].pressure, coords.getAxisValue(AMOTION_EVENT_AXIS_PRESSURE)); + // Note: originTimestamp is discarded when converting PredictionPoint to MotionEvent. + EXPECT_EQ(predictionPoints[i].targetTimestamp, + predictionMotionEvent.getHistoricalEventTime(i)); + EXPECT_EQ(AMOTION_EVENT_ACTION_MOVE, predictionMotionEvent.getAction()); + } +} + +TEST(MakeMotionEventTest, MakeLiftMotionEvent) { + const MotionEvent liftMotionEvent = makeLiftMotionEvent(); + ASSERT_EQ(1u, liftMotionEvent.getPointerCount()); + // Note: a MotionEvent's "history size" is one less than its number of samples. + ASSERT_EQ(0u, liftMotionEvent.getHistorySize()); + EXPECT_EQ(AMOTION_EVENT_ACTION_UP, liftMotionEvent.getAction()); +} + +// --- Ground-truth-generation helper functions. --- + +std::vector<GroundTruthPoint> generateConstantGroundTruthPoints( + const GroundTruthPoint& groundTruthPoint, size_t numPoints) { + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = groundTruthPoint.timestamp; + for (size_t i = 0; i < numPoints; ++i) { + groundTruthPoints.emplace_back(groundTruthPoint); + groundTruthPoints.back().timestamp = timestamp; + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + return groundTruthPoints; +} + +// This function uses the coordinate system (y, x), with +y pointing downwards and +x pointing +// rightwards. Angles are measured counterclockwise from down (+y). +std::vector<GroundTruthPoint> generateCircularArcGroundTruthPoints(Eigen::Vector2f initialPosition, + float initialAngle, + float velocity, + float turningAngle, + size_t numPoints) { + std::vector<GroundTruthPoint> groundTruthPoints; + // Create first point. + if (numPoints > 0) { + groundTruthPoints.push_back({{.position = initialPosition, .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}); + } + float trajectoryAngle = initialAngle; // measured counterclockwise from +y axis. + for (size_t i = 1; i < numPoints; ++i) { + const Eigen::Vector2f trajectory = + Eigen::Rotation2D(trajectoryAngle) * Eigen::Vector2f(1, 0); + groundTruthPoints.push_back( + {{.position = groundTruthPoints.back().position + velocity * trajectory, + .pressure = 0.0f}, + .timestamp = groundTruthPoints.back().timestamp + TEST_PREDICTION_INTERVAL_NANOS}); + trajectoryAngle += turningAngle; + } + return groundTruthPoints; +} + +TEST(GenerateConstantGroundTruthPointsTest, BasicTest) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10, 20), .pressure = 0.3f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(groundTruthPoint, /*numPoints=*/3); + + ASSERT_EQ(3u, groundTruthPoints.size()); + // First point. + EXPECT_EQ(groundTruthPoints[0].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[0].pressure, groundTruthPoint.pressure); + EXPECT_EQ(groundTruthPoints[0].timestamp, groundTruthPoint.timestamp); + // Second point. + EXPECT_EQ(groundTruthPoints[1].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[1].pressure, groundTruthPoint.pressure); + EXPECT_GT(groundTruthPoints[1].timestamp, groundTruthPoints[0].timestamp); + // Third point. + EXPECT_EQ(groundTruthPoints[2].position, groundTruthPoint.position); + EXPECT_EQ(groundTruthPoints[2].pressure, groundTruthPoint.pressure); + EXPECT_GT(groundTruthPoints[2].timestamp, groundTruthPoints[1].timestamp); +} + +TEST(GenerateCircularArcGroundTruthTest, StraightLineUpwards) { + const std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(0, 0), + /*initialAngle=*/M_PI, + /*velocity=*/1.0f, + /*turningAngle=*/0.0f, + /*numPoints=*/3); + + ASSERT_EQ(3u, groundTruthPoints.size()); + EXPECT_THAT(groundTruthPoints[0].position, Vector2fNear(Eigen::Vector2f(0, 0), 1e-6)); + EXPECT_THAT(groundTruthPoints[1].position, Vector2fNear(Eigen::Vector2f(-1, 0), 1e-6)); + EXPECT_THAT(groundTruthPoints[2].position, Vector2fNear(Eigen::Vector2f(-2, 0), 1e-6)); + // Check that timestamps are increasing between consecutive ground truth points. + EXPECT_GT(groundTruthPoints[1].timestamp, groundTruthPoints[0].timestamp); + EXPECT_GT(groundTruthPoints[2].timestamp, groundTruthPoints[1].timestamp); +} + +TEST(GenerateCircularArcGroundTruthTest, CounterclockwiseSquare) { + // Generate points in a counterclockwise unit square starting pointing right. + const std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(10, 100), + /*initialAngle=*/M_PI_2, + /*velocity=*/1.0f, + /*turningAngle=*/M_PI_2, + /*numPoints=*/5); + + ASSERT_EQ(5u, groundTruthPoints.size()); + EXPECT_THAT(groundTruthPoints[0].position, Vector2fNear(Eigen::Vector2f(10, 100), 1e-6)); + EXPECT_THAT(groundTruthPoints[1].position, Vector2fNear(Eigen::Vector2f(10, 101), 1e-6)); + EXPECT_THAT(groundTruthPoints[2].position, Vector2fNear(Eigen::Vector2f(9, 101), 1e-6)); + EXPECT_THAT(groundTruthPoints[3].position, Vector2fNear(Eigen::Vector2f(9, 100), 1e-6)); + EXPECT_THAT(groundTruthPoints[4].position, Vector2fNear(Eigen::Vector2f(10, 100), 1e-6)); +} + +// --- Prediction-generation helper functions. --- + +// Creates a sequence of predictions with values equal to those of the given GroundTruthPoint. +std::vector<PredictionPoint> generateConstantPredictions(const GroundTruthPoint& groundTruthPoint) { + std::vector<PredictionPoint> predictions; + nsecs_t predictionTimestamp = groundTruthPoint.timestamp + TEST_PREDICTION_INTERVAL_NANOS; + for (size_t j = 0; j < TEST_MAX_NUM_PREDICTIONS; ++j) { + predictions.push_back(PredictionPoint{{.position = groundTruthPoint.position, + .pressure = groundTruthPoint.pressure}, + .originTimestamp = groundTruthPoint.timestamp, + .targetTimestamp = predictionTimestamp}); + predictionTimestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + return predictions; +} + +// Generates TEST_MAX_NUM_PREDICTIONS predictions from the given most recent two ground truth points +// by linear extrapolation of position and pressure. The interval between consecutive predictions' +// timestamps is TEST_PREDICTION_INTERVAL_NANOS. +std::vector<PredictionPoint> generatePredictionsByLinearExtrapolation( + const GroundTruthPoint& firstGroundTruth, const GroundTruthPoint& secondGroundTruth) { + // Precompute deltas. + const Eigen::Vector2f trajectory = secondGroundTruth.position - firstGroundTruth.position; + const float deltaPressure = secondGroundTruth.pressure - firstGroundTruth.pressure; + // Compute predictions. + std::vector<PredictionPoint> predictions; + Eigen::Vector2f predictionPosition = secondGroundTruth.position; + float predictionPressure = secondGroundTruth.pressure; + nsecs_t predictionTargetTimestamp = secondGroundTruth.timestamp; + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS; ++i) { + predictionPosition += trajectory; + predictionPressure += deltaPressure; + predictionTargetTimestamp += TEST_PREDICTION_INTERVAL_NANOS; + predictions.push_back( + PredictionPoint{{.position = predictionPosition, .pressure = predictionPressure}, + .originTimestamp = secondGroundTruth.timestamp, + .targetTimestamp = predictionTargetTimestamp}); + } + return predictions; +} + +TEST(GeneratePredictionsTest, GenerateConstantPredictions) { + const GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10, 20), .pressure = 0.3f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + const std::vector<PredictionPoint> predictionPoints = + generateConstantPredictions(groundTruthPoint); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, predictionPoints.size()); + for (size_t i = 0; i < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + EXPECT_THAT(predictionPoints[i].position, Vector2fNear(groundTruthPoint.position, 1e-6)); + EXPECT_THAT(predictionPoints[i].pressure, FloatNear(groundTruthPoint.pressure, 1e-6)); + EXPECT_EQ(predictionPoints[i].originTimestamp, groundTruthPoint.timestamp); + EXPECT_EQ(predictionPoints[i].targetTimestamp, + groundTruthPoint.timestamp + + static_cast<nsecs_t>(i + 1) * TEST_PREDICTION_INTERVAL_NANOS); + } +} + +TEST(GeneratePredictionsTest, LinearExtrapolationFromTwoPoints) { + const nsecs_t initialTimestamp = TEST_INITIAL_TIMESTAMP; + const std::vector<PredictionPoint> predictionPoints = generatePredictionsByLinearExtrapolation( + GroundTruthPoint{{.position = Eigen::Vector2f(100, 200), .pressure = 0.9f}, + .timestamp = initialTimestamp}, + GroundTruthPoint{{.position = Eigen::Vector2f(105, 190), .pressure = 0.8f}, + .timestamp = initialTimestamp + TEST_PREDICTION_INTERVAL_NANOS}); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, predictionPoints.size()); + const nsecs_t originTimestamp = initialTimestamp + TEST_PREDICTION_INTERVAL_NANOS; + EXPECT_THAT(predictionPoints[0], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(110, 180), + .pressure = 0.7f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[1], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(115, 170), + .pressure = 0.6f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 2 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[2], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(120, 160), + .pressure = 0.5f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 3 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[3], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(125, 150), + .pressure = 0.4f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 4 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); + EXPECT_THAT(predictionPoints[4], + PredictionPointNear(PredictionPoint{{.position = Eigen::Vector2f(130, 140), + .pressure = 0.3f}, + .originTimestamp = originTimestamp, + .targetTimestamp = originTimestamp + + 5 * TEST_PREDICTION_INTERVAL_NANOS}, + 0.001)); +} + +// Generates predictions by linear extrapolation for each consecutive pair of ground truth points +// (see the comment for the above function for further explanation). Returns a vector of vectors of +// prediction points, where the first index is the source ground truth index, and the second is the +// prediction target index. +// +// The returned vector has size equal to the input vector, and the first element of the returned +// vector is always empty. +std::vector<std::vector<PredictionPoint>> generateAllPredictionsByLinearExtrapolation( + const std::vector<GroundTruthPoint>& groundTruthPoints) { + std::vector<std::vector<PredictionPoint>> allPredictions; + allPredictions.emplace_back(); + for (size_t i = 1; i < groundTruthPoints.size(); ++i) { + allPredictions.push_back(generatePredictionsByLinearExtrapolation(groundTruthPoints[i - 1], + groundTruthPoints[i])); + } + return allPredictions; +} + +TEST(GeneratePredictionsTest, GenerateAllPredictions) { + const nsecs_t initialTimestamp = TEST_INITIAL_TIMESTAMP; + std::vector<GroundTruthPoint> + groundTruthPoints{GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.5f}, + .timestamp = initialTimestamp}, + GroundTruthPoint{{.position = Eigen::Vector2f(1, -1), + .pressure = 0.51f}, + .timestamp = initialTimestamp + + 2 * TEST_PREDICTION_INTERVAL_NANOS}, + GroundTruthPoint{{.position = Eigen::Vector2f(2, -2), + .pressure = 0.52f}, + .timestamp = initialTimestamp + + 3 * TEST_PREDICTION_INTERVAL_NANOS}}; + + const std::vector<std::vector<PredictionPoint>> allPredictions = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + // Check format of allPredictions data. + ASSERT_EQ(groundTruthPoints.size(), allPredictions.size()); + EXPECT_TRUE(allPredictions[0].empty()); + EXPECT_EQ(TEST_MAX_NUM_PREDICTIONS, allPredictions[1].size()); + EXPECT_EQ(TEST_MAX_NUM_PREDICTIONS, allPredictions[2].size()); + + // Check positions of predictions generated from first pair of ground truth points. + EXPECT_THAT(allPredictions[1][0].position, Vector2fNear(Eigen::Vector2f(2, -2), 1e-9)); + EXPECT_THAT(allPredictions[1][1].position, Vector2fNear(Eigen::Vector2f(3, -3), 1e-9)); + EXPECT_THAT(allPredictions[1][2].position, Vector2fNear(Eigen::Vector2f(4, -4), 1e-9)); + EXPECT_THAT(allPredictions[1][3].position, Vector2fNear(Eigen::Vector2f(5, -5), 1e-9)); + EXPECT_THAT(allPredictions[1][4].position, Vector2fNear(Eigen::Vector2f(6, -6), 1e-9)); + + // Check pressures of predictions generated from first pair of ground truth points. + EXPECT_FLOAT_EQ(0.52f, allPredictions[1][0].pressure); + EXPECT_FLOAT_EQ(0.53f, allPredictions[1][1].pressure); + EXPECT_FLOAT_EQ(0.54f, allPredictions[1][2].pressure); + EXPECT_FLOAT_EQ(0.55f, allPredictions[1][3].pressure); + EXPECT_FLOAT_EQ(0.56f, allPredictions[1][4].pressure); +} + +// --- Prediction error helper functions. --- + +struct GeneralPositionErrors { + float alongTrajectoryErrorMean; + float alongTrajectoryErrorStd; + float offTrajectoryRmse; +}; + +// Inputs: +// • Vector of ground truth points +// • Vector of vectors of prediction points, where the first index is the source ground truth +// index, and the second is the prediction target index. +// +// Returns a vector of GeneralPositionErrors, indexed by prediction time delta bucket. +std::vector<GeneralPositionErrors> computeGeneralPositionErrors( + const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints) { + // Aggregate errors by time bucket (prediction target index). + std::vector<GeneralPositionErrors> generalPostitionErrors; + for (size_t predictionTargetIndex = 0; predictionTargetIndex < TEST_MAX_NUM_PREDICTIONS; + ++predictionTargetIndex) { + std::vector<float> alongTrajectoryErrors; + std::vector<float> alongTrajectorySquaredErrors; + std::vector<float> offTrajectoryErrors; + for (size_t sourceGroundTruthIndex = 1; sourceGroundTruthIndex < groundTruthPoints.size(); + ++sourceGroundTruthIndex) { + const size_t targetGroundTruthIndex = + sourceGroundTruthIndex + predictionTargetIndex + 1; + // Only include errors for points with a ground truth value. + if (targetGroundTruthIndex < groundTruthPoints.size()) { + const Eigen::Vector2f trajectory = + (groundTruthPoints[targetGroundTruthIndex].position - + groundTruthPoints[targetGroundTruthIndex - 1].position) + .normalized(); + const Eigen::Vector2f orthogonalTrajectory = + Eigen::Rotation2Df(M_PI_2) * trajectory; + const Eigen::Vector2f positionError = + predictionPoints[sourceGroundTruthIndex][predictionTargetIndex].position - + groundTruthPoints[targetGroundTruthIndex].position; + alongTrajectoryErrors.push_back(positionError.dot(trajectory)); + alongTrajectorySquaredErrors.push_back(alongTrajectoryErrors.back() * + alongTrajectoryErrors.back()); + offTrajectoryErrors.push_back(positionError.dot(orthogonalTrajectory)); + } + } + generalPostitionErrors.push_back( + {.alongTrajectoryErrorMean = average(alongTrajectoryErrors), + .alongTrajectoryErrorStd = standardDeviation(alongTrajectoryErrors), + .offTrajectoryRmse = rmse(offTrajectoryErrors)}); + } + return generalPostitionErrors; +} + +// Inputs: +// • Vector of ground truth points +// • Vector of vectors of prediction points, where the first index is the source ground truth +// index, and the second is the prediction target index. +// +// Returns a vector of pressure RMSEs, indexed by prediction time delta bucket. +std::vector<float> computePressureRmses( + const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints) { + // Aggregate errors by time bucket (prediction target index). + std::vector<float> pressureRmses; + for (size_t predictionTargetIndex = 0; predictionTargetIndex < TEST_MAX_NUM_PREDICTIONS; + ++predictionTargetIndex) { + std::vector<float> pressureErrors; + for (size_t sourceGroundTruthIndex = 1; sourceGroundTruthIndex < groundTruthPoints.size(); + ++sourceGroundTruthIndex) { + const size_t targetGroundTruthIndex = + sourceGroundTruthIndex + predictionTargetIndex + 1; + // Only include errors for points with a ground truth value. + if (targetGroundTruthIndex < groundTruthPoints.size()) { + pressureErrors.push_back( + predictionPoints[sourceGroundTruthIndex][predictionTargetIndex].pressure - + groundTruthPoints[targetGroundTruthIndex].pressure); + } + } + pressureRmses.push_back(rmse(pressureErrors)); + } + return pressureRmses; +} + +TEST(ErrorComputationHelperTest, ComputeGeneralPositionErrorsSimpleTest) { + std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + groundTruthPoints[3].position = Eigen::Vector2f(1, 0); + groundTruthPoints[4].position = Eigen::Vector2f(1, 1); + groundTruthPoints[5].position = Eigen::Vector2f(1, 3); + groundTruthPoints[6].position = Eigen::Vector2f(2, 3); + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + // The generated predictions look like: + // + // | Source | Target Ground Truth Index | + // | Index | 2 | 3 | 4 | 5 | 6 | + // |------------|--------|--------|--------|--------|--------| + // | 1 | (0, 0) | (0, 0) | (0, 0) | (0, 0) | (0, 0) | + // | 2 | | (0, 0) | (0, 0) | (0, 0) | (0, 0) | + // | 3 | | | (2, 0) | (3, 0) | (4, 0) | + // | 4 | | | | (1, 2) | (1, 3) | + // | 5 | | | | | (1, 5) | + // |---------------------------------------------------------| + // | Actual Ground Truth Values | + // | Position | (0, 0) | (1, 0) | (1, 1) | (1, 3) | (2, 3) | + // | Previous | (0, 0) | (0, 0) | (1, 0) | (1, 1) | (1, 3) | + // + // Note: this table organizes prediction targets by target ground truth index. Metrics are + // aggregated across points with the same prediction time bucket index, which is different. + // Each down-right diagonal from this table gives us points from a unique time bucket. + + // Initialize expected prediction errors from the table above. The first time bucket corresponds + // to the long diagonal of the table, and subsequent time buckets step up-right from there. + const std::vector<std::vector<float>> expectedAlongTrajectoryErrors{{0, -1, -1, -1, -1}, + {-1, -1, -3, -1}, + {-1, -3, 2}, + {-3, -2}, + {-2}}; + const std::vector<std::vector<float>> expectedOffTrajectoryErrors{{0, 0, 1, 0, 2}, + {0, 1, 2, 0}, + {1, 1, 3}, + {1, 3}, + {3}}; + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, generalPositionErrors.size()); + for (size_t i = 0; i < generalPositionErrors.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + EXPECT_FLOAT_EQ(average(expectedAlongTrajectoryErrors[i]), + generalPositionErrors[i].alongTrajectoryErrorMean); + EXPECT_FLOAT_EQ(standardDeviation(expectedAlongTrajectoryErrors[i]), + generalPositionErrors[i].alongTrajectoryErrorStd); + EXPECT_FLOAT_EQ(rmse(expectedOffTrajectoryErrors[i]), + generalPositionErrors[i].offTrajectoryRmse); + } +} + +TEST(ErrorComputationHelperTest, ComputePressureRmsesSimpleTest) { + // Generate ground truth points with pressures {0.0, 0.0, 0.0, 0.0, 0.5, 0.5, 0.5}. + // (We need TEST_MAX_NUM_PREDICTIONS + 2 to test all prediction time buckets.) + std::vector<GroundTruthPoint> groundTruthPoints = + generateConstantGroundTruthPoints(GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), + .pressure = 0.0f}, + .timestamp = TEST_INITIAL_TIMESTAMP}, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + for (size_t i = 4; i < groundTruthPoints.size(); ++i) { + groundTruthPoints[i].pressure = 0.5f; + } + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<float> pressureRmses = computePressureRmses(groundTruthPoints, predictionPoints); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, pressureRmses.size()); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{0.0f, 0.0f, -0.5f, 0.5f, 0.0f}), pressureRmses[0]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{0.0f, -0.5f, -0.5f, 1.0f}), pressureRmses[1]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f, -0.5f, -0.5f}), pressureRmses[2]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f, -0.5f}), pressureRmses[3]); + EXPECT_FLOAT_EQ(rmse(std::vector<float>{-0.5f}), pressureRmses[4]); +} + +// --- MotionPredictorMetricsManager tests. --- + +// Helper function that instantiates a MetricsManager with the given mock logged AtomFields. Takes +// vectors of ground truth and prediction points of the same length, and passes these points to the +// MetricsManager. The format of these vectors is expected to be: +// • groundTruthPoints: chronologically-ordered ground truth points, with at least 2 elements. +// • predictionPoints: the first index points to a vector of predictions corresponding to the +// source ground truth point with the same index. +// - The first element should be empty, because there are not expected to be predictions until +// we have received 2 ground truth points. +// - The last element may be empty, because there will be no future ground truth points to +// associate with those predictions (if not empty, it will be ignored). +// - To test all prediction buckets, there should be at least TEST_MAX_NUM_PREDICTIONS non-empty +// prediction sets (that is, excluding the first and last). Thus, groundTruthPoints and +// predictionPoints should have size at least TEST_MAX_NUM_PREDICTIONS + 2. +// +// The passed-in outAtomFields will contain the logged AtomFields when the function returns. +// +// This function returns void so that it can use test assertions. +void runMetricsManager(const std::vector<GroundTruthPoint>& groundTruthPoints, + const std::vector<std::vector<PredictionPoint>>& predictionPoints, + std::vector<AtomFields>& outAtomFields) { + MotionPredictorMetricsManager metricsManager(TEST_PREDICTION_INTERVAL_NANOS, + TEST_MAX_NUM_PREDICTIONS); + metricsManager.setMockLoggedAtomFields(&outAtomFields); + + // Validate structure of groundTruthPoints and predictionPoints. + ASSERT_EQ(predictionPoints.size(), groundTruthPoints.size()); + ASSERT_GE(groundTruthPoints.size(), 2u); + ASSERT_EQ(predictionPoints[0].size(), 0u); + for (size_t i = 1; i + 1 < predictionPoints.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + ASSERT_EQ(predictionPoints[i].size(), TEST_MAX_NUM_PREDICTIONS); + } + + // Pass ground truth points and predictions (for all except first and last ground truth). + for (size_t i = 0; i < groundTruthPoints.size(); ++i) { + metricsManager.onRecord(makeMotionEvent(groundTruthPoints[i])); + if ((i > 0) && (i + 1 < predictionPoints.size())) { + metricsManager.onPredict(makeMotionEvent(predictionPoints[i])); + } + } + // Send a stroke-end event to trigger the logging call. + metricsManager.onRecord(makeLiftMotionEvent()); +} + +// Vacuous test: +// • Input: no prediction data. +// • Expectation: no metrics should be logged. +TEST(MotionPredictorMetricsManagerTest, NoPredictions) { + std::vector<AtomFields> mockLoggedAtomFields; + MotionPredictorMetricsManager metricsManager(TEST_PREDICTION_INTERVAL_NANOS, + TEST_MAX_NUM_PREDICTIONS); + metricsManager.setMockLoggedAtomFields(&mockLoggedAtomFields); + + metricsManager.onRecord(makeMotionEvent( + GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), .pressure = 0}, .timestamp = 0})); + metricsManager.onRecord(makeLiftMotionEvent()); + + // Check that mockLoggedAtomFields is still empty (as it was initialized empty), ensuring that + // no metrics were logged. + EXPECT_EQ(0u, mockLoggedAtomFields.size()); +} + +// Perfect predictions test: +// • Input: constant input events, perfect predictions matching the input events. +// • Expectation: all error metrics should be zero, or NO_DATA_SENTINEL for "unreported" metrics. +// (For example, scale-invariant errors are only reported for the final time bucket.) +TEST(MotionPredictorMetricsManagerTest, ConstantGroundTruthPerfectPredictions) { + GroundTruthPoint groundTruthPoint{{.position = Eigen::Vector2f(10.0f, 20.0f), .pressure = 0.6f}, + .timestamp = TEST_INITIAL_TIMESTAMP}; + + // Generate ground truth and prediction points as described by the runMetricsManager comment. + std::vector<GroundTruthPoint> groundTruthPoints; + std::vector<std::vector<PredictionPoint>> predictionPoints; + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + groundTruthPoints.push_back(groundTruthPoint); + predictionPoints.push_back(i > 0 ? generateConstantPredictions(groundTruthPoint) + : std::vector<PredictionPoint>{}); + groundTruthPoint.timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + // Check that errors are all zero, or NO_DATA_SENTINEL for unreported metrics. + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // General errors: reported for every time bucket. + EXPECT_EQ(0, atom.alongTrajectoryErrorMeanMillipixels); + EXPECT_EQ(0, atom.alongTrajectoryErrorStdMillipixels); + EXPECT_EQ(0, atom.offTrajectoryRmseMillipixels); + EXPECT_EQ(0, atom.pressureRmseMilliunits); + // High-velocity errors: reported only for the last two time buckets. + // However, this data has zero velocity, so these metrics should all be NO_DATA_SENTINEL. + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityOffTrajectoryRmse); + // Scale-invariant errors: reported only for the last time bucket. + if (i + 1 == atomFields.size()) { + EXPECT_EQ(0, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(0, atom.scaleInvariantOffTrajectoryRmse); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantOffTrajectoryRmse); + } + } +} + +TEST(MotionPredictorMetricsManagerTest, QuadraticPressureLinearPredictions) { + // Generate ground truth points. + // + // Ground truth pressures are a quadratically increasing function from some initial value. + const float initialPressure = 0.5f; + const float quadraticCoefficient = 0.01f; + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = TEST_INITIAL_TIMESTAMP; + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + const float pressure = initialPressure + quadraticCoefficient * static_cast<float>(i * i); + groundTruthPoints.push_back( + GroundTruthPoint{{.position = Eigen::Vector2f(0, 0), .pressure = pressure}, + .timestamp = timestamp}); + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + // Note: the first index is the source ground truth index, and the second is the prediction + // target index. + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + const std::vector<float> pressureErrors = + computePressureRmses(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + // Check time bucket delta matches expectation based on index and prediction interval. + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // Check pressure error matches expectation. + EXPECT_NEAR(static_cast<int>(1000 * pressureErrors[i]), atom.pressureRmseMilliunits, 1); + } +} + +TEST(MotionPredictorMetricsManagerTest, QuadraticPositionLinearPredictionsGeneralErrors) { + // Generate ground truth points. + // + // Each component of the ground truth positions are an independent quadratically increasing + // function from some initial value. + const Eigen::Vector2f initialPosition(200, 300); + const Eigen::Vector2f quadraticCoefficients(-2, 3); + std::vector<GroundTruthPoint> groundTruthPoints; + nsecs_t timestamp = TEST_INITIAL_TIMESTAMP; + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + for (size_t i = 0; i < TEST_MAX_NUM_PREDICTIONS + 2; ++i) { + const Eigen::Vector2f position = + initialPosition + quadraticCoefficients * static_cast<float>(i * i); + groundTruthPoints.push_back( + GroundTruthPoint{{.position = position, .pressure = 0.5}, .timestamp = timestamp}); + timestamp += TEST_PREDICTION_INTERVAL_NANOS; + } + + // Note: the first index is the source ground truth index, and the second is the prediction + // target index. + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + // Check time bucket delta matches expectation based on index and prediction interval. + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + // Check general position errors match expectation. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorMean), + atom.alongTrajectoryErrorMeanMillipixels, 1); + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorStd), + atom.alongTrajectoryErrorStdMillipixels, 1); + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].offTrajectoryRmse), + atom.offTrajectoryRmseMillipixels, 1); + } +} + +// Counterclockwise regular octagonal section test: +// • Input – ground truth: constantly-spaced input events starting at a trajectory pointing exactly +// rightwards, and rotating by 45° counterclockwise after each input. +// • Input – predictions: simple linear extrapolations of previous two ground truth points. +// +// The code below uses the following terminology to distinguish references to ground truth events: +// • Source ground truth: the most recent ground truth point received at the time the prediction +// was made. +// • Target ground truth: the ground truth event that the prediction was attempting to match. +TEST(MotionPredictorMetricsManagerTest, CounterclockwiseOctagonGroundTruthLinearPredictions) { + // Select a stroke velocity that exceeds the high-velocity threshold of 1100 px/sec. + // For an input rate of 240 hz, 1100 px/sec * (1/240) sec/input ≈ 4.58 pixels per input. + const float strokeVelocity = 10; // pixels per input + + // As described in the runMetricsManager comment, we should have TEST_MAX_NUM_PREDICTIONS + 2 + // ground truth points. + std::vector<GroundTruthPoint> groundTruthPoints = generateCircularArcGroundTruthPoints( + /*initialPosition=*/Eigen::Vector2f(100, 100), + /*initialAngle=*/M_PI_2, + /*velocity=*/strokeVelocity, + /*turningAngle=*/-M_PI_4, + /*numPoints=*/TEST_MAX_NUM_PREDICTIONS + 2); + + std::vector<std::vector<PredictionPoint>> predictionPoints = + generateAllPredictionsByLinearExtrapolation(groundTruthPoints); + + std::vector<GeneralPositionErrors> generalPositionErrors = + computeGeneralPositionErrors(groundTruthPoints, predictionPoints); + + // Run test. + std::vector<AtomFields> atomFields; + runMetricsManager(groundTruthPoints, predictionPoints, atomFields); + + // Check logged metrics match expectations. + ASSERT_EQ(TEST_MAX_NUM_PREDICTIONS, atomFields.size()); + for (size_t i = 0; i < atomFields.size(); ++i) { + SCOPED_TRACE(testing::Message() << "i = " << i); + const AtomFields& atom = atomFields[i]; + const nsecs_t deltaTimeBucketNanos = TEST_PREDICTION_INTERVAL_NANOS * (i + 1); + EXPECT_EQ(deltaTimeBucketNanos / NANOS_PER_MILLIS, atom.deltaTimeBucketMilliseconds); + + // General errors: reported for every time bucket. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].alongTrajectoryErrorMean), + atom.alongTrajectoryErrorMeanMillipixels, 1); + // We allow for some floating point error in standard deviation (0.02 pixels). + EXPECT_NEAR(1000 * generalPositionErrors[i].alongTrajectoryErrorStd, + atom.alongTrajectoryErrorStdMillipixels, 20); + // All position errors are equal, so the standard deviation should be approximately zero. + EXPECT_NEAR(0, atom.alongTrajectoryErrorStdMillipixels, 20); + // Absolute value for RMSE, since it must be non-negative. + EXPECT_NEAR(static_cast<int>(1000 * generalPositionErrors[i].offTrajectoryRmse), + atom.offTrajectoryRmseMillipixels, 1); + + // High-velocity errors: reported only for the last two time buckets. + // + // Since our input stroke velocity is chosen to be above the high-velocity threshold, all + // data contributes to high-velocity errors, and thus high-velocity errors should be equal + // to general errors (where reported). + // + // As above, use absolute value for RMSE, since it must be non-negative. + if (i + 2 >= atomFields.size()) { + EXPECT_NEAR(static_cast<int>( + 1000 * std::abs(generalPositionErrors[i].alongTrajectoryErrorMean)), + atom.highVelocityAlongTrajectoryRmse, 1); + EXPECT_NEAR(static_cast<int>(1000 * + std::abs(generalPositionErrors[i].offTrajectoryRmse)), + atom.highVelocityOffTrajectoryRmse, 1); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.highVelocityOffTrajectoryRmse); + } + + // Scale-invariant errors: reported only for the last time bucket, where the reported value + // is the aggregation across all time buckets. + // + // The MetricsManager stores mMaxNumPredictions recent ground truth segments. Our ground + // truth segments here all have a length of strokeVelocity, so we can convert general errors + // to scale-invariant errors by dividing by `strokeVelocty * TEST_MAX_NUM_PREDICTIONS`. + // + // As above, use absolute value for RMSE, since it must be non-negative. + if (i + 1 == atomFields.size()) { + const float pathLength = strokeVelocity * TEST_MAX_NUM_PREDICTIONS; + std::vector<float> alongTrajectoryAbsoluteErrors; + std::vector<float> offTrajectoryAbsoluteErrors; + for (size_t j = 0; j < TEST_MAX_NUM_PREDICTIONS; ++j) { + alongTrajectoryAbsoluteErrors.push_back( + std::abs(generalPositionErrors[j].alongTrajectoryErrorMean)); + offTrajectoryAbsoluteErrors.push_back( + std::abs(generalPositionErrors[j].offTrajectoryRmse)); + } + EXPECT_NEAR(static_cast<int>(1000 * average(alongTrajectoryAbsoluteErrors) / + pathLength), + atom.scaleInvariantAlongTrajectoryRmse, 1); + EXPECT_NEAR(static_cast<int>(1000 * average(offTrajectoryAbsoluteErrors) / pathLength), + atom.scaleInvariantOffTrajectoryRmse, 1); + } else { + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantAlongTrajectoryRmse); + EXPECT_EQ(NO_DATA_SENTINEL, atom.scaleInvariantOffTrajectoryRmse); + } + } +} + +} // namespace +} // namespace android diff --git a/libs/input/tests/MotionPredictor_test.cpp b/libs/input/tests/MotionPredictor_test.cpp index 7a62f5ec58..4ac7ae920e 100644 --- a/libs/input/tests/MotionPredictor_test.cpp +++ b/libs/input/tests/MotionPredictor_test.cpp @@ -72,11 +72,20 @@ TEST(MotionPredictorTest, IsPredictionAvailable) { ASSERT_FALSE(predictor.isPredictionAvailable(/*deviceId=*/1, AINPUT_SOURCE_TOUCHSCREEN)); } +TEST(MotionPredictorTest, StationaryNoiseFloor) { + MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/1, + []() { return true /*enable prediction*/; }); + predictor.record(getMotionEvent(DOWN, 0, 1, 30ms)); + predictor.record(getMotionEvent(MOVE, 0, 1, 35ms)); // No movement. + std::unique_ptr<MotionEvent> predicted = predictor.predict(40 * NSEC_PER_MSEC); + ASSERT_EQ(nullptr, predicted); +} + TEST(MotionPredictorTest, Offset) { MotionPredictor predictor(/*predictionTimestampOffsetNanos=*/1, []() { return true /*enable prediction*/; }); predictor.record(getMotionEvent(DOWN, 0, 1, 30ms)); - predictor.record(getMotionEvent(MOVE, 0, 2, 35ms)); + predictor.record(getMotionEvent(MOVE, 0, 5, 35ms)); // Move enough to overcome the noise floor. std::unique_ptr<MotionEvent> predicted = predictor.predict(40 * NSEC_PER_MSEC); ASSERT_NE(nullptr, predicted); ASSERT_GE(predicted->getEventTime(), 41); diff --git a/libs/input/tests/VelocityTracker_test.cpp b/libs/input/tests/VelocityTracker_test.cpp index ae721093a0..73f25cc615 100644 --- a/libs/input/tests/VelocityTracker_test.cpp +++ b/libs/input/tests/VelocityTracker_test.cpp @@ -282,6 +282,11 @@ static void computeAndCheckAxisScrollVelocity( const std::vector<std::pair<std::chrono::nanoseconds, float>>& motions, std::optional<float> targetVelocity) { checkVelocity(computeVelocity(strategy, motions, AMOTION_EVENT_AXIS_SCROLL), targetVelocity); + // The strategy LSQ2 is not compatible with AXIS_SCROLL. In those situations, we should fall + // back to a strategy that supports differential axes. + checkVelocity(computeVelocity(VelocityTracker::Strategy::LSQ2, motions, + AMOTION_EVENT_AXIS_SCROLL), + targetVelocity); } static void computeAndCheckQuadraticEstimate(const std::vector<PlanarMotionEventEntry>& motions, diff --git a/libs/nativewindow/include/system/window.h b/libs/nativewindow/include/system/window.h index 0fee3c112e..edaa422e55 100644 --- a/libs/nativewindow/include/system/window.h +++ b/libs/nativewindow/include/system/window.h @@ -1066,12 +1066,33 @@ static inline int native_window_set_frame_rate(struct ANativeWindow* window, flo (int)compatibility, (int)changeFrameRateStrategy); } +struct ANativeWindowFrameTimelineInfo { + // Frame Id received from ANativeWindow_getNextFrameId. + uint64_t frameNumber; + + // VsyncId received from the Choreographer callback that started this frame. + int64_t frameTimelineVsyncId; + + // Input Event ID received from the input event that started this frame. + int32_t inputEventId; + + // The time which this frame rendering started (i.e. when Choreographer callback actually run) + int64_t startTimeNanos; + + // Whether or not to use the vsyncId to determine the refresh rate. Used for TextureView only. + int32_t useForRefreshRateSelection; + + // The VsyncId of a frame that was not drawn and squashed into this frame. + // Used for UI thread updates that were not picked up by RenderThread on time. + int64_t skippedFrameVsyncId; + + // The start time of a frame that was not drawn and squashed into this frame. + int64_t skippedFrameStartTimeNanos; +}; + static inline int native_window_set_frame_timeline_info( - struct ANativeWindow* window, uint64_t frameNumber, int64_t frameTimelineVsyncId, - int32_t inputEventId, int64_t startTimeNanos, int32_t useForRefreshRateSelection) { - return window->perform(window, NATIVE_WINDOW_SET_FRAME_TIMELINE_INFO, frameNumber, - frameTimelineVsyncId, inputEventId, startTimeNanos, - useForRefreshRateSelection); + struct ANativeWindow* window, struct ANativeWindowFrameTimelineInfo frameTimelineInfo) { + return window->perform(window, NATIVE_WINDOW_SET_FRAME_TIMELINE_INFO, frameTimelineInfo); } // ------------------------------------------------------------------------------------------------ diff --git a/libs/renderengine/skia/AutoBackendTexture.cpp b/libs/renderengine/skia/AutoBackendTexture.cpp index daa42c32b6..e8ad081e60 100644 --- a/libs/renderengine/skia/AutoBackendTexture.cpp +++ b/libs/renderengine/skia/AutoBackendTexture.cpp @@ -86,14 +86,38 @@ void AutoBackendTexture::releaseImageProc(SkImage::ReleaseContext releaseContext void logFatalTexture(const char* msg, const GrBackendTexture& tex, ui::Dataspace dataspace, SkColorType colorType) { - GrGLTextureInfo textureInfo; - bool retrievedTextureInfo = tex.getGLTextureInfo(&textureInfo); - LOG_ALWAYS_FATAL("%s isTextureValid:%d dataspace:%d" - "\n\tGrBackendTexture: (%i x %i) hasMipmaps: %i isProtected: %i texType: %i" - "\n\t\tGrGLTextureInfo: success: %i fTarget: %u fFormat: %u colorType %i", - msg, tex.isValid(), static_cast<int32_t>(dataspace), tex.width(), tex.height(), - tex.hasMipmaps(), tex.isProtected(), static_cast<int>(tex.textureType()), - retrievedTextureInfo, textureInfo.fTarget, textureInfo.fFormat, colorType); + switch (tex.backend()) { + case GrBackendApi::kOpenGL: { + GrGLTextureInfo textureInfo; + bool retrievedTextureInfo = tex.getGLTextureInfo(&textureInfo); + LOG_ALWAYS_FATAL("%s isTextureValid:%d dataspace:%d" + "\n\tGrBackendTexture: (%i x %i) hasMipmaps: %i isProtected: %i " + "texType: %i\n\t\tGrGLTextureInfo: success: %i fTarget: %u fFormat: %u" + " colorType %i", + msg, tex.isValid(), static_cast<int32_t>(dataspace), tex.width(), + tex.height(), tex.hasMipmaps(), tex.isProtected(), + static_cast<int>(tex.textureType()), retrievedTextureInfo, + textureInfo.fTarget, textureInfo.fFormat, colorType); + break; + } + case GrBackendApi::kVulkan: { + GrVkImageInfo imageInfo; + bool retrievedImageInfo = tex.getVkImageInfo(&imageInfo); + LOG_ALWAYS_FATAL("%s isTextureValid:%d dataspace:%d" + "\n\tGrBackendTexture: (%i x %i) hasMipmaps: %i isProtected: %i " + "texType: %i\n\t\tVkImageInfo: success: %i fFormat: %i " + "fSampleCount: %u fLevelCount: %u colorType %i", + msg, tex.isValid(), dataspace, tex.width(), tex.height(), + tex.hasMipmaps(), tex.isProtected(), + static_cast<int>(tex.textureType()), retrievedImageInfo, + imageInfo.fFormat, imageInfo.fSampleCount, imageInfo.fLevelCount, + colorType); + break; + } + default: + LOG_ALWAYS_FATAL("%s Unexpected backend %u", msg, static_cast<unsigned>(tex.backend())); + break; + } } sk_sp<SkImage> AutoBackendTexture::makeImage(ui::Dataspace dataspace, SkAlphaType alphaType, diff --git a/libs/renderengine/skia/SkiaRenderEngine.cpp b/libs/renderengine/skia/SkiaRenderEngine.cpp index edf734271a..d71e55f64c 100644 --- a/libs/renderengine/skia/SkiaRenderEngine.cpp +++ b/libs/renderengine/skia/SkiaRenderEngine.cpp @@ -58,9 +58,9 @@ #include <src/core/SkTraceEventCommon.h> #include <sync/sync.h> #include <ui/BlurRegion.h> -#include <ui/DataspaceUtils.h> #include <ui/DebugUtils.h> #include <ui/GraphicBuffer.h> +#include <ui/HdrRenderTypeUtils.h> #include <utils/Trace.h> #include <cmath> @@ -507,7 +507,8 @@ sk_sp<SkShader> SkiaRenderEngine::createRuntimeEffectShader( auto effect = shaders::LinearEffect{.inputDataspace = parameters.layer.sourceDataspace, .outputDataspace = parameters.outputDataSpace, - .undoPremultipliedAlpha = parameters.undoPremultipliedAlpha}; + .undoPremultipliedAlpha = parameters.undoPremultipliedAlpha, + .fakeOutputDataspace = parameters.fakeOutputDataspace}; auto effectIter = mRuntimeEffects.find(effect); sk_sp<SkRuntimeEffect> runtimeEffect = nullptr; @@ -662,6 +663,8 @@ void SkiaRenderEngine::drawLayersInternal( validateOutputBufferUsage(buffer->getBuffer()); auto grContext = getActiveGrContext(); + LOG_ALWAYS_FATAL_IF(grContext->abandoned(), "GrContext is abandoned/device lost at start of %s", + __func__); // any AutoBackendTexture deletions will now be deferred until cleanupPostRender is called DeferTextureCleanup dtc(mTextureCleanupMgr); @@ -901,12 +904,14 @@ void SkiaRenderEngine::drawLayersInternal( (display.outputDataspace & ui::Dataspace::TRANSFER_MASK) == static_cast<int32_t>(ui::Dataspace::TRANSFER_SRGB); - const ui::Dataspace runtimeEffectDataspace = !dimInLinearSpace && isExtendedHdr + const bool useFakeOutputDataspaceForRuntimeEffect = !dimInLinearSpace && isExtendedHdr; + + const ui::Dataspace fakeDataspace = useFakeOutputDataspaceForRuntimeEffect ? static_cast<ui::Dataspace>( (display.outputDataspace & ui::Dataspace::STANDARD_MASK) | ui::Dataspace::TRANSFER_GAMMA2_2 | (display.outputDataspace & ui::Dataspace::RANGE_MASK)) - : display.outputDataspace; + : ui::Dataspace::UNKNOWN; // If the input dataspace is range extended, the output dataspace transfer is sRGB // and dimmingStage is GAMMA_OETF, dim in linear space instead, and @@ -1013,7 +1018,8 @@ void SkiaRenderEngine::drawLayersInternal( .layerDimmingRatio = dimInLinearSpace ? layerDimmingRatio : 1.f, - .outputDataSpace = runtimeEffectDataspace})); + .outputDataSpace = display.outputDataspace, + .fakeOutputDataspace = fakeDataspace})); // Turn on dithering when dimming beyond this (arbitrary) threshold... static constexpr float kDimmingThreshold = 0.2f; @@ -1021,7 +1027,10 @@ void SkiaRenderEngine::drawLayersInternal( // Most HDR standards require at least 10-bits of color depth for source content, so we // can just extract the transfer function rather than dig into precise gralloc layout. // Furthermore, we can assume that the only 8-bit target we support is RGBA8888. - const bool requiresDownsample = isHdrDataspace(layer.sourceDataspace) && + const bool requiresDownsample = + getHdrRenderType(layer.sourceDataspace, + std::optional<ui::PixelFormat>(static_cast<ui::PixelFormat>( + buffer->getPixelFormat()))) != HdrRenderType::SDR && buffer->getPixelFormat() == PIXEL_FORMAT_RGBA_8888; if (layerDimmingRatio <= kDimmingThreshold || requiresDownsample) { paint.setDither(true); @@ -1077,7 +1086,8 @@ void SkiaRenderEngine::drawLayersInternal( .undoPremultipliedAlpha = false, .requiresLinearEffect = requiresLinearEffect, .layerDimmingRatio = layerDimmingRatio, - .outputDataSpace = runtimeEffectDataspace})); + .outputDataSpace = display.outputDataspace, + .fakeOutputDataspace = fakeDataspace})); } if (layer.disableBlending) { diff --git a/libs/renderengine/skia/SkiaRenderEngine.h b/libs/renderengine/skia/SkiaRenderEngine.h index 6457bfa9eb..723e73c29e 100644 --- a/libs/renderengine/skia/SkiaRenderEngine.h +++ b/libs/renderengine/skia/SkiaRenderEngine.h @@ -157,6 +157,7 @@ private: bool requiresLinearEffect; float layerDimmingRatio; const ui::Dataspace outputDataSpace; + const ui::Dataspace fakeOutputDataspace; }; sk_sp<SkShader> createRuntimeEffectShader(const RuntimeEffectShaderParameters&); diff --git a/libs/renderengine/skia/SkiaVkRenderEngine.cpp b/libs/renderengine/skia/SkiaVkRenderEngine.cpp index b99e3853ee..c16586bb6b 100644 --- a/libs/renderengine/skia/SkiaVkRenderEngine.cpp +++ b/libs/renderengine/skia/SkiaVkRenderEngine.cpp @@ -263,7 +263,7 @@ VulkanInterface initVulkanInterface(bool protectedContent = false) { VK_GET_INST_PROC(instance, EnumerateDeviceExtensionProperties); VK_GET_INST_PROC(instance, GetPhysicalDeviceProperties2); VK_GET_INST_PROC(instance, GetPhysicalDeviceExternalSemaphoreProperties); - VK_GET_INST_PROC(instance, GetPhysicalDeviceQueueFamilyProperties); + VK_GET_INST_PROC(instance, GetPhysicalDeviceQueueFamilyProperties2); VK_GET_INST_PROC(instance, GetPhysicalDeviceFeatures2); VK_GET_INST_PROC(instance, CreateDevice); @@ -342,17 +342,37 @@ VulkanInterface initVulkanInterface(bool protectedContent = false) { } uint32_t queueCount; - vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, &queueCount, nullptr); + vkGetPhysicalDeviceQueueFamilyProperties2(physicalDevice, &queueCount, nullptr); if (queueCount == 0) { BAIL("Could not find queues for physical device"); } - std::vector<VkQueueFamilyProperties> queueProps(queueCount); - vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, &queueCount, queueProps.data()); + std::vector<VkQueueFamilyProperties2> queueProps(queueCount); + std::vector<VkQueueFamilyGlobalPriorityPropertiesEXT> queuePriorityProps(queueCount); + VkQueueGlobalPriorityKHR queuePriority = VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_KHR; + // Even though we don't yet know if the VK_EXT_global_priority extension is available, + // we can safely add the request to the pNext chain, and if the extension is not + // available, it will be ignored. + for (uint32_t i = 0; i < queueCount; ++i) { + queuePriorityProps[i].sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_GLOBAL_PRIORITY_PROPERTIES_EXT; + queuePriorityProps[i].pNext = nullptr; + queueProps[i].pNext = &queuePriorityProps[i]; + } + vkGetPhysicalDeviceQueueFamilyProperties2(physicalDevice, &queueCount, queueProps.data()); int graphicsQueueIndex = -1; for (uint32_t i = 0; i < queueCount; ++i) { - if (queueProps[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) { + // Look at potential answers to the VK_EXT_global_priority query. If answers were + // provided, we may adjust the queuePriority. + if (queueProps[i].queueFamilyProperties.queueFlags & VK_QUEUE_GRAPHICS_BIT) { + for (uint32_t j = 0; j < queuePriorityProps[i].priorityCount; j++) { + if (queuePriorityProps[i].priorities[j] > queuePriority) { + queuePriority = queuePriorityProps[i].priorities[j]; + } + } + if (queuePriority == VK_QUEUE_GLOBAL_PRIORITY_REALTIME_KHR) { + interface.isRealtimePriority = true; + } graphicsQueueIndex = i; break; } @@ -419,12 +439,11 @@ VulkanInterface initVulkanInterface(bool protectedContent = false) { VK_STRUCTURE_TYPE_DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_EXT, nullptr, // If queue priority is supported, RE should always have realtime priority. - VK_QUEUE_GLOBAL_PRIORITY_REALTIME_EXT, + queuePriority, }; if (interface.grExtensions.hasExtension(VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME, 2)) { queueNextPtr = &queuePriorityCreateInfo; - interface.isRealtimePriority = true; } VkDeviceQueueCreateFlags deviceQueueCreateFlags = diff --git a/libs/shaders/shaders.cpp b/libs/shaders/shaders.cpp index c85517a976..ef039e5c36 100644 --- a/libs/shaders/shaders.cpp +++ b/libs/shaders/shaders.cpp @@ -168,8 +168,8 @@ void generateOOTF(ui::Dataspace inputDataspace, ui::Dataspace outputDataspace, void generateOETF(std::string& shader) { // Only support gamma 2.2 for now shader.append(R"( - float OETF(float3 linear) { - return sign(linear) * pow(abs(linear), (1.0 / 2.2)); + float3 OETF(float3 linear) { + return sign(linear) * pow(abs(linear), float3(1.0 / 2.2)); } )"); } diff --git a/libs/shaders/tests/Android.bp b/libs/shaders/tests/Android.bp index 1e4f45ac45..5639d744df 100644 --- a/libs/shaders/tests/Android.bp +++ b/libs/shaders/tests/Android.bp @@ -37,6 +37,7 @@ cc_test { shared_libs: [ "android.hardware.graphics.common@1.2", "libnativewindow", + "libbase", ], static_libs: [ "libarect", diff --git a/libs/tonemap/tests/Android.bp b/libs/tonemap/tests/Android.bp index 2abf51563c..5c5fc6c8b3 100644 --- a/libs/tonemap/tests/Android.bp +++ b/libs/tonemap/tests/Android.bp @@ -36,6 +36,7 @@ cc_test { ], shared_libs: [ "libnativewindow", + "libbase", ], static_libs: [ "libmath", diff --git a/services/surfaceflinger/Display/DisplayMap.h b/libs/ui/include/ui/DisplayMap.h index 0d5970676e..7eacb0a7f0 100644 --- a/services/surfaceflinger/Display/DisplayMap.h +++ b/libs/ui/include/ui/DisplayMap.h @@ -19,7 +19,7 @@ #include <ftl/small_map.h> #include <ftl/small_vector.h> -namespace android::display { +namespace android::ui { // The static capacities were chosen to exceed a typical number of physical and/or virtual displays. @@ -32,4 +32,4 @@ using PhysicalDisplayMap = ftl::SmallMap<Key, Value, 3>; template <typename T> using PhysicalDisplayVector = ftl::SmallVector<T, 3>; -} // namespace android::display +} // namespace android::ui diff --git a/libs/ui/include/ui/FenceTime.h b/libs/ui/include/ui/FenceTime.h index ac75f431a0..334106f0cf 100644 --- a/libs/ui/include/ui/FenceTime.h +++ b/libs/ui/include/ui/FenceTime.h @@ -142,6 +142,8 @@ private: std::atomic<nsecs_t> mSignalTime{Fence::SIGNAL_TIME_INVALID}; }; +using FenceTimePtr = std::shared_ptr<FenceTime>; + // A queue of FenceTimes that are expected to signal in FIFO order. // Only maintains a queue of weak pointers so it doesn't keep references // to Fences on its own. @@ -190,8 +192,15 @@ private: // before the new one is added. class FenceToFenceTimeMap { public: - // Create a new FenceTime with that wraps the provided Fence. - std::shared_ptr<FenceTime> createFenceTimeForTest(const sp<Fence>& fence); + using FencePair = std::pair<sp<Fence>, FenceTimePtr>; + + FencePair makePendingFenceForTest() { + const auto fence = sp<Fence>::make(); + return {fence, createFenceTimeForTest(fence)}; + } + + // Create a new FenceTime that wraps the provided Fence. + FenceTimePtr createFenceTimeForTest(const sp<Fence>&); // Signals all FenceTimes created through this class that are wrappers // around |fence|. @@ -205,7 +214,6 @@ private: std::unordered_map<Fence*, std::vector<std::weak_ptr<FenceTime>>> mMap; }; - -}; // namespace android +} // namespace android #endif // ANDROID_FENCE_TIME_H diff --git a/libs/ui/include_types/ui/HdrRenderTypeUtils.h b/libs/ui/include_types/ui/HdrRenderTypeUtils.h new file mode 100644 index 0000000000..b0af878cdb --- /dev/null +++ b/libs/ui/include_types/ui/HdrRenderTypeUtils.h @@ -0,0 +1,64 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <ui/GraphicTypes.h> + +namespace android { + +enum class HdrRenderType { + SDR, // just render to SDR + DISPLAY_HDR, // HDR by extended brightness + GENERIC_HDR // tonemapped HDR +}; + +/*** + * A helper function to classify how we treat the result based on params. + * + * @param dataspace the dataspace + * @param pixelFormat optional, in case there is no source buffer. + * @param hdrSdrRatio default is 1.f, render engine side doesn't take care of it. + * @return HdrRenderType + */ +inline HdrRenderType getHdrRenderType(ui::Dataspace dataspace, + std::optional<ui::PixelFormat> pixelFormat, + float hdrSdrRatio = 1.f) { + const auto transfer = dataspace & HAL_DATASPACE_TRANSFER_MASK; + const auto range = dataspace & HAL_DATASPACE_RANGE_MASK; + + if (transfer == HAL_DATASPACE_TRANSFER_ST2084 || transfer == HAL_DATASPACE_TRANSFER_HLG) { + return HdrRenderType::GENERIC_HDR; + } + + static const auto BT2020_LINEAR_EXT = static_cast<ui::Dataspace>(HAL_DATASPACE_STANDARD_BT2020 | + HAL_DATASPACE_TRANSFER_LINEAR | + HAL_DATASPACE_RANGE_EXTENDED); + + if ((dataspace == BT2020_LINEAR_EXT || dataspace == ui::Dataspace::V0_SCRGB) && + pixelFormat.has_value() && pixelFormat.value() == ui::PixelFormat::RGBA_FP16) { + return HdrRenderType::GENERIC_HDR; + } + + // Extended range layer with an hdr/sdr ratio of > 1.01f can "self-promote" to HDR. + if (range == HAL_DATASPACE_RANGE_EXTENDED && hdrSdrRatio > 1.01f) { + return HdrRenderType::DISPLAY_HDR; + } + + return HdrRenderType::SDR; +} + +} // namespace android
\ No newline at end of file diff --git a/libs/ui/tests/Android.bp b/libs/ui/tests/Android.bp index 831b64d877..8ce017d7a3 100644 --- a/libs/ui/tests/Android.bp +++ b/libs/ui/tests/Android.bp @@ -164,9 +164,9 @@ cc_test { } cc_test { - name: "DataspaceUtils_test", + name: "HdrRenderTypeUtils_test", shared_libs: ["libui"], - srcs: ["DataspaceUtils_test.cpp"], + srcs: ["HdrRenderTypeUtils_test.cpp"], cflags: [ "-Wall", "-Werror", diff --git a/libs/ui/tests/DataspaceUtils_test.cpp b/libs/ui/tests/DataspaceUtils_test.cpp deleted file mode 100644 index 3e0967182b..0000000000 --- a/libs/ui/tests/DataspaceUtils_test.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef LOG_TAG -#define LOG_TAG "DataspaceUtilsTest" - -#include <gtest/gtest.h> -#include <ui/DataspaceUtils.h> - -namespace android { - -class DataspaceUtilsTest : public testing::Test {}; - -TEST_F(DataspaceUtilsTest, isHdrDataspace) { - EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_ITU_HLG)); - EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_ITU_PQ)); - EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_PQ)); - EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_HLG)); - - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SRGB_LINEAR)); - // scRGB defines a very wide gamut but not an expanded luminance range - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SCRGB_LINEAR)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SRGB)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SCRGB)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_JFIF)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT601_625)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT601_525)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT709)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DCI_P3_LINEAR)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DCI_P3)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_P3_LINEAR)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_P3)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::ADOBE_RGB)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020_LINEAR)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020_ITU)); - EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_BT2020)); -} - -} // namespace android diff --git a/libs/ui/tests/HdrRenderTypeUtils_test.cpp b/libs/ui/tests/HdrRenderTypeUtils_test.cpp new file mode 100644 index 0000000000..efe819db76 --- /dev/null +++ b/libs/ui/tests/HdrRenderTypeUtils_test.cpp @@ -0,0 +1,65 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#undef LOG_TAG +#define LOG_TAG "HdrRenderTypeUtilsTest" + +#include <gtest/gtest.h> +#include <ui/HdrRenderTypeUtils.h> + +namespace android { + +class HdrRenderTypeUtilsTest : public testing::Test {}; + +TEST_F(HdrRenderTypeUtilsTest, getHdrRenderType) { + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_ITU_HLG, std::nullopt), + HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_ITU_PQ, std::nullopt), + HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_PQ, std::nullopt), HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_HLG, std::nullopt), + HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB, + std::optional<ui::PixelFormat>(ui::PixelFormat::RGBA_FP16)), + HdrRenderType::GENERIC_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB, + std::optional<ui::PixelFormat>(ui::PixelFormat::RGBA_8888), 2.f), + HdrRenderType::DISPLAY_HDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB_LINEAR, + std::optional<ui::PixelFormat>(ui::PixelFormat::RGBA_8888), 2.f), + HdrRenderType::DISPLAY_HDR); + + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SRGB_LINEAR, std::nullopt), HdrRenderType::SDR); + // scRGB defines a very wide gamut but not an expanded luminance range + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB_LINEAR, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SCRGB, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_SRGB, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_JFIF, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_BT601_625, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_BT601_525, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::V0_BT709, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DCI_P3_LINEAR, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DCI_P3, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DISPLAY_P3_LINEAR, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DISPLAY_P3, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::ADOBE_RGB, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_LINEAR, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::BT2020_ITU, std::nullopt), HdrRenderType::SDR); + EXPECT_EQ(getHdrRenderType(ui::Dataspace::DISPLAY_BT2020, std::nullopt), HdrRenderType::SDR); +} + +} // namespace android diff --git a/libs/ultrahdr/include/ultrahdr/jpegr.h b/libs/ultrahdr/include/ultrahdr/jpegr.h index a35fd30634..f80496a758 100644 --- a/libs/ultrahdr/include/ultrahdr/jpegr.h +++ b/libs/ultrahdr/include/ultrahdr/jpegr.h @@ -348,16 +348,6 @@ private: status_t extractPrimaryImageAndGainMap(jr_compressed_ptr compressed_jpegr_image, jr_compressed_ptr primary_image, jr_compressed_ptr gain_map); - /* - * This method is called in the decoding pipeline. It will read XMP metadata to find the start - * position of the compressed gain map, and will extract the compressed gain map. - * - * @param compressed_jpegr_image compressed JPEGR image - * @param dest destination of compressed gain map - * @return NO_ERROR if calculation succeeds, error code if error occurs. - */ - status_t extractGainMap(jr_compressed_ptr compressed_jpegr_image, - jr_compressed_ptr dest); /* * This method is called in the encoding pipeline. It will take the standard 8-bit JPEG image, diff --git a/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h b/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h index 064123210f..5420e1c9cf 100644 --- a/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h +++ b/libs/ultrahdr/include/ultrahdr/jpegrerrorcode.h @@ -44,6 +44,7 @@ enum { ERROR_JPEGR_INVALID_TRANS_FUNC = JPEGR_IO_ERROR_BASE - 6, ERROR_JPEGR_INVALID_METADATA = JPEGR_IO_ERROR_BASE - 7, ERROR_JPEGR_UNSUPPORTED_METADATA = JPEGR_IO_ERROR_BASE - 8, + ERROR_JPEGR_GAIN_MAP_IMAGE_NOT_FOUND = JPEGR_IO_ERROR_BASE - 9, JPEGR_RUNTIME_ERROR_BASE = -20000, ERROR_JPEGR_ENCODE_ERROR = JPEGR_RUNTIME_ERROR_BASE - 1, diff --git a/libs/ultrahdr/jpegr.cpp b/libs/ultrahdr/jpegr.cpp index 9c57f34c2a..fb24c9d206 100644 --- a/libs/ultrahdr/jpegr.cpp +++ b/libs/ultrahdr/jpegr.cpp @@ -539,9 +539,12 @@ status_t JpegR::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, jr_info_p return ERROR_JPEGR_INVALID_NULL_PTR; } - jpegr_compressed_struct primary_image, gain_map; - JPEGR_CHECK(extractPrimaryImageAndGainMap(compressed_jpegr_image, - &primary_image, &gain_map)); + jpegr_compressed_struct primary_image, gainmap_image; + status_t status = + extractPrimaryImageAndGainMap(compressed_jpegr_image, &primary_image, &gainmap_image); + if (status != NO_ERROR && status != ERROR_JPEGR_GAIN_MAP_IMAGE_NOT_FOUND) { + return status; + } JpegDecoderHelper jpeg_decoder; if (!jpeg_decoder.getCompressedImageParameters(primary_image.data, primary_image.length, @@ -550,7 +553,7 @@ status_t JpegR::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image, jr_info_p return ERROR_JPEGR_DECODE_ERROR; } - return NO_ERROR; + return status; } /* Decode API */ @@ -586,45 +589,56 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, return ERROR_JPEGR_INVALID_INPUT_TYPE; } + jpegr_compressed_struct primary_image, gainmap_image; + status_t status = + extractPrimaryImageAndGainMap(compressed_jpegr_image, &primary_image, &gainmap_image); + if (status != NO_ERROR) { + if (output_format != ULTRAHDR_OUTPUT_SDR || status != ERROR_JPEGR_GAIN_MAP_IMAGE_NOT_FOUND) { + ALOGE("received invalid compressed jpegr image"); + return status; + } + } + + JpegDecoderHelper jpeg_decoder; + if (!jpeg_decoder.decompressImage(primary_image.data, primary_image.length, + (output_format == ULTRAHDR_OUTPUT_SDR))) { + return ERROR_JPEGR_DECODE_ERROR; + } + if (output_format == ULTRAHDR_OUTPUT_SDR) { - JpegDecoderHelper jpeg_decoder; - if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length, - true)) { - return ERROR_JPEGR_DECODE_ERROR; + if ((jpeg_decoder.getDecompressedImageWidth() * + jpeg_decoder.getDecompressedImageHeight() * 4) > + jpeg_decoder.getDecompressedImageSize()) { + return ERROR_JPEGR_CALCULATION_ERROR; } - jpegr_uncompressed_struct uncompressed_rgba_image; - uncompressed_rgba_image.data = jpeg_decoder.getDecompressedImagePtr(); - uncompressed_rgba_image.width = jpeg_decoder.getDecompressedImageWidth(); - uncompressed_rgba_image.height = jpeg_decoder.getDecompressedImageHeight(); - memcpy(dest->data, uncompressed_rgba_image.data, - uncompressed_rgba_image.width * uncompressed_rgba_image.height * 4); - dest->width = uncompressed_rgba_image.width; - dest->height = uncompressed_rgba_image.height; - - if (gain_map == nullptr && exif == nullptr) { - return NO_ERROR; + } else { + if ((jpeg_decoder.getDecompressedImageWidth() * + jpeg_decoder.getDecompressedImageHeight() * 3 / 2) > + jpeg_decoder.getDecompressedImageSize()) { + return ERROR_JPEGR_CALCULATION_ERROR; } + } - if (exif != nullptr) { - if (exif->data == nullptr) { - return ERROR_JPEGR_INVALID_NULL_PTR; - } - if (exif->length < jpeg_decoder.getEXIFSize()) { - return ERROR_JPEGR_BUFFER_TOO_SMALL; - } - memcpy(exif->data, jpeg_decoder.getEXIFPtr(), jpeg_decoder.getEXIFSize()); - exif->length = jpeg_decoder.getEXIFSize(); + if (exif != nullptr) { + if (exif->data == nullptr) { + return ERROR_JPEGR_INVALID_NULL_PTR; } - if (gain_map == nullptr) { - return NO_ERROR; + if (exif->length < jpeg_decoder.getEXIFSize()) { + return ERROR_JPEGR_BUFFER_TOO_SMALL; } + memcpy(exif->data, jpeg_decoder.getEXIFPtr(), jpeg_decoder.getEXIFSize()); + exif->length = jpeg_decoder.getEXIFSize(); } - jpegr_compressed_struct compressed_map; - JPEGR_CHECK(extractGainMap(compressed_jpegr_image, &compressed_map)); + if (output_format == ULTRAHDR_OUTPUT_SDR) { + dest->width = jpeg_decoder.getDecompressedImageWidth(); + dest->height = jpeg_decoder.getDecompressedImageHeight(); + memcpy(dest->data, jpeg_decoder.getDecompressedImagePtr(), dest->width * dest->height * 4); + return NO_ERROR; + } JpegDecoderHelper gain_map_decoder; - if (!gain_map_decoder.decompressImage(compressed_map.data, compressed_map.length)) { + if (!gain_map_decoder.decompressImage(gainmap_image.data, gainmap_image.length)) { return ERROR_JPEGR_DECODE_ERROR; } if ((gain_map_decoder.getDecompressedImageWidth() * @@ -633,12 +647,17 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, return ERROR_JPEGR_CALCULATION_ERROR; } + jpegr_uncompressed_struct map; + map.data = gain_map_decoder.getDecompressedImagePtr(); + map.width = gain_map_decoder.getDecompressedImageWidth(); + map.height = gain_map_decoder.getDecompressedImageHeight(); + if (gain_map != nullptr) { - gain_map->width = gain_map_decoder.getDecompressedImageWidth(); - gain_map->height = gain_map_decoder.getDecompressedImageHeight(); + gain_map->width = map.width; + gain_map->height = map.height; int size = gain_map->width * gain_map->height; gain_map->data = malloc(size); - memcpy(gain_map->data, gain_map_decoder.getDecompressedImagePtr(), size); + memcpy(gain_map->data, map.data, size); } ultrahdr_metadata_struct uhdr_metadata; @@ -648,46 +667,16 @@ status_t JpegR::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image, } if (metadata != nullptr) { - metadata->version = uhdr_metadata.version; - metadata->minContentBoost = uhdr_metadata.minContentBoost; - metadata->maxContentBoost = uhdr_metadata.maxContentBoost; - metadata->gamma = uhdr_metadata.gamma; - metadata->offsetSdr = uhdr_metadata.offsetSdr; - metadata->offsetHdr = uhdr_metadata.offsetHdr; - metadata->hdrCapacityMin = uhdr_metadata.hdrCapacityMin; - metadata->hdrCapacityMax = uhdr_metadata.hdrCapacityMax; - } - - if (output_format == ULTRAHDR_OUTPUT_SDR) { - return NO_ERROR; + metadata->version = uhdr_metadata.version; + metadata->minContentBoost = uhdr_metadata.minContentBoost; + metadata->maxContentBoost = uhdr_metadata.maxContentBoost; + metadata->gamma = uhdr_metadata.gamma; + metadata->offsetSdr = uhdr_metadata.offsetSdr; + metadata->offsetHdr = uhdr_metadata.offsetHdr; + metadata->hdrCapacityMin = uhdr_metadata.hdrCapacityMin; + metadata->hdrCapacityMax = uhdr_metadata.hdrCapacityMax; } - JpegDecoderHelper jpeg_decoder; - if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length)) { - return ERROR_JPEGR_DECODE_ERROR; - } - if ((jpeg_decoder.getDecompressedImageWidth() * - jpeg_decoder.getDecompressedImageHeight() * 3 / 2) > - jpeg_decoder.getDecompressedImageSize()) { - return ERROR_JPEGR_CALCULATION_ERROR; - } - - if (exif != nullptr) { - if (exif->data == nullptr) { - return ERROR_JPEGR_INVALID_NULL_PTR; - } - if (exif->length < jpeg_decoder.getEXIFSize()) { - return ERROR_JPEGR_BUFFER_TOO_SMALL; - } - memcpy(exif->data, jpeg_decoder.getEXIFPtr(), jpeg_decoder.getEXIFSize()); - exif->length = jpeg_decoder.getEXIFSize(); - } - - jpegr_uncompressed_struct map; - map.data = gain_map_decoder.getDecompressedImagePtr(); - map.width = gain_map_decoder.getDecompressedImageWidth(); - map.height = gain_map_decoder.getDecompressedImageHeight(); - jpegr_uncompressed_struct uncompressed_yuv_420_image; uncompressed_yuv_420_image.data = jpeg_decoder.getDecompressedImagePtr(); uncompressed_yuv_420_image.width = jpeg_decoder.getDecompressedImageWidth(); @@ -1131,12 +1120,8 @@ status_t JpegR::extractPrimaryImageAndGainMap(jr_compressed_ptr compressed_jpegr const auto& jpeg_info = jpeg_info_builder.GetInfo(); const auto& image_ranges = jpeg_info.GetImageRanges(); - if (image_ranges.empty()) { - return ERROR_JPEGR_INVALID_INPUT_TYPE; - } - if (image_ranges.size() != 2) { - // Must be 2 JPEG Images + if (image_ranges.empty()) { return ERROR_JPEGR_INVALID_INPUT_TYPE; } @@ -1146,23 +1131,23 @@ status_t JpegR::extractPrimaryImageAndGainMap(jr_compressed_ptr compressed_jpegr primary_image->length = image_ranges[0].GetLength(); } + if (image_ranges.size() == 1) { + return ERROR_JPEGR_GAIN_MAP_IMAGE_NOT_FOUND; + } + if (gain_map != nullptr) { gain_map->data = static_cast<uint8_t*>(compressed_jpegr_image->data) + image_ranges[1].GetBegin(); gain_map->length = image_ranges[1].GetLength(); } - return NO_ERROR; -} - - -status_t JpegR::extractGainMap(jr_compressed_ptr compressed_jpegr_image, - jr_compressed_ptr dest) { - if (compressed_jpegr_image == nullptr || dest == nullptr) { - return ERROR_JPEGR_INVALID_NULL_PTR; + // TODO: choose primary image and gain map image carefully + if (image_ranges.size() > 2) { + ALOGW("Number of jpeg images present %d, primary, gain map images may not be correctly chosen", + (int)image_ranges.size()); } - return extractPrimaryImageAndGainMap(compressed_jpegr_image, nullptr, dest); + return NO_ERROR; } // JPEG/R structure: diff --git a/services/gpuservice/GpuService.cpp b/services/gpuservice/GpuService.cpp index 48d793a4d4..7fb06a2e30 100644 --- a/services/gpuservice/GpuService.cpp +++ b/services/gpuservice/GpuService.cpp @@ -67,6 +67,7 @@ GpuService::GpuService() mGpuWorkAsyncInitThread = std::make_unique<std::thread>([this]() { mGpuWork->initialize(); }); + property_set("persist.graphics.egl", ""); }; GpuService::~GpuService() { diff --git a/services/inputflinger/Android.bp b/services/inputflinger/Android.bp index 69df45bc3e..dc7c75a775 100644 --- a/services/inputflinger/Android.bp +++ b/services/inputflinger/Android.bp @@ -60,6 +60,7 @@ filegroup { name: "libinputflinger_sources", srcs: [ "InputCommonConverter.cpp", + "InputDeviceMetricsCollector.cpp", "InputProcessor.cpp", "PreferStylusOverTouchBlocker.cpp", "UnwantedInteractionBlocker.cpp", @@ -105,6 +106,13 @@ cc_defaults { "libstatspull", "libstatssocket", ], + include_dirs: [ + "bionic/libc/kernel/android/uapi/", + "bionic/libc/kernel/uapi", + ], + cflags: [ + "-D__ANDROID_HOST__", + ], }, }, } @@ -129,6 +137,7 @@ cc_library_shared { "libinputflinger_base", "libinputreader", "libinputreporter", + "libPlatformProperties", ], static_libs: [ "libinputdispatcher", @@ -228,6 +237,9 @@ phony { "inputflinger", "libinputflingerhost", + // rust targets + "libinput_rust_test", + // native fuzzers "inputflinger_latencytracker_fuzzer", "inputflinger_cursor_input_fuzzer", diff --git a/services/inputflinger/BlockingQueue.h b/services/inputflinger/BlockingQueue.h index c435bd2ebe..f848c82c42 100644 --- a/services/inputflinger/BlockingQueue.h +++ b/services/inputflinger/BlockingQueue.h @@ -18,14 +18,16 @@ #include <condition_variable> #include <functional> +#include <list> #include <mutex> -#include <vector> +#include <optional> #include "android-base/thread_annotations.h" namespace android { /** - * A FIFO queue that stores up to <i>capacity</i> objects. + * A thread-safe FIFO queue. This list-backed queue stores up to <i>capacity</i> objects if + * a capacity is provided at construction, and is otherwise unbounded. * Objects can always be added. Objects are added immediately. * If the queue is full, new objects cannot be added. * @@ -34,13 +36,13 @@ namespace android { template <class T> class BlockingQueue { public: - BlockingQueue(size_t capacity) : mCapacity(capacity) { - mQueue.reserve(mCapacity); - }; + explicit BlockingQueue() = default; + + explicit BlockingQueue(size_t capacity) : mCapacity(capacity){}; /** * Retrieve and remove the oldest object. - * Blocks execution while queue is empty. + * Blocks execution indefinitely while queue is empty. */ T pop() { std::unique_lock lock(mLock); @@ -52,26 +54,62 @@ public: }; /** + * Retrieve and remove the oldest object. + * Blocks execution for the given duration while queue is empty, and returns std::nullopt + * if the queue was empty for the entire duration. + */ + std::optional<T> popWithTimeout(std::chrono::nanoseconds duration) { + std::unique_lock lock(mLock); + android::base::ScopedLockAssertion assumeLock(mLock); + if (!mHasElements.wait_for(lock, duration, + [this]() REQUIRES(mLock) { return !this->mQueue.empty(); })) { + return {}; + } + T t = std::move(mQueue.front()); + mQueue.erase(mQueue.begin()); + return t; + }; + + /** * Add a new object to the queue. * Does not block. * Return true if an element was successfully added. * Return false if the queue is full. */ bool push(T&& t) { - { + { // acquire lock std::scoped_lock lock(mLock); - if (mQueue.size() == mCapacity) { + if (mCapacity && mQueue.size() == mCapacity) { return false; } mQueue.push_back(std::move(t)); - } + } // release lock + mHasElements.notify_one(); + return true; + }; + + /** + * Construct a new object into the queue. + * Does not block. + * Return true if an element was successfully added. + * Return false if the queue is full. + */ + template <class... Args> + bool emplace(Args&&... args) { + { // acquire lock + std::scoped_lock lock(mLock); + if (mCapacity && mQueue.size() == mCapacity) { + return false; + } + mQueue.emplace_back(args...); + } // release lock mHasElements.notify_one(); return true; }; - void erase(const std::function<bool(const T&)>& lambda) { + void erase_if(const std::function<bool(const T&)>& pred) { std::scoped_lock lock(mLock); - std::erase_if(mQueue, [&lambda](const auto& t) { return lambda(t); }); + std::erase_if(mQueue, pred); } /** @@ -94,7 +132,7 @@ public: } private: - const size_t mCapacity; + const std::optional<size_t> mCapacity; /** * Used to signal that mQueue is non-empty. */ @@ -103,7 +141,7 @@ private: * Lock for accessing and waiting on elements. */ std::mutex mLock; - std::vector<T> mQueue GUARDED_BY(mLock); + std::list<T> mQueue GUARDED_BY(mLock); }; } // namespace android diff --git a/services/inputflinger/InputCommonConverter.cpp b/services/inputflinger/InputCommonConverter.cpp index 2437d0fcfc..6ccd9e7697 100644 --- a/services/inputflinger/InputCommonConverter.cpp +++ b/services/inputflinger/InputCommonConverter.cpp @@ -258,12 +258,12 @@ static_assert(static_cast<common::Axis>(AMOTION_EVENT_AXIS_GENERIC_13) == common static_assert(static_cast<common::Axis>(AMOTION_EVENT_AXIS_GENERIC_14) == common::Axis::GENERIC_14); static_assert(static_cast<common::Axis>(AMOTION_EVENT_AXIS_GENERIC_15) == common::Axis::GENERIC_15); static_assert(static_cast<common::Axis>(AMOTION_EVENT_AXIS_GENERIC_16) == common::Axis::GENERIC_16); -// TODO(b/251196347): add GESTURE_{X,Y}_OFFSET, GESTURE_SCROLL_{X,Y}_DISTANCE, and -// GESTURE_PINCH_SCALE_FACTOR. +// TODO(b/251196347): add GESTURE_{X,Y}_OFFSET, GESTURE_SCROLL_{X,Y}_DISTANCE, +// GESTURE_PINCH_SCALE_FACTOR, and GESTURE_SWIPE_FINGER_COUNT. // If you added a new axis, consider whether this should also be exposed as a HAL axis. Update the // static_assert below and add the new axis here, or leave a comment summarizing your decision. static_assert(static_cast<common::Axis>(AMOTION_EVENT_MAXIMUM_VALID_AXIS_VALUE) == - static_cast<common::Axis>(AMOTION_EVENT_AXIS_GESTURE_PINCH_SCALE_FACTOR)); + static_cast<common::Axis>(AMOTION_EVENT_AXIS_GESTURE_SWIPE_FINGER_COUNT)); static common::VideoFrame getHalVideoFrame(const TouchVideoFrame& frame) { common::VideoFrame out; @@ -289,9 +289,9 @@ static std::vector<common::VideoFrame> convertVideoFrames( static void getHalPropertiesAndCoords(const NotifyMotionArgs& args, std::vector<common::PointerProperties>& outPointerProperties, std::vector<common::PointerCoords>& outPointerCoords) { - outPointerProperties.reserve(args.pointerCount); - outPointerCoords.reserve(args.pointerCount); - for (size_t i = 0; i < args.pointerCount; i++) { + outPointerProperties.reserve(args.getPointerCount()); + outPointerCoords.reserve(args.getPointerCount()); + for (size_t i = 0; i < args.getPointerCount(); i++) { common::PointerProperties properties; properties.id = args.pointerProperties[i].id; properties.toolType = getToolType(args.pointerProperties[i].toolType); diff --git a/services/inputflinger/InputDeviceMetricsCollector.cpp b/services/inputflinger/InputDeviceMetricsCollector.cpp new file mode 100644 index 0000000000..56a3fb44b4 --- /dev/null +++ b/services/inputflinger/InputDeviceMetricsCollector.cpp @@ -0,0 +1,466 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "InputDeviceMetricsCollector" +#include "InputDeviceMetricsCollector.h" + +#include "KeyCodeClassifications.h" + +#include <android-base/stringprintf.h> +#include <input/PrintTools.h> +#include <linux/input.h> + +namespace android { + +using android::base::StringPrintf; +using std::chrono::nanoseconds; +using std::chrono_literals::operator""ns; + +namespace { + +constexpr nanoseconds DEFAULT_USAGE_SESSION_TIMEOUT = std::chrono::minutes(2); + +/** + * Log debug messages about metrics events logged to statsd. + * Enable this via "adb shell setprop log.tag.InputDeviceMetricsCollector DEBUG" (requires restart) + */ +const bool DEBUG = __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG, ANDROID_LOG_INFO); + +constexpr size_t INTERACTIONS_QUEUE_CAPACITY = 500; + +int32_t linuxBusToInputDeviceBusEnum(int32_t linuxBus, bool isUsiStylus) { + if (isUsiStylus) { + // This is a stylus connected over the Universal Stylus Initiative (USI) protocol. + // For metrics purposes, we treat this protocol as a separate bus. + return util::INPUT_DEVICE_USAGE_REPORTED__DEVICE_BUS__USI; + } + + // When adding cases to this switch, also add them to the copy of this method in + // TouchpadInputMapper.cpp. + // TODO(b/286394420): deduplicate this method with the one in TouchpadInputMapper.cpp. + switch (linuxBus) { + case BUS_USB: + return util::INPUT_DEVICE_USAGE_REPORTED__DEVICE_BUS__USB; + case BUS_BLUETOOTH: + return util::INPUT_DEVICE_USAGE_REPORTED__DEVICE_BUS__BLUETOOTH; + default: + return util::INPUT_DEVICE_USAGE_REPORTED__DEVICE_BUS__OTHER; + } +} + +class : public InputDeviceMetricsLogger { + nanoseconds getCurrentTime() override { return nanoseconds(systemTime(SYSTEM_TIME_MONOTONIC)); } + + void logInputDeviceUsageReported(const InputDeviceInfo& info, + const DeviceUsageReport& report) override { + const int32_t durationMillis = + std::chrono::duration_cast<std::chrono::milliseconds>(report.usageDuration).count(); + const static std::vector<int32_t> empty; + const auto& identifier = info.getIdentifier(); + + ALOGD_IF(DEBUG, "Usage session reported for device: %s", identifier.name.c_str()); + ALOGD_IF(DEBUG, " Total duration: %dms", durationMillis); + ALOGD_IF(DEBUG, " Source breakdown:"); + + std::vector<int32_t> sources; + std::vector<int32_t> durationsPerSource; + for (auto& [src, dur] : report.sourceBreakdown) { + sources.push_back(ftl::to_underlying(src)); + int32_t durMillis = std::chrono::duration_cast<std::chrono::milliseconds>(dur).count(); + durationsPerSource.emplace_back(durMillis); + ALOGD_IF(DEBUG, " - usageSource: %s\t duration: %dms", + ftl::enum_string(src).c_str(), durMillis); + } + + ALOGD_IF(DEBUG, " Uid breakdown:"); + + std::vector<int32_t> uids; + std::vector<int32_t> durationsPerUid; + for (auto& [uid, dur] : report.uidBreakdown) { + uids.push_back(uid.val()); + int32_t durMillis = std::chrono::duration_cast<std::chrono::milliseconds>(dur).count(); + durationsPerUid.push_back(durMillis); + ALOGD_IF(DEBUG, " - uid: %s\t duration: %dms", uid.toString().c_str(), + durMillis); + } + util::stats_write(util::INPUTDEVICE_USAGE_REPORTED, identifier.vendor, identifier.product, + identifier.version, + linuxBusToInputDeviceBusEnum(identifier.bus, + info.getUsiVersion().has_value()), + durationMillis, sources, durationsPerSource, uids, durationsPerUid); + } +} sStatsdLogger; + +bool isIgnoredInputDeviceId(int32_t deviceId) { + switch (deviceId) { + case INVALID_INPUT_DEVICE_ID: + case VIRTUAL_KEYBOARD_ID: + return true; + default: + return false; + } +} + +} // namespace + +InputDeviceUsageSource getUsageSourceForKeyArgs(const InputDeviceInfo& info, + const NotifyKeyArgs& keyArgs) { + if (!isFromSource(keyArgs.source, AINPUT_SOURCE_KEYBOARD)) { + return InputDeviceUsageSource::UNKNOWN; + } + + if (isFromSource(keyArgs.source, AINPUT_SOURCE_DPAD) && + DPAD_ALL_KEYCODES.count(keyArgs.keyCode) != 0) { + return InputDeviceUsageSource::DPAD; + } + + if (isFromSource(keyArgs.source, AINPUT_SOURCE_GAMEPAD) && + GAMEPAD_KEYCODES.count(keyArgs.keyCode) != 0) { + return InputDeviceUsageSource::GAMEPAD; + } + + if (info.getKeyboardType() == AINPUT_KEYBOARD_TYPE_ALPHABETIC) { + return InputDeviceUsageSource::KEYBOARD; + } + + return InputDeviceUsageSource::BUTTONS; +} + +std::set<InputDeviceUsageSource> getUsageSourcesForMotionArgs(const NotifyMotionArgs& motionArgs) { + LOG_ALWAYS_FATAL_IF(motionArgs.getPointerCount() < 1, "Received motion args without pointers"); + std::set<InputDeviceUsageSource> sources; + + for (uint32_t i = 0; i < motionArgs.getPointerCount(); i++) { + const auto toolType = motionArgs.pointerProperties[i].toolType; + if (isFromSource(motionArgs.source, AINPUT_SOURCE_MOUSE)) { + if (toolType == ToolType::MOUSE) { + sources.emplace(InputDeviceUsageSource::MOUSE); + continue; + } + if (toolType == ToolType::FINGER) { + sources.emplace(InputDeviceUsageSource::TOUCHPAD); + continue; + } + if (isStylusToolType(toolType)) { + sources.emplace(InputDeviceUsageSource::STYLUS_INDIRECT); + continue; + } + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_MOUSE_RELATIVE) && + toolType == ToolType::MOUSE) { + sources.emplace(InputDeviceUsageSource::MOUSE_CAPTURED); + continue; + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_TOUCHPAD) && + toolType == ToolType::FINGER) { + sources.emplace(InputDeviceUsageSource::TOUCHPAD_CAPTURED); + continue; + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_BLUETOOTH_STYLUS) && + isStylusToolType(toolType)) { + sources.emplace(InputDeviceUsageSource::STYLUS_FUSED); + continue; + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_STYLUS) && isStylusToolType(toolType)) { + sources.emplace(InputDeviceUsageSource::STYLUS_DIRECT); + continue; + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_TOUCH_NAVIGATION)) { + sources.emplace(InputDeviceUsageSource::TOUCH_NAVIGATION); + continue; + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_JOYSTICK)) { + sources.emplace(InputDeviceUsageSource::JOYSTICK); + continue; + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_ROTARY_ENCODER)) { + sources.emplace(InputDeviceUsageSource::ROTARY_ENCODER); + continue; + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_TRACKBALL)) { + sources.emplace(InputDeviceUsageSource::TRACKBALL); + continue; + } + if (isFromSource(motionArgs.source, AINPUT_SOURCE_TOUCHSCREEN)) { + sources.emplace(InputDeviceUsageSource::TOUCHSCREEN); + continue; + } + sources.emplace(InputDeviceUsageSource::UNKNOWN); + } + + return sources; +} + +// --- InputDeviceMetricsCollector --- + +InputDeviceMetricsCollector::InputDeviceMetricsCollector(InputListenerInterface& listener) + : InputDeviceMetricsCollector(listener, sStatsdLogger, DEFAULT_USAGE_SESSION_TIMEOUT) {} + +InputDeviceMetricsCollector::InputDeviceMetricsCollector(InputListenerInterface& listener, + InputDeviceMetricsLogger& logger, + nanoseconds usageSessionTimeout) + : mNextListener(listener), + mLogger(logger), + mUsageSessionTimeout(usageSessionTimeout), + mInteractionsQueue(INTERACTIONS_QUEUE_CAPACITY) {} + +void InputDeviceMetricsCollector::notifyInputDevicesChanged( + const NotifyInputDevicesChangedArgs& args) { + reportCompletedSessions(); + onInputDevicesChanged(args.inputDeviceInfos); + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifyConfigurationChanged( + const NotifyConfigurationChangedArgs& args) { + reportCompletedSessions(); + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifyKey(const NotifyKeyArgs& args) { + reportCompletedSessions(); + const SourceProvider getSources = [&args](const InputDeviceInfo& info) { + return std::set{getUsageSourceForKeyArgs(info, args)}; + }; + onInputDeviceUsage(DeviceId{args.deviceId}, nanoseconds(args.eventTime), getSources); + + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifyMotion(const NotifyMotionArgs& args) { + reportCompletedSessions(); + onInputDeviceUsage(DeviceId{args.deviceId}, nanoseconds(args.eventTime), + [&args](const auto&) { return getUsageSourcesForMotionArgs(args); }); + + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifySwitch(const NotifySwitchArgs& args) { + reportCompletedSessions(); + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifySensor(const NotifySensorArgs& args) { + reportCompletedSessions(); + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifyVibratorState(const NotifyVibratorStateArgs& args) { + reportCompletedSessions(); + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifyDeviceReset(const NotifyDeviceResetArgs& args) { + reportCompletedSessions(); + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifyPointerCaptureChanged( + const NotifyPointerCaptureChangedArgs& args) { + reportCompletedSessions(); + mNextListener.notify(args); +} + +void InputDeviceMetricsCollector::notifyDeviceInteraction(int32_t deviceId, nsecs_t timestamp, + const std::set<Uid>& uids) { + if (isIgnoredInputDeviceId(deviceId)) { + return; + } + mInteractionsQueue.push(DeviceId{deviceId}, timestamp, uids); +} + +void InputDeviceMetricsCollector::dump(std::string& dump) { + dump += "InputDeviceMetricsCollector:\n"; + + dump += " Logged device IDs: " + dumpMapKeys(mLoggedDeviceInfos, &toString) + "\n"; + dump += " Devices with active usage sessions: " + + dumpMapKeys(mActiveUsageSessions, &toString) + "\n"; +} + +void InputDeviceMetricsCollector::onInputDevicesChanged(const std::vector<InputDeviceInfo>& infos) { + std::map<DeviceId, InputDeviceInfo> newDeviceInfos; + + for (const InputDeviceInfo& info : infos) { + if (isIgnoredInputDeviceId(info.getId())) { + continue; + } + newDeviceInfos.emplace(info.getId(), info); + } + + for (auto [deviceId, info] : mLoggedDeviceInfos) { + if (newDeviceInfos.count(deviceId) != 0) { + continue; + } + onInputDeviceRemoved(deviceId, info); + } + + std::swap(newDeviceInfos, mLoggedDeviceInfos); +} + +void InputDeviceMetricsCollector::onInputDeviceRemoved(DeviceId deviceId, + const InputDeviceInfo& info) { + auto it = mActiveUsageSessions.find(deviceId); + if (it == mActiveUsageSessions.end()) { + return; + } + // Report usage for that device if there is an active session. + auto& [_, activeSession] = *it; + mLogger.logInputDeviceUsageReported(info, activeSession.finishSession()); + mActiveUsageSessions.erase(it); + + // We don't remove this from mLoggedDeviceInfos because it will be updated in + // onInputDevicesChanged(). +} + +void InputDeviceMetricsCollector::onInputDeviceUsage(DeviceId deviceId, nanoseconds eventTime, + const SourceProvider& getSources) { + auto infoIt = mLoggedDeviceInfos.find(deviceId); + if (infoIt == mLoggedDeviceInfos.end()) { + // Do not track usage for devices that are not logged. + return; + } + + auto [sessionIt, _] = + mActiveUsageSessions.try_emplace(deviceId, mUsageSessionTimeout, eventTime); + for (InputDeviceUsageSource source : getSources(infoIt->second)) { + sessionIt->second.recordUsage(eventTime, source); + } +} + +void InputDeviceMetricsCollector::onInputDeviceInteraction(const Interaction& interaction) { + auto activeSessionIt = mActiveUsageSessions.find(std::get<DeviceId>(interaction)); + if (activeSessionIt == mActiveUsageSessions.end()) { + return; + } + + activeSessionIt->second.recordInteraction(interaction); +} + +void InputDeviceMetricsCollector::reportCompletedSessions() { + // Process all pending interactions. + for (auto interaction = mInteractionsQueue.pop(); interaction; + interaction = mInteractionsQueue.pop()) { + onInputDeviceInteraction(*interaction); + } + + const auto currentTime = mLogger.getCurrentTime(); + std::vector<DeviceId> completedUsageSessions; + + // Process usages for all active session to determine if any sessions have expired. + for (auto& [deviceId, activeSession] : mActiveUsageSessions) { + if (activeSession.checkIfCompletedAt(currentTime)) { + completedUsageSessions.emplace_back(deviceId); + } + } + + // Close out and log all expired usage sessions. + for (DeviceId deviceId : completedUsageSessions) { + const auto infoIt = mLoggedDeviceInfos.find(deviceId); + LOG_ALWAYS_FATAL_IF(infoIt == mLoggedDeviceInfos.end()); + + auto activeSessionIt = mActiveUsageSessions.find(deviceId); + LOG_ALWAYS_FATAL_IF(activeSessionIt == mActiveUsageSessions.end()); + auto& [_, activeSession] = *activeSessionIt; + mLogger.logInputDeviceUsageReported(infoIt->second, activeSession.finishSession()); + mActiveUsageSessions.erase(activeSessionIt); + } +} + +// --- InputDeviceMetricsCollector::ActiveSession --- + +InputDeviceMetricsCollector::ActiveSession::ActiveSession(nanoseconds usageSessionTimeout, + nanoseconds startTime) + : mUsageSessionTimeout(usageSessionTimeout), mDeviceSession({startTime, startTime}) {} + +void InputDeviceMetricsCollector::ActiveSession::recordUsage(nanoseconds eventTime, + InputDeviceUsageSource source) { + // We assume that event times for subsequent events are always monotonically increasing for each + // input device. + auto [activeSourceIt, inserted] = + mActiveSessionsBySource.try_emplace(source, eventTime, eventTime); + if (!inserted) { + activeSourceIt->second.end = eventTime; + } + mDeviceSession.end = eventTime; +} + +void InputDeviceMetricsCollector::ActiveSession::recordInteraction(const Interaction& interaction) { + const auto sessionExpiryTime = mDeviceSession.end + mUsageSessionTimeout; + const auto timestamp = std::get<nanoseconds>(interaction); + if (timestamp >= sessionExpiryTime) { + // This interaction occurred after the device's current active session is set to expire. + // Ignore it. + return; + } + + for (Uid uid : std::get<std::set<Uid>>(interaction)) { + auto [activeUidIt, inserted] = mActiveSessionsByUid.try_emplace(uid, timestamp, timestamp); + if (!inserted) { + activeUidIt->second.end = timestamp; + } + } +} + +bool InputDeviceMetricsCollector::ActiveSession::checkIfCompletedAt(nanoseconds timestamp) { + const auto sessionExpiryTime = timestamp - mUsageSessionTimeout; + std::vector<InputDeviceUsageSource> completedSourceSessionsForDevice; + for (auto& [source, session] : mActiveSessionsBySource) { + if (session.end <= sessionExpiryTime) { + completedSourceSessionsForDevice.emplace_back(source); + } + } + for (InputDeviceUsageSource source : completedSourceSessionsForDevice) { + auto it = mActiveSessionsBySource.find(source); + const auto& [_, session] = *it; + mSourceUsageBreakdown.emplace_back(source, session.end - session.start); + mActiveSessionsBySource.erase(it); + } + + std::vector<Uid> completedUidSessionsForDevice; + for (auto& [uid, session] : mActiveSessionsByUid) { + if (session.end <= sessionExpiryTime) { + completedUidSessionsForDevice.emplace_back(uid); + } + } + for (Uid uid : completedUidSessionsForDevice) { + auto it = mActiveSessionsByUid.find(uid); + const auto& [_, session] = *it; + mUidUsageBreakdown.emplace_back(uid, session.end - session.start); + mActiveSessionsByUid.erase(it); + } + + // This active session has expired if there are no more active source sessions tracked. + return mActiveSessionsBySource.empty(); +} + +InputDeviceMetricsLogger::DeviceUsageReport +InputDeviceMetricsCollector::ActiveSession::finishSession() { + const auto deviceUsageDuration = mDeviceSession.end - mDeviceSession.start; + + for (const auto& [source, sourceSession] : mActiveSessionsBySource) { + mSourceUsageBreakdown.emplace_back(source, sourceSession.end - sourceSession.start); + } + mActiveSessionsBySource.clear(); + + for (const auto& [uid, uidSession] : mActiveSessionsByUid) { + mUidUsageBreakdown.emplace_back(uid, uidSession.end - uidSession.start); + } + mActiveSessionsByUid.clear(); + + return {deviceUsageDuration, mSourceUsageBreakdown, mUidUsageBreakdown}; +} + +} // namespace android diff --git a/services/inputflinger/InputDeviceMetricsCollector.h b/services/inputflinger/InputDeviceMetricsCollector.h new file mode 100644 index 0000000000..1f7c5d9f80 --- /dev/null +++ b/services/inputflinger/InputDeviceMetricsCollector.h @@ -0,0 +1,199 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "InputListener.h" +#include "NotifyArgs.h" +#include "SyncQueue.h" + +#include <ftl/mixins.h> +#include <gui/WindowInfo.h> +#include <input/InputDevice.h> +#include <statslog.h> +#include <chrono> +#include <functional> +#include <map> +#include <set> +#include <vector> + +namespace android { + +/** + * Logs metrics about registered input devices and their usages. + * + * All methods in the InputListenerInterface must be called from a single thread. + */ +class InputDeviceMetricsCollectorInterface : public InputListenerInterface { +public: + /** + * Notify the metrics collector that there was an input device interaction with apps. + * Called from the InputDispatcher thread. + */ + virtual void notifyDeviceInteraction(int32_t deviceId, nsecs_t timestamp, + const std::set<gui::Uid>& uids) = 0; + /** + * Dump the state of the interaction blocker. + * This method may be called on any thread (usually by the input manager on a binder thread). + */ + virtual void dump(std::string& dump) = 0; +}; + +/** + * Enum representation of the InputDeviceUsageSource. + */ +enum class InputDeviceUsageSource : int32_t { + UNKNOWN = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__UNKNOWN, + BUTTONS = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__BUTTONS, + KEYBOARD = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__KEYBOARD, + DPAD = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__DPAD, + GAMEPAD = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__GAMEPAD, + JOYSTICK = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__JOYSTICK, + MOUSE = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__MOUSE, + MOUSE_CAPTURED = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__MOUSE_CAPTURED, + TOUCHPAD = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__TOUCHPAD, + TOUCHPAD_CAPTURED = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__TOUCHPAD_CAPTURED, + ROTARY_ENCODER = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__ROTARY_ENCODER, + STYLUS_DIRECT = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__STYLUS_DIRECT, + STYLUS_INDIRECT = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__STYLUS_INDIRECT, + STYLUS_FUSED = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__STYLUS_FUSED, + TOUCH_NAVIGATION = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__TOUCH_NAVIGATION, + TOUCHSCREEN = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__TOUCHSCREEN, + TRACKBALL = util::INPUT_DEVICE_USAGE_REPORTED__USAGE_SOURCES__TRACKBALL, + + ftl_first = UNKNOWN, + ftl_last = TRACKBALL, +}; + +/** Returns the InputDeviceUsageSource that corresponds to the key event. */ +InputDeviceUsageSource getUsageSourceForKeyArgs(const InputDeviceInfo&, const NotifyKeyArgs&); + +/** Returns the InputDeviceUsageSources that correspond to the motion event. */ +std::set<InputDeviceUsageSource> getUsageSourcesForMotionArgs(const NotifyMotionArgs&); + +/** The logging interface for the metrics collector, injected for testing. */ +class InputDeviceMetricsLogger { +public: + virtual std::chrono::nanoseconds getCurrentTime() = 0; + + // Describes the breakdown of an input device usage session by its usage sources. + // An input device can have more than one usage source. For example, some game controllers have + // buttons, joysticks, and touchpads. We track usage by these sources to get a better picture of + // the device usage. The source breakdown of a 10 minute usage session could look like this: + // { {GAMEPAD, <9 mins>}, {TOUCHPAD, <2 mins>}, {TOUCHPAD, <3 mins>} } + // This would indicate that the GAMEPAD source was used first, and that source usage session + // lasted for 9 mins. During that time, the TOUCHPAD was used for 2 mins, until its source + // usage session expired. The TOUCHPAD was then used again later for another 3 mins. + using SourceUsageBreakdown = + std::vector<std::pair<InputDeviceUsageSource, std::chrono::nanoseconds /*duration*/>>; + + // Describes the breakdown of an input device usage session by the UIDs that it interacted with. + using UidUsageBreakdown = + std::vector<std::pair<gui::Uid, std::chrono::nanoseconds /*duration*/>>; + + struct DeviceUsageReport { + std::chrono::nanoseconds usageDuration; + SourceUsageBreakdown sourceBreakdown; + UidUsageBreakdown uidBreakdown; + }; + + virtual void logInputDeviceUsageReported(const InputDeviceInfo&, const DeviceUsageReport&) = 0; + virtual ~InputDeviceMetricsLogger() = default; +}; + +class InputDeviceMetricsCollector : public InputDeviceMetricsCollectorInterface { +public: + explicit InputDeviceMetricsCollector(InputListenerInterface& listener); + ~InputDeviceMetricsCollector() override = default; + + // Test constructor + InputDeviceMetricsCollector(InputListenerInterface& listener, InputDeviceMetricsLogger& logger, + std::chrono::nanoseconds usageSessionTimeout); + + void notifyInputDevicesChanged(const NotifyInputDevicesChangedArgs& args) override; + void notifyConfigurationChanged(const NotifyConfigurationChangedArgs& args) override; + void notifyKey(const NotifyKeyArgs& args) override; + void notifyMotion(const NotifyMotionArgs& args) override; + void notifySwitch(const NotifySwitchArgs& args) override; + void notifySensor(const NotifySensorArgs& args) override; + void notifyVibratorState(const NotifyVibratorStateArgs& args) override; + void notifyDeviceReset(const NotifyDeviceResetArgs& args) override; + void notifyPointerCaptureChanged(const NotifyPointerCaptureChangedArgs& args) override; + + void notifyDeviceInteraction(int32_t deviceId, nsecs_t timestamp, + const std::set<gui::Uid>& uids) override; + void dump(std::string& dump) override; + +private: + InputListenerInterface& mNextListener; + InputDeviceMetricsLogger& mLogger; + const std::chrono::nanoseconds mUsageSessionTimeout; + + // Type-safe wrapper for input device id. + struct DeviceId : ftl::Constructible<DeviceId, std::int32_t>, + ftl::Equatable<DeviceId>, + ftl::Orderable<DeviceId> { + using Constructible::Constructible; + }; + static inline std::string toString(const DeviceId& id) { + return std::to_string(ftl::to_underlying(id)); + } + + using Uid = gui::Uid; + + std::map<DeviceId, InputDeviceInfo> mLoggedDeviceInfos; + + using Interaction = std::tuple<DeviceId, std::chrono::nanoseconds, std::set<Uid>>; + SyncQueue<Interaction> mInteractionsQueue; + + class ActiveSession { + public: + explicit ActiveSession(std::chrono::nanoseconds usageSessionTimeout, + std::chrono::nanoseconds startTime); + void recordUsage(std::chrono::nanoseconds eventTime, InputDeviceUsageSource source); + void recordInteraction(const Interaction&); + bool checkIfCompletedAt(std::chrono::nanoseconds timestamp); + InputDeviceMetricsLogger::DeviceUsageReport finishSession(); + + private: + struct UsageSession { + std::chrono::nanoseconds start{}; + std::chrono::nanoseconds end{}; + }; + + const std::chrono::nanoseconds mUsageSessionTimeout; + UsageSession mDeviceSession{}; + + std::map<InputDeviceUsageSource, UsageSession> mActiveSessionsBySource{}; + InputDeviceMetricsLogger::SourceUsageBreakdown mSourceUsageBreakdown{}; + + std::map<Uid, UsageSession> mActiveSessionsByUid{}; + InputDeviceMetricsLogger::UidUsageBreakdown mUidUsageBreakdown{}; + }; + + // The input devices that currently have active usage sessions. + std::map<DeviceId, ActiveSession> mActiveUsageSessions; + + void onInputDevicesChanged(const std::vector<InputDeviceInfo>& infos); + void onInputDeviceRemoved(DeviceId deviceId, const InputDeviceInfo& info); + using SourceProvider = std::function<std::set<InputDeviceUsageSource>(const InputDeviceInfo&)>; + void onInputDeviceUsage(DeviceId deviceId, std::chrono::nanoseconds eventTime, + const SourceProvider& getSources); + void onInputDeviceInteraction(const Interaction&); + void reportCompletedSessions(); +}; + +} // namespace android diff --git a/services/inputflinger/InputManager.cpp b/services/inputflinger/InputManager.cpp index ddebcad0d3..37b31875f8 100644 --- a/services/inputflinger/InputManager.cpp +++ b/services/inputflinger/InputManager.cpp @@ -23,6 +23,7 @@ #include "InputReaderFactory.h" #include "UnwantedInteractionBlocker.h" +#include <android/sysprop/InputProperties.sysprop.h> #include <binder/IPCThreadState.h> #include <log/log.h> @@ -32,6 +33,9 @@ namespace android { +static const bool ENABLE_INPUT_DEVICE_USAGE_METRICS = + sysprop::InputProperties::enable_input_device_usage_metrics().value_or(true); + using gui::FocusRequest; static int32_t exceptionCodeFromStatusT(status_t status) { @@ -55,12 +59,22 @@ static int32_t exceptionCodeFromStatusT(status_t status) { /** * The event flow is via the "InputListener" interface, as follows: - * InputReader -> UnwantedInteractionBlocker -> InputProcessor -> InputDispatcher + * InputReader + * -> UnwantedInteractionBlocker + * -> InputProcessor + * -> InputDeviceMetricsCollector + * -> InputDispatcher */ InputManager::InputManager(const sp<InputReaderPolicyInterface>& readerPolicy, InputDispatcherPolicyInterface& dispatcherPolicy) { mDispatcher = createInputDispatcher(dispatcherPolicy); - mProcessor = std::make_unique<InputProcessor>(*mDispatcher); + + if (ENABLE_INPUT_DEVICE_USAGE_METRICS) { + mCollector = std::make_unique<InputDeviceMetricsCollector>(*mDispatcher); + } + + mProcessor = ENABLE_INPUT_DEVICE_USAGE_METRICS ? std::make_unique<InputProcessor>(*mCollector) + : std::make_unique<InputProcessor>(*mDispatcher); mBlocker = std::make_unique<UnwantedInteractionBlocker>(*mProcessor); mReader = createInputReader(readerPolicy, *mBlocker); } @@ -113,6 +127,10 @@ InputProcessorInterface& InputManager::getProcessor() { return *mProcessor; } +InputDeviceMetricsCollectorInterface& InputManager::getMetricsCollector() { + return *mCollector; +} + InputDispatcherInterface& InputManager::getDispatcher() { return *mDispatcher; } @@ -131,6 +149,10 @@ void InputManager::dump(std::string& dump) { dump += '\n'; mProcessor->dump(dump); dump += '\n'; + if (ENABLE_INPUT_DEVICE_USAGE_METRICS) { + mCollector->dump(dump); + dump += '\n'; + } mDispatcher->dump(dump); dump += '\n'; } diff --git a/services/inputflinger/InputManager.h b/services/inputflinger/InputManager.h index b6ad419f31..9dc285f2c0 100644 --- a/services/inputflinger/InputManager.h +++ b/services/inputflinger/InputManager.h @@ -20,6 +20,7 @@ * Native input manager. */ +#include "InputDeviceMetricsCollector.h" #include "InputProcessor.h" #include "InputReaderBase.h" #include "include/UnwantedInteractionBlockerInterface.h" @@ -82,9 +83,12 @@ public: /* Gets the input reader. */ virtual InputReaderInterface& getReader() = 0; - /* Gets the input processor */ + /* Gets the input processor. */ virtual InputProcessorInterface& getProcessor() = 0; + /* Gets the metrics collector. */ + virtual InputDeviceMetricsCollectorInterface& getMetricsCollector() = 0; + /* Gets the input dispatcher. */ virtual InputDispatcherInterface& getDispatcher() = 0; @@ -108,6 +112,7 @@ public: InputReaderInterface& getReader() override; InputProcessorInterface& getProcessor() override; + InputDeviceMetricsCollectorInterface& getMetricsCollector() override; InputDispatcherInterface& getDispatcher() override; void monitor() override; void dump(std::string& dump) override; @@ -124,6 +129,8 @@ private: std::unique_ptr<InputProcessorInterface> mProcessor; + std::unique_ptr<InputDeviceMetricsCollectorInterface> mCollector; + std::unique_ptr<InputDispatcherInterface> mDispatcher; }; diff --git a/services/inputflinger/InputProcessor.cpp b/services/inputflinger/InputProcessor.cpp index 7a84be93b1..6dd267ce8f 100644 --- a/services/inputflinger/InputProcessor.cpp +++ b/services/inputflinger/InputProcessor.cpp @@ -322,7 +322,7 @@ void MotionClassifier::reset() { void MotionClassifier::reset(const NotifyDeviceResetArgs& args) { int32_t deviceId = args.deviceId; // Clear the pending events right away, to avoid unnecessary work done by the HAL. - mEvents.erase([deviceId](const ClassifierEvent& event) { + mEvents.erase_if([deviceId](const ClassifierEvent& event) { std::optional<int32_t> eventDeviceId = event.getDeviceId(); return eventDeviceId && (*eventDeviceId == deviceId); }); diff --git a/services/inputflinger/NotifyArgs.cpp b/services/inputflinger/NotifyArgs.cpp index 35d60eaaa2..6a25e72b5f 100644 --- a/services/inputflinger/NotifyArgs.cpp +++ b/services/inputflinger/NotifyArgs.cpp @@ -83,7 +83,6 @@ NotifyMotionArgs::NotifyMotionArgs( buttonState(buttonState), classification(classification), edgeFlags(edgeFlags), - pointerCount(pointerCount), xPrecision(xPrecision), yPrecision(yPrecision), xCursorPosition(xCursorPosition), @@ -92,36 +91,8 @@ NotifyMotionArgs::NotifyMotionArgs( readTime(readTime), videoFrames(videoFrames) { for (uint32_t i = 0; i < pointerCount; i++) { - this->pointerProperties[i].copyFrom(pointerProperties[i]); - this->pointerCoords[i].copyFrom(pointerCoords[i]); - } -} - -NotifyMotionArgs::NotifyMotionArgs(const NotifyMotionArgs& other) - : id(other.id), - eventTime(other.eventTime), - deviceId(other.deviceId), - source(other.source), - displayId(other.displayId), - policyFlags(other.policyFlags), - action(other.action), - actionButton(other.actionButton), - flags(other.flags), - metaState(other.metaState), - buttonState(other.buttonState), - classification(other.classification), - edgeFlags(other.edgeFlags), - pointerCount(other.pointerCount), - xPrecision(other.xPrecision), - yPrecision(other.yPrecision), - xCursorPosition(other.xCursorPosition), - yCursorPosition(other.yCursorPosition), - downTime(other.downTime), - readTime(other.readTime), - videoFrames(other.videoFrames) { - for (uint32_t i = 0; i < pointerCount; i++) { - pointerProperties[i].copyFrom(other.pointerProperties[i]); - pointerCoords[i].copyFrom(other.pointerCoords[i]); + this->pointerProperties.push_back(pointerProperties[i]); + this->pointerCoords.push_back(pointerCoords[i]); } } @@ -130,35 +101,22 @@ static inline bool isCursorPositionEqual(float lhs, float rhs) { } bool NotifyMotionArgs::operator==(const NotifyMotionArgs& rhs) const { - bool equal = id == rhs.id && eventTime == rhs.eventTime && readTime == rhs.readTime && + return id == rhs.id && eventTime == rhs.eventTime && readTime == rhs.readTime && deviceId == rhs.deviceId && source == rhs.source && displayId == rhs.displayId && policyFlags == rhs.policyFlags && action == rhs.action && actionButton == rhs.actionButton && flags == rhs.flags && metaState == rhs.metaState && buttonState == rhs.buttonState && classification == rhs.classification && - edgeFlags == rhs.edgeFlags && - pointerCount == rhs.pointerCount - // PointerProperties and PointerCoords are compared separately below - && xPrecision == rhs.xPrecision && yPrecision == rhs.yPrecision && + edgeFlags == rhs.edgeFlags && pointerProperties == rhs.pointerProperties && + pointerCoords == rhs.pointerCoords && xPrecision == rhs.xPrecision && + yPrecision == rhs.yPrecision && isCursorPositionEqual(xCursorPosition, rhs.xCursorPosition) && isCursorPositionEqual(yCursorPosition, rhs.yCursorPosition) && downTime == rhs.downTime && videoFrames == rhs.videoFrames; - if (!equal) { - return false; - } - - for (size_t i = 0; i < pointerCount; i++) { - equal = pointerProperties[i] == rhs.pointerProperties[i] && - pointerCoords[i] == rhs.pointerCoords[i]; - if (!equal) { - return false; - } - } - return true; } std::string NotifyMotionArgs::dump() const { std::string coords; - for (uint32_t i = 0; i < pointerCount; i++) { + for (uint32_t i = 0; i < getPointerCount(); i++) { if (!coords.empty()) { coords += ", "; } @@ -181,11 +139,10 @@ std::string NotifyMotionArgs::dump() const { coords += "}"; } return StringPrintf("NotifyMotionArgs(id=%" PRId32 ", eventTime=%" PRId64 ", deviceId=%" PRId32 - ", source=%s, action=%s, pointerCount=%" PRIu32 - " pointers=%s, flags=0x%08x)", + ", source=%s, action=%s, pointerCount=%zu pointers=%s, flags=0x%08x)", id, eventTime, deviceId, inputEventSourceToString(source).c_str(), - MotionEvent::actionToString(action).c_str(), pointerCount, coords.c_str(), - flags); + MotionEvent::actionToString(action).c_str(), getPointerCount(), + coords.c_str(), flags); } // --- NotifySwitchArgs --- diff --git a/services/inputflinger/PreferStylusOverTouchBlocker.cpp b/services/inputflinger/PreferStylusOverTouchBlocker.cpp index fbd296c131..ee0ab33559 100644 --- a/services/inputflinger/PreferStylusOverTouchBlocker.cpp +++ b/services/inputflinger/PreferStylusOverTouchBlocker.cpp @@ -22,7 +22,7 @@ namespace android { static std::pair<bool, bool> checkToolType(const NotifyMotionArgs& args) { bool hasStylus = false; bool hasTouch = false; - for (size_t i = 0; i < args.pointerCount; i++) { + for (size_t i = 0; i < args.getPointerCount(); i++) { // Make sure we are canceling stylus pointers const ToolType toolType = args.pointerProperties[i].toolType; if (isStylusToolType(toolType)) { diff --git a/services/inputflinger/SyncQueue.h b/services/inputflinger/SyncQueue.h new file mode 100644 index 0000000000..84ccace96e --- /dev/null +++ b/services/inputflinger/SyncQueue.h @@ -0,0 +1,66 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <utils/threads.h> +#include <list> +#include <mutex> +#include <optional> + +namespace android { + +/** A thread-safe FIFO queue. */ +template <class T> +class SyncQueue { +public: + SyncQueue() = default; + + SyncQueue(size_t capacity) : mCapacity(capacity) {} + + /** Retrieve and remove the oldest object. Returns std::nullopt if the queue is empty. */ + std::optional<T> pop() { + std::scoped_lock lock(mLock); + if (mQueue.empty()) { + return {}; + } + T t = std::move(mQueue.front()); + mQueue.erase(mQueue.begin()); + return t; + }; + + /** + * Add a new object to the queue. + * Return true if an element was successfully added. + * Return false if the queue is full. + */ + template <class... Args> + bool push(Args&&... args) { + std::scoped_lock lock(mLock); + if (mCapacity && mQueue.size() == mCapacity) { + return false; + } + mQueue.emplace_back(args...); + return true; + }; + +private: + const std::optional<size_t> mCapacity; + std::mutex mLock; + std::list<T> mQueue GUARDED_BY(mLock); +}; + +} // namespace android diff --git a/services/inputflinger/TEST_MAPPING b/services/inputflinger/TEST_MAPPING index f0b107222a..cdc4c088b5 100644 --- a/services/inputflinger/TEST_MAPPING +++ b/services/inputflinger/TEST_MAPPING @@ -49,15 +49,38 @@ "name": "CtsViewTestCases", "options": [ { - "include-filter": "android.view.cts.input", - "include-filter": "android.view.cts.HoverTest", - "include-filter": "android.view.cts.MotionEventTest", - "include-filter": "android.view.cts.PointerCaptureTest", - "include-filter": "android.view.cts.TooltipTest", - "include-filter": "android.view.cts.TouchDelegateTest", - "include-filter": "android.view.cts.VelocityTrackerTest", - "include-filter": "android.view.cts.VerifyInputEventTest", - "include-filter": "android.view.cts.ViewTest", + "include-filter": "android.view.cts.input" + } + ] + }, + { + "name": "CtsViewTestCases", + "options": [ + { + "include-filter": "android.view.cts.HoverTest" + }, + { + "include-filter": "android.view.cts.MotionEventTest" + }, + { + "include-filter": "android.view.cts.PointerCaptureTest" + }, + { + "include-filter": "android.view.cts.TooltipTest" + }, + { + "include-filter": "android.view.cts.TouchDelegateTest" + }, + { + "include-filter": "android.view.cts.VelocityTrackerTest" + }, + { + "include-filter": "android.view.cts.VerifyInputEventTest" + }, + { + "include-filter": "android.view.cts.ViewTest" + }, + { "include-filter": "android.view.cts.ViewUnbufferedTest" } ] @@ -66,7 +89,9 @@ "name": "CtsWidgetTestCases", "options": [ { - "include-filter": "android.widget.cts.NumberPickerTest", + "include-filter": "android.widget.cts.NumberPickerTest" + }, + { "include-filter": "android.widget.cts.SeekBarTest" } ] @@ -75,8 +100,17 @@ "name": "FrameworksCoreTests", "options": [ { - "include-filter": "android.hardware.input", - "include-filter": "android.view.VerifiedKeyEventTest", + "include-filter": "android.hardware.input" + } + ] + }, + { + "name": "FrameworksCoreTests", + "options": [ + { + "include-filter": "android.view.VerifiedKeyEventTest" + }, + { "include-filter": "android.view.VerifiedMotionEventTest" } ] @@ -153,15 +187,38 @@ "name": "CtsViewTestCases", "options": [ { - "include-filter": "android.view.cts.input", - "include-filter": "android.view.cts.HoverTest", - "include-filter": "android.view.cts.MotionEventTest", - "include-filter": "android.view.cts.PointerCaptureTest", - "include-filter": "android.view.cts.TooltipTest", - "include-filter": "android.view.cts.TouchDelegateTest", - "include-filter": "android.view.cts.VelocityTrackerTest", - "include-filter": "android.view.cts.VerifyInputEventTest", - "include-filter": "android.view.cts.ViewTest", + "include-filter": "android.view.cts.input" + } + ] + }, + { + "name": "CtsViewTestCases", + "options": [ + { + "include-filter": "android.view.cts.HoverTest" + }, + { + "include-filter": "android.view.cts.MotionEventTest" + }, + { + "include-filter": "android.view.cts.PointerCaptureTest" + }, + { + "include-filter": "android.view.cts.TooltipTest" + }, + { + "include-filter": "android.view.cts.TouchDelegateTest" + }, + { + "include-filter": "android.view.cts.VelocityTrackerTest" + }, + { + "include-filter": "android.view.cts.VerifyInputEventTest" + }, + { + "include-filter": "android.view.cts.ViewTest" + }, + { "include-filter": "android.view.cts.ViewUnbufferedTest" } ] @@ -170,7 +227,9 @@ "name": "CtsWidgetTestCases", "options": [ { - "include-filter": "android.widget.cts.NumberPickerTest", + "include-filter": "android.widget.cts.NumberPickerTest" + }, + { "include-filter": "android.widget.cts.SeekBarTest" } ] @@ -179,7 +238,9 @@ "name": "FrameworksCoreTests", "options": [ { - "include-filter": "android.view.VerifiedKeyEventTest", + "include-filter": "android.view.VerifiedKeyEventTest" + }, + { "include-filter": "android.view.VerifiedMotionEventTest" } ] diff --git a/services/inputflinger/UnwantedInteractionBlocker.cpp b/services/inputflinger/UnwantedInteractionBlocker.cpp index 02bc47d6bb..f889de58b8 100644 --- a/services/inputflinger/UnwantedInteractionBlocker.cpp +++ b/services/inputflinger/UnwantedInteractionBlocker.cpp @@ -117,7 +117,7 @@ static int getLinuxToolCode(ToolType toolType) { } static int32_t getActionUpForPointerId(const NotifyMotionArgs& args, int32_t pointerId) { - for (size_t i = 0; i < args.pointerCount; i++) { + for (size_t i = 0; i < args.getPointerCount(); i++) { if (pointerId == args.pointerProperties[i].id) { return AMOTION_EVENT_ACTION_POINTER_UP | (i << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT); @@ -156,9 +156,10 @@ NotifyMotionArgs removePointerIds(const NotifyMotionArgs& args, actionMasked == AMOTION_EVENT_ACTION_POINTER_UP; NotifyMotionArgs newArgs{args}; - newArgs.pointerCount = 0; + newArgs.pointerProperties.clear(); + newArgs.pointerCoords.clear(); int32_t newActionIndex = 0; - for (uint32_t i = 0; i < args.pointerCount; i++) { + for (uint32_t i = 0; i < args.getPointerCount(); i++) { const int32_t pointerId = args.pointerProperties[i].id; if (pointerIds.find(pointerId) != pointerIds.end()) { // skip this pointer @@ -170,19 +171,18 @@ NotifyMotionArgs removePointerIds(const NotifyMotionArgs& args, } continue; } - newArgs.pointerProperties[newArgs.pointerCount].copyFrom(args.pointerProperties[i]); - newArgs.pointerCoords[newArgs.pointerCount].copyFrom(args.pointerCoords[i]); + newArgs.pointerProperties.push_back(args.pointerProperties[i]); + newArgs.pointerCoords.push_back(args.pointerCoords[i]); if (i == actionIndex) { - newActionIndex = newArgs.pointerCount; + newActionIndex = newArgs.getPointerCount() - 1; } - newArgs.pointerCount++; } // Update POINTER_DOWN or POINTER_UP actions if (isPointerUpOrDownAction && newArgs.action != ACTION_UNKNOWN) { newArgs.action = actionMasked | (newActionIndex << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT); // Convert POINTER_DOWN and POINTER_UP to DOWN and UP if there's only 1 pointer remaining - if (newArgs.pointerCount == 1) { + if (newArgs.getPointerCount() == 1) { if (actionMasked == AMOTION_EVENT_ACTION_POINTER_DOWN) { newArgs.action = AMOTION_EVENT_ACTION_DOWN; } else if (actionMasked == AMOTION_EVENT_ACTION_POINTER_UP) { @@ -201,13 +201,14 @@ NotifyMotionArgs removePointerIds(const NotifyMotionArgs& args, */ static std::optional<NotifyMotionArgs> removeStylusPointerIds(const NotifyMotionArgs& args) { std::set<int32_t> stylusPointerIds; - for (uint32_t i = 0; i < args.pointerCount; i++) { + for (uint32_t i = 0; i < args.getPointerCount(); i++) { if (isStylusToolType(args.pointerProperties[i].toolType)) { stylusPointerIds.insert(args.pointerProperties[i].id); } } NotifyMotionArgs withoutStylusPointers = removePointerIds(args, stylusPointerIds); - if (withoutStylusPointers.pointerCount == 0 || withoutStylusPointers.action == ACTION_UNKNOWN) { + if (withoutStylusPointers.getPointerCount() == 0 || + withoutStylusPointers.action == ACTION_UNKNOWN) { return std::nullopt; } return withoutStylusPointers; @@ -272,7 +273,7 @@ std::optional<AndroidPalmFilterDeviceInfo> createPalmFilterDeviceInfo( std::vector<NotifyMotionArgs> cancelSuppressedPointers( const NotifyMotionArgs& args, const std::set<int32_t>& oldSuppressedPointerIds, const std::set<int32_t>& newSuppressedPointerIds) { - LOG_ALWAYS_FATAL_IF(args.pointerCount == 0, "0 pointers in %s", args.dump().c_str()); + LOG_ALWAYS_FATAL_IF(args.getPointerCount() == 0, "0 pointers in %s", args.dump().c_str()); // First, let's remove the old suppressed pointers. They've already been canceled previously. NotifyMotionArgs oldArgs = removePointerIds(args, oldSuppressedPointerIds); @@ -284,7 +285,7 @@ std::vector<NotifyMotionArgs> cancelSuppressedPointers( const int32_t actionMasked = MotionEvent::getActionMasked(args.action); // We will iteratively remove pointers from 'removedArgs'. NotifyMotionArgs removedArgs{oldArgs}; - for (uint32_t i = 0; i < oldArgs.pointerCount; i++) { + for (uint32_t i = 0; i < oldArgs.getPointerCount(); i++) { const int32_t pointerId = oldArgs.pointerProperties[i].id; if (newSuppressedPointerIds.find(pointerId) == newSuppressedPointerIds.end()) { // This is a pointer that should not be canceled. Move on. @@ -296,7 +297,7 @@ std::vector<NotifyMotionArgs> cancelSuppressedPointers( continue; } - if (removedArgs.pointerCount == 1) { + if (removedArgs.getPointerCount() == 1) { // We are about to remove the last pointer, which means there will be no more gesture // remaining. This is identical to canceling all pointers, so just send a single CANCEL // event, without any of the preceding POINTER_UP with FLAG_CANCELED events. @@ -314,7 +315,7 @@ std::vector<NotifyMotionArgs> cancelSuppressedPointers( } // Now 'removedArgs' contains only pointers that are valid. - if (removedArgs.pointerCount <= 0 || removedArgs.action == ACTION_UNKNOWN) { + if (removedArgs.getPointerCount() <= 0 || removedArgs.action == ACTION_UNKNOWN) { return out; } out.push_back(removedArgs); @@ -473,7 +474,7 @@ void UnwantedInteractionBlocker::monitor() { UnwantedInteractionBlocker::~UnwantedInteractionBlocker() {} void SlotState::update(const NotifyMotionArgs& args) { - for (size_t i = 0; i < args.pointerCount; i++) { + for (size_t i = 0; i < args.getPointerCount(); i++) { const int32_t pointerId = args.pointerProperties[i].id; const int32_t resolvedAction = resolveActionForPointer(i, args.action); processPointerId(pointerId, resolvedAction); @@ -571,7 +572,7 @@ std::vector<::ui::InProgressTouchEvdev> getTouches(const NotifyMotionArgs& args, const SlotState& newSlotState) { std::vector<::ui::InProgressTouchEvdev> touches; - for (size_t i = 0; i < args.pointerCount; i++) { + for (size_t i = 0; i < args.getPointerCount(); i++) { const int32_t pointerId = args.pointerProperties[i].id; touches.emplace_back(::ui::InProgressTouchEvdev()); touches.back().major = args.pointerCoords[i].getAxisValue(AMOTION_EVENT_AXIS_TOUCH_MAJOR); @@ -660,7 +661,7 @@ std::set<int32_t> PalmRejector::detectPalmPointers(const NotifyMotionArgs& args) // Now that we know which slots should be suppressed, let's convert those to pointer id's. std::set<int32_t> newSuppressedIds; - for (size_t i = 0; i < args.pointerCount; i++) { + for (size_t i = 0; i < args.getPointerCount(); i++) { const int32_t pointerId = args.pointerProperties[i].id; std::optional<size_t> slot = oldSlotState.getSlotForPointerId(pointerId); if (!slot) { diff --git a/services/inputflinger/benchmarks/InputDispatcher_benchmarks.cpp b/services/inputflinger/benchmarks/InputDispatcher_benchmarks.cpp index f65533ea79..b2e274d832 100644 --- a/services/inputflinger/benchmarks/InputDispatcher_benchmarks.cpp +++ b/services/inputflinger/benchmarks/InputDispatcher_benchmarks.cpp @@ -30,12 +30,14 @@ using android::os::InputEventInjectionSync; namespace android::inputdispatcher { +namespace { + // An arbitrary device id. constexpr int32_t DEVICE_ID = 1; // The default pid and uid for windows created by the test. -constexpr int32_t WINDOW_PID = 999; -constexpr int32_t WINDOW_UID = 1001; +constexpr gui::Pid WINDOW_PID{999}; +constexpr gui::Uid WINDOW_UID{1001}; static constexpr std::chrono::duration INJECT_EVENT_TIMEOUT = 5s; static constexpr std::chrono::nanoseconds DISPATCHING_TIMEOUT = 100ms; @@ -59,13 +61,13 @@ private: ALOGE("There is no focused window for %s", applicationHandle->getName().c_str()); } - void notifyWindowUnresponsive(const sp<IBinder>& connectionToken, std::optional<int32_t> pid, + void notifyWindowUnresponsive(const sp<IBinder>& connectionToken, std::optional<gui::Pid> pid, const std::string& reason) override { ALOGE("Window is not responding: %s", reason.c_str()); } void notifyWindowResponsive(const sp<IBinder>& connectionToken, - std::optional<int32_t> pid) override {} + std::optional<gui::Pid> pid) override {} void notifyInputChannelBroken(const sp<IBinder>&) override {} @@ -109,6 +111,9 @@ private: void notifyDropWindow(const sp<IBinder>&, float x, float y) override {} + void notifyDeviceInteraction(int32_t deviceId, nsecs_t timestamp, + const std::set<gui::Uid>& uids) override {} + InputDispatcherConfiguration mConfig; }; @@ -348,6 +353,8 @@ static void benchmarkOnWindowInfosChanged(benchmark::State& state) { dispatcher.stop(); } +} // namespace + BENCHMARK(benchmarkNotifyMotion); BENCHMARK(benchmarkInjectMotion); BENCHMARK(benchmarkOnWindowInfosChanged); diff --git a/services/inputflinger/dispatcher/Android.bp b/services/inputflinger/dispatcher/Android.bp index da4e42f4fe..492551ec06 100644 --- a/services/inputflinger/dispatcher/Android.bp +++ b/services/inputflinger/dispatcher/Android.bp @@ -94,6 +94,7 @@ cc_defaults { cc_library_static { name: "libinputdispatcher", + host_supported: true, defaults: [ "inputflinger_defaults", "libinputdispatcher_defaults", diff --git a/services/inputflinger/dispatcher/DebugConfig.cpp b/services/inputflinger/dispatcher/DebugConfig.cpp index 764194d3d0..12122fd354 100644 --- a/services/inputflinger/dispatcher/DebugConfig.cpp +++ b/services/inputflinger/dispatcher/DebugConfig.cpp @@ -30,11 +30,10 @@ const bool IS_DEBUGGABLE_BUILD = bool debugInboundEventDetails() { if (!IS_DEBUGGABLE_BUILD) { static const bool DEBUG_INBOUND_EVENT_DETAILS = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "InboundEvent", - ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "InboundEvent"); return DEBUG_INBOUND_EVENT_DETAILS; } - return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "InboundEvent", ANDROID_LOG_INFO); + return android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "InboundEvent"); } } // namespace android::inputdispatcher diff --git a/services/inputflinger/dispatcher/DebugConfig.h b/services/inputflinger/dispatcher/DebugConfig.h index 0e260a7a03..7a41d682f8 100644 --- a/services/inputflinger/dispatcher/DebugConfig.h +++ b/services/inputflinger/dispatcher/DebugConfig.h @@ -18,8 +18,7 @@ #define LOG_TAG "InputDispatcher" -#include <log/log.h> -#include <log/log_event_list.h> +#include <android-base/logging.h> namespace android::inputdispatcher { @@ -42,14 +41,14 @@ bool debugInboundEventDetails(); * Enable this via "adb shell setprop log.tag.InputDispatcherOutboundEvent DEBUG" (requires restart) */ const bool DEBUG_OUTBOUND_EVENT_DETAILS = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "OutboundEvent", ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "OutboundEvent"); /** * Log debug messages about the dispatch cycle. * Enable this via "adb shell setprop log.tag.InputDispatcherDispatchCycle DEBUG" (requires restart) */ const bool DEBUG_DISPATCH_CYCLE = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "DispatchCycle", ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "DispatchCycle"); /** * Log debug messages about channel creation @@ -57,28 +56,28 @@ const bool DEBUG_DISPATCH_CYCLE = * restart) */ const bool DEBUG_CHANNEL_CREATION = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "ChannelCreation", ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "ChannelCreation"); /** * Log debug messages about input event injection. * Enable this via "adb shell setprop log.tag.InputDispatcherInjection DEBUG" (requires restart) */ const bool DEBUG_INJECTION = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Injection", ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "Injection"); /** * Log debug messages about input focus tracking. * Enable this via "adb shell setprop log.tag.InputDispatcherFocus DEBUG" (requires restart) */ const bool DEBUG_FOCUS = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Focus", ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "Focus"); /** * Log debug messages about touch mode event * Enable this via "adb shell setprop log.tag.InputDispatcherTouchMode DEBUG" (requires restart) */ const bool DEBUG_TOUCH_MODE = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "TouchMode", ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "TouchMode"); /** * Log debug messages about touch occlusion @@ -90,13 +89,20 @@ constexpr bool DEBUG_TOUCH_OCCLUSION = true; * Enable this via "adb shell setprop log.tag.InputDispatcherAppSwitch DEBUG" (requires restart) */ const bool DEBUG_APP_SWITCH = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "AppSwitch", ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "AppSwitch"); /** * Log debug messages about hover events. * Enable this via "adb shell setprop log.tag.InputDispatcherHover DEBUG" (requires restart) */ const bool DEBUG_HOVER = - __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Hover", ANDROID_LOG_INFO); + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "Hover"); + +/** + * Crash if a bad stream from InputListener is detected. + * Enable this via "adb shell setprop log.tag.InputDispatcherVerifyEvents DEBUG" (requires restart) + */ +const bool DEBUG_VERIFY_EVENTS = + android::base::ShouldLog(android::base::LogSeverity::DEBUG, LOG_TAG "VerifyEvents"); } // namespace android::inputdispatcher diff --git a/services/inputflinger/dispatcher/Entry.cpp b/services/inputflinger/dispatcher/Entry.cpp index b625a1b95b..a670ebe04e 100644 --- a/services/inputflinger/dispatcher/Entry.cpp +++ b/services/inputflinger/dispatcher/Entry.cpp @@ -352,7 +352,7 @@ std::ostream& operator<<(std::ostream& out, const DispatchEntry& entry) { entry.transform.dump(transform, "transform"); out << ", resolvedFlags=" << entry.resolvedFlags << ", targetFlags=" << entry.targetFlags.string() << ", transform=" << transform - << "} original =" << entry.eventEntry->getDescription(); + << "} original: " << entry.eventEntry->getDescription(); return out; } diff --git a/services/inputflinger/dispatcher/InjectionState.cpp b/services/inputflinger/dispatcher/InjectionState.cpp index c2d3ad6b9d..053594b253 100644 --- a/services/inputflinger/dispatcher/InjectionState.cpp +++ b/services/inputflinger/dispatcher/InjectionState.cpp @@ -20,7 +20,7 @@ namespace android::inputdispatcher { -InjectionState::InjectionState(const std::optional<int32_t>& targetUid) +InjectionState::InjectionState(const std::optional<gui::Uid>& targetUid) : refCount(1), targetUid(targetUid), injectionResult(android::os::InputEventInjectionResult::PENDING), diff --git a/services/inputflinger/dispatcher/InjectionState.h b/services/inputflinger/dispatcher/InjectionState.h index d9e27ba50b..3a3f5aefba 100644 --- a/services/inputflinger/dispatcher/InjectionState.h +++ b/services/inputflinger/dispatcher/InjectionState.h @@ -26,12 +26,12 @@ namespace inputdispatcher { struct InjectionState { mutable int32_t refCount; - std::optional<int32_t> targetUid; + std::optional<gui::Uid> targetUid; android::os::InputEventInjectionResult injectionResult; // initially PENDING bool injectionIsAsync; // set to true if injection is not waiting for the result int32_t pendingForegroundDispatches; // the number of foreground dispatches in progress - explicit InjectionState(const std::optional<int32_t>& targetUid); + explicit InjectionState(const std::optional<gui::Uid>& targetUid); void release(); private: diff --git a/services/inputflinger/dispatcher/InputDispatcher.cpp b/services/inputflinger/dispatcher/InputDispatcher.cpp index 1cf2b5ffe0..59220a03ff 100644 --- a/services/inputflinger/dispatcher/InputDispatcher.cpp +++ b/services/inputflinger/dispatcher/InputDispatcher.cpp @@ -26,6 +26,7 @@ #include <android/os/IInputConstants.h> #include <binder/Binder.h> #include <ftl/enum.h> +#include <log/log_event_list.h> #if defined(__ANDROID__) #include <gui/SurfaceComposerClient.h> #endif @@ -125,6 +126,10 @@ inline const std::string binderToString(const sp<IBinder>& binder) { return StringPrintf("%p", binder.get()); } +static std::string uidString(const gui::Uid& uid) { + return uid.toString(); +} + inline int32_t getMotionEventActionPointerIndex(int32_t action) { return (action & AMOTION_EVENT_ACTION_POINTER_INDEX_MASK) >> AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT; @@ -556,7 +561,7 @@ bool canReceiveForegroundTouches(const WindowInfo& info) { return !info.inputConfig.test(gui::WindowInfo::InputConfig::NOT_TOUCHABLE) && !info.isSpy(); } -bool isWindowOwnedBy(const sp<WindowInfoHandle>& windowHandle, int32_t pid, int32_t uid) { +bool isWindowOwnedBy(const sp<WindowInfoHandle>& windowHandle, gui::Pid pid, gui::Uid uid) { if (windowHandle == nullptr) { return false; } @@ -576,14 +581,16 @@ std::optional<std::string> verifyTargetedInjection(const sp<WindowInfoHandle>& w // The event was not injected, or the injected event does not target a window. return {}; } - const int32_t uid = *entry.injectionState->targetUid; + const auto uid = *entry.injectionState->targetUid; if (window == nullptr) { - return StringPrintf("No valid window target for injection into uid %d.", uid); + return StringPrintf("No valid window target for injection into uid %s.", + uid.toString().c_str()); } if (entry.injectionState->targetUid != window->getInfo()->ownerUid) { - return StringPrintf("Injected event targeted at uid %d would be dispatched to window '%s' " - "owned by uid %d.", - uid, window->getName().c_str(), window->getInfo()->ownerUid); + return StringPrintf("Injected event targeted at uid %s would be dispatched to window '%s' " + "owned by uid %s.", + uid.toString().c_str(), window->getName().c_str(), + window->getInfo()->ownerUid.toString().c_str()); } return {}; } @@ -647,7 +654,6 @@ std::vector<TouchedWindow> getHoveringWindowsLocked(const TouchState* oldState, TouchedWindow touchedWindow; touchedWindow.windowHandle = oldWindow; touchedWindow.targetFlags = InputTarget::Flags::DISPATCH_AS_HOVER_EXIT; - touchedWindow.pointerIds.set(pointerId); out.push_back(touchedWindow); } } @@ -674,7 +680,7 @@ std::vector<TouchedWindow> getHoveringWindowsLocked(const TouchState* oldState, } touchedWindow.targetFlags = InputTarget::Flags::DISPATCH_AS_IS; } - touchedWindow.pointerIds.set(pointerId); + touchedWindow.addHoveringPointer(entry.deviceId, pointerId); if (canReceiveForegroundTouches(*newWindow->getInfo())) { touchedWindow.targetFlags |= InputTarget::Flags::FOREGROUND; } @@ -1963,7 +1969,7 @@ void InputDispatcher::dispatchEventLocked(nsecs_t currentTime, ALOGD("dispatchEventToCurrentInputTargets"); } - updateInteractionTokensLocked(*eventEntry, inputTargets); + processInteractionsLocked(*eventEntry, inputTargets); ALOG_ASSERT(eventEntry->dispatchInProgress); // should already have been set to true @@ -2196,8 +2202,8 @@ std::vector<Monitor> InputDispatcher::selectResponsiveMonitorsLocked( /** * In general, touch should be always split between windows. Some exceptions: * 1. Don't split touch is if we have an active pointer down, and a new pointer is going down that's - * from the same device, *and* the window that's receiving the current pointer does not support - * split touch. + * from the same device, *and* the window that's receiving the current pointer does not support + * split touch. * 2. Don't split mouse events */ bool InputDispatcher::shouldSplitTouch(const TouchState& touchState, @@ -2220,9 +2226,7 @@ bool InputDispatcher::shouldSplitTouch(const TouchState& touchState, continue; } - // Eventually, touchedWindow will contain the deviceId of each pointer that's currently - // being sent there. For now, use deviceId from touch state. - if (entry.deviceId == touchState.deviceId && touchedWindow.pointerIds.any()) { + if (touchedWindow.hasTouchingPointers(entry.deviceId)) { return false; } } @@ -2255,8 +2259,13 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( } bool isSplit = shouldSplitTouch(tempTouchState, entry); - const bool switchedDevice = (oldState != nullptr) && - (oldState->deviceId != entry.deviceId || oldState->source != entry.source); + bool switchedDevice = false; + if (oldState != nullptr) { + std::set<int32_t> oldActiveDevices = oldState->getActiveDeviceIds(); + const bool anotherDeviceIsActive = + oldActiveDevices.count(entry.deviceId) == 0 && !oldActiveDevices.empty(); + switchedDevice |= anotherDeviceIsActive; + } const bool isHoverAction = (maskedAction == AMOTION_EVENT_ACTION_HOVER_MOVE || maskedAction == AMOTION_EVENT_ACTION_HOVER_ENTER || @@ -2275,7 +2284,7 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( // from another device. However, if the new event is a down event, let's cancel the current // touch and let the new one take over. if (switchedDevice && wasDown && !isDown) { - LOG(INFO) << "Dropping event because a pointer for device " << oldState->deviceId + LOG(INFO) << "Dropping event because a pointer for another device " << " is already down in display " << displayId << ": " << entry.getDescription(); // TODO(b/211379801): test multiple simultaneous input streams. outInjectionResult = InputEventInjectionResult::FAILED; @@ -2285,8 +2294,6 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( if (newGesture) { // If a new gesture is starting, clear the touch state completely. tempTouchState.reset(); - tempTouchState.deviceId = entry.deviceId; - tempTouchState.source = entry.source; isSplit = false; } else if (switchedDevice && maskedAction == AMOTION_EVENT_ACTION_MOVE) { ALOGI("Dropping move event because a pointer for a different device is already active " @@ -2405,7 +2412,7 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( // still add a window to the touch state. We should avoid doing that, but some of the // later checks ("at least one foreground window") rely on this in order to dispatch // the event properly, so that needs to be updated, possibly by looking at InputTargets. - tempTouchState.addOrUpdateWindow(windowHandle, targetFlags, pointerIds, + tempTouchState.addOrUpdateWindow(windowHandle, targetFlags, entry.deviceId, pointerIds, isDownOrPointerDown ? std::make_optional(entry.eventTime) : std::nullopt); @@ -2429,8 +2436,8 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( if (isSplit) { wallpaperFlags |= InputTarget::Flags::SPLIT; } - tempTouchState.addOrUpdateWindow(wallpaper, wallpaperFlags, pointerIds, - entry.eventTime); + tempTouchState.addOrUpdateWindow(wallpaper, wallpaperFlags, entry.deviceId, + pointerIds, entry.eventTime); } } } @@ -2441,12 +2448,12 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( // which is a specific behaviour that we want. const int32_t pointerId = entry.pointerProperties[pointerIndex].id; for (TouchedWindow& touchedWindow : tempTouchState.windows) { - if (touchedWindow.pointerIds.test(pointerId) && - touchedWindow.pilferedPointerIds.count() > 0) { + if (touchedWindow.hasTouchingPointer(entry.deviceId, pointerId) && + touchedWindow.hasPilferingPointers(entry.deviceId)) { // This window is already pilfering some pointers, and this new pointer is also // going to it. Therefore, take over this pointer and don't give it to anyone // else. - touchedWindow.pilferedPointerIds.set(pointerId); + touchedWindow.addPilferingPointer(entry.deviceId, pointerId); } } @@ -2518,7 +2525,7 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( tempTouchState.getTouchedWindow(oldTouchedWindowHandle); addWindowTargetLocked(oldTouchedWindowHandle, InputTarget::Flags::DISPATCH_AS_SLIPPERY_EXIT, pointerIds, - touchedWindow.firstDownTimeInTarget, targets); + touchedWindow.getDownTimeInTarget(entry.deviceId), targets); // Make a slippery entrance into the new window. if (newTouchedWindowHandle->getInfo()->supportsSplitTouch()) { @@ -2539,13 +2546,14 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( targetFlags |= InputTarget::Flags::WINDOW_IS_PARTIALLY_OBSCURED; } - tempTouchState.addOrUpdateWindow(newTouchedWindowHandle, targetFlags, pointerIds, - entry.eventTime); + tempTouchState.addOrUpdateWindow(newTouchedWindowHandle, targetFlags, + entry.deviceId, pointerIds, entry.eventTime); // Check if the wallpaper window should deliver the corresponding event. slipWallpaperTouch(targetFlags, oldTouchedWindowHandle, newTouchedWindowHandle, - tempTouchState, pointerId, targets); - tempTouchState.removeTouchedPointerFromWindow(pointerId, oldTouchedWindowHandle); + tempTouchState, entry.deviceId, pointerId, targets); + tempTouchState.removeTouchingPointerFromWindow(entry.deviceId, pointerId, + oldTouchedWindowHandle); } } @@ -2559,7 +2567,8 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( if (mDragState && mDragState->dragWindow == touchedWindow.windowHandle) { continue; } - touchedWindow.pointerIds.set(entry.pointerProperties[pointerIndex].id); + touchedWindow.addTouchingPointer(entry.deviceId, + entry.pointerProperties[pointerIndex].id); } } } @@ -2571,7 +2580,7 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( for (const TouchedWindow& touchedWindow : hoveringWindows) { std::optional<InputTarget> target = createInputTargetLocked(touchedWindow.windowHandle, touchedWindow.targetFlags, - touchedWindow.firstDownTimeInTarget); + touchedWindow.getDownTimeInTarget(entry.deviceId)); if (!target) { continue; } @@ -2587,18 +2596,13 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( if (entry.injectionState != nullptr) { std::string errs; for (const TouchedWindow& touchedWindow : tempTouchState.windows) { - if (touchedWindow.targetFlags.test(InputTarget::Flags::DISPATCH_AS_OUTSIDE)) { - // Allow ACTION_OUTSIDE events generated by targeted injection to be - // dispatched to any uid, since the coords will be zeroed out later. - continue; - } const auto err = verifyTargetedInjection(touchedWindow.windowHandle, entry); if (err) errs += "\n - " + *err; } if (!errs.empty()) { ALOGW("Dropping targeted injection: At least one touched window is not owned by uid " - "%d:%s", - *entry.injectionState->targetUid, errs.c_str()); + "%s:%s", + entry.injectionState->targetUid->toString().c_str(), errs.c_str()); outInjectionResult = InputEventInjectionResult::TARGET_MISMATCH; return {}; } @@ -2610,7 +2614,7 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( sp<WindowInfoHandle> foregroundWindowHandle = tempTouchState.getFirstForegroundWindowHandle(); if (foregroundWindowHandle) { - const int32_t foregroundWindowUid = foregroundWindowHandle->getInfo()->ownerUid; + const auto foregroundWindowUid = foregroundWindowHandle->getInfo()->ownerUid; for (InputTarget& target : targets) { if (target.flags.test(InputTarget::Flags::DISPATCH_AS_OUTSIDE)) { sp<WindowInfoHandle> targetWindow = @@ -2633,17 +2637,30 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( // Output targets from the touch state. for (const TouchedWindow& touchedWindow : tempTouchState.windows) { - if (touchedWindow.pointerIds.none() && !touchedWindow.hasHoveringPointers(entry.deviceId)) { + if (!touchedWindow.hasTouchingPointers(entry.deviceId) && + !touchedWindow.hasHoveringPointers(entry.deviceId)) { // Windows with hovering pointers are getting persisted inside TouchState. // Do not send this event to those windows. continue; } addWindowTargetLocked(touchedWindow.windowHandle, touchedWindow.targetFlags, - touchedWindow.pointerIds, touchedWindow.firstDownTimeInTarget, - targets); + touchedWindow.getTouchingPointers(entry.deviceId), + touchedWindow.getDownTimeInTarget(entry.deviceId), targets); } + // During targeted injection, only allow owned targets to receive events + std::erase_if(targets, [&](const InputTarget& target) { + LOG_ALWAYS_FATAL_IF(target.windowHandle == nullptr); + const auto err = verifyTargetedInjection(target.windowHandle, entry); + if (err) { + LOG(WARNING) << "Dropping injected event from " << target.windowHandle->getName() + << ": " << (*err); + return true; + } + return false; + }); + if (targets.empty()) { LOG(INFO) << "Dropping event because no targets were found: " << entry.getDescription(); outInjectionResult = InputEventInjectionResult::FAILED; @@ -2683,12 +2700,10 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( } if (maskedAction == AMOTION_EVENT_ACTION_HOVER_ENTER || maskedAction == AMOTION_EVENT_ACTION_HOVER_MOVE) { - tempTouchState.deviceId = entry.deviceId; - tempTouchState.source = entry.source; } } else if (maskedAction == AMOTION_EVENT_ACTION_UP) { // Pointer went up. - tempTouchState.removeTouchedPointer(entry.pointerProperties[0].id); + tempTouchState.removeTouchingPointer(entry.deviceId, entry.pointerProperties[0].id); } else if (maskedAction == AMOTION_EVENT_ACTION_CANCEL) { // All pointers up or canceled. tempTouchState.reset(); @@ -2705,8 +2720,8 @@ std::vector<InputTarget> InputDispatcher::findTouchedWindowTargetsLocked( for (size_t i = 0; i < tempTouchState.windows.size();) { TouchedWindow& touchedWindow = tempTouchState.windows[i]; - touchedWindow.pointerIds.reset(pointerId); - if (touchedWindow.pointerIds.none()) { + touchedWindow.removeTouchingPointer(entry.deviceId, pointerId); + if (!touchedWindow.hasTouchingPointers(entry.deviceId)) { tempTouchState.windows.erase(tempTouchState.windows.begin() + i); continue; } @@ -2838,6 +2853,7 @@ std::optional<InputTarget> InputDispatcher::createInputTargetLocked( } InputTarget inputTarget; inputTarget.inputChannel = inputChannel; + inputTarget.windowHandle = windowHandle; inputTarget.flags = targetFlags; inputTarget.globalScaleFactor = windowHandle->getInfo()->globalScaleFactor; inputTarget.firstDownTimeInTarget = firstDownTimeInTarget; @@ -2845,7 +2861,7 @@ std::optional<InputTarget> InputDispatcher::createInputTargetLocked( if (displayInfoIt != mDisplayInfos.end()) { inputTarget.displayTransform = displayInfoIt->second.transform; } else { - // DisplayInfo not found for this window on display windowInfo->displayId. + // DisplayInfo not found for this window on display windowHandle->getInfo()->displayId. // TODO(b/198444055): Make this an error message after 'setInputWindows' API is removed. } return inputTarget; @@ -2961,8 +2977,8 @@ InputDispatcher::TouchOcclusionInfo InputDispatcher::computeTouchOcclusionInfoLo TouchOcclusionInfo info; info.hasBlockingOcclusion = false; info.obscuringOpacity = 0; - info.obscuringUid = -1; - std::map<int32_t, float> opacityByUid; + info.obscuringUid = gui::Uid::INVALID; + std::map<gui::Uid, float> opacityByUid; for (const sp<WindowInfoHandle>& otherHandle : windowHandles) { if (windowHandle == otherHandle) { break; // All future windows are below us. Exit early. @@ -2984,7 +3000,7 @@ InputDispatcher::TouchOcclusionInfo InputDispatcher::computeTouchOcclusionInfoLo break; } if (otherInfo->touchOcclusionMode == TouchOcclusionMode::USE_OPACITY) { - uint32_t uid = otherInfo->ownerUid; + const auto uid = otherInfo->ownerUid; float opacity = (opacityByUid.find(uid) == opacityByUid.end()) ? 0 : opacityByUid[uid]; // Given windows A and B: @@ -3008,29 +3024,30 @@ InputDispatcher::TouchOcclusionInfo InputDispatcher::computeTouchOcclusionInfoLo std::string InputDispatcher::dumpWindowForTouchOcclusion(const WindowInfo* info, bool isTouchedWindow) const { - return StringPrintf(INDENT2 "* %spackage=%s/%" PRId32 ", id=%" PRId32 ", mode=%s, alpha=%.2f, " + return StringPrintf(INDENT2 "* %spackage=%s/%s, id=%" PRId32 ", mode=%s, alpha=%.2f, " "frame=[%" PRId32 ",%" PRId32 "][%" PRId32 ",%" PRId32 "], touchableRegion=%s, window={%s}, inputConfig={%s}, " "hasToken=%s, applicationInfo.name=%s, applicationInfo.token=%s\n", isTouchedWindow ? "[TOUCHED] " : "", info->packageName.c_str(), - info->ownerUid, info->id, toString(info->touchOcclusionMode).c_str(), - info->alpha, info->frameLeft, info->frameTop, info->frameRight, - info->frameBottom, dumpRegion(info->touchableRegion).c_str(), - info->name.c_str(), info->inputConfig.string().c_str(), - toString(info->token != nullptr), info->applicationInfo.name.c_str(), + info->ownerUid.toString().c_str(), info->id, + toString(info->touchOcclusionMode).c_str(), info->alpha, info->frameLeft, + info->frameTop, info->frameRight, info->frameBottom, + dumpRegion(info->touchableRegion).c_str(), info->name.c_str(), + info->inputConfig.string().c_str(), toString(info->token != nullptr), + info->applicationInfo.name.c_str(), binderToString(info->applicationInfo.token).c_str()); } bool InputDispatcher::isTouchTrustedLocked(const TouchOcclusionInfo& occlusionInfo) const { if (occlusionInfo.hasBlockingOcclusion) { - ALOGW("Untrusted touch due to occlusion by %s/%d", occlusionInfo.obscuringPackage.c_str(), - occlusionInfo.obscuringUid); + ALOGW("Untrusted touch due to occlusion by %s/%s", occlusionInfo.obscuringPackage.c_str(), + occlusionInfo.obscuringUid.toString().c_str()); return false; } if (occlusionInfo.obscuringOpacity > mMaximumObscuringOpacityForTouch) { - ALOGW("Untrusted touch due to occlusion by %s/%d (obscuring opacity = " + ALOGW("Untrusted touch due to occlusion by %s/%s (obscuring opacity = " "%.2f, maximum allowed = %.2f)", - occlusionInfo.obscuringPackage.c_str(), occlusionInfo.obscuringUid, + occlusionInfo.obscuringPackage.c_str(), occlusionInfo.obscuringUid.toString().c_str(), occlusionInfo.obscuringOpacity, mMaximumObscuringOpacityForTouch); return false; } @@ -3300,11 +3317,8 @@ void InputDispatcher::enqueueDispatchEntryLocked(const std::shared_ptr<Connectio if (!connection->inputState.trackKey(keyEntry, dispatchEntry->resolvedAction, dispatchEntry->resolvedFlags)) { - if (DEBUG_DISPATCH_CYCLE) { - ALOGD("channel '%s' ~ enqueueDispatchEntryLocked: skipping inconsistent key " - "event", - connection->getInputChannelName().c_str()); - } + LOG(WARNING) << "channel " << connection->getInputChannelName() + << "~ dropping inconsistent event: " << *dispatchEntry; return; // skip the inconsistent event } break; @@ -3357,11 +3371,8 @@ void InputDispatcher::enqueueDispatchEntryLocked(const std::shared_ptr<Connectio if (!connection->inputState.trackMotion(motionEntry, dispatchEntry->resolvedAction, dispatchEntry->resolvedFlags)) { - if (DEBUG_DISPATCH_CYCLE) { - ALOGD("channel '%s' ~ enqueueDispatchEntryLocked: skipping inconsistent motion " - "event", - connection->getInputChannelName().c_str()); - } + LOG(WARNING) << "channel " << connection->getInputChannelName() + << "~ dropping inconsistent event: " << *dispatchEntry; return; // skip the inconsistent event } @@ -3416,38 +3427,49 @@ void InputDispatcher::enqueueDispatchEntryLocked(const std::shared_ptr<Connectio } /** - * This function is purely for debugging. It helps us understand where the user interaction - * was taking place. For example, if user is touching launcher, we will see a log that user - * started interacting with launcher. In that example, the event would go to the wallpaper as well. - * We will see both launcher and wallpaper in that list. - * Once the interaction with a particular set of connections starts, no new logs will be printed - * until the set of interacted connections changes. + * This function is for debugging and metrics collection. It has two roles. * - * The following items are skipped, to reduce the logspam: - * ACTION_OUTSIDE: any windows that are receiving ACTION_OUTSIDE are not logged - * ACTION_UP: any windows that receive ACTION_UP are not logged (for both keys and motions). - * This includes situations like the soft BACK button key. When the user releases (lifts up the - * finger) the back button, then navigation bar will inject KEYCODE_BACK with ACTION_UP. - * Both of those ACTION_UP events would not be logged + * The first role is to log input interaction with windows, which helps determine what the user was + * interacting with. For example, if user is touching launcher, we will see an input_interaction log + * that user started interacting with launcher window, as well as any other window that received + * that gesture, such as the wallpaper or other spy windows. A new input_interaction is only logged + * when the set of tokens that received the event changes. It is not logged again as long as the + * user is interacting with the same windows. + * + * The second role is to track input device activity for metrics collection. For each input event, + * we report the set of UIDs that the input device interacted with to the policy. Unlike for the + * input_interaction logs, the device interaction is reported even when the set of interaction + * tokens do not change. + * + * For these purposes, we do not count ACTION_OUTSIDE, ACTION_UP and ACTION_CANCEL actions as + * interaction. This includes up and cancel events for both keys and motions. */ -void InputDispatcher::updateInteractionTokensLocked(const EventEntry& entry, - const std::vector<InputTarget>& targets) { +void InputDispatcher::processInteractionsLocked(const EventEntry& entry, + const std::vector<InputTarget>& targets) { + int32_t deviceId; + nsecs_t eventTime; // Skip ACTION_UP events, and all events other than keys and motions if (entry.type == EventEntry::Type::KEY) { const KeyEntry& keyEntry = static_cast<const KeyEntry&>(entry); if (keyEntry.action == AKEY_EVENT_ACTION_UP) { return; } + deviceId = keyEntry.deviceId; + eventTime = keyEntry.eventTime; } else if (entry.type == EventEntry::Type::MOTION) { const MotionEntry& motionEntry = static_cast<const MotionEntry&>(entry); if (motionEntry.action == AMOTION_EVENT_ACTION_UP || - motionEntry.action == AMOTION_EVENT_ACTION_CANCEL) { + motionEntry.action == AMOTION_EVENT_ACTION_CANCEL || + MotionEvent::getActionMasked(motionEntry.action) == AMOTION_EVENT_ACTION_POINTER_UP) { return; } + deviceId = motionEntry.deviceId; + eventTime = motionEntry.eventTime; } else { return; // Not a key or a motion } + std::set<gui::Uid> interactionUids; std::unordered_set<sp<IBinder>, StrongPointerHash<IBinder>> newConnectionTokens; std::vector<std::shared_ptr<Connection>> newConnections; for (const InputTarget& target : targets) { @@ -3462,7 +3484,18 @@ void InputDispatcher::updateInteractionTokensLocked(const EventEntry& entry, } newConnectionTokens.insert(std::move(token)); newConnections.emplace_back(connection); + if (target.windowHandle) { + interactionUids.emplace(target.windowHandle->getInfo()->ownerUid); + } } + + auto command = [this, deviceId, eventTime, uids = std::move(interactionUids)]() + REQUIRES(mLock) { + scoped_unlock unlock(mLock); + mPolicy.notifyDeviceInteraction(deviceId, eventTime, uids); + }; + postCommandLocked(std::move(command)); + if (newConnectionTokens == mInteractionConnectionTokens) { return; // no change } @@ -4135,11 +4168,11 @@ std::unique_ptr<MotionEntry> InputDispatcher::splitMotionEvent( } } - if (action == AMOTION_EVENT_ACTION_DOWN) { - LOG_ALWAYS_FATAL_IF(splitDownTime != originalMotionEntry.eventTime, - "Split motion event has mismatching downTime and eventTime for " - "ACTION_DOWN, motionEntry=%s, splitDownTime=%" PRId64, - originalMotionEntry.getDescription().c_str(), splitDownTime); + if (action == AMOTION_EVENT_ACTION_DOWN && splitDownTime != originalMotionEntry.eventTime) { + logDispatchStateLocked(); + LOG_ALWAYS_FATAL("Split motion event has mismatching downTime and eventTime for " + "ACTION_DOWN, motionEntry=%s, splitDownTime=%" PRId64, + originalMotionEntry.getDescription().c_str(), splitDownTime); } int32_t newId = mIdGenerator.nextId(); @@ -4323,7 +4356,7 @@ void InputDispatcher::notifyMotion(const NotifyMotionArgs& args) { args.actionButton, args.flags, args.metaState, args.buttonState, args.edgeFlags, args.xPrecision, args.yPrecision, args.xCursorPosition, args.yCursorPosition, args.downTime); - for (uint32_t i = 0; i < args.pointerCount; i++) { + for (uint32_t i = 0; i < args.getPointerCount(); i++) { ALOGD(" Pointer %d: id=%d, toolType=%s, x=%f, y=%f, pressure=%f, size=%f, " "touchMajor=%f, touchMinor=%f, toolMajor=%f, toolMinor=%f, orientation=%f", i, args.pointerProperties[i].id, @@ -4340,13 +4373,27 @@ void InputDispatcher::notifyMotion(const NotifyMotionArgs& args) { } } - Result<void> motionCheck = validateMotionEvent(args.action, args.actionButton, - args.pointerCount, args.pointerProperties); + Result<void> motionCheck = + validateMotionEvent(args.action, args.actionButton, args.getPointerCount(), + args.pointerProperties.data()); if (!motionCheck.ok()) { LOG(ERROR) << "Invalid event: " << args.dump() << "; reason: " << motionCheck.error(); return; } + if (DEBUG_VERIFY_EVENTS) { + auto [it, _] = + mVerifiersByDisplay.try_emplace(args.displayId, + StringPrintf("display %" PRId32, args.displayId)); + Result<void> result = + it->second.processMovement(args.deviceId, args.action, args.getPointerCount(), + args.pointerProperties.data(), args.pointerCoords.data(), + args.flags); + if (!result.ok()) { + LOG(FATAL) << "Bad stream: " << result.error() << " caused by " << args.dump(); + } + } + uint32_t policyFlags = args.policyFlags; policyFlags |= POLICY_FLAG_TRUSTED; @@ -4366,7 +4413,7 @@ void InputDispatcher::notifyMotion(const NotifyMotionArgs& args) { const auto touchStateIt = mTouchStatesByDisplay.find(args.displayId); if (touchStateIt != mTouchStatesByDisplay.end()) { const TouchState& touchState = touchStateIt->second; - if (touchState.deviceId == args.deviceId && touchState.isDown()) { + if (touchState.hasTouchingPointers(args.deviceId)) { policyFlags |= POLICY_FLAG_PASS_TO_USER; } } @@ -4386,8 +4433,8 @@ void InputDispatcher::notifyMotion(const NotifyMotionArgs& args) { args.metaState, args.buttonState, args.classification, displayTransform, args.xPrecision, args.yPrecision, args.xCursorPosition, args.yCursorPosition, displayTransform, - args.downTime, args.eventTime, args.pointerCount, - args.pointerProperties, args.pointerCoords); + args.downTime, args.eventTime, args.getPointerCount(), + args.pointerProperties.data(), args.pointerCoords.data()); policyFlags |= POLICY_FLAG_FILTERED; if (!mPolicy.filterInputEvent(event, policyFlags)) { @@ -4405,8 +4452,9 @@ void InputDispatcher::notifyMotion(const NotifyMotionArgs& args) { args.buttonState, args.classification, args.edgeFlags, args.xPrecision, args.yPrecision, args.xCursorPosition, args.yCursorPosition, - args.downTime, args.pointerCount, - args.pointerProperties, args.pointerCoords); + args.downTime, args.getPointerCount(), + args.pointerProperties.data(), + args.pointerCoords.data()); if (args.id != android::os::IInputConstants::INVALID_INPUT_EVENT_ID && IdGenerator::getSource(args.id) == IdGenerator::Source::INPUT_READER && @@ -4515,7 +4563,7 @@ void InputDispatcher::notifyPointerCaptureChanged(const NotifyPointerCaptureChan } InputEventInjectionResult InputDispatcher::injectInputEvent(const InputEvent* event, - std::optional<int32_t> targetUid, + std::optional<gui::Uid> targetUid, InputEventInjectionSync syncMode, std::chrono::milliseconds timeout, uint32_t policyFlags) { @@ -4526,7 +4574,7 @@ InputEventInjectionResult InputDispatcher::injectInputEvent(const InputEvent* ev } if (debugInboundEventDetails()) { - LOG(DEBUG) << __func__ << ": targetUid=" << toString(targetUid) + LOG(DEBUG) << __func__ << ": targetUid=" << toString(targetUid, &uidString) << ", syncMode=" << ftl::enum_string(syncMode) << ", timeout=" << timeout.count() << "ms, policyFlags=0x" << std::hex << policyFlags << std::dec << ", event=" << *event; @@ -4971,8 +5019,8 @@ bool InputDispatcher::canWindowReceiveMotionLocked(const sp<WindowInfoHandle>& w ALOGD("%s", log.c_str()); } } - ALOGW("Dropping untrusted touch event due to %s/%d", occlusionInfo.obscuringPackage.c_str(), - occlusionInfo.obscuringUid); + ALOGW("Dropping untrusted touch event due to %s/%s", occlusionInfo.obscuringPackage.c_str(), + occlusionInfo.obscuringUid.toString().c_str()); return false; } @@ -5334,15 +5382,16 @@ void InputDispatcher::setInputFilterEnabled(bool enabled) { mLooper->wake(); } -bool InputDispatcher::setInTouchMode(bool inTouchMode, int32_t pid, int32_t uid, bool hasPermission, - int32_t displayId) { +bool InputDispatcher::setInTouchMode(bool inTouchMode, gui::Pid pid, gui::Uid uid, + bool hasPermission, int32_t displayId) { bool needWake = false; { std::scoped_lock lock(mLock); ALOGD_IF(DEBUG_TOUCH_MODE, - "Request to change touch mode to %s (calling pid=%d, uid=%d, " + "Request to change touch mode to %s (calling pid=%s, uid=%s, " "hasPermission=%s, target displayId=%d, mTouchModePerDisplay[displayId]=%s)", - toString(inTouchMode), pid, uid, toString(hasPermission), displayId, + toString(inTouchMode), pid.toString().c_str(), uid.toString().c_str(), + toString(hasPermission), displayId, mTouchModePerDisplay.count(displayId) == 0 ? "not set" : std::to_string(mTouchModePerDisplay[displayId]).c_str()); @@ -5354,9 +5403,9 @@ bool InputDispatcher::setInTouchMode(bool inTouchMode, int32_t pid, int32_t uid, if (!hasPermission) { if (!focusedWindowIsOwnedByLocked(pid, uid) && !recentWindowsAreOwnedByLocked(pid, uid)) { - ALOGD("Touch mode switch rejected, caller (pid=%d, uid=%d) doesn't own the focused " + ALOGD("Touch mode switch rejected, caller (pid=%s, uid=%s) doesn't own the focused " "window nor none of the previously interacted window", - pid, uid); + pid.toString().c_str(), uid.toString().c_str()); return false; } } @@ -5372,7 +5421,7 @@ bool InputDispatcher::setInTouchMode(bool inTouchMode, int32_t pid, int32_t uid, return true; } -bool InputDispatcher::focusedWindowIsOwnedByLocked(int32_t pid, int32_t uid) { +bool InputDispatcher::focusedWindowIsOwnedByLocked(gui::Pid pid, gui::Uid uid) { const sp<IBinder> focusedToken = mFocusResolver.getFocusedWindowToken(mFocusedDisplayId); if (focusedToken == nullptr) { return false; @@ -5381,7 +5430,7 @@ bool InputDispatcher::focusedWindowIsOwnedByLocked(int32_t pid, int32_t uid) { return isWindowOwnedBy(windowHandle, pid, uid); } -bool InputDispatcher::recentWindowsAreOwnedByLocked(int32_t pid, int32_t uid) { +bool InputDispatcher::recentWindowsAreOwnedByLocked(gui::Pid pid, gui::Uid uid) { return std::find_if(mInteractionConnectionTokens.begin(), mInteractionConnectionTokens.end(), [&](const sp<IBinder>& connectionToken) REQUIRES(mLock) { const sp<WindowInfoHandle> windowHandle = @@ -5426,14 +5475,22 @@ bool InputDispatcher::transferTouchFocus(const sp<IBinder>& fromToken, const sp< // Find the target touch state and touched window by fromToken. auto [state, touchedWindow, displayId] = findTouchStateWindowAndDisplayLocked(fromToken); + if (state == nullptr || touchedWindow == nullptr) { - ALOGD("Focus transfer failed because from window is not being touched."); + ALOGD("Touch transfer failed because from window is not being touched."); return false; } + std::set<int32_t> deviceIds = touchedWindow->getTouchingDeviceIds(); + if (deviceIds.size() != 1) { + LOG(DEBUG) << "Can't transfer touch. Currently touching devices: " << dumpSet(deviceIds) + << " for window: " << touchedWindow->dump(); + return false; + } + const int32_t deviceId = *deviceIds.begin(); sp<WindowInfoHandle> toWindowHandle = getWindowHandleLocked(toToken, displayId); if (toWindowHandle == nullptr) { - ALOGW("Cannot transfer focus because to window not found."); + ALOGW("Cannot transfer touch because to window not found."); return false; } @@ -5445,7 +5502,7 @@ bool InputDispatcher::transferTouchFocus(const sp<IBinder>& fromToken, const sp< // Erase old window. ftl::Flags<InputTarget::Flags> oldTargetFlags = touchedWindow->targetFlags; - std::bitset<MAX_POINTER_ID + 1> pointerIds = touchedWindow->pointerIds; + std::bitset<MAX_POINTER_ID + 1> pointerIds = touchedWindow->getTouchingPointers(deviceId); sp<WindowInfoHandle> fromWindowHandle = touchedWindow->windowHandle; state->removeWindowByToken(fromToken); @@ -5456,7 +5513,8 @@ bool InputDispatcher::transferTouchFocus(const sp<IBinder>& fromToken, const sp< if (canReceiveForegroundTouches(*toWindowHandle->getInfo())) { newTargetFlags |= InputTarget::Flags::FOREGROUND; } - state->addOrUpdateWindow(toWindowHandle, newTargetFlags, pointerIds, downTimeInTarget); + state->addOrUpdateWindow(toWindowHandle, newTargetFlags, deviceId, pointerIds, + downTimeInTarget); // Store the dragging window. if (isDragDrop) { @@ -5475,16 +5533,15 @@ bool InputDispatcher::transferTouchFocus(const sp<IBinder>& fromToken, const sp< std::shared_ptr<Connection> toConnection = getConnectionLocked(toToken); if (fromConnection != nullptr && toConnection != nullptr) { fromConnection->inputState.mergePointerStateTo(toConnection->inputState); - CancelationOptions - options(CancelationOptions::Mode::CANCEL_POINTER_EVENTS, - "transferring touch focus from this window to another window"); + CancelationOptions options(CancelationOptions::Mode::CANCEL_POINTER_EVENTS, + "transferring touch from this window to another window"); synthesizeCancelationEventsForConnectionLocked(fromConnection, options); synthesizePointerDownEventsForConnectionLocked(downTimeInTarget, toConnection, newTargetFlags); // Check if the wallpaper window should deliver the corresponding event. transferWallpaperTouch(oldTargetFlags, newTargetFlags, fromWindowHandle, toWindowHandle, - *state, pointerIds); + *state, deviceId, pointerIds); } } // release lock @@ -5666,10 +5723,11 @@ void InputDispatcher::dumpDispatchStateLocked(std::string& dump) const { windowInfo->applicationInfo.name.c_str(), binderToString(windowInfo->applicationInfo.token).c_str()); dump += dumpRegion(windowInfo->touchableRegion); - dump += StringPrintf(", ownerPid=%d, ownerUid=%d, dispatchingTimeout=%" PRId64 + dump += StringPrintf(", ownerPid=%s, ownerUid=%s, dispatchingTimeout=%" PRId64 "ms, hasToken=%s, " "touchOcclusionMode=%s\n", - windowInfo->ownerPid, windowInfo->ownerUid, + windowInfo->ownerPid.toString().c_str(), + windowInfo->ownerUid.toString().c_str(), millis(windowInfo->dispatchingTimeout), binderToString(windowInfo->token).c_str(), toString(windowInfo->touchOcclusionMode).c_str()); @@ -5861,7 +5919,7 @@ Result<std::unique_ptr<InputChannel>> InputDispatcher::createInputChannel(const Result<std::unique_ptr<InputChannel>> InputDispatcher::createInputMonitor(int32_t displayId, const std::string& name, - int32_t pid) { + gui::Pid pid) { std::shared_ptr<InputChannel> serverChannel; std::unique_ptr<InputChannel> clientChannel; status_t result = openInputChannelPair(name, serverChannel, clientChannel); @@ -5967,20 +6025,28 @@ status_t InputDispatcher::pilferPointersLocked(const sp<IBinder>& token) { } auto [statePtr, windowPtr, displayId] = findTouchStateWindowAndDisplayLocked(token); - if (statePtr == nullptr || windowPtr == nullptr || windowPtr->pointerIds.none()) { + if (statePtr == nullptr || windowPtr == nullptr) { ALOGW("Attempted to pilfer points from a channel without any on-going pointer streams." " Ignoring."); return BAD_VALUE; } + std::set<int32_t> deviceIds = windowPtr->getTouchingDeviceIds(); + if (deviceIds.size() != 1) { + LOG(WARNING) << "Can't pilfer. Currently touching devices: " << dumpSet(deviceIds) + << " in window: " << windowPtr->dump(); + return BAD_VALUE; + } + const int32_t deviceId = *deviceIds.begin(); TouchState& state = *statePtr; TouchedWindow& window = *windowPtr; // Send cancel events to all the input channels we're stealing from. CancelationOptions options(CancelationOptions::Mode::CANCEL_POINTER_EVENTS, "input channel stole pointer stream"); - options.deviceId = state.deviceId; + options.deviceId = deviceId; options.displayId = displayId; - options.pointerIds = window.pointerIds; + std::bitset<MAX_POINTER_ID + 1> pointerIds = window.getTouchingPointers(deviceId); + options.pointerIds = pointerIds; std::string canceledWindows; for (const TouchedWindow& w : state.windows) { const std::shared_ptr<InputChannel> channel = @@ -5997,9 +6063,9 @@ status_t InputDispatcher::pilferPointersLocked(const sp<IBinder>& token) { // Prevent the gesture from being sent to any other windows. // This only blocks relevant pointers to be sent to other windows - window.pilferedPointerIds |= window.pointerIds; + window.addPilferingPointers(deviceId, pointerIds); - state.cancelPointersForWindowsExcept(window.pointerIds, token); + state.cancelPointersForWindowsExcept(deviceId, pointerIds, token); return OK; } @@ -6053,7 +6119,7 @@ void InputDispatcher::setDisplayEligibilityForPointerCapture(int32_t displayId, } // release lock } -std::optional<int32_t> InputDispatcher::findMonitorPidByTokenLocked(const sp<IBinder>& token) { +std::optional<gui::Pid> InputDispatcher::findMonitorPidByTokenLocked(const sp<IBinder>& token) { for (const auto& [_, monitors] : mGlobalMonitorsByDisplay) { for (const Monitor& monitor : monitors) { if (monitor.inputChannel->getConnectionToken() == token) { @@ -6273,7 +6339,7 @@ void InputDispatcher::doInterceptKeyBeforeDispatchingCommand(const sp<IBinder>& } void InputDispatcher::sendWindowUnresponsiveCommandLocked(const sp<IBinder>& token, - std::optional<int32_t> pid, + std::optional<gui::Pid> pid, std::string reason) { auto command = [this, token, pid, r = std::move(reason)]() REQUIRES(mLock) { scoped_unlock unlock(mLock); @@ -6283,7 +6349,7 @@ void InputDispatcher::sendWindowUnresponsiveCommandLocked(const sp<IBinder>& tok } void InputDispatcher::sendWindowResponsiveCommandLocked(const sp<IBinder>& token, - std::optional<int32_t> pid) { + std::optional<gui::Pid> pid) { auto command = [this, token, pid]() REQUIRES(mLock) { scoped_unlock unlock(mLock); mPolicy.notifyWindowResponsive(token, pid); @@ -6299,7 +6365,7 @@ void InputDispatcher::sendWindowResponsiveCommandLocked(const sp<IBinder>& token void InputDispatcher::processConnectionUnresponsiveLocked(const Connection& connection, std::string reason) { const sp<IBinder>& connectionToken = connection.inputChannel->getConnectionToken(); - std::optional<int32_t> pid; + std::optional<gui::Pid> pid; if (connection.monitor) { ALOGW("Monitor %s is unresponsive: %s", connection.inputChannel->getName().c_str(), reason.c_str()); @@ -6321,7 +6387,7 @@ void InputDispatcher::processConnectionUnresponsiveLocked(const Connection& conn */ void InputDispatcher::processConnectionResponsiveLocked(const Connection& connection) { const sp<IBinder>& connectionToken = connection.inputChannel->getConnectionToken(); - std::optional<int32_t> pid; + std::optional<gui::Pid> pid; if (connection.monitor) { pid = findMonitorPidByTokenLocked(connectionToken); } else { @@ -6547,7 +6613,7 @@ void InputDispatcher::traceWaitQueueLength(const Connection& connection) { } } -void InputDispatcher::dump(std::string& dump) { +void InputDispatcher::dump(std::string& dump) const { std::scoped_lock _l(mLock); dump += "Input Dispatcher State:\n"; @@ -6572,7 +6638,7 @@ void InputDispatcher::monitor() { * this method can be safely called from any thread, as long as you've ensured that * the work you are interested in completing has already been queued. */ -bool InputDispatcher::waitForIdle() { +bool InputDispatcher::waitForIdle() const { /** * Timeout should represent the longest possible time that a device might spend processing * events and commands. @@ -6691,6 +6757,7 @@ void InputDispatcher::displayRemoved(int32_t displayId) { std::erase(mIneligibleDisplaysForPointerCapture, displayId); // Remove the associated touch mode state. mTouchModePerDisplay.erase(displayId); + mVerifiersByDisplay.erase(displayId); } // release lock // Wake up poll loop since it may need to make new input dispatching choices. @@ -6785,7 +6852,7 @@ void InputDispatcher::setMonitorDispatchingTimeoutForTest(std::chrono::nanosecon void InputDispatcher::slipWallpaperTouch(ftl::Flags<InputTarget::Flags> targetFlags, const sp<WindowInfoHandle>& oldWindowHandle, const sp<WindowInfoHandle>& newWindowHandle, - TouchState& state, int32_t pointerId, + TouchState& state, int32_t deviceId, int32_t pointerId, std::vector<InputTarget>& targets) const { std::bitset<MAX_POINTER_ID + 1> pointerIds; pointerIds.set(pointerId); @@ -6807,8 +6874,8 @@ void InputDispatcher::slipWallpaperTouch(ftl::Flags<InputTarget::Flags> targetFl addWindowTargetLocked(oldWallpaper, oldTouchedWindow.targetFlags | InputTarget::Flags::DISPATCH_AS_SLIPPERY_EXIT, - pointerIds, oldTouchedWindow.firstDownTimeInTarget, targets); - state.removeTouchedPointerFromWindow(pointerId, oldWallpaper); + pointerIds, oldTouchedWindow.getDownTimeInTarget(deviceId), targets); + state.removeTouchingPointerFromWindow(deviceId, pointerId, oldWallpaper); } if (newWallpaper != nullptr) { @@ -6816,7 +6883,7 @@ void InputDispatcher::slipWallpaperTouch(ftl::Flags<InputTarget::Flags> targetFl InputTarget::Flags::DISPATCH_AS_SLIPPERY_ENTER | InputTarget::Flags::WINDOW_IS_OBSCURED | InputTarget::Flags::WINDOW_IS_PARTIALLY_OBSCURED, - pointerIds); + deviceId, pointerIds); } } @@ -6824,7 +6891,7 @@ void InputDispatcher::transferWallpaperTouch(ftl::Flags<InputTarget::Flags> oldT ftl::Flags<InputTarget::Flags> newTargetFlags, const sp<WindowInfoHandle> fromWindowHandle, const sp<WindowInfoHandle> toWindowHandle, - TouchState& state, + TouchState& state, int32_t deviceId, std::bitset<MAX_POINTER_ID + 1> pointerIds) { const bool oldHasWallpaper = oldTargetFlags.test(InputTarget::Flags::FOREGROUND) && fromWindowHandle->getInfo()->inputConfig.test( @@ -6854,7 +6921,8 @@ void InputDispatcher::transferWallpaperTouch(ftl::Flags<InputTarget::Flags> oldT oldTargetFlags & (InputTarget::Flags::SPLIT | InputTarget::Flags::DISPATCH_AS_IS); wallpaperFlags |= InputTarget::Flags::WINDOW_IS_OBSCURED | InputTarget::Flags::WINDOW_IS_PARTIALLY_OBSCURED; - state.addOrUpdateWindow(newWallpaper, wallpaperFlags, pointerIds, downTimeInTarget); + state.addOrUpdateWindow(newWallpaper, wallpaperFlags, deviceId, pointerIds, + downTimeInTarget); std::shared_ptr<Connection> wallpaperConnection = getConnectionLocked(newWallpaper->getToken()); if (wallpaperConnection != nullptr) { diff --git a/services/inputflinger/dispatcher/InputDispatcher.h b/services/inputflinger/dispatcher/InputDispatcher.h index 6b22f2f24f..2b8b37e42a 100644 --- a/services/inputflinger/dispatcher/InputDispatcher.h +++ b/services/inputflinger/dispatcher/InputDispatcher.h @@ -87,9 +87,9 @@ public: std::chrono::nanoseconds staleEventTimeout); ~InputDispatcher() override; - void dump(std::string& dump) override; + void dump(std::string& dump) const override; void monitor() override; - bool waitForIdle() override; + bool waitForIdle() const override; status_t start() override; status_t stop() override; @@ -104,7 +104,7 @@ public: void notifyPointerCaptureChanged(const NotifyPointerCaptureChangedArgs& args) override; android::os::InputEventInjectionResult injectInputEvent( - const InputEvent* event, std::optional<int32_t> targetUid, + const InputEvent* event, std::optional<gui::Uid> targetUid, android::os::InputEventInjectionSync syncMode, std::chrono::milliseconds timeout, uint32_t policyFlags) override; @@ -119,7 +119,7 @@ public: void setFocusedDisplay(int32_t displayId) override; void setInputDispatchMode(bool enabled, bool frozen) override; void setInputFilterEnabled(bool enabled) override; - bool setInTouchMode(bool inTouchMode, int32_t pid, int32_t uid, bool hasPermission, + bool setInTouchMode(bool inTouchMode, gui::Pid pid, gui::Uid uid, bool hasPermission, int32_t displayId) override; void setMaximumObscuringOpacityForTouch(float opacity) override; @@ -132,7 +132,7 @@ public: void setFocusedWindow(const android::gui::FocusRequest&) override; base::Result<std::unique_ptr<InputChannel>> createInputMonitor(int32_t displayId, const std::string& name, - int32_t pid) override; + gui::Pid pid) override; status_t removeInputChannel(const sp<IBinder>& connectionToken) override; status_t pilferPointers(const sp<IBinder>& token) override; void requestPointerCapture(const sp<IBinder>& windowToken, bool enabled) override; @@ -169,10 +169,10 @@ private: InputDispatcherPolicyInterface& mPolicy; android::InputDispatcherConfiguration mConfig GUARDED_BY(mLock); - std::mutex mLock; + mutable std::mutex mLock; std::condition_variable mDispatcherIsAlive; - std::condition_variable mDispatcherEnteredIdle; + mutable std::condition_variable mDispatcherEnteredIdle; sp<Looper> mLooper; @@ -202,7 +202,7 @@ private: DropReason mLastDropReason GUARDED_BY(mLock); - const IdGenerator mIdGenerator; + const IdGenerator mIdGenerator GUARDED_BY(mLock); int64_t mWindowInfosVsyncId GUARDED_BY(mLock); @@ -271,7 +271,7 @@ private: mConnectionsByToken GUARDED_BY(mLock); // Find a monitor pid by the provided token. - std::optional<int32_t> findMonitorPidByTokenLocked(const sp<IBinder>& token) REQUIRES(mLock); + std::optional<gui::Pid> findMonitorPidByTokenLocked(const sp<IBinder>& token) REQUIRES(mLock); // Input channels that will receive a copy of all input events sent to the provided display. std::unordered_map<int32_t, std::vector<Monitor>> mGlobalMonitorsByDisplay GUARDED_BY(mLock); @@ -447,8 +447,8 @@ private: // when switching touch mode state). std::unordered_set<sp<IBinder>, StrongPointerHash<IBinder>> mInteractionConnectionTokens GUARDED_BY(mLock); - void updateInteractionTokensLocked(const EventEntry& entry, - const std::vector<InputTarget>& targets) REQUIRES(mLock); + void processInteractionsLocked(const EventEntry& entry, const std::vector<InputTarget>& targets) + REQUIRES(mLock); // Dispatch inbound events. bool dispatchConfigurationChangedLocked(nsecs_t currentTime, @@ -522,10 +522,10 @@ private: void processConnectionResponsiveLocked(const Connection& connection) REQUIRES(mLock); void sendWindowUnresponsiveCommandLocked(const sp<IBinder>& connectionToken, - std::optional<int32_t> pid, std::string reason) + std::optional<gui::Pid> pid, std::string reason) REQUIRES(mLock); void sendWindowResponsiveCommandLocked(const sp<IBinder>& connectionToken, - std::optional<int32_t> pid) REQUIRES(mLock); + std::optional<gui::Pid> pid) REQUIRES(mLock); // Optimization: AnrTracker is used to quickly find which connection is due for a timeout next. // AnrTracker must be kept in-sync with all responsive connection.waitQueues. @@ -574,7 +574,7 @@ private: bool hasBlockingOcclusion; float obscuringOpacity; std::string obscuringPackage; - int32_t obscuringUid; + gui::Uid obscuringUid = gui::Uid::INVALID; std::vector<std::string> debugInfo; }; @@ -649,7 +649,7 @@ private: // splitDownTime refers to the time of first 'down' event on that particular target std::unique_ptr<MotionEntry> splitMotionEvent(const MotionEntry& originalMotionEntry, std::bitset<MAX_POINTER_ID + 1> pointerIds, - nsecs_t splitDownTime); + nsecs_t splitDownTime) REQUIRES(mLock); // Reset and drop everything the dispatcher is doing. void resetAndDropEverythingLocked(const char* reason) REQUIRES(mLock); @@ -683,6 +683,7 @@ private: const std::string& reason) REQUIRES(mLock); void updateLastAnrStateLocked(const std::string& windowLabel, const std::string& reason) REQUIRES(mLock); + std::map<int32_t /*displayId*/, InputVerifier> mVerifiersByDisplay; bool afterKeyEventLockedInterruptable(const std::shared_ptr<Connection>& connection, DispatchEntry* dispatchEntry, KeyEntry& keyEntry, bool handled) REQUIRES(mLock); @@ -702,22 +703,22 @@ private: void traceWaitQueueLength(const Connection& connection); // Check window ownership - bool focusedWindowIsOwnedByLocked(int32_t pid, int32_t uid) REQUIRES(mLock); - bool recentWindowsAreOwnedByLocked(int32_t pid, int32_t uid) REQUIRES(mLock); + bool focusedWindowIsOwnedByLocked(gui::Pid pid, gui::Uid uid) REQUIRES(mLock); + bool recentWindowsAreOwnedByLocked(gui::Pid pid, gui::Uid uid) REQUIRES(mLock); sp<InputReporterInterface> mReporter; void slipWallpaperTouch(ftl::Flags<InputTarget::Flags> targetFlags, const sp<android::gui::WindowInfoHandle>& oldWindowHandle, const sp<android::gui::WindowInfoHandle>& newWindowHandle, - TouchState& state, int32_t pointerId, + TouchState& state, int32_t deviceId, int32_t pointerId, std::vector<InputTarget>& targets) const REQUIRES(mLock); void transferWallpaperTouch(ftl::Flags<InputTarget::Flags> oldTargetFlags, ftl::Flags<InputTarget::Flags> newTargetFlags, const sp<android::gui::WindowInfoHandle> fromWindowHandle, const sp<android::gui::WindowInfoHandle> toWindowHandle, - TouchState& state, std::bitset<MAX_POINTER_ID + 1> pointerIds) - REQUIRES(mLock); + TouchState& state, int32_t deviceId, + std::bitset<MAX_POINTER_ID + 1> pointerIds) REQUIRES(mLock); sp<android::gui::WindowInfoHandle> findWallpaperWindowBelow( const sp<android::gui::WindowInfoHandle>& windowHandle) const REQUIRES(mLock); diff --git a/services/inputflinger/dispatcher/InputState.cpp b/services/inputflinger/dispatcher/InputState.cpp index 4652c2dd12..2fcb89a731 100644 --- a/services/inputflinger/dispatcher/InputState.cpp +++ b/services/inputflinger/dispatcher/InputState.cpp @@ -93,11 +93,7 @@ bool InputState::trackMotion(const MotionEntry& entry, int32_t action, int32_t f mMotionMementos.erase(mMotionMementos.begin() + index); return true; } - if (DEBUG_OUTBOUND_EVENT_DETAILS) { - ALOGD("Dropping inconsistent motion up or cancel event: deviceId=%d, source=%08x, " - "displayId=%" PRId32 ", actionMasked=%d", - entry.deviceId, entry.source, entry.displayId, actionMasked); - } + return false; } @@ -150,11 +146,7 @@ bool InputState::trackMotion(const MotionEntry& entry, int32_t action, int32_t f return true; } } - if (DEBUG_OUTBOUND_EVENT_DETAILS) { - ALOGD("Dropping inconsistent motion pointer up/down or move event: " - "deviceId=%d, source=%08x, displayId=%" PRId32 ", actionMasked=%d", - entry.deviceId, entry.source, entry.displayId, actionMasked); - } + return false; } @@ -164,11 +156,7 @@ bool InputState::trackMotion(const MotionEntry& entry, int32_t action, int32_t f mMotionMementos.erase(mMotionMementos.begin() + index); return true; } - if (DEBUG_OUTBOUND_EVENT_DETAILS) { - ALOGD("Dropping inconsistent motion hover exit event: deviceId=%d, source=%08x, " - "displayId=%" PRId32, - entry.deviceId, entry.source, entry.displayId); - } + return false; } diff --git a/services/inputflinger/dispatcher/InputTarget.h b/services/inputflinger/dispatcher/InputTarget.h index 7b12f81c4e..3bf8b68f0e 100644 --- a/services/inputflinger/dispatcher/InputTarget.h +++ b/services/inputflinger/dispatcher/InputTarget.h @@ -17,6 +17,7 @@ #pragma once #include <ftl/flags.h> +#include <gui/WindowInfo.h> #include <gui/constants.h> #include <input/InputTransport.h> #include <ui/Transform.h> @@ -114,6 +115,10 @@ struct InputTarget { // Transform per pointerId. ui::Transform pointerTransforms[MAX_POINTERS]; + // The window that this input target is being dispatched to. It is possible for this to be + // null for cases like global monitors. + sp<gui::WindowInfoHandle> windowHandle; + void addPointers(std::bitset<MAX_POINTER_ID + 1> pointerIds, const ui::Transform& transform); void setDefaultPointerTransform(const ui::Transform& transform); diff --git a/services/inputflinger/dispatcher/Monitor.cpp b/services/inputflinger/dispatcher/Monitor.cpp index 43a82d5cca..204791eb03 100644 --- a/services/inputflinger/dispatcher/Monitor.cpp +++ b/services/inputflinger/dispatcher/Monitor.cpp @@ -19,7 +19,7 @@ namespace android::inputdispatcher { // --- Monitor --- -Monitor::Monitor(const std::shared_ptr<InputChannel>& inputChannel, int32_t pid) +Monitor::Monitor(const std::shared_ptr<InputChannel>& inputChannel, gui::Pid pid) : inputChannel(inputChannel), pid(pid) {} } // namespace android::inputdispatcher diff --git a/services/inputflinger/dispatcher/Monitor.h b/services/inputflinger/dispatcher/Monitor.h index 7b511915d0..1b1eb3a593 100644 --- a/services/inputflinger/dispatcher/Monitor.h +++ b/services/inputflinger/dispatcher/Monitor.h @@ -16,6 +16,7 @@ #pragma once +#include <gui/PidUid.h> #include <input/InputTransport.h> namespace android::inputdispatcher { @@ -23,9 +24,9 @@ namespace android::inputdispatcher { struct Monitor { std::shared_ptr<InputChannel> inputChannel; // never null - int32_t pid; + gui::Pid pid; - explicit Monitor(const std::shared_ptr<InputChannel>& inputChannel, int32_t pid); + explicit Monitor(const std::shared_ptr<InputChannel>& inputChannel, gui::Pid pid); }; } // namespace android::inputdispatcher diff --git a/services/inputflinger/dispatcher/TouchState.cpp b/services/inputflinger/dispatcher/TouchState.cpp index 0a61d48165..dadfdc122a 100644 --- a/services/inputflinger/dispatcher/TouchState.cpp +++ b/services/inputflinger/dispatcher/TouchState.cpp @@ -31,18 +31,34 @@ void TouchState::reset() { *this = TouchState(); } -void TouchState::removeTouchedPointer(int32_t pointerId) { +std::set<int32_t> TouchState::getActiveDeviceIds() const { + std::set<int32_t> out; + for (const TouchedWindow& w : windows) { + std::set<int32_t> deviceIds = w.getActiveDeviceIds(); + out.insert(deviceIds.begin(), deviceIds.end()); + } + return out; +} + +bool TouchState::hasTouchingPointers(int32_t deviceId) const { + return std::any_of(windows.begin(), windows.end(), [&](const TouchedWindow& window) { + return window.hasTouchingPointers(deviceId); + }); +} + +void TouchState::removeTouchingPointer(int32_t removedDeviceId, int32_t pointerId) { for (TouchedWindow& touchedWindow : windows) { - touchedWindow.removeTouchingPointer(pointerId); + touchedWindow.removeTouchingPointer(removedDeviceId, pointerId); } clearWindowsWithoutPointers(); } -void TouchState::removeTouchedPointerFromWindow( - int32_t pointerId, const sp<android::gui::WindowInfoHandle>& windowHandle) { +void TouchState::removeTouchingPointerFromWindow( + int32_t removedDeviceId, int32_t pointerId, + const sp<android::gui::WindowInfoHandle>& windowHandle) { for (TouchedWindow& touchedWindow : windows) { if (touchedWindow.windowHandle == windowHandle) { - touchedWindow.removeTouchingPointer(pointerId); + touchedWindow.removeTouchingPointer(removedDeviceId, pointerId); clearWindowsWithoutPointers(); return; } @@ -58,13 +74,14 @@ void TouchState::clearHoveringPointers() { void TouchState::clearWindowsWithoutPointers() { std::erase_if(windows, [](const TouchedWindow& w) { - return w.pointerIds.none() && !w.hasHoveringPointers(); + return !w.hasTouchingPointers() && !w.hasHoveringPointers(); }); } void TouchState::addOrUpdateWindow(const sp<WindowInfoHandle>& windowHandle, ftl::Flags<InputTarget::Flags> targetFlags, - std::bitset<MAX_POINTER_ID + 1> pointerIds, + int32_t addedDeviceId, + std::bitset<MAX_POINTER_ID + 1> touchingPointerIds, std::optional<nsecs_t> firstDownTimeInTarget) { for (TouchedWindow& touchedWindow : windows) { // We do not compare windows by token here because two windows that share the same token @@ -75,11 +92,11 @@ void TouchState::addOrUpdateWindow(const sp<WindowInfoHandle>& windowHandle, touchedWindow.targetFlags.clear(InputTarget::Flags::DISPATCH_AS_IS); } // For cases like hover enter/exit or DISPATCH_AS_OUTSIDE a touch window might not have - // downTime set initially. Need to update existing window when an pointer is down for - // the window. - touchedWindow.pointerIds |= pointerIds; - if (!touchedWindow.firstDownTimeInTarget.has_value()) { - touchedWindow.firstDownTimeInTarget = firstDownTimeInTarget; + // downTime set initially. Need to update existing window when a pointer is down for the + // window. + touchedWindow.addTouchingPointers(addedDeviceId, touchingPointerIds); + if (firstDownTimeInTarget) { + touchedWindow.trySetDownTimeInTarget(addedDeviceId, *firstDownTimeInTarget); } return; } @@ -87,8 +104,10 @@ void TouchState::addOrUpdateWindow(const sp<WindowInfoHandle>& windowHandle, TouchedWindow touchedWindow; touchedWindow.windowHandle = windowHandle; touchedWindow.targetFlags = targetFlags; - touchedWindow.pointerIds = pointerIds; - touchedWindow.firstDownTimeInTarget = firstDownTimeInTarget; + touchedWindow.addTouchingPointers(addedDeviceId, touchingPointerIds); + if (firstDownTimeInTarget) { + touchedWindow.trySetDownTimeInTarget(addedDeviceId, *firstDownTimeInTarget); + } windows.push_back(touchedWindow); } @@ -130,12 +149,12 @@ void TouchState::filterNonAsIsTouchWindows() { } } -void TouchState::cancelPointersForWindowsExcept(std::bitset<MAX_POINTER_ID + 1> pointerIds, +void TouchState::cancelPointersForWindowsExcept(int32_t touchedDeviceId, + std::bitset<MAX_POINTER_ID + 1> pointerIds, const sp<IBinder>& token) { - if (pointerIds.none()) return; - std::for_each(windows.begin(), windows.end(), [&pointerIds, &token](TouchedWindow& w) { + std::for_each(windows.begin(), windows.end(), [&](TouchedWindow& w) { if (w.windowHandle->getToken() != token) { - w.pointerIds &= ~pointerIds; + w.removeTouchingPointers(touchedDeviceId, pointerIds); } }); clearWindowsWithoutPointers(); @@ -149,24 +168,29 @@ void TouchState::cancelPointersForWindowsExcept(std::bitset<MAX_POINTER_ID + 1> */ void TouchState::cancelPointersForNonPilferingWindows() { // First, find all pointers that are being pilfered, across all windows - std::bitset<MAX_POINTER_ID + 1> allPilferedPointerIds; - std::for_each(windows.begin(), windows.end(), [&allPilferedPointerIds](const TouchedWindow& w) { - allPilferedPointerIds |= w.pilferedPointerIds; - }); + std::map<int32_t /*deviceId*/, std::bitset<MAX_POINTER_ID + 1>> allPilferedPointerIdsByDevice; + for (const TouchedWindow& w : windows) { + for (const auto& [iterDeviceId, pilferedPointerIds] : w.getPilferingPointers()) { + allPilferedPointerIdsByDevice[iterDeviceId] |= pilferedPointerIds; + } + }; // Optimization: most of the time, pilfering does not occur - if (allPilferedPointerIds.none()) return; + if (allPilferedPointerIdsByDevice.empty()) return; // Now, remove all pointers from every window that's being pilfered by other windows. // For example, if window A is pilfering pointer 1 (only), and window B is pilfering pointer 2 // (only), the remove pointer 2 from window A and pointer 1 from window B. Usually, the set of // pilfered pointers will be disjoint across all windows, but there's no reason to cause that // limitation here. - std::for_each(windows.begin(), windows.end(), [&allPilferedPointerIds](TouchedWindow& w) { - std::bitset<MAX_POINTER_ID + 1> pilferedByOtherWindows = - w.pilferedPointerIds ^ allPilferedPointerIds; - w.pointerIds &= ~pilferedByOtherWindows; - }); + for (const auto& [iterDeviceId, allPilferedPointerIds] : allPilferedPointerIdsByDevice) { + std::for_each(windows.begin(), windows.end(), [&](TouchedWindow& w) { + std::bitset<MAX_POINTER_ID + 1> pilferedByOtherWindows = + w.getPilferingPointers(iterDeviceId) ^ allPilferedPointerIds; + // Remove all pointers pilfered by other windows + w.removeTouchingPointers(iterDeviceId, pilferedByOtherWindows); + }); + } clearWindowsWithoutPointers(); } @@ -216,7 +240,7 @@ const TouchedWindow& TouchState::getTouchedWindow(const sp<WindowInfoHandle>& wi bool TouchState::isDown() const { return std::any_of(windows.begin(), windows.end(), - [](const TouchedWindow& window) { return window.pointerIds.any(); }); + [](const TouchedWindow& window) { return window.hasTouchingPointers(); }); } bool TouchState::hasHoveringPointers() const { @@ -245,19 +269,14 @@ void TouchState::removeHoveringPointer(int32_t hoveringDeviceId, int32_t hoverin void TouchState::removeAllPointersForDevice(int32_t removedDeviceId) { for (TouchedWindow& window : windows) { window.removeAllHoveringPointersForDevice(removedDeviceId); + window.removeAllTouchingPointersForDevice(removedDeviceId); } - if (deviceId == removedDeviceId) { - for (TouchedWindow& window : windows) { - window.removeAllTouchingPointers(); - } - } + clearWindowsWithoutPointers(); } std::string TouchState::dump() const { std::string out; - out += StringPrintf("deviceId=%d, source=%s\n", deviceId, - inputEventSourceToString(source).c_str()); if (!windows.empty()) { out += " Windows:\n"; for (size_t i = 0; i < windows.size(); i++) { diff --git a/services/inputflinger/dispatcher/TouchState.h b/services/inputflinger/dispatcher/TouchState.h index 15b840f125..9f29a4aa71 100644 --- a/services/inputflinger/dispatcher/TouchState.h +++ b/services/inputflinger/dispatcher/TouchState.h @@ -29,11 +29,6 @@ class WindowInfoHandle; namespace inputdispatcher { struct TouchState { - // id of the device that is currently down, others are rejected - int32_t deviceId = -1; - // source of the device that is current down, others are rejected - uint32_t source = 0; - std::vector<TouchedWindow> windows; TouchState() = default; @@ -43,24 +38,28 @@ struct TouchState { void reset(); void clearWindowsWithoutPointers(); - void removeTouchedPointer(int32_t pointerId); - void removeTouchedPointerFromWindow(int32_t pointerId, - const sp<android::gui::WindowInfoHandle>& windowHandle); + std::set<int32_t> getActiveDeviceIds() const; + + bool hasTouchingPointers(int32_t device) const; + void removeTouchingPointer(int32_t deviceId, int32_t pointerId); + void removeTouchingPointerFromWindow(int32_t deviceId, int32_t pointerId, + const sp<android::gui::WindowInfoHandle>& windowHandle); void addOrUpdateWindow(const sp<android::gui::WindowInfoHandle>& windowHandle, - ftl::Flags<InputTarget::Flags> targetFlags, - std::bitset<MAX_POINTER_ID + 1> pointerIds, + ftl::Flags<InputTarget::Flags> targetFlags, int32_t deviceId, + std::bitset<MAX_POINTER_ID + 1> touchingPointerIds, std::optional<nsecs_t> firstDownTimeInTarget = std::nullopt); void addHoveringPointerToWindow(const sp<android::gui::WindowInfoHandle>& windowHandle, int32_t deviceId, int32_t hoveringPointerId); void removeHoveringPointer(int32_t deviceId, int32_t hoveringPointerId); void clearHoveringPointers(); - void removeAllPointersForDevice(int32_t removedDeviceId); + void removeAllPointersForDevice(int32_t deviceId); void removeWindowByToken(const sp<IBinder>& token); void filterNonAsIsTouchWindows(); // Cancel pointers for current set of windows except the window with particular binder token. - void cancelPointersForWindowsExcept(std::bitset<MAX_POINTER_ID + 1> pointerIds, + void cancelPointersForWindowsExcept(int32_t deviceId, + std::bitset<MAX_POINTER_ID + 1> pointerIds, const sp<IBinder>& token); // Cancel pointers for current set of non-pilfering windows i.e. windows with isPilferingWindow // set to false. diff --git a/services/inputflinger/dispatcher/TouchedWindow.cpp b/services/inputflinger/dispatcher/TouchedWindow.cpp index d55d657667..ae165209bd 100644 --- a/services/inputflinger/dispatcher/TouchedWindow.cpp +++ b/services/inputflinger/dispatcher/TouchedWindow.cpp @@ -16,6 +16,7 @@ #include "TouchedWindow.h" +#include <android-base/logging.h> #include <android-base/stringprintf.h> #include <input/PrintTools.h> @@ -26,67 +27,236 @@ namespace android { namespace inputdispatcher { bool TouchedWindow::hasHoveringPointers() const { - return !mHoveringPointerIdsByDevice.empty(); + for (const auto& [_, state] : mDeviceStates) { + if (state.hoveringPointerIds.any()) { + return true; + } + } + return false; } bool TouchedWindow::hasHoveringPointers(int32_t deviceId) const { - return mHoveringPointerIdsByDevice.find(deviceId) != mHoveringPointerIdsByDevice.end(); + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { + return false; + } + const DeviceState& state = stateIt->second; + + return state.hoveringPointerIds.any(); } void TouchedWindow::clearHoveringPointers() { - mHoveringPointerIdsByDevice.clear(); + for (auto& [_, state] : mDeviceStates) { + state.hoveringPointerIds.reset(); + } + + std::erase_if(mDeviceStates, [](const auto& pair) { return !pair.second.hasPointers(); }); } bool TouchedWindow::hasHoveringPointer(int32_t deviceId, int32_t pointerId) const { - auto it = mHoveringPointerIdsByDevice.find(deviceId); - if (it == mHoveringPointerIdsByDevice.end()) { + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { return false; } - return it->second.test(pointerId); + const DeviceState& state = stateIt->second; + + return state.hoveringPointerIds.test(pointerId); } void TouchedWindow::addHoveringPointer(int32_t deviceId, int32_t pointerId) { - const auto [it, _] = mHoveringPointerIdsByDevice.insert({deviceId, {}}); - it->second.set(pointerId); + mDeviceStates[deviceId].hoveringPointerIds.set(pointerId); } -void TouchedWindow::removeTouchingPointer(int32_t pointerId) { - pointerIds.reset(pointerId); - pilferedPointerIds.reset(pointerId); - if (pointerIds.none()) { - firstDownTimeInTarget.reset(); +void TouchedWindow::addTouchingPointer(int32_t deviceId, int32_t pointerId) { + mDeviceStates[deviceId].touchingPointerIds.set(pointerId); +} + +void TouchedWindow::addTouchingPointers(int32_t deviceId, + std::bitset<MAX_POINTER_ID + 1> pointers) { + mDeviceStates[deviceId].touchingPointerIds |= pointers; +} + +bool TouchedWindow::hasTouchingPointers() const { + for (const auto& [_, state] : mDeviceStates) { + if (state.touchingPointerIds.any()) { + return true; + } } + return false; +} + +bool TouchedWindow::hasTouchingPointers(int32_t deviceId) const { + return getTouchingPointers(deviceId).any(); } -void TouchedWindow::removeAllTouchingPointers() { - pointerIds.reset(); +bool TouchedWindow::hasTouchingPointer(int32_t deviceId, int32_t pointerId) const { + return getTouchingPointers(deviceId).test(pointerId); +} + +std::bitset<MAX_POINTER_ID + 1> TouchedWindow::getTouchingPointers(int32_t deviceId) const { + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { + return {}; + } + const DeviceState& state = stateIt->second; + + return state.touchingPointerIds; +} + +void TouchedWindow::removeTouchingPointer(int32_t deviceId, int32_t pointerId) { + std::bitset<MAX_POINTER_ID + 1> pointerIds; + pointerIds.set(pointerId, true); + + removeTouchingPointers(deviceId, pointerIds); +} + +void TouchedWindow::removeTouchingPointers(int32_t deviceId, + std::bitset<MAX_POINTER_ID + 1> pointers) { + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { + return; + } + DeviceState& state = stateIt->second; + + state.touchingPointerIds &= ~pointers; + state.pilferingPointerIds &= ~pointers; + + if (!state.hasPointers()) { + mDeviceStates.erase(stateIt); + } +} + +std::set<int32_t> TouchedWindow::getTouchingDeviceIds() const { + std::set<int32_t> deviceIds; + for (const auto& [deviceId, _] : mDeviceStates) { + deviceIds.insert(deviceId); + } + return deviceIds; +} + +std::set<int32_t> TouchedWindow::getActiveDeviceIds() const { + std::set<int32_t> out; + for (const auto& [deviceId, _] : mDeviceStates) { + out.emplace(deviceId); + } + return out; +} + +bool TouchedWindow::hasPilferingPointers(int32_t deviceId) const { + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { + return false; + } + const DeviceState& state = stateIt->second; + + return state.pilferingPointerIds.any(); +} + +void TouchedWindow::addPilferingPointers(int32_t deviceId, + std::bitset<MAX_POINTER_ID + 1> pointerIds) { + mDeviceStates[deviceId].pilferingPointerIds |= pointerIds; +} + +void TouchedWindow::addPilferingPointer(int32_t deviceId, int32_t pointerId) { + mDeviceStates[deviceId].pilferingPointerIds.set(pointerId); +} + +std::bitset<MAX_POINTER_ID + 1> TouchedWindow::getPilferingPointers(int32_t deviceId) const { + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { + return {}; + } + const DeviceState& state = stateIt->second; + + return state.pilferingPointerIds; +} + +std::map<int32_t, std::bitset<MAX_POINTER_ID + 1>> TouchedWindow::getPilferingPointers() const { + std::map<int32_t, std::bitset<MAX_POINTER_ID + 1>> out; + for (const auto& [deviceId, state] : mDeviceStates) { + out.emplace(deviceId, state.pilferingPointerIds); + } + return out; +} + +std::optional<nsecs_t> TouchedWindow::getDownTimeInTarget(int32_t deviceId) const { + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { + return {}; + } + const DeviceState& state = stateIt->second; + return state.downTimeInTarget; +} + +void TouchedWindow::trySetDownTimeInTarget(int32_t deviceId, nsecs_t downTime) { + auto [stateIt, _] = mDeviceStates.try_emplace(deviceId); + DeviceState& state = stateIt->second; + + if (!state.downTimeInTarget) { + state.downTimeInTarget = downTime; + } +} + +void TouchedWindow::removeAllTouchingPointersForDevice(int32_t deviceId) { + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { + return; + } + DeviceState& state = stateIt->second; + + state.touchingPointerIds.reset(); + state.pilferingPointerIds.reset(); + state.downTimeInTarget.reset(); + + if (!state.hasPointers()) { + mDeviceStates.erase(stateIt); + } } void TouchedWindow::removeHoveringPointer(int32_t deviceId, int32_t pointerId) { - const auto it = mHoveringPointerIdsByDevice.find(deviceId); - if (it == mHoveringPointerIdsByDevice.end()) { + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { return; } - it->second.set(pointerId, false); + DeviceState& state = stateIt->second; + + state.hoveringPointerIds.set(pointerId, false); - if (it->second.none()) { - mHoveringPointerIdsByDevice.erase(deviceId); + if (!state.hasPointers()) { + mDeviceStates.erase(stateIt); } } void TouchedWindow::removeAllHoveringPointersForDevice(int32_t deviceId) { - mHoveringPointerIdsByDevice.erase(deviceId); + const auto stateIt = mDeviceStates.find(deviceId); + if (stateIt == mDeviceStates.end()) { + return; + } + DeviceState& state = stateIt->second; + + state.hoveringPointerIds.reset(); + + if (!state.hasPointers()) { + mDeviceStates.erase(stateIt); + } +} + +std::string TouchedWindow::deviceStateToString(const TouchedWindow::DeviceState& state) { + return StringPrintf("[touchingPointerIds=%s, " + "downTimeInTarget=%s, hoveringPointerIds=%s, pilferingPointerIds=%s]", + bitsetToString(state.touchingPointerIds).c_str(), + toString(state.downTimeInTarget).c_str(), + bitsetToString(state.hoveringPointerIds).c_str(), + bitsetToString(state.pilferingPointerIds).c_str()); } std::string TouchedWindow::dump() const { std::string out; - std::string hoveringPointers = - dumpMap(mHoveringPointerIdsByDevice, constToString, bitsetToString); - out += StringPrintf("name='%s', pointerIds=%s, targetFlags=%s, firstDownTimeInTarget=%s, " - "mHoveringPointerIdsByDevice=%s, pilferedPointerIds=%s\n", - windowHandle->getName().c_str(), bitsetToString(pointerIds).c_str(), - targetFlags.string().c_str(), toString(firstDownTimeInTarget).c_str(), - hoveringPointers.c_str(), bitsetToString(pilferedPointerIds).c_str()); + std::string deviceStates = + dumpMap(mDeviceStates, constToString, TouchedWindow::deviceStateToString); + out += StringPrintf("name='%s', targetFlags=%s, mDeviceStates=%s\n", + windowHandle->getName().c_str(), targetFlags.string().c_str(), + deviceStates.c_str()); return out; } diff --git a/services/inputflinger/dispatcher/TouchedWindow.h b/services/inputflinger/dispatcher/TouchedWindow.h index 43e716973c..81393fc2fc 100644 --- a/services/inputflinger/dispatcher/TouchedWindow.h +++ b/services/inputflinger/dispatcher/TouchedWindow.h @@ -20,6 +20,7 @@ #include <input/Input.h> #include <utils/BitSet.h> #include <bitset> +#include <set> #include "InputTarget.h" namespace android { @@ -30,28 +31,66 @@ namespace inputdispatcher { struct TouchedWindow { sp<gui::WindowInfoHandle> windowHandle; ftl::Flags<InputTarget::Flags> targetFlags; - std::bitset<MAX_POINTER_ID + 1> pointerIds; - // The pointer ids of the pointers that this window is currently pilfering - std::bitset<MAX_POINTER_ID + 1> pilferedPointerIds; - // Time at which the first action down occurred on this window. - // NOTE: This is not initialized in case of HOVER entry/exit and DISPATCH_AS_OUTSIDE scenario. - std::optional<nsecs_t> firstDownTimeInTarget; + // Hovering bool hasHoveringPointers() const; bool hasHoveringPointers(int32_t deviceId) const; - bool hasHoveringPointer(int32_t deviceId, int32_t pointerId) const; void addHoveringPointer(int32_t deviceId, int32_t pointerId); void removeHoveringPointer(int32_t deviceId, int32_t pointerId); - void removeTouchingPointer(int32_t pointerId); - void removeAllTouchingPointers(); + // Touching + bool hasTouchingPointer(int32_t deviceId, int32_t pointerId) const; + bool hasTouchingPointers() const; + bool hasTouchingPointers(int32_t deviceId) const; + std::bitset<MAX_POINTER_ID + 1> getTouchingPointers(int32_t deviceId) const; + void addTouchingPointer(int32_t deviceId, int32_t pointerId); + void addTouchingPointers(int32_t deviceId, std::bitset<MAX_POINTER_ID + 1> pointers); + void removeTouchingPointer(int32_t deviceId, int32_t pointerId); + void removeTouchingPointers(int32_t deviceId, std::bitset<MAX_POINTER_ID + 1> pointers); + /** + * Get the currently active touching device id. If there isn't exactly 1 touching device, return + * nullopt. + */ + std::set<int32_t> getTouchingDeviceIds() const; + /** + * The ids of devices that are currently touching or hovering. + */ + std::set<int32_t> getActiveDeviceIds() const; + + // Pilfering pointers + bool hasPilferingPointers(int32_t deviceId) const; + void addPilferingPointers(int32_t deviceId, std::bitset<MAX_POINTER_ID + 1> pointerIds); + void addPilferingPointer(int32_t deviceId, int32_t pointerId); + std::bitset<MAX_POINTER_ID + 1> getPilferingPointers(int32_t deviceId) const; + std::map<int32_t, std::bitset<MAX_POINTER_ID + 1>> getPilferingPointers() const; + + // Down time + std::optional<nsecs_t> getDownTimeInTarget(int32_t deviceId) const; + void trySetDownTimeInTarget(int32_t deviceId, nsecs_t downTime); + + void removeAllTouchingPointersForDevice(int32_t deviceId); void removeAllHoveringPointersForDevice(int32_t deviceId); void clearHoveringPointers(); std::string dump() const; private: - std::map<int32_t /*deviceId*/, std::bitset<MAX_POINTER_ID + 1>> mHoveringPointerIdsByDevice; + struct DeviceState { + std::bitset<MAX_POINTER_ID + 1> touchingPointerIds; + // The pointer ids of the pointers that this window is currently pilfering, by device + std::bitset<MAX_POINTER_ID + 1> pilferingPointerIds; + // Time at which the first action down occurred on this window, for each device + // NOTE: This is not initialized in case of HOVER entry/exit and DISPATCH_AS_OUTSIDE + // scenario. + std::optional<nsecs_t> downTimeInTarget; + std::bitset<MAX_POINTER_ID + 1> hoveringPointerIds; + + bool hasPointers() const { return touchingPointerIds.any() || hoveringPointerIds.any(); }; + }; + + std::map<int32_t /*deviceId*/, DeviceState> mDeviceStates; + + static std::string deviceStateToString(const TouchedWindow::DeviceState& state); }; } // namespace inputdispatcher diff --git a/services/inputflinger/dispatcher/include/InputDispatcherInterface.h b/services/inputflinger/dispatcher/include/InputDispatcherInterface.h index c752ddd699..6a07e59827 100644 --- a/services/inputflinger/dispatcher/include/InputDispatcherInterface.h +++ b/services/inputflinger/dispatcher/include/InputDispatcherInterface.h @@ -39,7 +39,7 @@ public: /* Dumps the state of the input dispatcher. * * This method may be called on any thread (usually by the input manager). */ - virtual void dump(std::string& dump) = 0; + virtual void dump(std::string& dump) const = 0; /* Called by the heatbeat to ensures that the dispatcher has not deadlocked. */ virtual void monitor() = 0; @@ -50,7 +50,7 @@ public: * Return true if the dispatcher is idle. * Return false if the timeout waiting for the dispatcher to become idle has expired. */ - virtual bool waitForIdle() = 0; + virtual bool waitForIdle() const = 0; /* Make the dispatcher start processing events. * @@ -76,7 +76,7 @@ public: * perform all necessary permission checks prior to injecting events. */ virtual android::os::InputEventInjectionResult injectInputEvent( - const InputEvent* event, std::optional<int32_t> targetUid, + const InputEvent* event, std::optional<gui::Uid> targetUid, android::os::InputEventInjectionSync syncMode, std::chrono::milliseconds timeout, uint32_t policyFlags) = 0; @@ -134,7 +134,7 @@ public: * * Returns true when changing touch mode state. */ - virtual bool setInTouchMode(bool inTouchMode, int32_t pid, int32_t uid, bool hasPermission, + virtual bool setInTouchMode(bool inTouchMode, gui::Pid pid, gui::Uid uid, bool hasPermission, int32_t displayId) = 0; /** @@ -182,7 +182,7 @@ public: */ virtual base::Result<std::unique_ptr<InputChannel>> createInputMonitor(int32_t displayId, const std::string& name, - int32_t pid) = 0; + gui::Pid pid) = 0; /* Removes input channels that will no longer receive input events. * diff --git a/services/inputflinger/dispatcher/include/InputDispatcherPolicyInterface.h b/services/inputflinger/dispatcher/include/InputDispatcherPolicyInterface.h index baea6f8eda..729d01f363 100644 --- a/services/inputflinger/dispatcher/include/InputDispatcherPolicyInterface.h +++ b/services/inputflinger/dispatcher/include/InputDispatcherPolicyInterface.h @@ -22,6 +22,7 @@ #include <gui/InputApplication.h> #include <input/Input.h> #include <utils/RefBase.h> +#include <set> namespace android { @@ -52,7 +53,7 @@ public: * pid of the owner. The string reason contains information about the input event that we * haven't received a response for. */ - virtual void notifyWindowUnresponsive(const sp<IBinder>& token, std::optional<int32_t> pid, + virtual void notifyWindowUnresponsive(const sp<IBinder>& token, std::optional<gui::Pid> pid, const std::string& reason) = 0; /* Notifies the system that a window just became responsive. This is only called after the @@ -60,7 +61,7 @@ public: * no longer should be shown to the user. The window is eligible to cause a new ANR in the * future. */ - virtual void notifyWindowResponsive(const sp<IBinder>& token, std::optional<int32_t> pid) = 0; + virtual void notifyWindowResponsive(const sp<IBinder>& token, std::optional<gui::Pid> pid) = 0; /* Notifies the system that an input channel is unrecoverably broken. */ virtual void notifyInputChannelBroken(const sp<IBinder>& token) = 0; @@ -135,6 +136,10 @@ public: /* Notifies the policy that the drag window has moved over to another window */ virtual void notifyDropWindow(const sp<IBinder>& token, float x, float y) = 0; + + /* Notifies the policy that there was an input device interaction with apps. */ + virtual void notifyDeviceInteraction(int32_t deviceId, nsecs_t timestamp, + const std::set<gui::Uid>& uids) = 0; }; } // namespace android diff --git a/services/inputflinger/include/InputReaderBase.h b/services/inputflinger/include/InputReaderBase.h index a93a2ea615..88e1d7db06 100644 --- a/services/inputflinger/include/InputReaderBase.h +++ b/services/inputflinger/include/InputReaderBase.h @@ -447,6 +447,9 @@ public: const std::string& inputDeviceDescriptor, ui::Rotation surfaceRotation) = 0; /* Notifies the input reader policy that a stylus gesture has started. */ virtual void notifyStylusGestureStarted(int32_t deviceId, nsecs_t eventTime) = 0; + + /* Returns true if any InputConnection is currently active. */ + virtual bool isInputMethodConnectionActive() = 0; }; } // namespace android diff --git a/services/inputflinger/include/KeyCodeClassifications.h b/services/inputflinger/include/KeyCodeClassifications.h new file mode 100644 index 0000000000..a09b02e17b --- /dev/null +++ b/services/inputflinger/include/KeyCodeClassifications.h @@ -0,0 +1,55 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <android/input.h> +#include <set> + +namespace android { + +/** The set of all Android key codes that are required for a device to be classified as a D-pad. */ +static const std::set<int32_t> DPAD_REQUIRED_KEYCODES = { + AKEYCODE_DPAD_UP, AKEYCODE_DPAD_DOWN, AKEYCODE_DPAD_LEFT, + AKEYCODE_DPAD_RIGHT, AKEYCODE_DPAD_CENTER, +}; + +/** The set of all Android key codes that correspond to D-pad keys. */ +static const std::set<int32_t> DPAD_ALL_KEYCODES = { + AKEYCODE_DPAD_UP, AKEYCODE_DPAD_DOWN, AKEYCODE_DPAD_LEFT, + AKEYCODE_DPAD_RIGHT, AKEYCODE_DPAD_CENTER, AKEYCODE_DPAD_UP_LEFT, + AKEYCODE_DPAD_UP_RIGHT, AKEYCODE_DPAD_DOWN_LEFT, AKEYCODE_DPAD_DOWN_RIGHT, +}; + +/** The set of all Android key codes that correspond to gamepad buttons. */ +static const std::set<int32_t> GAMEPAD_KEYCODES = { + AKEYCODE_BUTTON_A, AKEYCODE_BUTTON_B, AKEYCODE_BUTTON_C, // + AKEYCODE_BUTTON_X, AKEYCODE_BUTTON_Y, AKEYCODE_BUTTON_Z, // + AKEYCODE_BUTTON_L1, AKEYCODE_BUTTON_R1, // + AKEYCODE_BUTTON_L2, AKEYCODE_BUTTON_R2, // + AKEYCODE_BUTTON_THUMBL, AKEYCODE_BUTTON_THUMBR, // + AKEYCODE_BUTTON_START, AKEYCODE_BUTTON_SELECT, AKEYCODE_BUTTON_MODE, // +}; + +/** The set of all Android key codes that correspond to buttons (bit-switches) on a stylus. */ +static const std::set<int32_t> STYLUS_BUTTON_KEYCODES = { + AKEYCODE_STYLUS_BUTTON_PRIMARY, + AKEYCODE_STYLUS_BUTTON_SECONDARY, + AKEYCODE_STYLUS_BUTTON_TERTIARY, + AKEYCODE_STYLUS_BUTTON_TAIL, +}; + +} // namespace android diff --git a/services/inputflinger/include/NotifyArgs.h b/services/inputflinger/include/NotifyArgs.h index 7d29dd9cb2..736b1e07b7 100644 --- a/services/inputflinger/include/NotifyArgs.h +++ b/services/inputflinger/include/NotifyArgs.h @@ -104,9 +104,9 @@ struct NotifyMotionArgs { MotionClassification classification; int32_t edgeFlags; - uint32_t pointerCount; - PointerProperties pointerProperties[MAX_POINTERS]; - PointerCoords pointerCoords[MAX_POINTERS]; + // Vectors 'pointerProperties' and 'pointerCoords' must always have the same number of elements + std::vector<PointerProperties> pointerProperties; + std::vector<PointerCoords> pointerCoords; float xPrecision; float yPrecision; /** @@ -131,11 +131,13 @@ struct NotifyMotionArgs { float yCursorPosition, nsecs_t downTime, const std::vector<TouchVideoFrame>& videoFrames); - NotifyMotionArgs(const NotifyMotionArgs& other); + NotifyMotionArgs(const NotifyMotionArgs& other) = default; NotifyMotionArgs& operator=(const android::NotifyMotionArgs&) = default; bool operator==(const NotifyMotionArgs& rhs) const; + inline size_t getPointerCount() const { return pointerProperties.size(); } + std::string dump() const; }; diff --git a/services/inputflinger/include/NotifyArgsBuilders.h b/services/inputflinger/include/NotifyArgsBuilders.h new file mode 100644 index 0000000000..e4363a416c --- /dev/null +++ b/services/inputflinger/include/NotifyArgsBuilders.h @@ -0,0 +1,236 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <NotifyArgs.h> +#include <android/input.h> +#include <attestation/HmacKeyManager.h> +#include <gui/constants.h> +#include <input/Input.h> +#include <input/InputEventBuilders.h> +#include <utils/Timers.h> // for nsecs_t, systemTime + +#include <vector> + +namespace android { + +class MotionArgsBuilder { +public: + MotionArgsBuilder(int32_t action, int32_t source) { + mAction = action; + mSource = source; + mEventTime = systemTime(SYSTEM_TIME_MONOTONIC); + mDownTime = mEventTime; + } + + MotionArgsBuilder& deviceId(int32_t deviceId) { + mDeviceId = deviceId; + return *this; + } + + MotionArgsBuilder& downTime(nsecs_t downTime) { + mDownTime = downTime; + return *this; + } + + MotionArgsBuilder& eventTime(nsecs_t eventTime) { + mEventTime = eventTime; + return *this; + } + + MotionArgsBuilder& displayId(int32_t displayId) { + mDisplayId = displayId; + return *this; + } + + MotionArgsBuilder& policyFlags(int32_t policyFlags) { + mPolicyFlags = policyFlags; + return *this; + } + + MotionArgsBuilder& actionButton(int32_t actionButton) { + mActionButton = actionButton; + return *this; + } + + MotionArgsBuilder& buttonState(int32_t buttonState) { + mButtonState = buttonState; + return *this; + } + + MotionArgsBuilder& rawXCursorPosition(float rawXCursorPosition) { + mRawXCursorPosition = rawXCursorPosition; + return *this; + } + + MotionArgsBuilder& rawYCursorPosition(float rawYCursorPosition) { + mRawYCursorPosition = rawYCursorPosition; + return *this; + } + + MotionArgsBuilder& pointer(PointerBuilder pointer) { + mPointers.push_back(pointer); + return *this; + } + + MotionArgsBuilder& addFlag(uint32_t flags) { + mFlags |= flags; + return *this; + } + + MotionArgsBuilder& classification(MotionClassification classification) { + mClassification = classification; + return *this; + } + + NotifyMotionArgs build() { + std::vector<PointerProperties> pointerProperties; + std::vector<PointerCoords> pointerCoords; + for (const PointerBuilder& pointer : mPointers) { + pointerProperties.push_back(pointer.buildProperties()); + pointerCoords.push_back(pointer.buildCoords()); + } + + // Set mouse cursor position for the most common cases to avoid boilerplate. + if (mSource == AINPUT_SOURCE_MOUSE && + !MotionEvent::isValidCursorPosition(mRawXCursorPosition, mRawYCursorPosition)) { + mRawXCursorPosition = pointerCoords[0].getX(); + mRawYCursorPosition = pointerCoords[0].getY(); + } + + if (mAction == AMOTION_EVENT_ACTION_CANCEL) { + addFlag(AMOTION_EVENT_FLAG_CANCELED); + } + + return {InputEvent::nextId(), + mEventTime, + /*readTime=*/mEventTime, + mDeviceId, + mSource, + mDisplayId, + mPolicyFlags, + mAction, + mActionButton, + mFlags, + AMETA_NONE, + mButtonState, + mClassification, + /*edgeFlags=*/0, + static_cast<uint32_t>(mPointers.size()), + pointerProperties.data(), + pointerCoords.data(), + /*xPrecision=*/0, + /*yPrecision=*/0, + mRawXCursorPosition, + mRawYCursorPosition, + mDownTime, + /*videoFrames=*/{}}; + } + +private: + int32_t mAction; + int32_t mDeviceId{DEFAULT_DEVICE_ID}; + uint32_t mSource; + nsecs_t mDownTime; + nsecs_t mEventTime; + int32_t mDisplayId{ADISPLAY_ID_DEFAULT}; + uint32_t mPolicyFlags = DEFAULT_POLICY_FLAGS; + int32_t mActionButton{0}; + int32_t mButtonState{0}; + int32_t mFlags{0}; + MotionClassification mClassification{MotionClassification::NONE}; + float mRawXCursorPosition{AMOTION_EVENT_INVALID_CURSOR_POSITION}; + float mRawYCursorPosition{AMOTION_EVENT_INVALID_CURSOR_POSITION}; + + std::vector<PointerBuilder> mPointers; +}; + +class KeyArgsBuilder { +public: + KeyArgsBuilder(int32_t action, int32_t source) { + mAction = action; + mSource = source; + mEventTime = systemTime(SYSTEM_TIME_MONOTONIC); + mDownTime = mEventTime; + } + + KeyArgsBuilder& deviceId(int32_t deviceId) { + mDeviceId = deviceId; + return *this; + } + + KeyArgsBuilder& downTime(nsecs_t downTime) { + mDownTime = downTime; + return *this; + } + + KeyArgsBuilder& eventTime(nsecs_t eventTime) { + mEventTime = eventTime; + return *this; + } + + KeyArgsBuilder& displayId(int32_t displayId) { + mDisplayId = displayId; + return *this; + } + + KeyArgsBuilder& policyFlags(int32_t policyFlags) { + mPolicyFlags = policyFlags; + return *this; + } + + KeyArgsBuilder& addFlag(uint32_t flags) { + mFlags |= flags; + return *this; + } + + KeyArgsBuilder& keyCode(int32_t keyCode) { + mKeyCode = keyCode; + return *this; + } + + NotifyKeyArgs build() const { + return {InputEvent::nextId(), + mEventTime, + /*readTime=*/mEventTime, + mDeviceId, + mSource, + mDisplayId, + mPolicyFlags, + mAction, + mFlags, + mKeyCode, + mScanCode, + mMetaState, + mDownTime}; + } + +private: + int32_t mAction; + int32_t mDeviceId = DEFAULT_DEVICE_ID; + uint32_t mSource; + nsecs_t mDownTime; + nsecs_t mEventTime; + int32_t mDisplayId{ADISPLAY_ID_DEFAULT}; + uint32_t mPolicyFlags = DEFAULT_POLICY_FLAGS; + int32_t mFlags{0}; + int32_t mKeyCode{AKEYCODE_UNKNOWN}; + int32_t mScanCode{0}; + int32_t mMetaState{AMETA_NONE}; +}; + +} // namespace android diff --git a/services/inputflinger/reader/Android.bp b/services/inputflinger/reader/Android.bp index b0edb5746c..ccb8773ed0 100644 --- a/services/inputflinger/reader/Android.bp +++ b/services/inputflinger/reader/Android.bp @@ -98,12 +98,14 @@ cc_defaults { android: { shared_libs: [ "libinput", + "libstatspull", ], }, host: { static_libs: [ "libinput", "libbinder", + "libstatspull", ], }, }, diff --git a/services/inputflinger/reader/EventHub.cpp b/services/inputflinger/reader/EventHub.cpp index c468d45c02..44e80a732c 100644 --- a/services/inputflinger/reader/EventHub.cpp +++ b/services/inputflinger/reader/EventHub.cpp @@ -58,6 +58,8 @@ #include "EventHub.h" +#include "KeyCodeClassifications.h" + #define INDENT " " #define INDENT2 " " #define INDENT3 " " @@ -189,14 +191,6 @@ static std::string sha1(const std::string& in) { return out; } -/* The set of all Android key codes that correspond to buttons (bit-switches) on a stylus. */ -static constexpr std::array<int32_t, 4> STYLUS_BUTTON_KEYCODES = { - AKEYCODE_STYLUS_BUTTON_PRIMARY, - AKEYCODE_STYLUS_BUTTON_SECONDARY, - AKEYCODE_STYLUS_BUTTON_TERTIARY, - AKEYCODE_STYLUS_BUTTON_TAIL, -}; - /** * Return true if name matches "v4l-touch*" */ @@ -870,6 +864,30 @@ void EventHub::addDeviceInotify() { strerror(errno)); } +void EventHub::populateDeviceAbsoluteAxisInfo(Device& device) { + for (int axis = 0; axis <= ABS_MAX; axis++) { + if (!device.absBitmask.test(axis)) { + continue; + } + struct input_absinfo info {}; + if (ioctl(device.fd, EVIOCGABS(axis), &info)) { + ALOGE("Error reading absolute controller %d for device %s fd %d, errno=%d", axis, + device.identifier.name.c_str(), device.fd, errno); + continue; + } + if (info.minimum == info.maximum) { + continue; + } + RawAbsoluteAxisInfo& outAxisInfo = device.rawAbsoluteAxisInfoCache[axis]; + outAxisInfo.valid = true; + outAxisInfo.minValue = info.minimum; + outAxisInfo.maxValue = info.maximum; + outAxisInfo.flat = info.flat; + outAxisInfo.fuzz = info.fuzz; + outAxisInfo.resolution = info.resolution; + } +} + InputDeviceIdentifier EventHub::getDeviceIdentifier(int32_t deviceId) const { std::scoped_lock _l(mLock); Device* device = getDeviceLocked(deviceId); @@ -900,31 +918,20 @@ std::optional<PropertyMap> EventHub::getConfiguration(int32_t deviceId) const { status_t EventHub::getAbsoluteAxisInfo(int32_t deviceId, int axis, RawAbsoluteAxisInfo* outAxisInfo) const { outAxisInfo->clear(); - - if (axis >= 0 && axis <= ABS_MAX) { - std::scoped_lock _l(mLock); - - Device* device = getDeviceLocked(deviceId); - if (device != nullptr && device->hasValidFd() && device->absBitmask.test(axis)) { - struct input_absinfo info; - if (ioctl(device->fd, EVIOCGABS(axis), &info)) { - ALOGW("Error reading absolute controller %d for device %s fd %d, errno=%d", axis, - device->identifier.name.c_str(), device->fd, errno); - return -errno; - } - - if (info.minimum != info.maximum) { - outAxisInfo->valid = true; - outAxisInfo->minValue = info.minimum; - outAxisInfo->maxValue = info.maximum; - outAxisInfo->flat = info.flat; - outAxisInfo->fuzz = info.fuzz; - outAxisInfo->resolution = info.resolution; - } - return OK; - } + if (axis < 0 || axis > ABS_MAX) { + return -1; } - return -1; + std::scoped_lock _l(mLock); + Device* device = getDeviceLocked(deviceId); + if (device == nullptr) { + return -1; + } + auto it = device->rawAbsoluteAxisInfoCache.find(axis); + if (it == device->rawAbsoluteAxisInfoCache.end()) { + return -1; + } + *outAxisInfo = it->second; + return OK; } bool EventHub::hasRelativeAxis(int32_t deviceId, int axis) const { @@ -2060,15 +2067,6 @@ void EventHub::scanDevicesLocked() { // ---------------------------------------------------------------------------- -static const int32_t GAMEPAD_KEYCODES[] = { - AKEYCODE_BUTTON_A, AKEYCODE_BUTTON_B, AKEYCODE_BUTTON_C, // - AKEYCODE_BUTTON_X, AKEYCODE_BUTTON_Y, AKEYCODE_BUTTON_Z, // - AKEYCODE_BUTTON_L1, AKEYCODE_BUTTON_R1, // - AKEYCODE_BUTTON_L2, AKEYCODE_BUTTON_R2, // - AKEYCODE_BUTTON_THUMBL, AKEYCODE_BUTTON_THUMBR, // - AKEYCODE_BUTTON_START, AKEYCODE_BUTTON_SELECT, AKEYCODE_BUTTON_MODE, // -}; - status_t EventHub::registerFdForEpoll(int fd) { // TODO(b/121395353) - consider adding EPOLLRDHUP struct epoll_event eventItem = {}; @@ -2391,32 +2389,24 @@ void EventHub::openDeviceLocked(const std::string& devicePath) { device->classes |= InputDeviceClass::ALPHAKEY; } - // See if this device has a DPAD. - if (device->hasKeycodeLocked(AKEYCODE_DPAD_UP) && - device->hasKeycodeLocked(AKEYCODE_DPAD_DOWN) && - device->hasKeycodeLocked(AKEYCODE_DPAD_LEFT) && - device->hasKeycodeLocked(AKEYCODE_DPAD_RIGHT) && - device->hasKeycodeLocked(AKEYCODE_DPAD_CENTER)) { + // See if this device has a D-pad. + if (std::all_of(DPAD_REQUIRED_KEYCODES.begin(), DPAD_REQUIRED_KEYCODES.end(), + [&](int32_t keycode) { return device->hasKeycodeLocked(keycode); })) { device->classes |= InputDeviceClass::DPAD; } // See if this device has a gamepad. - for (size_t i = 0; i < sizeof(GAMEPAD_KEYCODES) / sizeof(GAMEPAD_KEYCODES[0]); i++) { - if (device->hasKeycodeLocked(GAMEPAD_KEYCODES[i])) { - device->classes |= InputDeviceClass::GAMEPAD; - break; - } + if (std::any_of(GAMEPAD_KEYCODES.begin(), GAMEPAD_KEYCODES.end(), + [&](int32_t keycode) { return device->hasKeycodeLocked(keycode); })) { + device->classes |= InputDeviceClass::GAMEPAD; } // See if this device has any stylus buttons that we would want to fuse with touch data. if (!device->classes.any(InputDeviceClass::TOUCH | InputDeviceClass::TOUCH_MT) && - !device->classes.any(InputDeviceClass::ALPHAKEY)) { - for (int32_t keycode : STYLUS_BUTTON_KEYCODES) { - if (device->hasKeycodeLocked(keycode)) { - device->classes |= InputDeviceClass::EXTERNAL_STYLUS; - break; - } - } + !device->classes.any(InputDeviceClass::ALPHAKEY) && + std::any_of(STYLUS_BUTTON_KEYCODES.begin(), STYLUS_BUTTON_KEYCODES.end(), + [&](int32_t keycode) { return device->hasKeycodeLocked(keycode); })) { + device->classes |= InputDeviceClass::EXTERNAL_STYLUS; } } @@ -2459,6 +2449,9 @@ void EventHub::openDeviceLocked(const std::string& devicePath) { device->configureFd(); + // read absolute axis info for all available axes for the device + populateDeviceAbsoluteAxisInfo(*device); + ALOGI("New device: id=%d, fd=%d, path='%s', name='%s', classes=%s, " "configuration='%s', keyLayout='%s', keyCharacterMap='%s', builtinKeyboard=%s, ", deviceId, fd, devicePath.c_str(), device->identifier.name.c_str(), diff --git a/services/inputflinger/reader/InputReader.cpp b/services/inputflinger/reader/InputReader.cpp index ea95f7857a..7f63355387 100644 --- a/services/inputflinger/reader/InputReader.cpp +++ b/services/inputflinger/reader/InputReader.cpp @@ -77,7 +77,7 @@ InputReader::InputReader(std::shared_ptr<EventHubInterface> eventHub, : mContext(this), mEventHub(eventHub), mPolicy(policy), - mQueuedListener(listener), + mNextListener(listener), mGlobalMetaState(AMETA_NONE), mLedMetaState(AMETA_NONE), mGeneration(1), @@ -140,7 +140,7 @@ void InputReader::loopOnce() { mReaderIsAliveCondition.notify_all(); if (!events.empty()) { - notifyArgs += processEventsLocked(events.data(), events.size()); + mPendingArgs += processEventsLocked(events.data(), events.size()); } if (mNextTimeout != LLONG_MAX) { @@ -150,16 +150,18 @@ void InputReader::loopOnce() { ALOGD("Timeout expired, latency=%0.3fms", (now - mNextTimeout) * 0.000001f); } mNextTimeout = LLONG_MAX; - notifyArgs += timeoutExpiredLocked(now); + mPendingArgs += timeoutExpiredLocked(now); } } if (oldGeneration != mGeneration) { inputDevicesChanged = true; inputDevices = getInputDevicesLocked(); - notifyArgs.emplace_back( + mPendingArgs.emplace_back( NotifyInputDevicesChangedArgs{mContext.getNextId(), inputDevices}); } + + std::swap(notifyArgs, mPendingArgs); } // release lock // Send out a message that the describes the changed input devices. @@ -175,8 +177,6 @@ void InputReader::loopOnce() { } } - notifyAll(std::move(notifyArgs)); - // Flush queued events out to the listener. // This must happen outside of the lock because the listener could potentially call // back into the InputReader's methods, such as getScanCodeState, or become blocked @@ -184,7 +184,9 @@ void InputReader::loopOnce() { // resulting in a deadlock. This situation is actually quite plausible because the // listener is actually the input dispatcher, which calls into the window manager, // which occasionally calls into the input reader. - mQueuedListener.flush(); + for (const NotifyArgs& args : notifyArgs) { + mNextListener.notify(args); + } } std::list<NotifyArgs> InputReader::processEventsLocked(const RawEvent* rawEvents, size_t count) { @@ -236,8 +238,8 @@ void InputReader::addDeviceLocked(nsecs_t when, int32_t eventHubId) { InputDeviceIdentifier identifier = mEventHub->getDeviceIdentifier(eventHubId); std::shared_ptr<InputDevice> device = createDeviceLocked(eventHubId, identifier); - notifyAll(device->configure(when, mConfig, /*changes=*/{})); - notifyAll(device->reset(when)); + mPendingArgs += device->configure(when, mConfig, /*changes=*/{}); + mPendingArgs += device->reset(when); if (device->isIgnored()) { ALOGI("Device added: id=%d, eventHubId=%d, name='%s', descriptor='%s' " @@ -310,12 +312,10 @@ void InputReader::removeDeviceLocked(nsecs_t when, int32_t eventHubId) { notifyExternalStylusPresenceChangedLocked(); } - std::list<NotifyArgs> resetEvents; if (device->hasEventHubDevices()) { - resetEvents += device->configure(when, mConfig, /*changes=*/{}); + mPendingArgs += device->configure(when, mConfig, /*changes=*/{}); } - resetEvents += device->reset(when); - notifyAll(std::move(resetEvents)); + mPendingArgs += device->reset(when); } std::shared_ptr<InputDevice> InputReader::createDeviceLocked( @@ -387,7 +387,7 @@ void InputReader::handleConfigurationChangedLocked(nsecs_t when) { updateGlobalMetaStateLocked(); // Enqueue configuration changed. - mQueuedListener.notifyConfigurationChanged({mContext.getNextId(), when}); + mPendingArgs.emplace_back(NotifyConfigurationChangedArgs{mContext.getNextId(), when}); } void InputReader::refreshConfigurationLocked(ConfigurationChanges changes) { @@ -409,7 +409,7 @@ void InputReader::refreshConfigurationLocked(ConfigurationChanges changes) { } else { for (auto& devicePair : mDevices) { std::shared_ptr<InputDevice>& device = devicePair.second; - notifyAll(device->configure(now, mConfig, changes)); + mPendingArgs += device->configure(now, mConfig, changes); } } @@ -419,18 +419,13 @@ void InputReader::refreshConfigurationLocked(ConfigurationChanges changes) { "There was no change in the pointer capture state."); } else { mCurrentPointerCaptureRequest = mConfig.pointerCaptureRequest; - mQueuedListener.notifyPointerCaptureChanged( - {mContext.getNextId(), now, mCurrentPointerCaptureRequest}); + mPendingArgs.emplace_back( + NotifyPointerCaptureChangedArgs{mContext.getNextId(), now, + mCurrentPointerCaptureRequest}); } } } -void InputReader::notifyAll(std::list<NotifyArgs>&& argsList) { - for (const NotifyArgs& args : argsList) { - mQueuedListener.notify(args); - } -} - void InputReader::updateGlobalMetaStateLocked() { mGlobalMetaState = 0; @@ -690,7 +685,7 @@ void InputReader::vibrate(int32_t deviceId, const VibrationSequence& sequence, s InputDevice* device = findInputDeviceLocked(deviceId); if (device) { - notifyAll(device->vibrate(sequence, repeat, token)); + mPendingArgs += device->vibrate(sequence, repeat, token); } } @@ -699,7 +694,7 @@ void InputReader::cancelVibrate(int32_t deviceId, int32_t token) { InputDevice* device = findInputDeviceLocked(deviceId); if (device) { - notifyAll(device->cancelVibrate(token)); + mPendingArgs += device->cancelVibrate(token); } } @@ -1040,6 +1035,16 @@ int32_t InputReader::ContextImpl::getLedMetaState() { return mReader->getLedMetaStateLocked(); } +void InputReader::ContextImpl::setPreventingTouchpadTaps(bool prevent) { + // lock is already held by the input loop + mReader->mPreventingTouchpadTaps = prevent; +} + +bool InputReader::ContextImpl::isPreventingTouchpadTaps() { + // lock is already held by the input loop + return mReader->mPreventingTouchpadTaps; +} + void InputReader::ContextImpl::disableVirtualKeysUntil(nsecs_t time) { // lock is already held by the input loop mReader->disableVirtualKeysUntilLocked(time); diff --git a/services/inputflinger/reader/controller/PeripheralController.cpp b/services/inputflinger/reader/controller/PeripheralController.cpp index a380b5eadf..eabf591dbf 100644 --- a/services/inputflinger/reader/controller/PeripheralController.cpp +++ b/services/inputflinger/reader/controller/PeripheralController.cpp @@ -16,8 +16,10 @@ #include <locale> #include <regex> -#include <set> +#include <sstream> +#include <string> +#include <android/sysprop/InputProperties.sysprop.h> #include <ftl/enum.h> #include "../Macros.h" @@ -45,6 +47,10 @@ static inline int32_t toArgb(int32_t brightness, int32_t red, int32_t green, int return (brightness & 0xff) << 24 | (red & 0xff) << 16 | (green & 0xff) << 8 | (blue & 0xff); } +static inline bool isKeyboardBacklightCustomLevelsEnabled() { + return sysprop::InputProperties::enable_keyboard_backlight_custom_levels().value_or(true); +} + /** * Input controller owned by InputReader device, implements the native API for querying input * lights, getting and setting the lights brightness and color, by interacting with EventHub @@ -272,11 +278,43 @@ void PeripheralController::populateDeviceInfo(InputDeviceInfo* deviceInfo) { for (const auto& [lightId, light] : mLights) { // Input device light doesn't support ordinal, always pass 1. InputDeviceLightInfo lightInfo(light->name, light->id, light->type, light->capabilityFlags, - /*ordinal=*/1); + /*ordinal=*/1, getPreferredBrightnessLevels(light.get())); deviceInfo->addLightInfo(lightInfo); } } +// TODO(b/281822656): Move to constructor and add as a parameter to avoid parsing repeatedly. +// Need to change lifecycle of Peripheral controller so that Input device configuration map is +// available at construction time before moving this logic to constructor. +std::set<BrightnessLevel> PeripheralController::getPreferredBrightnessLevels( + const Light* light) const { + std::set<BrightnessLevel> levels; + if (!isKeyboardBacklightCustomLevelsEnabled() || + light->type != InputDeviceLightType::KEYBOARD_BACKLIGHT) { + return levels; + } + std::optional<std::string> keyboardBacklightLevels = + mDeviceContext.getConfiguration().getString("keyboard.backlight.brightnessLevels"); + if (!keyboardBacklightLevels) { + return levels; + } + std::stringstream ss(*keyboardBacklightLevels); + while (ss.good()) { + std::string substr; + std::getline(ss, substr, ','); + char* end; + int32_t value = static_cast<int32_t>(strtol(substr.c_str(), &end, 10)); + if (*end != '\0' || value < 0 || value > 255) { + ALOGE("Error parsing keyboard backlight brightness levels, provided levels = %s", + keyboardBacklightLevels->c_str()); + levels.clear(); + break; + } + levels.insert(BrightnessLevel(value)); + } + return levels; +} + void PeripheralController::dump(std::string& dump) { dump += INDENT2 "Input Controller:\n"; if (!mLights.empty()) { @@ -550,5 +588,4 @@ std::optional<int32_t> PeripheralController::getLightPlayerId(int32_t lightId) { int32_t PeripheralController::getEventHubId() const { return getDeviceContext().getEventHubId(); } - } // namespace android diff --git a/services/inputflinger/reader/controller/PeripheralController.h b/services/inputflinger/reader/controller/PeripheralController.h index 8ac42c3792..07ade7c931 100644 --- a/services/inputflinger/reader/controller/PeripheralController.h +++ b/services/inputflinger/reader/controller/PeripheralController.h @@ -76,6 +76,7 @@ private: virtual void dump(std::string& dump) {} + void configureSuggestedBrightnessLevels(); std::optional<std::int32_t> getRawLightBrightness(int32_t rawLightId); void setRawLightBrightness(int32_t rawLightId, int32_t brightness); }; @@ -152,6 +153,8 @@ private: // Battery map from battery ID to battery std::unordered_map<int32_t, std::unique_ptr<Battery>> mBatteries; + + std::set<BrightnessLevel> getPreferredBrightnessLevels(const Light* light) const; }; } // namespace android diff --git a/services/inputflinger/reader/include/EventHub.h b/services/inputflinger/reader/include/EventHub.h index 20612c767c..024187f5b5 100644 --- a/services/inputflinger/reader/include/EventHub.h +++ b/services/inputflinger/reader/include/EventHub.h @@ -615,6 +615,7 @@ private: std::unique_ptr<PropertyMap> configuration; std::unique_ptr<VirtualKeyMap> virtualKeyMap; KeyMap keyMap; + std::unordered_map<int /*axis*/, RawAbsoluteAxisInfo> rawAbsoluteAxisInfoCache; bool ffEffectPlaying; int16_t ffEffectId; // initially -1 @@ -717,6 +718,13 @@ private: void addDeviceInputInotify(); void addDeviceInotify(); + /** + * AbsoluteAxisInfo remains unchanged for the lifetime of the device, hence + * we can read and store it with device + * @param device target device + */ + static void populateDeviceAbsoluteAxisInfo(Device& device); + // Protect all internal state. mutable std::mutex mLock; diff --git a/services/inputflinger/reader/include/InputDevice.h b/services/inputflinger/reader/include/InputDevice.h index 0b8a608891..2f8e5bd6cf 100644 --- a/services/inputflinger/reader/include/InputDevice.h +++ b/services/inputflinger/reader/include/InputDevice.h @@ -74,7 +74,7 @@ public: } inline bool hasMic() const { return mHasMic; } - inline bool isIgnored() { return !getMapperCount(); } + inline bool isIgnored() { return !getMapperCount() && !mController; } bool isEnabled(); [[nodiscard]] std::list<NotifyArgs> setEnabled(bool enabled, nsecs_t when); diff --git a/services/inputflinger/reader/include/InputReader.h b/services/inputflinger/reader/include/InputReader.h index 9112913565..e21715eb27 100644 --- a/services/inputflinger/reader/include/InputReader.h +++ b/services/inputflinger/reader/include/InputReader.h @@ -155,6 +155,9 @@ protected: int32_t getNextId() NO_THREAD_SAFETY_ANALYSIS override; void updateLedMetaState(int32_t metaState) REQUIRES(mReader->mLock) override; int32_t getLedMetaState() REQUIRES(mReader->mLock) REQUIRES(mLock) override; + void setPreventingTouchpadTaps(bool prevent) REQUIRES(mReader->mLock) + REQUIRES(mLock) override; + bool isPreventingTouchpadTaps() REQUIRES(mReader->mLock) REQUIRES(mLock) override; } mContext; friend class ContextImpl; @@ -171,7 +174,14 @@ private: // in parallel to passing it to the InputReader. std::shared_ptr<EventHubInterface> mEventHub; sp<InputReaderPolicyInterface> mPolicy; - QueuedInputListener mQueuedListener; + + // The next stage that should receive the events generated inside InputReader. + InputListenerInterface& mNextListener; + // As various events are generated inside InputReader, they are stored inside this list. The + // list can only be accessed with the lock, so the events inside it are well-ordered. + // Once the reader is done working, these events will be swapped into a temporary storage and + // sent to the 'mNextListener' without holding the lock. + std::list<NotifyArgs> mPendingArgs GUARDED_BY(mLock); InputReaderConfiguration mConfig GUARDED_BY(mLock); @@ -185,6 +195,9 @@ private: std::unordered_map<std::shared_ptr<InputDevice>, std::vector<int32_t> /*eventHubId*/> mDeviceToEventHubIdsMap GUARDED_BY(mLock); + // true if tap-to-click on touchpad currently disabled + bool mPreventingTouchpadTaps GUARDED_BY(mLock){false}; + // low-level input event decoding and device management [[nodiscard]] std::list<NotifyArgs> processEventsLocked(const RawEvent* rawEvents, size_t count) REQUIRES(mLock); @@ -236,8 +249,6 @@ private: ConfigurationChanges mConfigurationChangesToRefresh GUARDED_BY(mLock); void refreshConfigurationLocked(ConfigurationChanges changes) REQUIRES(mLock); - void notifyAll(std::list<NotifyArgs>&& argsList); - PointerCaptureRequest mCurrentPointerCaptureRequest GUARDED_BY(mLock); // state queries diff --git a/services/inputflinger/reader/include/InputReaderContext.h b/services/inputflinger/reader/include/InputReaderContext.h index 0beace19ab..aed75636f7 100644 --- a/services/inputflinger/reader/include/InputReaderContext.h +++ b/services/inputflinger/reader/include/InputReaderContext.h @@ -62,6 +62,9 @@ public: virtual void updateLedMetaState(int32_t metaState) = 0; virtual int32_t getLedMetaState() = 0; + + virtual void setPreventingTouchpadTaps(bool prevent) = 0; + virtual bool isPreventingTouchpadTaps() = 0; }; } // namespace android diff --git a/services/inputflinger/reader/mapper/KeyboardInputMapper.cpp b/services/inputflinger/reader/mapper/KeyboardInputMapper.cpp index 7388752805..5c42e10090 100644 --- a/services/inputflinger/reader/mapper/KeyboardInputMapper.cpp +++ b/services/inputflinger/reader/mapper/KeyboardInputMapper.cpp @@ -242,6 +242,7 @@ std::list<NotifyArgs> KeyboardInputMapper::processKey(nsecs_t when, nsecs_t read keyDown.downTime = when; mKeyDowns.push_back(keyDown); } + onKeyDownProcessed(); } else { // Remove key down. if (keyDownIndex) { @@ -419,4 +420,19 @@ std::list<NotifyArgs> KeyboardInputMapper::cancelAllDownKeys(nsecs_t when) { return out; } +void KeyboardInputMapper::onKeyDownProcessed() { + InputReaderContext& context = *getContext(); + if (context.isPreventingTouchpadTaps()) { + // avoid pinging java service unnecessarily + return; + } + // Ignore meta keys or multiple simultaneous down keys as they are likely to be keyboard + // shortcuts + bool shouldHideCursor = mKeyDowns.size() == 1 && !isMetaKey(mKeyDowns[0].keyCode); + if (shouldHideCursor && context.getPolicy()->isInputMethodConnectionActive()) { + context.fadePointer(); + context.setPreventingTouchpadTaps(true); + } +} + } // namespace android diff --git a/services/inputflinger/reader/mapper/KeyboardInputMapper.h b/services/inputflinger/reader/mapper/KeyboardInputMapper.h index cd3d3c49e9..96044eb0a8 100644 --- a/services/inputflinger/reader/mapper/KeyboardInputMapper.h +++ b/services/inputflinger/reader/mapper/KeyboardInputMapper.h @@ -104,6 +104,7 @@ private: void updateLedStateForModifier(LedState& ledState, int32_t led, int32_t modifier, bool reset); std::optional<DisplayViewport> findViewport(const InputReaderConfiguration& readerConfig); [[nodiscard]] std::list<NotifyArgs> cancelAllDownKeys(nsecs_t when); + void onKeyDownProcessed(); }; } // namespace android diff --git a/services/inputflinger/reader/mapper/TouchInputMapper.cpp b/services/inputflinger/reader/mapper/TouchInputMapper.cpp index f2b0a4b0a7..587e8450b1 100644 --- a/services/inputflinger/reader/mapper/TouchInputMapper.cpp +++ b/services/inputflinger/reader/mapper/TouchInputMapper.cpp @@ -130,7 +130,10 @@ TouchInputMapper::TouchInputMapper(InputDeviceContext& deviceContext, TouchInputMapper::~TouchInputMapper() {} uint32_t TouchInputMapper::getSources() const { - return mSource; + // The SOURCE_BLUETOOTH_STYLUS is added to events dynamically if the current stream is modified + // by the external stylus state. That's why we don't add it directly to mSource during + // configuration. + return mSource | (hasExternalStylus() ? AINPUT_SOURCE_BLUETOOTH_STYLUS : 0); } void TouchInputMapper::populateDeviceInfo(InputDeviceInfo& info) { @@ -932,9 +935,6 @@ void TouchInputMapper::configureInputDevice(nsecs_t when, bool* outResetNeeded) if (hasStylus()) { mSource |= AINPUT_SOURCE_STYLUS; } - if (hasExternalStylus()) { - mSource |= AINPUT_SOURCE_BLUETOOTH_STYLUS; - } } else if (mParameters.deviceType == Parameters::DeviceType::TOUCH_NAVIGATION) { mSource = AINPUT_SOURCE_TOUCH_NAVIGATION; mDeviceMode = DeviceMode::NAVIGATION; @@ -1653,6 +1653,10 @@ std::list<NotifyArgs> TouchInputMapper::cookAndDispatch(nsecs_t when, nsecs_t re mSource, mViewport.displayId, policyFlags, mLastCookedState.buttonState, mCurrentCookedState.buttonState); + if (mCurrentCookedState.cookedPointerData.pointerCount == 0) { + mCurrentStreamModifiedByExternalStylus = false; + } + // Clear some transient state. mCurrentRawState.rawVScroll = 0; mCurrentRawState.rawHScroll = 0; @@ -1704,6 +1708,10 @@ void TouchInputMapper::applyExternalStylusButtonState(nsecs_t when) { mExternalStylusButtonsApplied |= pressedButtons; mExternalStylusButtonsApplied &= ~releasedButtons; + + if (mExternalStylusButtonsApplied != 0 || releasedButtons != 0) { + mCurrentStreamModifiedByExternalStylus = true; + } } } @@ -1714,6 +1722,8 @@ void TouchInputMapper::applyExternalStylusTouchState(nsecs_t when) { return; } + mCurrentStreamModifiedByExternalStylus = true; + float pressure = lastPointerData.isTouching(*mFusedStylusPointerId) ? lastPointerData.pointerCoordsForId(*mFusedStylusPointerId) .getAxisValue(AMOTION_EVENT_AXIS_PRESSURE) @@ -3813,6 +3823,9 @@ NotifyMotionArgs TouchInputMapper::dispatchMotion( ALOG_ASSERT(false); } } + if (mCurrentStreamModifiedByExternalStylus) { + source |= AINPUT_SOURCE_BLUETOOTH_STYLUS; + } const int32_t displayId = getAssociatedDisplayId().value_or(ADISPLAY_ID_NONE); const bool showDirectStylusPointer = mConfig.stylusPointerIconEnabled && diff --git a/services/inputflinger/reader/mapper/TouchInputMapper.h b/services/inputflinger/reader/mapper/TouchInputMapper.h index d8b59ca39b..97f41cc57b 100644 --- a/services/inputflinger/reader/mapper/TouchInputMapper.h +++ b/services/inputflinger/reader/mapper/TouchInputMapper.h @@ -357,6 +357,8 @@ protected: bool mExternalStylusDataPending; // A subset of the buttons in mCurrentRawState that came from an external stylus. int32_t mExternalStylusButtonsApplied{0}; + // True if the current cooked pointer data was modified due to the state of an external stylus. + bool mCurrentStreamModifiedByExternalStylus{false}; // True if we sent a HOVER_ENTER event. bool mSentHoverEnter{false}; diff --git a/services/inputflinger/reader/mapper/TouchpadInputMapper.cpp b/services/inputflinger/reader/mapper/TouchpadInputMapper.cpp index c72425a90b..3e092d3fb7 100644 --- a/services/inputflinger/reader/mapper/TouchpadInputMapper.cpp +++ b/services/inputflinger/reader/mapper/TouchpadInputMapper.cpp @@ -16,8 +16,11 @@ #include "../Macros.h" +#include <algorithm> #include <chrono> +#include <iterator> #include <limits> +#include <map> #include <optional> #include <android-base/stringprintf.h> @@ -26,6 +29,8 @@ #include <input/PrintTools.h> #include <linux/input-event-codes.h> #include <log/log_main.h> +#include <stats_pull_atom_callback.h> +#include <statslog.h> #include "TouchCursorInputMapperCommon.h" #include "TouchpadInputMapper.h" #include "ui/Rotation.h" @@ -53,13 +58,14 @@ struct CurveSegment { }; const std::vector<CurveSegment> segments = { - {10.922, 3.19, 0}, - {31.750, 4.79, -17.526}, - {98.044, 7.28, -96.52}, - {std::numeric_limits<double>::infinity(), 15.04, -857.758}, + {32.002, 3.19, 0}, + {52.83, 4.79, -51.254}, + {119.124, 7.28, -182.737}, + {std::numeric_limits<double>::infinity(), 15.04, -1107.556}, }; -const std::vector<double> sensitivityFactors = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18}; +const std::vector<double> sensitivityFactors = {1, 2, 4, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 16, 18, 20}; std::vector<double> createAccelerationCurveForSensitivity(int32_t sensitivity, size_t propertySize) { @@ -169,6 +175,112 @@ void gestureInterpreterCallback(void* clientData, const Gesture* gesture) { mapper->consumeGesture(gesture); } +int32_t linuxBusToInputDeviceBusEnum(int32_t linuxBus, bool isUsiStylus) { + if (isUsiStylus) { + // This is a stylus connected over the Universal Stylus Initiative (USI) protocol. + // For metrics purposes, we treat this protocol as a separate bus. + return util::INPUT_DEVICE_USAGE_REPORTED__DEVICE_BUS__USI; + } + + // When adding cases to this switch, also add them to the copy of this method in + // InputDeviceMetricsCollector.cpp. + // TODO(b/286394420): deduplicate this method with the one in InputDeviceMetricsCollector.cpp. + switch (linuxBus) { + case BUS_USB: + return util::INPUT_DEVICE_USAGE_REPORTED__DEVICE_BUS__USB; + case BUS_BLUETOOTH: + return util::INPUT_DEVICE_USAGE_REPORTED__DEVICE_BUS__BLUETOOTH; + default: + return util::INPUT_DEVICE_USAGE_REPORTED__DEVICE_BUS__OTHER; + } +} + +class MetricsAccumulator { +public: + static MetricsAccumulator& getInstance() { + static MetricsAccumulator sAccumulator; + return sAccumulator; + } + + void recordFinger(const TouchpadInputMapper::MetricsIdentifier& id) { mCounters[id].fingers++; } + + void recordPalm(const TouchpadInputMapper::MetricsIdentifier& id) { mCounters[id].palms++; } + + // Checks whether a Gesture struct is for the end of a gesture that we log metrics for, and + // records it if so. + void processGesture(const TouchpadInputMapper::MetricsIdentifier& id, const Gesture& gesture) { + switch (gesture.type) { + case kGestureTypeFling: + if (gesture.details.fling.fling_state == GESTURES_FLING_START) { + // Indicates the end of a two-finger scroll gesture. + mCounters[id].twoFingerSwipeGestures++; + } + break; + case kGestureTypeSwipeLift: + mCounters[id].threeFingerSwipeGestures++; + break; + case kGestureTypeFourFingerSwipeLift: + mCounters[id].fourFingerSwipeGestures++; + break; + case kGestureTypePinch: + if (gesture.details.pinch.zoom_state == GESTURES_ZOOM_END) { + mCounters[id].pinchGestures++; + } + break; + default: + // We're not interested in any other gestures. + break; + } + } + +private: + MetricsAccumulator() { + AStatsManager_setPullAtomCallback(android::util::TOUCHPAD_USAGE, /*metadata=*/nullptr, + MetricsAccumulator::pullAtomCallback, /*cookie=*/nullptr); + } + + ~MetricsAccumulator() { AStatsManager_clearPullAtomCallback(android::util::TOUCHPAD_USAGE); } + + static AStatsManager_PullAtomCallbackReturn pullAtomCallback(int32_t atomTag, + AStatsEventList* outEventList, + void* cookie) { + LOG_ALWAYS_FATAL_IF(atomTag != android::util::TOUCHPAD_USAGE); + MetricsAccumulator& accumulator = MetricsAccumulator::getInstance(); + accumulator.produceAtoms(outEventList); + accumulator.resetCounters(); + return AStatsManager_PULL_SUCCESS; + } + + void produceAtoms(AStatsEventList* outEventList) const { + for (auto& [id, counters] : mCounters) { + auto [busId, vendorId, productId, versionId] = id; + addAStatsEvent(outEventList, android::util::TOUCHPAD_USAGE, vendorId, productId, + versionId, linuxBusToInputDeviceBusEnum(busId, /*isUsi=*/false), + counters.fingers, counters.palms, counters.twoFingerSwipeGestures, + counters.threeFingerSwipeGestures, counters.fourFingerSwipeGestures, + counters.pinchGestures); + } + } + + void resetCounters() { mCounters.clear(); } + + // Stores the counters for a specific touchpad model. Fields have the same meanings as those of + // the TouchpadUsage atom; see that definition for detailed documentation. + struct Counters { + int32_t fingers = 0; + int32_t palms = 0; + + int32_t twoFingerSwipeGestures = 0; + int32_t threeFingerSwipeGestures = 0; + int32_t fourFingerSwipeGestures = 0; + int32_t pinchGestures = 0; + }; + + // Metrics are aggregated by device model and version, so if two devices of the same model and + // version are connected at once, they will have the same counters. + std::map<TouchpadInputMapper::MetricsIdentifier, Counters> mCounters; +}; + } // namespace TouchpadInputMapper::TouchpadInputMapper(InputDeviceContext& deviceContext, @@ -178,7 +290,8 @@ TouchpadInputMapper::TouchpadInputMapper(InputDeviceContext& deviceContext, mPointerController(getContext()->getPointerController(getDeviceId())), mStateConverter(deviceContext, mMotionAccumulator), mGestureConverter(*getContext(), deviceContext, getDeviceId()), - mCapturedEventConverter(*getContext(), deviceContext, mMotionAccumulator, getDeviceId()) { + mCapturedEventConverter(*getContext(), deviceContext, mMotionAccumulator, getDeviceId()), + mMetricsId(metricsIdFromInputDeviceIdentifier(deviceContext.getDeviceIdentifier())) { RawAbsoluteAxisInfo slotAxisInfo; deviceContext.getAbsoluteAxisInfo(ABS_MT_SLOT, &slotAxisInfo); if (!slotAxisInfo.valid || slotAxisInfo.maxValue <= 0) { @@ -331,12 +444,39 @@ std::list<NotifyArgs> TouchpadInputMapper::process(const RawEvent* rawEvent) { } std::optional<SelfContainedHardwareState> state = mStateConverter.processRawEvent(rawEvent); if (state) { + updatePalmDetectionMetrics(); return sendHardwareState(rawEvent->when, rawEvent->readTime, *state); } else { return {}; } } +void TouchpadInputMapper::updatePalmDetectionMetrics() { + std::set<int32_t> currentTrackingIds; + for (size_t i = 0; i < mMotionAccumulator.getSlotCount(); i++) { + const MultiTouchMotionAccumulator::Slot& slot = mMotionAccumulator.getSlot(i); + if (!slot.isInUse()) { + continue; + } + currentTrackingIds.insert(slot.getTrackingId()); + if (slot.getToolType() == ToolType::PALM) { + mPalmTrackingIds.insert(slot.getTrackingId()); + } + } + std::vector<int32_t> liftedTouches; + std::set_difference(mLastFrameTrackingIds.begin(), mLastFrameTrackingIds.end(), + currentTrackingIds.begin(), currentTrackingIds.end(), + std::inserter(liftedTouches, liftedTouches.begin())); + for (int32_t trackingId : liftedTouches) { + if (mPalmTrackingIds.erase(trackingId) > 0) { + MetricsAccumulator::getInstance().recordPalm(mMetricsId); + } else { + MetricsAccumulator::getInstance().recordFinger(mMetricsId); + } + } + mLastFrameTrackingIds = currentTrackingIds; +} + std::list<NotifyArgs> TouchpadInputMapper::sendHardwareState(nsecs_t when, nsecs_t readTime, SelfContainedHardwareState schs) { ALOGD_IF(DEBUG_TOUCHPAD_GESTURES, "New hardware state: %s", schs.state.String().c_str()); @@ -363,8 +503,10 @@ void TouchpadInputMapper::consumeGesture(const Gesture* gesture) { std::list<NotifyArgs> TouchpadInputMapper::processGestures(nsecs_t when, nsecs_t readTime) { std::list<NotifyArgs> out = {}; + MetricsAccumulator& metricsAccumulator = MetricsAccumulator::getInstance(); for (Gesture& gesture : mGesturesToProcess) { out += mGestureConverter.handleGesture(when, readTime, gesture); + metricsAccumulator.processGesture(mMetricsId, gesture); } mGesturesToProcess.clear(); return out; diff --git a/services/inputflinger/reader/mapper/TouchpadInputMapper.h b/services/inputflinger/reader/mapper/TouchpadInputMapper.h index 23d0fd3cf3..73ca5afa04 100644 --- a/services/inputflinger/reader/mapper/TouchpadInputMapper.h +++ b/services/inputflinger/reader/mapper/TouchpadInputMapper.h @@ -18,6 +18,7 @@ #include <list> #include <memory> +#include <set> #include <vector> #include <PointerControllerInterface.h> @@ -58,10 +59,16 @@ public: void consumeGesture(const Gesture* gesture); + // A subset of InputDeviceIdentifier used for logging metrics, to avoid storing a copy of the + // strings in that bigger struct. + using MetricsIdentifier = std::tuple<uint16_t /*busId*/, uint16_t /*vendorId*/, + uint16_t /*productId*/, uint16_t /*version*/>; + private: void resetGestureInterpreter(nsecs_t when); explicit TouchpadInputMapper(InputDeviceContext& deviceContext, const InputReaderConfiguration& readerConfig); + void updatePalmDetectionMetrics(); [[nodiscard]] std::list<NotifyArgs> sendHardwareState(nsecs_t when, nsecs_t readTime, SelfContainedHardwareState schs); [[nodiscard]] std::list<NotifyArgs> processGestures(nsecs_t when, nsecs_t readTime); @@ -86,6 +93,15 @@ private: bool mProcessing = false; bool mResettingInterpreter = false; std::vector<Gesture> mGesturesToProcess; + + static MetricsIdentifier metricsIdFromInputDeviceIdentifier(const InputDeviceIdentifier& id) { + return std::make_tuple(id.bus, id.vendor, id.product, id.version); + } + const MetricsIdentifier mMetricsId; + // Tracking IDs for touches on the pad in the last evdev frame. + std::set<int32_t> mLastFrameTrackingIds; + // Tracking IDs for touches that have at some point been reported as palms by the touchpad. + std::set<int32_t> mPalmTrackingIds; }; } // namespace android diff --git a/services/inputflinger/reader/mapper/gestures/GestureConverter.cpp b/services/inputflinger/reader/mapper/gestures/GestureConverter.cpp index 7eca6fa0b4..3abf2bd60b 100644 --- a/services/inputflinger/reader/mapper/gestures/GestureConverter.cpp +++ b/services/inputflinger/reader/mapper/gestures/GestureConverter.cpp @@ -153,6 +153,9 @@ NotifyMotionArgs GestureConverter::handleMove(nsecs_t when, nsecs_t readTime, const Gesture& gesture) { float deltaX = gesture.details.move.dx; float deltaY = gesture.details.move.dy; + if (std::abs(deltaX) > 0 || std::abs(deltaY) > 0) { + enableTapToClick(); + } rotateDelta(mOrientation, &deltaX, &deltaY); mPointerController->setPresentation(PointerControllerInterface::Presentation::POINTER); @@ -191,6 +194,15 @@ std::list<NotifyArgs> GestureConverter::handleButtonsChange(nsecs_t when, nsecs_ coords.setAxisValue(AMOTION_EVENT_AXIS_Y, yCursorPosition); coords.setAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X, 0); coords.setAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y, 0); + + if (mReaderContext.isPreventingTouchpadTaps()) { + enableTapToClick(); + if (gesture.details.buttons.is_tap) { + // return early to prevent this tap + return out; + } + } + const uint32_t buttonsPressed = gesture.details.buttons.down; bool pointerDown = isPointerDown(mButtonState) || buttonsPressed & @@ -239,6 +251,11 @@ std::list<NotifyArgs> GestureConverter::handleButtonsChange(nsecs_t when, nsecs_ out.push_back(makeMotionArgs(when, readTime, AMOTION_EVENT_ACTION_UP, /* actionButton= */ 0, newButtonState, /* pointerCount= */ 1, mFingerProps.data(), &coords, xCursorPosition, yCursorPosition)); + // Send a HOVER_MOVE to tell the application that the mouse is hovering again. + out.push_back(makeMotionArgs(when, readTime, AMOTION_EVENT_ACTION_HOVER_MOVE, + /*actionButton=*/0, newButtonState, /*pointerCount=*/1, + mFingerProps.data(), &coords, xCursorPosition, + yCursorPosition)); } mButtonState = newButtonState; return out; @@ -332,6 +349,9 @@ std::list<NotifyArgs> GestureConverter::handleFling(nsecs_t when, nsecs_t readTi // magnitude, which will also result in the pointer icon being updated. // TODO(b/282023644): Add a signal in libgestures for when a stable contact has been // initiated with a touchpad. + if (!mReaderContext.isPreventingTouchpadTaps()) { + enableTapToClick(); + } return {handleMove(when, readTime, Gesture(kGestureMove, gesture.start_time, gesture.end_time, /*dx=*/0.f, @@ -385,6 +405,8 @@ NotifyMotionArgs GestureConverter::endScroll(nsecs_t when, nsecs_t readTime) { } mDownTime = when; + mFakeFingerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_GESTURE_SWIPE_FINGER_COUNT, + fingerCount); out.push_back(makeMotionArgs(when, readTime, AMOTION_EVENT_ACTION_DOWN, /* actionButton= */ 0, mButtonState, /* pointerCount= */ 1, mFingerProps.data(), mFakeFingerCoords.data(), xCursorPosition, @@ -441,6 +463,7 @@ NotifyMotionArgs GestureConverter::endScroll(nsecs_t when, nsecs_t readTime) { /* actionButton= */ 0, mButtonState, /* pointerCount= */ 1, mFingerProps.data(), mFakeFingerCoords.data(), xCursorPosition, yCursorPosition)); + mFakeFingerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_GESTURE_SWIPE_FINGER_COUNT, 0); mCurrentClassification = MotionClassification::NONE; mSwipeFingerCount = 0; return out; @@ -537,7 +560,7 @@ NotifyMotionArgs GestureConverter::makeMotionArgs(nsecs_t when, nsecs_t readTime /* policyFlags= */ POLICY_FLAG_WAKE, action, /* actionButton= */ actionButton, - /* flags= */ 0, + /* flags= */ action == AMOTION_EVENT_ACTION_CANCEL ? AMOTION_EVENT_FLAG_CANCELED : 0, mReaderContext.getGlobalMetaState(), buttonState, mCurrentClassification, @@ -553,4 +576,8 @@ NotifyMotionArgs GestureConverter::makeMotionArgs(nsecs_t when, nsecs_t readTime /* videoFrames= */ {}}; } +void GestureConverter::enableTapToClick() { + mReaderContext.setPreventingTouchpadTaps(false); +} + } // namespace android diff --git a/services/inputflinger/reader/mapper/gestures/GestureConverter.h b/services/inputflinger/reader/mapper/gestures/GestureConverter.h index b613b887cd..3ea3790620 100644 --- a/services/inputflinger/reader/mapper/gestures/GestureConverter.h +++ b/services/inputflinger/reader/mapper/gestures/GestureConverter.h @@ -78,6 +78,8 @@ private: const PointerCoords* pointerCoords, float xCursorPosition, float yCursorPosition); + void enableTapToClick(); + const int32_t mDeviceId; InputReaderContext& mReaderContext; std::shared_ptr<PointerControllerInterface> mPointerController; diff --git a/services/inputflinger/reporter/Android.bp b/services/inputflinger/reporter/Android.bp index 693ff063b1..b1e1aee02a 100644 --- a/services/inputflinger/reporter/Android.bp +++ b/services/inputflinger/reporter/Android.bp @@ -37,6 +37,7 @@ filegroup { cc_defaults { name: "libinputreporter_defaults", srcs: [":libinputreporter_sources"], + host_supported: true, shared_libs: [ "liblog", "libutils", diff --git a/services/inputflinger/tests/Android.bp b/services/inputflinger/tests/Android.bp index 569690ab78..1585fddfb3 100644 --- a/services/inputflinger/tests/Android.bp +++ b/services/inputflinger/tests/Android.bp @@ -48,6 +48,7 @@ cc_test { "FocusResolver_test.cpp", "GestureConverter_test.cpp", "HardwareStateConverter_test.cpp", + "InputDeviceMetricsCollector_test.cpp", "InputMapperTest.cpp", "InputProcessor_test.cpp", "InputProcessorConverter_test.cpp", @@ -58,8 +59,10 @@ cc_test { "NotifyArgs_test.cpp", "PreferStylusOverTouch_test.cpp", "PropertyProvider_test.cpp", + "SyncQueue_test.cpp", "TestInputListener.cpp", "TouchpadInputMapper_test.cpp", + "KeyboardInputMapper_test.cpp", "UinputDevice.cpp", "UnwantedInteractionBlocker_test.cpp", ], @@ -77,6 +80,9 @@ cc_test { ], }, host: { + sanitize: { + address: true, + }, include_dirs: [ "bionic/libc/kernel/android/uapi/", "bionic/libc/kernel/uapi", @@ -90,6 +96,7 @@ cc_test { }, }, sanitize: { + hwaddress: true, undefined: true, all_undefined: true, diag: { diff --git a/services/inputflinger/tests/BlockingQueue_test.cpp b/services/inputflinger/tests/BlockingQueue_test.cpp index fd9d9d5bd3..754a5c451e 100644 --- a/services/inputflinger/tests/BlockingQueue_test.cpp +++ b/services/inputflinger/tests/BlockingQueue_test.cpp @@ -22,6 +22,7 @@ namespace android { +using std::chrono_literals::operator""ns; // --- BlockingQueueTest --- @@ -34,6 +35,14 @@ TEST(BlockingQueueTest, Queue_AddAndRemove) { ASSERT_TRUE(queue.push(1)); ASSERT_EQ(queue.pop(), 1); + + ASSERT_TRUE(queue.emplace(2)); + ASSERT_EQ(queue.popWithTimeout(0ns), 2); + + ASSERT_TRUE(queue.push(3)); + ASSERT_EQ(queue.popWithTimeout(100ns), 3); + + ASSERT_EQ(std::nullopt, queue.popWithTimeout(0ns)); } /** @@ -87,7 +96,7 @@ TEST(BlockingQueueTest, Queue_Erases) { queue.push(3); queue.push(4); // Erase elements 2 and 4 - queue.erase([](int element) { return element == 2 || element == 4; }); + queue.erase_if([](int element) { return element == 2 || element == 4; }); // Should no longer receive elements 2 and 4 ASSERT_EQ(1, queue.pop()); ASSERT_EQ(3, queue.pop()); @@ -138,5 +147,9 @@ TEST(BlockingQueueTest, Queue_BlocksWhileWaitingForElements) { ASSERT_TRUE(hasReceivedElement); } +TEST(BlockingQueueTest, Queue_TimesOut) { + BlockingQueue<int> queue; + ASSERT_EQ(std::nullopt, queue.popWithTimeout(1ns)); +} } // namespace android diff --git a/services/inputflinger/tests/FakeInputReaderPolicy.cpp b/services/inputflinger/tests/FakeInputReaderPolicy.cpp index 3486d0f2a4..30222bf407 100644 --- a/services/inputflinger/tests/FakeInputReaderPolicy.cpp +++ b/services/inputflinger/tests/FakeInputReaderPolicy.cpp @@ -209,6 +209,14 @@ void FakeInputReaderPolicy::setStylusPointerIconEnabled(bool enabled) { mConfig.stylusPointerIconEnabled = enabled; } +void FakeInputReaderPolicy::setIsInputMethodConnectionActive(bool active) { + mIsInputMethodConnectionActive = active; +} + +bool FakeInputReaderPolicy::isInputMethodConnectionActive() { + return mIsInputMethodConnectionActive; +} + void FakeInputReaderPolicy::getReaderConfiguration(InputReaderConfiguration* outConfig) { *outConfig = mConfig; } diff --git a/services/inputflinger/tests/FakeInputReaderPolicy.h b/services/inputflinger/tests/FakeInputReaderPolicy.h index 85ff01a071..78bb2c3894 100644 --- a/services/inputflinger/tests/FakeInputReaderPolicy.h +++ b/services/inputflinger/tests/FakeInputReaderPolicy.h @@ -77,6 +77,8 @@ public: void setVelocityControlParams(const VelocityControlParameters& params); void setStylusButtonMotionEventsEnabled(bool enabled); void setStylusPointerIconEnabled(bool enabled); + void setIsInputMethodConnectionActive(bool active); + bool isInputMethodConnectionActive() override; private: void getReaderConfiguration(InputReaderConfiguration* outConfig) override; @@ -99,6 +101,7 @@ private: std::vector<DisplayViewport> mViewports; TouchAffineTransformation transform; std::optional<int32_t /*deviceId*/> mStylusGestureNotified GUARDED_BY(mLock){}; + bool mIsInputMethodConnectionActive{false}; uint32_t mNextPointerCaptureSequenceNumber{0}; }; diff --git a/services/inputflinger/tests/FocusResolver_test.cpp b/services/inputflinger/tests/FocusResolver_test.cpp index 5440a98db6..2ff9c3c784 100644 --- a/services/inputflinger/tests/FocusResolver_test.cpp +++ b/services/inputflinger/tests/FocusResolver_test.cpp @@ -31,6 +31,8 @@ using android::gui::WindowInfoHandle; namespace android::inputdispatcher { +namespace { + class FakeWindowHandle : public WindowInfoHandle { public: FakeWindowHandle(const std::string& name, const sp<IBinder>& token, bool focusable, @@ -49,6 +51,8 @@ public: } }; +} // namespace + TEST(FocusResolverTest, SetFocusedWindow) { sp<IBinder> focusableWindowToken = sp<BBinder>::make(); sp<IBinder> invisibleWindowToken = sp<BBinder>::make(); diff --git a/services/inputflinger/tests/GestureConverter_test.cpp b/services/inputflinger/tests/GestureConverter_test.cpp index a723636519..4df0f69481 100644 --- a/services/inputflinger/tests/GestureConverter_test.cpp +++ b/services/inputflinger/tests/GestureConverter_test.cpp @@ -169,7 +169,7 @@ TEST_F(GestureConverterTest, ButtonsChange) { /* down= */ GESTURES_BUTTON_NONE, /* up= */ GESTURES_BUTTON_RIGHT, /* is_tap= */ false); args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, rightUpGesture); - ASSERT_EQ(2u, args.size()); + ASSERT_EQ(3u, args.size()); ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_BUTTON_RELEASE), @@ -181,6 +181,10 @@ TEST_F(GestureConverterTest, ButtonsChange) { AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithButtonState(0), WithCoords(POINTER_X, POINTER_Y), WithToolType(ToolType::FINGER))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), WithButtonState(0), + WithCoords(POINTER_X, POINTER_Y), WithToolType(ToolType::FINGER))); } TEST_F(GestureConverterTest, DragWithButton) { @@ -225,7 +229,7 @@ TEST_F(GestureConverterTest, DragWithButton) { /* down= */ GESTURES_BUTTON_NONE, /* up= */ GESTURES_BUTTON_LEFT, /* is_tap= */ false); args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, upGesture); - ASSERT_EQ(2u, args.size()); + ASSERT_EQ(3u, args.size()); ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_BUTTON_RELEASE), @@ -237,6 +241,10 @@ TEST_F(GestureConverterTest, DragWithButton) { AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithButtonState(0), WithCoords(POINTER_X - 5, POINTER_Y + 10), WithToolType(ToolType::FINGER))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), WithButtonState(0), + WithCoords(POINTER_X - 5, POINTER_Y + 10), WithToolType(ToolType::FINGER))); } TEST_F(GestureConverterTest, Scroll) { @@ -332,7 +340,7 @@ TEST_F(GestureConverterTest, Scroll_Rotated) { WithToolType(ToolType::FINGER))); } -TEST_F(GestureConverterTest, Scroll_ClearsClassificationAndOffsetsAfterGesture) { +TEST_F(GestureConverterTest, Scroll_ClearsClassificationAfterGesture) { InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); @@ -349,29 +357,71 @@ TEST_F(GestureConverterTest, Scroll_ClearsClassificationAndOffsetsAfterGesture) Gesture moveGesture(kGestureMove, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, -5, 10); args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, moveGesture); ASSERT_EQ(1u, args.size()); - ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), - AllOf(WithMotionClassification(MotionClassification::NONE), - WithGestureScrollDistance(0, 0, EPSILON))); + EXPECT_THAT(std::get<NotifyMotionArgs>(args.front()), + WithMotionClassification(MotionClassification::NONE)); } -TEST_F(GestureConverterTest, ThreeFingerSwipe_ClearsClassificationAndOffsetsAfterGesture) { +TEST_F(GestureConverterTest, Scroll_ClearsScrollDistanceAfterGesture) { InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); - Gesture startGesture(kGestureSwipe, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /* dx= */ 0, - /* dy= */ 0); + Gesture startGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, -10); + std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, startGesture); + + Gesture continueGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 0, -5); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, continueGesture); + + Gesture flingGesture(kGestureFling, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, 1, 1, + GESTURES_FLING_START); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, flingGesture); + + // Move gestures don't use the fake finger array, so to test that gesture axes are cleared we + // need to use another gesture type, like pinch. + Gesture pinchGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dz=*/1, + GESTURES_ZOOM_START); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, pinchGesture); + ASSERT_FALSE(args.empty()); + EXPECT_THAT(std::get<NotifyMotionArgs>(args.front()), WithGestureScrollDistance(0, 0, EPSILON)); +} + +TEST_F(GestureConverterTest, ThreeFingerSwipe_ClearsClassificationAfterGesture) { + InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); + GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); + + Gesture startGesture(kGestureSwipe, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dx=*/0, + /*dy=*/0); std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, startGesture); Gesture liftGesture(kGestureSwipeLift, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME); args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, liftGesture); - Gesture moveGesture(kGestureMove, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /* dx= */ -5, - /* dy= */ 10); + Gesture moveGesture(kGestureMove, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dx=*/-5, + /*dy=*/10); args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, moveGesture); ASSERT_EQ(1u, args.size()); - ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), - AllOf(WithMotionClassification(MotionClassification::NONE), - WithGestureOffset(0, 0, EPSILON))); + EXPECT_THAT(std::get<NotifyMotionArgs>(args.front()), + WithMotionClassification(MotionClassification::NONE)); +} + +TEST_F(GestureConverterTest, ThreeFingerSwipe_ClearsGestureAxesAfterGesture) { + InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); + GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); + + Gesture startGesture(kGestureSwipe, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dx=*/5, + /*dy=*/5); + std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, startGesture); + + Gesture liftGesture(kGestureSwipeLift, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, liftGesture); + + // Move gestures don't use the fake finger array, so to test that gesture axes are cleared we + // need to use another gesture type, like pinch. + Gesture pinchGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dz=*/1, + GESTURES_ZOOM_START); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, pinchGesture); + ASSERT_FALSE(args.empty()); + EXPECT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(0))); } TEST_F(GestureConverterTest, ThreeFingerSwipe_Vertical) { @@ -392,6 +442,7 @@ TEST_F(GestureConverterTest, ThreeFingerSwipe_Vertical) { NotifyMotionArgs arg = std::get<NotifyMotionArgs>(args.front()); ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), WithGestureOffset(0, 0, EPSILON), + WithGestureSwipeFingerCount(3), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(1u), WithToolType(ToolType::FINGER))); PointerCoords finger0Start = arg.pointerCoords[0]; @@ -400,7 +451,7 @@ TEST_F(GestureConverterTest, ThreeFingerSwipe_Vertical) { ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_DOWN | 1 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(3), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(2u), WithToolType(ToolType::FINGER))); PointerCoords finger1Start = arg.pointerCoords[1]; @@ -409,7 +460,7 @@ TEST_F(GestureConverterTest, ThreeFingerSwipe_Vertical) { ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_DOWN | 2 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(3), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(3u), WithToolType(ToolType::FINGER))); PointerCoords finger2Start = arg.pointerCoords[2]; @@ -418,7 +469,7 @@ TEST_F(GestureConverterTest, ThreeFingerSwipe_Vertical) { arg = std::get<NotifyMotionArgs>(args.front()); ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_MOVE), - WithGestureOffset(0, -0.01, EPSILON), + WithGestureOffset(0, -0.01, EPSILON), WithGestureSwipeFingerCount(3), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(3u), WithToolType(ToolType::FINGER))); EXPECT_EQ(arg.pointerCoords[0].getX(), finger0Start.getX()); @@ -435,7 +486,7 @@ TEST_F(GestureConverterTest, ThreeFingerSwipe_Vertical) { arg = std::get<NotifyMotionArgs>(args.front()); ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_MOVE), - WithGestureOffset(0, -0.005, EPSILON), + WithGestureOffset(0, -0.005, EPSILON), WithGestureSwipeFingerCount(3), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(3u), WithToolType(ToolType::FINGER))); EXPECT_EQ(arg.pointerCoords[0].getX(), finger0Start.getX()); @@ -451,19 +502,20 @@ TEST_F(GestureConverterTest, ThreeFingerSwipe_Vertical) { ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_UP | 2 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(3), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(3u), WithToolType(ToolType::FINGER))); args.pop_front(); ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_UP | 1 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(3), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(2u), WithToolType(ToolType::FINGER))); args.pop_front(); ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithGestureOffset(0, 0, EPSILON), + WithGestureSwipeFingerCount(3), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(1u), WithToolType(ToolType::FINGER))); } @@ -559,6 +611,7 @@ TEST_F(GestureConverterTest, FourFingerSwipe_Horizontal) { NotifyMotionArgs arg = std::get<NotifyMotionArgs>(args.front()); ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), WithGestureOffset(0, 0, EPSILON), + WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(1u), WithToolType(ToolType::FINGER))); PointerCoords finger0Start = arg.pointerCoords[0]; @@ -567,7 +620,7 @@ TEST_F(GestureConverterTest, FourFingerSwipe_Horizontal) { ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_DOWN | 1 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(2u), WithToolType(ToolType::FINGER))); PointerCoords finger1Start = arg.pointerCoords[1]; @@ -576,7 +629,7 @@ TEST_F(GestureConverterTest, FourFingerSwipe_Horizontal) { ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_DOWN | 2 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(3u), WithToolType(ToolType::FINGER))); PointerCoords finger2Start = arg.pointerCoords[2]; @@ -585,7 +638,7 @@ TEST_F(GestureConverterTest, FourFingerSwipe_Horizontal) { ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_DOWN | 3 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(4u), WithToolType(ToolType::FINGER))); PointerCoords finger3Start = arg.pointerCoords[3]; @@ -594,7 +647,7 @@ TEST_F(GestureConverterTest, FourFingerSwipe_Horizontal) { arg = std::get<NotifyMotionArgs>(args.front()); ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_MOVE), - WithGestureOffset(0.01, 0, EPSILON), + WithGestureOffset(0.01, 0, EPSILON), WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(4u), WithToolType(ToolType::FINGER))); EXPECT_EQ(arg.pointerCoords[0].getX(), finger0Start.getX() + 10); @@ -613,7 +666,7 @@ TEST_F(GestureConverterTest, FourFingerSwipe_Horizontal) { arg = std::get<NotifyMotionArgs>(args.front()); ASSERT_THAT(arg, AllOf(WithMotionAction(AMOTION_EVENT_ACTION_MOVE), - WithGestureOffset(0.005, 0, EPSILON), + WithGestureOffset(0.005, 0, EPSILON), WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(4u), WithToolType(ToolType::FINGER))); EXPECT_EQ(arg.pointerCoords[0].getX(), finger0Start.getX() + 15); @@ -631,26 +684,27 @@ TEST_F(GestureConverterTest, FourFingerSwipe_Horizontal) { ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_UP | 3 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(4u), WithToolType(ToolType::FINGER))); args.pop_front(); ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_UP | 2 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(3u), WithToolType(ToolType::FINGER))); args.pop_front(); ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_POINTER_UP | 1 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT), - WithGestureOffset(0, 0, EPSILON), + WithGestureOffset(0, 0, EPSILON), WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(2u), WithToolType(ToolType::FINGER))); args.pop_front(); ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithGestureOffset(0, 0, EPSILON), + WithGestureSwipeFingerCount(4), WithMotionClassification(MotionClassification::MULTI_FINGER_SWIPE), WithPointerCount(1u), WithToolType(ToolType::FINGER))); } @@ -761,28 +815,52 @@ TEST_F(GestureConverterTest, Pinch_Outwards) { WithToolType(ToolType::FINGER))); } -TEST_F(GestureConverterTest, Pinch_ClearsClassificationAndScaleFactorAfterGesture) { +TEST_F(GestureConverterTest, Pinch_ClearsClassificationAfterGesture) { InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); - Gesture startGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /* dz= */ 1, + Gesture startGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dz=*/1, GESTURES_ZOOM_START); std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, startGesture); Gesture updateGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, - /* dz= */ 1.2, GESTURES_ZOOM_UPDATE); + /*dz=*/1.2, GESTURES_ZOOM_UPDATE); args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, updateGesture); - Gesture endGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /* dz= */ 1, + Gesture endGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dz=*/1, GESTURES_ZOOM_END); args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, endGesture); Gesture moveGesture(kGestureMove, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, -5, 10); args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, moveGesture); ASSERT_EQ(1u, args.size()); - ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), - AllOf(WithMotionClassification(MotionClassification::NONE), - WithGesturePinchScaleFactor(0, EPSILON))); + EXPECT_THAT(std::get<NotifyMotionArgs>(args.front()), + WithMotionClassification(MotionClassification::NONE)); +} + +TEST_F(GestureConverterTest, Pinch_ClearsScaleFactorAfterGesture) { + InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); + GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); + + Gesture startGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dz=*/1, + GESTURES_ZOOM_START); + std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, startGesture); + + Gesture updateGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, + /*dz=*/1.2, GESTURES_ZOOM_UPDATE); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, updateGesture); + + Gesture endGesture(kGesturePinch, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dz=*/1, + GESTURES_ZOOM_END); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, endGesture); + + // Move gestures don't use the fake finger array, so to test that gesture axes are cleared we + // need to use another gesture type, like scroll. + Gesture scrollGesture(kGestureScroll, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /*dx=*/1, + /*dy=*/0); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, scrollGesture); + ASSERT_FALSE(args.empty()); + EXPECT_THAT(std::get<NotifyMotionArgs>(args.front()), WithGesturePinchScaleFactor(0, EPSILON)); } TEST_F(GestureConverterTest, ResetWithButtonPressed) { @@ -905,4 +983,226 @@ TEST_F(GestureConverterTest, FlingTapDown) { ASSERT_TRUE(mFakePointerController->isPointerShown()); } +TEST_F(GestureConverterTest, Tap) { + // Tap should produce button press/release events + InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); + GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); + + Gesture flingGesture(kGestureFling, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /* vx= */ 0, + /* vy= */ 0, GESTURES_FLING_TAP_DOWN); + std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, flingGesture); + + ASSERT_EQ(1u, args.size()); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0, 0), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(0.0f))); + + Gesture tapGesture(kGestureButtonsChange, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, + /* down= */ GESTURES_BUTTON_LEFT, + /* up= */ GESTURES_BUTTON_LEFT, /* is_tap= */ true); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, tapGesture); + + ASSERT_EQ(5u, args.size()); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), WithCoords(POINTER_X, POINTER_Y), + WithRelativeMotion(0.f, 0.f), WithToolType(ToolType::FINGER), + WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), WithPressure(1.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_BUTTON_PRESS), + WithActionButton(AMOTION_EVENT_BUTTON_PRIMARY), + WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0.f, 0.f), + WithToolType(ToolType::FINGER), WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), + WithPressure(1.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_BUTTON_RELEASE), + WithActionButton(AMOTION_EVENT_BUTTON_PRIMARY), WithButtonState(0), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0.f, 0.f), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(1.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithCoords(POINTER_X, POINTER_Y), + WithRelativeMotion(0.f, 0.f), WithToolType(ToolType::FINGER), + WithButtonState(0), WithPressure(0.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0, 0), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(0.0f))); +} + +TEST_F(GestureConverterTest, Click) { + // Click should produce button press/release events + InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); + GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); + + Gesture flingGesture(kGestureFling, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /* vx= */ 0, + /* vy= */ 0, GESTURES_FLING_TAP_DOWN); + std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, flingGesture); + + ASSERT_EQ(1u, args.size()); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0, 0), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(0.0f))); + + Gesture buttonDownGesture(kGestureButtonsChange, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, + /* down= */ GESTURES_BUTTON_LEFT, + /* up= */ GESTURES_BUTTON_NONE, /* is_tap= */ false); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, buttonDownGesture); + + ASSERT_EQ(2u, args.size()); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), WithCoords(POINTER_X, POINTER_Y), + WithRelativeMotion(0.f, 0.f), WithToolType(ToolType::FINGER), + WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), WithPressure(1.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_BUTTON_PRESS), + WithActionButton(AMOTION_EVENT_BUTTON_PRIMARY), + WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0.f, 0.f), + WithToolType(ToolType::FINGER), WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), + WithPressure(1.0f))); + + Gesture buttonUpGesture(kGestureButtonsChange, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, + /* down= */ GESTURES_BUTTON_NONE, + /* up= */ GESTURES_BUTTON_LEFT, /* is_tap= */ false); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, buttonUpGesture); + + ASSERT_EQ(3u, args.size()); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_BUTTON_RELEASE), + WithActionButton(AMOTION_EVENT_BUTTON_PRIMARY), WithButtonState(0), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0.f, 0.f), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(1.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithCoords(POINTER_X, POINTER_Y), + WithRelativeMotion(0.f, 0.f), WithToolType(ToolType::FINGER), + WithButtonState(0), WithPressure(0.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0, 0), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(0.0f))); +} + +TEST_F(GestureConverterTest, TapWithTapToClickDisabled) { + // Tap should be ignored when disabled + mReader->getContext()->setPreventingTouchpadTaps(true); + + InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); + GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); + + Gesture flingGesture(kGestureFling, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /* vx= */ 0, + /* vy= */ 0, GESTURES_FLING_TAP_DOWN); + std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, flingGesture); + + ASSERT_EQ(1u, args.size()); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0, 0), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(0.0f))); + args.pop_front(); + + Gesture tapGesture(kGestureButtonsChange, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, + /* down= */ GESTURES_BUTTON_LEFT, + /* up= */ GESTURES_BUTTON_LEFT, /* is_tap= */ true); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, tapGesture); + + // no events should be generated + ASSERT_EQ(0u, args.size()); + + // Future taps should be re-enabled + ASSERT_FALSE(mReader->getContext()->isPreventingTouchpadTaps()); +} + +TEST_F(GestureConverterTest, ClickWithTapToClickDisabled) { + // Click should still produce button press/release events + mReader->getContext()->setPreventingTouchpadTaps(true); + + InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); + GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); + + Gesture flingGesture(kGestureFling, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, /* vx= */ 0, + /* vy= */ 0, GESTURES_FLING_TAP_DOWN); + std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, flingGesture); + + ASSERT_EQ(1u, args.size()); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0, 0), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(0.0f))); + + Gesture buttonDownGesture(kGestureButtonsChange, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, + /* down= */ GESTURES_BUTTON_LEFT, + /* up= */ GESTURES_BUTTON_NONE, /* is_tap= */ false); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, buttonDownGesture); + ASSERT_EQ(2u, args.size()); + + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), WithCoords(POINTER_X, POINTER_Y), + WithRelativeMotion(0.f, 0.f), WithToolType(ToolType::FINGER), + WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), WithPressure(1.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_BUTTON_PRESS), + WithActionButton(AMOTION_EVENT_BUTTON_PRIMARY), + WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0.f, 0.f), + WithToolType(ToolType::FINGER), WithButtonState(AMOTION_EVENT_BUTTON_PRIMARY), + WithPressure(1.0f))); + + Gesture buttonUpGesture(kGestureButtonsChange, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, + /* down= */ GESTURES_BUTTON_NONE, + /* up= */ GESTURES_BUTTON_LEFT, /* is_tap= */ false); + args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, buttonUpGesture); + + ASSERT_EQ(3u, args.size()); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_BUTTON_RELEASE), + WithActionButton(AMOTION_EVENT_BUTTON_PRIMARY), WithButtonState(0), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0.f, 0.f), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(1.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithCoords(POINTER_X, POINTER_Y), + WithRelativeMotion(0.f, 0.f), WithToolType(ToolType::FINGER), + WithButtonState(0), WithPressure(0.0f))); + args.pop_front(); + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), + WithCoords(POINTER_X, POINTER_Y), WithRelativeMotion(0, 0), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(0.0f))); + + // Future taps should be re-enabled + ASSERT_FALSE(mReader->getContext()->isPreventingTouchpadTaps()); +} + +TEST_F(GestureConverterTest, MoveEnablesTapToClick) { + // initially disable tap-to-click + mReader->getContext()->setPreventingTouchpadTaps(true); + + InputDeviceContext deviceContext(*mDevice, EVENTHUB_ID); + GestureConverter converter(*mReader->getContext(), deviceContext, DEVICE_ID); + + Gesture moveGesture(kGestureMove, ARBITRARY_GESTURE_TIME, ARBITRARY_GESTURE_TIME, -5, 10); + std::list<NotifyArgs> args = converter.handleGesture(ARBITRARY_TIME, READ_TIME, moveGesture); + ASSERT_EQ(1u, args.size()); + + ASSERT_THAT(std::get<NotifyMotionArgs>(args.front()), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_HOVER_MOVE), + WithCoords(POINTER_X - 5, POINTER_Y + 10), WithRelativeMotion(-5, 10), + WithToolType(ToolType::FINGER), WithButtonState(0), WithPressure(0.0f))); + + ASSERT_NO_FATAL_FAILURE(mFakePointerController->assertPosition(POINTER_X - 5, POINTER_Y + 10)); + + // Future taps should be re-enabled + ASSERT_FALSE(mReader->getContext()->isPreventingTouchpadTaps()); +} + } // namespace android diff --git a/services/inputflinger/tests/InputDeviceMetricsCollector_test.cpp b/services/inputflinger/tests/InputDeviceMetricsCollector_test.cpp new file mode 100644 index 0000000000..7ccfacaf91 --- /dev/null +++ b/services/inputflinger/tests/InputDeviceMetricsCollector_test.cpp @@ -0,0 +1,788 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../InputDeviceMetricsCollector.h" + +#include <NotifyArgsBuilders.h> +#include <gtest/gtest.h> +#include <gui/constants.h> +#include <input/InputEventBuilders.h> +#include <linux/input.h> + +#include <array> +#include <tuple> + +#include "TestInputListener.h" + +namespace android { + +using std::chrono_literals::operator""ns; +using std::chrono::nanoseconds; + +namespace { + +constexpr auto USAGE_TIMEOUT = 8765309ns; +constexpr auto TIME = 999999ns; +constexpr auto ALL_USAGE_SOURCES = ftl::enum_range<InputDeviceUsageSource>(); + +constexpr int32_t DEVICE_ID = 3; +constexpr int32_t DEVICE_ID_2 = 4; +constexpr int32_t VID = 0xFEED; +constexpr int32_t PID = 0xDEAD; +constexpr int32_t VERSION = 0xBEEF; +const std::string DEVICE_NAME = "Half Dome"; +const std::string LOCATION = "California"; +const std::string UNIQUE_ID = "Yosemite"; +constexpr uint32_t TOUCHSCREEN = AINPUT_SOURCE_TOUCHSCREEN; +constexpr uint32_t STYLUS = AINPUT_SOURCE_STYLUS; +constexpr uint32_t KEY_SOURCES = + AINPUT_SOURCE_KEYBOARD | AINPUT_SOURCE_DPAD | AINPUT_SOURCE_GAMEPAD; +constexpr int32_t POINTER_1_DOWN = + AMOTION_EVENT_ACTION_POINTER_DOWN | (1 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT); + +InputDeviceIdentifier generateTestIdentifier(int32_t id = DEVICE_ID) { + InputDeviceIdentifier identifier; + identifier.name = DEVICE_NAME + "_" + std::to_string(id); + identifier.location = LOCATION; + identifier.uniqueId = UNIQUE_ID; + identifier.vendor = VID; + identifier.product = PID; + identifier.version = VERSION; + identifier.bus = BUS_USB; + return identifier; +} + +InputDeviceInfo generateTestDeviceInfo(int32_t id = DEVICE_ID, + uint32_t sources = TOUCHSCREEN | STYLUS, + bool isAlphabetic = false) { + auto info = InputDeviceInfo(); + info.initialize(id, /*generation=*/1, /*controllerNumber=*/1, generateTestIdentifier(id), + "alias", /*isExternal=*/false, /*hasMic=*/false, ADISPLAY_ID_NONE); + info.addSource(sources); + info.setKeyboardType(isAlphabetic ? AINPUT_KEYBOARD_TYPE_ALPHABETIC + : AINPUT_KEYBOARD_TYPE_NON_ALPHABETIC); + return info; +} + +const InputDeviceInfo ALPHABETIC_KEYBOARD_INFO = + generateTestDeviceInfo(DEVICE_ID, KEY_SOURCES, /*isAlphabetic=*/true); +const InputDeviceInfo NON_ALPHABETIC_KEYBOARD_INFO = + generateTestDeviceInfo(DEVICE_ID, KEY_SOURCES, /*isAlphabetic=*/false); +const InputDeviceInfo TOUCHSCREEN_STYLUS_INFO = generateTestDeviceInfo(DEVICE_ID); +const InputDeviceInfo SECOND_TOUCHSCREEN_STYLUS_INFO = generateTestDeviceInfo(DEVICE_ID_2); + +std::set<gui::Uid> uids(std::initializer_list<int32_t> vals) { + std::set<gui::Uid> set; + for (const auto val : vals) { + set.emplace(val); + } + return set; +} + +} // namespace + +// --- InputDeviceMetricsCollectorDeviceClassificationTest --- + +class DeviceClassificationFixture : public ::testing::Test, + public ::testing::WithParamInterface<InputDeviceUsageSource> {}; + +TEST_P(DeviceClassificationFixture, ValidClassifications) { + const InputDeviceUsageSource usageSource = GetParam(); + + // Use a switch to ensure a test is added for all source classifications. + switch (usageSource) { + case InputDeviceUsageSource::UNKNOWN: { + ASSERT_EQ(InputDeviceUsageSource::UNKNOWN, + getUsageSourceForKeyArgs(generateTestDeviceInfo(), + KeyArgsBuilder(AKEY_EVENT_ACTION_DOWN, TOUCHSCREEN) + .build())); + + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::UNKNOWN}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_MOVE, AINPUT_SOURCE_KEYBOARD) + .pointer(PointerBuilder(/*id=*/1, ToolType::PALM) + .x(100) + .y(200)) + .build())); + break; + } + + case InputDeviceUsageSource::BUTTONS: { + ASSERT_EQ(InputDeviceUsageSource::BUTTONS, + getUsageSourceForKeyArgs(NON_ALPHABETIC_KEYBOARD_INFO, + KeyArgsBuilder(AKEY_EVENT_ACTION_DOWN, KEY_SOURCES) + .keyCode(AKEYCODE_STYLUS_BUTTON_TAIL) + .build())); + break; + } + + case InputDeviceUsageSource::KEYBOARD: { + ASSERT_EQ(InputDeviceUsageSource::KEYBOARD, + getUsageSourceForKeyArgs(ALPHABETIC_KEYBOARD_INFO, + KeyArgsBuilder(AKEY_EVENT_ACTION_DOWN, KEY_SOURCES) + .build())); + break; + } + + case InputDeviceUsageSource::DPAD: { + ASSERT_EQ(InputDeviceUsageSource::DPAD, + getUsageSourceForKeyArgs(NON_ALPHABETIC_KEYBOARD_INFO, + KeyArgsBuilder(AKEY_EVENT_ACTION_DOWN, KEY_SOURCES) + .keyCode(AKEYCODE_DPAD_CENTER) + .build())); + + ASSERT_EQ(InputDeviceUsageSource::DPAD, + getUsageSourceForKeyArgs(ALPHABETIC_KEYBOARD_INFO, + KeyArgsBuilder(AKEY_EVENT_ACTION_DOWN, KEY_SOURCES) + .keyCode(AKEYCODE_DPAD_CENTER) + .build())); + break; + } + + case InputDeviceUsageSource::GAMEPAD: { + ASSERT_EQ(InputDeviceUsageSource::GAMEPAD, + getUsageSourceForKeyArgs(NON_ALPHABETIC_KEYBOARD_INFO, + KeyArgsBuilder(AKEY_EVENT_ACTION_DOWN, KEY_SOURCES) + .keyCode(AKEYCODE_BUTTON_A) + .build())); + + ASSERT_EQ(InputDeviceUsageSource::GAMEPAD, + getUsageSourceForKeyArgs(ALPHABETIC_KEYBOARD_INFO, + KeyArgsBuilder(AKEY_EVENT_ACTION_DOWN, KEY_SOURCES) + .keyCode(AKEYCODE_BUTTON_A) + .build())); + break; + } + + case InputDeviceUsageSource::JOYSTICK: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::JOYSTICK}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_MOVE, AINPUT_SOURCE_JOYSTICK) + .pointer(PointerBuilder(/*id=*/1, ToolType::UNKNOWN) + .axis(AMOTION_EVENT_AXIS_GAS, 1.f)) + .build())); + break; + } + + case InputDeviceUsageSource::MOUSE: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::MOUSE}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_HOVER_MOVE, + AINPUT_SOURCE_MOUSE) + .pointer(PointerBuilder(/*id=*/1, ToolType::MOUSE) + .x(100) + .y(200)) + .build())); + break; + } + + case InputDeviceUsageSource::MOUSE_CAPTURED: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::MOUSE_CAPTURED}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_MOVE, + AINPUT_SOURCE_MOUSE_RELATIVE) + .pointer(PointerBuilder(/*id=*/1, ToolType::MOUSE) + .x(100) + .y(200) + .axis(AMOTION_EVENT_AXIS_RELATIVE_X, 100) + .axis(AMOTION_EVENT_AXIS_RELATIVE_Y, 200)) + .build())); + break; + } + + case InputDeviceUsageSource::TOUCHPAD: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::TOUCHPAD}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_MOVE, AINPUT_SOURCE_MOUSE) + .pointer(PointerBuilder(/*id=*/1, ToolType::FINGER) + .x(100) + .y(200)) + .build())); + break; + } + + case InputDeviceUsageSource::TOUCHPAD_CAPTURED: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::TOUCHPAD_CAPTURED}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_MOVE, AINPUT_SOURCE_TOUCHPAD) + .pointer(PointerBuilder(/*id=*/1, ToolType::FINGER) + .x(100) + .y(200) + .axis(AMOTION_EVENT_AXIS_RELATIVE_X, 1) + .axis(AMOTION_EVENT_AXIS_RELATIVE_Y, 2)) + .build())); + break; + } + + case InputDeviceUsageSource::ROTARY_ENCODER: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::ROTARY_ENCODER}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_SCROLL, + AINPUT_SOURCE_ROTARY_ENCODER) + .pointer(PointerBuilder(/*id=*/1, ToolType::UNKNOWN) + .axis(AMOTION_EVENT_AXIS_SCROLL, 10) + .axis(AMOTION_EVENT_AXIS_VSCROLL, 10)) + .build())); + break; + } + + case InputDeviceUsageSource::STYLUS_DIRECT: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::STYLUS_DIRECT}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_HOVER_ENTER, + STYLUS | TOUCHSCREEN) + .pointer(PointerBuilder(/*id=*/1, ToolType::STYLUS) + .x(100) + .y(200)) + .build())); + break; + } + + case InputDeviceUsageSource::STYLUS_INDIRECT: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::STYLUS_INDIRECT}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_HOVER_ENTER, + STYLUS | TOUCHSCREEN | AINPUT_SOURCE_MOUSE) + .pointer(PointerBuilder(/*id=*/1, ToolType::STYLUS) + .x(100) + .y(200)) + .build())); + break; + } + + case InputDeviceUsageSource::STYLUS_FUSED: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::STYLUS_FUSED}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_HOVER_ENTER, + AINPUT_SOURCE_BLUETOOTH_STYLUS | TOUCHSCREEN) + .pointer(PointerBuilder(/*id=*/1, ToolType::STYLUS) + .x(100) + .y(200)) + .build())); + break; + } + + case InputDeviceUsageSource::TOUCH_NAVIGATION: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::TOUCH_NAVIGATION}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_MOVE, + AINPUT_SOURCE_TOUCH_NAVIGATION) + .pointer(PointerBuilder(/*id=*/1, ToolType::FINGER) + .x(100) + .y(200)) + .build())); + break; + } + + case InputDeviceUsageSource::TOUCHSCREEN: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::TOUCHSCREEN}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(POINTER_1_DOWN, TOUCHSCREEN) + .pointer(PointerBuilder(/*id=*/1, ToolType::FINGER) + .x(100) + .y(200)) + .pointer(PointerBuilder(/*id=*/2, ToolType::FINGER) + .x(300) + .y(400)) + .build())); + break; + } + + case InputDeviceUsageSource::TRACKBALL: { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::TRACKBALL}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(AMOTION_EVENT_ACTION_SCROLL, + AINPUT_SOURCE_TRACKBALL) + .pointer(PointerBuilder(/*id=*/1, ToolType::UNKNOWN) + .axis(AMOTION_EVENT_AXIS_VSCROLL, 100) + .axis(AMOTION_EVENT_AXIS_HSCROLL, 200)) + .build())); + break; + } + } +} + +INSTANTIATE_TEST_SUITE_P(InputDeviceMetricsCollectorDeviceClassificationTest, + DeviceClassificationFixture, + ::testing::ValuesIn(ALL_USAGE_SOURCES.begin(), ALL_USAGE_SOURCES.end()), + [](const testing::TestParamInfo<InputDeviceUsageSource>& testParamInfo) { + return ftl::enum_string(testParamInfo.param); + }); + +TEST(InputDeviceMetricsCollectorDeviceClassificationTest, MixedClassificationTouchscreenStylus) { + std::set<InputDeviceUsageSource> srcs{InputDeviceUsageSource::TOUCHSCREEN, + InputDeviceUsageSource::STYLUS_DIRECT}; + ASSERT_EQ(srcs, + getUsageSourcesForMotionArgs( + MotionArgsBuilder(POINTER_1_DOWN, TOUCHSCREEN | STYLUS) + .pointer(PointerBuilder(/*id=*/1, ToolType::FINGER).x(100).y(200)) + .pointer(PointerBuilder(/*id=*/2, ToolType::STYLUS).x(300).y(400)) + .build())); +} + +// --- InputDeviceMetricsCollectorTest --- + +class InputDeviceMetricsCollectorTest : public testing::Test, public InputDeviceMetricsLogger { +protected: + TestInputListener mTestListener; + InputDeviceMetricsCollector mMetricsCollector{mTestListener, *this, USAGE_TIMEOUT}; + + void assertUsageLogged(const InputDeviceInfo& info, nanoseconds duration, + std::optional<SourceUsageBreakdown> sourceBreakdown = {}, + std::optional<UidUsageBreakdown> uidBreakdown = {}) { + ASSERT_GE(mLoggedUsageSessions.size(), 1u); + const auto& [loggedInfo, report] = *mLoggedUsageSessions.begin(); + ASSERT_EQ(info.getIdentifier(), loggedInfo.getIdentifier()); + ASSERT_EQ(duration, report.usageDuration); + if (sourceBreakdown) { + ASSERT_EQ(sourceBreakdown, report.sourceBreakdown); + } + if (uidBreakdown) { + ASSERT_EQ(uidBreakdown, report.uidBreakdown); + } + mLoggedUsageSessions.erase(mLoggedUsageSessions.begin()); + } + + void assertUsageNotLogged() { ASSERT_TRUE(mLoggedUsageSessions.empty()); } + + void setCurrentTime(nanoseconds time) { mCurrentTime = time; } + + nsecs_t currentTime() const { return mCurrentTime.count(); } + + NotifyMotionArgs generateMotionArgs(int32_t deviceId, + uint32_t source = AINPUT_SOURCE_TOUCHSCREEN, + std::vector<ToolType> toolTypes = {ToolType::FINGER}) { + MotionArgsBuilder builder(AMOTION_EVENT_ACTION_MOVE, source); + for (size_t i = 0; i < toolTypes.size(); i++) { + builder.pointer(PointerBuilder(i, toolTypes[i])); + } + return builder.deviceId(deviceId) + .eventTime(mCurrentTime.count()) + .downTime(mCurrentTime.count()) + .build(); + } + +private: + std::vector<std::tuple<InputDeviceInfo, DeviceUsageReport>> mLoggedUsageSessions; + nanoseconds mCurrentTime{TIME}; + + nanoseconds getCurrentTime() override { return mCurrentTime; } + + void logInputDeviceUsageReported(const InputDeviceInfo& info, + const DeviceUsageReport& report) override { + mLoggedUsageSessions.emplace_back(info, report); + } +}; + +TEST_F(InputDeviceMetricsCollectorTest, DontLogUsageWhenDeviceNotRegistered) { + // Device was used. + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mTestListener.assertNotifyMotionWasCalled(); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device was used again after the usage timeout expired, but we still don't log usage. + setCurrentTime(TIME + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mTestListener.assertNotifyMotionWasCalled(); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +TEST_F(InputDeviceMetricsCollectorTest, DontLogUsageForIgnoredDevices) { + constexpr static std::array<int32_t, 2> ignoredDevices{ + {INVALID_INPUT_DEVICE_ID, VIRTUAL_KEYBOARD_ID}}; + + for (int32_t ignoredDeviceId : ignoredDevices) { + mMetricsCollector.notifyInputDevicesChanged( + {/*id=*/0, {generateTestDeviceInfo(ignoredDeviceId)}}); + + // Device was used. + mMetricsCollector.notifyMotion(generateMotionArgs(ignoredDeviceId)); + mTestListener.assertNotifyMotionWasCalled(); + mMetricsCollector.notifyDeviceInteraction(ignoredDeviceId, TIME.count(), uids({0, 1, 2})); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device was used again after the usage timeout expired, but we still don't log usage. + setCurrentTime(TIME + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(ignoredDeviceId)); + mTestListener.assertNotifyMotionWasCalled(); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Remove the ignored device, and ensure we still don't log usage. + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {}}); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + } +} + +TEST_F(InputDeviceMetricsCollectorTest, LogsSingleEventUsageSession) { + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {TOUCHSCREEN_STYLUS_INFO}}); + + // Device was used. + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device was used again after the usage timeout. + setCurrentTime(TIME + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + // The usage session has zero duration because it consisted of only one event. + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 0ns)); +} + +TEST_F(InputDeviceMetricsCollectorTest, LogsMultipleEventUsageSession) { + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {TOUCHSCREEN_STYLUS_INFO}}); + + // Device was used. + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device was used again after some time. + setCurrentTime(TIME + 21ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + + setCurrentTime(TIME + 42ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + + // Device was used again after the usage timeout. + setCurrentTime(TIME + 42ns + 2 * USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 42ns)); +} + +TEST_F(InputDeviceMetricsCollectorTest, RemovingDeviceEndsUsageSession) { + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {TOUCHSCREEN_STYLUS_INFO}}); + + // Device was used. + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device was used again after some time. + setCurrentTime(TIME + 21ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + + // The device was removed before the usage timeout expired. + setCurrentTime(TIME + 42ns); + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {}}); + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 21ns)); +} + +TEST_F(InputDeviceMetricsCollectorTest, TracksUsageFromDifferentDevicesIndependently) { + mMetricsCollector.notifyInputDevicesChanged( + {/*id=*/0, {TOUCHSCREEN_STYLUS_INFO, SECOND_TOUCHSCREEN_STYLUS_INFO}}); + + // Device 1 was used. + setCurrentTime(TIME); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + setCurrentTime(TIME + 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device 2 was used. + setCurrentTime(TIME + 200ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID_2)); + setCurrentTime(TIME + 400ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID_2)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device 1 was used after its usage timeout expired. Its usage session is reported. + setCurrentTime(TIME + 300ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 100ns)); + + // Device 2 was used. + setCurrentTime(TIME + 350ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID_2)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device 1 was used. + setCurrentTime(TIME + 500ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Device 2 is not used for a while, but Device 1 is used again. + setCurrentTime(TIME + 400ns + (2 * USAGE_TIMEOUT)); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + // Since Device 2's usage session ended, its usage should be reported. + ASSERT_NO_FATAL_FAILURE( + assertUsageLogged(SECOND_TOUCHSCREEN_STYLUS_INFO, 150ns + USAGE_TIMEOUT)); + + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +TEST_F(InputDeviceMetricsCollectorTest, BreakdownUsageBySource) { + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {TOUCHSCREEN_STYLUS_INFO}}); + InputDeviceMetricsLogger::SourceUsageBreakdown expectedSourceBreakdown; + + // Use touchscreen. + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN)); + setCurrentTime(TIME + 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Use a stylus with the same input device. + setCurrentTime(TIME + 200ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, STYLUS, {ToolType::STYLUS})); + setCurrentTime(TIME + 400ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, STYLUS, {ToolType::STYLUS})); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Touchscreen was used again after its usage timeout expired. + // This should be tracked as a separate usage of the source in the breakdown. + setCurrentTime(TIME + 300ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + expectedSourceBreakdown.emplace_back(InputDeviceUsageSource::TOUCHSCREEN, 100ns); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Continue stylus and touchscreen usages. + setCurrentTime(TIME + 350ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, STYLUS, {ToolType::STYLUS})); + setCurrentTime(TIME + 450ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN)); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Touchscreen was used after the stylus's usage timeout expired. + // The stylus usage should be tracked in the source breakdown. + setCurrentTime(TIME + 400ns + USAGE_TIMEOUT + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN)); + expectedSourceBreakdown.emplace_back(InputDeviceUsageSource::STYLUS_DIRECT, + 150ns + USAGE_TIMEOUT); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Remove all devices to force the usage session to be logged. + setCurrentTime(TIME + 500ns + USAGE_TIMEOUT); + mMetricsCollector.notifyInputDevicesChanged({}); + expectedSourceBreakdown.emplace_back(InputDeviceUsageSource::TOUCHSCREEN, + 100ns + USAGE_TIMEOUT); + // Verify that only one usage session was logged for the device, and that session was broken + // down by source correctly. + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, + 400ns + USAGE_TIMEOUT + USAGE_TIMEOUT, + expectedSourceBreakdown)); + + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +TEST_F(InputDeviceMetricsCollectorTest, BreakdownUsageBySource_TrackSourceByDevice) { + mMetricsCollector.notifyInputDevicesChanged( + {/*id=*/0, {TOUCHSCREEN_STYLUS_INFO, SECOND_TOUCHSCREEN_STYLUS_INFO}}); + InputDeviceMetricsLogger::SourceUsageBreakdown expectedSourceBreakdown1; + InputDeviceMetricsLogger::SourceUsageBreakdown expectedSourceBreakdown2; + + // Use both devices, with different sources. + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN)); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID_2, STYLUS, {ToolType::STYLUS})); + setCurrentTime(TIME + 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN)); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID_2, STYLUS, {ToolType::STYLUS})); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Remove all devices to force the usage session to be logged. + mMetricsCollector.notifyInputDevicesChanged({}); + expectedSourceBreakdown1.emplace_back(InputDeviceUsageSource::TOUCHSCREEN, 100ns); + expectedSourceBreakdown2.emplace_back(InputDeviceUsageSource::STYLUS_DIRECT, 100ns); + ASSERT_NO_FATAL_FAILURE( + assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 100ns, expectedSourceBreakdown1)); + ASSERT_NO_FATAL_FAILURE( + assertUsageLogged(SECOND_TOUCHSCREEN_STYLUS_INFO, 100ns, expectedSourceBreakdown2)); + + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +TEST_F(InputDeviceMetricsCollectorTest, BreakdownUsageBySource_MultiSourceEvent) { + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {TOUCHSCREEN_STYLUS_INFO}}); + InputDeviceMetricsLogger::SourceUsageBreakdown expectedSourceBreakdown; + + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN | STYLUS, // + {ToolType::STYLUS})); + setCurrentTime(TIME + 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN | STYLUS, // + {ToolType::STYLUS, ToolType::FINGER})); + setCurrentTime(TIME + 200ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN | STYLUS, // + {ToolType::STYLUS, ToolType::FINGER})); + setCurrentTime(TIME + 300ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN | STYLUS, // + {ToolType::FINGER})); + setCurrentTime(TIME + 400ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID, TOUCHSCREEN | STYLUS, // + {ToolType::FINGER})); + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); + + // Remove all devices to force the usage session to be logged. + mMetricsCollector.notifyInputDevicesChanged({}); + expectedSourceBreakdown.emplace_back(InputDeviceUsageSource::STYLUS_DIRECT, 200ns); + expectedSourceBreakdown.emplace_back(InputDeviceUsageSource::TOUCHSCREEN, 300ns); + ASSERT_NO_FATAL_FAILURE( + assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 400ns, expectedSourceBreakdown)); + + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +TEST_F(InputDeviceMetricsCollectorTest, UidsNotTrackedWhenThereIsNoActiveSession) { + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {TOUCHSCREEN_STYLUS_INFO}}); + + // Notify interaction with UIDs before the device is used. + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1})); + + // Use the device. + setCurrentTime(TIME + 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + setCurrentTime(TIME + 200ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + + // Notify interaction for the wrong device. + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID_2, currentTime(), uids({42})); + + // Notify interaction after usage session would have expired. + // This interaction should not be tracked. + setCurrentTime(TIME + 200ns + USAGE_TIMEOUT); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({2, 3})); + + // Use the device again, by starting a new usage session. + setCurrentTime(TIME + 300ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + + // The first usage session is logged. + static const UidUsageBreakdown emptyBreakdown; + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 100ns, + /*sourceBreakdown=*/{}, + /*uidBreakdown=*/emptyBreakdown)); + + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +TEST_F(InputDeviceMetricsCollectorTest, BreakdownUsageByUid) { + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {TOUCHSCREEN_STYLUS_INFO}}); + UidUsageBreakdown expectedUidBreakdown; + + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1})); + + setCurrentTime(TIME + 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 2})); + setCurrentTime(TIME + 200ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 2, 3})); + + expectedUidBreakdown.emplace_back(1, 200ns); + expectedUidBreakdown.emplace_back(2, 100ns); + expectedUidBreakdown.emplace_back(3, 0ns); + + // Remove the device to force the usage session to be logged. + mMetricsCollector.notifyInputDevicesChanged({}); + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 200ns, + /*sourceBreakdown=*/{}, expectedUidBreakdown)); + + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +TEST_F(InputDeviceMetricsCollectorTest, BreakdownUsageByUid_TracksMultipleSessionsForUid) { + mMetricsCollector.notifyInputDevicesChanged({/*id=*/0, {TOUCHSCREEN_STYLUS_INFO}}); + UidUsageBreakdown expectedUidBreakdown; + + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 2})); + setCurrentTime(TIME + 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 2})); + + setCurrentTime(TIME + 200ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1})); + + setCurrentTime(TIME + 300ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 3})); + setCurrentTime(TIME + 400ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 3})); + + setCurrentTime(TIME + 200ns + USAGE_TIMEOUT); + expectedUidBreakdown.emplace_back(2, 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({4})); + + setCurrentTime(TIME + 300ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 4})); + + setCurrentTime(TIME + 400ns + USAGE_TIMEOUT); + expectedUidBreakdown.emplace_back(3, 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({2, 3})); + + setCurrentTime(TIME + 500ns + USAGE_TIMEOUT); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({3})); + + // Remove the device to force the usage session to be logged. + mMetricsCollector.notifyInputDevicesChanged({}); + expectedUidBreakdown.emplace_back(1, 300ns + USAGE_TIMEOUT); + expectedUidBreakdown.emplace_back(2, 0ns); + expectedUidBreakdown.emplace_back(3, 100ns); + expectedUidBreakdown.emplace_back(4, 100ns); + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 500ns + USAGE_TIMEOUT, + /*sourceBreakdown=*/{}, expectedUidBreakdown)); + + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +TEST_F(InputDeviceMetricsCollectorTest, BreakdownUsageByUid_TracksUidsByDevice) { + mMetricsCollector.notifyInputDevicesChanged( + {/*id=*/0, {TOUCHSCREEN_STYLUS_INFO, SECOND_TOUCHSCREEN_STYLUS_INFO}}); + UidUsageBreakdown expectedUidBreakdown1; + UidUsageBreakdown expectedUidBreakdown2; + + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 2})); + + setCurrentTime(TIME + 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID_2)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID_2, currentTime(), uids({1, 3})); + + setCurrentTime(TIME + 200ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID, currentTime(), uids({1, 2})); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID_2)); + mMetricsCollector.notifyDeviceInteraction(DEVICE_ID_2, currentTime(), uids({1, 3})); + + setCurrentTime(TIME + 200ns + USAGE_TIMEOUT); + expectedUidBreakdown1.emplace_back(1, 200ns); + expectedUidBreakdown1.emplace_back(2, 200ns); + expectedUidBreakdown2.emplace_back(1, 100ns); + expectedUidBreakdown2.emplace_back(3, 100ns); + mMetricsCollector.notifyMotion(generateMotionArgs(DEVICE_ID)); + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(TOUCHSCREEN_STYLUS_INFO, 200ns, + /*sourceBreakdown=*/{}, expectedUidBreakdown1)); + ASSERT_NO_FATAL_FAILURE(assertUsageLogged(SECOND_TOUCHSCREEN_STYLUS_INFO, 100ns, + /*sourceBreakdown=*/{}, expectedUidBreakdown2)); + + ASSERT_NO_FATAL_FAILURE(assertUsageNotLogged()); +} + +} // namespace android diff --git a/services/inputflinger/tests/InputDispatcher_test.cpp b/services/inputflinger/tests/InputDispatcher_test.cpp index f3aba79e30..6ab263d829 100644 --- a/services/inputflinger/tests/InputDispatcher_test.cpp +++ b/services/inputflinger/tests/InputDispatcher_test.cpp @@ -15,7 +15,9 @@ */ #include "../dispatcher/InputDispatcher.h" +#include "../BlockingQueue.h" +#include <NotifyArgsBuilders.h> #include <android-base/properties.h> #include <android-base/silent_death_test.h> #include <android-base/stringprintf.h> @@ -51,13 +53,16 @@ using testing::AllOf; static constexpr nsecs_t ARBITRARY_TIME = 1234; // An arbitrary device id. -static constexpr int32_t DEVICE_ID = 1; +static constexpr int32_t DEVICE_ID = DEFAULT_DEVICE_ID; static constexpr int32_t SECOND_DEVICE_ID = 2; // An arbitrary display id. static constexpr int32_t DISPLAY_ID = ADISPLAY_ID_DEFAULT; static constexpr int32_t SECOND_DISPLAY_ID = 1; +// Ensure common actions are interchangeable between keys and motions for convenience. +static_assert(AMOTION_EVENT_ACTION_DOWN == AKEY_EVENT_ACTION_DOWN); +static_assert(AMOTION_EVENT_ACTION_UP == AKEY_EVENT_ACTION_UP); static constexpr int32_t ACTION_DOWN = AMOTION_EVENT_ACTION_DOWN; static constexpr int32_t ACTION_MOVE = AMOTION_EVENT_ACTION_MOVE; static constexpr int32_t ACTION_UP = AMOTION_EVENT_ACTION_UP; @@ -90,18 +95,15 @@ static constexpr int32_t POINTER_2_UP = AMOTION_EVENT_ACTION_POINTER_UP | (2 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT); // The default pid and uid for windows created on the primary display by the test. -static constexpr int32_t WINDOW_PID = 999; -static constexpr int32_t WINDOW_UID = 1001; +static constexpr gui::Pid WINDOW_PID{999}; +static constexpr gui::Uid WINDOW_UID{1001}; // The default pid and uid for the windows created on the secondary display by the test. -static constexpr int32_t SECONDARY_WINDOW_PID = 1010; -static constexpr int32_t SECONDARY_WINDOW_UID = 1012; - -// The default policy flags to use for event injection by tests. -static constexpr uint32_t DEFAULT_POLICY_FLAGS = POLICY_FLAG_FILTERED | POLICY_FLAG_PASS_TO_USER; +static constexpr gui::Pid SECONDARY_WINDOW_PID{1010}; +static constexpr gui::Uid SECONDARY_WINDOW_UID{1012}; // An arbitrary pid of the gesture monitor window -static constexpr int32_t MONITOR_PID = 2001; +static constexpr gui::Pid MONITOR_PID{2001}; static constexpr std::chrono::duration STALE_EVENT_TIMEOUT = 1000ms; @@ -206,7 +208,10 @@ MATCHER_P(WithPointers, pointers, "MotionEvent with specified pointers") { class FakeInputDispatcherPolicy : public InputDispatcherPolicyInterface { InputDispatcherConfiguration mConfig; - using AnrResult = std::pair<sp<IBinder>, int32_t /*pid*/>; + struct AnrResult { + sp<IBinder> token{}; + gui::Pid pid{-1}; + }; public: FakeInputDispatcherPolicy() = default; @@ -296,15 +301,14 @@ public: void assertNotifyWindowUnresponsiveWasCalled(std::chrono::nanoseconds timeout, const sp<IBinder>& expectedToken, - int32_t expectedPid) { + gui::Pid expectedPid) { std::unique_lock lock(mLock); android::base::ScopedLockAssertion assumeLocked(mLock); AnrResult result; ASSERT_NO_FATAL_FAILURE(result = getAnrTokenLockedInterruptible(timeout, mAnrWindows, lock)); - const auto& [token, pid] = result; - ASSERT_EQ(expectedToken, token); - ASSERT_EQ(expectedPid, pid); + ASSERT_EQ(expectedToken, result.token); + ASSERT_EQ(expectedPid, result.pid); } /** Wrap call with ASSERT_NO_FATAL_FAILURE() to ensure the return value is valid. */ @@ -317,15 +321,14 @@ public: } void assertNotifyWindowResponsiveWasCalled(const sp<IBinder>& expectedToken, - int32_t expectedPid) { + gui::Pid expectedPid) { std::unique_lock lock(mLock); android::base::ScopedLockAssertion assumeLocked(mLock); AnrResult result; ASSERT_NO_FATAL_FAILURE( result = getAnrTokenLockedInterruptible(0s, mResponsiveWindows, lock)); - const auto& [token, pid] = result; - ASSERT_EQ(expectedToken, token); - ASSERT_EQ(expectedPid, pid); + ASSERT_EQ(expectedToken, result.token); + ASSERT_EQ(expectedPid, result.pid); } /** Wrap call with ASSERT_NO_FATAL_FAILURE() to ensure the return value is valid. */ @@ -381,7 +384,9 @@ public: mPointerCaptureRequest.reset(); } - void assertDropTargetEquals(const sp<IBinder>& targetToken) { + void assertDropTargetEquals(const InputDispatcherInterface& dispatcher, + const sp<IBinder>& targetToken) { + dispatcher.waitForIdle(); std::scoped_lock lock(mLock); ASSERT_TRUE(mNotifyDropWindowWasCalled); ASSERT_EQ(targetToken, mDropTargetWindowToken); @@ -415,6 +420,14 @@ public: ASSERT_FALSE(mPokedUserActivity) << "Expected user activity not to have been poked"; } + void assertNotifyDeviceInteractionWasCalled(int32_t deviceId, std::set<gui::Uid> uids) { + ASSERT_EQ(std::make_pair(deviceId, uids), mNotifiedInteractions.popWithTimeout(100ms)); + } + + void assertNotifyDeviceInteractionWasNotCalled() { + ASSERT_FALSE(mNotifiedInteractions.popWithTimeout(10ms)); + } + private: std::mutex mLock; std::unique_ptr<InputEvent> mFilteredEvent GUARDED_BY(mLock); @@ -440,6 +453,8 @@ private: std::chrono::milliseconds mInterceptKeyTimeout = 0ms; + BlockingQueue<std::pair<int32_t /*deviceId*/, std::set<gui::Uid>>> mNotifiedInteractions; + // All three ANR-related callbacks behave the same way, so we use this generic function to wait // for a specific container to become non-empty. When the container is non-empty, return the // first entry from the container and erase it. @@ -495,7 +510,7 @@ private: mConfigurationChangedTime = when; } - void notifyWindowUnresponsive(const sp<IBinder>& connectionToken, std::optional<int32_t> pid, + void notifyWindowUnresponsive(const sp<IBinder>& connectionToken, std::optional<gui::Pid> pid, const std::string&) override { std::scoped_lock lock(mLock); ASSERT_TRUE(pid.has_value()); @@ -504,7 +519,7 @@ private: } void notifyWindowResponsive(const sp<IBinder>& connectionToken, - std::optional<int32_t> pid) override { + std::optional<gui::Pid> pid) override { std::scoped_lock lock(mLock); ASSERT_TRUE(pid.has_value()); mResponsiveWindows.push({connectionToken, *pid}); @@ -611,6 +626,11 @@ private: mDropTargetWindowToken = token; } + void notifyDeviceInteraction(int32_t deviceId, nsecs_t timestamp, + const std::set<gui::Uid>& uids) override { + ASSERT_TRUE(mNotifiedInteractions.emplace(deviceId, uids)); + } + void assertFilterInputEventWasCalledInternal( const std::function<void(const InputEvent&)>& verify) { std::scoped_lock lock(mLock); @@ -848,6 +868,8 @@ TEST_F(InputDispatcherTest, NotifySwitch_CallsPolicy) { mFakePolicy->assertNotifySwitchWasCalled(args); } +namespace { + // --- InputDispatcherTest SetInputWindowTest --- static constexpr std::chrono::duration INJECT_EVENT_TIMEOUT = 500ms; // Default input dispatching timeout if there is no focused application or paused window @@ -1145,12 +1167,29 @@ public: mInfo.inputConfig = WindowInfo::InputConfig::DEFAULT; } - sp<FakeWindowHandle> clone( - const std::shared_ptr<InputApplicationHandle>& inputApplicationHandle, - const std::unique_ptr<InputDispatcher>& dispatcher, int32_t displayId) { - sp<FakeWindowHandle> handle = - sp<FakeWindowHandle>::make(inputApplicationHandle, dispatcher, - mInfo.name + "(Mirror)", displayId, mInfo.token); + sp<FakeWindowHandle> clone(int32_t displayId) { + sp<FakeWindowHandle> handle = sp<FakeWindowHandle>::make(mInfo.name + "(Mirror)"); + handle->mInfo = mInfo; + handle->mInfo.displayId = displayId; + handle->mInfo.id = sId++; + handle->mInputReceiver = mInputReceiver; + return handle; + } + + /** + * This is different from clone, because clone will make a "mirror" window - a window with the + * same token, but a different ID. The original window and the clone window are allowed to be + * sent to the dispatcher at the same time - they can coexist inside the dispatcher. + * This function will create a different object of WindowInfoHandle, but with the same + * properties as the original object - including the ID. + * You can use either the old or the new object to consume the events. + * IMPORTANT: The duplicated object is supposed to replace the original object, and not appear + * at the same time inside dispatcher. + */ + sp<FakeWindowHandle> duplicate() { + sp<FakeWindowHandle> handle = sp<FakeWindowHandle>::make(mName); + handle->mInfo = mInfo; + handle->mInputReceiver = mInputReceiver; return handle; } @@ -1407,21 +1446,23 @@ public: const std::string& getName() { return mName; } - void setOwnerInfo(int32_t ownerPid, int32_t ownerUid) { + void setOwnerInfo(gui::Pid ownerPid, gui::Uid ownerUid) { mInfo.ownerPid = ownerPid; mInfo.ownerUid = ownerUid; } - int32_t getPid() const { return mInfo.ownerPid; } + gui::Pid getPid() const { return mInfo.ownerPid; } void destroyReceiver() { mInputReceiver = nullptr; } int getChannelFd() { return mInputReceiver->getChannelFd(); } private: + FakeWindowHandle(std::string name) : mName(name){}; const std::string mName; - std::unique_ptr<FakeInputReceiver> mInputReceiver; + std::shared_ptr<FakeInputReceiver> mInputReceiver; static std::atomic<int32_t> sId; // each window gets a unique id, like in surfaceflinger + friend class sp<FakeWindowHandle>; }; std::atomic<int32_t> FakeWindowHandle::sId{1}; @@ -1431,7 +1472,7 @@ static InputEventInjectionResult injectKey( int32_t displayId = ADISPLAY_ID_NONE, InputEventInjectionSync syncMode = InputEventInjectionSync::WAIT_FOR_RESULT, std::chrono::milliseconds injectionTimeout = INJECT_EVENT_TIMEOUT, - bool allowKeyRepeat = true, std::optional<int32_t> targetUid = {}, + bool allowKeyRepeat = true, std::optional<gui::Uid> targetUid = {}, uint32_t policyFlags = DEFAULT_POLICY_FLAGS) { KeyEvent event; nsecs_t currentTime = systemTime(SYSTEM_TIME_MONOTONIC); @@ -1468,252 +1509,11 @@ static InputEventInjectionResult injectKeyUp(const std::unique_ptr<InputDispatch return injectKey(dispatcher, AKEY_EVENT_ACTION_UP, /*repeatCount=*/0, displayId); } -class PointerBuilder { -public: - PointerBuilder(int32_t id, ToolType toolType) { - mProperties.clear(); - mProperties.id = id; - mProperties.toolType = toolType; - mCoords.clear(); - } - - PointerBuilder& x(float x) { return axis(AMOTION_EVENT_AXIS_X, x); } - - PointerBuilder& y(float y) { return axis(AMOTION_EVENT_AXIS_Y, y); } - - PointerBuilder& axis(int32_t axis, float value) { - mCoords.setAxisValue(axis, value); - return *this; - } - - PointerProperties buildProperties() const { return mProperties; } - - PointerCoords buildCoords() const { return mCoords; } - -private: - PointerProperties mProperties; - PointerCoords mCoords; -}; - -class MotionEventBuilder { -public: - MotionEventBuilder(int32_t action, int32_t source) { - mAction = action; - mSource = source; - mEventTime = systemTime(SYSTEM_TIME_MONOTONIC); - mDownTime = mEventTime; - } - - MotionEventBuilder& deviceId(int32_t deviceId) { - mDeviceId = deviceId; - return *this; - } - - MotionEventBuilder& downTime(nsecs_t downTime) { - mDownTime = downTime; - return *this; - } - - MotionEventBuilder& eventTime(nsecs_t eventTime) { - mEventTime = eventTime; - return *this; - } - - MotionEventBuilder& displayId(int32_t displayId) { - mDisplayId = displayId; - return *this; - } - - MotionEventBuilder& actionButton(int32_t actionButton) { - mActionButton = actionButton; - return *this; - } - - MotionEventBuilder& buttonState(int32_t buttonState) { - mButtonState = buttonState; - return *this; - } - - MotionEventBuilder& rawXCursorPosition(float rawXCursorPosition) { - mRawXCursorPosition = rawXCursorPosition; - return *this; - } - - MotionEventBuilder& rawYCursorPosition(float rawYCursorPosition) { - mRawYCursorPosition = rawYCursorPosition; - return *this; - } - - MotionEventBuilder& pointer(PointerBuilder pointer) { - mPointers.push_back(pointer); - return *this; - } - - MotionEventBuilder& addFlag(uint32_t flags) { - mFlags |= flags; - return *this; - } - - MotionEvent build() { - std::vector<PointerProperties> pointerProperties; - std::vector<PointerCoords> pointerCoords; - for (const PointerBuilder& pointer : mPointers) { - pointerProperties.push_back(pointer.buildProperties()); - pointerCoords.push_back(pointer.buildCoords()); - } - - // Set mouse cursor position for the most common cases to avoid boilerplate. - if (mSource == AINPUT_SOURCE_MOUSE && - !MotionEvent::isValidCursorPosition(mRawXCursorPosition, mRawYCursorPosition)) { - mRawXCursorPosition = pointerCoords[0].getX(); - mRawYCursorPosition = pointerCoords[0].getY(); - } - - MotionEvent event; - ui::Transform identityTransform; - event.initialize(InputEvent::nextId(), mDeviceId, mSource, mDisplayId, INVALID_HMAC, - mAction, mActionButton, mFlags, /* edgeFlags */ 0, AMETA_NONE, - mButtonState, MotionClassification::NONE, identityTransform, - /* xPrecision */ 0, /* yPrecision */ 0, mRawXCursorPosition, - mRawYCursorPosition, identityTransform, mDownTime, mEventTime, - mPointers.size(), pointerProperties.data(), pointerCoords.data()); - - return event; - } - -private: - int32_t mAction; - int32_t mDeviceId = DEVICE_ID; - int32_t mSource; - nsecs_t mDownTime; - nsecs_t mEventTime; - int32_t mDisplayId{ADISPLAY_ID_DEFAULT}; - int32_t mActionButton{0}; - int32_t mButtonState{0}; - int32_t mFlags{0}; - float mRawXCursorPosition{AMOTION_EVENT_INVALID_CURSOR_POSITION}; - float mRawYCursorPosition{AMOTION_EVENT_INVALID_CURSOR_POSITION}; - - std::vector<PointerBuilder> mPointers; -}; - -class MotionArgsBuilder { -public: - MotionArgsBuilder(int32_t action, int32_t source) { - mAction = action; - mSource = source; - mEventTime = systemTime(SYSTEM_TIME_MONOTONIC); - mDownTime = mEventTime; - } - - MotionArgsBuilder& deviceId(int32_t deviceId) { - mDeviceId = deviceId; - return *this; - } - - MotionArgsBuilder& downTime(nsecs_t downTime) { - mDownTime = downTime; - return *this; - } - - MotionArgsBuilder& eventTime(nsecs_t eventTime) { - mEventTime = eventTime; - return *this; - } - - MotionArgsBuilder& displayId(int32_t displayId) { - mDisplayId = displayId; - return *this; - } - - MotionArgsBuilder& policyFlags(int32_t policyFlags) { - mPolicyFlags = policyFlags; - return *this; - } - - MotionArgsBuilder& actionButton(int32_t actionButton) { - mActionButton = actionButton; - return *this; - } - - MotionArgsBuilder& buttonState(int32_t buttonState) { - mButtonState = buttonState; - return *this; - } - - MotionArgsBuilder& rawXCursorPosition(float rawXCursorPosition) { - mRawXCursorPosition = rawXCursorPosition; - return *this; - } - - MotionArgsBuilder& rawYCursorPosition(float rawYCursorPosition) { - mRawYCursorPosition = rawYCursorPosition; - return *this; - } - - MotionArgsBuilder& pointer(PointerBuilder pointer) { - mPointers.push_back(pointer); - return *this; - } - - MotionArgsBuilder& addFlag(uint32_t flags) { - mFlags |= flags; - return *this; - } - - MotionArgsBuilder& classification(MotionClassification classification) { - mClassification = classification; - return *this; - } - - NotifyMotionArgs build() { - std::vector<PointerProperties> pointerProperties; - std::vector<PointerCoords> pointerCoords; - for (const PointerBuilder& pointer : mPointers) { - pointerProperties.push_back(pointer.buildProperties()); - pointerCoords.push_back(pointer.buildCoords()); - } - - // Set mouse cursor position for the most common cases to avoid boilerplate. - if (mSource == AINPUT_SOURCE_MOUSE && - !MotionEvent::isValidCursorPosition(mRawXCursorPosition, mRawYCursorPosition)) { - mRawXCursorPosition = pointerCoords[0].getX(); - mRawYCursorPosition = pointerCoords[0].getY(); - } - - NotifyMotionArgs args(InputEvent::nextId(), mEventTime, /*readTime=*/mEventTime, mDeviceId, - mSource, mDisplayId, mPolicyFlags, mAction, mActionButton, mFlags, - AMETA_NONE, mButtonState, mClassification, /*edgeFlags=*/0, - mPointers.size(), pointerProperties.data(), pointerCoords.data(), - /*xPrecision=*/0, /*yPrecision=*/0, mRawXCursorPosition, - mRawYCursorPosition, mDownTime, /*videoFrames=*/{}); - - return args; - } - -private: - int32_t mAction; - int32_t mDeviceId = DEVICE_ID; - int32_t mSource; - nsecs_t mDownTime; - nsecs_t mEventTime; - int32_t mDisplayId{ADISPLAY_ID_DEFAULT}; - int32_t mPolicyFlags = DEFAULT_POLICY_FLAGS; - int32_t mActionButton{0}; - int32_t mButtonState{0}; - int32_t mFlags{0}; - MotionClassification mClassification{MotionClassification::NONE}; - float mRawXCursorPosition{AMOTION_EVENT_INVALID_CURSOR_POSITION}; - float mRawYCursorPosition{AMOTION_EVENT_INVALID_CURSOR_POSITION}; - - std::vector<PointerBuilder> mPointers; -}; - static InputEventInjectionResult injectMotionEvent( const std::unique_ptr<InputDispatcher>& dispatcher, const MotionEvent& event, std::chrono::milliseconds injectionTimeout = INJECT_EVENT_TIMEOUT, InputEventInjectionSync injectionMode = InputEventInjectionSync::WAIT_FOR_RESULT, - std::optional<int32_t> targetUid = {}, uint32_t policyFlags = DEFAULT_POLICY_FLAGS) { + std::optional<gui::Uid> targetUid = {}, uint32_t policyFlags = DEFAULT_POLICY_FLAGS) { return dispatcher->injectInputEvent(&event, targetUid, injectionMode, injectionTimeout, policyFlags); } @@ -1726,20 +1526,22 @@ static InputEventInjectionResult injectMotionEvent( std::chrono::milliseconds injectionTimeout = INJECT_EVENT_TIMEOUT, InputEventInjectionSync injectionMode = InputEventInjectionSync::WAIT_FOR_RESULT, nsecs_t eventTime = systemTime(SYSTEM_TIME_MONOTONIC), - std::optional<int32_t> targetUid = {}, uint32_t policyFlags = DEFAULT_POLICY_FLAGS) { - MotionEvent event = MotionEventBuilder(action, source) - .displayId(displayId) - .eventTime(eventTime) - .rawXCursorPosition(cursorPosition.x) - .rawYCursorPosition(cursorPosition.y) - .pointer(PointerBuilder(/*id=*/0, ToolType::FINGER) - .x(position.x) - .y(position.y)) - .build(); + std::optional<gui::Uid> targetUid = {}, uint32_t policyFlags = DEFAULT_POLICY_FLAGS) { + MotionEventBuilder motionBuilder = + MotionEventBuilder(action, source) + .displayId(displayId) + .eventTime(eventTime) + .rawXCursorPosition(cursorPosition.x) + .rawYCursorPosition(cursorPosition.y) + .pointer( + PointerBuilder(/*id=*/0, ToolType::FINGER).x(position.x).y(position.y)); + if (MotionEvent::getActionMasked(action) == ACTION_DOWN) { + motionBuilder.downTime(eventTime); + } // Inject event until dispatch out. - return injectMotionEvent(dispatcher, event, injectionTimeout, injectionMode, targetUid, - policyFlags); + return injectMotionEvent(dispatcher, motionBuilder.build(), injectionTimeout, injectionMode, + targetUid, policyFlags); } static InputEventInjectionResult injectMotionDown( @@ -1833,6 +1635,8 @@ static NotifyPointerCaptureChangedArgs generatePointerCaptureChangedArgs( return NotifyPointerCaptureChangedArgs(/*id=*/0, systemTime(SYSTEM_TIME_MONOTONIC), request); } +} // namespace + /** * When a window unexpectedly disposes of its input channel, policy should be notified about the * broken channel. @@ -3866,6 +3670,9 @@ TEST_F(InputDispatcherTest, InterceptKeyByPolicy) { std::chrono::nanoseconds(interceptKeyTimeout).count()); } +/** + * Keys with ACTION_UP are delivered immediately, even if a long 'intercept key timeout' is set. + */ TEST_F(InputDispatcherTest, InterceptKeyIfKeyUp) { std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); sp<FakeWindowHandle> window = sp<FakeWindowHandle>::make(application, mDispatcher, @@ -3877,12 +3684,14 @@ TEST_F(InputDispatcherTest, InterceptKeyIfKeyUp) { window->consumeFocusEvent(true); - mFakePolicy->setInterceptKeyTimeout(150ms); mDispatcher->notifyKey(generateKeyArgs(AKEY_EVENT_ACTION_DOWN, ADISPLAY_ID_DEFAULT)); - mDispatcher->notifyKey(generateKeyArgs(AKEY_EVENT_ACTION_UP, ADISPLAY_ID_DEFAULT)); + window->consumeKeyDown(ADISPLAY_ID_DEFAULT); + // Set a value that's significantly larger than the default consumption timeout. If the + // implementation is correct, the actual value doesn't matter; it won't slow down the test. + mFakePolicy->setInterceptKeyTimeout(600ms); + mDispatcher->notifyKey(generateKeyArgs(AKEY_EVENT_ACTION_UP, ADISPLAY_ID_DEFAULT)); // Window should receive key event immediately when same key up. - window->consumeKeyDown(ADISPLAY_ID_DEFAULT); window->consumeKeyUp(ADISPLAY_ID_DEFAULT); } @@ -4153,6 +3962,72 @@ TEST_F(InputDispatcherTest, TouchpadThreeFingerSwipeNotSentToSingleWindow) { } /** + * Send a two-pointer gesture to a single window. The window's orientation changes in response to + * the first pointer. + * Ensure that the second pointer is not sent to the window. + * + * The subsequent gesture should be correctly delivered to the window. + */ +TEST_F(InputDispatcherTest, MultiplePointersWithRotatingWindow) { + std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); + sp<FakeWindowHandle> window = + sp<FakeWindowHandle>::make(application, mDispatcher, "Window", ADISPLAY_ID_DEFAULT); + window->setFrame(Rect(0, 0, 400, 400)); + mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {window}}}); + + const nsecs_t baseTime = systemTime(SYSTEM_TIME_MONOTONIC); + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN) + .downTime(baseTime + 10) + .eventTime(baseTime + 10) + .pointer(PointerBuilder(0, ToolType::FINGER).x(100).y(100)) + .build()); + + window->consumeMotionEvent(WithMotionAction(ACTION_DOWN)); + + // We need a new window object for the same window, because dispatcher will store objects by + // reference. That means that the testing code and the dispatcher will refer to the same shared + // object. Calling window->setTransform here would affect dispatcher's comparison + // of the old window to the new window, since both the old window and the new window would be + // updated to the same value. + sp<FakeWindowHandle> windowDup = window->duplicate(); + + // Change the transform so that the orientation is now different from original. + windowDup->setWindowTransform(0, -1, 1, 0); + + mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {windowDup}}}); + + window->consumeMotionEvent(WithMotionAction(ACTION_CANCEL)); + + mDispatcher->notifyMotion(MotionArgsBuilder(POINTER_1_DOWN, AINPUT_SOURCE_TOUCHSCREEN) + .downTime(baseTime + 10) + .eventTime(baseTime + 30) + .pointer(PointerBuilder(0, ToolType::FINGER).x(100).y(100)) + .pointer(PointerBuilder(1, ToolType::FINGER).x(200).y(200)) + .build()); + + // Finish the gesture and start a new one. Ensure the new gesture is sent to the window + mDispatcher->notifyMotion(MotionArgsBuilder(POINTER_1_UP, AINPUT_SOURCE_TOUCHSCREEN) + .downTime(baseTime + 10) + .eventTime(baseTime + 40) + .pointer(PointerBuilder(0, ToolType::FINGER).x(100).y(100)) + .pointer(PointerBuilder(1, ToolType::FINGER).x(200).y(200)) + .build()); + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_UP, AINPUT_SOURCE_TOUCHSCREEN) + .downTime(baseTime + 10) + .eventTime(baseTime + 50) + .pointer(PointerBuilder(0, ToolType::FINGER).x(100).y(100)) + .build()); + + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN) + .downTime(baseTime + 60) + .eventTime(baseTime + 60) + .pointer(PointerBuilder(0, ToolType::FINGER).x(40).y(40)) + .build()); + + windowDup->consumeMotionEvent(WithMotionAction(ACTION_DOWN)); +} + +/** * When there are multiple screens, such as screen projection to TV or screen recording, if the * cancel event occurs, the coordinates of the cancel event should be sent to the target screen, and * its coordinates should be converted by the transform of the windows of target screen. @@ -4172,8 +4047,7 @@ TEST_F(InputDispatcherTest, WhenMultiDisplayWindowSameToken_DispatchCancelToTarg ADISPLAY_ID_DEFAULT); windowDefaultDisplay->setWindowTransform(1, 0, 0, 1); - sp<FakeWindowHandle> windowSecondDisplay = - windowDefaultDisplay->clone(application, mDispatcher, SECOND_DISPLAY_ID); + sp<FakeWindowHandle> windowSecondDisplay = windowDefaultDisplay->clone(SECOND_DISPLAY_ID); windowSecondDisplay->setWindowTransform(2, 0, 0, 2); // Add the windows to the dispatcher @@ -4788,16 +4662,13 @@ TEST_F(InputDispatcherTest, TransferTouchFocus_CloneSurface) { sp<FakeWindowHandle>::make(application, mDispatcher, "D_1_W2", ADISPLAY_ID_DEFAULT); secondWindowInPrimary->setFrame(Rect(100, 0, 200, 100)); - sp<FakeWindowHandle> mirrorWindowInPrimary = - firstWindowInPrimary->clone(application, mDispatcher, ADISPLAY_ID_DEFAULT); + sp<FakeWindowHandle> mirrorWindowInPrimary = firstWindowInPrimary->clone(ADISPLAY_ID_DEFAULT); mirrorWindowInPrimary->setFrame(Rect(0, 100, 100, 200)); - sp<FakeWindowHandle> firstWindowInSecondary = - firstWindowInPrimary->clone(application, mDispatcher, SECOND_DISPLAY_ID); + sp<FakeWindowHandle> firstWindowInSecondary = firstWindowInPrimary->clone(SECOND_DISPLAY_ID); firstWindowInSecondary->setFrame(Rect(0, 0, 100, 100)); - sp<FakeWindowHandle> secondWindowInSecondary = - secondWindowInPrimary->clone(application, mDispatcher, SECOND_DISPLAY_ID); + sp<FakeWindowHandle> secondWindowInSecondary = secondWindowInPrimary->clone(SECOND_DISPLAY_ID); secondWindowInPrimary->setFrame(Rect(100, 0, 200, 100)); // Update window info, let it find window handle of second display first. @@ -4847,16 +4718,13 @@ TEST_F(InputDispatcherTest, TransferTouch_CloneSurface) { sp<FakeWindowHandle>::make(application, mDispatcher, "D_1_W2", ADISPLAY_ID_DEFAULT); secondWindowInPrimary->setFrame(Rect(100, 0, 200, 100)); - sp<FakeWindowHandle> mirrorWindowInPrimary = - firstWindowInPrimary->clone(application, mDispatcher, ADISPLAY_ID_DEFAULT); + sp<FakeWindowHandle> mirrorWindowInPrimary = firstWindowInPrimary->clone(ADISPLAY_ID_DEFAULT); mirrorWindowInPrimary->setFrame(Rect(0, 100, 100, 200)); - sp<FakeWindowHandle> firstWindowInSecondary = - firstWindowInPrimary->clone(application, mDispatcher, SECOND_DISPLAY_ID); + sp<FakeWindowHandle> firstWindowInSecondary = firstWindowInPrimary->clone(SECOND_DISPLAY_ID); firstWindowInSecondary->setFrame(Rect(0, 0, 100, 100)); - sp<FakeWindowHandle> secondWindowInSecondary = - secondWindowInPrimary->clone(application, mDispatcher, SECOND_DISPLAY_ID); + sp<FakeWindowHandle> secondWindowInSecondary = secondWindowInPrimary->clone(SECOND_DISPLAY_ID); secondWindowInPrimary->setFrame(Rect(100, 0, 200, 100)); // Update window info, let it find window handle of second display first. @@ -4911,6 +4779,7 @@ TEST_F(InputDispatcherTest, FocusedWindow_ReceivesFocusEventAndKeyEvent) { window->consumeKeyDown(ADISPLAY_ID_DEFAULT); // Should have poked user activity + mDispatcher->waitForIdle(); mFakePolicy->assertUserActivityPoked(); } @@ -4932,6 +4801,7 @@ TEST_F(InputDispatcherTest, FocusedWindow_DisableUserActivity) { window->consumeKeyDown(ADISPLAY_ID_DEFAULT); // Should have poked user activity + mDispatcher->waitForIdle(); mFakePolicy->assertUserActivityNotPoked(); } @@ -4999,6 +4869,26 @@ TEST_F(InputDispatcherTest, FocusedWindow_SystemKeyIgnoresDisableUserActivity) { mFakePolicy->assertUserActivityPoked(); } +TEST_F(InputDispatcherTest, InjectedTouchesPokeUserActivity) { + std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); + sp<FakeWindowHandle> window = sp<FakeWindowHandle>::make(application, mDispatcher, + "Fake Window", ADISPLAY_ID_DEFAULT); + + mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {window}}}); + + ASSERT_EQ(InputEventInjectionResult::SUCCEEDED, + injectMotionEvent(mDispatcher, AMOTION_EVENT_ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN, + ADISPLAY_ID_DEFAULT, {100, 100})) + << "Inject motion event should return InputEventInjectionResult::SUCCEEDED"; + + window->consumeMotionEvent( + AllOf(WithMotionAction(ACTION_DOWN), WithDisplayId(ADISPLAY_ID_DEFAULT))); + + // Should have poked user activity + mDispatcher->waitForIdle(); + mFakePolicy->assertUserActivityPoked(); +} + TEST_F(InputDispatcherTest, UnfocusedWindow_DoesNotReceiveFocusEventOrKeyEvent) { std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); sp<FakeWindowHandle> window = sp<FakeWindowHandle>::make(application, mDispatcher, @@ -5698,8 +5588,8 @@ TEST_F(InputDispatcherTest, DisplayRemoved) { * FLAG_WINDOW_IS_PARTIALLY_OBSCURED. */ TEST_F(InputDispatcherTest, SlipperyWindow_SetsFlagPartiallyObscured) { - constexpr int32_t SLIPPERY_PID = WINDOW_PID + 1; - constexpr int32_t SLIPPERY_UID = WINDOW_UID + 1; + constexpr gui::Pid SLIPPERY_PID{WINDOW_PID.val() + 1}; + constexpr gui::Uid SLIPPERY_UID{WINDOW_UID.val() + 1}; std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); mDispatcher->setFocusedApplication(ADISPLAY_ID_DEFAULT, application); @@ -5779,6 +5669,109 @@ TEST_F(InputDispatcherTest, TouchSlippingIntoWindowThatDropsTouches) { rightDropTouchesWindow->assertNoEvents(); } +TEST_F(InputDispatcherTest, NotifiesDeviceInteractionsWithMotions) { + using Uid = gui::Uid; + std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); + + sp<FakeWindowHandle> leftWindow = + sp<FakeWindowHandle>::make(application, mDispatcher, "Left", ADISPLAY_ID_DEFAULT); + leftWindow->setFrame(Rect(0, 0, 100, 100)); + leftWindow->setOwnerInfo(gui::Pid{1}, Uid{101}); + + sp<FakeWindowHandle> rightSpy = + sp<FakeWindowHandle>::make(application, mDispatcher, "Right spy", ADISPLAY_ID_DEFAULT); + rightSpy->setFrame(Rect(100, 0, 200, 100)); + rightSpy->setOwnerInfo(gui::Pid{2}, Uid{102}); + rightSpy->setSpy(true); + rightSpy->setTrustedOverlay(true); + + sp<FakeWindowHandle> rightWindow = + sp<FakeWindowHandle>::make(application, mDispatcher, "Right", ADISPLAY_ID_DEFAULT); + rightWindow->setFrame(Rect(100, 0, 200, 100)); + rightWindow->setOwnerInfo(gui::Pid{3}, Uid{103}); + + mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {rightSpy, rightWindow, leftWindow}}}); + + // Touch in the left window + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(50).y(50)) + .build()); + ASSERT_NO_FATAL_FAILURE(leftWindow->consumeMotionDown()); + mDispatcher->waitForIdle(); + ASSERT_NO_FATAL_FAILURE( + mFakePolicy->assertNotifyDeviceInteractionWasCalled(DEVICE_ID, {Uid{101}})); + + // Touch another finger over the right windows + mDispatcher->notifyMotion(MotionArgsBuilder(POINTER_1_DOWN, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(50).y(50)) + .pointer(PointerBuilder(1, ToolType::FINGER).x(150).y(50)) + .build()); + ASSERT_NO_FATAL_FAILURE(rightSpy->consumeMotionDown()); + ASSERT_NO_FATAL_FAILURE(rightWindow->consumeMotionDown()); + ASSERT_NO_FATAL_FAILURE(leftWindow->consumeMotionMove()); + mDispatcher->waitForIdle(); + ASSERT_NO_FATAL_FAILURE( + mFakePolicy->assertNotifyDeviceInteractionWasCalled(DEVICE_ID, + {Uid{101}, Uid{102}, Uid{103}})); + + // Release finger over left window. The UP actions are not treated as device interaction. + // The windows that did not receive the UP pointer will receive MOVE events, but since this + // is part of the UP action, we do not treat this as device interaction. + mDispatcher->notifyMotion(MotionArgsBuilder(POINTER_0_UP, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(50).y(50)) + .pointer(PointerBuilder(1, ToolType::FINGER).x(150).y(50)) + .build()); + ASSERT_NO_FATAL_FAILURE(leftWindow->consumeMotionUp()); + ASSERT_NO_FATAL_FAILURE(rightSpy->consumeMotionMove()); + ASSERT_NO_FATAL_FAILURE(rightWindow->consumeMotionMove()); + mDispatcher->waitForIdle(); + ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertNotifyDeviceInteractionWasNotCalled()); + + // Move remaining finger + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_MOVE, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(1, ToolType::FINGER).x(150).y(50)) + .build()); + ASSERT_NO_FATAL_FAILURE(rightSpy->consumeMotionMove()); + ASSERT_NO_FATAL_FAILURE(rightWindow->consumeMotionMove()); + mDispatcher->waitForIdle(); + ASSERT_NO_FATAL_FAILURE( + mFakePolicy->assertNotifyDeviceInteractionWasCalled(DEVICE_ID, {Uid{102}, Uid{103}})); + + // Release all fingers + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_UP, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(1, ToolType::FINGER).x(150).y(50)) + .build()); + ASSERT_NO_FATAL_FAILURE(rightSpy->consumeMotionUp()); + ASSERT_NO_FATAL_FAILURE(rightWindow->consumeMotionUp()); + mDispatcher->waitForIdle(); + ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertNotifyDeviceInteractionWasNotCalled()); +} + +TEST_F(InputDispatcherTest, NotifiesDeviceInteractionsWithKeys) { + std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); + + sp<FakeWindowHandle> window = + sp<FakeWindowHandle>::make(application, mDispatcher, "Window", ADISPLAY_ID_DEFAULT); + window->setFrame(Rect(0, 0, 100, 100)); + window->setOwnerInfo(gui::Pid{1}, gui::Uid{101}); + + mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {window}}}); + setFocusedWindow(window); + ASSERT_NO_FATAL_FAILURE(window->consumeFocusEvent(true)); + + mDispatcher->notifyKey(KeyArgsBuilder(ACTION_DOWN, AINPUT_SOURCE_KEYBOARD).build()); + ASSERT_NO_FATAL_FAILURE(window->consumeKeyDown(ADISPLAY_ID_DEFAULT)); + mDispatcher->waitForIdle(); + ASSERT_NO_FATAL_FAILURE( + mFakePolicy->assertNotifyDeviceInteractionWasCalled(DEVICE_ID, {gui::Uid{101}})); + + // The UP actions are not treated as device interaction. + mDispatcher->notifyKey(KeyArgsBuilder(ACTION_UP, AINPUT_SOURCE_KEYBOARD).build()); + ASSERT_NO_FATAL_FAILURE(window->consumeKeyUp(ADISPLAY_ID_DEFAULT)); + mDispatcher->waitForIdle(); + ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertNotifyDeviceInteractionWasNotCalled()); +} + class InputDispatcherKeyRepeatTest : public InputDispatcherTest { protected: static constexpr nsecs_t KEY_REPEAT_TIMEOUT = 40 * 1000000; // 40 ms @@ -6156,7 +6149,7 @@ protected: mDispatcher->notifyMotion(motionArgs); ASSERT_TRUE(mDispatcher->waitForIdle()); if (expectToBeFiltered) { - const auto xy = transform.transform(motionArgs.pointerCoords->getXYValue()); + const auto xy = transform.transform(motionArgs.pointerCoords[0].getXYValue()); mFakePolicy->assertFilterInputEventWasCalled(motionArgs, xy); } else { mFakePolicy->assertFilterInputEventWasNotCalled(); @@ -7174,6 +7167,55 @@ TEST_F(InputDispatcherSingleWindowAnr, mWindow->assertNoEvents(); } +/** + * Send an event to the app and have the app not respond right away. + * When ANR is raised, policy will tell the dispatcher to cancel the events for that window. + * So InputDispatcher will enqueue ACTION_CANCEL event as well. + * At some point, the window becomes responsive again. + * Ensure that subsequent events get dropped, and the next gesture is delivered. + */ +TEST_F(InputDispatcherSingleWindowAnr, TwoGesturesWithAnr) { + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(10).y(10)) + .build()); + + std::optional<uint32_t> sequenceNum = mWindow->receiveEvent(); // ACTION_DOWN + ASSERT_TRUE(sequenceNum); + const std::chrono::duration timeout = mWindow->getDispatchingTimeout(DISPATCHING_TIMEOUT); + mFakePolicy->assertNotifyWindowUnresponsiveWasCalled(timeout, mWindow); + + mWindow->finishEvent(*sequenceNum); + mWindow->consumeMotionEvent(WithMotionAction(ACTION_CANCEL)); + ASSERT_TRUE(mDispatcher->waitForIdle()); + mFakePolicy->assertNotifyWindowResponsiveWasCalled(mWindow->getToken(), mWindow->getPid()); + + // Now that the window is responsive, let's continue the gesture. + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_MOVE, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(11).y(11)) + .build()); + + mDispatcher->notifyMotion(MotionArgsBuilder(POINTER_1_DOWN, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(11).y(11)) + .pointer(PointerBuilder(1, ToolType::FINGER).x(3).y(3)) + .build()); + + mDispatcher->notifyMotion(MotionArgsBuilder(POINTER_1_UP, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(11).y(11)) + .pointer(PointerBuilder(1, ToolType::FINGER).x(3).y(3)) + .build()); + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_UP, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(11).y(11)) + .build()); + // We already canceled this pointer, so the window shouldn't get any new events. + mWindow->assertNoEvents(); + + // Start another one. + mDispatcher->notifyMotion(MotionArgsBuilder(ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN) + .pointer(PointerBuilder(0, ToolType::FINGER).x(15).y(15)) + .build()); + mWindow->consumeMotionEvent(WithMotionAction(ACTION_DOWN)); +} + class InputDispatcherMultiWindowAnr : public InputDispatcherTest { virtual void SetUp() override { InputDispatcherTest::SetUp(); @@ -7918,9 +7960,9 @@ protected: static_assert(1 - (1 - OPACITY_FAR_BELOW_THRESHOLD) * (1 - OPACITY_FAR_BELOW_THRESHOLD) < MAXIMUM_OBSCURING_OPACITY); - static const int32_t TOUCHED_APP_UID = 10001; - static const int32_t APP_B_UID = 10002; - static const int32_t APP_C_UID = 10003; + static constexpr gui::Uid TOUCHED_APP_UID{10001}; + static constexpr gui::Uid APP_B_UID{10002}; + static constexpr gui::Uid APP_C_UID{10003}; sp<FakeWindowHandle> mTouchWindow; @@ -7935,7 +7977,7 @@ protected: mTouchWindow.clear(); } - sp<FakeWindowHandle> getOccludingWindow(int32_t uid, std::string name, TouchOcclusionMode mode, + sp<FakeWindowHandle> getOccludingWindow(gui::Uid uid, std::string name, TouchOcclusionMode mode, float alpha = 1.0f) { sp<FakeWindowHandle> window = getWindow(uid, name); window->setTouchable(false); @@ -7944,12 +7986,12 @@ protected: return window; } - sp<FakeWindowHandle> getWindow(int32_t uid, std::string name) { + sp<FakeWindowHandle> getWindow(gui::Uid uid, std::string name) { std::shared_ptr<FakeApplicationHandle> app = std::make_shared<FakeApplicationHandle>(); sp<FakeWindowHandle> window = sp<FakeWindowHandle>::make(app, mDispatcher, name, ADISPLAY_ID_DEFAULT); // Generate an arbitrary PID based on the UID - window->setOwnerInfo(1777 + (uid % 10000), uid); + window->setOwnerInfo(gui::Pid{static_cast<pid_t>(1777 + (uid.val() % 10000))}, uid); return window; } @@ -8488,7 +8530,7 @@ TEST_F(InputDispatcherDragTests, DragAndDrop) { {150, 50})) << "Inject motion event should return InputEventInjectionResult::SUCCEEDED"; mDragWindow->consumeMotionUp(ADISPLAY_ID_DEFAULT); - mFakePolicy->assertDropTargetEquals(mSecondWindow->getToken()); + mFakePolicy->assertDropTargetEquals(*mDispatcher, mSecondWindow->getToken()); mWindow->assertNoEvents(); mSecondWindow->assertNoEvents(); } @@ -8519,7 +8561,7 @@ TEST_F(InputDispatcherDragTests, StylusDragAndDrop) { mDragWindow->consumeMotionMove(ADISPLAY_ID_DEFAULT); mWindow->assertNoEvents(); mSecondWindow->assertNoEvents(); - mFakePolicy->assertDropTargetEquals(mSecondWindow->getToken()); + mFakePolicy->assertDropTargetEquals(*mDispatcher, mSecondWindow->getToken()); // nothing to the window. ASSERT_EQ(InputEventInjectionResult::SUCCEEDED, @@ -8565,7 +8607,7 @@ TEST_F(InputDispatcherDragTests, DragAndDropOnInvalidWindow) { {150, 50})) << "Inject motion event should return InputEventInjectionResult::SUCCEEDED"; mDragWindow->consumeMotionUp(ADISPLAY_ID_DEFAULT); - mFakePolicy->assertDropTargetEquals(nullptr); + mFakePolicy->assertDropTargetEquals(*mDispatcher, nullptr); mWindow->assertNoEvents(); mSecondWindow->assertNoEvents(); } @@ -8648,7 +8690,7 @@ TEST_F(InputDispatcherDragTests, DragAndDropWhenSplitTouch) { injectMotionEvent(mDispatcher, secondFingerUpEvent, INJECT_EVENT_TIMEOUT, InputEventInjectionSync::WAIT_FOR_RESULT)); mDragWindow->consumeMotionUp(ADISPLAY_ID_DEFAULT); - mFakePolicy->assertDropTargetEquals(mWindow->getToken()); + mFakePolicy->assertDropTargetEquals(*mDispatcher, mWindow->getToken()); mWindow->assertNoEvents(); mSecondWindow->consumeMotionMove(); } @@ -8698,7 +8740,7 @@ TEST_F(InputDispatcherDragTests, DragAndDropWhenMultiDisplays) { {150, 50})) << "Inject motion event should return InputEventInjectionResult::SUCCEEDED"; mDragWindow->consumeMotionUp(ADISPLAY_ID_DEFAULT); - mFakePolicy->assertDropTargetEquals(mSecondWindow->getToken()); + mFakePolicy->assertDropTargetEquals(*mDispatcher, mSecondWindow->getToken()); mWindow->assertNoEvents(); mSecondWindow->assertNoEvents(); } @@ -8747,7 +8789,7 @@ TEST_F(InputDispatcherDragTests, MouseDragAndDrop) { .build())) << "Inject motion event should return InputEventInjectionResult::SUCCEEDED"; mDragWindow->consumeMotionUp(ADISPLAY_ID_DEFAULT); - mFakePolicy->assertDropTargetEquals(mSecondWindow->getToken()); + mFakePolicy->assertDropTargetEquals(*mDispatcher, mSecondWindow->getToken()); mWindow->assertNoEvents(); mSecondWindow->assertNoEvents(); } @@ -8792,7 +8834,7 @@ TEST_F(InputDispatcherDragTests, DragAndDropFinishedWhenCancelCurrentTouch) { ASSERT_TRUE(mDispatcher->waitForIdle()); // The D&D finished with nullptr - mFakePolicy->assertDropTargetEquals(nullptr); + mFakePolicy->assertDropTargetEquals(*mDispatcher, nullptr); // Remove drag window mDispatcher->onWindowInfosChanged({{*mWindow->getInfo(), *mSecondWindow->getInfo()}, {}, 0, 0}); @@ -8816,7 +8858,8 @@ TEST_F(InputDispatcherDragTests, DragAndDropFinishedWhenCancelCurrentTouch) { ASSERT_NO_FATAL_FAILURE(mWindow->consumeMotionMove()); ASSERT_EQ(InputEventInjectionResult::SUCCEEDED, - injectMotionUp(mDispatcher, AINPUT_SOURCE_TOUCHSCREEN, ADISPLAY_ID_DEFAULT, {50, 50})) + injectMotionUp(mDispatcher, AINPUT_SOURCE_TOUCHSCREEN, ADISPLAY_ID_DEFAULT, + {50, 50})) << "Inject motion event should return InputEventInjectionResult::SUCCEEDED"; ASSERT_NO_FATAL_FAILURE(mWindow->consumeMotionUp()); } @@ -8840,6 +8883,8 @@ TEST_F(InputDispatcherDropInputFeatureTest, WindowDropsInput) { mDispatcher->notifyMotion(generateMotionArgs(AMOTION_EVENT_ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN, ADISPLAY_ID_DEFAULT)); + mDispatcher->notifyMotion(generateMotionArgs(AMOTION_EVENT_ACTION_UP, AINPUT_SOURCE_TOUCHSCREEN, + ADISPLAY_ID_DEFAULT)); window->assertNoEvents(); // With the flag cleared, the window should get input @@ -8862,13 +8907,13 @@ TEST_F(InputDispatcherDropInputFeatureTest, ObscuredWindowDropsInput) { sp<FakeWindowHandle>::make(obscuringApplication, mDispatcher, "obscuringWindow", ADISPLAY_ID_DEFAULT); obscuringWindow->setFrame(Rect(0, 0, 50, 50)); - obscuringWindow->setOwnerInfo(111, 111); + obscuringWindow->setOwnerInfo(gui::Pid{111}, gui::Uid{111}); obscuringWindow->setTouchable(false); std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); sp<FakeWindowHandle> window = sp<FakeWindowHandle>::make(application, mDispatcher, "Test window", ADISPLAY_ID_DEFAULT); window->setDropInputIfObscured(true); - window->setOwnerInfo(222, 222); + window->setOwnerInfo(gui::Pid{222}, gui::Uid{222}); mDispatcher->setFocusedApplication(ADISPLAY_ID_DEFAULT, application); window->setFocusable(true); mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {obscuringWindow, window}}}); @@ -8881,6 +8926,8 @@ TEST_F(InputDispatcherDropInputFeatureTest, ObscuredWindowDropsInput) { mDispatcher->notifyMotion(generateMotionArgs(AMOTION_EVENT_ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN, ADISPLAY_ID_DEFAULT)); + mDispatcher->notifyMotion(generateMotionArgs(AMOTION_EVENT_ACTION_UP, AINPUT_SOURCE_TOUCHSCREEN, + ADISPLAY_ID_DEFAULT)); window->assertNoEvents(); // With the flag cleared, the window should get input @@ -8903,13 +8950,13 @@ TEST_F(InputDispatcherDropInputFeatureTest, UnobscuredWindowGetsInput) { sp<FakeWindowHandle>::make(obscuringApplication, mDispatcher, "obscuringWindow", ADISPLAY_ID_DEFAULT); obscuringWindow->setFrame(Rect(0, 0, 50, 50)); - obscuringWindow->setOwnerInfo(111, 111); + obscuringWindow->setOwnerInfo(gui::Pid{111}, gui::Uid{111}); obscuringWindow->setTouchable(false); std::shared_ptr<FakeApplicationHandle> application = std::make_shared<FakeApplicationHandle>(); sp<FakeWindowHandle> window = sp<FakeWindowHandle>::make(application, mDispatcher, "Test window", ADISPLAY_ID_DEFAULT); window->setDropInputIfObscured(true); - window->setOwnerInfo(222, 222); + window->setOwnerInfo(gui::Pid{222}, gui::Uid{222}); mDispatcher->setFocusedApplication(ADISPLAY_ID_DEFAULT, application); window->setFocusable(true); mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {obscuringWindow, window}}}); @@ -8922,6 +8969,8 @@ TEST_F(InputDispatcherDropInputFeatureTest, UnobscuredWindowGetsInput) { mDispatcher->notifyMotion(generateMotionArgs(AMOTION_EVENT_ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN, ADISPLAY_ID_DEFAULT)); + mDispatcher->notifyMotion(generateMotionArgs(AMOTION_EVENT_ACTION_UP, AINPUT_SOURCE_TOUCHSCREEN, + ADISPLAY_ID_DEFAULT)); window->assertNoEvents(); // When the window is no longer obscured because it went on top, it should get input @@ -8984,7 +9033,7 @@ protected: } } - void changeAndVerifyTouchModeInMainDisplayOnly(bool inTouchMode, int32_t pid, int32_t uid, + void changeAndVerifyTouchModeInMainDisplayOnly(bool inTouchMode, gui::Pid pid, gui::Uid uid, bool hasPermission) { ASSERT_TRUE(mDispatcher->setInTouchMode(inTouchMode, pid, uid, hasPermission, ADISPLAY_ID_DEFAULT)); @@ -9003,9 +9052,9 @@ TEST_F(InputDispatcherTouchModeChangedTests, FocusedWindowCanChangeTouchMode) { TEST_F(InputDispatcherTouchModeChangedTests, NonFocusedWindowOwnerCannotChangeTouchMode) { const WindowInfo& windowInfo = *mWindow->getInfo(); - int32_t ownerPid = windowInfo.ownerPid; - int32_t ownerUid = windowInfo.ownerUid; - mWindow->setOwnerInfo(/* pid */ -1, /* uid */ -1); + gui::Pid ownerPid = windowInfo.ownerPid; + gui::Uid ownerUid = windowInfo.ownerUid; + mWindow->setOwnerInfo(gui::Pid::INVALID, gui::Uid::INVALID); ASSERT_FALSE(mDispatcher->setInTouchMode(InputDispatcher::kDefaultInTouchMode, ownerPid, ownerUid, /*hasPermission=*/false, ADISPLAY_ID_DEFAULT)); @@ -9015,9 +9064,9 @@ TEST_F(InputDispatcherTouchModeChangedTests, NonFocusedWindowOwnerCannotChangeTo TEST_F(InputDispatcherTouchModeChangedTests, NonWindowOwnerMayChangeTouchModeOnPermissionGranted) { const WindowInfo& windowInfo = *mWindow->getInfo(); - int32_t ownerPid = windowInfo.ownerPid; - int32_t ownerUid = windowInfo.ownerUid; - mWindow->setOwnerInfo(/* pid */ -1, /* uid */ -1); + gui::Pid ownerPid = windowInfo.ownerPid; + gui::Uid ownerUid = windowInfo.ownerUid; + mWindow->setOwnerInfo(gui::Pid::INVALID, gui::Uid::INVALID); changeAndVerifyTouchModeInMainDisplayOnly(!InputDispatcher::kDefaultInTouchMode, ownerPid, ownerUid, /*hasPermission=*/true); } @@ -9228,10 +9277,10 @@ TEST_F(InputDispatcherSpyWindowTest, TouchableRegion) { */ TEST_F(InputDispatcherSpyWindowTest, WatchOutsideTouches) { auto window = createForeground(); - window->setOwnerInfo(12, 34); + window->setOwnerInfo(gui::Pid{12}, gui::Uid{34}); auto spy = createSpy(); spy->setWatchOutsideTouch(true); - spy->setOwnerInfo(56, 78); + spy->setOwnerInfo(gui::Pid{56}, gui::Uid{78}); spy->setFrame(Rect{0, 0, 20, 20}); mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {spy, window}}}); @@ -9644,7 +9693,7 @@ public: sp<FakeWindowHandle>::make(overlayApplication, mDispatcher, "Stylus interceptor window", ADISPLAY_ID_DEFAULT); overlay->setFocusable(false); - overlay->setOwnerInfo(111, 111); + overlay->setOwnerInfo(gui::Pid{111}, gui::Uid{111}); overlay->setTouchable(false); overlay->setInterceptsStylus(true); overlay->setTrustedOverlay(true); @@ -9655,7 +9704,7 @@ public: sp<FakeWindowHandle>::make(application, mDispatcher, "Application window", ADISPLAY_ID_DEFAULT); window->setFocusable(true); - window->setOwnerInfo(222, 222); + window->setOwnerInfo(gui::Pid{222}, gui::Uid{222}); mDispatcher->setFocusedApplication(ADISPLAY_ID_DEFAULT, application); mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {overlay, window}}}); @@ -9682,6 +9731,7 @@ public: using InputDispatcherStylusInterceptorDeathTest = InputDispatcherStylusInterceptorTest; TEST_F(InputDispatcherStylusInterceptorDeathTest, UntrustedOverlay_AbortsDispatcher) { + testing::GTEST_FLAG(death_test_style) = "threadsafe"; ScopedSilentDeath _silentDeath; auto [overlay, window] = setupStylusOverlayScenario(); @@ -9765,12 +9815,12 @@ TEST_F(InputDispatcherStylusInterceptorTest, StylusHandwritingScenario) { } struct User { - int32_t mPid; - int32_t mUid; + gui::Pid mPid; + gui::Uid mUid; uint32_t mPolicyFlags{DEFAULT_POLICY_FLAGS}; std::unique_ptr<InputDispatcher>& mDispatcher; - User(std::unique_ptr<InputDispatcher>& dispatcher, int32_t pid, int32_t uid) + User(std::unique_ptr<InputDispatcher>& dispatcher, gui::Pid pid, gui::Uid uid) : mPid(pid), mUid(uid), mDispatcher(dispatcher) {} InputEventInjectionResult injectTargetedMotion(int32_t action) const { @@ -9803,7 +9853,7 @@ struct User { using InputDispatcherTargetedInjectionTest = InputDispatcherTest; TEST_F(InputDispatcherTargetedInjectionTest, CanInjectIntoOwnedWindow) { - auto owner = User(mDispatcher, 10, 11); + auto owner = User(mDispatcher, gui::Pid{10}, gui::Uid{11}); auto window = owner.createWindow(); mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {window}}}); @@ -9820,11 +9870,11 @@ TEST_F(InputDispatcherTargetedInjectionTest, CanInjectIntoOwnedWindow) { } TEST_F(InputDispatcherTargetedInjectionTest, CannotInjectIntoUnownedWindow) { - auto owner = User(mDispatcher, 10, 11); + auto owner = User(mDispatcher, gui::Pid{10}, gui::Uid{11}); auto window = owner.createWindow(); mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {window}}}); - auto rando = User(mDispatcher, 20, 21); + auto rando = User(mDispatcher, gui::Pid{20}, gui::Uid{21}); EXPECT_EQ(InputEventInjectionResult::TARGET_MISMATCH, rando.injectTargetedMotion(AMOTION_EVENT_ACTION_DOWN)); @@ -9837,7 +9887,7 @@ TEST_F(InputDispatcherTargetedInjectionTest, CannotInjectIntoUnownedWindow) { } TEST_F(InputDispatcherTargetedInjectionTest, CanInjectIntoOwnedSpyWindow) { - auto owner = User(mDispatcher, 10, 11); + auto owner = User(mDispatcher, gui::Pid{10}, gui::Uid{11}); auto window = owner.createWindow(); auto spy = owner.createWindow(); spy->setSpy(true); @@ -9851,10 +9901,10 @@ TEST_F(InputDispatcherTargetedInjectionTest, CanInjectIntoOwnedSpyWindow) { } TEST_F(InputDispatcherTargetedInjectionTest, CannotInjectIntoUnownedSpyWindow) { - auto owner = User(mDispatcher, 10, 11); + auto owner = User(mDispatcher, gui::Pid{10}, gui::Uid{11}); auto window = owner.createWindow(); - auto rando = User(mDispatcher, 20, 21); + auto rando = User(mDispatcher, gui::Pid{20}, gui::Uid{21}); auto randosSpy = rando.createWindow(); randosSpy->setSpy(true); randosSpy->setTrustedOverlay(true); @@ -9869,10 +9919,10 @@ TEST_F(InputDispatcherTargetedInjectionTest, CannotInjectIntoUnownedSpyWindow) { } TEST_F(InputDispatcherTargetedInjectionTest, CanInjectIntoAnyWindowWhenNotTargeting) { - auto owner = User(mDispatcher, 10, 11); + auto owner = User(mDispatcher, gui::Pid{10}, gui::Uid{11}); auto window = owner.createWindow(); - auto rando = User(mDispatcher, 20, 21); + auto rando = User(mDispatcher, gui::Pid{20}, gui::Uid{21}); auto randosSpy = rando.createWindow(); randosSpy->setSpy(true); randosSpy->setTrustedOverlay(true); @@ -9893,21 +9943,21 @@ TEST_F(InputDispatcherTargetedInjectionTest, CanInjectIntoAnyWindowWhenNotTarget window->assertNoEvents(); } -TEST_F(InputDispatcherTargetedInjectionTest, CanGenerateActionOutsideToOtherUids) { - auto owner = User(mDispatcher, 10, 11); +TEST_F(InputDispatcherTargetedInjectionTest, CannotGenerateActionOutsideToOtherUids) { + auto owner = User(mDispatcher, gui::Pid{10}, gui::Uid{11}); auto window = owner.createWindow(); - auto rando = User(mDispatcher, 20, 21); + auto rando = User(mDispatcher, gui::Pid{20}, gui::Uid{21}); auto randosWindow = rando.createWindow(); randosWindow->setFrame(Rect{-10, -10, -5, -5}); randosWindow->setWatchOutsideTouch(true); mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {randosWindow, window}}}); - // We allow generation of ACTION_OUTSIDE events into windows owned by different uids. + // Do not allow generation of ACTION_OUTSIDE events into windows owned by different uids. EXPECT_EQ(InputEventInjectionResult::SUCCEEDED, owner.injectTargetedMotion(AMOTION_EVENT_ACTION_DOWN)); window->consumeMotionDown(); - randosWindow->consumeMotionOutside(); + randosWindow->assertNoEvents(); } } // namespace android::inputdispatcher diff --git a/services/inputflinger/tests/InputReader_test.cpp b/services/inputflinger/tests/InputReader_test.cpp index 38b32b3632..8941d16534 100644 --- a/services/inputflinger/tests/InputReader_test.cpp +++ b/services/inputflinger/tests/InputReader_test.cpp @@ -91,6 +91,9 @@ static constexpr int32_t ACTION_POINTER_1_DOWN = static constexpr int32_t ACTION_POINTER_1_UP = AMOTION_EVENT_ACTION_POINTER_UP | (1 << AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT); +static constexpr uint32_t STYLUS_FUSION_SOURCE = + AINPUT_SOURCE_TOUCHSCREEN | AINPUT_SOURCE_BLUETOOTH_STYLUS; + // Minimum timestamp separation between subsequent input events from a Bluetooth device. static constexpr nsecs_t MIN_BLUETOOTH_TIMESTAMP_DELTA = ms2ns(4); // Maximum smoothing time delta so that we don't generate events too far into the future. @@ -1334,19 +1337,8 @@ protected: mFakePolicy = sp<FakeInputReaderPolicy>::make(); mFakePointerController = std::make_shared<FakePointerController>(); mFakePolicy->setPointerController(mFakePointerController); - mTestListener = std::make_unique<TestInputListener>(/*eventHappenedTimeout=*/2000ms, - /*eventDidNotHappenTimeout=*/30ms); - - mReader = std::make_unique<InputReader>(std::make_shared<EventHub>(), mFakePolicy, - *mTestListener); - ASSERT_EQ(mReader->start(), OK); - // Since this test is run on a real device, all the input devices connected - // to the test device will show up in mReader. We wait for those input devices to - // show up before beginning the tests. - ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertInputDevicesChanged()); - ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyInputDevicesChangedWasCalled()); - ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyConfigurationChangedWasCalled()); + setupInputReader(); } void TearDown() override { @@ -1367,6 +1359,22 @@ protected: }); return it != inputDevices.end() ? std::make_optional(*it) : std::nullopt; } + + void setupInputReader() { + mTestListener = std::make_unique<TestInputListener>(/*eventHappenedTimeout=*/2000ms, + /*eventDidNotHappenTimeout=*/30ms); + + mReader = std::make_unique<InputReader>(std::make_shared<EventHub>(), mFakePolicy, + *mTestListener); + ASSERT_EQ(mReader->start(), OK); + + // Since this test is run on a real device, all the input devices connected + // to the test device will show up in mReader. We wait for those input devices to + // show up before beginning the tests. + ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertInputDevicesChanged()); + ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyInputDevicesChangedWasCalled()); + ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyConfigurationChangedWasCalled()); + } }; TEST_F(InputReaderIntegrationTest, TestInvalidDevice) { @@ -1540,7 +1548,7 @@ TEST_F(InputReaderIntegrationTest, SendsGearDownAndUpToInputListener) { // --- TouchIntegrationTest --- -class TouchIntegrationTest : public InputReaderIntegrationTest { +class BaseTouchIntegrationTest : public InputReaderIntegrationTest { protected: const std::string UNIQUE_ID = "local:0"; @@ -1574,8 +1582,8 @@ protected: NotifyMotionArgs args; ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasCalled(&args)); EXPECT_EQ(action, args.action); - ASSERT_EQ(points.size(), args.pointerCount); - for (size_t i = 0; i < args.pointerCount; i++) { + ASSERT_EQ(points.size(), args.getPointerCount()); + for (size_t i = 0; i < args.getPointerCount(); i++) { EXPECT_EQ(points[i].x, args.pointerCoords[i].getX()); EXPECT_EQ(points[i].y, args.pointerCoords[i].getY()); } @@ -1585,7 +1593,55 @@ protected: InputDeviceInfo mDeviceInfo; }; -TEST_F(TouchIntegrationTest, MultiTouchDeviceSource) { +enum class TouchIntegrationTestDisplays { DISPLAY_INTERNAL, DISPLAY_INPUT_PORT, DISPLAY_UNIQUE_ID }; + +class TouchIntegrationTest : public BaseTouchIntegrationTest, + public testing::WithParamInterface<TouchIntegrationTestDisplays> { +protected: + static constexpr std::optional<uint8_t> DISPLAY_PORT = 0; + const std::string INPUT_PORT = "uinput_touch/input0"; + + void SetUp() override { +#if !defined(__ANDROID__) + GTEST_SKIP(); +#endif + if (GetParam() == TouchIntegrationTestDisplays::DISPLAY_INTERNAL) { + BaseTouchIntegrationTest::SetUp(); + return; + } + + // setup policy with a input-port or UniqueId association to the display + bool isInputPortAssociation = + GetParam() == TouchIntegrationTestDisplays::DISPLAY_INPUT_PORT; + + mFakePolicy = sp<FakeInputReaderPolicy>::make(); + if (isInputPortAssociation) { + mFakePolicy->addInputPortAssociation(INPUT_PORT, DISPLAY_PORT.value()); + } else { + mFakePolicy->addInputUniqueIdAssociation(INPUT_PORT, UNIQUE_ID); + } + mFakePointerController = std::make_shared<FakePointerController>(); + mFakePolicy->setPointerController(mFakePointerController); + + InputReaderIntegrationTest::setupInputReader(); + + mDevice = createUinputDevice<UinputTouchScreen>(Rect(0, 0, DISPLAY_WIDTH, DISPLAY_HEIGHT), + INPUT_PORT); + ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertInputDevicesChanged()); + + // Add a display linked to a physical port or UniqueId. + setDisplayInfoAndReconfigure(DISPLAY_ID, DISPLAY_WIDTH, DISPLAY_HEIGHT, ui::ROTATION_0, + UNIQUE_ID, isInputPortAssociation ? DISPLAY_PORT : NO_PORT, + ViewportType::INTERNAL); + ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertInputDevicesChanged()); + ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyConfigurationChangedWasCalled()); + const auto info = findDeviceByName(mDevice->getName()); + ASSERT_TRUE(info); + mDeviceInfo = *info; + } +}; + +TEST_P(TouchIntegrationTest, MultiTouchDeviceSource) { // The UinputTouchScreen is an MT device that supports MT_TOOL_TYPE and also supports stylus // buttons. It should show up as a touchscreen, stylus, and keyboard (for reporting button // presses). @@ -1593,7 +1649,7 @@ TEST_F(TouchIntegrationTest, MultiTouchDeviceSource) { mDeviceInfo.getSources()); } -TEST_F(TouchIntegrationTest, InputEvent_ProcessSingleTouch) { +TEST_P(TouchIntegrationTest, InputEvent_ProcessSingleTouch) { NotifyMotionArgs args; const Point centerPoint = mDevice->getCenterPoint(); @@ -1617,7 +1673,7 @@ TEST_F(TouchIntegrationTest, InputEvent_ProcessSingleTouch) { ASSERT_EQ(AMOTION_EVENT_ACTION_UP, args.action); } -TEST_F(TouchIntegrationTest, InputEvent_ProcessMultiTouch) { +TEST_P(TouchIntegrationTest, InputEvent_ProcessMultiTouch) { NotifyMotionArgs args; const Point centerPoint = mDevice->getCenterPoint(); @@ -1673,7 +1729,7 @@ TEST_F(TouchIntegrationTest, InputEvent_ProcessMultiTouch) { * palms, and wants to cancel Pointer 1, then it is safe to simply drop POINTER_1_UP event without * losing information about non-palm pointers. */ -TEST_F(TouchIntegrationTest, MultiTouch_PointerMoveAndSecondPointerUp) { +TEST_P(TouchIntegrationTest, MultiTouch_PointerMoveAndSecondPointerUp) { NotifyMotionArgs args; const Point centerPoint = mDevice->getCenterPoint(); @@ -1716,7 +1772,7 @@ TEST_F(TouchIntegrationTest, MultiTouch_PointerMoveAndSecondPointerUp) { * In this scenario, the movement of the second pointer just prior to liftoff is ignored, and never * gets sent to the listener. */ -TEST_F(TouchIntegrationTest, MultiTouch_PointerMoveAndSecondPointerMoveAndUp) { +TEST_P(TouchIntegrationTest, MultiTouch_PointerMoveAndSecondPointerMoveAndUp) { NotifyMotionArgs args; const Point centerPoint = mDevice->getCenterPoint(); @@ -1756,7 +1812,7 @@ TEST_F(TouchIntegrationTest, MultiTouch_PointerMoveAndSecondPointerMoveAndUp) { assertReceivedMotion(AMOTION_EVENT_ACTION_MOVE, {centerPoint + Point(5, 5)}); } -TEST_F(TouchIntegrationTest, InputEvent_ProcessPalm) { +TEST_P(TouchIntegrationTest, InputEvent_ProcessPalm) { NotifyMotionArgs args; const Point centerPoint = mDevice->getCenterPoint(); @@ -1807,7 +1863,7 @@ TEST_F(TouchIntegrationTest, InputEvent_ProcessPalm) { ASSERT_EQ(AMOTION_EVENT_ACTION_UP, args.action); } -TEST_F(TouchIntegrationTest, NotifiesPolicyWhenStylusGestureStarted) { +TEST_P(TouchIntegrationTest, NotifiesPolicyWhenStylusGestureStarted) { const Point centerPoint = mDevice->getCenterPoint(); // Send down with the pen tool selected. The policy should be notified of the stylus presence. @@ -1859,19 +1915,24 @@ TEST_F(TouchIntegrationTest, NotifiesPolicyWhenStylusGestureStarted) { ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertStylusGestureNotified(mDeviceInfo.getId())); } +INSTANTIATE_TEST_SUITE_P(TouchIntegrationTestDisplayVariants, TouchIntegrationTest, + testing::Values(TouchIntegrationTestDisplays::DISPLAY_INTERNAL, + TouchIntegrationTestDisplays::DISPLAY_INPUT_PORT, + TouchIntegrationTestDisplays::DISPLAY_UNIQUE_ID)); + // --- StylusButtonIntegrationTest --- // Verify the behavior of button presses reported by various kinds of styluses, including buttons // reported by the touchscreen's device, by a fused external stylus, and by an un-fused external // stylus. template <typename UinputStylusDevice> -class StylusButtonIntegrationTest : public TouchIntegrationTest { +class StylusButtonIntegrationTest : public BaseTouchIntegrationTest { protected: void SetUp() override { #if !defined(__ANDROID__) GTEST_SKIP(); #endif - TouchIntegrationTest::SetUp(); + BaseTouchIntegrationTest::SetUp(); mTouchscreen = mDevice.get(); mTouchscreenInfo = mDeviceInfo; @@ -1909,8 +1970,8 @@ private: std::unique_ptr<UinputStylusDevice> mStylusDeviceLifecycleTracker{}; // Hide the base class's device to expose it with a different name for readability. - using TouchIntegrationTest::mDevice; - using TouchIntegrationTest::mDeviceInfo; + using BaseTouchIntegrationTest::mDevice; + using BaseTouchIntegrationTest::mDeviceInfo; }; using StylusButtonIntegrationTestTypes = @@ -2162,7 +2223,23 @@ TYPED_TEST(StylusButtonIntegrationTest, DISABLED_StylusButtonMotionEventsDisable // Verify the behavior of an external stylus. An external stylus can report pressure or button // data independently of the touchscreen, which is then sent as a MotionEvent as part of an // ongoing stylus gesture that is being emitted by the touchscreen. -using ExternalStylusIntegrationTest = TouchIntegrationTest; +using ExternalStylusIntegrationTest = BaseTouchIntegrationTest; + +TEST_F(ExternalStylusIntegrationTest, ExternalStylusConnectionChangesTouchscreenSource) { + // Create an external stylus capable of reporting pressure data that + // should be fused with a touch pointer. + std::unique_ptr<UinputExternalStylusWithPressure> stylus = + createUinputDevice<UinputExternalStylusWithPressure>(); + ASSERT_NO_FATAL_FAILURE(mFakePolicy->assertInputDevicesChanged()); + ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyConfigurationChangedWasCalled()); + const auto stylusInfo = findDeviceByName(stylus->getName()); + ASSERT_TRUE(stylusInfo); + + // Connecting an external stylus changes the source of the touchscreen. + const auto deviceInfo = findDeviceByName(mDevice->getName()); + ASSERT_TRUE(deviceInfo); + ASSERT_TRUE(isFromSource(deviceInfo->getSources(), STYLUS_FUSION_SOURCE)); +} TEST_F(ExternalStylusIntegrationTest, DISABLED_FusedExternalStylusPressureReported) { const Point centerPoint = mDevice->getCenterPoint(); @@ -2193,17 +2270,17 @@ TEST_F(ExternalStylusIntegrationTest, DISABLED_FusedExternalStylusPressureReport mDevice->sendDown(centerPoint); mDevice->sendSync(); ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasCalled( - AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), - WithToolType(ToolType::STYLUS), WithButtonState(0), - WithDeviceId(touchscreenId), WithPressure(100.f / RAW_PRESSURE_MAX)))); + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), WithToolType(ToolType::STYLUS), + WithButtonState(0), WithSource(STYLUS_FUSION_SOURCE), WithDeviceId(touchscreenId), + WithPressure(100.f / RAW_PRESSURE_MAX)))); // Change the pressure on the external stylus, and ensure the touchscreen generates a MOVE // event with the updated pressure. stylus->setPressure(200); ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasCalled( - AllOf(WithMotionAction(AMOTION_EVENT_ACTION_MOVE), - WithToolType(ToolType::STYLUS), WithButtonState(0), - WithDeviceId(touchscreenId), WithPressure(200.f / RAW_PRESSURE_MAX)))); + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_MOVE), WithToolType(ToolType::STYLUS), + WithButtonState(0), WithSource(STYLUS_FUSION_SOURCE), WithDeviceId(touchscreenId), + WithPressure(200.f / RAW_PRESSURE_MAX)))); // The external stylus did not generate any events. ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasNotCalled()); @@ -2248,8 +2325,8 @@ TEST_F(ExternalStylusIntegrationTest, DISABLED_FusedExternalStylusPressureNotRep // it shows up as a finger pointer. ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasCalled( AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), - WithToolType(ToolType::FINGER), WithDeviceId(touchscreenId), - WithPressure(1.f)))); + WithSource(AINPUT_SOURCE_TOUCHSCREEN | AINPUT_SOURCE_STYLUS), + WithToolType(ToolType::FINGER), WithDeviceId(touchscreenId), WithPressure(1.f)))); // Change the pressure on the external stylus. Since the pressure was not present at the start // of the gesture, it is ignored for now. @@ -2261,6 +2338,7 @@ TEST_F(ExternalStylusIntegrationTest, DISABLED_FusedExternalStylusPressureNotRep mDevice->sendSync(); ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasCalled( AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), + WithSource(AINPUT_SOURCE_TOUCHSCREEN | AINPUT_SOURCE_STYLUS), WithToolType(ToolType::FINGER)))); // Start a new gesture. Since we have a valid pressure value, it shows up as a stylus. @@ -2269,9 +2347,9 @@ TEST_F(ExternalStylusIntegrationTest, DISABLED_FusedExternalStylusPressureNotRep mDevice->sendDown(centerPoint); mDevice->sendSync(); ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasCalled( - AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), - WithToolType(ToolType::STYLUS), WithButtonState(0), - WithDeviceId(touchscreenId), WithPressure(200.f / RAW_PRESSURE_MAX)))); + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), WithSource(STYLUS_FUSION_SOURCE), + WithToolType(ToolType::STYLUS), WithButtonState(0), WithDeviceId(touchscreenId), + WithPressure(200.f / RAW_PRESSURE_MAX)))); // The external stylus did not generate any events. ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasNotCalled()); @@ -2303,14 +2381,15 @@ TEST_F(ExternalStylusIntegrationTest, DISABLED_UnfusedExternalStylus) { std::chrono::milliseconds(ns2ms(EXTERNAL_STYLUS_DATA_TIMEOUT)); mDevice->sendSync(); ASSERT_NO_FATAL_FAILURE( - mTestListener - ->assertNotifyMotionWasCalled(AllOf(WithMotionAction(AMOTION_EVENT_ACTION_DOWN), - WithToolType( - ToolType::FINGER), - WithButtonState(0), - WithDeviceId(touchscreenId), - WithPressure(1.f)), - waitUntil)); + mTestListener->assertNotifyMotionWasCalled(AllOf(WithMotionAction( + AMOTION_EVENT_ACTION_DOWN), + WithToolType(ToolType::FINGER), + WithSource(AINPUT_SOURCE_TOUCHSCREEN | + AINPUT_SOURCE_STYLUS), + WithButtonState(0), + WithDeviceId(touchscreenId), + WithPressure(1.f)), + waitUntil)); // The external stylus did not generate any events. ASSERT_NO_FATAL_FAILURE(mTestListener->assertNotifyMotionWasNotCalled()); @@ -3989,7 +4068,7 @@ TEST_F(CursorInputMapperTest, Process_ShouldSetAllFieldsAndIncludeGlobalMetaStat ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, args.metaState); ASSERT_EQ(AMOTION_EVENT_BUTTON_PRIMARY, args.buttonState); ASSERT_EQ(0, args.edgeFlags); - ASSERT_EQ(uint32_t(1), args.pointerCount); + ASSERT_EQ(uint32_t(1), args.getPointerCount()); ASSERT_EQ(0, args.pointerProperties[0].id); ASSERT_EQ(ToolType::MOUSE, args.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertCursorPointerCoords(args.pointerCoords[0], 0.0f, 0.0f, 1.0f)); @@ -4007,7 +4086,7 @@ TEST_F(CursorInputMapperTest, Process_ShouldSetAllFieldsAndIncludeGlobalMetaStat ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, args.metaState); ASSERT_EQ(AMOTION_EVENT_BUTTON_PRIMARY, args.buttonState); ASSERT_EQ(0, args.edgeFlags); - ASSERT_EQ(uint32_t(1), args.pointerCount); + ASSERT_EQ(uint32_t(1), args.getPointerCount()); ASSERT_EQ(0, args.pointerProperties[0].id); ASSERT_EQ(ToolType::MOUSE, args.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertCursorPointerCoords(args.pointerCoords[0], 0.0f, 0.0f, 1.0f)); @@ -4028,7 +4107,7 @@ TEST_F(CursorInputMapperTest, Process_ShouldSetAllFieldsAndIncludeGlobalMetaStat ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, args.metaState); ASSERT_EQ(0, args.buttonState); ASSERT_EQ(0, args.edgeFlags); - ASSERT_EQ(uint32_t(1), args.pointerCount); + ASSERT_EQ(uint32_t(1), args.getPointerCount()); ASSERT_EQ(0, args.pointerProperties[0].id); ASSERT_EQ(ToolType::MOUSE, args.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertCursorPointerCoords(args.pointerCoords[0], 0.0f, 0.0f, 0.0f)); @@ -4046,7 +4125,7 @@ TEST_F(CursorInputMapperTest, Process_ShouldSetAllFieldsAndIncludeGlobalMetaStat ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, args.metaState); ASSERT_EQ(0, args.buttonState); ASSERT_EQ(0, args.edgeFlags); - ASSERT_EQ(uint32_t(1), args.pointerCount); + ASSERT_EQ(uint32_t(1), args.getPointerCount()); ASSERT_EQ(0, args.pointerProperties[0].id); ASSERT_EQ(ToolType::MOUSE, args.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertCursorPointerCoords(args.pointerCoords[0], 0.0f, 0.0f, 0.0f)); @@ -5309,7 +5388,7 @@ TEST_F(SingleTouchInputMapperTest, Process_WhenVirtualKeyIsPressedAndMovedOutOfB ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5333,7 +5412,7 @@ TEST_F(SingleTouchInputMapperTest, Process_WhenVirtualKeyIsPressedAndMovedOutOfB ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5356,7 +5435,7 @@ TEST_F(SingleTouchInputMapperTest, Process_WhenVirtualKeyIsPressedAndMovedOutOfB ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5406,7 +5485,7 @@ TEST_F(SingleTouchInputMapperTest, Process_WhenTouchStartsOutsideDisplayAndMoves ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5429,7 +5508,7 @@ TEST_F(SingleTouchInputMapperTest, Process_WhenTouchStartsOutsideDisplayAndMoves ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5474,7 +5553,7 @@ TEST_F(SingleTouchInputMapperTest, Process_NormalSingleTouchGesture_VirtualDispl ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5501,7 +5580,7 @@ TEST_F(SingleTouchInputMapperTest, Process_NormalSingleTouchGesture_VirtualDispl ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5526,7 +5605,7 @@ TEST_F(SingleTouchInputMapperTest, Process_NormalSingleTouchGesture_VirtualDispl ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5569,7 +5648,7 @@ TEST_F(SingleTouchInputMapperTest, Process_NormalSingleTouchGesture) { ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5594,7 +5673,7 @@ TEST_F(SingleTouchInputMapperTest, Process_NormalSingleTouchGesture) { ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -5617,7 +5696,7 @@ TEST_F(SingleTouchInputMapperTest, Process_NormalSingleTouchGesture) { ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -6972,7 +7051,7 @@ public: NotifyMotionArgs motionArgs; ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], point.x, point.y, 1, 0, 0, 0, 0, 0, 0, 0)); } @@ -7044,7 +7123,7 @@ TEST_F(TouchDisplayProjectionTest, EmitsTouchDownAfterEnteringPhysicalDisplay) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_NO_FATAL_FAILURE( assertPointerCoords(motionArgs.pointerCoords[0], 11, 21, 1, 0, 0, 0, 0, 0, 0, 0)); @@ -7342,12 +7421,10 @@ public: protected: StylusState mStylusState{}; - static constexpr uint32_t EXPECTED_SOURCE = - AINPUT_SOURCE_TOUCHSCREEN | AINPUT_SOURCE_BLUETOOTH_STYLUS; void testStartFusedStylusGesture(SingleTouchInputMapper& mapper) { auto toolTypeSource = - AllOf(WithSource(EXPECTED_SOURCE), WithToolType(ToolType::STYLUS)); + AllOf(WithSource(STYLUS_FUSION_SOURCE), WithToolType(ToolType::STYLUS)); // The first pointer is withheld. processDown(mapper, 100, 200); @@ -7381,7 +7458,7 @@ protected: processUp(mapper); processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled( - AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithSource(EXPECTED_SOURCE), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithSource(STYLUS_FUSION_SOURCE), WithToolType(ToolType::STYLUS)))); mStylusState.pressure = 0.f; @@ -7391,8 +7468,10 @@ protected: } void testUnsuccessfulFusionGesture(SingleTouchInputMapper& mapper) { + // When stylus fusion is not successful, events should be reported with the original source. + // In this case, it is from a touchscreen. auto toolTypeSource = - AllOf(WithSource(EXPECTED_SOURCE), WithToolType(ToolType::FINGER)); + AllOf(WithSource(AINPUT_SOURCE_TOUCHSCREEN), WithToolType(ToolType::FINGER)); // The first pointer is withheld when an external stylus is connected, // and a timeout is requested. @@ -7432,7 +7511,7 @@ private: TEST_F(ExternalStylusFusionTest, UsesBluetoothStylusSource) { SingleTouchInputMapper& mapper = initializeInputMapperWithExternalStylus(); - ASSERT_EQ(EXPECTED_SOURCE, mapper.getSources()); + ASSERT_EQ(STYLUS_FUSION_SOURCE, mapper.getSources()); } TEST_F(ExternalStylusFusionTest, UnsuccessfulFusion) { @@ -7449,8 +7528,7 @@ TEST_F(ExternalStylusFusionTest, SuccessfulFusion_TouchFirst) { // before the touch is reported by the touchscreen. TEST_F(ExternalStylusFusionTest, SuccessfulFusion_PressureFirst) { SingleTouchInputMapper& mapper = initializeInputMapperWithExternalStylus(); - auto toolTypeSource = - AllOf(WithSource(EXPECTED_SOURCE), WithToolType(ToolType::STYLUS)); + auto toolTypeSource = AllOf(WithSource(STYLUS_FUSION_SOURCE), WithToolType(ToolType::STYLUS)); // The external stylus reports pressure first. It is ignored for now. mStylusState.pressure = 1.f; @@ -7492,8 +7570,7 @@ TEST_F(ExternalStylusFusionTest, FusionIsRepeatedForEachNewGesture) { TEST_F(ExternalStylusFusionTest, FusedPointerReportsPressureChanges) { SingleTouchInputMapper& mapper = initializeInputMapperWithExternalStylus(); - auto toolTypeSource = - AllOf(WithSource(EXPECTED_SOURCE), WithToolType(ToolType::STYLUS)); + auto toolTypeSource = AllOf(WithSource(STYLUS_FUSION_SOURCE), WithToolType(ToolType::STYLUS)); mStylusState.pressure = 0.8f; processExternalStylusState(mapper); @@ -7554,7 +7631,7 @@ TEST_F(ExternalStylusFusionTest, FusedPointerReportsPressureChanges) { processUp(mapper); processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled( - AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithSource(EXPECTED_SOURCE), + AllOf(WithMotionAction(AMOTION_EVENT_ACTION_UP), WithSource(STYLUS_FUSION_SOURCE), WithToolType(ToolType::STYLUS)))); ASSERT_NO_FATAL_FAILURE(mReader->getContext()->assertTimeoutWasNotRequested()); @@ -7563,7 +7640,7 @@ TEST_F(ExternalStylusFusionTest, FusedPointerReportsPressureChanges) { TEST_F(ExternalStylusFusionTest, FusedPointerReportsToolTypeChanges) { SingleTouchInputMapper& mapper = initializeInputMapperWithExternalStylus(); - auto source = WithSource(EXPECTED_SOURCE); + auto source = WithSource(STYLUS_FUSION_SOURCE); mStylusState.pressure = 1.f; mStylusState.toolType = ToolType::ERASER; @@ -7616,8 +7693,7 @@ TEST_F(ExternalStylusFusionTest, FusedPointerReportsToolTypeChanges) { TEST_F(ExternalStylusFusionTest, FusedPointerReportsButtons) { SingleTouchInputMapper& mapper = initializeInputMapperWithExternalStylus(); - auto toolTypeSource = - AllOf(WithSource(EXPECTED_SOURCE), WithToolType(ToolType::STYLUS)); + auto toolTypeSource = AllOf(WithSource(STYLUS_FUSION_SOURCE), WithToolType(ToolType::STYLUS)); ASSERT_NO_FATAL_FAILURE(testStartFusedStylusGesture(mapper)); @@ -7832,7 +7908,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -7851,7 +7927,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -7882,7 +7958,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -7911,7 +7987,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -7934,7 +8010,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(1, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -7959,7 +8035,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(1, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -7986,7 +8062,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8015,7 +8091,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8038,7 +8114,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8061,7 +8137,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithoutTrackin ASSERT_EQ(AMETA_SHIFT_LEFT_ON | AMETA_SHIFT_ON, motionArgs.metaState); ASSERT_EQ(0, motionArgs.buttonState); ASSERT_EQ(0, motionArgs.edgeFlags); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8149,7 +8225,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8157,7 +8233,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_1_DOWN, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8179,7 +8255,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8198,7 +8274,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_0_UP, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8210,7 +8286,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(1, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8225,7 +8301,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(1, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8243,7 +8319,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_0_DOWN, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8262,7 +8338,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_1_UP, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8274,7 +8350,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8286,7 +8362,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithTrackingId ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_UP, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8319,7 +8395,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8327,7 +8403,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_1_DOWN, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8347,7 +8423,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8367,7 +8443,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_0_UP, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8379,7 +8455,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(1, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8392,7 +8468,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(1, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8408,7 +8484,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_0_DOWN, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8428,7 +8504,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_1_UP, motionArgs.action); - ASSERT_EQ(size_t(2), motionArgs.pointerCount); + ASSERT_EQ(size_t(2), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(1, motionArgs.pointerProperties[1].id); @@ -8440,7 +8516,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8452,7 +8528,7 @@ TEST_F(MultiTouchInputMapperTest, Process_NormalMultiTouchGesture_WithSlots) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_UP, motionArgs.action); - ASSERT_EQ(size_t(1), motionArgs.pointerCount); + ASSERT_EQ(size_t(1), motionArgs.getPointerCount()); ASSERT_EQ(0, motionArgs.pointerProperties[0].id); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(motionArgs.pointerCoords[0], @@ -8593,7 +8669,7 @@ TEST_F(MultiTouchInputMapperTest, Process_TouchAndToolAxes_SummedLinearCalibrati ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&args)); ASSERT_EQ(ACTION_POINTER_1_DOWN, args.action); - ASSERT_EQ(size_t(2), args.pointerCount); + ASSERT_EQ(size_t(2), args.getPointerCount()); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(args.pointerCoords[0], x, y, 1.0f, size, touch, touch, tool, tool, 0, 0)); ASSERT_NO_FATAL_FAILURE(assertPointerCoords(args.pointerCoords[1], @@ -9852,7 +9928,7 @@ TEST_F(MultiTouchInputMapperTest, Process_ShouldHandlePalmToolType_TwoPointers) processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // First finger up. It used to be in palm mode, and we already generated ACTION_POINTER_UP for // it. Second finger receive move. @@ -9861,7 +9937,7 @@ TEST_F(MultiTouchInputMapperTest, Process_ShouldHandlePalmToolType_TwoPointers) processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // Second finger keeps moving. processSlot(mapper, SECOND_SLOT); @@ -9870,7 +9946,7 @@ TEST_F(MultiTouchInputMapperTest, Process_ShouldHandlePalmToolType_TwoPointers) processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // Second finger up. processId(mapper, INVALID_TRACKING_ID); @@ -9944,7 +10020,7 @@ TEST_F(MultiTouchInputMapperTest, Process_ShouldHandlePalmToolType_ShouldCancelW ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // third finger move processId(mapper, THIRD_TRACKING_ID); @@ -9959,7 +10035,7 @@ TEST_F(MultiTouchInputMapperTest, Process_ShouldHandlePalmToolType_ShouldCancelW processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // second finger up, third finger receive move. processSlot(mapper, SECOND_SLOT); @@ -9967,7 +10043,7 @@ TEST_F(MultiTouchInputMapperTest, Process_ShouldHandlePalmToolType_ShouldCancelW processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // third finger up. processSlot(mapper, THIRD_SLOT); @@ -10024,7 +10100,7 @@ TEST_F(MultiTouchInputMapperTest, Process_ShouldHandlePalmToolType_KeepFirstPoin processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // second finger up. processSlot(mapper, SECOND_SLOT); @@ -10070,7 +10146,7 @@ TEST_F(MultiTouchInputMapperTest, Process_MultiTouch_WithInvalidTrackingId) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // First finger move. processId(mapper, FIRST_TRACKING_ID); @@ -10079,7 +10155,7 @@ TEST_F(MultiTouchInputMapperTest, Process_MultiTouch_WithInvalidTrackingId) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); // Second finger down. processSlot(mapper, SECOND_SLOT); @@ -10089,7 +10165,7 @@ TEST_F(MultiTouchInputMapperTest, Process_MultiTouch_WithInvalidTrackingId) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_1_DOWN, motionArgs.action); - ASSERT_EQ(uint32_t(2), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(2), motionArgs.getPointerCount()); // second finger up with some unexpected data. processSlot(mapper, SECOND_SLOT); @@ -10098,7 +10174,7 @@ TEST_F(MultiTouchInputMapperTest, Process_MultiTouch_WithInvalidTrackingId) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ACTION_POINTER_1_UP, motionArgs.action); - ASSERT_EQ(uint32_t(2), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(2), motionArgs.getPointerCount()); // first finger up with some unexpected data. processSlot(mapper, FIRST_SLOT); @@ -10108,7 +10184,7 @@ TEST_F(MultiTouchInputMapperTest, Process_MultiTouch_WithInvalidTrackingId) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(AMOTION_EVENT_ACTION_UP, motionArgs.action); - ASSERT_EQ(uint32_t(1), motionArgs.pointerCount); + ASSERT_EQ(uint32_t(1), motionArgs.getPointerCount()); } TEST_F(MultiTouchInputMapperTest, Reset_PreservesLastTouchState) { @@ -10363,7 +10439,7 @@ TEST_F(MultiTouchInputMapperTest, Process_TouchpadCapture) { NotifyMotionArgs args; ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&args)); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, args.action); - ASSERT_EQ(1U, args.pointerCount); + ASSERT_EQ(1U, args.getPointerCount()); ASSERT_EQ(0, args.pointerProperties[0].id); ASSERT_EQ(AINPUT_SOURCE_TOUCHPAD, args.source); ASSERT_NO_FATAL_FAILURE( @@ -10378,7 +10454,7 @@ TEST_F(MultiTouchInputMapperTest, Process_TouchpadCapture) { // expect coord[0] to contain previous location, coord[1] to contain new touch 1 location ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&args)); ASSERT_EQ(ACTION_POINTER_1_DOWN, args.action); - ASSERT_EQ(2U, args.pointerCount); + ASSERT_EQ(2U, args.getPointerCount()); ASSERT_EQ(0, args.pointerProperties[0].id); ASSERT_EQ(1, args.pointerProperties[1].id); ASSERT_NO_FATAL_FAILURE( @@ -10446,7 +10522,7 @@ TEST_F(MultiTouchInputMapperTest, Process_TouchpadCapture) { // expect coord[0] to contain new location of touch 1, and properties[0].id to contain 1 ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&args)); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, args.action); - ASSERT_EQ(1U, args.pointerCount); + ASSERT_EQ(1U, args.getPointerCount()); ASSERT_EQ(1, args.pointerProperties[0].id); ASSERT_NO_FATAL_FAILURE( assertPointerCoords(args.pointerCoords[0], 320, 900, 1, 0, 0, 0, 0, 0, 0, 0)); @@ -10666,7 +10742,7 @@ TEST_F(MultiTouchPointerModeTest, PointerGestureMaxSwipeWidthSwipe) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(MotionClassification::NONE, motionArgs.classification); @@ -10688,7 +10764,7 @@ TEST_F(MultiTouchPointerModeTest, PointerGestureMaxSwipeWidthSwipe) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(MotionClassification::TWO_FINGER_SWIPE, motionArgs.classification); @@ -10726,7 +10802,7 @@ TEST_F(MultiTouchPointerModeTest, PointerGestureMaxSwipeWidthLowResolutionSwipe) processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(MotionClassification::NONE, motionArgs.classification); @@ -10748,7 +10824,7 @@ TEST_F(MultiTouchPointerModeTest, PointerGestureMaxSwipeWidthLowResolutionSwipe) processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(MotionClassification::TWO_FINGER_SWIPE, motionArgs.classification); @@ -10782,7 +10858,7 @@ TEST_F(MultiTouchPointerModeTest, PointerGestureMaxSwipeWidthFreeform) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(MotionClassification::NONE, motionArgs.classification); @@ -10807,16 +10883,16 @@ TEST_F(MultiTouchPointerModeTest, PointerGestureMaxSwipeWidthFreeform) { ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); // The previous PRESS gesture is cancelled, because it is transformed to freeform - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_CANCEL, motionArgs.action); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(MotionClassification::NONE, motionArgs.classification); - ASSERT_EQ(2U, motionArgs.pointerCount); + ASSERT_EQ(2U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_POINTER_DOWN, motionArgs.action & AMOTION_EVENT_ACTION_MASK); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(MotionClassification::NONE, motionArgs.classification); @@ -10846,7 +10922,7 @@ TEST_F(MultiTouchPointerModeTest, PointerGestureMaxSwipeWidthFreeform) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); - ASSERT_EQ(2U, motionArgs.pointerCount); + ASSERT_EQ(2U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); ASSERT_EQ(ToolType::FINGER, motionArgs.pointerProperties[0].toolType); ASSERT_EQ(MotionClassification::NONE, motionArgs.classification); @@ -10875,7 +10951,7 @@ TEST_F(MultiTouchPointerModeTest, TwoFingerSwipeOffsets) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_DOWN, motionArgs.action); ASSERT_EQ(MotionClassification::NONE, motionArgs.classification); ASSERT_EQ(0, motionArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_GESTURE_X_OFFSET)); @@ -10897,7 +10973,7 @@ TEST_F(MultiTouchPointerModeTest, TwoFingerSwipeOffsets) { processSync(mapper); ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs)); - ASSERT_EQ(1U, motionArgs.pointerCount); + ASSERT_EQ(1U, motionArgs.getPointerCount()); ASSERT_EQ(AMOTION_EVENT_ACTION_MOVE, motionArgs.action); ASSERT_EQ(MotionClassification::TWO_FINGER_SWIPE, motionArgs.classification); ASSERT_LT(motionArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_GESTURE_X_OFFSET), 0); @@ -11144,6 +11220,101 @@ TEST_F(LightControllerTest, MonoKeyboardBacklight) { ASSERT_EQ(controller.getLightColor(lights[0].id).value_or(-1), LIGHT_BRIGHTNESS); } +TEST_F(LightControllerTest, Ignore_MonoLight_WithPreferredBacklightLevels) { + RawLightInfo infoMono = {.id = 1, + .name = "mono_light", + .maxBrightness = 255, + .flags = InputLightClass::BRIGHTNESS, + .path = ""}; + mFakeEventHub->addRawLightInfo(infoMono.id, std::move(infoMono)); + mFakeEventHub->addConfigurationProperty(EVENTHUB_ID, "keyboard.backlight.brightnessLevels", + "0,100,200"); + + PeripheralController& controller = addControllerAndConfigure<PeripheralController>(); + std::list<NotifyArgs> unused = + mDevice->configure(ARBITRARY_TIME, mFakePolicy->getReaderConfiguration(), + /*changes=*/{}); + + InputDeviceInfo info; + controller.populateDeviceInfo(&info); + std::vector<InputDeviceLightInfo> lights = info.getLights(); + ASSERT_EQ(1U, lights.size()); + ASSERT_EQ(0U, lights[0].preferredBrightnessLevels.size()); +} + +TEST_F(LightControllerTest, KeyboardBacklight_WithNoPreferredBacklightLevels) { + RawLightInfo infoMono = {.id = 1, + .name = "mono_keyboard_backlight", + .maxBrightness = 255, + .flags = InputLightClass::BRIGHTNESS | + InputLightClass::KEYBOARD_BACKLIGHT, + .path = ""}; + mFakeEventHub->addRawLightInfo(infoMono.id, std::move(infoMono)); + + PeripheralController& controller = addControllerAndConfigure<PeripheralController>(); + std::list<NotifyArgs> unused = + mDevice->configure(ARBITRARY_TIME, mFakePolicy->getReaderConfiguration(), + /*changes=*/{}); + + InputDeviceInfo info; + controller.populateDeviceInfo(&info); + std::vector<InputDeviceLightInfo> lights = info.getLights(); + ASSERT_EQ(1U, lights.size()); + ASSERT_EQ(0U, lights[0].preferredBrightnessLevels.size()); +} + +TEST_F(LightControllerTest, KeyboardBacklight_WithPreferredBacklightLevels) { + RawLightInfo infoMono = {.id = 1, + .name = "mono_keyboard_backlight", + .maxBrightness = 255, + .flags = InputLightClass::BRIGHTNESS | + InputLightClass::KEYBOARD_BACKLIGHT, + .path = ""}; + mFakeEventHub->addRawLightInfo(infoMono.id, std::move(infoMono)); + mFakeEventHub->addConfigurationProperty(EVENTHUB_ID, "keyboard.backlight.brightnessLevels", + "0,100,200"); + + PeripheralController& controller = addControllerAndConfigure<PeripheralController>(); + std::list<NotifyArgs> unused = + mDevice->configure(ARBITRARY_TIME, mFakePolicy->getReaderConfiguration(), + /*changes=*/{}); + + InputDeviceInfo info; + controller.populateDeviceInfo(&info); + std::vector<InputDeviceLightInfo> lights = info.getLights(); + ASSERT_EQ(1U, lights.size()); + ASSERT_EQ(3U, lights[0].preferredBrightnessLevels.size()); + std::set<BrightnessLevel>::iterator it = lights[0].preferredBrightnessLevels.begin(); + ASSERT_EQ(BrightnessLevel(0), *it); + std::advance(it, 1); + ASSERT_EQ(BrightnessLevel(100), *it); + std::advance(it, 1); + ASSERT_EQ(BrightnessLevel(200), *it); +} + +TEST_F(LightControllerTest, KeyboardBacklight_WithWrongPreferredBacklightLevels) { + RawLightInfo infoMono = {.id = 1, + .name = "mono_keyboard_backlight", + .maxBrightness = 255, + .flags = InputLightClass::BRIGHTNESS | + InputLightClass::KEYBOARD_BACKLIGHT, + .path = ""}; + mFakeEventHub->addRawLightInfo(infoMono.id, std::move(infoMono)); + mFakeEventHub->addConfigurationProperty(EVENTHUB_ID, "keyboard.backlight.brightnessLevels", + "0,100,200,300,400,500"); + + PeripheralController& controller = addControllerAndConfigure<PeripheralController>(); + std::list<NotifyArgs> unused = + mDevice->configure(ARBITRARY_TIME, mFakePolicy->getReaderConfiguration(), + /*changes=*/{}); + + InputDeviceInfo info; + controller.populateDeviceInfo(&info); + std::vector<InputDeviceLightInfo> lights = info.getLights(); + ASSERT_EQ(1U, lights.size()); + ASSERT_EQ(0U, lights[0].preferredBrightnessLevels.size()); +} + TEST_F(LightControllerTest, RGBLight) { RawLightInfo infoRed = {.id = 1, .name = "red", diff --git a/services/inputflinger/tests/InstrumentedInputReader.h b/services/inputflinger/tests/InstrumentedInputReader.h index 7f8d5562ef..fef58ecfe4 100644 --- a/services/inputflinger/tests/InstrumentedInputReader.h +++ b/services/inputflinger/tests/InstrumentedInputReader.h @@ -103,12 +103,16 @@ protected: mExternalStylusDevices = devices; } + void setPreventingTouchpadTaps(bool prevent) override { mPreventingTouchpadTaps = prevent; } + bool isPreventingTouchpadTaps() override { return mPreventingTouchpadTaps; } + private: int32_t mGlobalMetaState; bool mUpdateGlobalMetaStateWasCalled; int32_t mGeneration; std::optional<nsecs_t> mRequestedTimeout; std::vector<InputDeviceInfo> mExternalStylusDevices; + bool mPreventingTouchpadTaps{false}; } mFakeContext; friend class InputReaderTest; diff --git a/services/inputflinger/tests/InterfaceMocks.h b/services/inputflinger/tests/InterfaceMocks.h index d720a902dc..b6720c5712 100644 --- a/services/inputflinger/tests/InterfaceMocks.h +++ b/services/inputflinger/tests/InterfaceMocks.h @@ -49,6 +49,9 @@ public: MOCK_METHOD(void, updateLedMetaState, (int32_t metaState), (override)); MOCK_METHOD(int32_t, getLedMetaState, (), (override)); + + MOCK_METHOD(void, setPreventingTouchpadTaps, (bool prevent), (override)); + MOCK_METHOD(bool, isPreventingTouchpadTaps, (), (override)); }; class MockEventHubInterface : public EventHubInterface { diff --git a/services/inputflinger/tests/KeyboardInputMapper_test.cpp b/services/inputflinger/tests/KeyboardInputMapper_test.cpp new file mode 100644 index 0000000000..08a5559de4 --- /dev/null +++ b/services/inputflinger/tests/KeyboardInputMapper_test.cpp @@ -0,0 +1,152 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "KeyboardInputMapper.h" + +#include <gtest/gtest.h> + +#include "InputMapperTest.h" +#include "InterfaceMocks.h" + +#define TAG "KeyboardInputMapper_test" + +namespace android { + +using testing::_; +using testing::DoAll; +using testing::Return; +using testing::SetArgPointee; + +/** + * Unit tests for KeyboardInputMapper. + */ +class KeyboardInputMapperUnitTest : public InputMapperUnitTest { +protected: + sp<FakeInputReaderPolicy> mFakePolicy; + const std::unordered_map<int32_t, int32_t> mKeyCodeMap{{KEY_0, AKEYCODE_0}, + {KEY_A, AKEYCODE_A}, + {KEY_LEFTCTRL, AKEYCODE_CTRL_LEFT}, + {KEY_LEFTALT, AKEYCODE_ALT_LEFT}, + {KEY_RIGHTALT, AKEYCODE_ALT_RIGHT}, + {KEY_LEFTSHIFT, AKEYCODE_SHIFT_LEFT}, + {KEY_RIGHTSHIFT, AKEYCODE_SHIFT_RIGHT}, + {KEY_FN, AKEYCODE_FUNCTION}, + {KEY_LEFTCTRL, AKEYCODE_CTRL_LEFT}, + {KEY_RIGHTCTRL, AKEYCODE_CTRL_RIGHT}, + {KEY_LEFTMETA, AKEYCODE_META_LEFT}, + {KEY_RIGHTMETA, AKEYCODE_META_RIGHT}, + {KEY_CAPSLOCK, AKEYCODE_CAPS_LOCK}, + {KEY_NUMLOCK, AKEYCODE_NUM_LOCK}, + {KEY_SCROLLLOCK, AKEYCODE_SCROLL_LOCK}}; + + void SetUp() override { + InputMapperUnitTest::SetUp(); + + // set key-codes expected in tests + for (const auto& [scanCode, outKeycode] : mKeyCodeMap) { + EXPECT_CALL(mMockEventHub, mapKey(EVENTHUB_ID, scanCode, _, _, _, _, _)) + .WillRepeatedly(DoAll(SetArgPointee<4>(outKeycode), Return(NO_ERROR))); + } + + mFakePolicy = sp<FakeInputReaderPolicy>::make(); + EXPECT_CALL(mMockInputReaderContext, getPolicy).WillRepeatedly(Return(mFakePolicy.get())); + + mMapper = createInputMapper<KeyboardInputMapper>(*mDeviceContext, mReaderConfiguration, + AINPUT_SOURCE_KEYBOARD, + AINPUT_KEYBOARD_TYPE_ALPHABETIC); + } + + void testPointerVisibilityForKeys(const std::vector<int32_t>& keyCodes, bool expectVisible) { + EXPECT_CALL(mMockInputReaderContext, fadePointer) + .Times(expectVisible ? 0 : keyCodes.size()); + for (int32_t keyCode : keyCodes) { + process(EV_KEY, keyCode, 1); + process(EV_SYN, SYN_REPORT, 0); + process(EV_KEY, keyCode, 0); + process(EV_SYN, SYN_REPORT, 0); + } + } + + void testTouchpadTapStateForKeys(const std::vector<int32_t>& keyCodes, + const bool expectPrevent) { + EXPECT_CALL(mMockInputReaderContext, isPreventingTouchpadTaps).Times(keyCodes.size()); + if (expectPrevent) { + EXPECT_CALL(mMockInputReaderContext, setPreventingTouchpadTaps(true)) + .Times(keyCodes.size()); + } + for (int32_t keyCode : keyCodes) { + process(EV_KEY, keyCode, 1); + process(EV_SYN, SYN_REPORT, 0); + process(EV_KEY, keyCode, 0); + process(EV_SYN, SYN_REPORT, 0); + } + } +}; + +/** + * Pointer visibility should remain unaffected if there is no active Input Method Connection + */ +TEST_F(KeyboardInputMapperUnitTest, KeystrokesWithoutIMeConnectionDoesNotHidePointer) { + testPointerVisibilityForKeys({KEY_0, KEY_A, KEY_LEFTCTRL}, /* expectVisible= */ true); +} + +/** + * Pointer should hide if there is a active Input Method Connection + */ +TEST_F(KeyboardInputMapperUnitTest, AlphanumericKeystrokesWithIMeConnectionHidePointer) { + mFakePolicy->setIsInputMethodConnectionActive(true); + testPointerVisibilityForKeys({KEY_0, KEY_A}, /* expectVisible= */ false); +} + +/** + * Pointer visibility should remain unaffected by meta keys even if Input Method Connection is + * active + */ +TEST_F(KeyboardInputMapperUnitTest, MetaKeystrokesWithIMeConnectionDoesNotHidePointer) { + mFakePolicy->setIsInputMethodConnectionActive(true); + std::vector<int32_t> metaKeys{KEY_LEFTALT, KEY_RIGHTALT, KEY_LEFTSHIFT, KEY_RIGHTSHIFT, + KEY_FN, KEY_LEFTCTRL, KEY_RIGHTCTRL, KEY_LEFTMETA, + KEY_RIGHTMETA, KEY_CAPSLOCK, KEY_NUMLOCK, KEY_SCROLLLOCK}; + testPointerVisibilityForKeys(metaKeys, /* expectVisible= */ true); +} + +/** + * Touchpad tap should not be disabled if there is no active Input Method Connection + */ +TEST_F(KeyboardInputMapperUnitTest, KeystrokesWithoutIMeConnectionDontDisableTouchpadTap) { + testTouchpadTapStateForKeys({KEY_0, KEY_A, KEY_LEFTCTRL}, /* expectPrevent= */ false); +} + +/** + * Touchpad tap should be disabled if there is a active Input Method Connection + */ +TEST_F(KeyboardInputMapperUnitTest, AlphanumericKeystrokesWithIMeConnectionDisableTouchpadTap) { + mFakePolicy->setIsInputMethodConnectionActive(true); + testTouchpadTapStateForKeys({KEY_0, KEY_A}, /* expectPrevent= */ true); +} + +/** + * Touchpad tap should not be disabled by meta keys even if Input Method Connection is active + */ +TEST_F(KeyboardInputMapperUnitTest, MetaKeystrokesWithIMeConnectionDontDisableTouchpadTap) { + mFakePolicy->setIsInputMethodConnectionActive(true); + std::vector<int32_t> metaKeys{KEY_LEFTALT, KEY_RIGHTALT, KEY_LEFTSHIFT, KEY_RIGHTSHIFT, + KEY_FN, KEY_LEFTCTRL, KEY_RIGHTCTRL, KEY_LEFTMETA, + KEY_RIGHTMETA, KEY_CAPSLOCK, KEY_NUMLOCK, KEY_SCROLLLOCK}; + testTouchpadTapStateForKeys(metaKeys, /* expectPrevent= */ false); +} + +} // namespace android diff --git a/services/inputflinger/tests/SyncQueue_test.cpp b/services/inputflinger/tests/SyncQueue_test.cpp new file mode 100644 index 0000000000..b57ccc24be --- /dev/null +++ b/services/inputflinger/tests/SyncQueue_test.cpp @@ -0,0 +1,92 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../SyncQueue.h" + +#include <gtest/gtest.h> +#include <thread> + +namespace android { + +// --- SyncQueueTest --- + +// Validate basic pop and push operation. +TEST(SyncQueueTest, AddAndRemove) { + SyncQueue<int> queue; + + queue.push(1); + ASSERT_EQ(queue.pop(), 1); + + queue.push(3); + ASSERT_EQ(queue.pop(), 3); + + ASSERT_EQ(std::nullopt, queue.pop()); +} + +// Make sure the queue maintains FIFO order. +// Add elements and remove them, and check the order. +TEST(SyncQueueTest, isFIFO) { + SyncQueue<int> queue; + + constexpr int numItems = 10; + for (int i = 0; i < numItems; i++) { + queue.push(static_cast<int>(i)); + } + for (int i = 0; i < numItems; i++) { + ASSERT_EQ(queue.pop(), static_cast<int>(i)); + } +} + +// Make sure the queue has strict capacity limits. +TEST(SyncQueueTest, QueueReachesCapacity) { + constexpr size_t capacity = 3; + SyncQueue<int> queue(capacity); + + // First 3 elements should be added successfully + ASSERT_TRUE(queue.push(1)); + ASSERT_TRUE(queue.push(2)); + ASSERT_TRUE(queue.push(3)); + ASSERT_FALSE(queue.push(4)) << "Queue should reach capacity at size " << capacity; +} + +TEST(SyncQueueTest, AllowsMultipleThreads) { + SyncQueue<int> queue; + + // Test with a large number of items to increase likelihood that threads overlap + constexpr int numItems = 100; + + // Fill queue from a different thread + std::thread fillQueue([&queue]() { + for (int i = 0; i < numItems; i++) { + queue.push(static_cast<int>(i)); + } + }); + + // Make sure all elements are received in correct order + for (int i = 0; i < numItems; i++) { + // Since popping races with the thread that's filling the queue, + // keep popping until we get something back + std::optional<int> popped; + do { + popped = queue.pop(); + } while (!popped); + ASSERT_EQ(popped, static_cast<int>(i)); + } + + fillQueue.join(); +} + +} // namespace android diff --git a/services/inputflinger/tests/TestInputListenerMatchers.h b/services/inputflinger/tests/TestInputListenerMatchers.h index db6f2548e8..70bad7c66b 100644 --- a/services/inputflinger/tests/TestInputListenerMatchers.h +++ b/services/inputflinger/tests/TestInputListenerMatchers.h @@ -23,6 +23,8 @@ #include <gtest/gtest.h> #include <input/Input.h> +#include "TestConstants.h" + namespace android { MATCHER_P(WithMotionAction, action, "MotionEvent with specified action") { @@ -69,8 +71,8 @@ MATCHER_P(WithKeyCode, keyCode, "KeyEvent with specified key code") { } MATCHER_P(WithPointerCount, count, "MotionEvent with specified number of pointers") { - *result_listener << "expected " << count << " pointer(s), but got " << arg.pointerCount; - return arg.pointerCount == count; + *result_listener << "expected " << count << " pointer(s), but got " << arg.getPointerCount(); + return arg.getPointerCount() == count; } MATCHER_P2(WithPointerId, index, id, "MotionEvent with specified pointer ID for pointer index") { @@ -136,6 +138,15 @@ MATCHER_P2(WithGesturePinchScaleFactor, factor, epsilon, return fabs(argScaleFactor - factor) <= epsilon; } +MATCHER_P(WithGestureSwipeFingerCount, count, + "InputEvent with specified touchpad swipe finger count") { + const auto argFingerCount = + arg.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_GESTURE_SWIPE_FINGER_COUNT); + *result_listener << "expected gesture swipe finger count " << count << " but got " + << argFingerCount; + return fabs(argFingerCount - count) <= EPSILON; +} + MATCHER_P(WithPressure, pressure, "InputEvent with specified pressure") { const auto argPressure = arg.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_PRESSURE); *result_listener << "expected pressure " << pressure << ", but got " << argPressure; diff --git a/services/inputflinger/tests/TouchpadInputMapper_test.cpp b/services/inputflinger/tests/TouchpadInputMapper_test.cpp index 92cd462c9a..02abf9f0f0 100644 --- a/services/inputflinger/tests/TouchpadInputMapper_test.cpp +++ b/services/inputflinger/tests/TouchpadInputMapper_test.cpp @@ -139,7 +139,8 @@ TEST_F(TouchpadInputMapperTest, HoverAndLeftButtonPress) { VariantWith<NotifyMotionArgs>(WithMotionAction(ACTION_DOWN)), VariantWith<NotifyMotionArgs>(WithMotionAction(BUTTON_PRESS)), VariantWith<NotifyMotionArgs>(WithMotionAction(BUTTON_RELEASE)), - VariantWith<NotifyMotionArgs>(WithMotionAction(ACTION_UP)))); + VariantWith<NotifyMotionArgs>(WithMotionAction(ACTION_UP)), + VariantWith<NotifyMotionArgs>(WithMotionAction(HOVER_MOVE)))); // Liftoff args.clear(); diff --git a/services/inputflinger/tests/UinputDevice.cpp b/services/inputflinger/tests/UinputDevice.cpp index 5a654c907d..e8aaa18b23 100644 --- a/services/inputflinger/tests/UinputDevice.cpp +++ b/services/inputflinger/tests/UinputDevice.cpp @@ -171,10 +171,11 @@ void UinputKeyboardWithHidUsage::configureDevice(int fd, uinput_user_dev* device // --- UinputTouchScreen --- -UinputTouchScreen::UinputTouchScreen(const Rect& size) +UinputTouchScreen::UinputTouchScreen(const Rect& size, const std::string& physicalPort) : UinputKeyboard(DEVICE_NAME, PRODUCT_ID, {BTN_TOUCH, BTN_TOOL_PEN, BTN_STYLUS, BTN_STYLUS2, BTN_STYLUS3}), - mSize(size) {} + mSize(size), + mPhysicalPort(physicalPort) {} void UinputTouchScreen::configureDevice(int fd, uinput_user_dev* device) { UinputKeyboard::configureDevice(fd, device); @@ -189,6 +190,9 @@ void UinputTouchScreen::configureDevice(int fd, uinput_user_dev* device) { ioctl(fd, UI_SET_ABSBIT, ABS_MT_TRACKING_ID); ioctl(fd, UI_SET_ABSBIT, ABS_MT_TOOL_TYPE); ioctl(fd, UI_SET_PROPBIT, INPUT_PROP_DIRECT); + if (!mPhysicalPort.empty()) { + ioctl(fd, UI_SET_PHYS, mPhysicalPort.c_str()); + } device->absmin[ABS_MT_SLOT] = RAW_SLOT_MIN; device->absmax[ABS_MT_SLOT] = RAW_SLOT_MAX; diff --git a/services/inputflinger/tests/UinputDevice.h b/services/inputflinger/tests/UinputDevice.h index 55996b8bfe..f5507ec385 100644 --- a/services/inputflinger/tests/UinputDevice.h +++ b/services/inputflinger/tests/UinputDevice.h @@ -214,11 +214,12 @@ public: const Point getCenterPoint(); protected: - explicit UinputTouchScreen(const Rect& size); + explicit UinputTouchScreen(const Rect& size, const std::string& physicalPort = ""); private: void configureDevice(int fd, uinput_user_dev* device) override; const Rect mSize; + const std::string mPhysicalPort; }; } // namespace android diff --git a/services/inputflinger/tests/UnwantedInteractionBlocker_test.cpp b/services/inputflinger/tests/UnwantedInteractionBlocker_test.cpp index 1fff2c7590..da0815f088 100644 --- a/services/inputflinger/tests/UnwantedInteractionBlocker_test.cpp +++ b/services/inputflinger/tests/UnwantedInteractionBlocker_test.cpp @@ -138,9 +138,10 @@ TEST(DeviceInfoConversionTest, TabletDeviceTest) { static void assertArgs(const NotifyMotionArgs& args, int32_t action, const std::vector<std::pair<int32_t /*pointerId*/, PointerData>>& pointers) { - ASSERT_EQ(action, args.action); - ASSERT_EQ(pointers.size(), args.pointerCount); - for (size_t i = 0; i < args.pointerCount; i++) { + ASSERT_EQ(action, args.action) + << "Expected " << MotionEvent::actionToString(action) << " but got " << args.action; + ASSERT_EQ(pointers.size(), args.getPointerCount()); + for (size_t i = 0; i < args.getPointerCount(); i++) { const auto& [pointerId, pointerData] = pointers[i]; ASSERT_EQ(pointerId, args.pointerProperties[i].id); ASSERT_EQ(pointerData.x, args.pointerCoords[i].getX()); @@ -196,7 +197,7 @@ TEST(RemovePointerIdsTest, RemoveAllPointersDuringMove) { AMOTION_EVENT_ACTION_MOVE, {{1, 2, 3}, {4, 5, 6}}); NotifyMotionArgs noPointers = removePointerIds(args, {0, 1}); - ASSERT_EQ(0u, noPointers.pointerCount); + ASSERT_EQ(0u, noPointers.getPointerCount()); } /** @@ -771,7 +772,7 @@ TEST_F(PalmRejectorTest, TwoPointersAreCanceled) { ASSERT_EQ(POINTER_0_UP, argsList[0].action); ASSERT_EQ(FLAG_CANCELED, argsList[0].flags); ASSERT_EQ(MOVE, argsList[1].action); - ASSERT_EQ(1u, argsList[1].pointerCount); + ASSERT_EQ(1u, argsList[1].getPointerCount()); ASSERT_EQ(0, argsList[1].flags); mPalmRejector->processMotion( @@ -958,7 +959,7 @@ TEST_F(PalmRejectorFakeFilterTest, OneOfTwoPointersIsCanceled) { {{1433.0, 751.0, 43.0}, {1072.0, 766.0, 13.0}})); ASSERT_EQ(1u, argsList.size()); ASSERT_EQ(MOVE, argsList[0].action); - ASSERT_EQ(1u, argsList[0].pointerCount); + ASSERT_EQ(1u, argsList[0].getPointerCount()); ASSERT_EQ(1433, argsList[0].pointerCoords[0].getX()); ASSERT_EQ(751, argsList[0].pointerCoords[0].getY()); } @@ -986,7 +987,7 @@ TEST_F(PalmRejectorFakeFilterTest, NewDownEventAfterCancel) { ASSERT_EQ(1u, argsList.size()); // Cancel all ASSERT_EQ(CANCEL, argsList[0].action); - ASSERT_EQ(2u, argsList[0].pointerCount); + ASSERT_EQ(2u, argsList[0].getPointerCount()); ASSERT_EQ(FLAG_CANCELED, argsList[0].flags); // Future move events are ignored @@ -1001,7 +1002,7 @@ TEST_F(PalmRejectorFakeFilterTest, NewDownEventAfterCancel) { {{1433.0, 751.0, 43.0}, {1072.0, 766.0, 13.0}, {1000, 700, 10}})); ASSERT_EQ(1u, argsList.size()); ASSERT_EQ(DOWN, argsList[0].action); - ASSERT_EQ(1u, argsList[0].pointerCount); + ASSERT_EQ(1u, argsList[0].getPointerCount()); ASSERT_EQ(2, argsList[0].pointerProperties[0].id); } diff --git a/services/inputflinger/tests/fuzzers/Android.bp b/services/inputflinger/tests/fuzzers/Android.bp index 55c2db6c91..47b0824fd0 100644 --- a/services/inputflinger/tests/fuzzers/Android.bp +++ b/services/inputflinger/tests/fuzzers/Android.bp @@ -21,52 +21,35 @@ package { default_applicable_licenses: ["frameworks_native_license"], } -cc_fuzz { - name: "inputflinger_latencytracker_fuzzer", - defaults: [ - "inputflinger_defaults", - ], - include_dirs: [ - "frameworks/native/services/inputflinger", - ], - shared_libs: [ - "libbase", - "libbinder", - "liblog", - "libutils", - "libinput", - "libinputflinger", - ], - srcs: [ - "LatencyTrackerFuzzer.cpp", - ], - fuzz_config: { - cc: ["android-framework-input@google.com"], - }, -} - cc_defaults { name: "inputflinger_fuzz_defaults", defaults: [ "inputflinger_defaults", + "libinputflinger_defaults", ], + host_supported: true, include_dirs: [ "frameworks/native/services/inputflinger", ], shared_libs: [ - "android.hardware.input.classifier@1.0", - "android.hardware.input.processor-V1-ndk", - "libbase", - "libbinder", - "libcutils", - "liblog", - "libutils", - "libinput", - "libinputflinger", "libinputreader", "libinputflinger_base", - "libstatslog", ], + sanitize: { + hwaddress: true, + undefined: true, + all_undefined: true, + diag: { + undefined: true, + }, + }, + target: { + host: { + sanitize: { + address: true, + }, + }, + }, header_libs: [ "libbatteryservice_headers", "libinputreader_headers", @@ -145,3 +128,17 @@ cc_fuzz { "InputClassifierFuzzer.cpp", ], } + +cc_fuzz { + name: "inputflinger_latencytracker_fuzzer", + defaults: [ + "inputflinger_fuzz_defaults", + "libinputdispatcher_defaults", + ], + shared_libs: [ + "libinputreporter", + ], + srcs: [ + "LatencyTrackerFuzzer.cpp", + ], +} diff --git a/services/inputflinger/tests/fuzzers/BlockingQueueFuzzer.cpp b/services/inputflinger/tests/fuzzers/BlockingQueueFuzzer.cpp index d2595bfc9d..e9016bb0c6 100644 --- a/services/inputflinger/tests/fuzzers/BlockingQueueFuzzer.cpp +++ b/services/inputflinger/tests/fuzzers/BlockingQueueFuzzer.cpp @@ -47,12 +47,21 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *data, size_t size) { filled > numPops ? filled -= numPops : filled = 0; }, [&]() -> void { + // Pops blocks if it is empty, so only pop up to num elements inserted. + size_t numPops = fdp.ConsumeIntegralInRange<size_t>(0, filled); + for (size_t i = 0; i < numPops; i++) { + queue.popWithTimeout( + std::chrono::nanoseconds{fdp.ConsumeIntegral<int64_t>()}); + } + filled > numPops ? filled -= numPops : filled = 0; + }, + [&]() -> void { queue.clear(); filled = 0; }, [&]() -> void { int32_t eraseElement = fdp.ConsumeIntegral<int32_t>(); - queue.erase([&](int32_t element) { + queue.erase_if([&](int32_t element) { if (element == eraseElement) { filled--; return true; diff --git a/services/inputflinger/tests/fuzzers/MapperHelpers.h b/services/inputflinger/tests/fuzzers/MapperHelpers.h index 1e44e0fba0..1ecaa648f7 100644 --- a/services/inputflinger/tests/fuzzers/MapperHelpers.h +++ b/services/inputflinger/tests/fuzzers/MapperHelpers.h @@ -289,6 +289,7 @@ public: } void setTouchAffineTransformation(const TouchAffineTransformation t) { mTransform = t; } void notifyStylusGestureStarted(int32_t, nsecs_t) {} + bool isInputMethodConnectionActive() override { return mFdp->ConsumeBool(); } }; class FuzzInputListener : public virtual InputListenerInterface { @@ -339,6 +340,9 @@ public: void updateLedMetaState(int32_t metaState) override{}; int32_t getLedMetaState() override { return mFdp->ConsumeIntegral<int32_t>(); }; void notifyStylusGestureStarted(int32_t, nsecs_t) {} + + void setPreventingTouchpadTaps(bool prevent) {} + bool isPreventingTouchpadTaps() { return mFdp->ConsumeBool(); }; }; } // namespace android diff --git a/services/sensorservice/AidlSensorHalWrapper.cpp b/services/sensorservice/AidlSensorHalWrapper.cpp index f5b360f3b6..e60db93431 100644 --- a/services/sensorservice/AidlSensorHalWrapper.cpp +++ b/services/sensorservice/AidlSensorHalWrapper.cpp @@ -308,8 +308,12 @@ status_t AidlSensorHalWrapper::configureDirectChannel(int32_t sensorHandle, int3 } int32_t token; - mSensors->configDirectReport(sensorHandle, channelHandle, rate, &token); - return token; + status_t status = convertToStatus( + mSensors->configDirectReport(sensorHandle, channelHandle, rate, &token)); + if (status == OK && rate != ISensors::RateLevel::STOP) { + status = static_cast<status_t>(token); + } + return status; } void AidlSensorHalWrapper::writeWakeLockHandled(uint32_t count) { diff --git a/services/sensorservice/SensorService.h b/services/sensorservice/SensorService.h index 0aa1bcbd72..545f6c25d5 100644 --- a/services/sensorservice/SensorService.h +++ b/services/sensorservice/SensorService.h @@ -61,7 +61,7 @@ // For older HALs which don't support batching, use a smaller socket buffer size. #define SOCKET_BUFFER_SIZE_NON_BATCHED (4 * 1024) -#define SENSOR_REGISTRATIONS_BUF_SIZE 200 +#define SENSOR_REGISTRATIONS_BUF_SIZE 500 // Apps that targets S+ and do not have HIGH_SAMPLING_RATE_SENSORS permission will be capped // at 200 Hz. The cap also applies to all requests when the mic toggle is flipped to on, regardless diff --git a/services/surfaceflinger/Android.bp b/services/surfaceflinger/Android.bp index 5683a9280f..0e6d8b1456 100644 --- a/services/surfaceflinger/Android.bp +++ b/services/surfaceflinger/Android.bp @@ -185,6 +185,7 @@ filegroup { "Scheduler/MessageQueue.cpp", "Scheduler/RefreshRateSelector.cpp", "Scheduler/Scheduler.cpp", + "Scheduler/SmallAreaDetectionAllowMappings.cpp", "Scheduler/VSyncDispatchTimerQueue.cpp", "Scheduler/VSyncPredictor.cpp", "Scheduler/VSyncReactor.cpp", diff --git a/services/surfaceflinger/ClientCache.h b/services/surfaceflinger/ClientCache.h index b56b252d9f..fefc04054d 100644 --- a/services/surfaceflinger/ClientCache.h +++ b/services/surfaceflinger/ClientCache.h @@ -29,7 +29,9 @@ #include <set> #include <unordered_map> -#define BUFFER_CACHE_MAX_SIZE 64 +// 4096 is based on 64 buffers * 64 layers. Once this limit is reached, the least recently used +// buffer is uncached before the new buffer is cached. +#define BUFFER_CACHE_MAX_SIZE 4096 namespace android { diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/CompositionRefreshArgs.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/CompositionRefreshArgs.h index d93e25e4ca..09bc46747a 100644 --- a/services/surfaceflinger/CompositionEngine/include/compositionengine/CompositionRefreshArgs.h +++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/CompositionRefreshArgs.h @@ -37,6 +37,15 @@ struct BorderRenderInfo { half4 color; std::vector<int32_t> layerIds; }; + +// Interface of composition engine power hint callback. +struct ICEPowerCallback { + virtual void notifyCpuLoadUp() = 0; + +protected: + ~ICEPowerCallback() = default; +}; + /** * A parameter object for refreshing a set of outputs */ @@ -96,6 +105,8 @@ struct CompositionRefreshArgs { std::vector<BorderRenderInfo> borderInfoList; bool hasTrustedPresentationListener = false; + + ICEPowerCallback* powerCallback = nullptr; }; } // namespace android::compositionengine diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/OutputCompositionState.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/OutputCompositionState.h index a3fda61ecb..28c6e92b06 100644 --- a/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/OutputCompositionState.h +++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/OutputCompositionState.h @@ -32,6 +32,7 @@ // TODO(b/129481165): remove the #pragma below and fix conversion issues #pragma clang diagnostic pop // ignored "-Wconversion -Wextra" +#include <compositionengine/CompositionRefreshArgs.h> #include <compositionengine/ProjectionSpace.h> #include <renderengine/BorderRenderInfo.h> #include <ui/LayerStack.h> @@ -167,6 +168,8 @@ struct OutputCompositionState { uint64_t lastOutputLayerHash = 0; uint64_t outputLayerHash = 0; + ICEPowerCallback* powerCallback = nullptr; + // Debugging void dump(std::string& result) const; }; diff --git a/services/surfaceflinger/CompositionEngine/src/Output.cpp b/services/surfaceflinger/CompositionEngine/src/Output.cpp index 793959cea6..1205a2ce71 100644 --- a/services/surfaceflinger/CompositionEngine/src/Output.cpp +++ b/services/surfaceflinger/CompositionEngine/src/Output.cpp @@ -843,6 +843,7 @@ void Output::writeCompositionState(const compositionengine::CompositionRefreshAr editState().earliestPresentTime = refreshArgs.earliestPresentTime; editState().expectedPresentTime = refreshArgs.expectedPresentTime; + editState().powerCallback = refreshArgs.powerCallback; compositionengine::OutputLayer* peekThroughLayer = nullptr; sp<GraphicBuffer> previousOverride = nullptr; diff --git a/services/surfaceflinger/CompositionEngine/src/OutputCompositionState.cpp b/services/surfaceflinger/CompositionEngine/src/OutputCompositionState.cpp index c512a1e97f..9713e79fe3 100644 --- a/services/surfaceflinger/CompositionEngine/src/OutputCompositionState.cpp +++ b/services/surfaceflinger/CompositionEngine/src/OutputCompositionState.cpp @@ -67,7 +67,7 @@ void OutputCompositionState::dump(std::string& out) const { out.append("\n "); out.append("\n "); - dumpVal(out, "treate170mAsSrgb", treat170mAsSrgb); + dumpVal(out, "treat170mAsSrgb", treat170mAsSrgb); out.append("\n"); for (const auto& borderRenderInfo : borderInfoList) { diff --git a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp index 0ac0ecb727..fc5f8ca1d3 100644 --- a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp +++ b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include <DisplayHardware/Hal.h> #include <android-base/stringprintf.h> #include <compositionengine/DisplayColorProfile.h> @@ -26,7 +25,7 @@ #include <cstdint> #include "system/graphics-base-v1.0.h" -#include <ui/DataspaceUtils.h> +#include <ui/HdrRenderTypeUtils.h> // TODO(b/129481165): remove the #pragma below and fix conversion issues #pragma clang diagnostic push @@ -331,10 +330,18 @@ void OutputLayer::updateCompositionState( (state.dataspace & HAL_DATASPACE_RANGE_MASK) | HAL_DATASPACE_TRANSFER_SRGB); } + auto pixelFormat = layerFEState->buffer ? std::make_optional(static_cast<ui::PixelFormat>( + layerFEState->buffer->getPixelFormat())) + : std::nullopt; + + // get HdrRenderType after the dataspace gets changed. + auto hdrRenderType = + getHdrRenderType(state.dataspace, pixelFormat, layerFEState->desiredHdrSdrRatio); + // For hdr content, treat the white point as the display brightness - HDR content should not be // boosted or dimmed. // If the layer explicitly requests to disable dimming, then don't dim either. - if (isHdrDataspace(state.dataspace) || + if (hdrRenderType == HdrRenderType::GENERIC_HDR || getOutput().getState().displayBrightnessNits == getOutput().getState().sdrWhitePointNits || getOutput().getState().displayBrightnessNits == 0.f || !layerFEState->dimmingEnabled) { state.dimmingRatio = 1.f; @@ -343,8 +350,7 @@ void OutputLayer::updateCompositionState( float layerBrightnessNits = getOutput().getState().sdrWhitePointNits; // RANGE_EXTENDED can "self-promote" to HDR, but is still rendered for a particular // range that we may need to re-adjust to the current display conditions - if ((state.dataspace & HAL_DATASPACE_RANGE_MASK) == HAL_DATASPACE_RANGE_EXTENDED && - layerFEState->currentHdrSdrRatio > 1.01f) { + if (hdrRenderType == HdrRenderType::DISPLAY_HDR) { layerBrightnessNits *= layerFEState->currentHdrSdrRatio; } state.dimmingRatio = diff --git a/services/surfaceflinger/CompositionEngine/src/planner/CachedSet.cpp b/services/surfaceflinger/CompositionEngine/src/planner/CachedSet.cpp index 8ced0aca36..7547be94e3 100644 --- a/services/surfaceflinger/CompositionEngine/src/planner/CachedSet.cpp +++ b/services/surfaceflinger/CompositionEngine/src/planner/CachedSet.cpp @@ -162,6 +162,9 @@ void CachedSet::render(renderengine::RenderEngine& renderEngine, TexturePool& te const OutputCompositionState& outputState, bool deviceHandlesColorTransform) { ATRACE_CALL(); + if (outputState.powerCallback) { + outputState.powerCallback->notifyCpuLoadUp(); + } const Rect& viewport = outputState.layerStackSpace.getContent(); const ui::Dataspace& outputDataspace = outputState.dataspace; const ui::Transform::RotationFlags orientation = @@ -177,18 +180,19 @@ void CachedSet::render(renderengine::RenderEngine& renderEngine, TexturePool& te .targetLuminanceNits = outputState.displayBrightnessNits, }; - LayerFE::ClientCompositionTargetSettings targetSettings{ - .clip = Region(viewport), - .needsFiltering = false, - .isSecure = outputState.isSecure, - .supportsProtectedContent = false, - .viewport = viewport, - .dataspace = outputDataspace, - .realContentIsVisible = true, - .clearContent = false, - .blurSetting = LayerFE::ClientCompositionTargetSettings::BlurSetting::Enabled, - .whitePointNits = outputState.displayBrightnessNits, - }; + LayerFE::ClientCompositionTargetSettings + targetSettings{.clip = Region(viewport), + .needsFiltering = false, + .isSecure = outputState.isSecure, + .supportsProtectedContent = false, + .viewport = viewport, + .dataspace = outputDataspace, + .realContentIsVisible = true, + .clearContent = false, + .blurSetting = + LayerFE::ClientCompositionTargetSettings::BlurSetting::Enabled, + .whitePointNits = outputState.displayBrightnessNits, + .treat170mAsSrgb = outputState.treat170mAsSrgb}; std::vector<renderengine::LayerSettings> layerSettings; renderengine::LayerSettings highlight; diff --git a/services/surfaceflinger/CompositionEngine/tests/MockPowerAdvisor.h b/services/surfaceflinger/CompositionEngine/tests/MockPowerAdvisor.h index 961ec808e8..f74ef4c60e 100644 --- a/services/surfaceflinger/CompositionEngine/tests/MockPowerAdvisor.h +++ b/services/surfaceflinger/CompositionEngine/tests/MockPowerAdvisor.h @@ -34,6 +34,7 @@ public: MOCK_METHOD(void, setExpensiveRenderingExpected, (DisplayId displayId, bool expected), (override)); MOCK_METHOD(bool, isUsingExpensiveRendering, (), (override)); + MOCK_METHOD(void, notifyCpuLoadUp, (), (override)); MOCK_METHOD(void, notifyDisplayUpdateImminentAndCpuReset, (), (override)); MOCK_METHOD(bool, usePowerHintSession, (), (override)); MOCK_METHOD(bool, supportsPowerHintSession, (), (override)); diff --git a/services/surfaceflinger/Display/PhysicalDisplay.h b/services/surfaceflinger/Display/PhysicalDisplay.h index cba10146b7..ef36234942 100644 --- a/services/surfaceflinger/Display/PhysicalDisplay.h +++ b/services/surfaceflinger/Display/PhysicalDisplay.h @@ -21,9 +21,9 @@ #include <binder/IBinder.h> #include <ui/DisplayId.h> +#include <ui/DisplayMap.h> #include <utils/StrongPointer.h> -#include "DisplayMap.h" #include "DisplaySnapshot.h" namespace android::display { @@ -66,7 +66,7 @@ private: DisplaySnapshot mSnapshot; }; -using PhysicalDisplays = PhysicalDisplayMap<PhysicalDisplayId, PhysicalDisplay>; +using PhysicalDisplays = ui::PhysicalDisplayMap<PhysicalDisplayId, PhysicalDisplay>; // Combinator for ftl::Optional<PhysicalDisplayId>::and_then. constexpr auto getPhysicalDisplay(const PhysicalDisplays& displays) { diff --git a/services/surfaceflinger/DisplayDevice.cpp b/services/surfaceflinger/DisplayDevice.cpp index f6ca9e2856..195d90c947 100644 --- a/services/surfaceflinger/DisplayDevice.cpp +++ b/services/surfaceflinger/DisplayDevice.cpp @@ -39,7 +39,6 @@ #include <system/window.h> #include <ui/GraphicTypes.h> -#include "Display/DisplaySnapshot.h" #include "DisplayDevice.h" #include "FrontEnd/DisplayInfo.h" #include "Layer.h" @@ -214,10 +213,7 @@ void DisplayDevice::setActiveMode(DisplayModeId modeId, Fps displayFps, Fps rend ATRACE_INT(mRenderFrameRateFPSTrace.c_str(), renderFps.getIntValue()); mRefreshRateSelector->setActiveMode(modeId, renderFps); - - if (mRefreshRateOverlay) { - mRefreshRateOverlay->changeRefreshRate(displayFps, renderFps); - } + updateRefreshRateOverlayRate(displayFps, renderFps); } status_t DisplayDevice::initiateModeChange(const ActiveModeInfo& info, @@ -231,10 +227,18 @@ status_t DisplayDevice::initiateModeChange(const ActiveModeInfo& info, return BAD_VALUE; } mUpcomingActiveMode = info; - ATRACE_INT(mActiveModeFPSHwcTrace.c_str(), info.modeOpt->modePtr->getFps().getIntValue()); - return mHwComposer.setActiveModeWithConstraints(getPhysicalId(), - info.modeOpt->modePtr->getHwcId(), constraints, - outTimeline); + mIsModeSetPending = true; + + const auto& pendingMode = *info.modeOpt->modePtr; + ATRACE_INT(mActiveModeFPSHwcTrace.c_str(), pendingMode.getFps().getIntValue()); + + return mHwComposer.setActiveModeWithConstraints(getPhysicalId(), pendingMode.getHwcId(), + constraints, outTimeline); +} + +void DisplayDevice::finalizeModeChange(DisplayModeId modeId, Fps displayFps, Fps renderFps) { + setActiveMode(modeId, displayFps, renderFps); + mIsModeSetPending = false; } nsecs_t DisplayDevice::getVsyncPeriodFromHWC() const { @@ -439,7 +443,7 @@ void DisplayDevice::enableRefreshRateOverlay(bool enable, bool setByHwc, bool sh mRefreshRateOverlay = std::make_unique<RefreshRateOverlay>(fpsRange, features); mRefreshRateOverlay->setLayerStack(getLayerStack()); mRefreshRateOverlay->setViewport(getSize()); - updateRefreshRateOverlayRate(getActiveMode().modePtr->getFps(), getActiveMode().fps); + updateRefreshRateOverlayRate(getActiveMode().modePtr->getFps(), getActiveMode().fps, setByHwc); } void DisplayDevice::updateRefreshRateOverlayRate(Fps displayFps, Fps renderFps, bool setByHwc) { diff --git a/services/surfaceflinger/DisplayDevice.h b/services/surfaceflinger/DisplayDevice.h index dc5f8a85af..6d2fe5430c 100644 --- a/services/surfaceflinger/DisplayDevice.h +++ b/services/surfaceflinger/DisplayDevice.h @@ -217,6 +217,8 @@ public: return mUpcomingActiveMode; } + bool isModeSetPending() const REQUIRES(kMainThreadContext) { return mIsModeSetPending; } + scheduler::FrameRateMode getActiveMode() const REQUIRES(kMainThreadContext) { return mRefreshRateSelector->getActiveMode(); } @@ -228,6 +230,9 @@ public: hal::VsyncPeriodChangeTimeline* outTimeline) REQUIRES(kMainThreadContext); + void finalizeModeChange(DisplayModeId, Fps displayFps, Fps renderFps) + REQUIRES(kMainThreadContext); + scheduler::RefreshRateSelector& refreshRateSelector() const { return *mRefreshRateSelector; } // Extends the lifetime of the RefreshRateSelector, so it can outlive this DisplayDevice. @@ -302,7 +307,9 @@ private: ActiveModeInfo mDesiredActiveMode GUARDED_BY(mActiveModeLock); TracedOrdinal<bool> mDesiredActiveModeChanged GUARDED_BY(mActiveModeLock) = {ftl::Concat("DesiredActiveModeChanged-", getId().value).c_str(), false}; + ActiveModeInfo mUpcomingActiveMode GUARDED_BY(kMainThreadContext); + bool mIsModeSetPending GUARDED_BY(kMainThreadContext) = false; }; struct DisplayDeviceState { diff --git a/services/surfaceflinger/DisplayHardware/AidlComposerHal.h b/services/surfaceflinger/DisplayHardware/AidlComposerHal.h index b8ae26f879..8d21b491c3 100644 --- a/services/surfaceflinger/DisplayHardware/AidlComposerHal.h +++ b/services/surfaceflinger/DisplayHardware/AidlComposerHal.h @@ -17,8 +17,9 @@ #pragma once #include "ComposerHal.h" + #include <ftl/shared_mutex.h> -#include <ftl/small_map.h> +#include <ui/DisplayMap.h> #include <functional> #include <optional> @@ -272,9 +273,9 @@ private: // Invalid displayId used as a key to mReaders when mSingleReader is true. static constexpr int64_t kSingleReaderKey = 0; - // TODO (b/256881188): Use display::PhysicalDisplayMap instead of hard-coded `3` - ftl::SmallMap<Display, ComposerClientWriter, 3> mWriters GUARDED_BY(mMutex); - ftl::SmallMap<Display, ComposerClientReader, 3> mReaders GUARDED_BY(mMutex); + ui::PhysicalDisplayMap<Display, ComposerClientWriter> mWriters GUARDED_BY(mMutex); + ui::PhysicalDisplayMap<Display, ComposerClientReader> mReaders GUARDED_BY(mMutex); + // Protect access to mWriters and mReaders with a shared_mutex. Adding and // removing a display require exclusive access, since the iterator or the // writer/reader may be invalidated. Other calls need shared access while diff --git a/services/surfaceflinger/DisplayHardware/PowerAdvisor.cpp b/services/surfaceflinger/DisplayHardware/PowerAdvisor.cpp index f8b466c93c..9c7576e76d 100644 --- a/services/surfaceflinger/DisplayHardware/PowerAdvisor.cpp +++ b/services/surfaceflinger/DisplayHardware/PowerAdvisor.cpp @@ -138,6 +138,21 @@ void PowerAdvisor::setExpensiveRenderingExpected(DisplayId displayId, bool expec } } +void PowerAdvisor::notifyCpuLoadUp() { + // Only start sending this notification once the system has booted so we don't introduce an + // early-boot dependency on Power HAL + if (!mBootFinished.load()) { + return; + } + if (usePowerHintSession() && ensurePowerHintSessionRunning()) { + std::lock_guard lock(mHintSessionMutex); + auto ret = mHintSession->sendHint(SessionHint::CPU_LOAD_UP); + if (!ret.isOk()) { + mHintSessionRunning = false; + } + } +} + void PowerAdvisor::notifyDisplayUpdateImminentAndCpuReset() { // Only start sending this notification once the system has booted so we don't introduce an // early-boot dependency on Power HAL diff --git a/services/surfaceflinger/DisplayHardware/PowerAdvisor.h b/services/surfaceflinger/DisplayHardware/PowerAdvisor.h index f0d3fd8518..cfaa135299 100644 --- a/services/surfaceflinger/DisplayHardware/PowerAdvisor.h +++ b/services/surfaceflinger/DisplayHardware/PowerAdvisor.h @@ -49,6 +49,7 @@ public: virtual void onBootFinished() = 0; virtual void setExpensiveRenderingExpected(DisplayId displayId, bool expected) = 0; virtual bool isUsingExpensiveRendering() = 0; + virtual void notifyCpuLoadUp() = 0; virtual void notifyDisplayUpdateImminentAndCpuReset() = 0; // Checks both if it supports and if it's enabled virtual bool usePowerHintSession() = 0; @@ -108,6 +109,7 @@ public: void onBootFinished() override; void setExpensiveRenderingExpected(DisplayId displayId, bool expected) override; bool isUsingExpensiveRendering() override { return mNotifiedExpensiveRendering; }; + void notifyCpuLoadUp() override; void notifyDisplayUpdateImminentAndCpuReset() override; bool usePowerHintSession() override; bool supportsPowerHintSession() override; diff --git a/services/surfaceflinger/FrontEnd/DisplayInfo.h b/services/surfaceflinger/FrontEnd/DisplayInfo.h index 218a64a8d6..6502f36f70 100644 --- a/services/surfaceflinger/FrontEnd/DisplayInfo.h +++ b/services/surfaceflinger/FrontEnd/DisplayInfo.h @@ -19,6 +19,9 @@ #include <sstream> #include <gui/DisplayInfo.h> +#include <ui/DisplayMap.h> +#include <ui/LayerStack.h> +#include <ui/Transform.h> namespace android::surfaceflinger::frontend { @@ -44,4 +47,6 @@ struct DisplayInfo { } }; +using DisplayInfos = ui::DisplayMap<ui::LayerStack, DisplayInfo>; + } // namespace android::surfaceflinger::frontend diff --git a/services/surfaceflinger/FrontEnd/LayerHierarchy.cpp b/services/surfaceflinger/FrontEnd/LayerHierarchy.cpp index 5913d4b589..163d34575c 100644 --- a/services/surfaceflinger/FrontEnd/LayerHierarchy.cpp +++ b/services/surfaceflinger/FrontEnd/LayerHierarchy.cpp @@ -16,7 +16,7 @@ #define ATRACE_TAG ATRACE_TAG_GRAPHICS #undef LOG_TAG -#define LOG_TAG "LayerHierarchy" +#define LOG_TAG "SurfaceFlinger" #include "LayerHierarchy.h" #include "LayerLog.h" diff --git a/services/surfaceflinger/FrontEnd/LayerHierarchy.h b/services/surfaceflinger/FrontEnd/LayerHierarchy.h index b25b731356..5389adab6a 100644 --- a/services/surfaceflinger/FrontEnd/LayerHierarchy.h +++ b/services/surfaceflinger/FrontEnd/LayerHierarchy.h @@ -42,10 +42,10 @@ class LayerHierarchyBuilder; class LayerHierarchy { public: enum Variant : uint32_t { - Attached, - Detached, - Relative, - Mirror, + Attached, // child of the parent + Detached, // child of the parent but currently relative parented to another layer + Relative, // relative child of the parent + Mirror, // mirrored from another layer ftl_first = Attached, ftl_last = Mirror, }; diff --git a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp index cd9515cd02..1712137a7e 100644 --- a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp +++ b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp @@ -17,7 +17,7 @@ #define ATRACE_TAG ATRACE_TAG_GRAPHICS #undef LOG_TAG -#define LOG_TAG "LayerLifecycleManager" +#define LOG_TAG "SurfaceFlinger" #include "LayerLifecycleManager.h" #include "Client.h" // temporarily needed for LayerCreationArgs @@ -28,6 +28,14 @@ namespace android::surfaceflinger::frontend { using namespace ftl::flag_operators; +namespace { +// Returns true if the layer is root of a display and can be mirrored by mirroringLayer +bool canMirrorRootLayer(RequestedLayerState& mirroringLayer, RequestedLayerState& rootLayer) { + return rootLayer.isRoot() && rootLayer.layerStack == mirroringLayer.layerStackToMirror && + rootLayer.id != mirroringLayer.id; +} +} // namespace + void LayerLifecycleManager::addLayers(std::vector<std::unique_ptr<RequestedLayerState>> newLayers) { if (newLayers.empty()) { return; @@ -43,14 +51,19 @@ void LayerLifecycleManager::addLayers(std::vector<std::unique_ptr<RequestedLayer it->second.owner.getDebugString().c_str()); } mAddedLayers.push_back(newLayer.get()); + mChangedLayers.push_back(newLayer.get()); layer.parentId = linkLayer(layer.parentId, layer.id); layer.relativeParentId = linkLayer(layer.relativeParentId, layer.id); if (layer.layerStackToMirror != ui::INVALID_LAYER_STACK) { + // Set mirror layer's default layer stack to -1 so it doesn't end up rendered on a + // display accidentally. + layer.layerStack = ui::INVALID_LAYER_STACK; + // if this layer is mirroring a display, then walk though all the existing root layers // for the layer stack and add them as children to be mirrored. mDisplayMirroringLayers.emplace_back(layer.id); for (auto& rootLayer : mLayers) { - if (rootLayer->isRoot() && rootLayer->layerStack == layer.layerStackToMirror) { + if (canMirrorRootLayer(layer, *rootLayer)) { layer.mirrorIds.emplace_back(rootLayer->id); linkLayer(rootLayer->id, layer.id); } @@ -190,6 +203,10 @@ void LayerLifecycleManager::applyTransactions(const std::vector<TransactionState continue; } + if (layer->changes.get() == 0) { + mChangedLayers.push_back(layer); + } + if (transaction.flags & ISurfaceComposer::eAnimation) { layer->changes |= RequestedLayerState::Changes::Animation; } @@ -232,6 +249,7 @@ void LayerLifecycleManager::applyTransactions(const std::vector<TransactionState bgColorLayer->what |= layer_state_t::eColorChanged | layer_state_t::eDataspaceChanged | layer_state_t::eAlphaChanged; bgColorLayer->changes |= RequestedLayerState::Changes::Content; + mChangedLayers.push_back(bgColorLayer); mGlobalChanges |= RequestedLayerState::Changes::Content; } } @@ -278,6 +296,7 @@ void LayerLifecycleManager::commitChanges() { } } mDestroyedLayers.clear(); + mChangedLayers.clear(); mGlobalChanges.clear(); } @@ -298,10 +317,25 @@ const std::vector<std::unique_ptr<RequestedLayerState>>& LayerLifecycleManager:: return mDestroyedLayers; } +const std::vector<RequestedLayerState*>& LayerLifecycleManager::getChangedLayers() const { + return mChangedLayers; +} + const ftl::Flags<RequestedLayerState::Changes> LayerLifecycleManager::getGlobalChanges() const { return mGlobalChanges; } +const RequestedLayerState* LayerLifecycleManager::getLayerFromId(uint32_t id) const { + if (id == UNASSIGNED_LAYER_ID) { + return nullptr; + } + auto it = mIdToLayer.find(id); + if (it == mIdToLayer.end()) { + return nullptr; + } + return &it->second.owner; +} + RequestedLayerState* LayerLifecycleManager::getLayerFromId(uint32_t id) { if (id == UNASSIGNED_LAYER_ID) { return nullptr; @@ -383,10 +417,9 @@ void LayerLifecycleManager::fixRelativeZLoop(uint32_t relativeRootId) { // and updates its list of layers that its mirroring. This function should be called when a new // root layer is added, removed or moved to another display. void LayerLifecycleManager::updateDisplayMirrorLayers(RequestedLayerState& rootLayer) { - for (uint32_t mirrorLayerId : mDisplayMirroringLayers) { - RequestedLayerState* mirrorLayer = getLayerFromId(mirrorLayerId); - bool canBeMirrored = - rootLayer.isRoot() && rootLayer.layerStack == mirrorLayer->layerStackToMirror; + for (uint32_t mirroringLayerId : mDisplayMirroringLayers) { + RequestedLayerState* mirrorLayer = getLayerFromId(mirroringLayerId); + bool canBeMirrored = canMirrorRootLayer(*mirrorLayer, rootLayer); bool currentlyMirrored = std::find(mirrorLayer->mirrorIds.begin(), mirrorLayer->mirrorIds.end(), rootLayer.id) != mirrorLayer->mirrorIds.end(); diff --git a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.h b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.h index f0d2c22e5a..48571bf923 100644 --- a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.h +++ b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.h @@ -76,7 +76,9 @@ public: void removeLifecycleListener(std::shared_ptr<ILifecycleListener>); const std::vector<std::unique_ptr<RequestedLayerState>>& getLayers() const; const std::vector<std::unique_ptr<RequestedLayerState>>& getDestroyedLayers() const; + const std::vector<RequestedLayerState*>& getChangedLayers() const; const ftl::Flags<RequestedLayerState::Changes> getGlobalChanges() const; + const RequestedLayerState* getLayerFromId(uint32_t) const; private: friend class LayerLifecycleManagerTest; @@ -111,6 +113,8 @@ private: // Keeps track of all the layers that were added in order. Changes will be cleared once // committed. std::vector<RequestedLayerState*> mAddedLayers; + // Keeps track of new and layers with states changes since last commit. + std::vector<RequestedLayerState*> mChangedLayers; }; } // namespace android::surfaceflinger::frontend diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshot.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshot.cpp index a9925843df..f0826c6db3 100644 --- a/services/surfaceflinger/FrontEnd/LayerSnapshot.cpp +++ b/services/surfaceflinger/FrontEnd/LayerSnapshot.cpp @@ -16,7 +16,7 @@ #define ATRACE_TAG ATRACE_TAG_GRAPHICS #undef LOG_TAG -#define LOG_TAG "LayerSnapshot" +#define LOG_TAG "SurfaceFlinger" #include "LayerSnapshot.h" @@ -24,6 +24,23 @@ namespace android::surfaceflinger::frontend { using namespace ftl::flag_operators; +namespace { + +void updateSurfaceDamage(const RequestedLayerState& requested, bool hasReadyFrame, + bool forceFullDamage, Region& outSurfaceDamageRegion) { + if (!hasReadyFrame) { + outSurfaceDamageRegion.clear(); + return; + } + if (forceFullDamage) { + outSurfaceDamageRegion = Region::INVALID_REGION; + } else { + outSurfaceDamageRegion = requested.surfaceDamageRegion; + } +} + +} // namespace + LayerSnapshot::LayerSnapshot(const RequestedLayerState& state, const LayerHierarchy::TraversalPath& path) : path(path) { @@ -46,14 +63,16 @@ LayerSnapshot::LayerSnapshot(const RequestedLayerState& state, premultipliedAlpha = state.premultipliedAlpha; inputInfo.name = state.name; inputInfo.id = static_cast<int32_t>(uniqueSequence); - inputInfo.ownerUid = static_cast<int32_t>(state.ownerUid); - inputInfo.ownerPid = state.ownerPid; + inputInfo.ownerUid = gui::Uid{state.ownerUid}; + inputInfo.ownerPid = gui::Pid{state.ownerPid}; uid = state.ownerUid; pid = state.ownerPid; changes = RequestedLayerState::Changes::Created; + clientChanges = 0; mirrorRootPath = path.variant == LayerHierarchy::Variant::Mirror ? path : LayerHierarchy::TraversalPath::ROOT; + reachablilty = LayerSnapshot::Reachablilty::Unreachable; } // As documented in libhardware header, formats in the range @@ -131,6 +150,10 @@ bool LayerSnapshot::isHiddenByPolicy() const { } bool LayerSnapshot::getIsVisible() const { + if (reachablilty != LayerSnapshot::Reachablilty::Reachable) { + return false; + } + if (handleSkipScreenshotFlag & outputFilter.toInternalDisplay) { return false; } @@ -148,12 +171,16 @@ bool LayerSnapshot::getIsVisible() const { std::string LayerSnapshot::getIsVisibleReason() const { // not visible - if (handleSkipScreenshotFlag & outputFilter.toInternalDisplay) return "eLayerSkipScreenshot"; - if (!hasSomethingToDraw()) return "!hasSomethingToDraw"; - if (invalidTransform) return "invalidTransform"; + if (reachablilty == LayerSnapshot::Reachablilty::Unreachable) + return "layer not reachable from root"; + if (reachablilty == LayerSnapshot::Reachablilty::ReachableByRelativeParent) + return "layer only reachable via relative parent"; if (isHiddenByPolicyFromParent) return "hidden by parent or layer flag"; if (isHiddenByPolicyFromRelativeParent) return "hidden by relative parent"; + if (handleSkipScreenshotFlag & outputFilter.toInternalDisplay) return "eLayerSkipScreenshot"; + if (invalidTransform) return "invalidTransform"; if (color.a == 0.0f && !hasBlur()) return "alpha = 0 and no blur"; + if (!hasSomethingToDraw()) return "!hasSomethingToDraw"; // visible std::stringstream reason; @@ -177,8 +204,9 @@ bool LayerSnapshot::isTransformValid(const ui::Transform& t) { } bool LayerSnapshot::hasInputInfo() const { - return inputInfo.token != nullptr || - inputInfo.inputConfig.test(gui::WindowInfo::InputConfig::NO_INPUT_CHANNEL); + return (inputInfo.token != nullptr || + inputInfo.inputConfig.test(gui::WindowInfo::InputConfig::NO_INPUT_CHANNEL)) && + reachablilty == Reachablilty::Reachable; } std::string LayerSnapshot::getDebugString() const { @@ -191,8 +219,16 @@ std::string LayerSnapshot::getDebugString() const { << " geomLayerTransform={tx=" << geomLayerTransform.tx() << ",ty=" << geomLayerTransform.ty() << "}" << "}"; - debug << " input{ touchCropId=" << touchCropId - << " replaceTouchableRegionWithCrop=" << inputInfo.replaceTouchableRegionWithCrop << "}"; + if (hasInputInfo()) { + debug << " input{" + << "(" << inputInfo.inputConfig.string() << ")"; + if (touchCropId != UNASSIGNED_LAYER_ID) debug << " touchCropId=" << touchCropId; + if (inputInfo.replaceTouchableRegionWithCrop) debug << " replaceTouchableRegionWithCrop"; + auto touchableRegion = inputInfo.touchableRegion.getBounds(); + debug << " touchableRegion={" << touchableRegion.left << "," << touchableRegion.top << "," + << touchableRegion.bottom << "," << touchableRegion.right << "}" + << "}"; + } return debug.str(); } @@ -203,4 +239,172 @@ FloatRect LayerSnapshot::sourceBounds() const { return geomBufferSize.toFloatRect(); } +Hwc2::IComposerClient::BlendMode LayerSnapshot::getBlendMode( + const RequestedLayerState& requested) const { + auto blendMode = Hwc2::IComposerClient::BlendMode::NONE; + if (alpha != 1.0f || !contentOpaque) { + blendMode = requested.premultipliedAlpha ? Hwc2::IComposerClient::BlendMode::PREMULTIPLIED + : Hwc2::IComposerClient::BlendMode::COVERAGE; + } + return blendMode; +} + +void LayerSnapshot::merge(const RequestedLayerState& requested, bool forceUpdate, + bool displayChanges, bool forceFullDamage, + uint32_t displayRotationFlags) { + clientChanges = requested.what; + changes = requested.changes; + contentDirty = requested.what & layer_state_t::CONTENT_DIRTY; + // TODO(b/238781169) scope down the changes to only buffer updates. + hasReadyFrame = requested.hasReadyFrame(); + sidebandStreamHasFrame = requested.hasSidebandStreamFrame(); + updateSurfaceDamage(requested, hasReadyFrame, forceFullDamage, surfaceDamage); + + if (forceUpdate || requested.what & layer_state_t::eTransparentRegionChanged) { + transparentRegionHint = requested.transparentRegion; + } + if (forceUpdate || requested.what & layer_state_t::eFlagsChanged) { + layerOpaqueFlagSet = + (requested.flags & layer_state_t::eLayerOpaque) == layer_state_t::eLayerOpaque; + } + if (forceUpdate || requested.what & layer_state_t::eBufferTransformChanged) { + geomBufferTransform = requested.bufferTransform; + } + if (forceUpdate || requested.what & layer_state_t::eTransformToDisplayInverseChanged) { + geomBufferUsesDisplayInverseTransform = requested.transformToDisplayInverse; + } + if (forceUpdate || requested.what & layer_state_t::eDataspaceChanged) { + dataspace = requested.dataspace; + } + if (forceUpdate || requested.what & layer_state_t::eExtendedRangeBrightnessChanged) { + currentHdrSdrRatio = requested.currentHdrSdrRatio; + desiredHdrSdrRatio = requested.desiredHdrSdrRatio; + } + if (forceUpdate || requested.what & layer_state_t::eCachingHintChanged) { + cachingHint = requested.cachingHint; + } + if (forceUpdate || requested.what & layer_state_t::eHdrMetadataChanged) { + hdrMetadata = requested.hdrMetadata; + } + if (forceUpdate || requested.what & layer_state_t::eSidebandStreamChanged) { + sidebandStream = requested.sidebandStream; + } + if (forceUpdate || requested.what & layer_state_t::eShadowRadiusChanged) { + shadowRadius = requested.shadowRadius; + shadowSettings.length = requested.shadowRadius; + } + if (forceUpdate || requested.what & layer_state_t::eFrameRateSelectionPriority) { + frameRateSelectionPriority = requested.frameRateSelectionPriority; + } + if (forceUpdate || requested.what & layer_state_t::eColorSpaceAgnosticChanged) { + isColorspaceAgnostic = requested.colorSpaceAgnostic; + } + if (forceUpdate || requested.what & layer_state_t::eDimmingEnabledChanged) { + dimmingEnabled = requested.dimmingEnabled; + } + if (forceUpdate || requested.what & layer_state_t::eCropChanged) { + geomCrop = requested.crop; + } + + if (forceUpdate || + requested.what & + (layer_state_t::eFlagsChanged | layer_state_t::eBufferChanged | + layer_state_t::eSidebandStreamChanged)) { + compositionType = requested.getCompositionType(); + } + + if (forceUpdate || requested.what & layer_state_t::eInputInfoChanged) { + if (requested.windowInfoHandle) { + inputInfo = *requested.windowInfoHandle->getInfo(); + } else { + inputInfo = {}; + // b/271132344 revisit this and see if we can always use the layers uid/pid + inputInfo.name = requested.name; + inputInfo.ownerUid = requested.ownerUid; + inputInfo.ownerPid = requested.ownerPid; + } + inputInfo.id = static_cast<int32_t>(uniqueSequence); + touchCropId = requested.touchCropId; + } + + if (forceUpdate || + requested.what & + (layer_state_t::eColorChanged | layer_state_t::eBufferChanged | + layer_state_t::eSidebandStreamChanged)) { + color.rgb = requested.getColor().rgb; + } + + if (forceUpdate || requested.what & layer_state_t::eBufferChanged) { + acquireFence = + (requested.externalTexture && + requested.bufferData->flags.test(BufferData::BufferDataChange::fenceChanged)) + ? requested.bufferData->acquireFence + : Fence::NO_FENCE; + buffer = requested.externalTexture ? requested.externalTexture->getBuffer() : nullptr; + externalTexture = requested.externalTexture; + frameNumber = (requested.bufferData) ? requested.bufferData->frameNumber : 0; + hasProtectedContent = requested.externalTexture && + requested.externalTexture->getUsage() & GRALLOC_USAGE_PROTECTED; + geomUsesSourceCrop = hasBufferOrSidebandStream(); + } + + if (forceUpdate || + requested.what & + (layer_state_t::eCropChanged | layer_state_t::eBufferCropChanged | + layer_state_t::eBufferTransformChanged | + layer_state_t::eTransformToDisplayInverseChanged) || + requested.changes.test(RequestedLayerState::Changes::BufferSize) || displayChanges) { + bufferSize = requested.getBufferSize(displayRotationFlags); + geomBufferSize = bufferSize; + croppedBufferSize = requested.getCroppedBufferSize(bufferSize); + geomContentCrop = requested.getBufferCrop(); + } + + if (forceUpdate || + requested.what & + (layer_state_t::eFlagsChanged | layer_state_t::eDestinationFrameChanged | + layer_state_t::ePositionChanged | layer_state_t::eMatrixChanged | + layer_state_t::eBufferTransformChanged | + layer_state_t::eTransformToDisplayInverseChanged) || + requested.changes.test(RequestedLayerState::Changes::BufferSize) || displayChanges) { + localTransform = requested.getTransform(displayRotationFlags); + localTransformInverse = localTransform.inverse(); + } + + if (forceUpdate || requested.what & (layer_state_t::eColorChanged) || + requested.changes.test(RequestedLayerState::Changes::BufferSize)) { + color.rgb = requested.getColor().rgb; + } + + if (forceUpdate || + requested.what & + (layer_state_t::eBufferChanged | layer_state_t::eDataspaceChanged | + layer_state_t::eApiChanged)) { + isHdrY410 = requested.dataspace == ui::Dataspace::BT2020_ITU_PQ && + requested.api == NATIVE_WINDOW_API_MEDIA && + requested.bufferData->getPixelFormat() == HAL_PIXEL_FORMAT_RGBA_1010102; + } + + if (forceUpdate || + requested.what & + (layer_state_t::eBufferChanged | layer_state_t::eDataspaceChanged | + layer_state_t::eApiChanged | layer_state_t::eShadowRadiusChanged | + layer_state_t::eBlurRegionsChanged | layer_state_t::eStretchChanged)) { + forceClientComposition = isHdrY410 || shadowSettings.length > 0 || + requested.blurRegions.size() > 0 || stretchEffect.hasEffect(); + } + + if (forceUpdate || + requested.what & + (layer_state_t::eColorChanged | layer_state_t::eShadowRadiusChanged | + layer_state_t::eBlurRegionsChanged | layer_state_t::eBackgroundBlurRadiusChanged | + layer_state_t::eCornerRadiusChanged | layer_state_t::eAlphaChanged | + layer_state_t::eFlagsChanged | layer_state_t::eBufferChanged | + layer_state_t::eSidebandStreamChanged)) { + contentOpaque = isContentOpaque(); + isOpaque = contentOpaque && !roundedCorner.hasRoundedCorners() && color.a == 1.f; + blendMode = getBlendMode(requested); + } +} + } // namespace android::surfaceflinger::frontend diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshot.h b/services/surfaceflinger/FrontEnd/LayerSnapshot.h index b167d3ea1b..2f45d52162 100644 --- a/services/surfaceflinger/FrontEnd/LayerSnapshot.h +++ b/services/surfaceflinger/FrontEnd/LayerSnapshot.h @@ -18,6 +18,7 @@ #include <compositionengine/LayerFECompositionState.h> #include <renderengine/LayerSettings.h> +#include "DisplayHardware/ComposerHal.h" #include "LayerHierarchy.h" #include "RequestedLayerState.h" #include "Scheduler/LayerInfo.h" @@ -57,6 +58,7 @@ struct LayerSnapshot : public compositionengine::LayerFECompositionState { bool isHiddenByPolicyFromParent = false; bool isHiddenByPolicyFromRelativeParent = false; ftl::Flags<RequestedLayerState::Changes> changes; + uint64_t clientChanges = 0; // Some consumers of this snapshot (input, layer traces) rely on each snapshot to be unique. // For mirrored layers, snapshots will have the same sequence so this unique id provides // an alternative identifier when needed. @@ -93,11 +95,37 @@ struct LayerSnapshot : public compositionengine::LayerFECompositionState { bool handleSkipScreenshotFlag = false; int32_t frameRateSelectionPriority; LayerHierarchy::TraversalPath mirrorRootPath; - bool unreachable = true; uint32_t touchCropId; - uid_t uid; - pid_t pid; + gui::Uid uid = gui::Uid::INVALID; + gui::Pid pid = gui::Pid::INVALID; ChildState childState; + enum class Reachablilty : uint32_t { + // Can traverse the hierarchy from a root node and reach this snapshot + Reachable, + // Cannot traverse the hierarchy from a root node and reach this snapshot + Unreachable, + // Can only reach this node from a relative parent. This means the nodes parents are + // not reachable. + // See example scenario: + // ROOT + // ├── 1 + // │ ├── 11 + // │ │ └── 111 + // │ ├── 12 + // │ │ └ - 111 (relative) + // │ ├── 13 + // │ └── 14 + // │ └ * 12 (mirroring) + // └── 2 + // 111 will create two snapshots, first when visited from 1 -> 12 or 1 -> 11 and the + // second when visited from 1 -> 14 -> 12. Because its parent 11 doesn't exist in the + // mirrored hierarchy, the second snapshot will be marked as ReachableByRelativeParent. + // This snapshot doesn't have any valid properties because it cannot inherit from its + // parent. Therefore, snapshots that are not reachable will be ignored for composition + // and input. + ReachableByRelativeParent + }; + Reachablilty reachablilty; static bool isOpaqueFormat(PixelFormat format); static bool isTransformValid(const ui::Transform& t); @@ -116,6 +144,10 @@ struct LayerSnapshot : public compositionengine::LayerFECompositionState { std::string getIsVisibleReason() const; bool hasInputInfo() const; FloatRect sourceBounds() const; + Hwc2::IComposerClient::BlendMode getBlendMode(const RequestedLayerState& requested) const; + + void merge(const RequestedLayerState& requested, bool forceUpdate, bool displayChanges, + bool forceFullDamage, uint32_t displayRotationFlags); }; } // namespace android::surfaceflinger::frontend diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp index 985c6f9fc9..21f0a672b7 100644 --- a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp +++ b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp @@ -17,30 +17,30 @@ // #define LOG_NDEBUG 0 #define ATRACE_TAG ATRACE_TAG_GRAPHICS #undef LOG_TAG -#define LOG_TAG "LayerSnapshotBuilder" - -#include "LayerSnapshotBuilder.h" -#include <gui/TraceUtils.h> -#include <ui/FloatRect.h> +#define LOG_TAG "SurfaceFlinger" #include <numeric> #include <optional> +#include <ftl/small_map.h> #include <gui/TraceUtils.h> +#include <ui/DisplayMap.h> +#include <ui/FloatRect.h> + #include "DisplayHardware/HWC2.h" #include "DisplayHardware/Hal.h" #include "LayerLog.h" #include "LayerSnapshotBuilder.h" #include "TimeStats/TimeStats.h" -#include "ftl/small_map.h" +#include "Tracing/TransactionTracing.h" namespace android::surfaceflinger::frontend { using namespace ftl::flag_operators; namespace { -FloatRect getMaxDisplayBounds( - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displays) { + +FloatRect getMaxDisplayBounds(const DisplayInfos& displays) { const ui::Size maxSize = [&displays] { if (displays.empty()) return ui::Size{5000, 5000}; @@ -258,19 +258,6 @@ auto getBlendMode(const LayerSnapshot& snapshot, const RequestedLayerState& requ return blendMode; } -void updateSurfaceDamage(const RequestedLayerState& requested, bool hasReadyFrame, - bool forceFullDamage, Region& outSurfaceDamageRegion) { - if (!hasReadyFrame) { - outSurfaceDamageRegion.clear(); - return; - } - if (forceFullDamage) { - outSurfaceDamageRegion = Region::INVALID_REGION; - } else { - outSurfaceDamageRegion = requested.surfaceDamageRegion; - } -} - void updateVisibility(LayerSnapshot& snapshot, bool visible) { snapshot.isVisible = visible; @@ -288,6 +275,8 @@ void updateVisibility(LayerSnapshot& snapshot, bool visible) { const bool visibleForInput = snapshot.hasInputInfo() ? snapshot.canReceiveInput() : snapshot.isVisible; snapshot.inputInfo.setInputConfig(gui::WindowInfo::InputConfig::NOT_VISIBLE, !visibleForInput); + LLOGV(snapshot.sequence, "updating visibility %s %s", visible ? "true" : "false", + snapshot.getDebugString().c_str()); } bool needsInputInfo(const LayerSnapshot& snapshot, const RequestedLayerState& requested) { @@ -330,18 +319,31 @@ void updateMetadata(LayerSnapshot& snapshot, const RequestedLayerState& requeste void clearChanges(LayerSnapshot& snapshot) { snapshot.changes.clear(); + snapshot.clientChanges = 0; snapshot.contentDirty = false; snapshot.hasReadyFrame = false; snapshot.sidebandStreamHasFrame = false; snapshot.surfaceDamage.clear(); } +// TODO (b/259407931): Remove. +uint32_t getPrimaryDisplayRotationFlags( + const ui::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displays) { + for (auto& [_, display] : displays) { + if (display.isPrimary) { + return display.rotationFlags; + } + } + return 0; +} + } // namespace LayerSnapshot LayerSnapshotBuilder::getRootSnapshot() { LayerSnapshot snapshot; snapshot.path = LayerHierarchy::TraversalPath::ROOT; snapshot.changes = ftl::Flags<RequestedLayerState::Changes>(); + snapshot.clientChanges = 0; snapshot.isHiddenByPolicyFromParent = false; snapshot.isHiddenByPolicyFromRelativeParent = false; snapshot.parentTransform.reset(); @@ -375,44 +377,45 @@ LayerSnapshotBuilder::LayerSnapshotBuilder(Args args) : LayerSnapshotBuilder() { } bool LayerSnapshotBuilder::tryFastUpdate(const Args& args) { - if (args.forceUpdate != ForceUpdateFlags::NONE || args.displayChanges) { - // force update requested, or we have display changes, so skip the fast path - return false; - } + const bool forceUpdate = args.forceUpdate != ForceUpdateFlags::NONE; - if (args.layerLifecycleManager.getGlobalChanges().get() == 0) { + if (args.layerLifecycleManager.getGlobalChanges().get() == 0 && !forceUpdate && + !args.displayChanges) { return true; } - if (args.layerLifecycleManager.getGlobalChanges() != RequestedLayerState::Changes::Content) { - // We have changes that require us to walk the hierarchy and update child layers. - // No fast path for you. - return false; - } - // There are only content changes which do not require any child layer snapshots to be updated. ALOGV("%s", __func__); ATRACE_NAME("FastPath"); - // Collect layers with changes - ftl::SmallMap<uint32_t, RequestedLayerState*, 10> layersWithChanges; - for (auto& layer : args.layerLifecycleManager.getLayers()) { - if (layer->changes.test(RequestedLayerState::Changes::Content)) { - layersWithChanges.emplace_or_replace(layer->id, layer.get()); + uint32_t primaryDisplayRotationFlags = getPrimaryDisplayRotationFlags(args.displays); + if (forceUpdate || args.displayChanges) { + for (auto& snapshot : mSnapshots) { + const RequestedLayerState* requested = + args.layerLifecycleManager.getLayerFromId(snapshot->path.id); + if (!requested) continue; + snapshot->merge(*requested, forceUpdate, args.displayChanges, args.forceFullDamage, + primaryDisplayRotationFlags); } + return false; } - // Walk through the snapshots, clearing previous change flags and updating the snapshots - // if needed. - for (auto& snapshot : mSnapshots) { - auto it = layersWithChanges.find(snapshot->path.id); - if (it != layersWithChanges.end()) { - ALOGV("%s fast path snapshot changes = %s", __func__, - mRootSnapshot.changes.string().c_str()); - LayerHierarchy::TraversalPath root = LayerHierarchy::TraversalPath::ROOT; - updateSnapshot(*snapshot, args, *it->second, mRootSnapshot, root); + // Walk through all the updated requested layer states and update the corresponding snapshots. + for (const RequestedLayerState* requested : args.layerLifecycleManager.getChangedLayers()) { + auto range = mIdToSnapshots.equal_range(requested->id); + for (auto it = range.first; it != range.second; it++) { + it->second->merge(*requested, forceUpdate, args.displayChanges, args.forceFullDamage, + primaryDisplayRotationFlags); } } + + if ((args.layerLifecycleManager.getGlobalChanges().get() & + ~(RequestedLayerState::Changes::Content | RequestedLayerState::Changes::Buffer).get()) != + 0) { + // We have changes that require us to walk the hierarchy and update child layers. + // No fast path for you. + return false; + } return true; } @@ -430,20 +433,28 @@ void LayerSnapshotBuilder::updateSnapshots(const Args& args) { if (args.forceUpdate == ForceUpdateFlags::HIERARCHY) { mRootSnapshot.changes |= RequestedLayerState::Changes::Hierarchy | RequestedLayerState::Changes::Visibility; + mRootSnapshot.clientChanges |= layer_state_t::eReparent; } + + for (auto& snapshot : mSnapshots) { + if (snapshot->reachablilty == LayerSnapshot::Reachablilty::Reachable) { + snapshot->reachablilty = LayerSnapshot::Reachablilty::Unreachable; + } + } + LayerHierarchy::TraversalPath root = LayerHierarchy::TraversalPath::ROOT; if (args.root.getLayer()) { // The hierarchy can have a root layer when used for screenshots otherwise, it will have // multiple children. LayerHierarchy::ScopedAddToTraversalPath addChildToPath(root, args.root.getLayer()->id, LayerHierarchy::Variant::Attached); - updateSnapshotsInHierarchy(args, args.root, root, mRootSnapshot); + updateSnapshotsInHierarchy(args, args.root, root, mRootSnapshot, /*depth=*/0); } else { for (auto& [childHierarchy, variant] : args.root.mChildren) { LayerHierarchy::ScopedAddToTraversalPath addChildToPath(root, childHierarchy->getLayer()->id, variant); - updateSnapshotsInHierarchy(args, *childHierarchy, root, mRootSnapshot); + updateSnapshotsInHierarchy(args, *childHierarchy, root, mRootSnapshot, /*depth=*/0); } } @@ -469,13 +480,26 @@ void LayerSnapshotBuilder::updateSnapshots(const Args& args) { auto it = mSnapshots.begin(); while (it < mSnapshots.end()) { auto& traversalPath = it->get()->path; - if (!it->get()->unreachable && - destroyedLayerIds.find(traversalPath.id) == destroyedLayerIds.end()) { + const bool unreachable = + it->get()->reachablilty == LayerSnapshot::Reachablilty::Unreachable; + const bool isClone = traversalPath.isClone(); + const bool layerIsDestroyed = + destroyedLayerIds.find(traversalPath.id) != destroyedLayerIds.end(); + const bool destroySnapshot = (unreachable && isClone) || layerIsDestroyed; + + if (!destroySnapshot) { it++; continue; } - mIdToSnapshot.erase(traversalPath); + mPathToSnapshot.erase(traversalPath); + + auto range = mIdToSnapshots.equal_range(traversalPath.id); + auto matchingSnapshot = + std::find_if(range.first, range.second, [&traversalPath](auto& snapshotWithId) { + return snapshotWithId.second->path == traversalPath; + }); + mIdToSnapshots.erase(matchingSnapshot); mNeedsTouchableRegionCrop.erase(traversalPath); mSnapshots.back()->globalZ = it->get()->globalZ; std::iter_swap(it, mSnapshots.end() - 1); @@ -496,12 +520,24 @@ void LayerSnapshotBuilder::update(const Args& args) { const LayerSnapshot& LayerSnapshotBuilder::updateSnapshotsInHierarchy( const Args& args, const LayerHierarchy& hierarchy, - LayerHierarchy::TraversalPath& traversalPath, const LayerSnapshot& parentSnapshot) { + LayerHierarchy::TraversalPath& traversalPath, const LayerSnapshot& parentSnapshot, + int depth) { + if (depth > 50) { + TransactionTraceWriter::getInstance().invoke("layer_builder_stack_overflow_", + /*overwrite=*/false); + LOG_ALWAYS_FATAL("Cycle detected in LayerSnapshotBuilder. See " + "builder_stack_overflow_transactions.winscope"); + } + const RequestedLayerState* layer = hierarchy.getLayer(); LayerSnapshot* snapshot = getSnapshot(traversalPath); const bool newSnapshot = snapshot == nullptr; + uint32_t primaryDisplayRotationFlags = getPrimaryDisplayRotationFlags(args.displays); if (newSnapshot) { snapshot = createSnapshot(traversalPath, *layer, parentSnapshot); + snapshot->merge(*layer, /*forceUpdate=*/true, /*displayChanges=*/true, args.forceFullDamage, + primaryDisplayRotationFlags); + snapshot->changes |= RequestedLayerState::Changes::Created; } scheduler::LayerInfo::FrameRate oldFrameRate = snapshot->frameRate; if (traversalPath.isRelative()) { @@ -519,7 +555,8 @@ const LayerSnapshot& LayerSnapshotBuilder::updateSnapshotsInHierarchy( childHierarchy->getLayer()->id, variant); const LayerSnapshot& childSnapshot = - updateSnapshotsInHierarchy(args, *childHierarchy, traversalPath, *snapshot); + updateSnapshotsInHierarchy(args, *childHierarchy, traversalPath, *snapshot, + depth + 1); updateChildState(*snapshot, childSnapshot, args); } @@ -538,8 +575,8 @@ LayerSnapshot* LayerSnapshotBuilder::getSnapshot(uint32_t layerId) const { } LayerSnapshot* LayerSnapshotBuilder::getSnapshot(const LayerHierarchy::TraversalPath& id) const { - auto it = mIdToSnapshot.find(id); - return it == mIdToSnapshot.end() ? nullptr : it->second; + auto it = mPathToSnapshot.find(id); + return it == mPathToSnapshot.end() ? nullptr : it->second; } LayerSnapshot* LayerSnapshotBuilder::createSnapshot(const LayerHierarchy::TraversalPath& path, @@ -551,7 +588,9 @@ LayerSnapshot* LayerSnapshotBuilder::createSnapshot(const LayerHierarchy::Traver if (path.isClone() && path.variant != LayerHierarchy::Variant::Mirror) { snapshot->mirrorRootPath = parentSnapshot.mirrorRootPath; } - mIdToSnapshot[path] = snapshot; + mPathToSnapshot[path] = snapshot; + + mIdToSnapshots.emplace(path.id, snapshot); return snapshot; } @@ -566,20 +605,15 @@ bool LayerSnapshotBuilder::sortSnapshotsByZ(const Args& args) { } mResortSnapshots = false; - for (auto& snapshot : mSnapshots) { - snapshot->unreachable = snapshot->path.isClone(); - } - size_t globalZ = 0; args.root.traverseInZOrder( [this, &globalZ](const LayerHierarchy&, const LayerHierarchy::TraversalPath& traversalPath) -> bool { LayerSnapshot* snapshot = getSnapshot(traversalPath); if (!snapshot) { - return false; + return true; } - snapshot->unreachable = false; if (snapshot->getIsVisible() || snapshot->hasInputInfo()) { updateVisibility(*snapshot, snapshot->getIsVisible()); size_t oldZ = snapshot->globalZ; @@ -602,7 +636,7 @@ bool LayerSnapshotBuilder::sortSnapshotsByZ(const Args& args) { mSnapshots[globalZ]->globalZ = globalZ; /* mark unreachable snapshots as explicitly invisible */ updateVisibility(*mSnapshots[globalZ], false); - if (mSnapshots[globalZ]->unreachable) { + if (mSnapshots[globalZ]->reachablilty == LayerSnapshot::Reachablilty::Unreachable) { hasUnreachableSnapshots = true; } globalZ++; @@ -626,7 +660,9 @@ void LayerSnapshotBuilder::updateRelativeState(LayerSnapshot& snapshot, snapshot.relativeLayerMetadata = parentSnapshot.relativeLayerMetadata; } } - snapshot.isVisible = snapshot.getIsVisible(); + if (snapshot.reachablilty == LayerSnapshot::Reachablilty::Unreachable) { + snapshot.reachablilty = LayerSnapshot::Reachablilty::ReachableByRelativeParent; + } } void LayerSnapshotBuilder::updateChildState(LayerSnapshot& snapshot, @@ -667,17 +703,6 @@ void LayerSnapshotBuilder::resetRelativeState(LayerSnapshot& snapshot) { snapshot.relativeLayerMetadata.mMap.clear(); } -// TODO (b/259407931): Remove. -uint32_t getPrimaryDisplayRotationFlags( - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displays) { - for (auto& [_, display] : displays) { - if (display.isPrimary) { - return display.rotationFlags; - } - } - return 0; -} - void LayerSnapshotBuilder::updateSnapshot(LayerSnapshot& snapshot, const Args& args, const RequestedLayerState& requested, const LayerSnapshot& parentSnapshot, @@ -687,82 +712,69 @@ void LayerSnapshotBuilder::updateSnapshot(LayerSnapshot& snapshot, const Args& a (RequestedLayerState::Changes::Hierarchy | RequestedLayerState::Changes::Geometry | RequestedLayerState::Changes::Visibility | RequestedLayerState::Changes::Metadata | RequestedLayerState::Changes::AffectsChildren | - RequestedLayerState::Changes::FrameRate); - snapshot.changes |= parentChanges | requested.changes; + RequestedLayerState::Changes::FrameRate | RequestedLayerState::Changes::GameMode); + snapshot.changes |= parentChanges; + if (args.displayChanges) snapshot.changes |= RequestedLayerState::Changes::Geometry; + snapshot.reachablilty = LayerSnapshot::Reachablilty::Reachable; + snapshot.clientChanges |= (parentSnapshot.clientChanges & layer_state_t::AFFECTS_CHILDREN); snapshot.isHiddenByPolicyFromParent = parentSnapshot.isHiddenByPolicyFromParent || parentSnapshot.invalidTransform || requested.isHiddenByPolicy() || (args.excludeLayerIds.find(path.id) != args.excludeLayerIds.end()); - snapshot.contentDirty = requested.what & layer_state_t::CONTENT_DIRTY; - // TODO(b/238781169) scope down the changes to only buffer updates. - snapshot.hasReadyFrame = requested.hasReadyFrame(); - snapshot.sidebandStreamHasFrame = requested.hasSidebandStreamFrame(); - updateSurfaceDamage(requested, snapshot.hasReadyFrame, args.forceFullDamage, - snapshot.surfaceDamage); - snapshot.outputFilter.layerStack = parentSnapshot.path == LayerHierarchy::TraversalPath::ROOT - ? requested.layerStack - : parentSnapshot.outputFilter.layerStack; - uint32_t primaryDisplayRotationFlags = getPrimaryDisplayRotationFlags(args.displays); const bool forceUpdate = args.forceUpdate == ForceUpdateFlags::ALL || + snapshot.clientChanges & layer_state_t::eReparent || snapshot.changes.any(RequestedLayerState::Changes::Visibility | RequestedLayerState::Changes::Created); - // always update the buffer regardless of visibility - if (forceUpdate || requested.what & layer_state_t::BUFFER_CHANGES || args.displayChanges) { - snapshot.acquireFence = - (requested.externalTexture && - requested.bufferData->flags.test(BufferData::BufferDataChange::fenceChanged)) - ? requested.bufferData->acquireFence - : Fence::NO_FENCE; - snapshot.buffer = - requested.externalTexture ? requested.externalTexture->getBuffer() : nullptr; - snapshot.bufferSize = requested.getBufferSize(primaryDisplayRotationFlags); - snapshot.geomBufferSize = snapshot.bufferSize; - snapshot.croppedBufferSize = requested.getCroppedBufferSize(snapshot.bufferSize); - snapshot.dataspace = requested.dataspace; - snapshot.externalTexture = requested.externalTexture; - snapshot.frameNumber = (requested.bufferData) ? requested.bufferData->frameNumber : 0; - snapshot.geomBufferTransform = requested.bufferTransform; - snapshot.geomBufferUsesDisplayInverseTransform = requested.transformToDisplayInverse; - snapshot.geomContentCrop = requested.getBufferCrop(); - snapshot.geomUsesSourceCrop = snapshot.hasBufferOrSidebandStream(); - snapshot.hasProtectedContent = requested.externalTexture && - requested.externalTexture->getUsage() & GRALLOC_USAGE_PROTECTED; - snapshot.isHdrY410 = requested.dataspace == ui::Dataspace::BT2020_ITU_PQ && - requested.api == NATIVE_WINDOW_API_MEDIA && - requested.bufferData->getPixelFormat() == HAL_PIXEL_FORMAT_RGBA_1010102; - snapshot.sidebandStream = requested.sidebandStream; - snapshot.transparentRegionHint = requested.transparentRegion; - snapshot.color.rgb = requested.getColor().rgb; - snapshot.currentHdrSdrRatio = requested.currentHdrSdrRatio; - snapshot.desiredHdrSdrRatio = requested.desiredHdrSdrRatio; + if (forceUpdate || snapshot.clientChanges & layer_state_t::eLayerStackChanged) { + // If root layer, use the layer stack otherwise get the parent's layer stack. + snapshot.outputFilter.layerStack = + parentSnapshot.path == LayerHierarchy::TraversalPath::ROOT + ? requested.layerStack + : parentSnapshot.outputFilter.layerStack; } if (snapshot.isHiddenByPolicyFromParent && !snapshot.changes.test(RequestedLayerState::Changes::Created)) { if (forceUpdate || - snapshot.changes.any(RequestedLayerState::Changes::Hierarchy | - RequestedLayerState::Changes::Geometry | + snapshot.changes.any(RequestedLayerState::Changes::Geometry | RequestedLayerState::Changes::Input)) { updateInput(snapshot, requested, parentSnapshot, path, args); } return; } - if (forceUpdate || snapshot.changes.any(RequestedLayerState::Changes::AffectsChildren)) { - // If root layer, use the layer stack otherwise get the parent's layer stack. + if (forceUpdate || snapshot.changes.any(RequestedLayerState::Changes::Mirror)) { + // Display mirrors are always placed in a VirtualDisplay so we never want to capture layers + // marked as skip capture + snapshot.handleSkipScreenshotFlag = parentSnapshot.handleSkipScreenshotFlag || + (requested.layerStackToMirror != ui::INVALID_LAYER_STACK); + } + + if (forceUpdate || snapshot.clientChanges & layer_state_t::eAlphaChanged) { snapshot.color.a = parentSnapshot.color.a * requested.color.a; snapshot.alpha = snapshot.color.a; snapshot.inputInfo.alpha = snapshot.color.a; + } + if (forceUpdate || snapshot.clientChanges & layer_state_t::eFlagsChanged) { snapshot.isSecure = parentSnapshot.isSecure || (requested.flags & layer_state_t::eLayerSecure); - snapshot.isTrustedOverlay = parentSnapshot.isTrustedOverlay || requested.isTrustedOverlay; snapshot.outputFilter.toInternalDisplay = parentSnapshot.outputFilter.toInternalDisplay || (requested.flags & layer_state_t::eLayerSkipScreenshot); + } + + if (forceUpdate || snapshot.clientChanges & layer_state_t::eTrustedOverlayChanged) { + snapshot.isTrustedOverlay = parentSnapshot.isTrustedOverlay || requested.isTrustedOverlay; + } + + if (forceUpdate || snapshot.clientChanges & layer_state_t::eStretchChanged) { snapshot.stretchEffect = (requested.stretchEffect.hasEffect()) ? requested.stretchEffect : parentSnapshot.stretchEffect; + } + + if (forceUpdate || snapshot.clientChanges & layer_state_t::eColorTransformChanged) { if (!parentSnapshot.colorTransformIsIdentity) { snapshot.colorTransform = parentSnapshot.colorTransform * requested.colorTransform; snapshot.colorTransformIsIdentity = false; @@ -770,16 +782,20 @@ void LayerSnapshotBuilder::updateSnapshot(LayerSnapshot& snapshot, const Args& a snapshot.colorTransform = requested.colorTransform; snapshot.colorTransformIsIdentity = !requested.hasColorTransform; } + } + + if (forceUpdate || snapshot.changes.test(RequestedLayerState::Changes::GameMode)) { snapshot.gameMode = requested.metadata.has(gui::METADATA_GAME_MODE) ? requested.gameMode : parentSnapshot.gameMode; - // Display mirrors are always placed in a VirtualDisplay so we never want to capture layers - // marked as skip capture - snapshot.handleSkipScreenshotFlag = parentSnapshot.handleSkipScreenshotFlag || - (requested.layerStackToMirror != ui::INVALID_LAYER_STACK); + updateMetadata(snapshot, requested, args); + if (args.includeMetadata) { + snapshot.layerMetadata = parentSnapshot.layerMetadata; + snapshot.layerMetadata.merge(requested.metadata); + } } - if (forceUpdate || snapshot.changes.any(RequestedLayerState::Changes::AffectsChildren) || + if (forceUpdate || snapshot.clientChanges & layer_state_t::eFixedTransformHintChanged || args.displayChanges) { snapshot.fixedTransformHint = requested.fixedTransformHint != ui::Transform::ROT_INVALID ? requested.fixedTransformHint @@ -795,9 +811,7 @@ void LayerSnapshotBuilder::updateSnapshot(LayerSnapshot& snapshot, const Args& a } } - if (forceUpdate || - snapshot.changes.any(RequestedLayerState::Changes::FrameRate | - RequestedLayerState::Changes::Hierarchy)) { + if (forceUpdate || snapshot.changes.any(RequestedLayerState::Changes::FrameRate)) { snapshot.frameRate = (requested.requestedFrameRate.rate.isValid() || (requested.requestedFrameRate.type == scheduler::LayerInfo::FrameRateCompatibility::NoVote)) @@ -805,23 +819,10 @@ void LayerSnapshotBuilder::updateSnapshot(LayerSnapshot& snapshot, const Args& a : parentSnapshot.frameRate; } - if (forceUpdate || requested.what & layer_state_t::eMetadataChanged) { - updateMetadata(snapshot, requested, args); - } - - if (forceUpdate || requested.changes.get() != 0) { - snapshot.compositionType = requested.getCompositionType(); - snapshot.dimmingEnabled = requested.dimmingEnabled; - snapshot.layerOpaqueFlagSet = - (requested.flags & layer_state_t::eLayerOpaque) == layer_state_t::eLayerOpaque; - snapshot.cachingHint = requested.cachingHint; - snapshot.frameRateSelectionPriority = requested.frameRateSelectionPriority; - } - - if (forceUpdate || snapshot.changes.any(RequestedLayerState::Changes::Content) || - snapshot.changes.any(RequestedLayerState::Changes::AffectsChildren)) { - snapshot.color.rgb = requested.getColor().rgb; - snapshot.isColorspaceAgnostic = requested.colorSpaceAgnostic; + if (forceUpdate || + snapshot.clientChanges & + (layer_state_t::eBackgroundBlurRadiusChanged | layer_state_t::eBlurRegionsChanged | + layer_state_t::eAlphaChanged)) { snapshot.backgroundBlurRadius = args.supportsBlur ? static_cast<int>(parentSnapshot.color.a * (float)requested.backgroundBlurRadius) : 0; @@ -829,29 +830,30 @@ void LayerSnapshotBuilder::updateSnapshot(LayerSnapshot& snapshot, const Args& a for (auto& region : snapshot.blurRegions) { region.alpha = region.alpha * snapshot.color.a; } - snapshot.hdrMetadata = requested.hdrMetadata; } - if (forceUpdate || - snapshot.changes.any(RequestedLayerState::Changes::Hierarchy | - RequestedLayerState::Changes::Geometry)) { + if (forceUpdate || snapshot.changes.any(RequestedLayerState::Changes::Geometry)) { + uint32_t primaryDisplayRotationFlags = getPrimaryDisplayRotationFlags(args.displays); updateLayerBounds(snapshot, requested, parentSnapshot, primaryDisplayRotationFlags); + } + + if (forceUpdate || snapshot.clientChanges & layer_state_t::eCornerRadiusChanged || + snapshot.changes.any(RequestedLayerState::Changes::Geometry)) { updateRoundedCorner(snapshot, requested, parentSnapshot); } + if (forceUpdate || snapshot.clientChanges & layer_state_t::eShadowRadiusChanged || + snapshot.changes.any(RequestedLayerState::Changes::Geometry)) { + updateShadows(snapshot, requested, args.globalShadowSettings); + } + if (forceUpdate || - snapshot.changes.any(RequestedLayerState::Changes::Hierarchy | - RequestedLayerState::Changes::Geometry | + snapshot.changes.any(RequestedLayerState::Changes::Geometry | RequestedLayerState::Changes::Input)) { updateInput(snapshot, requested, parentSnapshot, path, args); } // computed snapshot properties - updateShadows(snapshot, requested, args.globalShadowSettings); - if (args.includeMetadata) { - snapshot.layerMetadata = parentSnapshot.layerMetadata; - snapshot.layerMetadata.merge(requested.metadata); - } snapshot.forceClientComposition = snapshot.isHdrY410 || snapshot.shadowSettings.length > 0 || requested.blurRegions.size() > 0 || snapshot.stretchEffect.hasEffect(); snapshot.contentOpaque = snapshot.isContentOpaque(); @@ -907,10 +909,6 @@ void LayerSnapshotBuilder::updateLayerBounds(LayerSnapshot& snapshot, const RequestedLayerState& requested, const LayerSnapshot& parentSnapshot, uint32_t primaryDisplayRotationFlags) { - snapshot.croppedBufferSize = requested.getCroppedBufferSize(snapshot.bufferSize); - snapshot.geomCrop = requested.crop; - snapshot.localTransform = requested.getTransform(primaryDisplayRotationFlags); - snapshot.localTransformInverse = snapshot.localTransform.inverse(); snapshot.geomLayerTransform = parentSnapshot.geomLayerTransform * snapshot.localTransform; const bool transformWasInvalid = snapshot.invalidTransform; snapshot.invalidTransform = !LayerSnapshot::isTransformValid(snapshot.geomLayerTransform); @@ -967,11 +965,8 @@ void LayerSnapshotBuilder::updateLayerBounds(LayerSnapshot& snapshot, } } -void LayerSnapshotBuilder::updateShadows(LayerSnapshot& snapshot, - const RequestedLayerState& requested, +void LayerSnapshotBuilder::updateShadows(LayerSnapshot& snapshot, const RequestedLayerState&, const renderengine::ShadowSettings& globalShadowSettings) { - snapshot.shadowRadius = requested.shadowRadius; - snapshot.shadowSettings.length = requested.shadowRadius; if (snapshot.shadowRadius > 0.f) { snapshot.shadowSettings = globalShadowSettings; @@ -1000,8 +995,8 @@ void LayerSnapshotBuilder::updateInput(LayerSnapshot& snapshot, snapshot.inputInfo = {}; // b/271132344 revisit this and see if we can always use the layers uid/pid snapshot.inputInfo.name = requested.name; - snapshot.inputInfo.ownerUid = static_cast<int32_t>(requested.ownerUid); - snapshot.inputInfo.ownerPid = requested.ownerPid; + snapshot.inputInfo.ownerUid = gui::Uid{requested.ownerUid}; + snapshot.inputInfo.ownerPid = gui::Pid{requested.ownerPid}; } snapshot.touchCropId = requested.touchCropId; @@ -1051,10 +1046,11 @@ void LayerSnapshotBuilder::updateInput(LayerSnapshot& snapshot, snapshot.inputInfo.inputConfig |= gui::WindowInfo::InputConfig::DROP_INPUT; } - auto cropLayerSnapshot = getSnapshot(requested.touchCropId); - if (cropLayerSnapshot) { + if (requested.touchCropId != UNASSIGNED_LAYER_ID || path.isClone()) { mNeedsTouchableRegionCrop.insert(path); - } else if (snapshot.inputInfo.replaceTouchableRegionWithCrop) { + } + auto cropLayerSnapshot = getSnapshot(requested.touchCropId); + if (!cropLayerSnapshot && snapshot.inputInfo.replaceTouchableRegionWithCrop) { FloatRect inputBounds = getInputBounds(snapshot, /*fillParentBounds=*/true).first; Rect inputBoundsInDisplaySpace = getInputBoundsInDisplaySpace(snapshot, inputBounds, displayInfo.transform); @@ -1074,8 +1070,6 @@ void LayerSnapshotBuilder::updateInput(LayerSnapshot& snapshot, // Cloned layers shouldn't handle watch outside since their z order is not determined by // WM or the client. snapshot.inputInfo.inputConfig.clear(gui::WindowInfo::InputConfig::WATCH_OUTSIDE_TOUCH); - - mNeedsTouchableRegionCrop.insert(path); } } @@ -1132,7 +1126,7 @@ void LayerSnapshotBuilder::updateTouchableRegionCrop(const Args& args) { RequestedLayerState::Changes::Input; if (args.forceUpdate != ForceUpdateFlags::ALL && - !args.layerLifecycleManager.getGlobalChanges().any(AFFECTS_INPUT)) { + !args.layerLifecycleManager.getGlobalChanges().any(AFFECTS_INPUT) && !args.displayChanges) { return; } @@ -1141,6 +1135,8 @@ void LayerSnapshotBuilder::updateTouchableRegionCrop(const Args& args) { if (!snapshot) { continue; } + LLOGV(snapshot->sequence, "updateTouchableRegionCrop=%s", + snapshot->getDebugString().c_str()); const std::optional<frontend::DisplayInfo> displayInfoOpt = args.displays.get(snapshot->outputFilter.layerStack); static frontend::DisplayInfo sDefaultInfo = {.isSecure = false}; diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.h b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.h index 148c98e2b1..c81a5d2b9e 100644 --- a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.h +++ b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.h @@ -16,7 +16,6 @@ #pragma once -#include "Display/DisplayMap.h" #include "FrontEnd/DisplayInfo.h" #include "FrontEnd/LayerLifecycleManager.h" #include "LayerHierarchy.h" @@ -45,7 +44,7 @@ public: const LayerLifecycleManager& layerLifecycleManager; ForceUpdateFlags forceUpdate = ForceUpdateFlags::NONE; bool includeMetadata = false; - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displays; + const DisplayInfos& displays; // Set to true if there were display changes since last update. bool displayChanges = false; const renderengine::ShadowSettings& globalShadowSettings; @@ -97,7 +96,7 @@ private: const LayerSnapshot& updateSnapshotsInHierarchy(const Args&, const LayerHierarchy& hierarchy, LayerHierarchy::TraversalPath& traversalPath, - const LayerSnapshot& parentSnapshot); + const LayerSnapshot& parentSnapshot, int depth); void updateSnapshot(LayerSnapshot&, const Args&, const RequestedLayerState&, const LayerSnapshot& parentSnapshot, const LayerHierarchy::TraversalPath&); static void updateRelativeState(LayerSnapshot& snapshot, const LayerSnapshot& parentSnapshot, @@ -123,7 +122,9 @@ private: std::unordered_map<LayerHierarchy::TraversalPath, LayerSnapshot*, LayerHierarchy::TraversalPathHash> - mIdToSnapshot; + mPathToSnapshot; + std::multimap<uint32_t, LayerSnapshot*> mIdToSnapshots; + // Track snapshots that needs touchable region crop from other snapshots std::unordered_set<LayerHierarchy::TraversalPath, LayerHierarchy::TraversalPathHash> mNeedsTouchableRegionCrop; diff --git a/services/surfaceflinger/FrontEnd/RequestedLayerState.cpp b/services/surfaceflinger/FrontEnd/RequestedLayerState.cpp index 3d9979f329..9a1ba2dd29 100644 --- a/services/surfaceflinger/FrontEnd/RequestedLayerState.cpp +++ b/services/surfaceflinger/FrontEnd/RequestedLayerState.cpp @@ -13,10 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +// #define LOG_NDEBUG 0 #define ATRACE_TAG ATRACE_TAG_GRAPHICS #undef LOG_TAG -#define LOG_TAG "RequestedLayerState" +#define LOG_TAG "SurfaceFlinger" #include <log/log.h> #include <private/android_filesystem_config.h> @@ -130,12 +131,17 @@ void RequestedLayerState::merge(const ResolvedComposerState& resolvedComposerSta const uint32_t oldFlags = flags; const half oldAlpha = color.a; const bool hadBuffer = externalTexture != nullptr; + uint64_t oldFramenumber = hadBuffer ? bufferData->frameNumber : 0; + const ui::Size oldBufferSize = hadBuffer + ? ui::Size(externalTexture->getWidth(), externalTexture->getHeight()) + : ui::Size(); const bool hadSideStream = sidebandStream != nullptr; const layer_state_t& clientState = resolvedComposerState.state; const bool hadBlur = hasBlur(); uint64_t clientChanges = what | layer_state_t::diff(clientState); layer_state_t::merge(clientState); what = clientChanges; + LLOGV(layerId, "requested=%" PRIu64 "flags=%" PRIu64, clientState.what, clientChanges); if (clientState.what & layer_state_t::eFlagsChanged) { if ((oldFlags ^ flags) & layer_state_t::eLayerHidden) { @@ -146,15 +152,19 @@ void RequestedLayerState::merge(const ResolvedComposerState& resolvedComposerSta changes |= RequestedLayerState::Changes::Geometry; } } + if (clientState.what & layer_state_t::eBufferChanged) { externalTexture = resolvedComposerState.externalTexture; - barrierProducerId = std::max(bufferData->producerId, barrierProducerId); - barrierFrameNumber = std::max(bufferData->frameNumber, barrierFrameNumber); - // TODO(b/277265947) log and flush transaction trace when we detect out of order updates - const bool hasBuffer = externalTexture != nullptr; if (hasBuffer || hasBuffer != hadBuffer) { changes |= RequestedLayerState::Changes::Buffer; + const ui::Size newBufferSize = hasBuffer + ? ui::Size(externalTexture->getWidth(), externalTexture->getHeight()) + : ui::Size(); + if (oldBufferSize != newBufferSize) { + changes |= RequestedLayerState::Changes::BufferSize; + changes |= RequestedLayerState::Changes::Geometry; + } } if (hasBuffer != hadBuffer) { @@ -162,6 +172,28 @@ void RequestedLayerState::merge(const ResolvedComposerState& resolvedComposerSta RequestedLayerState::Changes::VisibleRegion | RequestedLayerState::Changes::Visibility | RequestedLayerState::Changes::Input; } + + if (hasBuffer) { + const bool frameNumberChanged = + bufferData->flags.test(BufferData::BufferDataChange::frameNumberChanged); + const uint64_t frameNumber = + frameNumberChanged ? bufferData->frameNumber : oldFramenumber + 1; + bufferData->frameNumber = frameNumber; + + if ((barrierProducerId > bufferData->producerId) || + ((barrierProducerId == bufferData->producerId) && + (barrierFrameNumber > bufferData->frameNumber))) { + ALOGE("Out of order buffers detected for %s producedId=%d frameNumber=%" PRIu64 + " -> producedId=%d frameNumber=%" PRIu64, + getDebugString().c_str(), bufferData->producerId, bufferData->frameNumber, + bufferData->producerId, frameNumber); + TransactionTraceWriter::getInstance().invoke("out_of_order_buffers_", + /*overwrite=*/false); + } + + barrierProducerId = std::max(bufferData->producerId, barrierProducerId); + barrierFrameNumber = std::max(bufferData->frameNumber, barrierFrameNumber); + } } if (clientState.what & layer_state_t::eSidebandStreamChanged) { @@ -260,7 +292,7 @@ void RequestedLayerState::merge(const ResolvedComposerState& resolvedComposerSta // child layers. if (static_cast<int32_t>(gameMode) != requestedGameMode) { gameMode = static_cast<gui::GameMode>(requestedGameMode); - changes |= RequestedLayerState::Changes::AffectsChildren; + changes |= RequestedLayerState::Changes::GameMode; } } } @@ -351,7 +383,7 @@ bool RequestedLayerState::isHiddenByPolicy() const { return (flags & layer_state_t::eLayerHidden) == layer_state_t::eLayerHidden; }; half4 RequestedLayerState::getColor() const { - if ((sidebandStream != nullptr) || (externalTexture != nullptr)) { + if (sidebandStream || externalTexture) { return {0._hf, 0._hf, 0._hf, color.a}; } return color; diff --git a/services/surfaceflinger/FrontEnd/RequestedLayerState.h b/services/surfaceflinger/FrontEnd/RequestedLayerState.h index 0ef50bc60a..02e3bac18b 100644 --- a/services/surfaceflinger/FrontEnd/RequestedLayerState.h +++ b/services/surfaceflinger/FrontEnd/RequestedLayerState.h @@ -54,6 +54,8 @@ struct RequestedLayerState : layer_state_t { Buffer = 1u << 15, SidebandStream = 1u << 16, Animation = 1u << 17, + BufferSize = 1u << 18, + GameMode = 1u << 19, }; static Rect reduce(const Rect& win, const Region& exclude); RequestedLayerState(const LayerCreationArgs&); @@ -91,10 +93,10 @@ struct RequestedLayerState : layer_state_t { const uint32_t textureName; // The owner of the layer. If created from a non system process, it will be the calling uid. // If created from a system process, the value can be passed in. - const uid_t ownerUid; + const gui::Uid ownerUid; // The owner pid of the layer. If created from a non system process, it will be the calling pid. // If created from a system process, the value can be passed in. - const pid_t ownerPid; + const gui::Pid ownerPid; bool dataspaceRequested; bool hasColorTransform; bool premultipliedAlpha{true}; diff --git a/services/surfaceflinger/FrontEnd/TransactionHandler.cpp b/services/surfaceflinger/FrontEnd/TransactionHandler.cpp index 9cbe0bb224..6e78e93650 100644 --- a/services/surfaceflinger/FrontEnd/TransactionHandler.cpp +++ b/services/surfaceflinger/FrontEnd/TransactionHandler.cpp @@ -16,7 +16,7 @@ // #define LOG_NDEBUG 0 #undef LOG_TAG -#define LOG_TAG "TransactionHandler" +#define LOG_TAG "SurfaceFlinger" #define ATRACE_TAG ATRACE_TAG_GRAPHICS #include <cutils/trace.h> @@ -186,21 +186,36 @@ bool TransactionHandler::hasPendingTransactions() { } void TransactionHandler::onTransactionQueueStalled(uint64_t transactionId, - sp<ITransactionCompletedListener>& listener, - const std::string& reason) { - if (std::find(mStalledTransactions.begin(), mStalledTransactions.end(), transactionId) != - mStalledTransactions.end()) { - return; - } + StalledTransactionInfo stalledTransactionInfo) { + std::lock_guard lock{mStalledMutex}; + mStalledTransactions.emplace(transactionId, std::move(stalledTransactionInfo)); +} + +void TransactionHandler::removeFromStalledTransactions(uint64_t transactionId) { + std::lock_guard lock{mStalledMutex}; + mStalledTransactions.erase(transactionId); +} - mStalledTransactions.push_back(transactionId); - listener->onTransactionQueueStalled(String8(reason.c_str())); +std::optional<TransactionHandler::StalledTransactionInfo> +TransactionHandler::getStalledTransactionInfo(pid_t pid) { + std::lock_guard lock{mStalledMutex}; + for (auto [_, stalledTransactionInfo] : mStalledTransactions) { + if (pid == stalledTransactionInfo.pid) { + return stalledTransactionInfo; + } + } + return std::nullopt; } -void TransactionHandler::removeFromStalledTransactions(uint64_t id) { - auto it = std::find(mStalledTransactions.begin(), mStalledTransactions.end(), id); - if (it != mStalledTransactions.end()) { - mStalledTransactions.erase(it); +void TransactionHandler::onLayerDestroyed(uint32_t layerId) { + std::lock_guard lock{mStalledMutex}; + for (auto it = mStalledTransactions.begin(); it != mStalledTransactions.end();) { + if (it->second.layerId == layerId) { + it = mStalledTransactions.erase(it); + } else { + it++; + } } } + } // namespace android::surfaceflinger::frontend diff --git a/services/surfaceflinger/FrontEnd/TransactionHandler.h b/services/surfaceflinger/FrontEnd/TransactionHandler.h index 865835f92d..ff54dc5450 100644 --- a/services/surfaceflinger/FrontEnd/TransactionHandler.h +++ b/services/surfaceflinger/FrontEnd/TransactionHandler.h @@ -18,6 +18,7 @@ #include <semaphore.h> #include <cstdint> +#include <optional> #include <vector> #include <LocklessQueue.h> @@ -61,9 +62,18 @@ public: std::vector<TransactionState> flushTransactions(); void addTransactionReadyFilter(TransactionFilter&&); void queueTransaction(TransactionState&&); - void onTransactionQueueStalled(uint64_t transactionId, sp<ITransactionCompletedListener>&, - const std::string& reason); + + struct StalledTransactionInfo { + pid_t pid; + uint32_t layerId; + std::string layerName; + uint64_t bufferId; + uint64_t frameNumber; + }; + void onTransactionQueueStalled(uint64_t transactionId, StalledTransactionInfo); void removeFromStalledTransactions(uint64_t transactionId); + std::optional<StalledTransactionInfo> getStalledTransactionInfo(pid_t pid); + void onLayerDestroyed(uint32_t layerId); private: // For unit tests @@ -79,7 +89,10 @@ private: LocklessQueue<TransactionState> mLocklessTransactionQueue; std::atomic<size_t> mPendingTransactionCount = 0; ftl::SmallVector<TransactionFilter, 2> mTransactionReadyFilters; - std::vector<uint64_t> mStalledTransactions; + + std::mutex mStalledMutex; + std::unordered_map<uint64_t /* transactionId */, StalledTransactionInfo> mStalledTransactions + GUARDED_BY(mStalledMutex); }; } // namespace surfaceflinger::frontend } // namespace android diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp index c1f9a37f46..2c8d559cc1 100644 --- a/services/surfaceflinger/Layer.cpp +++ b/services/surfaceflinger/Layer.cpp @@ -50,10 +50,10 @@ #include <stdlib.h> #include <sys/types.h> #include <system/graphics-base-v1.0.h> -#include <ui/DataspaceUtils.h> #include <ui/DebugUtils.h> #include <ui/FloatRect.h> #include <ui/GraphicBuffer.h> +#include <ui/HdrRenderTypeUtils.h> #include <ui/PixelFormat.h> #include <ui/Rect.h> #include <ui/Transform.h> @@ -1348,6 +1348,8 @@ void Layer::setFrameTimelineVsyncForBufferTransaction(const FrameTimelineInfo& i mDrawingState.bufferSurfaceFrameTX = createSurfaceFrameForBuffer(info, postTime, mTransactionName); } + + setFrameTimelineVsyncForSkippedFrames(info, postTime, mTransactionName); } void Layer::setFrameTimelineVsyncForBufferlessTransaction(const FrameTimelineInfo& info, @@ -1379,11 +1381,13 @@ void Layer::setFrameTimelineVsyncForBufferlessTransaction(const FrameTimelineInf it->second = createSurfaceFrameForTransaction(info, postTime); } } + + setFrameTimelineVsyncForSkippedFrames(info, postTime, mTransactionName); } void Layer::addSurfaceFrameDroppedForBuffer( - std::shared_ptr<frametimeline::SurfaceFrame>& surfaceFrame) { - surfaceFrame->setDropTime(systemTime()); + std::shared_ptr<frametimeline::SurfaceFrame>& surfaceFrame, nsecs_t dropTime) { + surfaceFrame->setDropTime(dropTime); surfaceFrame->setPresentState(PresentState::Dropped); mFlinger->mFrameTimeline->addSurfaceFrame(surfaceFrame); } @@ -1433,6 +1437,32 @@ std::shared_ptr<frametimeline::SurfaceFrame> Layer::createSurfaceFrameForBuffer( return surfaceFrame; } +void Layer::setFrameTimelineVsyncForSkippedFrames(const FrameTimelineInfo& info, nsecs_t postTime, + std::string debugName) { + if (info.skippedFrameVsyncId == FrameTimelineInfo::INVALID_VSYNC_ID) { + return; + } + + FrameTimelineInfo skippedFrameTimelineInfo = info; + skippedFrameTimelineInfo.vsyncId = info.skippedFrameVsyncId; + + auto surfaceFrame = + mFlinger->mFrameTimeline->createSurfaceFrameForToken(skippedFrameTimelineInfo, + mOwnerPid, mOwnerUid, + getSequence(), mName, debugName, + /*isBuffer*/ false, getGameMode()); + surfaceFrame->setActualStartTime(skippedFrameTimelineInfo.skippedFrameStartTimeNanos); + // For Transactions, the post time is considered to be both queue and acquire fence time. + surfaceFrame->setActualQueueTime(postTime); + surfaceFrame->setAcquireFenceTime(postTime); + const auto fps = mFlinger->mScheduler->getFrameRateOverride(getOwnerUid()); + if (fps) { + surfaceFrame->setRenderRate(*fps); + } + onSurfaceFrameCreated(surfaceFrame); + addSurfaceFrameDroppedForBuffer(surfaceFrame, postTime); +} + bool Layer::setFrameRateForLayerTreeLegacy(FrameRate frameRate) { if (mDrawingState.frameRateForLayerTree == frameRate) { return false; @@ -2421,8 +2451,8 @@ void Layer::handleDropInputMode(gui::WindowInfo& info) const { WindowInfo Layer::fillInputInfo(const InputDisplayArgs& displayArgs) { if (!hasInputInfo()) { mDrawingState.inputInfo.name = getName(); - mDrawingState.inputInfo.ownerUid = mOwnerUid; - mDrawingState.inputInfo.ownerPid = mOwnerPid; + mDrawingState.inputInfo.ownerUid = gui::Uid{mOwnerUid}; + mDrawingState.inputInfo.ownerPid = gui::Pid{mOwnerPid}; mDrawingState.inputInfo.inputConfig |= WindowInfo::InputConfig::NO_INPUT_CHANNEL; mDrawingState.inputInfo.displayId = getLayerStack().id; } @@ -3066,7 +3096,7 @@ bool Layer::setBuffer(std::shared_ptr<renderengine::ExternalTexture>& buffer, decrementPendingBufferCount(); if (mDrawingState.bufferSurfaceFrameTX != nullptr && mDrawingState.bufferSurfaceFrameTX->getPresentState() != PresentState::Presented) { - addSurfaceFrameDroppedForBuffer(mDrawingState.bufferSurfaceFrameTX); + addSurfaceFrameDroppedForBuffer(mDrawingState.bufferSurfaceFrameTX, systemTime()); mDrawingState.bufferSurfaceFrameTX.reset(); } } else if (EARLY_RELEASE_ENABLED && mLastClientCompositionFence != nullptr) { @@ -3094,6 +3124,16 @@ bool Layer::setBuffer(std::shared_ptr<renderengine::ExternalTexture>& buffer, return true; } + if ((mDrawingState.producerId > bufferData.producerId) || + ((mDrawingState.producerId == bufferData.producerId) && + (mDrawingState.frameNumber > frameNumber))) { + ALOGE("Out of order buffers detected for %s producedId=%d frameNumber=%" PRIu64 + " -> producedId=%d frameNumber=%" PRIu64, + getDebugName(), mDrawingState.producerId, mDrawingState.frameNumber, + bufferData.producerId, frameNumber); + TransactionTraceWriter::getInstance().invoke("out_of_order_buffers_", /*overwrite=*/false); + } + mDrawingState.producerId = bufferData.producerId; mDrawingState.barrierProducerId = std::max(mDrawingState.producerId, mDrawingState.barrierProducerId); @@ -3101,7 +3141,6 @@ bool Layer::setBuffer(std::shared_ptr<renderengine::ExternalTexture>& buffer, mDrawingState.barrierFrameNumber = std::max(mDrawingState.frameNumber, mDrawingState.barrierFrameNumber); - // TODO(b/277265947) log and flush transaction trace when we detect out of order updates mDrawingState.releaseBufferListener = bufferData.releaseBufferListener; mDrawingState.buffer = std::move(buffer); mDrawingState.acquireFence = bufferData.flags.test(BufferData::BufferDataChange::fenceChanged) @@ -3138,6 +3177,14 @@ bool Layer::setBuffer(std::shared_ptr<renderengine::ExternalTexture>& buffer, } mDrawingState.releaseBufferEndpoint = bufferData.releaseBufferEndpoint; + + // If the layer had been updated a TextureView, this would make sure the present time could be + // same to TextureView update when it's a small dirty, and get the correct heuristic rate. + if (mFlinger->mScheduler->supportSmallDirtyDetection()) { + if (mDrawingState.useVsyncIdForRefreshRateSelection) { + mUsedVsyncIdForRefreshRateSelection = true; + } + } return true; } @@ -3160,10 +3207,38 @@ void Layer::recordLayerHistoryBufferUpdate(const scheduler::LayerProps& layerPro mDrawingState.latchedVsyncId); if (prediction.has_value()) { ATRACE_FORMAT_INSTANT("predictedPresentTime"); + mMaxTimeForUseVsyncId = prediction->presentTime + + scheduler::LayerHistory::kMaxPeriodForHistory.count(); return prediction->presentTime; } } + if (!mFlinger->mScheduler->supportSmallDirtyDetection()) { + return static_cast<nsecs_t>(0); + } + + // If the layer is not an application and didn't set an explicit rate or desiredPresentTime, + // return "0" to tell the layer history that it will use the max refresh rate without + // calculating the adaptive rate. + if (mWindowType != WindowInfo::Type::APPLICATION && + mWindowType != WindowInfo::Type::BASE_APPLICATION) { + return static_cast<nsecs_t>(0); + } + + // Return the valid present time only when the layer potentially updated a TextureView so + // LayerHistory could heuristically calculate the rate if the UI is continually updating. + if (mUsedVsyncIdForRefreshRateSelection) { + const auto prediction = + mFlinger->mFrameTimeline->getTokenManager()->getPredictionsForToken( + mDrawingState.latchedVsyncId); + if (prediction.has_value()) { + if (mMaxTimeForUseVsyncId >= prediction->presentTime) { + return prediction->presentTime; + } + mUsedVsyncIdForRefreshRateSelection = false; + } + } + return static_cast<nsecs_t>(0); }(); @@ -3223,6 +3298,7 @@ bool Layer::setSurfaceDamageRegion(const Region& surfaceDamage) { mDrawingState.surfaceDamageRegion = surfaceDamage; mDrawingState.modified = true; setTransactionFlags(eTransactionNeeded); + setIsSmallDirty(); return true; } @@ -3598,7 +3674,7 @@ std::pair<FloatRect, bool> Layer::getInputBounds(bool fillParentBounds) const { return {inputBounds, inputBoundsValid}; } -bool Layer::simpleBufferUpdate(const layer_state_t& s) const { +bool Layer::isSimpleBufferUpdate(const layer_state_t& s) const { const uint64_t requiredFlags = layer_state_t::eBufferChanged; const uint64_t deniedFlags = layer_state_t::eProducerDisconnect | layer_state_t::eLayerChanged | @@ -3607,51 +3683,42 @@ bool Layer::simpleBufferUpdate(const layer_state_t& s) const { layer_state_t::eLayerStackChanged | layer_state_t::eAutoRefreshChanged | layer_state_t::eReparent; - const uint64_t allowedFlags = layer_state_t::eHasListenerCallbacksChanged | - layer_state_t::eFrameRateSelectionPriority | layer_state_t::eFrameRateChanged | - layer_state_t::eSurfaceDamageRegionChanged | layer_state_t::eApiChanged | - layer_state_t::eMetadataChanged | layer_state_t::eDropInputModeChanged | - layer_state_t::eInputInfoChanged; - if ((s.what & requiredFlags) != requiredFlags) { - ALOGV("%s: false [missing required flags 0x%" PRIx64 "]", __func__, - (s.what | requiredFlags) & ~s.what); + ATRACE_FORMAT_INSTANT("%s: false [missing required flags 0x%" PRIx64 "]", __func__, + (s.what | requiredFlags) & ~s.what); return false; } if (s.what & deniedFlags) { - ALOGV("%s: false [has denied flags 0x%" PRIx64 "]", __func__, s.what & deniedFlags); + ATRACE_FORMAT_INSTANT("%s: false [has denied flags 0x%" PRIx64 "]", __func__, + s.what & deniedFlags); return false; } - if (s.what & allowedFlags) { - ALOGV("%s: [has allowed flags 0x%" PRIx64 "]", __func__, s.what & allowedFlags); - } - if (s.what & layer_state_t::ePositionChanged) { if (mRequestedTransform.tx() != s.x || mRequestedTransform.ty() != s.y) { - ALOGV("%s: false [ePositionChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [ePositionChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eAlphaChanged) { if (mDrawingState.color.a != s.color.a) { - ALOGV("%s: false [eAlphaChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eAlphaChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eColorTransformChanged) { if (mDrawingState.colorTransform != s.colorTransform) { - ALOGV("%s: false [eColorTransformChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eColorTransformChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eBackgroundColorChanged) { if (mDrawingState.bgColorLayer || s.bgColor.a != 0) { - ALOGV("%s: false [eBackgroundColorChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eBackgroundColorChanged changed]", __func__); return false; } } @@ -3661,91 +3728,92 @@ bool Layer::simpleBufferUpdate(const layer_state_t& s) const { mRequestedTransform.dtdy() != s.matrix.dtdy || mRequestedTransform.dtdx() != s.matrix.dtdx || mRequestedTransform.dsdy() != s.matrix.dsdy) { - ALOGV("%s: false [eMatrixChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eMatrixChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eCornerRadiusChanged) { if (mDrawingState.cornerRadius != s.cornerRadius) { - ALOGV("%s: false [eCornerRadiusChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eCornerRadiusChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eBackgroundBlurRadiusChanged) { if (mDrawingState.backgroundBlurRadius != static_cast<int>(s.backgroundBlurRadius)) { - ALOGV("%s: false [eBackgroundBlurRadiusChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eBackgroundBlurRadiusChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eBufferTransformChanged) { if (mDrawingState.bufferTransform != s.bufferTransform) { - ALOGV("%s: false [eBufferTransformChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eBufferTransformChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eTransformToDisplayInverseChanged) { if (mDrawingState.transformToDisplayInverse != s.transformToDisplayInverse) { - ALOGV("%s: false [eTransformToDisplayInverseChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eTransformToDisplayInverseChanged changed]", + __func__); return false; } } if (s.what & layer_state_t::eCropChanged) { if (mDrawingState.crop != s.crop) { - ALOGV("%s: false [eCropChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eCropChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eDataspaceChanged) { if (mDrawingState.dataspace != s.dataspace) { - ALOGV("%s: false [eDataspaceChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eDataspaceChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eHdrMetadataChanged) { if (mDrawingState.hdrMetadata != s.hdrMetadata) { - ALOGV("%s: false [eHdrMetadataChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eHdrMetadataChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eSidebandStreamChanged) { if (mDrawingState.sidebandStream != s.sidebandStream) { - ALOGV("%s: false [eSidebandStreamChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eSidebandStreamChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eColorSpaceAgnosticChanged) { if (mDrawingState.colorSpaceAgnostic != s.colorSpaceAgnostic) { - ALOGV("%s: false [eColorSpaceAgnosticChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eColorSpaceAgnosticChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eShadowRadiusChanged) { if (mDrawingState.shadowRadius != s.shadowRadius) { - ALOGV("%s: false [eShadowRadiusChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eShadowRadiusChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eFixedTransformHintChanged) { if (mDrawingState.fixedTransformHint != s.fixedTransformHint) { - ALOGV("%s: false [eFixedTransformHintChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eFixedTransformHintChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eTrustedOverlayChanged) { if (mDrawingState.isTrustedOverlay != s.isTrustedOverlay) { - ALOGV("%s: false [eTrustedOverlayChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eTrustedOverlayChanged changed]", __func__); return false; } } @@ -3754,28 +3822,28 @@ bool Layer::simpleBufferUpdate(const layer_state_t& s) const { StretchEffect temp = s.stretchEffect; temp.sanitize(); if (mDrawingState.stretchEffect != temp) { - ALOGV("%s: false [eStretchChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eStretchChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eBufferCropChanged) { if (mDrawingState.bufferCrop != s.bufferCrop) { - ALOGV("%s: false [eBufferCropChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eBufferCropChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eDestinationFrameChanged) { if (mDrawingState.destinationFrame != s.destinationFrame) { - ALOGV("%s: false [eDestinationFrameChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eDestinationFrameChanged changed]", __func__); return false; } } if (s.what & layer_state_t::eDimmingEnabledChanged) { if (mDrawingState.dimmingEnabled != s.dimmingEnabled) { - ALOGV("%s: false [eDimmingEnabledChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eDimmingEnabledChanged changed]", __func__); return false; } } @@ -3783,12 +3851,11 @@ bool Layer::simpleBufferUpdate(const layer_state_t& s) const { if (s.what & layer_state_t::eExtendedRangeBrightnessChanged) { if (mDrawingState.currentHdrSdrRatio != s.currentHdrSdrRatio || mDrawingState.desiredHdrSdrRatio != s.desiredHdrSdrRatio) { - ALOGV("%s: false [eExtendedRangeBrightnessChanged changed]", __func__); + ATRACE_FORMAT_INSTANT("%s: false [eExtendedRangeBrightnessChanged changed]", __func__); return false; } } - ALOGV("%s: true", __func__); return true; } @@ -4278,6 +4345,26 @@ void Layer::updateLastLatchTime(nsecs_t latchTime) { mLastLatchTime = latchTime; } +void Layer::setIsSmallDirty() { + if (!mFlinger->mScheduler->supportSmallDirtyDetection()) { + return; + } + + if (mWindowType != WindowInfo::Type::APPLICATION && + mWindowType != WindowInfo::Type::BASE_APPLICATION) { + return; + } + Rect bounds = mDrawingState.surfaceDamageRegion.getBounds(); + if (!bounds.isValid()) { + return; + } + + // If the damage region is a small dirty, this could give the hint for the layer history that + // it could suppress the heuristic rate when calculating. + mSmallDirty = mFlinger->mScheduler->isSmallDirtyArea(mOwnerUid, + bounds.getWidth() * bounds.getHeight()); +} + // --------------------------------------------------------------------------- std::ostream& operator<<(std::ostream& stream, const Layer::FrameRate& rate) { diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h index f7596e20e5..895d25aa03 100644 --- a/services/surfaceflinger/Layer.h +++ b/services/surfaceflinger/Layer.h @@ -785,8 +785,8 @@ public: void setFrameTimelineVsyncForBufferlessTransaction(const FrameTimelineInfo& info, nsecs_t postTime); - void addSurfaceFrameDroppedForBuffer( - std::shared_ptr<frametimeline::SurfaceFrame>& surfaceFrame); + void addSurfaceFrameDroppedForBuffer(std::shared_ptr<frametimeline::SurfaceFrame>& surfaceFrame, + nsecs_t dropTime); void addSurfaceFramePresentedForBuffer( std::shared_ptr<frametimeline::SurfaceFrame>& surfaceFrame, nsecs_t acquireFenceTime, nsecs_t currentLatchTime); @@ -795,6 +795,8 @@ public: const FrameTimelineInfo& info, nsecs_t postTime); std::shared_ptr<frametimeline::SurfaceFrame> createSurfaceFrameForBuffer( const FrameTimelineInfo& info, nsecs_t queueTime, std::string debugName); + void setFrameTimelineVsyncForSkippedFrames(const FrameTimelineInfo& info, nsecs_t postTime, + std::string debugName); bool setTrustedPresentationInfo(TrustedPresentationThresholds const& thresholds, TrustedPresentationListener const& listener); @@ -840,6 +842,14 @@ public: mutable bool contentDirty{false}; Region surfaceDamageRegion; + // True when the surfaceDamageRegion is recognized as a small area update. + bool mSmallDirty{false}; + // Used to check if mUsedVsyncIdForRefreshRateSelection should be expired when it stop updating. + nsecs_t mMaxTimeForUseVsyncId = 0; + // True when DrawState.useVsyncIdForRefreshRateSelection previously set to true during updating + // buffer. + bool mUsedVsyncIdForRefreshRateSelection{false}; + // Layer serial number. This gives layers an explicit ordering, so we // have a stable sort order when their layer stack and Z-order are // the same. @@ -866,7 +876,7 @@ public: std::string getPendingBufferCounterName() { return mBlastTransactionName; } bool updateGeometry(); - bool simpleBufferUpdate(const layer_state_t&) const; + bool isSimpleBufferUpdate(const layer_state_t& s) const; static bool isOpaqueFormat(PixelFormat format); @@ -902,6 +912,7 @@ public: .transform = getTransform(), .setFrameRateVote = getFrameRateForLayerTree(), .frameRateSelectionPriority = getFrameRateSelectionPriority(), + .isSmallDirty = mSmallDirty, }; }; bool hasBuffer() const { return mBufferInfo.mBuffer != nullptr; } @@ -915,6 +926,9 @@ public: // Exposed so SurfaceFlinger can assert that it's held const sp<SurfaceFlinger> mFlinger; + // Check if the damage region is a small dirty. + void setIsSmallDirty(); + protected: // For unit tests friend class TestableSurfaceFlinger; diff --git a/services/surfaceflinger/LayerFE.cpp b/services/surfaceflinger/LayerFE.cpp index f855f278c3..88808664a4 100644 --- a/services/surfaceflinger/LayerFE.cpp +++ b/services/surfaceflinger/LayerFE.cpp @@ -249,10 +249,13 @@ void LayerFE::prepareBufferStateClientComposition( layerSettings.frameNumber = mSnapshot->frameNumber; layerSettings.bufferId = mSnapshot->externalTexture->getId(); + const bool useFiltering = targetSettings.needsFiltering || + mSnapshot->geomLayerTransform.needsBilinearFiltering(); + // Query the texture matrix given our current filtering mode. float textureMatrix[16]; getDrawingTransformMatrix(layerSettings.source.buffer.buffer, mSnapshot->geomContentCrop, - mSnapshot->geomBufferTransform, targetSettings.needsFiltering, + mSnapshot->geomBufferTransform, useFiltering, textureMatrix); if (mSnapshot->geomBufferUsesDisplayInverseTransform) { @@ -303,7 +306,7 @@ void LayerFE::prepareBufferStateClientComposition( mat4::translate(vec4(translateX, translateY, 0.f, 1.f)) * mat4::scale(vec4(scaleWidth, scaleHeight, 1.0f, 1.0f)); - layerSettings.source.buffer.useTextureFiltering = targetSettings.needsFiltering; + layerSettings.source.buffer.useTextureFiltering = useFiltering; layerSettings.source.buffer.textureTransform = mat4(static_cast<const float*>(textureMatrix)) * tr; diff --git a/services/surfaceflinger/LayerProtoHelper.cpp b/services/surfaceflinger/LayerProtoHelper.cpp index 3472d201f3..1c7581b093 100644 --- a/services/surfaceflinger/LayerProtoHelper.cpp +++ b/services/surfaceflinger/LayerProtoHelper.cpp @@ -178,6 +178,7 @@ void LayerProtoHelper::writeToProto( InputWindowInfoProto* proto = getInputWindowInfoProto(); proto->set_layout_params_flags(inputInfo.layoutParamsFlags.get()); + proto->set_input_config(inputInfo.inputConfig.get()); using U = std::underlying_type_t<WindowInfo::Type>; // TODO(b/129481165): This static assert can be safely removed once conversion warnings // are re-enabled. @@ -427,7 +428,7 @@ void LayerProtoHelper::writeSnapshotToProto(LayerProto* layerInfo, layerInfo->set_is_relative_of(requestedState.isRelativeOf); - layerInfo->set_owner_uid(requestedState.ownerUid); + layerInfo->set_owner_uid(requestedState.ownerUid.val()); if ((traceFlags & LayerTracing::TRACE_INPUT) && snapshot.hasInputInfo()) { LayerProtoHelper::writeToProto(snapshot.inputInfo, {}, @@ -446,7 +447,7 @@ void LayerProtoHelper::writeSnapshotToProto(LayerProto* layerInfo, } google::protobuf::RepeatedPtrField<DisplayProto> LayerProtoHelper::writeDisplayInfoToProto( - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displayInfos) { + const frontend::DisplayInfos& displayInfos) { google::protobuf::RepeatedPtrField<DisplayProto> displays; displays.Reserve(displayInfos.size()); for (const auto& [layerStack, displayInfo] : displayInfos) { diff --git a/services/surfaceflinger/LayerProtoHelper.h b/services/surfaceflinger/LayerProtoHelper.h index b84a49b27c..346685f259 100644 --- a/services/surfaceflinger/LayerProtoHelper.h +++ b/services/surfaceflinger/LayerProtoHelper.h @@ -26,6 +26,8 @@ #include <ui/Region.h> #include <ui/Transform.h> #include <cstdint> + +#include "FrontEnd/DisplayInfo.h" #include "FrontEnd/LayerHierarchy.h" #include "FrontEnd/LayerSnapshot.h" @@ -65,15 +67,15 @@ public: const frontend::RequestedLayerState& requestedState, const frontend::LayerSnapshot& snapshot, uint32_t traceFlags); static google::protobuf::RepeatedPtrField<DisplayProto> writeDisplayInfoToProto( - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displayInfos); + const frontend::DisplayInfos&); }; class LayerProtoFromSnapshotGenerator { public: - LayerProtoFromSnapshotGenerator( - const frontend::LayerSnapshotBuilder& snapshotBuilder, - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displayInfos, - const std::unordered_map<uint32_t, sp<Layer>>& legacyLayers, uint32_t traceFlags) + LayerProtoFromSnapshotGenerator(const frontend::LayerSnapshotBuilder& snapshotBuilder, + const frontend::DisplayInfos& displayInfos, + const std::unordered_map<uint32_t, sp<Layer>>& legacyLayers, + uint32_t traceFlags) : mSnapshotBuilder(snapshotBuilder), mLegacyLayers(legacyLayers), mDisplayInfos(displayInfos), @@ -88,7 +90,7 @@ private: const frontend::LayerSnapshotBuilder& mSnapshotBuilder; const std::unordered_map<uint32_t, sp<Layer>>& mLegacyLayers; - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& mDisplayInfos; + const frontend::DisplayInfos& mDisplayInfos; uint32_t mTraceFlags; LayersProto mLayersProto; // winscope expects all the layers, so provide a snapshot even if it not currently drawing diff --git a/services/surfaceflinger/Scheduler/Android.bp b/services/surfaceflinger/Scheduler/Android.bp index d5d868839f..6d2586ae9c 100644 --- a/services/surfaceflinger/Scheduler/Android.bp +++ b/services/surfaceflinger/Scheduler/Android.bp @@ -40,6 +40,7 @@ cc_library_static { name: "libscheduler", defaults: ["libscheduler_defaults"], srcs: [ + "src/FrameTargeter.cpp", "src/PresentLatencyTracker.cpp", "src/Timer.cpp", ], @@ -52,6 +53,7 @@ cc_test { test_suites: ["device-tests"], defaults: ["libscheduler_defaults"], srcs: [ + "tests/FrameTargeterTest.cpp", "tests/PresentLatencyTrackerTest.cpp", "tests/TimerTest.cpp", ], diff --git a/services/surfaceflinger/Scheduler/ISchedulerCallback.h b/services/surfaceflinger/Scheduler/ISchedulerCallback.h index 92c2189244..badbf53753 100644 --- a/services/surfaceflinger/Scheduler/ISchedulerCallback.h +++ b/services/surfaceflinger/Scheduler/ISchedulerCallback.h @@ -25,7 +25,7 @@ namespace android::scheduler { struct ISchedulerCallback { - virtual void setVsyncEnabled(PhysicalDisplayId, bool) = 0; + virtual void requestHardwareVsync(PhysicalDisplayId, bool enabled) = 0; virtual void requestDisplayModes(std::vector<display::DisplayModeRequest>) = 0; virtual void kernelTimerChanged(bool expired) = 0; virtual void triggerOnFrameRateOverridesChanged() = 0; diff --git a/services/surfaceflinger/Scheduler/LayerHistory.cpp b/services/surfaceflinger/Scheduler/LayerHistory.cpp index beaf9724a3..6adffc9466 100644 --- a/services/surfaceflinger/Scheduler/LayerHistory.cpp +++ b/services/surfaceflinger/Scheduler/LayerHistory.cpp @@ -320,4 +320,11 @@ auto LayerHistory::findLayer(int32_t id) -> std::pair<LayerStatus, LayerPair*> { return {LayerStatus::NotFound, nullptr}; } +bool LayerHistory::isSmallDirtyArea(uint32_t dirtyArea, float threshold) const { + const float ratio = (float)dirtyArea / mDisplayArea; + const bool isSmallDirty = ratio <= threshold; + ATRACE_FORMAT_INSTANT("small dirty=%s, ratio=%.3f", isSmallDirty ? "true" : "false", ratio); + return isSmallDirty; +} + } // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/LayerHistory.h b/services/surfaceflinger/Scheduler/LayerHistory.h index 69caf9ffd2..6f07e3ba0e 100644 --- a/services/surfaceflinger/Scheduler/LayerHistory.h +++ b/services/surfaceflinger/Scheduler/LayerHistory.h @@ -43,6 +43,7 @@ struct LayerProps; class LayerHistory { public: using LayerVoteType = RefreshRateSelector::LayerVoteType; + static constexpr std::chrono::nanoseconds kMaxPeriodForHistory = 1s; LayerHistory(); ~LayerHistory(); @@ -87,6 +88,8 @@ public: void attachChoreographer(int32_t layerId, const sp<EventThreadConnection>& choreographerConnection); + bool isSmallDirtyArea(uint32_t dirtyArea, float threshold) const; + private: friend class LayerHistoryTest; friend class TestableScheduler; diff --git a/services/surfaceflinger/Scheduler/LayerInfo.cpp b/services/surfaceflinger/Scheduler/LayerInfo.cpp index bae3739501..875e87084f 100644 --- a/services/surfaceflinger/Scheduler/LayerInfo.cpp +++ b/services/surfaceflinger/Scheduler/LayerInfo.cpp @@ -63,7 +63,8 @@ void LayerInfo::setLastPresentTime(nsecs_t lastPresentTime, nsecs_t now, LayerUp case LayerUpdateType::Buffer: FrameTimeData frameTime = {.presentTime = lastPresentTime, .queueTime = mLastUpdatedTime, - .pendingModeChange = pendingModeChange}; + .pendingModeChange = pendingModeChange, + .isSmallDirty = props.isSmallDirty}; mFrameTimes.push_back(frameTime); if (mFrameTimes.size() > HISTORY_SIZE) { mFrameTimes.pop_front(); @@ -99,21 +100,38 @@ LayerInfo::Frequent LayerInfo::isFrequent(nsecs_t now) const { // classification. bool isFrequent = true; bool isInfrequent = true; + int32_t smallDirtyCount = 0; const auto n = mFrameTimes.size() - 1; for (size_t i = 0; i < kFrequentLayerWindowSize - 1; i++) { if (mFrameTimes[n - i].queueTime - mFrameTimes[n - i - 1].queueTime < kMaxPeriodForFrequentLayerNs.count()) { isInfrequent = false; + if (mFrameTimes[n - i].presentTime == 0 && mFrameTimes[n - i].isSmallDirty) { + smallDirtyCount++; + } } else { isFrequent = false; } } + // Vote the small dirty when a layer contains at least HISTORY_SIZE of small dirty updates. + bool isSmallDirty = false; + if (smallDirtyCount >= kNumSmallDirtyThreshold) { + if (mLastSmallDirtyCount >= HISTORY_SIZE) { + isSmallDirty = true; + } else { + mLastSmallDirtyCount++; + } + } else { + mLastSmallDirtyCount = 0; + } + if (isFrequent || isInfrequent) { // If the layer was previously inconclusive, we clear // the history as indeterminate layers changed to frequent, // and we should not look at the stale data. - return {isFrequent, isFrequent && !mIsFrequencyConclusive, /* isConclusive */ true}; + return {isFrequent, isFrequent && !mIsFrequencyConclusive, /* isConclusive */ true, + isSmallDirty}; } // If we can't determine whether the layer is frequent or not, we return @@ -202,6 +220,7 @@ std::optional<nsecs_t> LayerInfo::calculateAverageFrameTime() const { nsecs_t totalDeltas = 0; int numDeltas = 0; + int32_t smallDirtyCount = 0; auto prevFrame = mFrameTimes.begin(); for (auto it = mFrameTimes.begin() + 1; it != mFrameTimes.end(); ++it) { const auto currDelta = getFrameTime(*it) - getFrameTime(*prevFrame); @@ -210,6 +229,13 @@ std::optional<nsecs_t> LayerInfo::calculateAverageFrameTime() const { continue; } + // If this is a small area update, we don't want to consider it for calculating the average + // frame time. Instead, we let the bigger frame updates to drive the calculation. + if (it->isSmallDirty && currDelta < kMinPeriodBetweenSmallDirtyFrames) { + smallDirtyCount++; + continue; + } + prevFrame = it; if (currDelta > kMaxPeriodBetweenFrames) { @@ -221,6 +247,10 @@ std::optional<nsecs_t> LayerInfo::calculateAverageFrameTime() const { numDeltas++; } + if (smallDirtyCount > 0) { + ATRACE_FORMAT_INSTANT("small dirty = %" PRIu32, smallDirtyCount); + } + if (numDeltas == 0) { return std::nullopt; } @@ -286,6 +316,7 @@ LayerInfo::LayerVote LayerInfo::getRefreshRateVote(const RefreshRateSelector& se ATRACE_FORMAT_INSTANT("infrequent"); ALOGV("%s is infrequent", mName.c_str()); mLastRefreshRate.infrequent = true; + mLastSmallDirtyCount = 0; // Infrequent layers vote for minimal refresh rate for // battery saving purposes and also to prevent b/135718869. return {LayerHistory::LayerVoteType::Min, Fps()}; @@ -295,6 +326,13 @@ LayerInfo::LayerVote LayerInfo::getRefreshRateVote(const RefreshRateSelector& se clearHistory(now); } + // Return no vote if the recent frames are small dirty. + if (frequent.isSmallDirty && !mLastRefreshRate.reported.isValid()) { + ATRACE_FORMAT_INSTANT("NoVote (small dirty)"); + ALOGV("%s is small dirty", mName.c_str()); + return {LayerHistory::LayerVoteType::NoVote, Fps()}; + } + auto refreshRate = calculateRefreshRateIfPossible(selector, now); if (refreshRate.has_value()) { ALOGV("%s calculated refresh rate: %s", mName.c_str(), to_string(*refreshRate).c_str()); diff --git a/services/surfaceflinger/Scheduler/LayerInfo.h b/services/surfaceflinger/Scheduler/LayerInfo.h index c5a60573f5..122796b3d5 100644 --- a/services/surfaceflinger/Scheduler/LayerInfo.h +++ b/services/surfaceflinger/Scheduler/LayerInfo.h @@ -57,6 +57,7 @@ class LayerInfo { static constexpr Fps kMinFpsForFrequentLayer = 10_Hz; static constexpr auto kMaxPeriodForFrequentLayerNs = std::chrono::nanoseconds(kMinFpsForFrequentLayer.getPeriodNsecs()) + 1ms; + static constexpr size_t kNumSmallDirtyThreshold = 2; friend class LayerHistoryTest; friend class LayerInfoTest; @@ -195,6 +196,7 @@ private: nsecs_t presentTime; // desiredPresentTime, if provided nsecs_t queueTime; // buffer queue time bool pendingModeChange; + bool isSmallDirty; }; // Holds information about the calculated and reported refresh rate @@ -259,6 +261,8 @@ private: bool clearHistory; // Represents whether we were able to determine isFrequent conclusively bool isConclusive; + // Represents whether the latest frames are small dirty. + bool isSmallDirty = false; }; Frequent isFrequent(nsecs_t now) const; bool isAnimating(nsecs_t now) const; @@ -277,6 +281,11 @@ private: // this period apart from each other, the interval between them won't be // taken into account when calculating average frame rate. static constexpr nsecs_t kMaxPeriodBetweenFrames = kMinFpsForFrequentLayer.getPeriodNsecs(); + // Used for sanitizing the heuristic data. If frames are small dirty updating and are less + // than this period apart from each other, the interval between them won't be + // taken into account when calculating average frame rate. + static constexpr nsecs_t kMinPeriodBetweenSmallDirtyFrames = (60_Hz).getPeriodNsecs(); + LayerHistory::LayerVoteType mDefaultVote; LayerVote mLayerVote; @@ -291,12 +300,16 @@ private: std::chrono::time_point<std::chrono::steady_clock> mFrameTimeValidSince = std::chrono::steady_clock::now(); static constexpr size_t HISTORY_SIZE = RefreshRateHistory::HISTORY_SIZE; - static constexpr std::chrono::nanoseconds HISTORY_DURATION = 1s; + static constexpr std::chrono::nanoseconds HISTORY_DURATION = LayerHistory::kMaxPeriodForHistory; std::unique_ptr<LayerProps> mLayerProps; RefreshRateHistory mRefreshRateHistory; + // This will be accessed from only one thread when counting a layer is frequent or infrequent, + // and to determine whether a layer is in small dirty updating. + mutable int32_t mLastSmallDirtyCount = 0; + mutable std::unordered_map<LayerHistory::LayerVoteType, std::string> mTraceTags; // Shared for all LayerInfo instances @@ -309,6 +322,7 @@ struct LayerProps { ui::Transform transform; LayerInfo::FrameRate setFrameRateVote; int32_t frameRateSelectionPriority = -1; + bool isSmallDirty = false; }; } // namespace scheduler diff --git a/services/surfaceflinger/Scheduler/RefreshRateSelector.cpp b/services/surfaceflinger/Scheduler/RefreshRateSelector.cpp index f136e9f9df..6b7d7df5a3 100644 --- a/services/surfaceflinger/Scheduler/RefreshRateSelector.cpp +++ b/services/surfaceflinger/Scheduler/RefreshRateSelector.cpp @@ -148,8 +148,8 @@ std::string toString(const RefreshRateSelector::PolicyVariant& policy) { } // namespace auto RefreshRateSelector::createFrameRateModes( - std::function<bool(const DisplayMode&)>&& filterModes, const FpsRange& renderRange) const - -> std::vector<FrameRateMode> { + const Policy& policy, std::function<bool(const DisplayMode&)>&& filterModes, + const FpsRange& renderRange) const -> std::vector<FrameRateMode> { struct Key { Fps fps; int32_t group; @@ -202,11 +202,25 @@ auto RefreshRateSelector::createFrameRateModes( ALOGV("%s: including %s (%s)", __func__, to_string(fps).c_str(), to_string(mode->getFps()).c_str()); } else { - // We might need to update the map as we found a lower refresh rate - if (isStrictlyLess(mode->getFps(), existingIter->second->second->getFps())) { + // If the primary physical range is a single rate, prefer to stay in that rate + // even if there is a lower physical refresh rate available. This would cause more + // cases to stay within the primary physical range + const Fps existingModeFps = existingIter->second->second->getFps(); + const bool existingModeIsPrimaryRange = policy.primaryRangeIsSingleRate() && + policy.primaryRanges.physical.includes(existingModeFps); + const bool newModeIsPrimaryRange = policy.primaryRangeIsSingleRate() && + policy.primaryRanges.physical.includes(mode->getFps()); + if (newModeIsPrimaryRange == existingModeIsPrimaryRange) { + // We might need to update the map as we found a lower refresh rate + if (isStrictlyLess(mode->getFps(), existingModeFps)) { + existingIter->second = it; + ALOGV("%s: changing %s (%s) as we found a lower physical rate", __func__, + to_string(fps).c_str(), to_string(mode->getFps()).c_str()); + } + } else if (newModeIsPrimaryRange) { existingIter->second = it; - ALOGV("%s: changing %s (%s)", __func__, to_string(fps).c_str(), - to_string(mode->getFps()).c_str()); + ALOGV("%s: changing %s (%s) to stay in the primary range", __func__, + to_string(fps).c_str(), to_string(mode->getFps()).c_str()); } } } @@ -302,6 +316,19 @@ float RefreshRateSelector::calculateNonExactMatchingLayerScoreLocked(const Layer if (layer.vote == LayerVoteType::ExplicitExactOrMultiple || layer.vote == LayerVoteType::Heuristic) { + using fps_approx_ops::operator<; + if (refreshRate < 60_Hz) { + const bool favorsAtLeast60 = + std::find_if(mFrameRatesThatFavorsAtLeast60.begin(), + mFrameRatesThatFavorsAtLeast60.end(), [&](Fps fps) { + using fps_approx_ops::operator==; + return fps == layer.desiredRefreshRate; + }) != mFrameRatesThatFavorsAtLeast60.end(); + if (favorsAtLeast60) { + return 0; + } + } + const float multiplier = refreshRate.getValue() / layer.desiredRefreshRate.getValue(); // We only want to score this layer as a fractional pair if the content is not @@ -487,10 +514,8 @@ auto RefreshRateSelector::getRankedFrameRatesLocked(const std::vector<LayerRequi // If the primary range consists of a single refresh rate then we can only // move out the of range if layers explicitly request a different refresh // rate. - const bool primaryRangeIsSingleRate = - isApproxEqual(policy->primaryRanges.physical.min, policy->primaryRanges.physical.max); - - if (!signals.touch && signals.idle && !(primaryRangeIsSingleRate && hasExplicitVoteLayers)) { + if (!signals.touch && signals.idle && + !(policy->primaryRangeIsSingleRate() && hasExplicitVoteLayers)) { ALOGV("Idle"); const auto ranking = rankFrameRates(activeMode.getGroup(), RefreshRateOrder::Ascending); ATRACE_FORMAT_INSTANT("%s (Idle)", to_string(ranking.front().frameRateMode.fps).c_str()); @@ -564,8 +589,11 @@ auto RefreshRateSelector::getRankedFrameRatesLocked(const std::vector<LayerRequi continue; } - const bool inPrimaryRange = policy->primaryRanges.render.includes(fps); - if ((primaryRangeIsSingleRate || !inPrimaryRange) && + const bool inPrimaryPhysicalRange = + policy->primaryRanges.physical.includes(modePtr->getFps()); + const bool inPrimaryRenderRange = policy->primaryRanges.render.includes(fps); + if (((policy->primaryRangeIsSingleRate() && !inPrimaryPhysicalRange) || + !inPrimaryRenderRange) && !(layer.focused && (layer.vote == LayerVoteType::ExplicitDefault || layer.vote == LayerVoteType::ExplicitExact))) { @@ -676,7 +704,7 @@ auto RefreshRateSelector::getRankedFrameRatesLocked(const std::vector<LayerRequi return score.overallScore == 0; }); - if (primaryRangeIsSingleRate) { + if (policy->primaryRangeIsSingleRate()) { // If we never scored any layers, then choose the rate from the primary // range instead of picking a random score from the app range. if (noLayerScore) { @@ -1221,10 +1249,19 @@ void RefreshRateSelector::constructAvailableRefreshRates() { (supportsFrameRateOverride() || ranges.render.includes(mode.getFps())); }; - const auto frameRateModes = createFrameRateModes(filterModes, ranges.render); + auto frameRateModes = createFrameRateModes(*policy, filterModes, ranges.render); + if (frameRateModes.empty()) { + ALOGW("No matching frame rate modes for %s range. policy: %s", rangeName, + policy->toString().c_str()); + // TODO(b/292105422): Ideally DisplayManager should not send render ranges smaller than + // the min supported. See b/292047939. + // For not we just ignore the render ranges. + frameRateModes = createFrameRateModes(*policy, filterModes, {}); + } LOG_ALWAYS_FATAL_IF(frameRateModes.empty(), - "No matching frame rate modes for %s range. policy: %s", rangeName, - policy->toString().c_str()); + "No matching frame rate modes for %s range even after ignoring the " + "render range. policy: %s", + rangeName, policy->toString().c_str()); const auto stringifyModes = [&] { std::string str; diff --git a/services/surfaceflinger/Scheduler/RefreshRateSelector.h b/services/surfaceflinger/Scheduler/RefreshRateSelector.h index 5052e6e257..b25919e7a0 100644 --- a/services/surfaceflinger/Scheduler/RefreshRateSelector.h +++ b/services/surfaceflinger/Scheduler/RefreshRateSelector.h @@ -18,6 +18,7 @@ #include <algorithm> #include <numeric> +#include <set> #include <type_traits> #include <utility> #include <variant> @@ -100,6 +101,11 @@ public: } bool operator!=(const Policy& other) const { return !(*this == other); } + + bool primaryRangeIsSingleRate() const { + return isApproxEqual(primaryRanges.physical.min, primaryRanges.physical.max); + } + std::string toString() const; }; @@ -467,8 +473,8 @@ private: } std::vector<FrameRateMode> createFrameRateModes( - std::function<bool(const DisplayMode&)>&& filterModes, const FpsRange&) const - REQUIRES(mLock); + const Policy&, std::function<bool(const DisplayMode&)>&& filterModes, + const FpsRange&) const REQUIRES(mLock); // The display modes of the active display. The DisplayModeIterators below are pointers into // this container, so must be invalidated whenever the DisplayModes change. The Policy below @@ -500,6 +506,12 @@ private: const std::vector<Fps> mKnownFrameRates; const Config mConfig; + + // A list of known frame rates that favors at least 60Hz if there is no exact match display + // refresh rate + const std::vector<Fps> mFrameRatesThatFavorsAtLeast60 = {23.976_Hz, 25_Hz, 29.97_Hz, 50_Hz, + 59.94_Hz}; + Config::FrameRateOverride mFrameRateOverrideConfig; struct GetRankedFrameRatesCache { diff --git a/services/surfaceflinger/Scheduler/Scheduler.cpp b/services/surfaceflinger/Scheduler/Scheduler.cpp index 9319543666..68927bd806 100644 --- a/services/surfaceflinger/Scheduler/Scheduler.cpp +++ b/services/surfaceflinger/Scheduler/Scheduler.cpp @@ -25,12 +25,14 @@ #include <android/hardware/configstore/1.0/ISurfaceFlingerConfigs.h> #include <android/hardware/configstore/1.1/ISurfaceFlingerConfigs.h> #include <configstore/Utils.h> +#include <ftl/concat.h> #include <ftl/enum.h> #include <ftl/fake_guard.h> #include <ftl/small_map.h> #include <gui/TraceUtils.h> #include <gui/WindowInfo.h> #include <system/window.h> +#include <ui/DisplayMap.h> #include <utils/Timers.h> #include <FrameTimeline/FrameTimeline.h> @@ -44,7 +46,6 @@ #include <numeric> #include "../Layer.h" -#include "Display/DisplayMap.h" #include "EventThread.h" #include "FrameRateOverrideMappings.h" #include "FrontEnd/LayerHandle.h" @@ -114,8 +115,12 @@ void Scheduler::setPacesetterDisplay(std::optional<PhysicalDisplayId> pacesetter } void Scheduler::registerDisplay(PhysicalDisplayId displayId, RefreshRateSelectorPtr selectorPtr) { - registerDisplayInternal(displayId, std::move(selectorPtr), - std::make_shared<VsyncSchedule>(displayId, mFeatures)); + auto schedulePtr = std::make_shared<VsyncSchedule>(displayId, mFeatures, + [this](PhysicalDisplayId id, bool enable) { + onHardwareVsyncRequest(id, enable); + }); + + registerDisplayInternal(displayId, std::move(selectorPtr), std::move(schedulePtr)); } void Scheduler::registerDisplayInternal(PhysicalDisplayId displayId, @@ -123,14 +128,22 @@ void Scheduler::registerDisplayInternal(PhysicalDisplayId displayId, VsyncSchedulePtr schedulePtr) { demotePacesetterDisplay(); - std::shared_ptr<VsyncSchedule> pacesetterVsyncSchedule; - { + auto [pacesetterVsyncSchedule, isNew] = [&]() FTL_FAKE_GUARD(kMainThreadContext) { std::scoped_lock lock(mDisplayLock); - mDisplays.emplace_or_replace(displayId, std::move(selectorPtr), std::move(schedulePtr)); + const bool isNew = mDisplays + .emplace_or_replace(displayId, displayId, std::move(selectorPtr), + std::move(schedulePtr), mFeatures) + .second; + + return std::make_pair(promotePacesetterDisplayLocked(), isNew); + }(); - pacesetterVsyncSchedule = promotePacesetterDisplayLocked(); - } applyNewVsyncSchedule(std::move(pacesetterVsyncSchedule)); + + // Disable hardware VSYNC if the registration is new, as opposed to a renewal. + if (isNew) { + onHardwareVsyncRequest(displayId, false); + } } void Scheduler::unregisterDisplay(PhysicalDisplayId displayId) { @@ -159,14 +172,53 @@ void Scheduler::run() { void Scheduler::onFrameSignal(ICompositor& compositor, VsyncId vsyncId, TimePoint expectedVsyncTime) { - const TimePoint frameTime = SchedulerClock::now(); + const FrameTargeter::BeginFrameArgs beginFrameArgs = + {.frameBeginTime = SchedulerClock::now(), + .vsyncId = vsyncId, + // TODO(b/255601557): Calculate per display. + .expectedVsyncTime = expectedVsyncTime, + .sfWorkDuration = mVsyncModulator->getVsyncConfig().sfWorkDuration}; - if (!compositor.commit(frameTime, vsyncId, expectedVsyncTime)) { - return; + LOG_ALWAYS_FATAL_IF(!mPacesetterDisplayId); + const auto pacesetterId = *mPacesetterDisplayId; + const auto pacesetterOpt = mDisplays.get(pacesetterId); + + FrameTargeter& pacesetterTargeter = *pacesetterOpt->get().targeterPtr; + pacesetterTargeter.beginFrame(beginFrameArgs, *pacesetterOpt->get().schedulePtr); + + FrameTargets targets; + targets.try_emplace(pacesetterId, &pacesetterTargeter.target()); + + for (const auto& [id, display] : mDisplays) { + if (id == pacesetterId) continue; + + const FrameTargeter& targeter = *display.targeterPtr; + targets.try_emplace(id, &targeter.target()); + } + + if (!compositor.commit(pacesetterId, targets)) return; + + // TODO(b/256196556): Choose the frontrunner display. + FrameTargeters targeters; + targeters.try_emplace(pacesetterId, &pacesetterTargeter); + + for (auto& [id, display] : mDisplays) { + if (id == pacesetterId) continue; + + FrameTargeter& targeter = *display.targeterPtr; + targeter.beginFrame(beginFrameArgs, *display.schedulePtr); + + targeters.try_emplace(id, &targeter); } - compositor.composite(frameTime, vsyncId); + const auto resultsPerDisplay = compositor.composite(pacesetterId, targeters); compositor.sample(); + + for (const auto& [id, targeter] : targeters) { + const auto resultOpt = resultsPerDisplay.get(id); + LOG_ALWAYS_FATAL_IF(!resultOpt); + targeter->endFrame(*resultOpt); + } } std::optional<Fps> Scheduler::getFrameRateOverride(uid_t uid) const { @@ -176,23 +228,23 @@ std::optional<Fps> Scheduler::getFrameRateOverride(uid_t uid) const { .getFrameRateOverrideForUid(uid, supportsFrameRateOverrideByContent); } -bool Scheduler::isVsyncValid(TimePoint expectedVsyncTimestamp, uid_t uid) const { +bool Scheduler::isVsyncValid(TimePoint expectedVsyncTime, uid_t uid) const { const auto frameRate = getFrameRateOverride(uid); if (!frameRate.has_value()) { return true; } ATRACE_FORMAT("%s uid: %d frameRate: %s", __func__, uid, to_string(*frameRate).c_str()); - return getVsyncSchedule()->getTracker().isVSyncInPhase(expectedVsyncTimestamp.ns(), *frameRate); + return getVsyncSchedule()->getTracker().isVSyncInPhase(expectedVsyncTime.ns(), *frameRate); } -bool Scheduler::isVsyncInPhase(TimePoint timePoint, const Fps frameRate) const { - return getVsyncSchedule()->getTracker().isVSyncInPhase(timePoint.ns(), frameRate); +bool Scheduler::isVsyncInPhase(TimePoint expectedVsyncTime, Fps frameRate) const { + return getVsyncSchedule()->getTracker().isVSyncInPhase(expectedVsyncTime.ns(), frameRate); } impl::EventThread::ThrottleVsyncCallback Scheduler::makeThrottleVsyncCallback() const { - return [this](nsecs_t expectedVsyncTimestamp, uid_t uid) { - return !isVsyncValid(TimePoint::fromNs(expectedVsyncTimestamp), uid); + return [this](nsecs_t expectedVsyncTime, uid_t uid) { + return !isVsyncValid(TimePoint::fromNs(expectedVsyncTime), uid); }; } @@ -407,13 +459,13 @@ void Scheduler::setVsyncConfig(const VsyncConfig& config, Period vsyncPeriod) { void Scheduler::enableHardwareVsync(PhysicalDisplayId id) { auto schedule = getVsyncSchedule(id); LOG_ALWAYS_FATAL_IF(!schedule); - schedule->enableHardwareVsync(mSchedulerCallback); + schedule->enableHardwareVsync(); } void Scheduler::disableHardwareVsync(PhysicalDisplayId id, bool disallow) { auto schedule = getVsyncSchedule(id); LOG_ALWAYS_FATAL_IF(!schedule); - schedule->disableHardwareVsync(mSchedulerCallback, disallow); + schedule->disableHardwareVsync(disallow); } void Scheduler::resyncAllToHardwareVsync(bool allowToEnable) { @@ -440,12 +492,32 @@ void Scheduler::resyncToHardwareVsyncLocked(PhysicalDisplayId id, bool allowToEn refreshRate = display.selectorPtr->getActiveMode().modePtr->getFps(); } if (refreshRate->isValid()) { - display.schedulePtr->startPeriodTransition(mSchedulerCallback, refreshRate->getPeriod(), - false /* force */); + constexpr bool kForce = false; + display.schedulePtr->startPeriodTransition(refreshRate->getPeriod(), kForce); } } } +void Scheduler::onHardwareVsyncRequest(PhysicalDisplayId id, bool enabled) { + static const auto& whence = __func__; + ATRACE_NAME(ftl::Concat(whence, ' ', id.value, ' ', enabled).c_str()); + + // On main thread to serialize reads/writes of pending hardware VSYNC state. + static_cast<void>( + schedule([=]() FTL_FAKE_GUARD(mDisplayLock) FTL_FAKE_GUARD(kMainThreadContext) { + ATRACE_NAME(ftl::Concat(whence, ' ', id.value, ' ', enabled).c_str()); + + if (const auto displayOpt = mDisplays.get(id)) { + auto& display = displayOpt->get(); + display.schedulePtr->setPendingHardwareVsyncState(enabled); + + if (display.powerMode != hal::PowerMode::OFF) { + mSchedulerCallback.requestHardwareVsync(id, enabled); + } + } + })); +} + void Scheduler::setRenderRate(PhysicalDisplayId id, Fps renderFrameRate) { std::scoped_lock lock(mDisplayLock); ftl::FakeGuard guard(kMainThreadContext); @@ -491,18 +563,25 @@ bool Scheduler::addResyncSample(PhysicalDisplayId id, nsecs_t timestamp, ALOGW("%s: Invalid display %s!", __func__, to_string(id).c_str()); return false; } - return schedule->addResyncSample(mSchedulerCallback, TimePoint::fromNs(timestamp), - hwcVsyncPeriod); + return schedule->addResyncSample(TimePoint::fromNs(timestamp), hwcVsyncPeriod); } void Scheduler::addPresentFence(PhysicalDisplayId id, std::shared_ptr<FenceTime> fence) { - auto schedule = getVsyncSchedule(id); - LOG_ALWAYS_FATAL_IF(!schedule); - const bool needMoreSignals = schedule->getController().addPresentFence(std::move(fence)); - if (needMoreSignals) { - schedule->enableHardwareVsync(mSchedulerCallback); + const auto scheduleOpt = + (ftl::FakeGuard(mDisplayLock), mDisplays.get(id)).and_then([](const Display& display) { + return display.powerMode == hal::PowerMode::OFF + ? std::nullopt + : std::make_optional(display.schedulePtr); + }); + + if (!scheduleOpt) return; + const auto& schedule = scheduleOpt->get(); + + if (const bool needMoreSignals = schedule->getController().addPresentFence(std::move(fence))) { + schedule->enableHardwareVsync(); } else { - schedule->disableHardwareVsync(mSchedulerCallback, false /* disallow */); + constexpr bool kDisallow = false; + schedule->disableHardwareVsync(kDisallow); } } @@ -566,9 +645,13 @@ void Scheduler::setDisplayPowerMode(PhysicalDisplayId id, hal::PowerMode powerMo } { std::scoped_lock lock(mDisplayLock); - auto vsyncSchedule = getVsyncScheduleLocked(id); - LOG_ALWAYS_FATAL_IF(!vsyncSchedule); - vsyncSchedule->getController().setDisplayPowerMode(powerMode); + + const auto displayOpt = mDisplays.get(id); + LOG_ALWAYS_FATAL_IF(!displayOpt); + auto& display = displayOpt->get(); + + display.powerMode = powerMode; + display.schedulePtr->getController().setDisplayPowerMode(powerMode); } if (!isPacesetter) return; @@ -626,7 +709,7 @@ void Scheduler::kernelIdleTimerCallback(TimerState state) { ftl::FakeGuard guard(kMainThreadContext); for (const auto& [_, display] : mDisplays) { constexpr bool kDisallow = false; - display.schedulePtr->disableHardwareVsync(mSchedulerCallback, kDisallow); + display.schedulePtr->disableHardwareVsync(kDisallow); } } @@ -681,6 +764,24 @@ void Scheduler::dump(utils::Dumper& dumper) const { mFrameRateOverrideMappings.dump(dumper); dumper.eol(); + + { + utils::Dumper::Section section(dumper, "Frame Targeting"sv); + + std::scoped_lock lock(mDisplayLock); + ftl::FakeGuard guard(kMainThreadContext); + + for (const auto& [id, display] : mDisplays) { + utils::Dumper::Section + section(dumper, + id == mPacesetterDisplayId + ? ftl::Concat("Pacesetter Display ", id.value).c_str() + : ftl::Concat("Follower Display ", id.value).c_str()); + + display.targeterPtr->dump(dumper); + dumper.eol(); + } + } } void Scheduler::dumpVsync(std::string& out) const { @@ -745,8 +846,8 @@ std::shared_ptr<VsyncSchedule> Scheduler::promotePacesetterDisplayLocked( newVsyncSchedulePtr = pacesetter.schedulePtr; const Fps refreshRate = pacesetter.selectorPtr->getActiveMode().modePtr->getFps(); - newVsyncSchedulePtr->startPeriodTransition(mSchedulerCallback, refreshRate.getPeriod(), - true /* force */); + constexpr bool kForce = true; + newVsyncSchedulePtr->startPeriodTransition(refreshRate.getPeriod(), kForce); } return newVsyncSchedulePtr; } @@ -846,7 +947,7 @@ auto Scheduler::chooseDisplayModes() const -> DisplayModeChoiceMap { ATRACE_CALL(); using RankedRefreshRates = RefreshRateSelector::RankedFrameRates; - display::PhysicalDisplayVector<RankedRefreshRates> perDisplayRanking; + ui::PhysicalDisplayVector<RankedRefreshRates> perDisplayRanking; const auto globalSignals = makeGlobalSignals(); Fps pacesetterFps; @@ -947,4 +1048,20 @@ void Scheduler::setPreferredRefreshRateForUid(FrameRateOverride frameRateOverrid mFrameRateOverrideMappings.setPreferredRefreshRateForUid(frameRateOverride); } +void Scheduler::updateSmallAreaDetection( + std::vector<std::pair<uid_t, float>>& uidThresholdMappings) { + mSmallAreaDetectionAllowMappings.update(uidThresholdMappings); +} + +void Scheduler::setSmallAreaDetectionThreshold(uid_t uid, float threshold) { + mSmallAreaDetectionAllowMappings.setThesholdForUid(uid, threshold); +} + +bool Scheduler::isSmallDirtyArea(uid_t uid, uint32_t dirtyArea) { + std::optional<float> oThreshold = mSmallAreaDetectionAllowMappings.getThresholdForUid(uid); + if (oThreshold) return mLayerHistory.isSmallDirtyArea(dirtyArea, oThreshold.value()); + + return false; +} + } // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/Scheduler.h b/services/surfaceflinger/Scheduler/Scheduler.h index f13c878b67..20cc77f834 100644 --- a/services/surfaceflinger/Scheduler/Scheduler.h +++ b/services/surfaceflinger/Scheduler/Scheduler.h @@ -35,11 +35,12 @@ #include <ftl/fake_guard.h> #include <ftl/optional.h> #include <scheduler/Features.h> +#include <scheduler/FrameTargeter.h> #include <scheduler/Time.h> #include <scheduler/VsyncConfig.h> #include <ui/DisplayId.h> +#include <ui/DisplayMap.h> -#include "Display/DisplayMap.h" #include "Display/DisplayModeRequest.h" #include "EventThread.h" #include "FrameRateOverrideMappings.h" @@ -48,6 +49,7 @@ #include "MessageQueue.h" #include "OneShotTimer.h" #include "RefreshRateSelector.h" +#include "SmallAreaDetectionAllowMappings.h" #include "Utils/Dumper.h" #include "VsyncModulator.h" @@ -219,7 +221,7 @@ public: // otherwise. bool addResyncSample(PhysicalDisplayId, nsecs_t timestamp, std::optional<nsecs_t> hwcVsyncPeriod); - void addPresentFence(PhysicalDisplayId, std::shared_ptr<FenceTime>) EXCLUDES(mDisplayLock) + void addPresentFence(PhysicalDisplayId, std::shared_ptr<FenceTime>) REQUIRES(kMainThreadContext); // Layers are registered on creation, and unregistered when the weak reference expires. @@ -249,9 +251,18 @@ public: return std::const_pointer_cast<VsyncSchedule>(std::as_const(*this).getVsyncSchedule(idOpt)); } + TimePoint expectedPresentTimeForPacesetter() const EXCLUDES(mDisplayLock) { + std::scoped_lock lock(mDisplayLock); + return pacesetterDisplayLocked() + .transform([](const Display& display) { + return display.targeterPtr->target().expectedPresentTime(); + }) + .value_or(TimePoint()); + } + // Returns true if a given vsync timestamp is considered valid vsync // for a given uid - bool isVsyncValid(TimePoint expectedVsyncTimestamp, uid_t uid) const; + bool isVsyncValid(TimePoint expectedVsyncTime, uid_t uid) const; bool isVsyncInPhase(TimePoint expectedVsyncTime, Fps frameRate) const; @@ -279,6 +290,13 @@ public: void setGameModeRefreshRateForUid(FrameRateOverride); + void updateSmallAreaDetection(std::vector<std::pair<uid_t, float>>& uidThresholdMappings); + + void setSmallAreaDetectionThreshold(uid_t uid, float threshold); + + // Returns true if the dirty area is less than threshold. + bool isSmallDirtyArea(uid_t uid, uint32_t dirtyArea); + // Retrieves the overridden refresh rate for a given uid. std::optional<Fps> getFrameRateOverride(uid_t) const EXCLUDES(mDisplayLock); @@ -295,6 +313,11 @@ public: return mLayerHistory.getLayerFramerate(now, id); } + // Returns true if the small dirty detection is enabled. + bool supportSmallDirtyDetection() const { + return mFeatures.test(Feature::kSmallDirtyContentDetection); + } + private: friend class TestableScheduler; @@ -303,7 +326,8 @@ private: enum class TouchState { Inactive, Active }; // impl::MessageQueue overrides: - void onFrameSignal(ICompositor&, VsyncId, TimePoint expectedVsyncTime) override; + void onFrameSignal(ICompositor&, VsyncId, TimePoint expectedVsyncTime) override + REQUIRES(kMainThreadContext, mDisplayLock); // Create a connection on the given EventThread. ConnectionHandle createConnection(std::unique_ptr<EventThread>); @@ -317,6 +341,9 @@ private: void touchTimerCallback(TimerState); void displayPowerTimerCallback(TimerState); + // VsyncSchedule delegate. + void onHardwareVsyncRequest(PhysicalDisplayId, bool enable); + void resyncToHardwareVsyncLocked(PhysicalDisplayId, bool allowToEnable, std::optional<Fps> refreshRate = std::nullopt) REQUIRES(kMainThreadContext, mDisplayLock); @@ -371,7 +398,7 @@ private: } }; - using DisplayModeChoiceMap = display::PhysicalDisplayMap<PhysicalDisplayId, DisplayModeChoice>; + using DisplayModeChoiceMap = ui::PhysicalDisplayMap<PhysicalDisplayId, DisplayModeChoice>; // See mDisplayLock for thread safety. DisplayModeChoiceMap chooseDisplayModes() const @@ -423,19 +450,32 @@ private: // must lock for writes but not reads. See also mPolicyLock for locking order. mutable std::mutex mDisplayLock; + using FrameTargeterPtr = std::unique_ptr<FrameTargeter>; + struct Display { - Display(RefreshRateSelectorPtr selectorPtr, VsyncSchedulePtr schedulePtr) - : selectorPtr(std::move(selectorPtr)), schedulePtr(std::move(schedulePtr)) {} + Display(PhysicalDisplayId displayId, RefreshRateSelectorPtr selectorPtr, + VsyncSchedulePtr schedulePtr, FeatureFlags features) + : displayId(displayId), + selectorPtr(std::move(selectorPtr)), + schedulePtr(std::move(schedulePtr)), + targeterPtr(std::make_unique< + FrameTargeter>(displayId, + features.test(Feature::kBackpressureGpuComposition))) {} + + const PhysicalDisplayId displayId; // Effectively const except in move constructor. RefreshRateSelectorPtr selectorPtr; VsyncSchedulePtr schedulePtr; + FrameTargeterPtr targeterPtr; + + hal::PowerMode powerMode = hal::PowerMode::OFF; }; using DisplayRef = std::reference_wrapper<Display>; using ConstDisplayRef = std::reference_wrapper<const Display>; - display::PhysicalDisplayMap<PhysicalDisplayId, Display> mDisplays GUARDED_BY(mDisplayLock) + ui::PhysicalDisplayMap<PhysicalDisplayId, Display> mDisplays GUARDED_BY(mDisplayLock) GUARDED_BY(kMainThreadContext); ftl::Optional<PhysicalDisplayId> mPacesetterDisplayId GUARDED_BY(mDisplayLock) @@ -502,6 +542,7 @@ private: static constexpr std::chrono::nanoseconds MAX_VSYNC_APPLIED_TIME = 200ms; FrameRateOverrideMappings mFrameRateOverrideMappings; + SmallAreaDetectionAllowMappings mSmallAreaDetectionAllowMappings; }; } // namespace scheduler diff --git a/services/surfaceflinger/Scheduler/SmallAreaDetectionAllowMappings.cpp b/services/surfaceflinger/Scheduler/SmallAreaDetectionAllowMappings.cpp new file mode 100644 index 0000000000..95cd5d199a --- /dev/null +++ b/services/surfaceflinger/Scheduler/SmallAreaDetectionAllowMappings.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <sys/types.h> + +#include "SmallAreaDetectionAllowMappings.h" + +namespace android::scheduler { +void SmallAreaDetectionAllowMappings::update( + std::vector<std::pair<uid_t, float>>& uidThresholdMappings) { + std::lock_guard lock(mLock); + mMap.clear(); + for (std::pair<uid_t, float> row : uidThresholdMappings) { + if (!isValidThreshold(row.second)) continue; + + mMap.emplace(row.first, row.second); + } +} + +void SmallAreaDetectionAllowMappings::setThesholdForUid(uid_t uid, float threshold) { + if (!isValidThreshold(threshold)) return; + + std::lock_guard lock(mLock); + mMap.emplace(uid, threshold); +} + +std::optional<float> SmallAreaDetectionAllowMappings::getThresholdForUid(uid_t uid) { + std::lock_guard lock(mLock); + const auto iter = mMap.find(uid); + if (iter != mMap.end()) { + return iter->second; + } + return std::nullopt; +} +} // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/SmallAreaDetectionAllowMappings.h b/services/surfaceflinger/Scheduler/SmallAreaDetectionAllowMappings.h new file mode 100644 index 0000000000..cbab69091f --- /dev/null +++ b/services/surfaceflinger/Scheduler/SmallAreaDetectionAllowMappings.h @@ -0,0 +1,39 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <android-base/thread_annotations.h> +#include <sys/types.h> +#include <optional> +#include <unordered_map> +#include <vector> + +namespace android::scheduler { +class SmallAreaDetectionAllowMappings { + using UidThresholdMap = std::unordered_map<uid_t, float>; + +public: + void update(std::vector<std::pair<uid_t, float>>& uidThresholdMappings); + void setThesholdForUid(uid_t uid, float threshold) EXCLUDES(mLock); + std::optional<float> getThresholdForUid(uid_t uid) EXCLUDES(mLock); + +private: + static bool isValidThreshold(float threshold) { return threshold >= 0.0f && threshold <= 1.0f; } + mutable std::mutex mLock; + UidThresholdMap mMap GUARDED_BY(mLock); +}; +} // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/VsyncSchedule.cpp b/services/surfaceflinger/Scheduler/VsyncSchedule.cpp index 84671aea0d..ff3f29dbbf 100644 --- a/services/surfaceflinger/Scheduler/VsyncSchedule.cpp +++ b/services/surfaceflinger/Scheduler/VsyncSchedule.cpp @@ -22,7 +22,6 @@ #include "VsyncSchedule.h" -#include "ISchedulerCallback.h" #include "Utils/Dumper.h" #include "VSyncDispatchTimerQueue.h" #include "VSyncPredictor.h" @@ -54,8 +53,10 @@ private: VSyncCallbackRegistration mRegistration; }; -VsyncSchedule::VsyncSchedule(PhysicalDisplayId id, FeatureFlags features) +VsyncSchedule::VsyncSchedule(PhysicalDisplayId id, FeatureFlags features, + RequestHardwareVsync requestHardwareVsync) : mId(id), + mRequestHardwareVsync(std::move(requestHardwareVsync)), mTracker(createTracker(id)), mDispatch(createDispatch(mTracker)), mController(createController(id, *mTracker, features)), @@ -64,8 +65,9 @@ VsyncSchedule::VsyncSchedule(PhysicalDisplayId id, FeatureFlags features) : nullptr) {} VsyncSchedule::VsyncSchedule(PhysicalDisplayId id, TrackerPtr tracker, DispatchPtr dispatch, - ControllerPtr controller) + ControllerPtr controller, RequestHardwareVsync requestHardwareVsync) : mId(id), + mRequestHardwareVsync(std::move(requestHardwareVsync)), mTracker(std::move(tracker)), mDispatch(std::move(dispatch)), mController(std::move(controller)) {} @@ -135,14 +137,13 @@ VsyncSchedule::ControllerPtr VsyncSchedule::createController(PhysicalDisplayId i return reactor; } -void VsyncSchedule::startPeriodTransition(ISchedulerCallback& callback, Period period, bool force) { +void VsyncSchedule::startPeriodTransition(Period period, bool force) { std::lock_guard<std::mutex> lock(mHwVsyncLock); mController->startPeriodTransition(period.ns(), force); - enableHardwareVsyncLocked(callback); + enableHardwareVsyncLocked(); } -bool VsyncSchedule::addResyncSample(ISchedulerCallback& callback, TimePoint timestamp, - ftl::Optional<Period> hwcVsyncPeriod) { +bool VsyncSchedule::addResyncSample(TimePoint timestamp, ftl::Optional<Period> hwcVsyncPeriod) { bool needsHwVsync = false; bool periodFlushed = false; { @@ -154,31 +155,32 @@ bool VsyncSchedule::addResyncSample(ISchedulerCallback& callback, TimePoint time } } if (needsHwVsync) { - enableHardwareVsync(callback); + enableHardwareVsync(); } else { - disableHardwareVsync(callback, false /* disallow */); + constexpr bool kDisallow = false; + disableHardwareVsync(kDisallow); } return periodFlushed; } -void VsyncSchedule::enableHardwareVsync(ISchedulerCallback& callback) { +void VsyncSchedule::enableHardwareVsync() { std::lock_guard<std::mutex> lock(mHwVsyncLock); - enableHardwareVsyncLocked(callback); + enableHardwareVsyncLocked(); } -void VsyncSchedule::enableHardwareVsyncLocked(ISchedulerCallback& callback) { +void VsyncSchedule::enableHardwareVsyncLocked() { if (mHwVsyncState == HwVsyncState::Disabled) { getTracker().resetModel(); - callback.setVsyncEnabled(mId, true); + mRequestHardwareVsync(mId, true); mHwVsyncState = HwVsyncState::Enabled; } } -void VsyncSchedule::disableHardwareVsync(ISchedulerCallback& callback, bool disallow) { +void VsyncSchedule::disableHardwareVsync(bool disallow) { std::lock_guard<std::mutex> lock(mHwVsyncLock); switch (mHwVsyncState) { case HwVsyncState::Enabled: - callback.setVsyncEnabled(mId, false); + mRequestHardwareVsync(mId, false); [[fallthrough]]; case HwVsyncState::Disabled: mHwVsyncState = disallow ? HwVsyncState::Disallowed : HwVsyncState::Disabled; diff --git a/services/surfaceflinger/Scheduler/VsyncSchedule.h b/services/surfaceflinger/Scheduler/VsyncSchedule.h index 763d058e28..0757b5789d 100644 --- a/services/surfaceflinger/Scheduler/VsyncSchedule.h +++ b/services/surfaceflinger/Scheduler/VsyncSchedule.h @@ -16,16 +16,20 @@ #pragma once +#include <functional> #include <memory> #include <string> -#include <ThreadContext.h> #include <android-base/thread_annotations.h> #include <ftl/enum.h> #include <ftl/optional.h> +#include <ui/DisplayId.h> + #include <scheduler/Features.h> +#include <scheduler/IVsyncSource.h> #include <scheduler/Time.h> -#include <ui/DisplayId.h> + +#include "ThreadContext.h" namespace android { class EventThreadTest; @@ -38,8 +42,6 @@ class SchedulerFuzzer; namespace android::scheduler { -struct ISchedulerCallback; - // TODO(b/185535769): Rename classes, and remove aliases. class VSyncDispatch; class VSyncTracker; @@ -49,13 +51,16 @@ using VsyncDispatch = VSyncDispatch; using VsyncTracker = VSyncTracker; // Schedule that synchronizes to hardware VSYNC of a physical display. -class VsyncSchedule { +class VsyncSchedule final : public IVsyncSource { public: - VsyncSchedule(PhysicalDisplayId, FeatureFlags); + using RequestHardwareVsync = std::function<void(PhysicalDisplayId, bool enabled)>; + + VsyncSchedule(PhysicalDisplayId, FeatureFlags, RequestHardwareVsync); ~VsyncSchedule(); - Period period() const; - TimePoint vsyncDeadlineAfter(TimePoint) const; + // IVsyncSource overrides: + Period period() const override; + TimePoint vsyncDeadlineAfter(TimePoint) const override; // Inform the schedule that the period is changing and the schedule needs to recalibrate // itself. The schedule will end the period transition internally. This will @@ -64,13 +69,12 @@ public: // \param [in] period The period that the system is changing into. // \param [in] force True to force a transition even if it is not a // change. - void startPeriodTransition(ISchedulerCallback&, Period period, bool force); + void startPeriodTransition(Period period, bool force); // Pass a VSYNC sample to VsyncController. Return true if // VsyncController detected that the VSYNC period changed. Enable or disable // hardware VSYNCs depending on whether more samples are needed. - bool addResyncSample(ISchedulerCallback&, TimePoint timestamp, - ftl::Optional<Period> hwcVsyncPeriod); + bool addResyncSample(TimePoint timestamp, ftl::Optional<Period> hwcVsyncPeriod); // TODO(b/185535769): Hide behind API. const VsyncTracker& getTracker() const { return *mTracker; } @@ -89,12 +93,12 @@ public: // Turn on hardware VSYNCs, unless mHwVsyncState is Disallowed, in which // case this call is ignored. - void enableHardwareVsync(ISchedulerCallback&) EXCLUDES(mHwVsyncLock); + void enableHardwareVsync() EXCLUDES(mHwVsyncLock); // Disable hardware VSYNCs. If `disallow` is true, future calls to // enableHardwareVsync are ineffective until isHardwareVsyncAllowed is // called with `makeAllowed` set to true. - void disableHardwareVsync(ISchedulerCallback&, bool disallow) EXCLUDES(mHwVsyncLock); + void disableHardwareVsync(bool disallow) EXCLUDES(mHwVsyncLock); // If true, enableHardwareVsync can enable hardware VSYNC (if not already // enabled). If false, enableHardwareVsync does nothing. @@ -107,8 +111,11 @@ public: protected: using ControllerPtr = std::unique_ptr<VsyncController>; + static void NoOpRequestHardwareVsync(PhysicalDisplayId, bool) {} + // For tests. - VsyncSchedule(PhysicalDisplayId, TrackerPtr, DispatchPtr, ControllerPtr); + VsyncSchedule(PhysicalDisplayId, TrackerPtr, DispatchPtr, ControllerPtr, + RequestHardwareVsync = NoOpRequestHardwareVsync); private: friend class TestableScheduler; @@ -120,7 +127,7 @@ private: static DispatchPtr createDispatch(TrackerPtr); static ControllerPtr createController(PhysicalDisplayId, VsyncTracker&, FeatureFlags); - void enableHardwareVsyncLocked(ISchedulerCallback&) REQUIRES(mHwVsyncLock); + void enableHardwareVsyncLocked() REQUIRES(mHwVsyncLock); mutable std::mutex mHwVsyncLock; enum class HwVsyncState { @@ -147,6 +154,7 @@ private: using TracerPtr = std::unique_ptr<PredictedVsyncTracer>; const PhysicalDisplayId mId; + const RequestHardwareVsync mRequestHardwareVsync; const TrackerPtr mTracker; const DispatchPtr mDispatch; const ControllerPtr mController; diff --git a/services/surfaceflinger/Scheduler/include/scheduler/Features.h b/services/surfaceflinger/Scheduler/include/scheduler/Features.h index b3a6a606c3..7c72ac6afc 100644 --- a/services/surfaceflinger/Scheduler/include/scheduler/Features.h +++ b/services/surfaceflinger/Scheduler/include/scheduler/Features.h @@ -23,10 +23,12 @@ namespace android::scheduler { enum class Feature : std::uint8_t { - kPresentFences = 0b1, - kKernelIdleTimer = 0b10, - kContentDetection = 0b100, - kTracePredictedVsync = 0b1000, + kPresentFences = 1 << 0, + kKernelIdleTimer = 1 << 1, + kContentDetection = 1 << 2, + kTracePredictedVsync = 1 << 3, + kBackpressureGpuComposition = 1 << 4, + kSmallDirtyContentDetection = 1 << 5, }; using FeatureFlags = ftl::Flags<Feature>; diff --git a/services/surfaceflinger/Scheduler/include/scheduler/FrameTargeter.h b/services/surfaceflinger/Scheduler/include/scheduler/FrameTargeter.h new file mode 100644 index 0000000000..ae74205720 --- /dev/null +++ b/services/surfaceflinger/Scheduler/include/scheduler/FrameTargeter.h @@ -0,0 +1,150 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <array> +#include <atomic> +#include <memory> + +#include <ui/DisplayId.h> +#include <ui/Fence.h> +#include <ui/FenceTime.h> + +#include <scheduler/Time.h> +#include <scheduler/VsyncId.h> +#include <scheduler/interface/CompositeResult.h> + +// TODO(b/185536303): Pull to FTL. +#include "../../../TracedOrdinal.h" +#include "../../../Utils/Dumper.h" + +namespace android::scheduler { + +struct IVsyncSource; + +// Read-only interface to the metrics computed by FrameTargeter for the latest frame. +class FrameTarget { +public: + VsyncId vsyncId() const { return mVsyncId; } + + // The time when the frame actually began, as opposed to when it had been scheduled to begin. + TimePoint frameBeginTime() const { return mFrameBeginTime; } + + // Relative to when the frame actually began, as opposed to when it had been scheduled to begin. + Duration expectedFrameDuration() const { return mExpectedPresentTime - mFrameBeginTime; } + + TimePoint expectedPresentTime() const { return mExpectedPresentTime; } + + // The time of the VSYNC that preceded this frame. See `presentFenceForPastVsync` for details. + TimePoint pastVsyncTime(Period vsyncPeriod) const; + + // Equivalent to `pastVsyncTime` unless running N VSYNCs ahead. + TimePoint previousFrameVsyncTime(Period vsyncPeriod) const { + return mExpectedPresentTime - vsyncPeriod; + } + + // The present fence for the frame that had targeted the most recent VSYNC before this frame. + // If the target VSYNC for any given frame is more than `vsyncPeriod` in the future, then the + // VSYNC of at least one previous frame has not yet passed. In other words, this is NOT the + // `presentFenceForPreviousFrame` if running N VSYNCs ahead, but the one that should have been + // signaled by now (unless that frame missed). + const FenceTimePtr& presentFenceForPastVsync(Period vsyncPeriod) const; + + // Equivalent to `presentFenceForPastVsync` unless running N VSYNCs ahead. + const FenceTimePtr& presentFenceForPreviousFrame() const { + return mPresentFences.front().fenceTime; + } + + bool wouldPresentEarly(Period vsyncPeriod) const; + + bool isFramePending() const { return mFramePending; } + bool didMissFrame() const { return mFrameMissed; } + bool didMissHwcFrame() const { return mHwcFrameMissed && !mGpuFrameMissed; } + +protected: + explicit FrameTarget(const std::string& displayLabel); + ~FrameTarget() = default; + + VsyncId mVsyncId; + TimePoint mFrameBeginTime; + TimePoint mExpectedPresentTime; + + TracedOrdinal<bool> mFramePending; + TracedOrdinal<bool> mFrameMissed; + TracedOrdinal<bool> mHwcFrameMissed; + TracedOrdinal<bool> mGpuFrameMissed; + + struct FenceWithFenceTime { + sp<Fence> fence = Fence::NO_FENCE; + FenceTimePtr fenceTime = FenceTime::NO_FENCE; + }; + std::array<FenceWithFenceTime, 2> mPresentFences; + +private: + template <int N> + inline bool targetsVsyncsAhead(Period vsyncPeriod) const { + static_assert(N > 1); + return expectedFrameDuration() > (N - 1) * vsyncPeriod; + } +}; + +// Computes a display's per-frame metrics about past/upcoming targeting of present deadlines. +class FrameTargeter final : private FrameTarget { +public: + FrameTargeter(PhysicalDisplayId displayId, bool backpressureGpuComposition) + : FrameTarget(to_string(displayId)), + mBackpressureGpuComposition(backpressureGpuComposition) {} + + const FrameTarget& target() const { return *this; } + + struct BeginFrameArgs { + TimePoint frameBeginTime; + VsyncId vsyncId; + TimePoint expectedVsyncTime; + Duration sfWorkDuration; + }; + + void beginFrame(const BeginFrameArgs&, const IVsyncSource&); + + // TODO(b/241285191): Merge with FrameTargeter::endFrame. + FenceTimePtr setPresentFence(sp<Fence>); + + void endFrame(const CompositeResult&); + + void dump(utils::Dumper&) const; + +private: + friend class FrameTargeterTest; + + // For tests. + using IsFencePendingFuncPtr = bool (*)(const FenceTimePtr&, int graceTimeMs); + void beginFrame(const BeginFrameArgs&, const IVsyncSource&, IsFencePendingFuncPtr); + FenceTimePtr setPresentFence(sp<Fence>, FenceTimePtr); + + static bool isFencePending(const FenceTimePtr&, int graceTimeMs); + + const bool mBackpressureGpuComposition; + + TimePoint mScheduledPresentTime; + CompositionCoverageFlags mCompositionCoverage; + + std::atomic_uint mFrameMissedCount = 0; + std::atomic_uint mHwcFrameMissedCount = 0; + std::atomic_uint mGpuFrameMissedCount = 0; +}; + +} // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/include/scheduler/IVsyncSource.h b/services/surfaceflinger/Scheduler/include/scheduler/IVsyncSource.h new file mode 100644 index 0000000000..bb2de75ac4 --- /dev/null +++ b/services/surfaceflinger/Scheduler/include/scheduler/IVsyncSource.h @@ -0,0 +1,31 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <scheduler/Time.h> + +namespace android::scheduler { + +struct IVsyncSource { + virtual Period period() const = 0; + virtual TimePoint vsyncDeadlineAfter(TimePoint) const = 0; + +protected: + ~IVsyncSource() = default; +}; + +} // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/include/scheduler/VsyncId.h b/services/surfaceflinger/Scheduler/include/scheduler/VsyncId.h index c64a3cdc6f..6ca4e85041 100644 --- a/services/surfaceflinger/Scheduler/include/scheduler/VsyncId.h +++ b/services/surfaceflinger/Scheduler/include/scheduler/VsyncId.h @@ -18,17 +18,17 @@ #include <cstdint> -namespace android { +#include <ftl/mixins.h> -// TODO(b/185536303): Import StrongTyping.h into FTL so it can be used here. +namespace android { // Sequential frame identifier, also known as FrameTimeline token. -struct VsyncId { - int64_t value = -1; +// +// TODO(b/241285191): Pull to <gui/FrameTimelineInfo.h> and use VsyncId over int64_t everywhere. +struct VsyncId : ftl::DefaultConstructible<VsyncId, int64_t, -1>, + ftl::Incrementable<VsyncId>, + ftl::Equatable<VsyncId> { + using DefaultConstructible::DefaultConstructible; }; -inline bool operator==(VsyncId lhs, VsyncId rhs) { - return lhs.value == rhs.value; -} - } // namespace android diff --git a/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositeResult.h b/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositeResult.h new file mode 100644 index 0000000000..87c704ece3 --- /dev/null +++ b/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositeResult.h @@ -0,0 +1,32 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <ui/DisplayId.h> +#include <ui/DisplayMap.h> + +#include <scheduler/interface/CompositionCoverage.h> + +namespace android { + +struct CompositeResult { + CompositionCoverageFlags compositionCoverage; +}; + +using CompositeResultsPerDisplay = ui::PhysicalDisplayMap<PhysicalDisplayId, CompositeResult>; + +} // namespace android diff --git a/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositionCoverage.h b/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositionCoverage.h index 3d0f1a9d33..767462dfce 100644 --- a/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositionCoverage.h +++ b/services/surfaceflinger/Scheduler/include/scheduler/interface/CompositionCoverage.h @@ -19,6 +19,8 @@ #include <cstdint> #include <ftl/flags.h> +#include <ui/DisplayId.h> +#include <ui/DisplayMap.h> namespace android { @@ -34,4 +36,14 @@ enum class CompositionCoverage : std::uint8_t { using CompositionCoverageFlags = ftl::Flags<CompositionCoverage>; +using CompositionCoveragePerDisplay = ui::DisplayMap<DisplayId, CompositionCoverageFlags>; + +inline CompositionCoverageFlags multiDisplayUnion(const CompositionCoveragePerDisplay& displays) { + CompositionCoverageFlags coverage; + for (const auto& [id, flags] : displays) { + coverage |= flags; + } + return coverage; +} + } // namespace android diff --git a/services/surfaceflinger/Scheduler/include/scheduler/interface/ICompositor.h b/services/surfaceflinger/Scheduler/include/scheduler/interface/ICompositor.h index cc419259ef..12ee36e084 100644 --- a/services/surfaceflinger/Scheduler/include/scheduler/interface/ICompositor.h +++ b/services/surfaceflinger/Scheduler/include/scheduler/interface/ICompositor.h @@ -16,10 +16,23 @@ #pragma once +#include <ui/DisplayId.h> +#include <ui/DisplayMap.h> + #include <scheduler/Time.h> #include <scheduler/VsyncId.h> +#include <scheduler/interface/CompositeResult.h> namespace android { +namespace scheduler { + +class FrameTarget; +class FrameTargeter; + +using FrameTargets = ui::PhysicalDisplayMap<PhysicalDisplayId, const scheduler::FrameTarget*>; +using FrameTargeters = ui::PhysicalDisplayMap<PhysicalDisplayId, scheduler::FrameTargeter*>; + +} // namespace scheduler struct ICompositor { // Configures physical displays, processing hotplug and/or mode setting via the Composer HAL. @@ -27,11 +40,12 @@ struct ICompositor { // Commits transactions for layers and displays. Returns whether any state has been invalidated, // i.e. whether a frame should be composited for each display. - virtual bool commit(TimePoint frameTime, VsyncId, TimePoint expectedVsyncTime) = 0; + virtual bool commit(PhysicalDisplayId pacesetterId, const scheduler::FrameTargets&) = 0; // Composites a frame for each display. CompositionEngine performs GPU and/or HAL composition // via RenderEngine and the Composer HAL, respectively. - virtual void composite(TimePoint frameTime, VsyncId) = 0; + virtual CompositeResultsPerDisplay composite(PhysicalDisplayId pacesetterId, + const scheduler::FrameTargeters&) = 0; // Samples the composited frame via RegionSamplingThread. virtual void sample() = 0; diff --git a/services/surfaceflinger/Scheduler/src/FrameTargeter.cpp b/services/surfaceflinger/Scheduler/src/FrameTargeter.cpp new file mode 100644 index 0000000000..7a18654346 --- /dev/null +++ b/services/surfaceflinger/Scheduler/src/FrameTargeter.cpp @@ -0,0 +1,154 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gui/TraceUtils.h> + +#include <scheduler/FrameTargeter.h> +#include <scheduler/IVsyncSource.h> + +namespace android::scheduler { + +FrameTarget::FrameTarget(const std::string& displayLabel) + : mFramePending("PrevFramePending " + displayLabel, false), + mFrameMissed("PrevFrameMissed " + displayLabel, false), + mHwcFrameMissed("PrevHwcFrameMissed " + displayLabel, false), + mGpuFrameMissed("PrevGpuFrameMissed " + displayLabel, false) {} + +TimePoint FrameTarget::pastVsyncTime(Period vsyncPeriod) const { + // TODO(b/267315508): Generalize to N VSYNCs. + const int shift = static_cast<int>(targetsVsyncsAhead<2>(vsyncPeriod)); + return mExpectedPresentTime - Period::fromNs(vsyncPeriod.ns() << shift); +} + +const FenceTimePtr& FrameTarget::presentFenceForPastVsync(Period vsyncPeriod) const { + // TODO(b/267315508): Generalize to N VSYNCs. + const size_t i = static_cast<size_t>(targetsVsyncsAhead<2>(vsyncPeriod)); + return mPresentFences[i].fenceTime; +} + +bool FrameTarget::wouldPresentEarly(Period vsyncPeriod) const { + // TODO(b/241285475): Since this is called during `composite`, the calls to `targetsVsyncsAhead` + // should use `TimePoint::now()` in case of delays since `mFrameBeginTime`. + + // TODO(b/267315508): Generalize to N VSYNCs. + if (targetsVsyncsAhead<3>(vsyncPeriod)) { + return true; + } + + const auto fence = presentFenceForPastVsync(vsyncPeriod); + return fence->isValid() && fence->getSignalTime() != Fence::SIGNAL_TIME_PENDING; +} + +void FrameTargeter::beginFrame(const BeginFrameArgs& args, const IVsyncSource& vsyncSource) { + return beginFrame(args, vsyncSource, &FrameTargeter::isFencePending); +} + +void FrameTargeter::beginFrame(const BeginFrameArgs& args, const IVsyncSource& vsyncSource, + IsFencePendingFuncPtr isFencePendingFuncPtr) { + mVsyncId = args.vsyncId; + mFrameBeginTime = args.frameBeginTime; + + // The `expectedVsyncTime`, which was predicted when this frame was scheduled, is normally in + // the future relative to `frameBeginTime`, but may not be for delayed frames. Adjust + // `mExpectedPresentTime` accordingly, but not `mScheduledPresentTime`. + const TimePoint lastScheduledPresentTime = mScheduledPresentTime; + mScheduledPresentTime = args.expectedVsyncTime; + + const Period vsyncPeriod = vsyncSource.period(); + + // Calculate the expected present time once and use the cached value throughout this frame to + // make sure all layers are seeing this same value. + if (args.expectedVsyncTime >= args.frameBeginTime) { + mExpectedPresentTime = args.expectedVsyncTime; + } else { + mExpectedPresentTime = vsyncSource.vsyncDeadlineAfter(args.frameBeginTime); + if (args.sfWorkDuration > vsyncPeriod) { + // Inflate the expected present time if we're targeting the next VSYNC. + mExpectedPresentTime += vsyncPeriod; + } + } + + ATRACE_FORMAT("%s %" PRId64 " vsyncIn %.2fms%s", __func__, ftl::to_underlying(args.vsyncId), + ticks<std::milli, float>(mExpectedPresentTime - TimePoint::now()), + mExpectedPresentTime == args.expectedVsyncTime ? "" : " (adjusted)"); + + const FenceTimePtr& pastPresentFence = presentFenceForPastVsync(vsyncPeriod); + + // In cases where the present fence is about to fire, give it a small grace period instead of + // giving up on the frame. + // + // TODO(b/280667110): The grace period should depend on `sfWorkDuration` and `vsyncPeriod` being + // approximately equal, not whether backpressure propagation is enabled. + const int graceTimeForPresentFenceMs = static_cast<int>( + mBackpressureGpuComposition || !mCompositionCoverage.test(CompositionCoverage::Gpu)); + + // Pending frames may trigger backpressure propagation. + const auto& isFencePending = *isFencePendingFuncPtr; + mFramePending = pastPresentFence != FenceTime::NO_FENCE && + isFencePending(pastPresentFence, graceTimeForPresentFenceMs); + + // A frame is missed if the prior frame is still pending. If no longer pending, then we still + // count the frame as missed if the predicted present time was further in the past than when the + // fence actually fired. Add some slop to correct for drift. This should generally be smaller + // than a typical frame duration, but should not be so small that it reports reasonable drift as + // a missed frame. + mFrameMissed = mFramePending || [&] { + const nsecs_t pastPresentTime = pastPresentFence->getSignalTime(); + if (pastPresentTime < 0) return false; + const nsecs_t frameMissedSlop = vsyncPeriod.ns() / 2; + return lastScheduledPresentTime.ns() < pastPresentTime - frameMissedSlop; + }(); + + mHwcFrameMissed = mFrameMissed && mCompositionCoverage.test(CompositionCoverage::Hwc); + mGpuFrameMissed = mFrameMissed && mCompositionCoverage.test(CompositionCoverage::Gpu); + + if (mFrameMissed) mFrameMissedCount++; + if (mHwcFrameMissed) mHwcFrameMissedCount++; + if (mGpuFrameMissed) mGpuFrameMissedCount++; +} + +void FrameTargeter::endFrame(const CompositeResult& result) { + mCompositionCoverage = result.compositionCoverage; +} + +FenceTimePtr FrameTargeter::setPresentFence(sp<Fence> presentFence) { + auto presentFenceTime = std::make_shared<FenceTime>(presentFence); + return setPresentFence(std::move(presentFence), std::move(presentFenceTime)); +} + +FenceTimePtr FrameTargeter::setPresentFence(sp<Fence> presentFence, FenceTimePtr presentFenceTime) { + mPresentFences[1] = mPresentFences[0]; + mPresentFences[0] = {std::move(presentFence), presentFenceTime}; + return presentFenceTime; +} + +void FrameTargeter::dump(utils::Dumper& dumper) const { + // There are scripts and tests that expect this (rather than "name=value") format. + dumper.dump({}, "Total missed frame count: " + std::to_string(mFrameMissedCount)); + dumper.dump({}, "HWC missed frame count: " + std::to_string(mHwcFrameMissedCount)); + dumper.dump({}, "GPU missed frame count: " + std::to_string(mGpuFrameMissedCount)); +} + +bool FrameTargeter::isFencePending(const FenceTimePtr& fence, int graceTimeMs) { + ATRACE_CALL(); + const status_t status = fence->wait(graceTimeMs); + + // This is the same as Fence::Status::Unsignaled, but it saves a call to getStatus, + // which calls wait(0) again internally. + return status == -ETIME; +} + +} // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/tests/FrameTargeterTest.cpp b/services/surfaceflinger/Scheduler/tests/FrameTargeterTest.cpp new file mode 100644 index 0000000000..1e038d1753 --- /dev/null +++ b/services/surfaceflinger/Scheduler/tests/FrameTargeterTest.cpp @@ -0,0 +1,301 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <ftl/optional.h> +#include <gtest/gtest.h> + +#include <scheduler/Fps.h> +#include <scheduler/FrameTargeter.h> +#include <scheduler/IVsyncSource.h> + +using namespace std::chrono_literals; + +namespace android::scheduler { +namespace { + +struct VsyncSource final : IVsyncSource { + VsyncSource(Period period, TimePoint deadline) : vsyncPeriod(period), vsyncDeadline(deadline) {} + + const Period vsyncPeriod; + const TimePoint vsyncDeadline; + + Period period() const override { return vsyncPeriod; } + TimePoint vsyncDeadlineAfter(TimePoint) const override { return vsyncDeadline; } +}; + +} // namespace + +class FrameTargeterTest : public testing::Test { +public: + const auto& target() const { return mTargeter.target(); } + + struct Frame { + Frame(FrameTargeterTest* testPtr, VsyncId vsyncId, TimePoint& frameBeginTime, + Duration frameDuration, Fps refreshRate, + FrameTargeter::IsFencePendingFuncPtr isFencePendingFuncPtr = Frame::fenceSignaled, + const ftl::Optional<VsyncSource>& vsyncSourceOpt = std::nullopt) + : testPtr(testPtr), frameBeginTime(frameBeginTime), period(refreshRate.getPeriod()) { + const FrameTargeter::BeginFrameArgs args{.frameBeginTime = frameBeginTime, + .vsyncId = vsyncId, + .expectedVsyncTime = + frameBeginTime + frameDuration, + .sfWorkDuration = 10ms}; + + testPtr->mTargeter.beginFrame(args, + vsyncSourceOpt + .or_else([&] { + return std::make_optional( + VsyncSource(period, + args.expectedVsyncTime)); + }) + .value(), + isFencePendingFuncPtr); + } + + FenceTimePtr end(CompositionCoverage coverage = CompositionCoverage::Hwc) { + if (ended) return nullptr; + ended = true; + + auto [fence, fenceTime] = testPtr->mFenceMap.makePendingFenceForTest(); + testPtr->mTargeter.setPresentFence(std::move(fence), fenceTime); + + testPtr->mTargeter.endFrame({.compositionCoverage = coverage}); + return fenceTime; + } + + ~Frame() { + end(); + frameBeginTime += period; + } + + static bool fencePending(const FenceTimePtr&, int) { return true; } + static bool fenceSignaled(const FenceTimePtr&, int) { return false; } + + FrameTargeterTest* const testPtr; + + TimePoint& frameBeginTime; + const Period period; + + bool ended = false; + }; + +private: + FenceToFenceTimeMap mFenceMap; + + static constexpr bool kBackpressureGpuComposition = true; + FrameTargeter mTargeter{PhysicalDisplayId::fromPort(13), kBackpressureGpuComposition}; +}; + +TEST_F(FrameTargeterTest, targetsFrames) { + VsyncId vsyncId{42}; + { + TimePoint frameBeginTime(989ms); + const Frame frame(this, vsyncId++, frameBeginTime, 10ms, 60_Hz); + + EXPECT_EQ(target().vsyncId(), VsyncId{42}); + EXPECT_EQ(target().frameBeginTime(), TimePoint(989ms)); + EXPECT_EQ(target().expectedPresentTime(), TimePoint(999ms)); + EXPECT_EQ(target().expectedFrameDuration(), 10ms); + } + { + TimePoint frameBeginTime(1100ms); + const Frame frame(this, vsyncId++, frameBeginTime, 11ms, 60_Hz); + + EXPECT_EQ(target().vsyncId(), VsyncId{43}); + EXPECT_EQ(target().frameBeginTime(), TimePoint(1100ms)); + EXPECT_EQ(target().expectedPresentTime(), TimePoint(1111ms)); + EXPECT_EQ(target().expectedFrameDuration(), 11ms); + } +} + +TEST_F(FrameTargeterTest, inflatesExpectedPresentTime) { + // Negative such that `expectedVsyncTime` is in the past. + constexpr Duration kFrameDuration = -3ms; + TimePoint frameBeginTime(777ms); + + constexpr Fps kRefreshRate = 120_Hz; + const VsyncSource vsyncSource(kRefreshRate.getPeriod(), frameBeginTime + 5ms); + const Frame frame(this, VsyncId{123}, frameBeginTime, kFrameDuration, kRefreshRate, + Frame::fenceSignaled, vsyncSource); + + EXPECT_EQ(target().expectedPresentTime(), vsyncSource.vsyncDeadline + vsyncSource.vsyncPeriod); +} + +TEST_F(FrameTargeterTest, recallsPastVsync) { + VsyncId vsyncId{111}; + TimePoint frameBeginTime(1000ms); + constexpr Fps kRefreshRate = 60_Hz; + constexpr Period kPeriod = kRefreshRate.getPeriod(); + constexpr Duration kFrameDuration = 13ms; + + for (int n = 5; n-- > 0;) { + Frame frame(this, vsyncId++, frameBeginTime, kFrameDuration, kRefreshRate); + const auto fence = frame.end(); + + EXPECT_EQ(target().pastVsyncTime(kPeriod), frameBeginTime + kFrameDuration - kPeriod); + EXPECT_EQ(target().presentFenceForPastVsync(kPeriod), fence); + } +} + +TEST_F(FrameTargeterTest, recallsPastVsyncTwoVsyncsAhead) { + VsyncId vsyncId{222}; + TimePoint frameBeginTime(2000ms); + constexpr Fps kRefreshRate = 120_Hz; + constexpr Period kPeriod = kRefreshRate.getPeriod(); + constexpr Duration kFrameDuration = 10ms; + + FenceTimePtr previousFence = FenceTime::NO_FENCE; + + for (int n = 5; n-- > 0;) { + Frame frame(this, vsyncId++, frameBeginTime, kFrameDuration, kRefreshRate); + const auto fence = frame.end(); + + EXPECT_EQ(target().pastVsyncTime(kPeriod), frameBeginTime + kFrameDuration - 2 * kPeriod); + EXPECT_EQ(target().presentFenceForPastVsync(kPeriod), previousFence); + + previousFence = fence; + } +} + +TEST_F(FrameTargeterTest, doesNotDetectEarlyPresentIfNoFence) { + constexpr Period kPeriod = (60_Hz).getPeriod(); + EXPECT_EQ(target().presentFenceForPastVsync(kPeriod), FenceTime::NO_FENCE); + EXPECT_FALSE(target().wouldPresentEarly(kPeriod)); +} + +TEST_F(FrameTargeterTest, detectsEarlyPresent) { + VsyncId vsyncId{333}; + TimePoint frameBeginTime(3000ms); + constexpr Fps kRefreshRate = 60_Hz; + constexpr Period kPeriod = kRefreshRate.getPeriod(); + + // The target is not early while past present fences are pending. + for (int n = 3; n-- > 0;) { + const Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); + EXPECT_FALSE(target().wouldPresentEarly(kPeriod)); + } + + // The target is early if the past present fence was signaled. + Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); + const auto fence = frame.end(); + fence->signalForTest(frameBeginTime.ns()); + + EXPECT_TRUE(target().wouldPresentEarly(kPeriod)); +} + +TEST_F(FrameTargeterTest, detectsEarlyPresentTwoVsyncsAhead) { + VsyncId vsyncId{444}; + TimePoint frameBeginTime(4000ms); + constexpr Fps kRefreshRate = 120_Hz; + constexpr Period kPeriod = kRefreshRate.getPeriod(); + + // The target is not early while past present fences are pending. + for (int n = 3; n-- > 0;) { + const Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); + EXPECT_FALSE(target().wouldPresentEarly(kPeriod)); + } + + Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); + const auto fence = frame.end(); + fence->signalForTest(frameBeginTime.ns()); + + // The target is two VSYNCs ahead, so the past present fence is still pending. + EXPECT_FALSE(target().wouldPresentEarly(kPeriod)); + + { const Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); } + + // The target is early if the past present fence was signaled. + EXPECT_TRUE(target().wouldPresentEarly(kPeriod)); +} + +TEST_F(FrameTargeterTest, detectsEarlyPresentThreeVsyncsAhead) { + TimePoint frameBeginTime(5000ms); + constexpr Fps kRefreshRate = 144_Hz; + constexpr Period kPeriod = kRefreshRate.getPeriod(); + + const Frame frame(this, VsyncId{555}, frameBeginTime, 16ms, kRefreshRate); + + // The target is more than two VSYNCs ahead, but present fences are not tracked that far back. + EXPECT_TRUE(target().wouldPresentEarly(kPeriod)); +} + +TEST_F(FrameTargeterTest, detectsMissedFrames) { + VsyncId vsyncId{555}; + TimePoint frameBeginTime(5000ms); + constexpr Fps kRefreshRate = 60_Hz; + constexpr Period kPeriod = kRefreshRate.getPeriod(); + + EXPECT_FALSE(target().isFramePending()); + EXPECT_FALSE(target().didMissFrame()); + EXPECT_FALSE(target().didMissHwcFrame()); + + { + const Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); + EXPECT_FALSE(target().isFramePending()); + + // The frame did not miss if the past present fence is invalid. + EXPECT_FALSE(target().didMissFrame()); + EXPECT_FALSE(target().didMissHwcFrame()); + } + { + Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate, Frame::fencePending); + EXPECT_TRUE(target().isFramePending()); + + // The frame missed if the past present fence is pending. + EXPECT_TRUE(target().didMissFrame()); + EXPECT_TRUE(target().didMissHwcFrame()); + + frame.end(CompositionCoverage::Gpu); + } + { + const Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate, Frame::fencePending); + EXPECT_TRUE(target().isFramePending()); + + // The GPU frame missed if the past present fence is pending. + EXPECT_TRUE(target().didMissFrame()); + EXPECT_FALSE(target().didMissHwcFrame()); + } + { + Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); + EXPECT_FALSE(target().isFramePending()); + + const auto fence = frame.end(); + const auto expectedPresentTime = target().expectedPresentTime(); + fence->signalForTest(expectedPresentTime.ns() + kPeriod.ns() / 2 + 1); + } + { + Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); + EXPECT_FALSE(target().isFramePending()); + + const auto fence = frame.end(); + const auto expectedPresentTime = target().expectedPresentTime(); + fence->signalForTest(expectedPresentTime.ns() + kPeriod.ns() / 2); + + // The frame missed if the past present fence was signaled but not within slop. + EXPECT_TRUE(target().didMissFrame()); + EXPECT_TRUE(target().didMissHwcFrame()); + } + { + Frame frame(this, vsyncId++, frameBeginTime, 10ms, kRefreshRate); + EXPECT_FALSE(target().isFramePending()); + + // The frame did not miss if the past present fence was signaled within slop. + EXPECT_FALSE(target().didMissFrame()); + EXPECT_FALSE(target().didMissHwcFrame()); + } +} + +} // namespace android::scheduler diff --git a/services/surfaceflinger/Scheduler/tests/PresentLatencyTrackerTest.cpp b/services/surfaceflinger/Scheduler/tests/PresentLatencyTrackerTest.cpp index 8952ca99ab..df2ea83112 100644 --- a/services/surfaceflinger/Scheduler/tests/PresentLatencyTrackerTest.cpp +++ b/services/surfaceflinger/Scheduler/tests/PresentLatencyTrackerTest.cpp @@ -23,16 +23,6 @@ #include <ui/FenceTime.h> namespace android::scheduler { -namespace { - -using FencePair = std::pair<sp<Fence>, std::shared_ptr<FenceTime>>; - -FencePair makePendingFence(FenceToFenceTimeMap& fenceMap) { - const auto fence = sp<Fence>::make(); - return {fence, fenceMap.createFenceTimeForTest(fence)}; -} - -} // namespace TEST(PresentLatencyTrackerTest, skipsInvalidFences) { PresentLatencyTracker tracker; @@ -43,7 +33,7 @@ TEST(PresentLatencyTrackerTest, skipsInvalidFences) { EXPECT_EQ(tracker.trackPendingFrame(kCompositeTime, FenceTime::NO_FENCE), Duration::zero()); FenceToFenceTimeMap fenceMap; - const auto [fence, fenceTime] = makePendingFence(fenceMap); + const auto [fence, fenceTime] = fenceMap.makePendingFenceForTest(); EXPECT_EQ(tracker.trackPendingFrame(kCompositeTime, fenceTime), Duration::zero()); fenceTime->signalForTest(9999); @@ -56,8 +46,9 @@ TEST(PresentLatencyTrackerTest, tracksPendingFrames) { PresentLatencyTracker tracker; FenceToFenceTimeMap fenceMap; - std::array<FencePair, PresentLatencyTracker::kMaxPendingFrames> fences; - std::generate(fences.begin(), fences.end(), [&fenceMap] { return makePendingFence(fenceMap); }); + std::array<FenceToFenceTimeMap::FencePair, PresentLatencyTracker::kMaxPendingFrames> fences; + std::generate(fences.begin(), fences.end(), + [&fenceMap] { return fenceMap.makePendingFenceForTest(); }); // The present latency is 0 if all fences are pending. const TimePoint kCompositeTime = TimePoint::fromNs(1234); @@ -71,7 +62,7 @@ TEST(PresentLatencyTrackerTest, tracksPendingFrames) { fences[i].second->signalForTest(kCompositeTime.ns() + static_cast<nsecs_t>(i)); } - const auto fence = makePendingFence(fenceMap); + const auto fence = fenceMap.makePendingFenceForTest(); // ...then the present latency is measured using the latest frame. constexpr Duration kPresentLatency = Duration::fromNs(static_cast<nsecs_t>(kPresentCount) - 1); diff --git a/services/surfaceflinger/ScreenCaptureOutput.cpp b/services/surfaceflinger/ScreenCaptureOutput.cpp index 09dac23410..ef9b457fc9 100644 --- a/services/surfaceflinger/ScreenCaptureOutput.cpp +++ b/services/surfaceflinger/ScreenCaptureOutput.cpp @@ -24,19 +24,37 @@ namespace android { +namespace { + +ui::Size getDisplaySize(ui::Rotation orientation, const Rect& sourceCrop) { + if (orientation == ui::Rotation::Rotation90 || orientation == ui::Rotation::Rotation270) { + return {sourceCrop.getHeight(), sourceCrop.getWidth()}; + } + return {sourceCrop.getWidth(), sourceCrop.getHeight()}; +} + +Rect getOrientedDisplaySpaceRect(ui::Rotation orientation, int reqWidth, int reqHeight) { + if (orientation == ui::Rotation::Rotation90 || orientation == ui::Rotation::Rotation270) { + return {reqHeight, reqWidth}; + } + return {reqWidth, reqHeight}; +} + +} // namespace + std::shared_ptr<ScreenCaptureOutput> createScreenCaptureOutput(ScreenCaptureOutputArgs args) { std::shared_ptr<ScreenCaptureOutput> output = compositionengine::impl::createOutputTemplated< ScreenCaptureOutput, compositionengine::CompositionEngine, const RenderArea&, - const compositionengine::Output::ColorProfile&, bool>(args.compositionEngine, - args.renderArea, - args.colorProfile, - args.regionSampling); + const compositionengine::Output::ColorProfile&, + bool>(args.compositionEngine, args.renderArea, args.colorProfile, args.regionSampling, + args.dimInGammaSpaceForEnhancedScreenshots); output->editState().isSecure = args.renderArea.isSecure(); output->setCompositionEnabled(true); output->setLayerFilter({args.layerStack}); output->setRenderSurface(std::make_unique<ScreenCaptureRenderSurface>(std::move(args.buffer))); output->setDisplayBrightness(args.sdrWhitePointNits, args.displayBrightnessNits); output->editState().clientTargetBrightness = args.targetBrightness; + output->editState().treat170mAsSrgb = args.treat170mAsSrgb; output->setDisplayColorProfile(std::make_unique<compositionengine::impl::DisplayColorProfile>( compositionengine::DisplayColorProfileCreationArgsBuilder() @@ -45,10 +63,10 @@ std::shared_ptr<ScreenCaptureOutput> createScreenCaptureOutput(ScreenCaptureOutp const Rect& sourceCrop = args.renderArea.getSourceCrop(); const ui::Rotation orientation = ui::Transform::toRotation(args.renderArea.getRotationFlags()); - const Rect orientedDisplaySpaceRect{args.renderArea.getReqWidth(), - args.renderArea.getReqHeight()}; - output->setProjection(orientation, sourceCrop, orientedDisplaySpaceRect); - output->setDisplaySize({sourceCrop.getWidth(), sourceCrop.getHeight()}); + output->setDisplaySize(getDisplaySize(orientation, sourceCrop)); + output->setProjection(orientation, sourceCrop, + getOrientedDisplaySpaceRect(orientation, args.renderArea.getReqWidth(), + args.renderArea.getReqHeight())); { std::string name = args.regionSampling ? "RegionSampling" : "ScreenCaptureOutput"; @@ -62,8 +80,11 @@ std::shared_ptr<ScreenCaptureOutput> createScreenCaptureOutput(ScreenCaptureOutp ScreenCaptureOutput::ScreenCaptureOutput( const RenderArea& renderArea, const compositionengine::Output::ColorProfile& colorProfile, - bool regionSampling) - : mRenderArea(renderArea), mColorProfile(colorProfile), mRegionSampling(regionSampling) {} + bool regionSampling, bool dimInGammaSpaceForEnhancedScreenshots) + : mRenderArea(renderArea), + mColorProfile(colorProfile), + mRegionSampling(regionSampling), + mDimInGammaSpaceForEnhancedScreenshots(dimInGammaSpaceForEnhancedScreenshots) {} void ScreenCaptureOutput::updateColorProfile(const compositionengine::CompositionRefreshArgs&) { auto& outputState = editState(); @@ -76,6 +97,14 @@ renderengine::DisplaySettings ScreenCaptureOutput::generateClientCompositionDisp auto clientCompositionDisplay = compositionengine::impl::Output::generateClientCompositionDisplaySettings(); clientCompositionDisplay.clip = mRenderArea.getSourceCrop(); + + auto renderIntent = static_cast<ui::RenderIntent>(clientCompositionDisplay.renderIntent); + if (mDimInGammaSpaceForEnhancedScreenshots && renderIntent != ui::RenderIntent::COLORIMETRIC && + renderIntent != ui::RenderIntent::TONE_MAP_COLORIMETRIC) { + clientCompositionDisplay.dimmingStage = + aidl::android::hardware::graphics::composer3::DimmingStage::GAMMA_OETF; + } + return clientCompositionDisplay; } diff --git a/services/surfaceflinger/ScreenCaptureOutput.h b/services/surfaceflinger/ScreenCaptureOutput.h index 3c307b0733..fc095def99 100644 --- a/services/surfaceflinger/ScreenCaptureOutput.h +++ b/services/surfaceflinger/ScreenCaptureOutput.h @@ -36,6 +36,8 @@ struct ScreenCaptureOutputArgs { // Counterintuitively, when targetBrightness > 1.0 then dim the scene. float targetBrightness; bool regionSampling; + bool treat170mAsSrgb; + bool dimInGammaSpaceForEnhancedScreenshots; }; // ScreenCaptureOutput is used to compose a set of layers into a preallocated buffer. @@ -46,7 +48,7 @@ class ScreenCaptureOutput : public compositionengine::impl::Output { public: ScreenCaptureOutput(const RenderArea& renderArea, const compositionengine::Output::ColorProfile& colorProfile, - bool regionSampling); + bool regionSampling, bool dimInGammaSpaceForEnhancedScreenshots); void updateColorProfile(const compositionengine::CompositionRefreshArgs&) override; @@ -62,6 +64,7 @@ private: const RenderArea& mRenderArea; const compositionengine::Output::ColorProfile& mColorProfile; const bool mRegionSampling; + const bool mDimInGammaSpaceForEnhancedScreenshots; }; std::shared_ptr<ScreenCaptureOutput> createScreenCaptureOutput(ScreenCaptureOutputArgs); diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp index 7c85452462..6c41129e08 100644 --- a/services/surfaceflinger/SurfaceFlinger.cpp +++ b/services/surfaceflinger/SurfaceFlinger.cpp @@ -77,9 +77,9 @@ #include <processgroup/processgroup.h> #include <renderengine/RenderEngine.h> #include <renderengine/impl/ExternalTexture.h> +#include <scheduler/FrameTargeter.h> #include <sys/types.h> #include <ui/ColorSpace.h> -#include <ui/DataspaceUtils.h> #include <ui/DebugUtils.h> #include <ui/DisplayId.h> #include <ui/DisplayMode.h> @@ -87,6 +87,7 @@ #include <ui/DisplayState.h> #include <ui/DynamicDisplayInfo.h> #include <ui/GraphicBufferAllocator.h> +#include <ui/HdrRenderTypeUtils.h> #include <ui/LayerStack.h> #include <ui/PixelFormat.h> #include <ui/StaticDisplayInfo.h> @@ -96,6 +97,7 @@ #include <utils/Timers.h> #include <utils/misc.h> +#include <unistd.h> #include <algorithm> #include <cerrno> #include <cinttypes> @@ -116,7 +118,6 @@ #include "Client.h" #include "ClientCache.h" #include "Colorizer.h" -#include "Display/DisplayMap.h" #include "DisplayDevice.h" #include "DisplayHardware/ComposerHal.h" #include "DisplayHardware/FramebufferSurface.h" @@ -450,6 +451,9 @@ SurfaceFlinger::SurfaceFlinger(Factory& factory) : SurfaceFlinger(factory, SkipI property_get("debug.sf.treat_170m_as_sRGB", value, "0"); mTreat170mAsSrgb = atoi(value); + property_get("debug.sf.dim_in_gamma_in_enhanced_screenshots", value, 0); + mDimInGammaSpaceForEnhancedScreenshots = atoi(value); + mIgnoreHwcPhysicalDisplayOrientation = base::GetBoolProperty("debug.sf.ignore_hwc_physical_display_orientation"s, false); @@ -910,6 +914,27 @@ void SurfaceFlinger::init() FTL_FAKE_GUARD(kMainThreadContext) { ALOGE("Run StartPropertySetThread failed!"); } + if (mTransactionTracing) { + TransactionTraceWriter::getInstance().setWriterFunction([&](const std::string& prefix, + bool overwrite) { + auto writeFn = [&]() { + const std::string filename = + TransactionTracing::DIR_NAME + prefix + TransactionTracing::FILE_NAME; + if (overwrite && access(filename.c_str(), F_OK) == 0) { + ALOGD("TransactionTraceWriter: file=%s already exists", filename.c_str()); + return; + } + mTransactionTracing->flush(); + mTransactionTracing->writeToFile(filename); + }; + if (std::this_thread::get_id() == mMainThreadId) { + writeFn(); + } else { + mScheduler->schedule(writeFn).get(); + } + }); + } + ALOGV("Done initializing"); } @@ -1175,9 +1200,9 @@ status_t SurfaceFlinger::getDisplayStats(const sp<IBinder>& displayToken, } void SurfaceFlinger::setDesiredActiveMode(display::DisplayModeRequest&& request, bool force) { - ATRACE_CALL(); - const auto displayId = request.mode.modePtr->getPhysicalDisplayId(); + ATRACE_NAME(ftl::Concat(__func__, ' ', displayId.value).c_str()); + const auto display = getDisplayDeviceLocked(displayId); if (!display) { ALOGW("%s: display is no longer valid", __func__); @@ -1205,17 +1230,24 @@ void SurfaceFlinger::setDesiredActiveMode(display::DisplayModeRequest&& request, // As we called to set period, we will call to onRefreshRateChangeCompleted once // VsyncController model is locked. mScheduler->modulateVsync(displayId, &VsyncModulator::onRefreshRateChangeInitiated); - updatePhaseConfiguration(mode.fps); + + if (displayId == mActiveDisplayId) { + updatePhaseConfiguration(mode.fps); + } + mScheduler->setModeChangePending(true); break; case DisplayDevice::DesiredActiveModeAction::InitiateRenderRateSwitch: mScheduler->setRenderRate(displayId, mode.fps); - updatePhaseConfiguration(mode.fps); - mRefreshRateStats->setRefreshRate(mode.fps); - if (display->getPhysicalId() == mActiveDisplayId && emitEvent) { - mScheduler->onPrimaryDisplayModeChanged(mAppConnectionHandle, mode); + + if (displayId == mActiveDisplayId) { + updatePhaseConfiguration(mode.fps); + mRefreshRateStats->setRefreshRate(mode.fps); } + if (emitEvent) { + dispatchDisplayModeChangeEvent(displayId, mode); + } break; case DisplayDevice::DesiredActiveModeAction::None: break; @@ -1271,24 +1303,20 @@ status_t SurfaceFlinger::setActiveModeFromBackdoor(const sp<display::DisplayToke return future.get(); } -void SurfaceFlinger::updateInternalStateWithChangedMode() { - ATRACE_CALL(); - - const auto display = getDefaultDisplayDeviceLocked(); - if (!display) { - return; - } +void SurfaceFlinger::finalizeDisplayModeChange(DisplayDevice& display) { + const auto displayId = display.getPhysicalId(); + ATRACE_NAME(ftl::Concat(__func__, ' ', displayId.value).c_str()); - const auto upcomingModeInfo = display->getUpcomingActiveMode(); + const auto upcomingModeInfo = display.getUpcomingActiveMode(); if (!upcomingModeInfo.modeOpt) { // There is no pending mode change. This can happen if the active // display changed and the mode change happened on a different display. return; } - if (display->getActiveMode().modePtr->getResolution() != + if (display.getActiveMode().modePtr->getResolution() != upcomingModeInfo.modeOpt->modePtr->getResolution()) { - auto& state = mCurrentState.displays.editValueFor(display->getDisplayToken()); + auto& state = mCurrentState.displays.editValueFor(display.getDisplayToken()); // We need to generate new sequenceId in order to recreate the display (and this // way the framebuffer). state.sequenceId = DisplayDeviceState{}.sequenceId; @@ -1299,27 +1327,24 @@ void SurfaceFlinger::updateInternalStateWithChangedMode() { return; } - mPhysicalDisplays.get(display->getPhysicalId()) - .transform(&PhysicalDisplay::snapshotRef) - .transform(ftl::unit_fn([&](const display::DisplaySnapshot& snapshot) { - FTL_FAKE_GUARD(kMainThreadContext, - display->setActiveMode(upcomingModeInfo.modeOpt->modePtr->getId(), - upcomingModeInfo.modeOpt->modePtr->getFps(), - upcomingModeInfo.modeOpt->fps)); - })); - - const Fps refreshRate = upcomingModeInfo.modeOpt->fps; - mRefreshRateStats->setRefreshRate(refreshRate); - updatePhaseConfiguration(refreshRate); + const auto& activeMode = *upcomingModeInfo.modeOpt; + display.finalizeModeChange(activeMode.modePtr->getId(), activeMode.modePtr->getFps(), + activeMode.fps); + + if (displayId == mActiveDisplayId) { + mRefreshRateStats->setRefreshRate(activeMode.fps); + updatePhaseConfiguration(activeMode.fps); + } if (upcomingModeInfo.event != scheduler::DisplayModeEvent::None) { - mScheduler->onPrimaryDisplayModeChanged(mAppConnectionHandle, *upcomingModeInfo.modeOpt); + dispatchDisplayModeChangeEvent(displayId, activeMode); } } void SurfaceFlinger::clearDesiredActiveModeState(const sp<DisplayDevice>& display) { display->clearDesiredActiveModeState(); if (display->getPhysicalId() == mActiveDisplayId) { + // TODO(b/255635711): Check for pending mode changes on other displays. mScheduler->setModeChangePending(false); } } @@ -1333,21 +1358,18 @@ void SurfaceFlinger::desiredActiveModeChangeDone(const sp<DisplayDevice>& displa clearDesiredActiveModeState(display); mScheduler->resyncToHardwareVsync(displayId, true /* allowToEnable */, displayFps); mScheduler->setRenderRate(displayId, renderFps); - updatePhaseConfiguration(renderFps); + + if (displayId == mActiveDisplayId) { + updatePhaseConfiguration(renderFps); + } } -void SurfaceFlinger::setActiveModeInHwcIfNeeded() { +void SurfaceFlinger::initiateDisplayModeChanges() { ATRACE_CALL(); std::optional<PhysicalDisplayId> displayToUpdateImmediately; for (const auto& [id, physical] : mPhysicalDisplays) { - const auto& snapshot = physical.snapshot(); - - if (snapshot.connectionType() != ui::DisplayConnectionType::Internal) { - continue; - } - const auto display = getDisplayDeviceLocked(id); if (!display) continue; @@ -1358,14 +1380,13 @@ void SurfaceFlinger::setActiveModeInHwcIfNeeded() { continue; } - if (id != mActiveDisplayId) { - // Display is no longer the active display, so abort the mode change. + if (!shouldApplyRefreshRateSelectorPolicy(*display)) { clearDesiredActiveModeState(display); continue; } const auto desiredModeId = desiredActiveMode->modeOpt->modePtr->getId(); - const auto displayModePtrOpt = snapshot.displayModes().get(desiredModeId); + const auto displayModePtrOpt = physical.snapshot().displayModes().get(desiredModeId); if (!displayModePtrOpt) { ALOGW("Desired display mode is no longer supported. Mode ID = %d", @@ -1415,19 +1436,18 @@ void SurfaceFlinger::setActiveModeInHwcIfNeeded() { if (outTimeline.refreshRequired) { scheduleComposite(FrameHint::kNone); - mSetActiveModePending = true; } else { - // Updating the internal state should be done outside the loop, - // because it can recreate a DisplayDevice and modify mDisplays - // which will invalidate the iterator. + // TODO(b/255635711): Remove `displayToUpdateImmediately` to `finalizeDisplayModeChange` + // for all displays. This was only needed when the loop iterated over `mDisplays` rather + // than `mPhysicalDisplays`. displayToUpdateImmediately = display->getPhysicalId(); } } if (displayToUpdateImmediately) { - updateInternalStateWithChangedMode(); - const auto display = getDisplayDeviceLocked(*displayToUpdateImmediately); + finalizeDisplayModeChange(*display); + const auto desiredActiveMode = display->getDesiredActiveMode(); if (desiredActiveMode && display->getActiveMode() == desiredActiveMode->modeOpt) { desiredActiveModeChangeDone(display); @@ -2140,65 +2160,7 @@ void SurfaceFlinger::onRefreshRateChangedDebug(const RefreshRateChangedDebugData } } -void SurfaceFlinger::setVsyncEnabled(PhysicalDisplayId id, bool enabled) { - const char* const whence = __func__; - ATRACE_FORMAT("%s (%d) for %" PRIu64, whence, enabled, id.value); - - // On main thread to avoid race conditions with display power state. - static_cast<void>(mScheduler->schedule([=]() FTL_FAKE_GUARD(mStateLock) { - { - ftl::FakeGuard guard(kMainThreadContext); - if (auto schedule = mScheduler->getVsyncSchedule(id)) { - schedule->setPendingHardwareVsyncState(enabled); - } - } - - ATRACE_FORMAT("%s (%d) for %" PRIu64 " (main thread)", whence, enabled, id.value); - if (const auto display = getDisplayDeviceLocked(id); display && display->isPoweredOn()) { - setHWCVsyncEnabled(id, enabled); - } - })); -} - -bool SurfaceFlinger::wouldPresentEarly(TimePoint frameTime, Period vsyncPeriod) const { - const bool isThreeVsyncsAhead = mExpectedPresentTime - frameTime > 2 * vsyncPeriod; - return isThreeVsyncsAhead || - getPreviousPresentFence(frameTime, vsyncPeriod)->getSignalTime() != - Fence::SIGNAL_TIME_PENDING; -} - -auto SurfaceFlinger::getPreviousPresentFence(TimePoint frameTime, Period vsyncPeriod) const - -> const FenceTimePtr& { - const bool isTwoVsyncsAhead = mExpectedPresentTime - frameTime > vsyncPeriod; - const size_t i = static_cast<size_t>(isTwoVsyncsAhead); - return mPreviousPresentFences[i].fenceTime; -} - -bool SurfaceFlinger::isFencePending(const FenceTimePtr& fence, int graceTimeMs) { - ATRACE_CALL(); - if (fence == FenceTime::NO_FENCE) { - return false; - } - - const status_t status = fence->wait(graceTimeMs); - // This is the same as Fence::Status::Unsignaled, but it saves a getStatus() call, - // which calls wait(0) again internally - return status == -ETIME; -} - -TimePoint SurfaceFlinger::calculateExpectedPresentTime(TimePoint frameTime) const { - const auto& schedule = mScheduler->getVsyncSchedule(); - - const TimePoint vsyncDeadline = schedule->vsyncDeadlineAfter(frameTime); - if (mScheduler->vsyncModulator().getVsyncConfig().sfOffset > 0) { - return vsyncDeadline; - } - - // Inflate the expected present time if we're targeting the next vsync. - return vsyncDeadline + schedule->period(); -} - -void SurfaceFlinger::configure() FTL_FAKE_GUARD(kMainThreadContext) { +void SurfaceFlinger::configure() { Mutex::Autolock lock(mStateLock); if (configureLocked()) { setTransactionFlags(eDisplayTransactionNeeded); @@ -2208,6 +2170,7 @@ void SurfaceFlinger::configure() FTL_FAKE_GUARD(kMainThreadContext) { bool SurfaceFlinger::updateLayerSnapshotsLegacy(VsyncId vsyncId, frontend::Update& update, bool transactionsFlushed, bool& outTransactionsAreEmpty) { + ATRACE_CALL(); bool needsTraversal = false; if (transactionsFlushed) { needsTraversal |= commitMirrorDisplays(vsyncId); @@ -2260,7 +2223,7 @@ void SurfaceFlinger::updateLayerHistory(const frontend::LayerSnapshot& snapshot) bool SurfaceFlinger::updateLayerSnapshots(VsyncId vsyncId, frontend::Update& update, bool transactionsFlushed, bool& outTransactionsAreEmpty) { using Changes = frontend::RequestedLayerState::Changes; - ATRACE_NAME("updateLayerSnapshots"); + ATRACE_CALL(); { mLayerLifecycleManager.addLayers(std::move(update.newLayers)); mLayerLifecycleManager.applyTransactions(update.transactions); @@ -2371,117 +2334,77 @@ bool SurfaceFlinger::updateLayerSnapshots(VsyncId vsyncId, frontend::Update& upd return mustComposite; } -bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expectedVsyncTime) - FTL_FAKE_GUARD(kMainThreadContext) { - // The expectedVsyncTime, which was predicted when this frame was scheduled, is normally in the - // future relative to frameTime, but may not be for delayed frames. Adjust mExpectedPresentTime - // accordingly, but not mScheduledPresentTime. - const TimePoint lastScheduledPresentTime = mScheduledPresentTime; - mScheduledPresentTime = expectedVsyncTime; +bool SurfaceFlinger::commit(PhysicalDisplayId pacesetterId, + const scheduler::FrameTargets& frameTargets) { + const scheduler::FrameTarget& pacesetterFrameTarget = *frameTargets.get(pacesetterId)->get(); - // Calculate the expected present time once and use the cached value throughout this frame to - // make sure all layers are seeing this same value. - mExpectedPresentTime = expectedVsyncTime >= frameTime ? expectedVsyncTime - : calculateExpectedPresentTime(frameTime); + const VsyncId vsyncId = pacesetterFrameTarget.vsyncId(); + ATRACE_NAME(ftl::Concat(__func__, ' ', ftl::to_underlying(vsyncId)).c_str()); - ATRACE_FORMAT("%s %" PRId64 " vsyncIn %.2fms%s", __func__, vsyncId.value, - ticks<std::milli, float>(mExpectedPresentTime - TimePoint::now()), - mExpectedPresentTime == expectedVsyncTime ? "" : " (adjusted)"); - - const Period vsyncPeriod = mScheduler->getVsyncSchedule()->period(); - const FenceTimePtr& previousPresentFence = getPreviousPresentFence(frameTime, vsyncPeriod); - - // When backpressure propagation is enabled, we want to give a small grace period of 1ms - // for the present fence to fire instead of just giving up on this frame to handle cases - // where present fence is just about to get signaled. - const int graceTimeForPresentFenceMs = static_cast<int>( - mBackpressureGpuComposition || !mCompositionCoverage.test(CompositionCoverage::Gpu)); - - // Pending frames may trigger backpressure propagation. - const TracedOrdinal<bool> framePending = {"PrevFramePending", - isFencePending(previousPresentFence, - graceTimeForPresentFenceMs)}; - - // Frame missed counts for metrics tracking. - // A frame is missed if the prior frame is still pending. If no longer pending, - // then we still count the frame as missed if the predicted present time - // was further in the past than when the fence actually fired. - - // Add some slop to correct for drift. This should generally be - // smaller than a typical frame duration, but should not be so small - // that it reports reasonable drift as a missed frame. - const nsecs_t frameMissedSlop = vsyncPeriod.ns() / 2; - const nsecs_t previousPresentTime = previousPresentFence->getSignalTime(); - const TracedOrdinal<bool> frameMissed = {"PrevFrameMissed", - framePending || - (previousPresentTime >= 0 && - (lastScheduledPresentTime.ns() < - previousPresentTime - frameMissedSlop))}; - const TracedOrdinal<bool> hwcFrameMissed = {"PrevHwcFrameMissed", - frameMissed && - mCompositionCoverage.test( - CompositionCoverage::Hwc)}; - - const TracedOrdinal<bool> gpuFrameMissed = {"PrevGpuFrameMissed", - frameMissed && - mCompositionCoverage.test( - CompositionCoverage::Gpu)}; - - if (frameMissed) { - mFrameMissedCount++; + if (pacesetterFrameTarget.didMissFrame()) { mTimeStats->incrementMissedFrames(); } - if (hwcFrameMissed) { - mHwcFrameMissedCount++; - } - - if (gpuFrameMissed) { - mGpuFrameMissedCount++; - } - if (mTracingEnabledChanged) { mLayerTracingEnabled = mLayerTracing.isEnabled(); mTracingEnabledChanged = false; } - // If we are in the middle of a mode change and the fence hasn't - // fired yet just wait for the next commit. - if (mSetActiveModePending) { - if (framePending) { - mScheduler->scheduleFrame(); - return false; - } + // If a mode set is pending and the fence hasn't fired yet, wait for the next commit. + if (std::any_of(frameTargets.begin(), frameTargets.end(), + [this](const auto& pair) FTL_FAKE_GUARD(mStateLock) + FTL_FAKE_GUARD(kMainThreadContext) { + if (!pair.second->isFramePending()) return false; - // We received the present fence from the HWC, so we assume it successfully updated - // the mode, hence we update SF. - mSetActiveModePending = false; - { - Mutex::Autolock lock(mStateLock); - updateInternalStateWithChangedMode(); + if (const auto display = getDisplayDeviceLocked(pair.first)) { + return display->isModeSetPending(); + } + + return false; + })) { + mScheduler->scheduleFrame(); + return false; + } + + { + Mutex::Autolock lock(mStateLock); + + for (const auto [id, target] : frameTargets) { + // TODO(b/241285876): This is `nullptr` when the DisplayDevice is about to be removed in + // this commit, since the PhysicalDisplay has already been removed. Rather than checking + // for `nullptr` below, change Scheduler::onFrameSignal to filter out the FrameTarget of + // the removed display. + const auto display = getDisplayDeviceLocked(id); + + if (display && display->isModeSetPending()) { + finalizeDisplayModeChange(*display); + } } } - if (framePending) { - if (mBackpressureGpuComposition || (hwcFrameMissed && !gpuFrameMissed)) { + if (pacesetterFrameTarget.isFramePending()) { + if (mBackpressureGpuComposition || pacesetterFrameTarget.didMissHwcFrame()) { scheduleCommit(FrameHint::kNone); return false; } } + const Period vsyncPeriod = mScheduler->getVsyncSchedule()->period(); + // Save this once per commit + composite to ensure consistency // TODO (b/240619471): consider removing active display check once AOD is fixed const auto activeDisplay = FTL_FAKE_GUARD(mStateLock, getDisplayDeviceLocked(mActiveDisplayId)); mPowerHintSessionEnabled = mPowerAdvisor->usePowerHintSession() && activeDisplay && activeDisplay->getPowerMode() == hal::PowerMode::ON; if (mPowerHintSessionEnabled) { - mPowerAdvisor->setCommitStart(frameTime); - mPowerAdvisor->setExpectedPresentTime(mExpectedPresentTime); + mPowerAdvisor->setCommitStart(pacesetterFrameTarget.frameBeginTime()); + mPowerAdvisor->setExpectedPresentTime(pacesetterFrameTarget.expectedPresentTime()); // Frame delay is how long we should have minus how long we actually have. const Duration idealSfWorkDuration = mScheduler->vsyncModulator().getVsyncConfig().sfWorkDuration; - const Duration frameDelay = idealSfWorkDuration - (mExpectedPresentTime - frameTime); + const Duration frameDelay = + idealSfWorkDuration - pacesetterFrameTarget.expectedFrameDuration(); mPowerAdvisor->setFrameDelay(frameDelay); mPowerAdvisor->setTotalFrameTargetWorkDuration(idealSfWorkDuration); @@ -2501,7 +2424,8 @@ bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expe // Composite if transactions were committed, or if requested by HWC. bool mustComposite = mMustComposite.exchange(false); { - mFrameTimeline->setSfWakeUp(vsyncId.value, frameTime.ns(), + mFrameTimeline->setSfWakeUp(ftl::to_underlying(vsyncId), + pacesetterFrameTarget.frameBeginTime().ns(), Fps::fromPeriodNsecs(vsyncPeriod.ns())); const bool flushTransactions = clearTransactionFlags(eTransactionFlushNeeded); @@ -2509,9 +2433,11 @@ bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expe if (flushTransactions) { updates = flushLifecycleUpdates(); if (mTransactionTracing) { - mTransactionTracing->addCommittedTransactions(vsyncId.value, frameTime.ns(), - updates, mFrontEndDisplayInfos, - mFrontEndDisplayInfosChanged); + mTransactionTracing + ->addCommittedTransactions(ftl::to_underlying(vsyncId), + pacesetterFrameTarget.frameBeginTime().ns(), + updates, mFrontEndDisplayInfos, + mFrontEndDisplayInfosChanged); } } bool transactionsAreEmpty; @@ -2546,15 +2472,15 @@ bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expe { Mutex::Autolock lock(mStateLock); mScheduler->chooseRefreshRateForContent(); - setActiveModeInHwcIfNeeded(); + initiateDisplayModeChanges(); } updateCursorAsync(); - updateInputFlinger(vsyncId, frameTime); + updateInputFlinger(vsyncId, pacesetterFrameTarget.frameBeginTime()); if (mLayerTracingEnabled && !mLayerTracing.flagIsSet(LayerTracing::TRACE_COMPOSITION)) { // This will block and tracing should only be enabled for debugging. - addToLayerTracing(mVisibleRegionsDirty, frameTime.ns(), vsyncId.value); + addToLayerTracing(mVisibleRegionsDirty, pacesetterFrameTarget.frameBeginTime(), vsyncId); } mLastCommittedVsyncId = vsyncId; @@ -2563,26 +2489,42 @@ bool SurfaceFlinger::commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expe return mustComposite && CC_LIKELY(mBootStage != BootStage::BOOTLOADER); } -void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) - FTL_FAKE_GUARD(kMainThreadContext) { - ATRACE_FORMAT("%s %" PRId64, __func__, vsyncId.value); +CompositeResultsPerDisplay SurfaceFlinger::composite( + PhysicalDisplayId pacesetterId, const scheduler::FrameTargeters& frameTargeters) { + const scheduler::FrameTarget& pacesetterTarget = + frameTargeters.get(pacesetterId)->get()->target(); + + const VsyncId vsyncId = pacesetterTarget.vsyncId(); + ATRACE_NAME(ftl::Concat(__func__, ' ', ftl::to_underlying(vsyncId)).c_str()); compositionengine::CompositionRefreshArgs refreshArgs; + refreshArgs.powerCallback = this; const auto& displays = FTL_FAKE_GUARD(mStateLock, mDisplays); refreshArgs.outputs.reserve(displays.size()); + + // Add outputs for physical displays. + for (const auto& [id, targeter] : frameTargeters) { + ftl::FakeGuard guard(mStateLock); + + if (const auto display = getCompositionDisplayLocked(id)) { + refreshArgs.outputs.push_back(display); + } + } + std::vector<DisplayId> displayIds; for (const auto& [_, display] : displays) { - bool dropFrame = false; + displayIds.push_back(display->getId()); + display->tracePowerMode(); + + // Add outputs for virtual displays. if (display->isVirtual()) { - Fps refreshRate = display->getAdjustedRefreshRate(); - using fps_approx_ops::operator>; - dropFrame = (refreshRate > 0_Hz) && !mScheduler->isVsyncInPhase(frameTime, refreshRate); - } - if (!dropFrame) { - refreshArgs.outputs.push_back(display->getCompositionDisplay()); + const Fps refreshRate = display->getAdjustedRefreshRate(); + + if (!refreshRate.isValid() || + mScheduler->isVsyncInPhase(pacesetterTarget.frameBeginTime(), refreshRate)) { + refreshArgs.outputs.push_back(display->getCompositionDisplay()); + } } - display->tracePowerMode(); - displayIds.push_back(display->getId()); } mPowerAdvisor->setDisplays(displayIds); @@ -2642,23 +2584,25 @@ void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) if (!getHwComposer().getComposer()->isSupported( Hwc2::Composer::OptionalFeature::ExpectedPresentTime) && - wouldPresentEarly(frameTime, vsyncPeriod)) { - const auto prevVsyncTime = mExpectedPresentTime - vsyncPeriod; + pacesetterTarget.wouldPresentEarly(vsyncPeriod)) { const auto hwcMinWorkDuration = mVsyncConfiguration->getCurrentConfigs().hwcMinWorkDuration; - refreshArgs.earliestPresentTime = prevVsyncTime - hwcMinWorkDuration; + // TODO(b/255601557): Calculate and pass per-display values for each FrameTarget. + refreshArgs.earliestPresentTime = + pacesetterTarget.previousFrameVsyncTime(vsyncPeriod) - hwcMinWorkDuration; } refreshArgs.scheduledFrameTime = mScheduler->getScheduledFrameTime(); - refreshArgs.expectedPresentTime = mExpectedPresentTime.ns(); + refreshArgs.expectedPresentTime = pacesetterTarget.expectedPresentTime().ns(); refreshArgs.hasTrustedPresentationListener = mNumTrustedPresentationListeners > 0; // Store the present time just before calling to the composition engine so we could notify // the scheduler. const auto presentTime = systemTime(); - std::vector<std::pair<Layer*, LayerFE*>> layers = - moveSnapshotsToCompositionArgs(refreshArgs, /*cursorOnly=*/false, vsyncId.value); + constexpr bool kCursorOnly = false; + const auto layers = moveSnapshotsToCompositionArgs(refreshArgs, kCursorOnly); + mCompositionEngine->present(refreshArgs); moveSnapshotsFromCompositionArgs(refreshArgs, layers); @@ -2675,14 +2619,14 @@ void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) } } - mTimeStats->recordFrameDuration(frameTime.ns(), systemTime()); + mTimeStats->recordFrameDuration(pacesetterTarget.frameBeginTime().ns(), systemTime()); // Send a power hint after presentation is finished. if (mPowerHintSessionEnabled) { // Now that the current frame has been presented above, PowerAdvisor needs the present time // of the previous frame (whose fence is signaled by now) to determine how long the HWC had // waited on that fence to retire before presenting. - const auto& previousPresentFence = mPreviousPresentFences[0].fenceTime; + const auto& previousPresentFence = pacesetterTarget.presentFenceForPreviousFrame(); mPowerAdvisor->setSfPresentTiming(TimePoint::fromNs(previousPresentFence->getSignalTime()), TimePoint::now()); @@ -2693,23 +2637,27 @@ void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) scheduleComposite(FrameHint::kNone); } - postComposition(presentTime); + postComposition(pacesetterId, frameTargeters, presentTime); - const bool hadGpuComposited = mCompositionCoverage.test(CompositionCoverage::Gpu); + const bool hadGpuComposited = + multiDisplayUnion(mCompositionCoverage).test(CompositionCoverage::Gpu); mCompositionCoverage.clear(); TimeStats::ClientCompositionRecord clientCompositionRecord; + for (const auto& [_, display] : displays) { const auto& state = display->getCompositionDisplay()->getState(); + CompositionCoverageFlags& flags = + mCompositionCoverage.try_emplace(display->getId()).first->second; if (state.usesDeviceComposition) { - mCompositionCoverage |= CompositionCoverage::Hwc; + flags |= CompositionCoverage::Hwc; } if (state.reusedClientComposition) { - mCompositionCoverage |= CompositionCoverage::GpuReuse; + flags |= CompositionCoverage::GpuReuse; } else if (state.usesClientComposition) { - mCompositionCoverage |= CompositionCoverage::Gpu; + flags |= CompositionCoverage::Gpu; } clientCompositionRecord.predicted |= @@ -2718,10 +2666,11 @@ void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) (state.strategyPrediction == CompositionStrategyPredictionState::SUCCESS); } - const bool hasGpuComposited = mCompositionCoverage.test(CompositionCoverage::Gpu); + const auto coverage = multiDisplayUnion(mCompositionCoverage); + const bool hasGpuComposited = coverage.test(CompositionCoverage::Gpu); clientCompositionRecord.hadClientComposition = hasGpuComposited; - clientCompositionRecord.reused = mCompositionCoverage.test(CompositionCoverage::GpuReuse); + clientCompositionRecord.reused = coverage.test(CompositionCoverage::GpuReuse); clientCompositionRecord.changed = hadGpuComposited != hasGpuComposited; mTimeStats->pushCompositionStrategyState(clientCompositionRecord); @@ -2730,13 +2679,13 @@ void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) // TODO(b/160583065): Enable skip validation when SF caches all client composition layers. const bool hasGpuUseOrReuse = - mCompositionCoverage.any(CompositionCoverage::Gpu | CompositionCoverage::GpuReuse); + coverage.any(CompositionCoverage::Gpu | CompositionCoverage::GpuReuse); mScheduler->modulateVsync({}, &VsyncModulator::onDisplayRefresh, hasGpuUseOrReuse); mLayersWithQueuedFrames.clear(); if (mLayerTracingEnabled && mLayerTracing.flagIsSet(LayerTracing::TRACE_COMPOSITION)) { // This will block and should only be used for debugging. - addToLayerTracing(mVisibleRegionsDirty, frameTime.ns(), vsyncId.value); + addToLayerTracing(mVisibleRegionsDirty, pacesetterTarget.frameBeginTime(), vsyncId); } if (mVisibleRegionsDirty) mHdrLayerInfoChanged = true; @@ -2749,6 +2698,17 @@ void SurfaceFlinger::composite(TimePoint frameTime, VsyncId vsyncId) if (mPowerHintSessionEnabled) { mPowerAdvisor->setCompositeEnd(TimePoint::now()); } + + CompositeResultsPerDisplay resultsPerDisplay; + + // Filter out virtual displays. + for (const auto& [id, coverage] : mCompositionCoverage) { + if (const auto idOpt = PhysicalDisplayId::tryCast(id)) { + resultsPerDisplay.try_emplace(*idOpt, CompositeResult{coverage}); + } + } + + return resultsPerDisplay; } void SurfaceFlinger::updateLayerGeometry() { @@ -2776,7 +2736,14 @@ bool SurfaceFlinger::isHdrLayer(const frontend::LayerSnapshot& snapshot) const { return false; } } - if (isHdrDataspace(snapshot.dataspace)) { + // RANGE_EXTENDED layer may identify themselves as being "HDR" + // via a desired hdr/sdr ratio + auto pixelFormat = snapshot.buffer + ? std::make_optional(static_cast<ui::PixelFormat>(snapshot.buffer->getPixelFormat())) + : std::nullopt; + + if (getHdrRenderType(snapshot.dataspace, pixelFormat, snapshot.desiredHdrSdrRatio) != + HdrRenderType::SDR) { return true; } // If the layer is not allowed to be dimmed, treat it as HDR. WindowManager may disable @@ -2786,12 +2753,6 @@ bool SurfaceFlinger::isHdrLayer(const frontend::LayerSnapshot& snapshot) const { if (!snapshot.dimmingEnabled) { return true; } - // RANGE_EXTENDED layers may identify themselves as being "HDR" via a desired sdr/hdr ratio - if ((snapshot.dataspace & (int32_t)Dataspace::RANGE_MASK) == - (int32_t)Dataspace::RANGE_EXTENDED && - snapshot.desiredHdrSdrRatio > 1.01f) { - return true; - } return false; } @@ -2832,38 +2793,56 @@ ui::Rotation SurfaceFlinger::getPhysicalDisplayOrientation(DisplayId displayId, return ui::ROTATION_0; } -void SurfaceFlinger::postComposition(nsecs_t callTime) { +void SurfaceFlinger::postComposition(PhysicalDisplayId pacesetterId, + const scheduler::FrameTargeters& frameTargeters, + nsecs_t presentStartTime) { ATRACE_CALL(); ALOGV(__func__); - const auto* defaultDisplay = FTL_FAKE_GUARD(mStateLock, getDefaultDisplayDeviceLocked()).get(); + ui::PhysicalDisplayMap<PhysicalDisplayId, std::shared_ptr<FenceTime>> presentFences; + ui::PhysicalDisplayMap<PhysicalDisplayId, const sp<Fence>> gpuCompositionDoneFences; - std::shared_ptr<FenceTime> glCompositionDoneFenceTime; - if (defaultDisplay && - defaultDisplay->getCompositionDisplay()->getState().usesClientComposition) { - glCompositionDoneFenceTime = - std::make_shared<FenceTime>(defaultDisplay->getCompositionDisplay() - ->getRenderSurface() - ->getClientTargetAcquireFence()); - } else { - glCompositionDoneFenceTime = FenceTime::NO_FENCE; + for (const auto& [id, targeter] : frameTargeters) { + auto presentFence = getHwComposer().getPresentFence(id); + + if (id == pacesetterId) { + mTransactionCallbackInvoker.addPresentFence(presentFence); + } + + if (auto fenceTime = targeter->setPresentFence(std::move(presentFence)); + fenceTime->isValid()) { + presentFences.try_emplace(id, std::move(fenceTime)); + } + + ftl::FakeGuard guard(mStateLock); + if (const auto display = getCompositionDisplayLocked(id); + display && display->getState().usesClientComposition) { + gpuCompositionDoneFences + .try_emplace(id, display->getRenderSurface()->getClientTargetAcquireFence()); + } } - mPreviousPresentFences[1] = mPreviousPresentFences[0]; + const auto pacesetterDisplay = FTL_FAKE_GUARD(mStateLock, getDisplayDeviceLocked(pacesetterId)); - auto presentFence = defaultDisplay - ? getHwComposer().getPresentFence(defaultDisplay->getPhysicalId()) - : Fence::NO_FENCE; + std::shared_ptr<FenceTime> pacesetterPresentFenceTime = + presentFences.get(pacesetterId) + .transform([](const FenceTimePtr& ptr) { return ptr; }) + .value_or(FenceTime::NO_FENCE); - auto presentFenceTime = std::make_shared<FenceTime>(presentFence); - mPreviousPresentFences[0] = {presentFence, presentFenceTime}; + std::shared_ptr<FenceTime> pacesetterGpuCompositionDoneFenceTime = + gpuCompositionDoneFences.get(pacesetterId) + .transform([](sp<Fence> fence) { + return std::make_shared<FenceTime>(std::move(fence)); + }) + .value_or(FenceTime::NO_FENCE); const TimePoint presentTime = TimePoint::now(); // Set presentation information before calling Layer::releasePendingBuffer, such that jank // information from previous' frame classification is already available when sending jank info // to clients, so they get jank classification as early as possible. - mFrameTimeline->setSfPresent(presentTime.ns(), presentFenceTime, glCompositionDoneFenceTime); + mFrameTimeline->setSfPresent(presentTime.ns(), pacesetterPresentFenceTime, + pacesetterGpuCompositionDoneFenceTime); // We use the CompositionEngine::getLastFrameRefreshTimestamp() which might // be sampled a little later than when we started doing work for this frame, @@ -2871,9 +2850,9 @@ void SurfaceFlinger::postComposition(nsecs_t callTime) { const TimePoint compositeTime = TimePoint::fromNs(mCompositionEngine->getLastFrameRefreshTimestamp()); const Duration presentLatency = - !getHwComposer().hasCapability(Capability::PRESENT_FENCE_IS_NOT_RELIABLE) - ? mPresentLatencyTracker.trackPendingFrame(compositeTime, presentFenceTime) - : Duration::zero(); + getHwComposer().hasCapability(Capability::PRESENT_FENCE_IS_NOT_RELIABLE) + ? Duration::zero() + : mPresentLatencyTracker.trackPendingFrame(compositeTime, pacesetterPresentFenceTime); const auto schedule = mScheduler->getVsyncSchedule(); const TimePoint vsyncDeadline = schedule->vsyncDeadlineAfter(presentTime); @@ -2883,7 +2862,7 @@ void SurfaceFlinger::postComposition(nsecs_t callTime) { const CompositorTiming compositorTiming(vsyncDeadline.ns(), vsyncPeriod.ns(), vsyncPhase, presentLatency.ns()); - display::DisplayMap<ui::LayerStack, const DisplayDevice*> layerStackToDisplay; + ui::DisplayMap<ui::LayerStack, const DisplayDevice*> layerStackToDisplay; { if (!mLayersWithBuffersRemoved.empty() || mNumTrustedPresentationListeners > 0) { Mutex::Autolock lock(mStateLock); @@ -2910,8 +2889,8 @@ void SurfaceFlinger::postComposition(nsecs_t callTime) { mLayersWithBuffersRemoved.clear(); for (const auto& layer: mLayersWithQueuedFrames) { - layer->onPostComposition(defaultDisplay, glCompositionDoneFenceTime, presentFenceTime, - compositorTiming); + layer->onPostComposition(pacesetterDisplay.get(), pacesetterGpuCompositionDoneFenceTime, + pacesetterPresentFenceTime, compositorTiming); layer->releasePendingBuffer(presentTime.ns()); } @@ -2974,34 +2953,28 @@ void SurfaceFlinger::postComposition(nsecs_t callTime) { mHdrLayerInfoChanged = false; - mTransactionCallbackInvoker.addPresentFence(std::move(presentFence)); mTransactionCallbackInvoker.sendCallbacks(false /* onCommitOnly */); mTransactionCallbackInvoker.clearCompletedTransactions(); mTimeStats->incrementTotalFrames(); - mTimeStats->setPresentFenceGlobal(presentFenceTime); + mTimeStats->setPresentFenceGlobal(pacesetterPresentFenceTime); - { + for (auto&& [id, presentFence] : presentFences) { ftl::FakeGuard guard(mStateLock); - for (const auto& [id, physicalDisplay] : mPhysicalDisplays) { - if (auto displayDevice = getDisplayDeviceLocked(id); - displayDevice && displayDevice->isPoweredOn() && physicalDisplay.isInternal()) { - auto presentFenceTimeI = defaultDisplay && defaultDisplay->getPhysicalId() == id - ? std::move(presentFenceTime) - : std::make_shared<FenceTime>(getHwComposer().getPresentFence(id)); - if (presentFenceTimeI->isValid()) { - mScheduler->addPresentFence(id, std::move(presentFenceTimeI)); - } - } + const bool isInternalDisplay = + mPhysicalDisplays.get(id).transform(&PhysicalDisplay::isInternal).value_or(false); + + if (isInternalDisplay) { + mScheduler->addPresentFence(id, std::move(presentFence)); } } - const bool isDisplayConnected = - defaultDisplay && getHwComposer().isConnected(defaultDisplay->getPhysicalId()); + const bool hasPacesetterDisplay = + pacesetterDisplay && getHwComposer().isConnected(pacesetterId); if (!hasSyncFramework) { - if (isDisplayConnected && defaultDisplay->isPoweredOn()) { - mScheduler->enableHardwareVsync(defaultDisplay->getPhysicalId()); + if (hasPacesetterDisplay && pacesetterDisplay->isPoweredOn()) { + mScheduler->enableHardwareVsync(pacesetterId); } } @@ -3009,7 +2982,7 @@ void SurfaceFlinger::postComposition(nsecs_t callTime) { const size_t appConnections = mScheduler->getEventThreadConnectionCount(mAppConnectionHandle); mTimeStats->recordDisplayEventConnectionCount(sfConnections + appConnections); - if (isDisplayConnected && !defaultDisplay->isPoweredOn()) { + if (hasPacesetterDisplay && !pacesetterDisplay->isPoweredOn()) { getRenderEngine().cleanupPostRender(); return; } @@ -3040,7 +3013,7 @@ void SurfaceFlinger::postComposition(nsecs_t callTime) { if (!layer->hasTrustedPresentationListener()) { return; } - const frontend::LayerSnapshot* snapshot = (mLayerLifecycleManagerEnabled) + const frontend::LayerSnapshot* snapshot = mLayerLifecycleManagerEnabled ? mLayerSnapshotBuilder.getSnapshot(layer->sequence) : layer->getLayerSnapshot(); std::optional<const DisplayDevice*> displayOpt = std::nullopt; @@ -3049,7 +3022,8 @@ void SurfaceFlinger::postComposition(nsecs_t callTime) { } const DisplayDevice* display = displayOpt.value_or(nullptr); layer->updateTrustedPresentationState(display, snapshot, - nanoseconds_to_milliseconds(callTime), false); + nanoseconds_to_milliseconds(presentStartTime), + false); }); } @@ -3281,6 +3255,16 @@ void SurfaceFlinger::dispatchDisplayHotplugEvent(PhysicalDisplayId displayId, bo mScheduler->onHotplugReceived(mSfConnectionHandle, displayId, connected); } +void SurfaceFlinger::dispatchDisplayModeChangeEvent(PhysicalDisplayId displayId, + const scheduler::FrameRateMode& mode) { + // TODO(b/255635821): Merge code paths and move to Scheduler. + const auto onDisplayModeChanged = displayId == mActiveDisplayId + ? &scheduler::Scheduler::onPrimaryDisplayModeChanged + : &scheduler::Scheduler::onNonPrimaryDisplayModeChanged; + + ((*mScheduler).*onDisplayModeChanged)(mAppConnectionHandle, mode); +} + sp<DisplayDevice> SurfaceFlinger::setupNewDisplayDeviceInternal( const wp<IBinder>& displayToken, std::shared_ptr<compositionengine::Display> compositionDisplay, @@ -3380,14 +3364,8 @@ sp<DisplayDevice> SurfaceFlinger::setupNewDisplayDeviceInternal( Dataspace::UNKNOWN}); if (const auto& physical = state.physical) { - mPhysicalDisplays.get(physical->id) - .transform(&PhysicalDisplay::snapshotRef) - .transform(ftl::unit_fn([&](const display::DisplaySnapshot& snapshot) { - FTL_FAKE_GUARD(kMainThreadContext, - display->setActiveMode(physical->activeMode->getId(), - physical->activeMode->getFps(), - physical->activeMode->getFps())); - })); + const auto& mode = *physical->activeMode; + display->setActiveMode(mode.getId(), mode.getFps(), mode.getFps()); } display->setLayerFilter(makeLayerFilterForDisplay(display->getId(), state.layerStack)); @@ -3782,7 +3760,8 @@ void SurfaceFlinger::updateInputFlinger(VsyncId vsyncId, TimePoint frameTime) { mWindowInfosListenerInvoker ->windowInfosChanged(gui::WindowInfosUpdate{std::move(windowInfos), std::move(displayInfos), - vsyncId.value, frameTime.ns()}, + ftl::to_underlying(vsyncId), + frameTime.ns()}, std::move( inputWindowCommands.windowInfosReportedListeners), /* forceImmediateCall= */ visibleWindowsChanged || @@ -3872,11 +3851,17 @@ void SurfaceFlinger::updateCursorAsync() { refreshArgs.outputs.push_back(display->getCompositionDisplay()); } } - auto layers = moveSnapshotsToCompositionArgs(refreshArgs, /*cursorOnly=*/true, 0); + + constexpr bool kCursorOnly = true; + const auto layers = moveSnapshotsToCompositionArgs(refreshArgs, kCursorOnly); mCompositionEngine->updateCursorAsync(refreshArgs); moveSnapshotsFromCompositionArgs(refreshArgs, layers); } +void SurfaceFlinger::requestHardwareVsync(PhysicalDisplayId displayId, bool enable) { + getHwComposer().setVsyncEnabled(displayId, enable ? hal::Vsync::ENABLE : hal::Vsync::DISABLE); +} + void SurfaceFlinger::requestDisplayModes(std::vector<display::DisplayModeRequest> modeRequests) { if (mBootStage != BootStage::FINISHED) { ALOGV("Currently in the boot stage, skipping display mode changes"); @@ -3899,12 +3884,9 @@ void SurfaceFlinger::requestDisplayModes(std::vector<display::DisplayModeRequest if (!display) continue; - const bool isInternalDisplay = mPhysicalDisplays.get(displayId) - .transform(&PhysicalDisplay::isInternal) - .value_or(false); - - if (isInternalDisplay && displayId != mActiveDisplayId) { - ALOGV("%s(%s): Inactive display", __func__, to_string(displayId).c_str()); + if (ftl::FakeGuard guard(kMainThreadContext); + !shouldApplyRefreshRateSelectorPolicy(*display)) { + ALOGV("%s(%s): Skipped applying policy", __func__, to_string(displayId).c_str()); continue; } @@ -3912,7 +3894,7 @@ void SurfaceFlinger::requestDisplayModes(std::vector<display::DisplayModeRequest setDesiredActiveMode(std::move(request)); } else { ALOGV("%s: Mode %d is disallowed for display %s", __func__, modePtr->getId().value(), - to_string(display->getId()).c_str()); + to_string(displayId).c_str()); } } } @@ -3926,6 +3908,10 @@ void SurfaceFlinger::triggerOnFrameRateOverridesChanged() { mScheduler->onFrameRateOverridesChanged(mAppConnectionHandle, displayId); } +void SurfaceFlinger::notifyCpuLoadUp() { + mPowerAdvisor->notifyCpuLoadUp(); +} + void SurfaceFlinger::initScheduler(const sp<const DisplayDevice>& display) { using namespace scheduler; @@ -3942,6 +3928,9 @@ void SurfaceFlinger::initScheduler(const sp<const DisplayDevice>& display) { if (sysprop::use_content_detection_for_refresh_rate(false)) { features |= Feature::kContentDetection; + if (base::GetBoolProperty("debug.sf.enable_small_dirty_detection"s, false)) { + features |= Feature::kSmallDirtyContentDetection; + } } if (base::GetBoolProperty("debug.sf.show_predicted_vsync"s, false)) { features |= Feature::kTracePredictedVsync; @@ -3953,6 +3942,9 @@ void SurfaceFlinger::initScheduler(const sp<const DisplayDevice>& display) { if (display->refreshRateSelector().kernelIdleTimerController()) { features |= Feature::kKernelIdleTimer; } + if (mBackpressureGpuComposition) { + features |= Feature::kBackpressureGpuComposition; + } auto modulatorPtr = sp<VsyncModulator>::make(mVsyncConfiguration->getCurrentConfigs()); @@ -3960,8 +3952,6 @@ void SurfaceFlinger::initScheduler(const sp<const DisplayDevice>& display) { static_cast<ISchedulerCallback&>(*this), features, std::move(modulatorPtr)); mScheduler->registerDisplay(display->getPhysicalId(), display->holdRefreshRateSelector()); - - setVsyncEnabled(display->getPhysicalId(), false); mScheduler->startTimers(); const auto configs = mVsyncConfiguration->getCurrentConfigs(); @@ -4272,33 +4262,38 @@ void SurfaceFlinger::setTransactionFlags(uint32_t mask, TransactionSchedule sche TransactionHandler::TransactionReadiness SurfaceFlinger::transactionReadyTimelineCheck( const TransactionHandler::TransactionFlushState& flushState) { - using TransactionReadiness = TransactionHandler::TransactionReadiness; const auto& transaction = *flushState.transaction; - TimePoint desiredPresentTime = TimePoint::fromNs(transaction.desiredPresentTime); + + const TimePoint desiredPresentTime = TimePoint::fromNs(transaction.desiredPresentTime); + const TimePoint expectedPresentTime = mScheduler->expectedPresentTimeForPacesetter(); + + using TransactionReadiness = TransactionHandler::TransactionReadiness; + // Do not present if the desiredPresentTime has not passed unless it is more than // one second in the future. We ignore timestamps more than 1 second in the future // for stability reasons. - if (!transaction.isAutoTimestamp && desiredPresentTime >= mExpectedPresentTime && - desiredPresentTime < mExpectedPresentTime + 1s) { + if (!transaction.isAutoTimestamp && desiredPresentTime >= expectedPresentTime && + desiredPresentTime < expectedPresentTime + 1s) { ATRACE_FORMAT("not current desiredPresentTime: %" PRId64 " expectedPresentTime: %" PRId64, - desiredPresentTime, mExpectedPresentTime); + desiredPresentTime, expectedPresentTime); return TransactionReadiness::NotReady; } - if (!mScheduler->isVsyncValid(mExpectedPresentTime, transaction.originUid)) { - ATRACE_FORMAT("!isVsyncValid expectedPresentTime: %" PRId64 " uid: %d", - mExpectedPresentTime, transaction.originUid); + if (!mScheduler->isVsyncValid(expectedPresentTime, transaction.originUid)) { + ATRACE_FORMAT("!isVsyncValid expectedPresentTime: %" PRId64 " uid: %d", expectedPresentTime, + transaction.originUid); return TransactionReadiness::NotReady; } // If the client didn't specify desiredPresentTime, use the vsyncId to determine the // expected present time of this transaction. if (transaction.isAutoTimestamp && - frameIsEarly(mExpectedPresentTime, VsyncId{transaction.frameTimelineInfo.vsyncId})) { + frameIsEarly(expectedPresentTime, VsyncId{transaction.frameTimelineInfo.vsyncId})) { ATRACE_FORMAT("frameIsEarly vsyncId: %" PRId64 " expectedPresentTime: %" PRId64, - transaction.frameTimelineInfo.vsyncId, mExpectedPresentTime); + transaction.frameTimelineInfo.vsyncId, expectedPresentTime); return TransactionReadiness::NotReady; } + return TransactionReadiness::Ready; } @@ -4381,9 +4376,13 @@ TransactionHandler::TransactionReadiness SurfaceFlinger::transactionReadyBufferC (flushState.queueProcessTime - transaction.postTime) > std::chrono::nanoseconds(4s).count()) { mTransactionHandler - .onTransactionQueueStalled(transaction.id, listener, - "Buffer processing hung up due to stuck " - "fence. Indicates GPU hang"); + .onTransactionQueueStalled(transaction.id, + {.pid = layer->getOwnerPid(), + .layerId = static_cast<uint32_t>( + layer->getSequence()), + .layerName = layer->getDebugName(), + .bufferId = s.bufferData->getId(), + .frameNumber = s.bufferData->frameNumber}); } ATRACE_FORMAT("fence unsignaled %s", layer->getDebugName()); return TraverseBuffersReturnValues::STOP_TRAVERSAL; @@ -4437,7 +4436,7 @@ bool SurfaceFlinger::transactionFlushNeeded() { bool SurfaceFlinger::frameIsEarly(TimePoint expectedPresentTime, VsyncId vsyncId) const { const auto prediction = - mFrameTimeline->getTokenManager()->getPredictionsForToken(vsyncId.value); + mFrameTimeline->getTokenManager()->getPredictionsForToken(ftl::to_underlying(vsyncId)); if (!prediction) { return false; } @@ -4458,26 +4457,27 @@ bool SurfaceFlinger::frameIsEarly(TimePoint expectedPresentTime, VsyncId vsyncId bool SurfaceFlinger::shouldLatchUnsignaled(const sp<Layer>& layer, const layer_state_t& state, size_t numStates, bool firstTransaction) const { if (enableLatchUnsignaledConfig == LatchUnsignaledConfig::Disabled) { - ALOGV("%s: false (LatchUnsignaledConfig::Disabled)", __func__); + ATRACE_FORMAT_INSTANT("%s: false (LatchUnsignaledConfig::Disabled)", __func__); return false; } if (enableLatchUnsignaledConfig == LatchUnsignaledConfig::Always) { - ALOGV("%s: true (LatchUnsignaledConfig::Always)", __func__); + ATRACE_FORMAT_INSTANT("%s: true (LatchUnsignaledConfig::Always)", __func__); return true; } // We only want to latch unsignaled when a single layer is updated in this // transaction (i.e. not a blast sync transaction). if (numStates != 1) { - ALOGV("%s: false (numStates=%zu)", __func__, numStates); + ATRACE_FORMAT_INSTANT("%s: false (numStates=%zu)", __func__, numStates); return false; } if (enableLatchUnsignaledConfig == LatchUnsignaledConfig::AutoSingleLayer) { if (!firstTransaction) { - ALOGV("%s: false (LatchUnsignaledConfig::AutoSingleLayer; not first transaction)", - __func__); + ATRACE_FORMAT_INSTANT("%s: false (LatchUnsignaledConfig::AutoSingleLayer; not first " + "transaction)", + __func__); return false; } @@ -4485,19 +4485,14 @@ bool SurfaceFlinger::shouldLatchUnsignaled(const sp<Layer>& layer, const layer_s // as it leads to jank due to RenderEngine waiting for unsignaled buffer // or window animations being slow. if (mScheduler->vsyncModulator().isVsyncConfigEarly()) { - ALOGV("%s: false (LatchUnsignaledConfig::AutoSingleLayer; isVsyncConfigEarly)", - __func__); + ATRACE_FORMAT_INSTANT("%s: false (LatchUnsignaledConfig::AutoSingleLayer; " + "isVsyncConfigEarly)", + __func__); return false; } } - if (!layer->simpleBufferUpdate(state)) { - ALOGV("%s: false (!simpleBufferUpdate)", __func__); - return false; - } - - ALOGV("%s: true", __func__); - return true; + return layer->isSimpleBufferUpdate(state); } status_t SurfaceFlinger::setTransactionState( @@ -4562,6 +4557,9 @@ status_t SurfaceFlinger::setTransactionState( resolvedState.externalTexture = getExternalTextureFromBufferData(*resolvedState.state.bufferData, layerName.c_str(), transactionId); + if (resolvedState.externalTexture) { + resolvedState.state.bufferData->buffer = resolvedState.externalTexture->getBuffer(); + } mBufferCountTracker.increment(resolvedState.state.surface->localBinder()); } resolvedState.layerId = LayerHandle::getLayerId(resolvedState.state.surface); @@ -5411,6 +5409,8 @@ void SurfaceFlinger::onHandleDestroyed(BBinder* handle, sp<Layer>& layer, uint32 mDestroyedHandles.emplace_back(layerId); } + mTransactionHandler.onLayerDestroyed(layerId); + Mutex::Autolock lock(mStateLock); markLayerPendingRemovalLocked(layer); layer->onHandleDestroyed(); @@ -5521,11 +5521,14 @@ void SurfaceFlinger::setPowerModeInternal(const sp<DisplayDevice>& display, hal: getHwComposer().setPowerMode(displayId, mode); if (displayId == mActiveDisplayId && mode != hal::PowerMode::DOZE_SUSPEND) { - setHWCVsyncEnabled(displayId, - mScheduler->getVsyncSchedule(displayId) - ->getPendingHardwareVsyncState()); + const bool enable = + mScheduler->getVsyncSchedule(displayId)->getPendingHardwareVsyncState(); + requestHardwareVsync(displayId, enable); + mScheduler->enableSyntheticVsync(false); - mScheduler->resyncToHardwareVsync(displayId, true /* allowToEnable */, refreshRate); + + constexpr bool kAllowToEnable = true; + mScheduler->resyncToHardwareVsync(displayId, kAllowToEnable, refreshRate); } mVisibleRegionsDirty = true; @@ -5534,25 +5537,29 @@ void SurfaceFlinger::setPowerModeInternal(const sp<DisplayDevice>& display, hal: // Turn off the display if (displayId == mActiveDisplayId) { - if (setSchedFifo(false) != NO_ERROR) { - ALOGW("Failed to set SCHED_OTHER after powering off active display: %s", - strerror(errno)); - } - if (setSchedAttr(false) != NO_ERROR) { - ALOGW("Failed set uclamp.min after powering off active display: %s", - strerror(errno)); - } + if (const auto display = getActivatableDisplay()) { + onActiveDisplayChangedLocked(activeDisplay.get(), *display); + } else { + if (setSchedFifo(false) != NO_ERROR) { + ALOGW("Failed to set SCHED_OTHER after powering off active display: %s", + strerror(errno)); + } + if (setSchedAttr(false) != NO_ERROR) { + ALOGW("Failed set uclamp.min after powering off active display: %s", + strerror(errno)); + } - if (*currentModeOpt != hal::PowerMode::DOZE_SUSPEND) { - mScheduler->disableHardwareVsync(displayId, true); - mScheduler->enableSyntheticVsync(); + if (*currentModeOpt != hal::PowerMode::DOZE_SUSPEND) { + mScheduler->disableHardwareVsync(displayId, true); + mScheduler->enableSyntheticVsync(); + } } } - // Make sure HWVsync is disabled before turning off the display - setHWCVsyncEnabled(displayId, false); - + // Disable VSYNC before turning off the display. + requestHardwareVsync(displayId, false); getHwComposer().setPowerMode(displayId, mode); + mVisibleRegionsDirty = true; // from this point on, SF will stop drawing on this display } else if (mode == hal::PowerMode::DOZE || mode == hal::PowerMode::ON) { @@ -5580,9 +5587,10 @@ void SurfaceFlinger::setPowerModeInternal(const sp<DisplayDevice>& display, hal: if (displayId == mActiveDisplayId) { mTimeStats->setPowerMode(mode); mRefreshRateStats->setPowerMode(mode); - mScheduler->setDisplayPowerMode(displayId, mode); } + mScheduler->setDisplayPowerMode(displayId, mode); + ALOGD("Finished setting power mode %d on display %s", mode, to_string(displayId).c_str()); } @@ -6037,10 +6045,6 @@ void SurfaceFlinger::dumpAllLocked(const DumpArgs& args, const std::string& comp dumpVsync(result); result.append("\n"); - StringAppendF(&result, "Total missed frame count: %u\n", mFrameMissedCount.load()); - StringAppendF(&result, "HWC missed frame count: %u\n", mHwcFrameMissedCount.load()); - StringAppendF(&result, "GPU missed frame count: %u\n\n", mGpuFrameMissedCount.load()); - /* * Dump the visible layer list */ @@ -6154,7 +6158,7 @@ void SurfaceFlinger::dumpAllLocked(const DumpArgs& args, const std::string& comp result.append("Window Infos:\n"); auto windowInfosDebug = mWindowInfosListenerInvoker->getDebugInfo(); StringAppendF(&result, " max send vsync id: %" PRId64 "\n", - windowInfosDebug.maxSendDelayVsyncId.value); + ftl::to_underlying(windowInfosDebug.maxSendDelayVsyncId)); StringAppendF(&result, " max send delay (ns): %" PRId64 " ns\n", windowInfosDebug.maxSendDelayDuration); StringAppendF(&result, " unsent messages: %zu\n", windowInfosDebug.pendingMessageCount); @@ -6473,13 +6477,16 @@ status_t SurfaceFlinger::onTransact(uint32_t code, const Parcel& data, Parcel* r ALOGD("LayerTracing enabled"); tracingEnabledChanged = mLayerTracing.enable(); if (tracingEnabledChanged) { - int64_t startingTime = - (fixedStartingTime) ? fixedStartingTime : systemTime(); + const TimePoint startingTime = fixedStartingTime + ? TimePoint::fromNs(fixedStartingTime) + : TimePoint::now(); + mScheduler - ->schedule([&]() FTL_FAKE_GUARD(mStateLock) FTL_FAKE_GUARD( - kMainThreadContext) { - addToLayerTracing(true /* visibleRegionDirty */, startingTime, - mLastCommittedVsyncId.value); + ->schedule([this, startingTime]() FTL_FAKE_GUARD( + mStateLock) FTL_FAKE_GUARD(kMainThreadContext) { + constexpr bool kVisibleRegionDirty = true; + addToLayerTracing(kVisibleRegionDirty, startingTime, + mLastCommittedVsyncId); }) .wait(); } @@ -6737,7 +6744,7 @@ status_t SurfaceFlinger::onTransact(uint32_t code, const Parcel& data, Parcel* r mTransactionTracing->setBufferSize( TransactionTracing::ACTIVE_TRACING_BUFFER_SIZE); } else { - mTransactionTracing->writeToFile(); + TransactionTraceWriter::getInstance().invoke("", /* overwrite= */ true); mTransactionTracing->setBufferSize( TransactionTracing::CONTINUOUS_TRACING_BUFFER_SIZE); } @@ -7365,6 +7372,7 @@ ftl::SharedFuture<FenceResult> SurfaceFlinger::renderScreenImpl( ATRACE_CALL(); auto layers = getLayerSnapshots(); + for (auto& [_, layerFE] : layers) { frontend::LayerSnapshot* snapshot = layerFE->mSnapshot.get(); captureResults.capturedSecureLayers |= (snapshot->isVisible && snapshot->isSecure); @@ -7416,16 +7424,27 @@ ftl::SharedFuture<FenceResult> SurfaceFlinger::renderScreenImpl( pickBestDataspace(requestedDataspace, display, captureResults.capturedHdrLayers, renderArea->getHintForSeamlessTransition()); sdrWhitePointNits = state.sdrWhitePointNits; - displayBrightnessNits = state.displayBrightnessNits; - if (sdrWhitePointNits > 1.0f) { - // Restrict the amount of HDR "headroom" in the screenshot to avoid over-dimming - // the SDR portion. 2.0 chosen by experimentation - constexpr float kMaxScreenshotHeadroom = 2.0f; - displayBrightnessNits = - std::min(sdrWhitePointNits * kMaxScreenshotHeadroom, displayBrightnessNits); + + if (!captureResults.capturedHdrLayers) { + displayBrightnessNits = sdrWhitePointNits; + } else { + displayBrightnessNits = state.displayBrightnessNits; + // Only clamp the display brightness if this is not a seamless transition. Otherwise + // for seamless transitions it's important to match the current display state as the + // buffer will be shown under these same conditions, and we want to avoid any + // flickers + if (sdrWhitePointNits > 1.0f && !renderArea->getHintForSeamlessTransition()) { + // Restrict the amount of HDR "headroom" in the screenshot to avoid over-dimming + // the SDR portion. 2.0 chosen by experimentation + constexpr float kMaxScreenshotHeadroom = 2.0f; + displayBrightnessNits = std::min(sdrWhitePointNits * kMaxScreenshotHeadroom, + displayBrightnessNits); + } } - if (requestedDataspace == ui::Dataspace::UNKNOWN) { + // Screenshots leaving the device should be colorimetric + if (requestedDataspace == ui::Dataspace::UNKNOWN && + renderArea->getHintForSeamlessTransition()) { renderIntent = state.renderIntent; } } @@ -7469,6 +7488,10 @@ ftl::SharedFuture<FenceResult> SurfaceFlinger::renderScreenImpl( } } + // Screenshots leaving the device must not dim in gamma space. + const bool dimInGammaSpaceForEnhancedScreenshots = mDimInGammaSpaceForEnhancedScreenshots && + renderArea->getHintForSeamlessTransition(); + std::shared_ptr<ScreenCaptureOutput> output = createScreenCaptureOutput( ScreenCaptureOutputArgs{.compositionEngine = *compositionEngine, .colorProfile = colorProfile, @@ -7478,7 +7501,10 @@ ftl::SharedFuture<FenceResult> SurfaceFlinger::renderScreenImpl( .sdrWhitePointNits = sdrWhitePointNits, .displayBrightnessNits = displayBrightnessNits, .targetBrightness = targetBrightness, - .regionSampling = regionSampling}); + .regionSampling = regionSampling, + .treat170mAsSrgb = mTreat170mAsSrgb, + .dimInGammaSpaceForEnhancedScreenshots = + dimInGammaSpaceForEnhancedScreenshots}); const float colorSaturation = grayscale ? 0 : 1; compositionengine::CompositionRefreshArgs refreshArgs{ @@ -7603,6 +7629,7 @@ status_t SurfaceFlinger::setDesiredDisplayModeSpecsInternal( const sp<DisplayDevice>& display, const scheduler::RefreshRateSelector::PolicyVariant& policy) { const auto displayId = display->getPhysicalId(); + ATRACE_NAME(ftl::Concat(__func__, ' ', displayId.value).c_str()); Mutex::Autolock lock(mStateLock); @@ -7623,19 +7650,33 @@ status_t SurfaceFlinger::setDesiredDisplayModeSpecsInternal( break; } - const bool isInternalDisplay = mPhysicalDisplays.get(displayId) - .transform(&PhysicalDisplay::isInternal) - .value_or(false); - - if (isInternalDisplay && displayId != mActiveDisplayId) { - // The policy will be be applied when the display becomes active. - ALOGV("%s(%s): Inactive display", __func__, to_string(displayId).c_str()); + if (!shouldApplyRefreshRateSelectorPolicy(*display)) { + ALOGV("%s(%s): Skipped applying policy", __func__, to_string(displayId).c_str()); return NO_ERROR; } return applyRefreshRateSelectorPolicy(displayId, selector); } +bool SurfaceFlinger::shouldApplyRefreshRateSelectorPolicy(const DisplayDevice& display) const { + if (display.isPoweredOn() || mPhysicalDisplays.size() == 1) return true; + + LOG_ALWAYS_FATAL_IF(display.isVirtual()); + const auto displayId = display.getPhysicalId(); + + // The display is powered off, and this is a multi-display device. If the display is the + // inactive internal display of a dual-display foldable, then the policy will be applied + // when it becomes active upon powering on. + // + // TODO(b/255635711): Remove this function (i.e. returning `false` as a special case) once + // concurrent mode setting across multiple (potentially powered off) displays is supported. + // + return displayId == mActiveDisplayId || + !mPhysicalDisplays.get(displayId) + .transform(&PhysicalDisplay::isInternal) + .value_or(false); +} + status_t SurfaceFlinger::applyRefreshRateSelectorPolicy( PhysicalDisplayId displayId, const scheduler::RefreshRateSelector& selector, bool force) { const scheduler::RefreshRateSelector::Policy currentPolicy = selector.getCurrentPolicy(); @@ -7831,23 +7872,37 @@ status_t SurfaceFlinger::setOverrideFrameRate(uid_t uid, float frameRate) { return NO_ERROR; } +status_t SurfaceFlinger::updateSmallAreaDetection( + std::vector<std::pair<uid_t, float>>& uidThresholdMappings) { + mScheduler->updateSmallAreaDetection(uidThresholdMappings); + return NO_ERROR; +} + +status_t SurfaceFlinger::setSmallAreaDetectionThreshold(uid_t uid, float threshold) { + mScheduler->setSmallAreaDetectionThreshold(uid, threshold); + return NO_ERROR; +} + void SurfaceFlinger::enableRefreshRateOverlay(bool enable) { bool setByHwc = getHwComposer().hasCapability(Capability::REFRESH_RATE_CHANGED_CALLBACK_DEBUG); for (const auto& [id, display] : mPhysicalDisplays) { if (display.snapshot().connectionType() == ui::DisplayConnectionType::Internal) { - if (setByHwc) { - const auto status = - getHwComposer().setRefreshRateChangedCallbackDebugEnabled(id, enable); - if (status != NO_ERROR) { - ALOGE("Error updating the refresh rate changed callback debug enabled"); - return; - } - } - if (const auto device = getDisplayDeviceLocked(id)) { - device->enableRefreshRateOverlay(enable, setByHwc, mRefreshRateOverlaySpinner, - mRefreshRateOverlayRenderRate, - mRefreshRateOverlayShowInMiddle); + const auto enableOverlay = [&](const bool setByHwc) FTL_FAKE_GUARD( + kMainThreadContext) { + device->enableRefreshRateOverlay(enable, setByHwc, mRefreshRateOverlaySpinner, + mRefreshRateOverlayRenderRate, + mRefreshRateOverlayShowInMiddle); + }; + enableOverlay(setByHwc); + if (setByHwc) { + const auto status = + getHwComposer().setRefreshRateChangedCallbackDebugEnabled(id, enable); + if (status != NO_ERROR) { + ALOGE("Error updating the refresh rate changed callback debug enabled"); + enableOverlay(/*setByHwc*/ false); + } + } } } } @@ -7955,6 +8010,29 @@ void SurfaceFlinger::sample() { void SurfaceFlinger::onActiveDisplaySizeChanged(const DisplayDevice& activeDisplay) { mScheduler->onActiveDisplayAreaChanged(activeDisplay.getWidth() * activeDisplay.getHeight()); getRenderEngine().onActiveDisplaySizeChanged(activeDisplay.getSize()); + + // Notify layers to update small dirty flag. + if (mScheduler->supportSmallDirtyDetection()) { + mCurrentState.traverse([&](Layer* layer) { + if (layer->getLayerStack() == activeDisplay.getLayerStack()) { + layer->setIsSmallDirty(); + } + }); + } +} + +sp<DisplayDevice> SurfaceFlinger::getActivatableDisplay() const { + if (mPhysicalDisplays.size() == 1) return nullptr; + + // TODO(b/255635821): Choose the pacesetter display, considering both internal and external + // displays. For now, pick the other internal display, assuming a dual-display foldable. + return findDisplay([this](const DisplayDevice& display) REQUIRES(mStateLock) { + const auto idOpt = PhysicalDisplayId::tryCast(display.getId()); + return idOpt && *idOpt != mActiveDisplayId && display.isPoweredOn() && + mPhysicalDisplays.get(*idOpt) + .transform(&PhysicalDisplay::isInternal) + .value_or(false); + }); } void SurfaceFlinger::onActiveDisplayChangedLocked(const DisplayDevice* inactiveDisplayPtr, @@ -7975,7 +8053,9 @@ void SurfaceFlinger::onActiveDisplayChangedLocked(const DisplayDevice* inactiveD resetPhaseConfiguration(activeDisplay.getActiveMode().fps); + // TODO(b/255635711): Check for pending mode changes on other displays. mScheduler->setModeChangePending(false); + mScheduler->setPacesetterDisplay(mActiveDisplayId); onActiveDisplaySizeChanged(activeDisplay); @@ -8003,6 +8083,12 @@ status_t SurfaceFlinger::removeWindowInfosListener( return NO_ERROR; } +status_t SurfaceFlinger::getStalledTransactionInfo( + int pid, std::optional<TransactionHandler::StalledTransactionInfo>& result) { + result = mTransactionHandler.getStalledTransactionInfo(pid); + return NO_ERROR; +} + std::shared_ptr<renderengine::ExternalTexture> SurfaceFlinger::getExternalTextureFromBufferData( BufferData& bufferData, const char* layerName, uint64_t transactionId) { if (bufferData.buffer && @@ -8070,7 +8156,13 @@ bool SurfaceFlinger::commitMirrorDisplays(VsyncId vsyncId) { // Set mirror layer's default layer stack to -1 so it doesn't end up rendered on a display // accidentally. sp<Layer> rootMirrorLayer = LayerHandle::getLayer(mirrorDisplay.rootHandle); - rootMirrorLayer->setLayerStack(ui::LayerStack::fromValue(-1)); + ssize_t idx = mCurrentState.layersSortedByZ.indexOf(rootMirrorLayer); + bool ret = rootMirrorLayer->setLayerStack(ui::LayerStack::fromValue(-1)); + if (idx >= 0 && ret) { + mCurrentState.layersSortedByZ.removeAt(idx); + mCurrentState.layersSortedByZ.add(rootMirrorLayer); + } + for (const auto& layer : mDrawingState.layersSortedByZ) { if (layer->getLayerStack() != mirrorDisplay.layerStack || layer->isInternalDisplayOverlay()) { @@ -8136,7 +8228,7 @@ void SurfaceFlinger::updateLayerMetadataSnapshot() { void SurfaceFlinger::moveSnapshotsFromCompositionArgs( compositionengine::CompositionRefreshArgs& refreshArgs, - std::vector<std::pair<Layer*, LayerFE*>>& layers) { + const std::vector<std::pair<Layer*, LayerFE*>>& layers) { if (mLayerLifecycleManagerEnabled) { std::vector<std::unique_ptr<frontend::LayerSnapshot>>& snapshots = mLayerSnapshotBuilder.getSnapshots(); @@ -8153,7 +8245,7 @@ void SurfaceFlinger::moveSnapshotsFromCompositionArgs( } std::vector<std::pair<Layer*, LayerFE*>> SurfaceFlinger::moveSnapshotsToCompositionArgs( - compositionengine::CompositionRefreshArgs& refreshArgs, bool cursorOnly, int64_t vsyncId) { + compositionengine::CompositionRefreshArgs& refreshArgs, bool cursorOnly) { std::vector<std::pair<Layer*, LayerFE*>> layers; if (mLayerLifecycleManagerEnabled) { nsecs_t currentTime = systemTime(); @@ -8230,7 +8322,7 @@ SurfaceFlinger::getLayerSnapshotsForScreenshots( if (layerStack && snapshot->outputFilter.layerStack != *layerStack) { return; } - if (uid != CaptureArgs::UNSET_UID && snapshot->uid != uid) { + if (uid != CaptureArgs::UNSET_UID && snapshot->uid != gui::Uid(uid)) { return; } if (!snapshot->hasSomethingToDraw()) { @@ -8353,7 +8445,7 @@ frontend::Update SurfaceFlinger::flushLifecycleUpdates() { return update; } -void SurfaceFlinger::addToLayerTracing(bool visibleRegionDirty, int64_t time, int64_t vsyncId) { +void SurfaceFlinger::addToLayerTracing(bool visibleRegionDirty, TimePoint time, VsyncId vsyncId) { const uint32_t tracingFlags = mLayerTracing.getFlags(); LayersProto layers(dumpDrawingStateProto(tracingFlags)); if (tracingFlags & LayerTracing::TRACE_EXTRA) { @@ -8364,7 +8456,8 @@ void SurfaceFlinger::addToLayerTracing(bool visibleRegionDirty, int64_t time, in dumpHwc(hwcDump); } auto displays = dumpDisplayProto(); - mLayerTracing.notify(visibleRegionDirty, time, vsyncId, &layers, std::move(hwcDump), &displays); + mLayerTracing.notify(visibleRegionDirty, time.ns(), ftl::to_underlying(vsyncId), &layers, + std::move(hwcDump), &displays); } // gui::ISurfaceComposer @@ -8504,33 +8597,35 @@ binder::Status SurfaceComposerAIDL::getStaticDisplayInfo(int64_t displayId, outInfo->secure = info.secure; outInfo->installOrientation = static_cast<gui::Rotation>(info.installOrientation); - gui::DeviceProductInfo dinfo; - std::optional<DeviceProductInfo> dpi = info.deviceProductInfo; - dinfo.name = std::move(dpi->name); - dinfo.manufacturerPnpId = - std::vector<uint8_t>(dpi->manufacturerPnpId.begin(), dpi->manufacturerPnpId.end()); - dinfo.productId = dpi->productId; - dinfo.relativeAddress = - std::vector<uint8_t>(dpi->relativeAddress.begin(), dpi->relativeAddress.end()); - if (const auto* model = - std::get_if<DeviceProductInfo::ModelYear>(&dpi->manufactureOrModelDate)) { - gui::DeviceProductInfo::ModelYear modelYear; - modelYear.year = model->year; - dinfo.manufactureOrModelDate.set<Tag::modelYear>(modelYear); - } else if (const auto* manufacture = std::get_if<DeviceProductInfo::ManufactureYear>( - &dpi->manufactureOrModelDate)) { - gui::DeviceProductInfo::ManufactureYear date; - date.modelYear.year = manufacture->year; - dinfo.manufactureOrModelDate.set<Tag::manufactureYear>(date); - } else if (const auto* manufacture = std::get_if<DeviceProductInfo::ManufactureWeekAndYear>( - &dpi->manufactureOrModelDate)) { - gui::DeviceProductInfo::ManufactureWeekAndYear date; - date.manufactureYear.modelYear.year = manufacture->year; - date.week = manufacture->week; - dinfo.manufactureOrModelDate.set<Tag::manufactureWeekAndYear>(date); - } - - outInfo->deviceProductInfo = dinfo; + if (const std::optional<DeviceProductInfo> dpi = info.deviceProductInfo) { + gui::DeviceProductInfo dinfo; + dinfo.name = std::move(dpi->name); + dinfo.manufacturerPnpId = std::vector<uint8_t>(dpi->manufacturerPnpId.begin(), + dpi->manufacturerPnpId.end()); + dinfo.productId = dpi->productId; + dinfo.relativeAddress = + std::vector<uint8_t>(dpi->relativeAddress.begin(), dpi->relativeAddress.end()); + if (const auto* model = + std::get_if<DeviceProductInfo::ModelYear>(&dpi->manufactureOrModelDate)) { + gui::DeviceProductInfo::ModelYear modelYear; + modelYear.year = model->year; + dinfo.manufactureOrModelDate.set<Tag::modelYear>(modelYear); + } else if (const auto* manufacture = std::get_if<DeviceProductInfo::ManufactureYear>( + &dpi->manufactureOrModelDate)) { + gui::DeviceProductInfo::ManufactureYear date; + date.modelYear.year = manufacture->year; + dinfo.manufactureOrModelDate.set<Tag::manufactureYear>(date); + } else if (const auto* manufacture = + std::get_if<DeviceProductInfo::ManufactureWeekAndYear>( + &dpi->manufactureOrModelDate)) { + gui::DeviceProductInfo::ManufactureWeekAndYear date; + date.manufactureYear.modelYear.year = manufacture->year; + date.week = manufacture->week; + dinfo.manufactureOrModelDate.set<Tag::manufactureWeekAndYear>(date); + } + + outInfo->deviceProductInfo = dinfo; + } } return binderStatusFromStatusT(status); } @@ -9063,6 +9158,40 @@ binder::Status SurfaceComposerAIDL::setOverrideFrameRate(int32_t uid, float fram return binderStatusFromStatusT(status); } +binder::Status SurfaceComposerAIDL::updateSmallAreaDetection(const std::vector<int32_t>& uids, + const std::vector<float>& thresholds) { + status_t status; + const int c_uid = IPCThreadState::self()->getCallingUid(); + if (c_uid == AID_ROOT || c_uid == AID_SYSTEM) { + if (uids.size() != thresholds.size()) return binderStatusFromStatusT(BAD_VALUE); + + std::vector<std::pair<uid_t, float>> mappings; + const size_t size = uids.size(); + mappings.reserve(size); + for (int i = 0; i < size; i++) { + auto row = std::make_pair(static_cast<uid_t>(uids[i]), thresholds[i]); + mappings.push_back(row); + } + status = mFlinger->updateSmallAreaDetection(mappings); + } else { + ALOGE("updateSmallAreaDetection() permission denied for uid: %d", c_uid); + status = PERMISSION_DENIED; + } + return binderStatusFromStatusT(status); +} + +binder::Status SurfaceComposerAIDL::setSmallAreaDetectionThreshold(int32_t uid, float threshold) { + status_t status; + const int c_uid = IPCThreadState::self()->getCallingUid(); + if (c_uid == AID_ROOT || c_uid == AID_SYSTEM) { + status = mFlinger->setSmallAreaDetectionThreshold(uid, threshold); + } else { + ALOGE("setSmallAreaDetectionThreshold() permission denied for uid: %d", c_uid); + status = PERMISSION_DENIED; + } + return binderStatusFromStatusT(status); +} + binder::Status SurfaceComposerAIDL::getGpuContextPriority(int32_t* outPriority) { *outPriority = mFlinger->getGpuContextPriority(); return binder::Status::ok(); @@ -9104,6 +9233,28 @@ binder::Status SurfaceComposerAIDL::removeWindowInfosListener( return binderStatusFromStatusT(status); } +binder::Status SurfaceComposerAIDL::getStalledTransactionInfo( + int pid, std::optional<gui::StalledTransactionInfo>* outInfo) { + const int callingPid = IPCThreadState::self()->getCallingPid(); + const int callingUid = IPCThreadState::self()->getCallingUid(); + if (!checkPermission(sAccessSurfaceFlinger, callingPid, callingUid)) { + return binderStatusFromStatusT(PERMISSION_DENIED); + } + + std::optional<TransactionHandler::StalledTransactionInfo> stalledTransactionInfo; + status_t status = mFlinger->getStalledTransactionInfo(pid, stalledTransactionInfo); + if (stalledTransactionInfo) { + gui::StalledTransactionInfo result; + result.layerName = String16{stalledTransactionInfo->layerName.c_str()}, + result.bufferId = stalledTransactionInfo->bufferId, + result.frameNumber = stalledTransactionInfo->frameNumber, + outInfo->emplace(std::move(result)); + } else { + outInfo->reset(); + } + return binderStatusFromStatusT(status); +} + status_t SurfaceComposerAIDL::checkAccessPermission(bool usePermissionCache) { if (!mFlinger->callingThreadHasUnscopedSurfaceFlingerAccess(usePermissionCache)) { IPCThreadState* ipc = IPCThreadState::self(); diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h index d4700a4e25..47ada25d26 100644 --- a/services/surfaceflinger/SurfaceFlinger.h +++ b/services/surfaceflinger/SurfaceFlinger.h @@ -43,6 +43,7 @@ #include <renderengine/LayerSettings.h> #include <serviceutils/PriorityDumper.h> #include <system/graphics.h> +#include <ui/DisplayMap.h> #include <ui/FenceTime.h> #include <ui/PixelFormat.h> #include <ui/Size.h> @@ -62,7 +63,6 @@ #include <scheduler/interface/ICompositor.h> #include <ui/FenceResult.h> -#include "Display/DisplayMap.h" #include "Display/PhysicalDisplay.h" #include "DisplayDevice.h" #include "DisplayHardware/HWC2.h" @@ -194,7 +194,8 @@ class SurfaceFlinger : public BnSurfaceComposer, private IBinder::DeathRecipient, private HWC2::ComposerCallback, private ICompositor, - private scheduler::ISchedulerCallback { + private scheduler::ISchedulerCallback, + private compositionengine::ICEPowerCallback { public: struct SkipInitializationTag {}; @@ -322,6 +323,11 @@ public: // on this behavior to increase contrast for some media sources. bool mTreat170mAsSrgb = false; + // If true, then screenshots with an enhanced render intent will dim in gamma space. + // The purpose is to ensure that screenshots appear correct during system animations for devices + // that require that dimming must occur in gamma space. + bool mDimInGammaSpaceForEnhancedScreenshots = false; + // Allows to ignore physical orientation provided through hwc API in favour of // 'ro.surface_flinger.primary_display_orientation'. // TODO(b/246793311): Clean up a temporary property @@ -608,6 +614,10 @@ private: status_t setOverrideFrameRate(uid_t uid, float frameRate); + status_t updateSmallAreaDetection(std::vector<std::pair<uid_t, float>>& uidThresholdMappings); + + status_t setSmallAreaDetectionThreshold(uid_t uid, float threshold); + int getGpuContextPriority(); status_t getMaxAcquiredBufferCount(int* buffers) const; @@ -617,6 +627,9 @@ private: status_t removeWindowInfosListener( const sp<gui::IWindowInfosListener>& windowInfosListener) const; + status_t getStalledTransactionInfo( + int pid, std::optional<TransactionHandler::StalledTransactionInfo>& result); + // Implements IBinder::DeathRecipient. void binderDied(const wp<IBinder>& who) override; @@ -632,20 +645,24 @@ private: void onRefreshRateChangedDebug(const RefreshRateChangedDebugData&) override; // ICompositor overrides: - void configure() override; - bool commit(TimePoint frameTime, VsyncId, TimePoint expectedVsyncTime) override; - void composite(TimePoint frameTime, VsyncId) override; + void configure() override REQUIRES(kMainThreadContext); + bool commit(PhysicalDisplayId pacesetterId, const scheduler::FrameTargets&) override + REQUIRES(kMainThreadContext); + CompositeResultsPerDisplay composite(PhysicalDisplayId pacesetterId, + const scheduler::FrameTargeters&) override + REQUIRES(kMainThreadContext); + void sample() override; // ISchedulerCallback overrides: - - // Toggles hardware VSYNC by calling into HWC. - // TODO(b/241286146): Rename for self-explanatory API. - void setVsyncEnabled(PhysicalDisplayId, bool) override; + void requestHardwareVsync(PhysicalDisplayId, bool) override; void requestDisplayModes(std::vector<display::DisplayModeRequest>) override; void kernelTimerChanged(bool expired) override; void triggerOnFrameRateOverridesChanged() override; + // ICEPowerCallback overrides: + void notifyCpuLoadUp() override; + // Toggles the kernel idle timer on or off depending the policy decisions around refresh rates. void toggleKernelIdleTimer() REQUIRES(mStateLock); @@ -673,11 +690,10 @@ private: REQUIRES(mStateLock); status_t setActiveModeFromBackdoor(const sp<display::DisplayToken>&, DisplayModeId); - // Sets the active mode and a new refresh rate in SF. - void updateInternalStateWithChangedMode() REQUIRES(mStateLock, kMainThreadContext); - // Calls to setActiveMode on the main thread if there is a pending mode change - // that needs to be applied. - void setActiveModeInHwcIfNeeded() REQUIRES(mStateLock, kMainThreadContext); + + void initiateDisplayModeChanges() REQUIRES(mStateLock, kMainThreadContext); + void finalizeDisplayModeChange(DisplayDevice&) REQUIRES(mStateLock, kMainThreadContext); + void clearDesiredActiveModeState(const sp<DisplayDevice>&) REQUIRES(mStateLock); // Called when active mode is no longer is progress void desiredActiveModeChangeDone(const sp<DisplayDevice>&) REQUIRES(mStateLock); @@ -694,6 +710,9 @@ private: const sp<DisplayDevice>&, const scheduler::RefreshRateSelector::PolicyVariant&) EXCLUDES(mStateLock) REQUIRES(kMainThreadContext); + bool shouldApplyRefreshRateSelectorPolicy(const DisplayDevice&) const + REQUIRES(mStateLock, kMainThreadContext); + // TODO(b/241285191): Look up RefreshRateSelector on Scheduler to remove redundant parameter. status_t applyRefreshRateSelectorPolicy(PhysicalDisplayId, const scheduler::RefreshRateSelector&, @@ -711,10 +730,9 @@ private: void updateLayerGeometry(); void updateLayerMetadataSnapshot(); std::vector<std::pair<Layer*, LayerFE*>> moveSnapshotsToCompositionArgs( - compositionengine::CompositionRefreshArgs& refreshArgs, bool cursorOnly, - int64_t vsyncId); + compositionengine::CompositionRefreshArgs& refreshArgs, bool cursorOnly); void moveSnapshotsFromCompositionArgs(compositionengine::CompositionRefreshArgs& refreshArgs, - std::vector<std::pair<Layer*, LayerFE*>>& layers); + const std::vector<std::pair<Layer*, LayerFE*>>& layers); bool updateLayerSnapshotsLegacy(VsyncId vsyncId, frontend::Update& update, bool transactionsFlushed, bool& out) REQUIRES(kMainThreadContext); @@ -884,6 +902,14 @@ private: return findDisplay([id](const auto& display) { return display.getId() == id; }); } + std::shared_ptr<compositionengine::Display> getCompositionDisplayLocked(DisplayId id) const + REQUIRES(mStateLock) { + if (const auto display = getDisplayDeviceLocked(id)) { + return display->getCompositionDisplay(); + } + return nullptr; + } + // Returns the primary display or (for foldables) the active display, assuming that the inner // and outer displays have mutually exclusive power states. sp<const DisplayDevice> getDefaultDisplayDeviceLocked() const REQUIRES(mStateLock) { @@ -923,7 +949,8 @@ private: template <typename Predicate> sp<DisplayDevice> findDisplay(Predicate p) const REQUIRES(mStateLock) { const auto it = std::find_if(mDisplays.begin(), mDisplays.end(), - [&](const auto& pair) { return p(*pair.second); }); + [&](const auto& pair) + REQUIRES(mStateLock) { return p(*pair.second); }); return it == mDisplays.end() ? nullptr : it->second; } @@ -957,7 +984,8 @@ private: /* * Compositing */ - void postComposition(nsecs_t callTime) REQUIRES(kMainThreadContext); + void postComposition(PhysicalDisplayId pacesetterId, const scheduler::FrameTargeters&, + nsecs_t presentStartTime) REQUIRES(kMainThreadContext); /* * Display management @@ -991,32 +1019,15 @@ private: const DisplayDeviceState& drawingState) REQUIRES(mStateLock, kMainThreadContext); - void dispatchDisplayHotplugEvent(PhysicalDisplayId displayId, bool connected); + void dispatchDisplayHotplugEvent(PhysicalDisplayId, bool connected); + void dispatchDisplayModeChangeEvent(PhysicalDisplayId, const scheduler::FrameRateMode&) + REQUIRES(mStateLock); /* * VSYNC */ nsecs_t getVsyncPeriodFromHWC() const REQUIRES(mStateLock); - void setHWCVsyncEnabled(PhysicalDisplayId id, bool enabled) { - hal::Vsync halState = enabled ? hal::Vsync::ENABLE : hal::Vsync::DISABLE; - getHwComposer().setVsyncEnabled(id, halState); - } - - using FenceTimePtr = std::shared_ptr<FenceTime>; - - bool wouldPresentEarly(TimePoint frameTime, Period) const REQUIRES(kMainThreadContext); - - const FenceTimePtr& getPreviousPresentFence(TimePoint frameTime, Period) const - REQUIRES(kMainThreadContext); - - // Blocks the thread waiting for up to graceTimeMs in case the fence is about to signal. - static bool isFencePending(const FenceTimePtr&, int graceTimeMs); - - // Calculates the expected present time for this frame. For negative offsets, performs a - // correction using the predicted vsync for the next frame instead. - TimePoint calculateExpectedPresentTime(TimePoint frameTime) const; - /* * Display identification */ @@ -1052,6 +1063,9 @@ private: VirtualDisplayId acquireVirtualDisplay(ui::Size, ui::PixelFormat) REQUIRES(mStateLock); void releaseVirtualDisplay(VirtualDisplayId); + // Returns a display other than `mActiveDisplayId` that can be activated, if any. + sp<DisplayDevice> getActivatableDisplay() const REQUIRES(mStateLock, kMainThreadContext); + void onActiveDisplayChangedLocked(const DisplayDevice* inactiveDisplayPtr, const DisplayDevice& activeDisplay) REQUIRES(mStateLock, kMainThreadContext); @@ -1087,7 +1101,7 @@ private: void dumpOffscreenLayersProto(LayersProto& layersProto, uint32_t traceFlags = LayerTracing::TRACE_ALL) const; google::protobuf::RepeatedPtrField<DisplayProto> dumpDisplayProto() const; - void addToLayerTracing(bool visibleRegionDirty, int64_t time, int64_t vsyncId) + void addToLayerTracing(bool visibleRegionDirty, TimePoint, VsyncId) REQUIRES(kMainThreadContext); // Dumps state from HW Composer @@ -1225,7 +1239,7 @@ private: // never removed, so take precedence over external and virtual displays. // // May be read from any thread, but must only be written from the main thread. - display::DisplayMap<wp<IBinder>, const sp<DisplayDevice>> mDisplays GUARDED_BY(mStateLock); + ui::DisplayMap<wp<IBinder>, const sp<DisplayDevice>> mDisplays GUARDED_BY(mStateLock); display::PhysicalDisplays mPhysicalDisplays GUARDED_BY(mStateLock); @@ -1262,9 +1276,6 @@ private: // If blurs should be enabled on this device. bool mSupportsBlur = false; - std::atomic<uint32_t> mFrameMissedCount = 0; - std::atomic<uint32_t> mHwcFrameMissedCount = 0; - std::atomic<uint32_t> mGpuFrameMissedCount = 0; TransactionCallbackInvoker mTransactionCallbackInvoker; @@ -1311,7 +1322,7 @@ private: std::unique_ptr<compositionengine::CompositionEngine> mCompositionEngine; - CompositionCoverageFlags mCompositionCoverage; + CompositionCoveragePerDisplay mCompositionCoverage; // mMaxRenderTargetSize is only set once in init() so it doesn't need to be protected by // any mutex. @@ -1332,18 +1343,6 @@ private: std::unique_ptr<scheduler::RefreshRateStats> mRefreshRateStats; scheduler::PresentLatencyTracker mPresentLatencyTracker GUARDED_BY(kMainThreadContext); - struct FenceWithFenceTime { - sp<Fence> fence = Fence::NO_FENCE; - FenceTimePtr fenceTime = FenceTime::NO_FENCE; - }; - std::array<FenceWithFenceTime, 2> mPreviousPresentFences; - - TimePoint mScheduledPresentTime GUARDED_BY(kMainThreadContext); - TimePoint mExpectedPresentTime GUARDED_BY(kMainThreadContext); - - // below flags are set by main thread only - bool mSetActiveModePending = false; - bool mLumaSampling = true; sp<RegionSamplingThread> mRegionSamplingThread; sp<FpsReporter> mFpsReporter; @@ -1443,7 +1442,7 @@ private: std::unordered_map<uint32_t, sp<Layer>> mLegacyLayers; TransactionHandler mTransactionHandler; - display::DisplayMap<ui::LayerStack, frontend::DisplayInfo> mFrontEndDisplayInfos; + ui::DisplayMap<ui::LayerStack, frontend::DisplayInfo> mFrontEndDisplayInfos; bool mFrontEndDisplayInfosChanged = false; // WindowInfo ids visible during the last commit. @@ -1555,12 +1554,17 @@ public: const sp<IBinder>& displayToken, std::optional<gui::DisplayDecorationSupport>* outSupport) override; binder::Status setOverrideFrameRate(int32_t uid, float frameRate) override; + binder::Status updateSmallAreaDetection(const std::vector<int32_t>& uids, + const std::vector<float>& thresholds) override; + binder::Status setSmallAreaDetectionThreshold(int32_t uid, float threshold) override; binder::Status getGpuContextPriority(int32_t* outPriority) override; binder::Status getMaxAcquiredBufferCount(int32_t* buffers) override; binder::Status addWindowInfosListener(const sp<gui::IWindowInfosListener>& windowInfosListener, gui::WindowInfosListenerInfo* outInfo) override; binder::Status removeWindowInfosListener( const sp<gui::IWindowInfosListener>& windowInfosListener) override; + binder::Status getStalledTransactionInfo(int pid, + std::optional<gui::StalledTransactionInfo>* outInfo); private: static const constexpr bool kUsePermissionCache = true; diff --git a/services/surfaceflinger/TEST_MAPPING b/services/surfaceflinger/TEST_MAPPING index 155a27531b..fc6c4f3b1f 100644 --- a/services/surfaceflinger/TEST_MAPPING +++ b/services/surfaceflinger/TEST_MAPPING @@ -1,4 +1,9 @@ { + "imports": [ + { + "path": "frameworks/native/libs/gui" + } + ], "presubmit": [ { "name": "libsurfaceflinger_unittest" @@ -7,14 +12,6 @@ "name": "libcompositionengine_test" }, { - "name": "libgui_test", - "options": [ - { - "native-test-flag": "--gtest_filter=\"InputSurfacesTest*:MultiDisplayTests*\"" - } - ] - }, - { "name": "libscheduler_test" } ], diff --git a/services/surfaceflinger/Tracing/TransactionProtoParser.cpp b/services/surfaceflinger/Tracing/TransactionProtoParser.cpp index 06941809ba..b1e3d6378a 100644 --- a/services/surfaceflinger/Tracing/TransactionProtoParser.cpp +++ b/services/surfaceflinger/Tracing/TransactionProtoParser.cpp @@ -195,6 +195,7 @@ proto::LayerState TransactionProtoParser::toProto( windowInfoProto->set_layout_params_flags(inputInfo->layoutParamsFlags.get()); windowInfoProto->set_layout_params_type( static_cast<int32_t>(inputInfo->layoutParamsType)); + windowInfoProto->set_input_config(inputInfo->inputConfig.get()); LayerProtoHelper::writeToProto(inputInfo->touchableRegion, windowInfoProto->mutable_touchable_region()); windowInfoProto->set_surface_inset(inputInfo->surfaceInset); @@ -467,11 +468,9 @@ void TransactionProtoParser::fromProto(const proto::LayerState& proto, static_cast<gui::WindowInfo::Type>(windowInfoProto.layout_params_type()); LayerProtoHelper::readFromProto(windowInfoProto.touchable_region(), inputInfo.touchableRegion); + inputInfo.inputConfig = + ftl::Flags<gui::WindowInfo::InputConfig>(windowInfoProto.input_config()); inputInfo.surfaceInset = windowInfoProto.surface_inset(); - inputInfo.setInputConfig(gui::WindowInfo::InputConfig::NOT_FOCUSABLE, - !windowInfoProto.focusable()); - inputInfo.setInputConfig(gui::WindowInfo::InputConfig::DUPLICATE_TOUCH_TO_WALLPAPER, - windowInfoProto.has_wallpaper()); inputInfo.globalScaleFactor = windowInfoProto.global_scale_factor(); const proto::Transform& transformProto = windowInfoProto.transform(); inputInfo.transform.set(transformProto.dsdx(), transformProto.dtdx(), transformProto.dtdy(), @@ -601,7 +600,7 @@ frontend::DisplayInfo TransactionProtoParser::fromProto(const proto::DisplayInfo void TransactionProtoParser::fromProto( const google::protobuf::RepeatedPtrField<proto::DisplayInfo>& proto, - display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& outDisplayInfos) { + frontend::DisplayInfos& outDisplayInfos) { outDisplayInfos.clear(); for (const proto::DisplayInfo& displayInfo : proto) { outDisplayInfos.emplace_or_replace(ui::LayerStack::fromValue(displayInfo.layer_stack()), diff --git a/services/surfaceflinger/Tracing/TransactionProtoParser.h b/services/surfaceflinger/Tracing/TransactionProtoParser.h index d6c98e1120..457c3bec40 100644 --- a/services/surfaceflinger/Tracing/TransactionProtoParser.h +++ b/services/surfaceflinger/Tracing/TransactionProtoParser.h @@ -18,9 +18,8 @@ #include <gui/fake/BufferData.h> #include <layerproto/TransactionProto.h> #include <utils/RefBase.h> -#include "Display/DisplayMap.h" -#include "FrontEnd/DisplayInfo.h" +#include "FrontEnd/DisplayInfo.h" #include "FrontEnd/LayerCreationArgs.h" #include "TransactionState.h" @@ -56,9 +55,8 @@ public: void fromProto(const proto::LayerCreationArgs&, LayerCreationArgs& outArgs); std::unique_ptr<FlingerDataMapper> mMapper; static frontend::DisplayInfo fromProto(const proto::DisplayInfo&); - static void fromProto( - const google::protobuf::RepeatedPtrField<proto::DisplayInfo>&, - display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& outDisplayInfos); + static void fromProto(const google::protobuf::RepeatedPtrField<proto::DisplayInfo>&, + frontend::DisplayInfos& outDisplayInfos); private: proto::DisplayState toProto(const DisplayState&); diff --git a/services/surfaceflinger/Tracing/TransactionTracing.cpp b/services/surfaceflinger/Tracing/TransactionTracing.cpp index 87a633fc9d..7e330b97dd 100644 --- a/services/surfaceflinger/Tracing/TransactionTracing.cpp +++ b/services/surfaceflinger/Tracing/TransactionTracing.cpp @@ -28,6 +28,7 @@ #include "TransactionTracing.h" namespace android { +ANDROID_SINGLETON_STATIC_INSTANCE(android::TransactionTraceWriter) TransactionTracing::TransactionTracing() : mProtoParser(std::make_unique<TransactionProtoParser::FlingerDataMapper>()) { @@ -56,7 +57,7 @@ TransactionTracing::~TransactionTracing() { writeToFile(); } -status_t TransactionTracing::writeToFile(std::string filename) { +status_t TransactionTracing::writeToFile(const std::string& filename) { std::scoped_lock lock(mTraceLock); proto::TransactionTraceFile fileProto = createTraceFileProto(); addStartingStateToProtoLocked(fileProto); @@ -92,10 +93,10 @@ void TransactionTracing::addQueuedTransaction(const TransactionState& transactio mTransactionQueue.push(state); } -void TransactionTracing::addCommittedTransactions( - int64_t vsyncId, nsecs_t commitTime, frontend::Update& newUpdate, - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displayInfos, - bool displayInfoChanged) { +void TransactionTracing::addCommittedTransactions(int64_t vsyncId, nsecs_t commitTime, + frontend::Update& newUpdate, + const frontend::DisplayInfos& displayInfos, + bool displayInfoChanged) { CommittedUpdates update; update.vsyncId = vsyncId; update.timestamp = commitTime; @@ -115,6 +116,7 @@ void TransactionTracing::addCommittedTransactions( } mPendingUpdates.emplace_back(update); tryPushToTracingThread(); + mLastUpdatedVsyncId = vsyncId; } void TransactionTracing::loop() { @@ -218,19 +220,29 @@ void TransactionTracing::addEntry(const std::vector<CommittedUpdates>& committed mTransactionsAddedToBufferCv.notify_one(); } -void TransactionTracing::flush(int64_t vsyncId) { - while (!mPendingUpdates.empty() || !mPendingDestroyedLayers.empty()) { - tryPushToTracingThread(); +void TransactionTracing::flush() { + { + std::scoped_lock lock(mMainThreadLock); + // Collect any pending transactions and wait for transactions to be added to + mUpdates.insert(mUpdates.end(), std::make_move_iterator(mPendingUpdates.begin()), + std::make_move_iterator(mPendingUpdates.end())); + mPendingUpdates.clear(); + mDestroyedLayers.insert(mDestroyedLayers.end(), mPendingDestroyedLayers.begin(), + mPendingDestroyedLayers.end()); + mPendingDestroyedLayers.clear(); + mTransactionsAvailableCv.notify_one(); } std::unique_lock<std::mutex> lock(mTraceLock); base::ScopedLockAssertion assumeLocked(mTraceLock); - mTransactionsAddedToBufferCv.wait(lock, [&]() REQUIRES(mTraceLock) { - proto::TransactionTraceEntry entry; - if (mBuffer.used() > 0) { - entry.ParseFromString(mBuffer.back()); - } - return mBuffer.used() > 0 && entry.vsync_id() >= vsyncId; - }); + mTransactionsAddedToBufferCv.wait_for(lock, std::chrono::milliseconds(100), + [&]() REQUIRES(mTraceLock) { + proto::TransactionTraceEntry entry; + if (mBuffer.used() > 0) { + entry.ParseFromString(mBuffer.back()); + } + return mBuffer.used() > 0 && + entry.vsync_id() >= mLastUpdatedVsyncId; + }); } void TransactionTracing::onLayerRemoved(int32_t layerId) { diff --git a/services/surfaceflinger/Tracing/TransactionTracing.h b/services/surfaceflinger/Tracing/TransactionTracing.h index f27e7a9663..a59dc6e7dc 100644 --- a/services/surfaceflinger/Tracing/TransactionTracing.h +++ b/services/surfaceflinger/Tracing/TransactionTracing.h @@ -19,13 +19,13 @@ #include <android-base/thread_annotations.h> #include <layerproto/TransactionProto.h> #include <utils/Errors.h> +#include <utils/Singleton.h> #include <utils/Timers.h> #include <memory> #include <mutex> #include <thread> -#include "Display/DisplayMap.h" #include "FrontEnd/DisplayInfo.h" #include "FrontEnd/LayerCreationArgs.h" #include "FrontEnd/Update.h" @@ -59,14 +59,14 @@ public: ~TransactionTracing(); void addQueuedTransaction(const TransactionState&); - void addCommittedTransactions( - int64_t vsyncId, nsecs_t commitTime, frontend::Update& update, - const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displayInfos, - bool displayInfoChanged); - status_t writeToFile(std::string filename = FILE_NAME); + void addCommittedTransactions(int64_t vsyncId, nsecs_t commitTime, frontend::Update& update, + const frontend::DisplayInfos&, bool displayInfoChanged); + status_t writeToFile(const std::string& filename = FILE_PATH); void setBufferSize(size_t bufferSizeInBytes); void onLayerRemoved(int layerId); void dump(std::string&) const; + // Wait until all the committed transactions for the specified vsync id are added to the buffer. + void flush() EXCLUDES(mMainThreadLock); static constexpr auto CONTINUOUS_TRACING_BUFFER_SIZE = 512 * 1024; static constexpr auto ACTIVE_TRACING_BUFFER_SIZE = 100 * 1024 * 1024; // version 1 - switching to support new frontend @@ -76,7 +76,9 @@ private: friend class TransactionTracingTest; friend class SurfaceFlinger; - static constexpr auto FILE_NAME = "/data/misc/wmtrace/transactions_trace.winscope"; + static constexpr auto DIR_NAME = "/data/misc/wmtrace/"; + static constexpr auto FILE_NAME = "transactions_trace.winscope"; + static constexpr auto FILE_PATH = "/data/misc/wmtrace/transactions_trace.winscope"; mutable std::mutex mTraceLock; RingBuffer<proto::TransactionTraceFile, proto::TransactionTraceEntry> mBuffer @@ -88,8 +90,7 @@ private: nsecs_t mStartingTimestamp GUARDED_BY(mTraceLock); std::unordered_map<int, proto::LayerCreationArgs> mCreatedLayers GUARDED_BY(mTraceLock); std::map<uint32_t /* layerId */, TracingLayerState> mStartingStates GUARDED_BY(mTraceLock); - display::DisplayMap<ui::LayerStack, frontend::DisplayInfo> mStartingDisplayInfos - GUARDED_BY(mTraceLock); + frontend::DisplayInfos mStartingDisplayInfos GUARDED_BY(mTraceLock); std::set<uint32_t /* layerId */> mRemovedLayerHandlesAtStart GUARDED_BY(mTraceLock); TransactionProtoParser mProtoParser; @@ -106,7 +107,7 @@ private: std::vector<LayerCreationArgs> createdLayers; std::vector<uint32_t> destroyedLayerHandles; bool displayInfoChanged; - display::DisplayMap<ui::LayerStack, frontend::DisplayInfo> displayInfos; + frontend::DisplayInfos displayInfos; int64_t vsyncId; int64_t timestamp; }; @@ -115,6 +116,7 @@ private: std::vector<uint32_t /* layerId */> mDestroyedLayers GUARDED_BY(mMainThreadLock); std::vector<uint32_t /* layerId */> mPendingDestroyedLayers; // only accessed by main thread + int64_t mLastUpdatedVsyncId = -1; proto::TransactionTraceFile createTraceFileProto() const; void loop(); @@ -125,10 +127,21 @@ private: void addStartingStateToProtoLocked(proto::TransactionTraceFile& proto) REQUIRES(mTraceLock); void updateStartingStateLocked(const proto::TransactionTraceEntry& entry) REQUIRES(mTraceLock); // TEST - // Wait until all the committed transactions for the specified vsync id are added to the buffer. - void flush(int64_t vsyncId) EXCLUDES(mMainThreadLock); // Return buffer contents as trace file proto proto::TransactionTraceFile writeToProto() EXCLUDES(mMainThreadLock); }; +class TransactionTraceWriter : public Singleton<TransactionTraceWriter> { + friend class Singleton<TransactionTracing>; + std::function<void(const std::string& prefix, bool overwrite)> mWriterFunction = + [](const std::string&, bool) {}; + +public: + void setWriterFunction( + std::function<void(const std::string& prefix, bool overwrite)> function) { + mWriterFunction = std::move(function); + } + void invoke(const std::string& prefix, bool overwrite) { mWriterFunction(prefix, overwrite); } +}; + } // namespace android diff --git a/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.cpp b/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.cpp index 55004c5e70..519ef44552 100644 --- a/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.cpp +++ b/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.cpp @@ -41,7 +41,7 @@ namespace android { using namespace ftl::flag_operators; bool LayerTraceGenerator::generate(const proto::TransactionTraceFile& traceFile, - const char* outputLayersTracePath) { + const char* outputLayersTracePath, bool onlyLastEntry) { if (traceFile.entry_size() == 0) { ALOGD("Trace file is empty"); return false; @@ -53,7 +53,7 @@ bool LayerTraceGenerator::generate(const proto::TransactionTraceFile& traceFile, frontend::LayerLifecycleManager lifecycleManager; frontend::LayerHierarchyBuilder hierarchyBuilder{{}}; frontend::LayerSnapshotBuilder snapshotBuilder; - display::DisplayMap<ui::LayerStack, frontend::DisplayInfo> displayInfos; + ui::DisplayMap<ui::LayerStack, frontend::DisplayInfo> displayInfos; renderengine::ShadowSettings globalShadowSettings{.ambientColor = {1, 1, 1, 1}}; char value[PROPERTY_VALUE_MAX]; @@ -158,9 +158,11 @@ bool LayerTraceGenerator::generate(const proto::TransactionTraceFile& traceFile, layerTracing.getFlags()) .generate(hierarchyBuilder.getHierarchy()); auto displayProtos = LayerProtoHelper::writeDisplayInfoToProto(displayInfos); - layerTracing.notify(visibleRegionsDirty, entry.elapsed_realtime_nanos(), entry.vsync_id(), - &layersProto, {}, &displayProtos); - layerTracing.appendToStream(out); + if (!onlyLastEntry || (i == traceFile.entry_size() - 1)) { + layerTracing.notify(visibleRegionsDirty, entry.elapsed_realtime_nanos(), + entry.vsync_id(), &layersProto, {}, &displayProtos); + layerTracing.appendToStream(out); + } } layerTracing.disable("", /*writeToFile=*/false); out.close(); diff --git a/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.h b/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.h index ee1ea6ce8b..e41d1e6e0b 100644 --- a/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.h +++ b/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.h @@ -21,6 +21,7 @@ namespace android { class LayerTraceGenerator { public: - bool generate(const proto::TransactionTraceFile&, const char* outputLayersTracePath); + bool generate(const proto::TransactionTraceFile&, const char* outputLayersTracePath, + bool onlyLastEntry); }; } // namespace android
\ No newline at end of file diff --git a/services/surfaceflinger/Tracing/tools/main.cpp b/services/surfaceflinger/Tracing/tools/main.cpp index c440c19ccc..5ca87e4cb3 100644 --- a/services/surfaceflinger/Tracing/tools/main.cpp +++ b/services/surfaceflinger/Tracing/tools/main.cpp @@ -26,9 +26,9 @@ using namespace android; int main(int argc, char** argv) { - if (argc > 3) { + if (argc > 4) { std::cout << "Usage: " << argv[0] - << " [transaction-trace-path] [output-layers-trace-path]\n"; + << " [transaction-trace-path] [output-layers-trace-path] [--last-entry-only]\n"; return -1; } @@ -48,12 +48,16 @@ int main(int argc, char** argv) { } const char* outputLayersTracePath = - (argc == 3) ? argv[2] : "/data/misc/wmtrace/layers_trace.winscope"; - ; + (argc >= 3) ? argv[2] : "/data/misc/wmtrace/layers_trace.winscope"; + + const bool generateLastEntryOnly = + argc >= 4 && std::string_view(argv[3]) == "--last-entry-only"; + ALOGD("Generating %s...", outputLayersTracePath); std::cout << "Generating " << outputLayersTracePath << "\n"; - if (!LayerTraceGenerator().generate(transactionTraceFile, outputLayersTracePath)) { + if (!LayerTraceGenerator().generate(transactionTraceFile, outputLayersTracePath, + generateLastEntryOnly)) { std::cout << "Error: Failed to generate layers trace " << outputLayersTracePath; return -1; } diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h b/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h index 9b9ac96cf7..ded6cebde3 100644 --- a/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h +++ b/services/surfaceflinger/fuzzer/surfaceflinger_fuzzers_utils.h @@ -286,8 +286,11 @@ public: private: // ICompositor overrides: void configure() override {} - bool commit(TimePoint, VsyncId, TimePoint) override { return false; } - void composite(TimePoint, VsyncId) override {} + bool commit(PhysicalDisplayId, const scheduler::FrameTargets&) override { return false; } + CompositeResultsPerDisplay composite(PhysicalDisplayId, + const scheduler::FrameTargeters&) override { + return {}; + } void sample() override {} // MessageQueue overrides: @@ -474,25 +477,25 @@ public: &outWideColorGamutPixelFormat); } - void overrideHdrTypes(sp<IBinder> &display, FuzzedDataProvider *fdp) { + void overrideHdrTypes(const sp<IBinder>& display, FuzzedDataProvider* fdp) { std::vector<ui::Hdr> hdrTypes; hdrTypes.push_back(fdp->PickValueInArray(kHdrTypes)); mFlinger->overrideHdrTypes(display, hdrTypes); } - void getDisplayedContentSample(sp<IBinder> &display, FuzzedDataProvider *fdp) { + void getDisplayedContentSample(const sp<IBinder>& display, FuzzedDataProvider* fdp) { DisplayedFrameStats outDisplayedFrameStats; mFlinger->getDisplayedContentSample(display, fdp->ConsumeIntegral<uint64_t>(), fdp->ConsumeIntegral<uint64_t>(), &outDisplayedFrameStats); } - void getDisplayStats(sp<IBinder> &display) { + void getDisplayStats(const sp<IBinder>& display) { android::DisplayStatInfo stats; mFlinger->getDisplayStats(display, &stats); } - void getDisplayState(sp<IBinder> &display) { + void getDisplayState(const sp<IBinder>& display) { ui::DisplayState displayState; mFlinger->getDisplayState(display, &displayState); } @@ -506,12 +509,12 @@ public: android::ui::DynamicDisplayInfo dynamicDisplayInfo; mFlinger->getDynamicDisplayInfoFromId(displayId, &dynamicDisplayInfo); } - void getDisplayNativePrimaries(sp<IBinder> &display) { + void getDisplayNativePrimaries(const sp<IBinder>& display) { android::ui::DisplayPrimaries displayPrimaries; mFlinger->getDisplayNativePrimaries(display, displayPrimaries); } - void getDesiredDisplayModeSpecs(sp<IBinder> &display) { + void getDesiredDisplayModeSpecs(const sp<IBinder>& display) { gui::DisplayModeSpecs _; mFlinger->getDesiredDisplayModeSpecs(display, &_); } @@ -523,7 +526,7 @@ public: return ids.front(); } - std::pair<sp<IBinder>, int64_t> fuzzBoot(FuzzedDataProvider *fdp) { + std::pair<sp<IBinder>, PhysicalDisplayId> fuzzBoot(FuzzedDataProvider* fdp) { mFlinger->callingThreadHasUnscopedSurfaceFlingerAccess(fdp->ConsumeBool()); const sp<Client> client = sp<Client>::make(mFlinger); @@ -550,13 +553,13 @@ public: mFlinger->bootFinished(); - return {display, physicalDisplayId.value}; + return {display, physicalDisplayId}; } void fuzzSurfaceFlinger(const uint8_t *data, size_t size) { FuzzedDataProvider mFdp(data, size); - auto [display, displayId] = fuzzBoot(&mFdp); + const auto [display, displayId] = fuzzBoot(&mFdp); sp<IGraphicBufferProducer> bufferProducer = sp<mock::GraphicBufferProducer>::make(); @@ -564,8 +567,8 @@ public: getDisplayStats(display); getDisplayState(display); - getStaticDisplayInfo(displayId); - getDynamicDisplayInfo(displayId); + getStaticDisplayInfo(displayId.value); + getDynamicDisplayInfo(displayId.value); getDisplayNativePrimaries(display); mFlinger->setAutoLowLatencyMode(display, mFdp.ConsumeBool()); @@ -604,7 +607,10 @@ public: mFlinger->commitTransactions(); mFlinger->flushTransactionQueues(getFuzzedVsyncId(mFdp)); - mFlinger->postComposition(systemTime()); + + scheduler::FrameTargeter frameTargeter(displayId, mFdp.ConsumeBool()); + mFlinger->postComposition(displayId, ftl::init::map(displayId, &frameTargeter), + mFdp.ConsumeIntegral<nsecs_t>()); } mFlinger->setTransactionFlags(mFdp.ConsumeIntegral<uint32_t>()); @@ -622,8 +628,6 @@ public: mFlinger->getMaxAcquiredBufferCountForCurrentRefreshRate(mFdp.ConsumeIntegral<uid_t>()); - mFlinger->calculateExpectedPresentTime({}); - mFlinger->enableHalVirtualDisplays(mFdp.ConsumeBool()); fuzzDumpsysAndDebug(&mFdp); @@ -788,7 +792,7 @@ public: } private: - void setVsyncEnabled(PhysicalDisplayId, bool) override {} + void requestHardwareVsync(PhysicalDisplayId, bool) override {} void requestDisplayModes(std::vector<display::DisplayModeRequest>) override {} void kernelTimerChanged(bool) override {} void triggerOnFrameRateOverridesChanged() override {} diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_layer_fuzzer.cpp b/services/surfaceflinger/fuzzer/surfaceflinger_layer_fuzzer.cpp index 921cae4e41..9f0bddea1e 100644 --- a/services/surfaceflinger/fuzzer/surfaceflinger_layer_fuzzer.cpp +++ b/services/surfaceflinger/fuzzer/surfaceflinger_layer_fuzzer.cpp @@ -106,7 +106,7 @@ void LayerFuzzer::invokeEffectLayer() { effectLayer->addSurfaceFramePresentedForBuffer(surfaceFrame, mFdp.ConsumeIntegral<int64_t>() /*acquireTime*/, mFdp.ConsumeIntegral<int64_t>() /*currentTime*/); - effectLayer->addSurfaceFrameDroppedForBuffer(surfaceFrame1); + effectLayer->addSurfaceFrameDroppedForBuffer(surfaceFrame1, mFdp.ConsumeIntegral<nsecs_t>()); parent.clear(); client.clear(); diff --git a/services/surfaceflinger/fuzzer/surfaceflinger_scheduler_fuzzer.cpp b/services/surfaceflinger/fuzzer/surfaceflinger_scheduler_fuzzer.cpp index f17d2e1cb4..4d1a5ffa67 100644 --- a/services/surfaceflinger/fuzzer/surfaceflinger_scheduler_fuzzer.cpp +++ b/services/surfaceflinger/fuzzer/surfaceflinger_scheduler_fuzzer.cpp @@ -19,6 +19,7 @@ #include <fuzzer/FuzzedDataProvider.h> #include <processgroup/sched_policy.h> +#include <scheduler/IVsyncSource.h> #include <scheduler/PresentLatencyTracker.h> #include "Scheduler/OneShotTimer.h" @@ -42,13 +43,14 @@ constexpr nsecs_t kVsyncPeriods[] = {(30_Hz).getPeriodNsecs(), (60_Hz).getPeriod (120_Hz).getPeriodNsecs()}; constexpr auto kLayerVoteTypes = ftl::enum_range<scheduler::RefreshRateSelector::LayerVoteType>(); +constexpr auto kCompositionCoverage = ftl::enum_range<CompositionCoverage>(); constexpr PowerMode kPowerModes[] = {PowerMode::ON, PowerMode::DOZE, PowerMode::OFF, PowerMode::DOZE_SUSPEND, PowerMode::ON_SUSPEND}; constexpr uint16_t kRandomStringLength = 256; constexpr std::chrono::duration kSyncPeriod(16ms); -constexpr PhysicalDisplayId DEFAULT_DISPLAY_ID = PhysicalDisplayId::fromPort(42u); +constexpr PhysicalDisplayId kDisplayId = PhysicalDisplayId::fromPort(42u); template <typename T> void dump(T* component, FuzzedDataProvider* fdp) { @@ -56,6 +58,10 @@ void dump(T* component, FuzzedDataProvider* fdp) { component->dump(res); } +inline sp<Fence> makeFakeFence() { + return sp<Fence>::make(memfd_create("fd", MFD_ALLOW_SEALING)); +} + class SchedulerFuzzer { public: SchedulerFuzzer(const uint8_t* data, size_t size) : mFdp(data, size){}; @@ -65,6 +71,7 @@ private: void fuzzRefreshRateSelection(); void fuzzRefreshRateSelector(); void fuzzPresentLatencyTracker(); + void fuzzFrameTargeter(); void fuzzVSyncModulator(); void fuzzVSyncPredictor(); void fuzzVSyncReactor(); @@ -170,9 +177,8 @@ void SchedulerFuzzer::fuzzVSyncPredictor() { uint16_t now = mFdp.ConsumeIntegral<uint16_t>(); uint16_t historySize = mFdp.ConsumeIntegralInRange<uint16_t>(1, UINT16_MAX); uint16_t minimumSamplesForPrediction = mFdp.ConsumeIntegralInRange<uint16_t>(1, UINT16_MAX); - scheduler::VSyncPredictor tracker{DEFAULT_DISPLAY_ID, - mFdp.ConsumeIntegral<uint16_t>() /*period*/, historySize, - minimumSamplesForPrediction, + scheduler::VSyncPredictor tracker{kDisplayId, mFdp.ConsumeIntegral<uint16_t>() /*period*/, + historySize, minimumSamplesForPrediction, mFdp.ConsumeIntegral<uint32_t>() /*outlierTolerancePercent*/}; uint16_t period = mFdp.ConsumeIntegral<uint16_t>(); tracker.setPeriod(period); @@ -244,7 +250,7 @@ void SchedulerFuzzer::fuzzLayerHistory() { void SchedulerFuzzer::fuzzVSyncReactor() { std::shared_ptr<FuzzImplVSyncTracker> vSyncTracker = std::make_shared<FuzzImplVSyncTracker>(); - scheduler::VSyncReactor reactor(DEFAULT_DISPLAY_ID, + scheduler::VSyncReactor reactor(kDisplayId, std::make_unique<ClockWrapper>( std::make_shared<FuzzImplClock>()), *vSyncTracker, mFdp.ConsumeIntegral<uint8_t>() /*pendingLimit*/, @@ -256,13 +262,13 @@ void SchedulerFuzzer::fuzzVSyncReactor() { reactor.addHwVsyncTimestamp(0, std::nullopt, &periodFlushed); reactor.addHwVsyncTimestamp(mFdp.ConsumeIntegral<nsecs_t>() /*newPeriod*/, std::nullopt, &periodFlushed); - sp<Fence> fence = sp<Fence>::make(memfd_create("fd", MFD_ALLOW_SEALING)); - std::shared_ptr<FenceTime> ft = std::make_shared<FenceTime>(fence); + + const auto fence = std::make_shared<FenceTime>(makeFakeFence()); vSyncTracker->addVsyncTimestamp(mFdp.ConsumeIntegral<nsecs_t>()); FenceTime::Snapshot snap(mFdp.ConsumeIntegral<nsecs_t>()); - ft->applyTrustedSnapshot(snap); + fence->applyTrustedSnapshot(snap); reactor.setIgnorePresentFences(mFdp.ConsumeBool()); - reactor.addPresentFence(ft); + reactor.addPresentFence(fence); dump<scheduler::VSyncReactor>(&reactor, &mFdp); } @@ -392,14 +398,45 @@ void SchedulerFuzzer::fuzzRefreshRateSelector() { void SchedulerFuzzer::fuzzPresentLatencyTracker() { scheduler::PresentLatencyTracker tracker; - tracker.trackPendingFrame(TimePoint::fromNs(mFdp.ConsumeIntegral<nsecs_t>()), - FenceTime::NO_FENCE); + + int i = 5; + while (i-- > 0) { + tracker.trackPendingFrame(getFuzzedTimePoint(mFdp), + std::make_shared<FenceTime>(makeFakeFence())); + } +} + +void SchedulerFuzzer::fuzzFrameTargeter() { + scheduler::FrameTargeter frameTargeter(kDisplayId, mFdp.ConsumeBool()); + + const struct VsyncSource final : scheduler::IVsyncSource { + explicit VsyncSource(FuzzedDataProvider& fuzzer) : fuzzer(fuzzer) {} + FuzzedDataProvider& fuzzer; + + Period period() const { return getFuzzedDuration(fuzzer); } + TimePoint vsyncDeadlineAfter(TimePoint) const { return getFuzzedTimePoint(fuzzer); } + } vsyncSource{mFdp}; + + int i = 10; + while (i-- > 0) { + frameTargeter.beginFrame({.frameBeginTime = getFuzzedTimePoint(mFdp), + .vsyncId = getFuzzedVsyncId(mFdp), + .expectedVsyncTime = getFuzzedTimePoint(mFdp), + .sfWorkDuration = getFuzzedDuration(mFdp)}, + vsyncSource); + + frameTargeter.setPresentFence(makeFakeFence()); + + frameTargeter.endFrame( + {.compositionCoverage = mFdp.PickValueInArray(kCompositionCoverage.values)}); + } } void SchedulerFuzzer::process() { fuzzRefreshRateSelection(); fuzzRefreshRateSelector(); fuzzPresentLatencyTracker(); + fuzzFrameTargeter(); fuzzVSyncModulator(); fuzzVSyncPredictor(); fuzzVSyncReactor(); diff --git a/services/surfaceflinger/layerproto/common.proto b/services/surfaceflinger/layerproto/common.proto index a6d8d61600..5e20d4d0f5 100644 --- a/services/surfaceflinger/layerproto/common.proto +++ b/services/surfaceflinger/layerproto/common.proto @@ -70,6 +70,7 @@ message InputWindowInfoProto { bool replace_touchable_region_with_crop = 14; RectProto touchable_region_crop = 15; TransformProto transform = 16; + uint32 input_config = 17; } message BlurRegion { diff --git a/services/surfaceflinger/layerproto/transactions.proto b/services/surfaceflinger/layerproto/transactions.proto index b0cee9b398..d03afa05ab 100644 --- a/services/surfaceflinger/layerproto/transactions.proto +++ b/services/surfaceflinger/layerproto/transactions.proto @@ -256,13 +256,14 @@ message LayerState { int32 layout_params_type = 2; RegionProto touchable_region = 3; int32 surface_inset = 4; - bool focusable = 5; - bool has_wallpaper = 6; + bool focusable = 5; // unused + bool has_wallpaper = 6; // unused float global_scale_factor = 7; uint32 crop_layer_id = 8; bool replace_touchable_region_with_crop = 9; RectProto touchable_region_crop = 10; Transform transform = 11; + uint32 input_config = 12; } WindowInfo window_info_handle = 27; float bg_color_alpha = 28; diff --git a/services/surfaceflinger/tests/ScreenCapture_test.cpp b/services/surfaceflinger/tests/ScreenCapture_test.cpp index 013694fd47..96cc333a08 100644 --- a/services/surfaceflinger/tests/ScreenCapture_test.cpp +++ b/services/surfaceflinger/tests/ScreenCapture_test.cpp @@ -19,6 +19,7 @@ #pragma clang diagnostic ignored "-Wconversion" #include <private/android_filesystem_config.h> +#include <ui/DisplayState.h> #include "LayerTransactionTest.h" @@ -32,11 +33,11 @@ protected: const auto ids = SurfaceComposerClient::getPhysicalDisplayIds(); ASSERT_FALSE(ids.empty()); - const auto display = SurfaceComposerClient::getPhysicalDisplayToken(ids.front()); - ASSERT_FALSE(display == nullptr); + mDisplayToken = SurfaceComposerClient::getPhysicalDisplayToken(ids.front()); + ASSERT_FALSE(mDisplayToken == nullptr); ui::DisplayMode mode; - ASSERT_EQ(NO_ERROR, SurfaceComposerClient::getActiveDisplayMode(display, &mode)); + ASSERT_EQ(NO_ERROR, SurfaceComposerClient::getActiveDisplayMode(mDisplayToken, &mode)); const ui::Size& resolution = mode.resolution; mDisplaySize = resolution; @@ -57,7 +58,7 @@ protected: TransactionUtils::fillSurfaceRGBA8(mFGSurfaceControl, 195, 63, 63); asTransaction([&](Transaction& t) { - t.setDisplayLayerStack(display, ui::DEFAULT_LAYER_STACK); + t.setDisplayLayerStack(mDisplayToken, ui::DEFAULT_LAYER_STACK); t.setLayer(mBGSurfaceControl, INT32_MAX - 2).show(mBGSurfaceControl); @@ -71,11 +72,18 @@ protected: LayerTransactionTest::TearDown(); mBGSurfaceControl = 0; mFGSurfaceControl = 0; + + // Restore display rotation + asTransaction([&](Transaction& t) { + Rect displayBounds{mDisplaySize}; + t.setDisplayProjection(mDisplayToken, ui::ROTATION_0, displayBounds, displayBounds); + }); } sp<SurfaceControl> mBGSurfaceControl; sp<SurfaceControl> mFGSurfaceControl; std::unique_ptr<ScreenCapture> mCapture; + sp<IBinder> mDisplayToken; ui::Size mDisplaySize; }; @@ -870,6 +878,42 @@ TEST_F(ScreenCaptureTest, CaptureOffscreen) { mCapture->expectColor(Rect(0, 0, 32, 32), Color::RED); } +TEST_F(ScreenCaptureTest, CaptureDisplayWith90DegRotation) { + asTransaction([&](Transaction& t) { + Rect newDisplayBounds{mDisplaySize.height, mDisplaySize.width}; + t.setDisplayProjection(mDisplayToken, ui::ROTATION_90, newDisplayBounds, newDisplayBounds); + }); + + DisplayCaptureArgs displayCaptureArgs; + displayCaptureArgs.displayToken = mDisplayToken; + displayCaptureArgs.width = mDisplaySize.width; + displayCaptureArgs.height = mDisplaySize.height; + displayCaptureArgs.useIdentityTransform = true; + ScreenCapture::captureDisplay(&mCapture, displayCaptureArgs); + + mCapture->expectBGColor(0, 0); + mCapture->expectFGColor(mDisplaySize.width - 65, 65); +} + +TEST_F(ScreenCaptureTest, CaptureDisplayWith270DegRotation) { + asTransaction([&](Transaction& t) { + Rect newDisplayBounds{mDisplaySize.height, mDisplaySize.width}; + t.setDisplayProjection(mDisplayToken, ui::ROTATION_270, newDisplayBounds, newDisplayBounds); + }); + + DisplayCaptureArgs displayCaptureArgs; + displayCaptureArgs.displayToken = mDisplayToken; + displayCaptureArgs.width = mDisplaySize.width; + displayCaptureArgs.height = mDisplaySize.height; + displayCaptureArgs.useIdentityTransform = true; + ScreenCapture::captureDisplay(&mCapture, displayCaptureArgs); + + std::this_thread::sleep_for(std::chrono::seconds{5}); + + mCapture->expectBGColor(mDisplayWidth - 1, mDisplaySize.height - 1); + mCapture->expectFGColor(65, mDisplaySize.height - 65); +} + TEST_F(ScreenCaptureTest, CaptureNonHdrLayer) { sp<SurfaceControl> layer; ASSERT_NO_FATAL_FAILURE(layer = createLayer("test layer", 32, 32, diff --git a/services/surfaceflinger/tests/tracing/TransactionTraceTestSuite.cpp b/services/surfaceflinger/tests/tracing/TransactionTraceTestSuite.cpp index 0b2e5a3c9d..9526948bff 100644 --- a/services/surfaceflinger/tests/tracing/TransactionTraceTestSuite.cpp +++ b/services/surfaceflinger/tests/tracing/TransactionTraceTestSuite.cpp @@ -59,8 +59,8 @@ protected: std::string actualLayersTracePath = std::string(temp_dir.path) + "/" + expectedLayersFilename + "_actual"; - EXPECT_TRUE( - LayerTraceGenerator().generate(mTransactionTrace, actualLayersTracePath.c_str())) + EXPECT_TRUE(LayerTraceGenerator().generate(mTransactionTrace, actualLayersTracePath.c_str(), + /*onlyLastEntry=*/true)) << "Failed to generate layers trace from " << transactionTracePath; EXPECT_TRUE(std::filesystem::exists(std::filesystem::path(actualLayersTracePath))); parseLayersTraceFromFile(actualLayersTracePath.c_str(), mActualLayersTraceProto); @@ -86,9 +86,9 @@ protected: std::vector<std::filesystem::path> TransactionTraceTestSuite::sTransactionTraces{}; struct LayerInfo { - int32_t id; + uint64_t id; std::string name; - int32_t parent; + uint64_t parent; int z; uint64_t curr_frame; float x; @@ -119,8 +119,8 @@ inline void PrintTo(const LayerInfo& info, ::std::ostream* os) { } struct find_id { - int id; - find_id(int id) : id(id) {} + uint64_t id; + find_id(uint64_t id) : id(id) {} bool operator()(LayerInfo const& m) const { return m.id == id; } }; @@ -136,9 +136,9 @@ static LayerInfo getLayerInfoFromProto(::android::surfaceflinger::LayerProto& pr touchableRegionBounds = touchableRegion.bounds(); } - return {proto.id(), + return {static_cast<uint64_t>(proto.id()), proto.name(), - proto.parent(), + static_cast<uint64_t>(proto.parent()), proto.z(), proto.curr_frame(), proto.has_position() ? proto.position().x() : -1, @@ -150,7 +150,7 @@ static LayerInfo getLayerInfoFromProto(::android::surfaceflinger::LayerProto& pr static std::vector<LayerInfo> getLayerInfosFromProto( android::surfaceflinger::LayersTraceProto& entry) { - std::unordered_map<int32_t /* snapshotId*/, int32_t /*layerId*/> snapshotIdToLayerId; + std::unordered_map<uint64_t /* snapshotId*/, uint64_t /*layerId*/> snapshotIdToLayerId; std::vector<LayerInfo> layers; layers.reserve(static_cast<size_t>(entry.layers().layers_size())); bool mapSnapshotIdToLayerId = false; @@ -158,7 +158,12 @@ static std::vector<LayerInfo> getLayerInfosFromProto( auto layer = entry.layers().layers(i); LayerInfo layerInfo = getLayerInfoFromProto(layer); - snapshotIdToLayerId[layerInfo.id] = static_cast<int32_t>(layer.original_id()); + uint64_t layerId = layerInfo.name.find("(Mirror)") == std::string::npos + ? static_cast<uint64_t>(layer.original_id()) + : static_cast<uint64_t>(layer.original_id()) | 1ull << 63; + + snapshotIdToLayerId[layerInfo.id] = layerId; + if (layer.original_id() != 0) { mapSnapshotIdToLayerId = true; } @@ -172,7 +177,7 @@ static std::vector<LayerInfo> getLayerInfosFromProto( for (auto& layer : layers) { layer.id = snapshotIdToLayerId[layer.id]; auto it = snapshotIdToLayerId.find(layer.parent); - layer.parent = it == snapshotIdToLayerId.end() ? -1 : it->second; + layer.parent = it == snapshotIdToLayerId.end() ? static_cast<uint64_t>(-1) : it->second; } return layers; } @@ -189,7 +194,6 @@ TEST_P(TransactionTraceTestSuite, validateEndState) { std::vector<LayerInfo> expectedLayers = getLayerInfosFromProto(expectedLastEntry); std::vector<LayerInfo> actualLayers = getLayerInfosFromProto(actualLastEntry); - ; size_t i = 0; for (; i < actualLayers.size() && i < expectedLayers.size(); i++) { @@ -197,9 +201,9 @@ TEST_P(TransactionTraceTestSuite, validateEndState) { find_id(expectedLayers[i].id)); EXPECT_NE(it, actualLayers.end()); EXPECT_EQ(expectedLayers[i], *it); - ALOGV("Validating %s[%d] parent=%d z=%d frame=%" PRIu64, expectedLayers[i].name.c_str(), - expectedLayers[i].id, expectedLayers[i].parent, expectedLayers[i].z, - expectedLayers[i].curr_frame); + ALOGV("Validating %s[%" PRIu64 "] parent=%" PRIu64 " z=%d frame=%" PRIu64, + expectedLayers[i].name.c_str(), expectedLayers[i].id, expectedLayers[i].parent, + expectedLayers[i].z, expectedLayers[i].curr_frame); } EXPECT_EQ(expectedLayers.size(), actualLayers.size()); @@ -208,9 +212,9 @@ TEST_P(TransactionTraceTestSuite, validateEndState) { for (size_t j = 0; j < actualLayers.size(); j++) { if (std::find_if(expectedLayers.begin(), expectedLayers.end(), find_id(actualLayers[j].id)) == expectedLayers.end()) { - ALOGD("actualLayers [%d]:%s parent=%d z=%d frame=%" PRIu64, actualLayers[j].id, - actualLayers[j].name.c_str(), actualLayers[j].parent, actualLayers[j].z, - actualLayers[j].curr_frame); + ALOGD("actualLayers [%" PRIu64 "]:%s parent=%" PRIu64 " z=%d frame=%" PRIu64, + actualLayers[j].id, actualLayers[j].name.c_str(), actualLayers[j].parent, + actualLayers[j].z, actualLayers[j].curr_frame); } } FAIL(); @@ -220,9 +224,9 @@ TEST_P(TransactionTraceTestSuite, validateEndState) { for (size_t j = 0; j < expectedLayers.size(); j++) { if (std::find_if(actualLayers.begin(), actualLayers.end(), find_id(expectedLayers[j].id)) == actualLayers.end()) { - ALOGD("expectedLayers [%d]:%s parent=%d z=%d frame=%" PRIu64, expectedLayers[j].id, - expectedLayers[j].name.c_str(), expectedLayers[j].parent, expectedLayers[j].z, - expectedLayers[j].curr_frame); + ALOGD("expectedLayers [%" PRIu64 "]:%s parent=%" PRIu64 " z=%d frame=%" PRIu64, + expectedLayers[j].id, expectedLayers[j].name.c_str(), + expectedLayers[j].parent, expectedLayers[j].z, expectedLayers[j].curr_frame); } } FAIL(); diff --git a/services/surfaceflinger/tests/tracing/readme.md b/services/surfaceflinger/tests/tracing/readme.md index 3e80a741c4..f545a3c12d 100644 --- a/services/surfaceflinger/tests/tracing/readme.md +++ b/services/surfaceflinger/tests/tracing/readme.md @@ -14,7 +14,9 @@ corresponding layer trace in testdata. #### Workflow #### Add transaction traces that resulted in front end bugs along with the layer trace after fixing the issue. The layer trace -can be generated by using the layertracegenerator tool. The +can be generated by using the layertracegenerator tool. Use the +--last-entry-only flag to generate only the last entry in the +trace. This will keep the test data to a manageable size. The main goal of this test suite is to add regression tests with minimal effort. diff --git a/services/surfaceflinger/tests/tracing/testdata/layers_trace_b282110579.winscope b/services/surfaceflinger/tests/tracing/testdata/layers_trace_b282110579.winscope Binary files differnew file mode 100644 index 0000000000..3246453971 --- /dev/null +++ b/services/surfaceflinger/tests/tracing/testdata/layers_trace_b282110579.winscope diff --git a/services/surfaceflinger/tests/tracing/testdata/transactions_trace_b282110579.winscope b/services/surfaceflinger/tests/tracing/testdata/transactions_trace_b282110579.winscope Binary files differnew file mode 100644 index 0000000000..ecb94319d2 --- /dev/null +++ b/services/surfaceflinger/tests/tracing/testdata/transactions_trace_b282110579.winscope diff --git a/services/surfaceflinger/tests/unittests/Android.bp b/services/surfaceflinger/tests/unittests/Android.bp index db81bad968..ceb69df644 100644 --- a/services/surfaceflinger/tests/unittests/Android.bp +++ b/services/surfaceflinger/tests/unittests/Android.bp @@ -100,17 +100,18 @@ cc_test { "LayerTestUtils.cpp", "MessageQueueTest.cpp", "PowerAdvisorTest.cpp", + "SmallAreaDetectionAllowMappingsTest.cpp", "SurfaceFlinger_CreateDisplayTest.cpp", "SurfaceFlinger_DestroyDisplayTest.cpp", "SurfaceFlinger_DisplayModeSwitching.cpp", "SurfaceFlinger_DisplayTransactionCommitTest.cpp", "SurfaceFlinger_ExcludeDolbyVisionTest.cpp", + "SurfaceFlinger_FoldableTest.cpp", "SurfaceFlinger_GetDisplayNativePrimariesTest.cpp", "SurfaceFlinger_GetDisplayStatsTest.cpp", "SurfaceFlinger_HdrOutputControlTest.cpp", "SurfaceFlinger_HotplugTest.cpp", "SurfaceFlinger_InitializeDisplaysTest.cpp", - "SurfaceFlinger_MultiDisplayPacesetterTest.cpp", "SurfaceFlinger_NotifyPowerBoostTest.cpp", "SurfaceFlinger_PowerHintTest.cpp", "SurfaceFlinger_SetDisplayStateTest.cpp", diff --git a/services/surfaceflinger/tests/unittests/DisplayDevice_InitiateModeChange.cpp b/services/surfaceflinger/tests/unittests/DisplayDevice_InitiateModeChange.cpp index 60ad7a3a03..2d87ddd488 100644 --- a/services/surfaceflinger/tests/unittests/DisplayDevice_InitiateModeChange.cpp +++ b/services/surfaceflinger/tests/unittests/DisplayDevice_InitiateModeChange.cpp @@ -78,7 +78,7 @@ TEST_F(InitiateModeChangeTest, setDesiredActiveMode_setNewMode) { mDisplay->setDesiredActiveMode( {scheduler::FrameRateMode{90_Hz, kMode90}, Event::None})); ASSERT_NE(std::nullopt, mDisplay->getDesiredActiveMode()); - EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, mDisplay->getDesiredActiveMode()->modeOpt); + EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, *mDisplay->getDesiredActiveMode()->modeOpt); EXPECT_EQ(Event::None, mDisplay->getDesiredActiveMode()->event); // Setting another mode should be cached but return None @@ -86,7 +86,7 @@ TEST_F(InitiateModeChangeTest, setDesiredActiveMode_setNewMode) { mDisplay->setDesiredActiveMode( {scheduler::FrameRateMode{120_Hz, kMode120}, Event::None})); ASSERT_NE(std::nullopt, mDisplay->getDesiredActiveMode()); - EXPECT_FRAME_RATE_MODE(kMode120, 120_Hz, mDisplay->getDesiredActiveMode()->modeOpt); + EXPECT_FRAME_RATE_MODE(kMode120, 120_Hz, *mDisplay->getDesiredActiveMode()->modeOpt); EXPECT_EQ(Event::None, mDisplay->getDesiredActiveMode()->event); } @@ -105,7 +105,7 @@ TEST_F(InitiateModeChangeTest, initiateModeChange) NO_THREAD_SAFETY_ANALYSIS { mDisplay->setDesiredActiveMode( {scheduler::FrameRateMode{90_Hz, kMode90}, Event::None})); ASSERT_NE(std::nullopt, mDisplay->getDesiredActiveMode()); - EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, mDisplay->getDesiredActiveMode()->modeOpt); + EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, *mDisplay->getDesiredActiveMode()->modeOpt); EXPECT_EQ(Event::None, mDisplay->getDesiredActiveMode()->event); hal::VsyncPeriodChangeConstraints constraints{ @@ -136,7 +136,7 @@ NO_THREAD_SAFETY_ANALYSIS { mDisplay->setDesiredActiveMode( {scheduler::FrameRateMode{90_Hz, kMode90}, Event::None})); ASSERT_NE(std::nullopt, mDisplay->getDesiredActiveMode()); - EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, mDisplay->getDesiredActiveMode()->modeOpt); + EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, *mDisplay->getDesiredActiveMode()->modeOpt); EXPECT_EQ(Event::None, mDisplay->getDesiredActiveMode()->event); hal::VsyncPeriodChangeConstraints constraints{ @@ -154,7 +154,7 @@ NO_THREAD_SAFETY_ANALYSIS { mDisplay->setDesiredActiveMode( {scheduler::FrameRateMode{120_Hz, kMode120}, Event::None})); ASSERT_NE(std::nullopt, mDisplay->getDesiredActiveMode()); - EXPECT_FRAME_RATE_MODE(kMode120, 120_Hz, mDisplay->getDesiredActiveMode()->modeOpt); + EXPECT_FRAME_RATE_MODE(kMode120, 120_Hz, *mDisplay->getDesiredActiveMode()->modeOpt); EXPECT_EQ(Event::None, mDisplay->getDesiredActiveMode()->event); EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, *mDisplay->getUpcomingActiveMode().modeOpt); diff --git a/services/surfaceflinger/tests/unittests/DisplayTransactionTestHelpers.h b/services/surfaceflinger/tests/unittests/DisplayTransactionTestHelpers.h index e64cb38b16..ee12276994 100644 --- a/services/surfaceflinger/tests/unittests/DisplayTransactionTestHelpers.h +++ b/services/surfaceflinger/tests/unittests/DisplayTransactionTestHelpers.h @@ -371,10 +371,11 @@ struct HwcDisplayVariant { // Called by tests to inject a HWC display setup template <bool kInitPowerMode = true> static void injectHwcDisplay(DisplayTransactionTest* test) { - EXPECT_CALL(*test->mComposer, getDisplayCapabilities(HWC_DISPLAY_ID, _)) - .WillOnce(DoAll(SetArgPointee<1>(std::vector<DisplayCapability>({})), - Return(Error::NONE))); if constexpr (kInitPowerMode) { + EXPECT_CALL(*test->mComposer, getDisplayCapabilities(HWC_DISPLAY_ID, _)) + .WillOnce(DoAll(SetArgPointee<1>(std::vector<DisplayCapability>({})), + Return(Error::NONE))); + EXPECT_CALL(*test->mComposer, setPowerMode(HWC_DISPLAY_ID, INIT_POWER_MODE)) .WillOnce(Return(Error::NONE)); } diff --git a/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp b/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp index 85d86a7acc..69128c02b6 100644 --- a/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp +++ b/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp @@ -959,6 +959,77 @@ TEST_F(LayerHistoryTest, heuristicLayerNotOscillating) { recordFramesAndExpect(layer, time, 27.1_Hz, 30_Hz, PRESENT_TIME_HISTORY_SIZE); } +TEST_F(LayerHistoryTest, smallDirtyLayer) { + auto layer = createLayer(); + + EXPECT_CALL(*layer, isVisible()).WillRepeatedly(Return(true)); + EXPECT_CALL(*layer, getFrameRateForLayerTree()).WillRepeatedly(Return(Layer::FrameRate())); + + nsecs_t time = systemTime(); + + EXPECT_EQ(1, layerCount()); + EXPECT_EQ(0, activeLayerCount()); + EXPECT_EQ(0, frequentLayerCount(time)); + + LayerHistory::Summary summary; + + // layer is active but infrequent. + for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) { + auto props = layer->getLayerProps(); + if (i % 3 == 0) { + props.isSmallDirty = false; + } else { + props.isSmallDirty = true; + } + + history().record(layer->getSequence(), props, time, time, + LayerHistory::LayerUpdateType::Buffer); + time += HI_FPS_PERIOD; + summary = summarizeLayerHistory(time); + } + + ASSERT_EQ(1, summary.size()); + ASSERT_EQ(LayerHistory::LayerVoteType::Heuristic, summary[0].vote); + EXPECT_GE(HI_FPS, summary[0].desiredRefreshRate); +} + +TEST_F(LayerHistoryTest, smallDirtyInMultiLayer) { + auto layer1 = createLayer("UI"); + auto layer2 = createLayer("Video"); + + EXPECT_CALL(*layer1, isVisible()).WillRepeatedly(Return(true)); + EXPECT_CALL(*layer1, getFrameRateForLayerTree()).WillRepeatedly(Return(Layer::FrameRate())); + + EXPECT_CALL(*layer2, isVisible()).WillRepeatedly(Return(true)); + EXPECT_CALL(*layer2, getFrameRateForLayerTree()) + .WillRepeatedly( + Return(Layer::FrameRate(30_Hz, Layer::FrameRateCompatibility::Default))); + + nsecs_t time = systemTime(); + + EXPECT_EQ(2, layerCount()); + EXPECT_EQ(0, activeLayerCount()); + EXPECT_EQ(0, frequentLayerCount(time)); + + LayerHistory::Summary summary; + + // layer1 is updating small dirty. + for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE + FREQUENT_LAYER_WINDOW_SIZE + 1; i++) { + auto props = layer1->getLayerProps(); + props.isSmallDirty = true; + history().record(layer1->getSequence(), props, 0 /*presentTime*/, time, + LayerHistory::LayerUpdateType::Buffer); + history().record(layer2->getSequence(), layer2->getLayerProps(), time, time, + LayerHistory::LayerUpdateType::Buffer); + time += HI_FPS_PERIOD; + summary = summarizeLayerHistory(time); + } + + ASSERT_EQ(1, summary.size()); + ASSERT_EQ(LayerHistory::LayerVoteType::ExplicitDefault, summary[0].vote); + ASSERT_EQ(30_Hz, summary[0].desiredRefreshRate); +} + class LayerHistoryTestParameterized : public LayerHistoryTest, public testing::WithParamInterface<std::chrono::nanoseconds> { }; diff --git a/services/surfaceflinger/tests/unittests/LayerSnapshotTest.cpp b/services/surfaceflinger/tests/unittests/LayerSnapshotTest.cpp index b8a7446b3a..5da893ee62 100644 --- a/services/surfaceflinger/tests/unittests/LayerSnapshotTest.cpp +++ b/services/surfaceflinger/tests/unittests/LayerSnapshotTest.cpp @@ -105,7 +105,7 @@ protected: LayerHierarchyBuilder mHierarchyBuilder{{}}; LayerSnapshotBuilder mSnapshotBuilder; - display::DisplayMap<ui::LayerStack, frontend::DisplayInfo> mFrontEndDisplayInfos; + DisplayInfos mFrontEndDisplayInfos; renderengine::ShadowSettings globalShadowSettings; static const std::vector<uint32_t> STARTING_ZORDER; }; @@ -228,6 +228,7 @@ TEST_F(LayerSnapshotTest, AlphaInheritedByChildren) { setAlpha(1, 0.5); setAlpha(122, 0.5); UPDATE_AND_VERIFY(mSnapshotBuilder, STARTING_ZORDER); + EXPECT_EQ(getSnapshot(1)->alpha, 0.5f); EXPECT_EQ(getSnapshot(12)->alpha, 0.5f); EXPECT_EQ(getSnapshot(1221)->alpha, 0.25f); } @@ -236,28 +237,30 @@ TEST_F(LayerSnapshotTest, AlphaInheritedByChildren) { TEST_F(LayerSnapshotTest, UpdateClearsPreviousChangeStates) { setCrop(1, Rect(1, 2, 3, 4)); UPDATE_AND_VERIFY(mSnapshotBuilder, STARTING_ZORDER); - EXPECT_TRUE(getSnapshot(1)->changes.get() != 0); - EXPECT_TRUE(getSnapshot(11)->changes.get() != 0); + EXPECT_TRUE(getSnapshot(1)->changes.test(RequestedLayerState::Changes::Geometry)); + EXPECT_TRUE(getSnapshot(11)->changes.test(RequestedLayerState::Changes::Geometry)); setCrop(2, Rect(1, 2, 3, 4)); UPDATE_AND_VERIFY(mSnapshotBuilder, STARTING_ZORDER); - EXPECT_TRUE(getSnapshot(2)->changes.get() != 0); - EXPECT_TRUE(getSnapshot(1)->changes.get() == 0); - EXPECT_TRUE(getSnapshot(11)->changes.get() == 0); + EXPECT_TRUE(getSnapshot(2)->changes.test(RequestedLayerState::Changes::Geometry)); + EXPECT_FALSE(getSnapshot(1)->changes.test(RequestedLayerState::Changes::Geometry)); + EXPECT_FALSE(getSnapshot(11)->changes.test(RequestedLayerState::Changes::Geometry)); } TEST_F(LayerSnapshotTest, FastPathClearsPreviousChangeStates) { setColor(11, {1._hf, 0._hf, 0._hf}); UPDATE_AND_VERIFY(mSnapshotBuilder, STARTING_ZORDER); - EXPECT_TRUE(getSnapshot(11)->changes.get() != 0); - EXPECT_TRUE(getSnapshot(1)->changes.get() == 0); + EXPECT_EQ(getSnapshot(11)->changes, RequestedLayerState::Changes::Content); + EXPECT_EQ(getSnapshot(11)->clientChanges, layer_state_t::eColorChanged); + EXPECT_EQ(getSnapshot(1)->changes.get(), 0u); UPDATE_AND_VERIFY(mSnapshotBuilder, STARTING_ZORDER); - EXPECT_TRUE(getSnapshot(11)->changes.get() == 0); + EXPECT_EQ(getSnapshot(11)->changes.get(), 0u); } TEST_F(LayerSnapshotTest, FastPathSetsChangeFlagToContent) { setColor(1, {1._hf, 0._hf, 0._hf}); UPDATE_AND_VERIFY(mSnapshotBuilder, STARTING_ZORDER); EXPECT_EQ(getSnapshot(1)->changes, RequestedLayerState::Changes::Content); + EXPECT_EQ(getSnapshot(1)->clientChanges, layer_state_t::eColorChanged); } TEST_F(LayerSnapshotTest, GameMode) { @@ -270,7 +273,9 @@ TEST_F(LayerSnapshotTest, GameMode) { transactions.back().states.front().layerId = 1; transactions.back().states.front().state.layerId = static_cast<int32_t>(1); mLifecycleManager.applyTransactions(transactions); + EXPECT_EQ(mLifecycleManager.getGlobalChanges(), RequestedLayerState::Changes::GameMode); UPDATE_AND_VERIFY(mSnapshotBuilder, STARTING_ZORDER); + EXPECT_EQ(getSnapshot(1)->clientChanges, layer_state_t::eMetadataChanged); EXPECT_EQ(static_cast<int32_t>(getSnapshot(1)->gameMode), 42); EXPECT_EQ(static_cast<int32_t>(getSnapshot(11)->gameMode), 42); } @@ -309,7 +314,7 @@ TEST_F(LayerSnapshotTest, NoLayerVoteForParentWithChildVotes) { EXPECT_EQ(getSnapshot(1)->frameRate.type, scheduler::LayerInfo::FrameRateCompatibility::NoVote); } -TEST_F(LayerSnapshotTest, canCropTouchableRegion) { +TEST_F(LayerSnapshotTest, CanCropTouchableRegion) { // ROOT // ├── 1 // │ ├── 11 diff --git a/services/surfaceflinger/tests/unittests/MessageQueueTest.cpp b/services/surfaceflinger/tests/unittests/MessageQueueTest.cpp index 8f1b450b06..9aa089f900 100644 --- a/services/surfaceflinger/tests/unittests/MessageQueueTest.cpp +++ b/services/surfaceflinger/tests/unittests/MessageQueueTest.cpp @@ -20,9 +20,10 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> +#include <scheduler/interface/ICompositor.h> + #include "FrameTimeline.h" #include "Scheduler/MessageQueue.h" -#include "SurfaceFlinger.h" #include "mock/MockVSyncDispatch.h" namespace android { @@ -34,8 +35,11 @@ using CallbackToken = scheduler::VSyncDispatch::CallbackToken; struct NoOpCompositor final : ICompositor { void configure() override {} - bool commit(TimePoint, VsyncId, TimePoint) override { return false; } - void composite(TimePoint, VsyncId) override {} + bool commit(PhysicalDisplayId, const scheduler::FrameTargets&) override { return false; } + CompositeResultsPerDisplay composite(PhysicalDisplayId, + const scheduler::FrameTargeters&) override { + return {}; + } void sample() override {} } gNoOpCompositor; @@ -137,7 +141,7 @@ TEST_F(MessageQueueTest, commitTwiceWithCallback) { generateTokenForPredictions(frametimeline::TimelineItem(kStartTime.ns(), kEndTime.ns(), kPresentTime.ns()))) - .WillOnce(Return(vsyncId.value)); + .WillOnce(Return(ftl::to_underlying(vsyncId))); EXPECT_CALL(*mEventQueue.mHandler, dispatchFrame(vsyncId, kPresentTime)).Times(1); EXPECT_NO_FATAL_FAILURE( mEventQueue.vsyncCallback(kPresentTime.ns(), kStartTime.ns(), kEndTime.ns())); diff --git a/services/surfaceflinger/tests/unittests/RefreshRateSelectorTest.cpp b/services/surfaceflinger/tests/unittests/RefreshRateSelectorTest.cpp index d63e187ac4..0397b9936f 100644 --- a/services/surfaceflinger/tests/unittests/RefreshRateSelectorTest.cpp +++ b/services/surfaceflinger/tests/unittests/RefreshRateSelectorTest.cpp @@ -222,6 +222,7 @@ protected: makeModes(kMode60, kMode90, kMode72_G1, kMode120_G1, kMode30_G1, kMode25_G1, kMode50); static inline const DisplayModes kModes_60_120 = makeModes(kMode60, kMode120); static inline const DisplayModes kModes_1_5_10 = makeModes(kMode1, kMode5, kMode10); + static inline const DisplayModes kModes_60_90_120 = makeModes(kMode60, kMode90, kMode120); // This is a typical TV configuration. static inline const DisplayModes kModes_24_25_30_50_60_Frac = @@ -1413,7 +1414,9 @@ TEST_P(RefreshRateSelectorTest, getBestFrameRateMode_ExplicitDefault) { ss << "ExplicitDefault " << desired; lr.name = ss.str(); - EXPECT_EQ(expected, selector.getBestFrameRateMode(layers)->getFps()); + const auto bestFps = selector.getBestFrameRateMode(layers)->getFps(); + EXPECT_EQ(expected, bestFps) + << "expected " << expected << " for " << desired << " but got " << bestFps; } } @@ -1422,7 +1425,7 @@ TEST_P(RefreshRateSelectorTest, std::vector<LayerRequirement> layers = {{.weight = 1.f}}; auto& lr = layers[0]; - // Test that 23.976 will choose 24 if 23.976 is not supported + // Test that 23.976 will prefer 60 over 59.94 and 30 { auto selector = createSelector(makeModes(kMode24, kMode25, kMode30, kMode30Frac, kMode60, kMode60Frac), @@ -1431,7 +1434,7 @@ TEST_P(RefreshRateSelectorTest, lr.vote = LayerVoteType::ExplicitExactOrMultiple; lr.desiredRefreshRate = 23.976_Hz; lr.name = "ExplicitExactOrMultiple 23.976 Hz"; - EXPECT_EQ(kModeId24, selector.getBestFrameRateMode(layers)->getId()); + EXPECT_EQ(kModeId60, selector.getBestFrameRateMode(layers)->getId()); } // Test that 24 will choose 23.976 if 24 is not supported @@ -1456,13 +1459,13 @@ TEST_P(RefreshRateSelectorTest, EXPECT_EQ(kModeId60Frac, selector.getBestFrameRateMode(layers)->getId()); } - // Test that 29.97 will choose 30 if 59.94 is not supported + // Test that 29.97 will choose 60 if 59.94 is not supported { auto selector = createSelector(makeModes(kMode30, kMode60), kModeId60); lr.desiredRefreshRate = 29.97_Hz; lr.name = "ExplicitExactOrMultiple 29.97 Hz"; - EXPECT_EQ(kModeId30, selector.getBestFrameRateMode(layers)->getId()); + EXPECT_EQ(kModeId60, selector.getBestFrameRateMode(layers)->getId()); } // Test that 59.94 will choose 60 if 59.94 is not supported @@ -2516,6 +2519,71 @@ TEST_P(RefreshRateSelectorTest, isFractionalPairOrMultiple) { EXPECT_FALSE(RefreshRateSelector::isFractionalPairOrMultiple(29.97_Hz, 59.94_Hz)); } +TEST_P(RefreshRateSelectorTest, test23976Chooses120) { + auto selector = createSelector(kModes_60_90_120, kModeId120); + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "23.976 ExplicitExactOrMultiple"; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + layers[0].desiredRefreshRate = 23.976_Hz; + EXPECT_FRAME_RATE_MODE(kMode120, 120_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); +} + +TEST_P(RefreshRateSelectorTest, test23976Chooses60IfThresholdIs120) { + auto selector = + createSelector(kModes_60_90_120, kModeId120, {.frameRateMultipleThreshold = 120}); + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "23.976 ExplicitExactOrMultiple"; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + layers[0].desiredRefreshRate = 23.976_Hz; + EXPECT_FRAME_RATE_MODE(kMode60, 60_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); +} + +TEST_P(RefreshRateSelectorTest, test25Chooses60) { + auto selector = createSelector(kModes_60_90_120, kModeId120); + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "25 ExplicitExactOrMultiple"; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + layers[0].desiredRefreshRate = 25.00_Hz; + EXPECT_FRAME_RATE_MODE(kMode60, 60_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); +} + +TEST_P(RefreshRateSelectorTest, test2997Chooses60) { + auto selector = createSelector(kModes_60_90_120, kModeId120); + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "29.97 ExplicitExactOrMultiple"; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + layers[0].desiredRefreshRate = 29.97_Hz; + EXPECT_FRAME_RATE_MODE(kMode60, 60_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); +} + +TEST_P(RefreshRateSelectorTest, test50Chooses120) { + auto selector = createSelector(kModes_60_90_120, kModeId120); + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "50 ExplicitExactOrMultiple"; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + layers[0].desiredRefreshRate = 50.00_Hz; + EXPECT_FRAME_RATE_MODE(kMode120, 120_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); +} + +TEST_P(RefreshRateSelectorTest, test50Chooses60IfThresholdIs120) { + auto selector = + createSelector(kModes_60_90_120, kModeId120, {.frameRateMultipleThreshold = 120}); + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "50 ExplicitExactOrMultiple"; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + layers[0].desiredRefreshRate = 50.00_Hz; + EXPECT_FRAME_RATE_MODE(kMode60, 60_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); +} + +TEST_P(RefreshRateSelectorTest, test5994Chooses60) { + auto selector = createSelector(kModes_60_90_120, kModeId120); + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "59.94 ExplicitExactOrMultiple"; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + layers[0].desiredRefreshRate = 59.94_Hz; + EXPECT_FRAME_RATE_MODE(kMode60, 60_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); +} + TEST_P(RefreshRateSelectorTest, getFrameRateOverrides_noLayers) { auto selector = createSelector(kModes_30_60_72_90_120, kModeId120); @@ -3042,5 +3110,84 @@ TEST_P(RefreshRateSelectorTest, frameRateNotInRange) { EXPECT_FRAME_RATE_MODE(kMode60, 60_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); } +TEST_P(RefreshRateSelectorTest, frameRateIsLowerThanMinSupported) { + if (GetParam() != Config::FrameRateOverride::Enabled) { + return; + } + + auto selector = createSelector(kModes_60_90, kModeId60); + + constexpr Fps kMin = RefreshRateSelector::kMinSupportedFrameRate; + constexpr FpsRanges kLowerThanMin = {{60_Hz, 90_Hz}, {kMin / 2, kMin / 2}}; + + EXPECT_EQ(SetPolicyResult::Changed, + selector.setDisplayManagerPolicy( + {DisplayModeId(kModeId60), kLowerThanMin, kLowerThanMin})); +} + +// b/296079213 +TEST_P(RefreshRateSelectorTest, frameRateOverrideInBlockingZone60_120) { + auto selector = createSelector(kModes_60_120, kModeId120); + + const FpsRange only120 = {120_Hz, 120_Hz}; + const FpsRange allRange = {0_Hz, 120_Hz}; + EXPECT_EQ(SetPolicyResult::Changed, + selector.setDisplayManagerPolicy( + {kModeId120, {only120, allRange}, {allRange, allRange}})); + + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "30Hz ExplicitExactOrMultiple"; + layers[0].desiredRefreshRate = 30_Hz; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + + if (GetParam() != Config::FrameRateOverride::Enabled) { + EXPECT_FRAME_RATE_MODE(kMode120, 120_Hz, + selector.getBestScoredFrameRate(layers).frameRateMode); + } else { + EXPECT_FRAME_RATE_MODE(kMode120, 30_Hz, + selector.getBestScoredFrameRate(layers).frameRateMode); + } +} + +TEST_P(RefreshRateSelectorTest, frameRateOverrideInBlockingZone60_90) { + auto selector = createSelector(kModes_60_90, kModeId90); + + const FpsRange only90 = {90_Hz, 90_Hz}; + const FpsRange allRange = {0_Hz, 90_Hz}; + EXPECT_EQ(SetPolicyResult::Changed, + selector.setDisplayManagerPolicy( + {kModeId90, {only90, allRange}, {allRange, allRange}})); + + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "30Hz ExplicitExactOrMultiple"; + layers[0].desiredRefreshRate = 30_Hz; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + + if (GetParam() != Config::FrameRateOverride::Enabled) { + EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, + selector.getBestScoredFrameRate(layers).frameRateMode); + } else { + EXPECT_FRAME_RATE_MODE(kMode90, 30_Hz, + selector.getBestScoredFrameRate(layers).frameRateMode); + } +} + +TEST_P(RefreshRateSelectorTest, frameRateOverrideInBlockingZone60_90_NonDivisor) { + auto selector = createSelector(kModes_60_90, kModeId90); + + const FpsRange only90 = {90_Hz, 90_Hz}; + const FpsRange allRange = {0_Hz, 90_Hz}; + EXPECT_EQ(SetPolicyResult::Changed, + selector.setDisplayManagerPolicy( + {kModeId90, {only90, allRange}, {allRange, allRange}})); + + std::vector<LayerRequirement> layers = {{.weight = 1.f}}; + layers[0].name = "60Hz ExplicitExactOrMultiple"; + layers[0].desiredRefreshRate = 60_Hz; + layers[0].vote = LayerVoteType::ExplicitExactOrMultiple; + + EXPECT_FRAME_RATE_MODE(kMode90, 90_Hz, selector.getBestScoredFrameRate(layers).frameRateMode); +} + } // namespace } // namespace android::scheduler diff --git a/services/surfaceflinger/tests/unittests/SchedulerTest.cpp b/services/surfaceflinger/tests/unittests/SchedulerTest.cpp index 965e37873f..682c998542 100644 --- a/services/surfaceflinger/tests/unittests/SchedulerTest.cpp +++ b/services/surfaceflinger/tests/unittests/SchedulerTest.cpp @@ -155,6 +155,33 @@ TEST_F(SchedulerTest, validConnectionHandle) { EXPECT_EQ(kEventConnections, mScheduler->getEventThreadConnectionCount(mConnectionHandle)); } +TEST_F(SchedulerTest, registerDisplay) FTL_FAKE_GUARD(kMainThreadContext) { + // Hardware VSYNC should not change if the display is already registered. + EXPECT_CALL(mSchedulerCallback, requestHardwareVsync(kDisplayId1, false)).Times(0); + mScheduler->registerDisplay(kDisplayId1, + std::make_shared<RefreshRateSelector>(kDisplay1Modes, + kDisplay1Mode60->getId())); + + // TODO(b/241285191): Restore once VsyncSchedule::getPendingHardwareVsyncState is called by + // Scheduler::setDisplayPowerMode rather than SF::setPowerModeInternal. +#if 0 + // Hardware VSYNC should be disabled for newly registered displays. + EXPECT_CALL(mSchedulerCallback, requestHardwareVsync(kDisplayId2, false)).Times(1); + EXPECT_CALL(mSchedulerCallback, requestHardwareVsync(kDisplayId3, false)).Times(1); +#endif + + mScheduler->registerDisplay(kDisplayId2, + std::make_shared<RefreshRateSelector>(kDisplay2Modes, + kDisplay2Mode60->getId())); + mScheduler->registerDisplay(kDisplayId3, + std::make_shared<RefreshRateSelector>(kDisplay3Modes, + kDisplay3Mode60->getId())); + + EXPECT_FALSE(mScheduler->getVsyncSchedule(kDisplayId1)->getPendingHardwareVsyncState()); + EXPECT_FALSE(mScheduler->getVsyncSchedule(kDisplayId2)->getPendingHardwareVsyncState()); + EXPECT_FALSE(mScheduler->getVsyncSchedule(kDisplayId3)->getPendingHardwareVsyncState()); +} + TEST_F(SchedulerTest, chooseRefreshRateForContentIsNoopWhenModeSwitchingIsNotSupported) { // The layer is registered at creation time and deregistered at destruction time. sp<MockLayer> layer = sp<MockLayer>::make(mFlinger.flinger()); diff --git a/services/surfaceflinger/tests/unittests/SmallAreaDetectionAllowMappingsTest.cpp b/services/surfaceflinger/tests/unittests/SmallAreaDetectionAllowMappingsTest.cpp new file mode 100644 index 0000000000..b910485c06 --- /dev/null +++ b/services/surfaceflinger/tests/unittests/SmallAreaDetectionAllowMappingsTest.cpp @@ -0,0 +1,61 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#undef LOG_TAG +#define LOG_TAG "SmallAreaDetectionAllowMappingsTest" + +#include <gtest/gtest.h> + +#include "Scheduler/SmallAreaDetectionAllowMappings.h" + +namespace android::scheduler { + +class SmallAreaDetectionMappingsAllowTest : public testing::Test { +protected: + SmallAreaDetectionAllowMappings mMappings; +}; + +namespace { +TEST_F(SmallAreaDetectionMappingsAllowTest, testUpdate) { + const uid_t uid1 = 10100; + const uid_t uid2 = 10101; + const float threshold1 = 0.05f; + const float threshold2 = 0.07f; + std::vector<std::pair<uid_t, float>> mappings; + mappings.reserve(2); + mappings.push_back(std::make_pair(uid1, threshold1)); + mappings.push_back(std::make_pair(uid2, threshold2)); + + mMappings.update(mappings); + ASSERT_EQ(mMappings.getThresholdForUid(uid1).value(), threshold1); + ASSERT_EQ(mMappings.getThresholdForUid(uid2).value(), threshold2); +} + +TEST_F(SmallAreaDetectionMappingsAllowTest, testSetThesholdForUid) { + const uid_t uid = 10111; + const float threshold = 0.05f; + + mMappings.setThesholdForUid(uid, threshold); + ASSERT_EQ(mMappings.getThresholdForUid(uid), threshold); +} + +TEST_F(SmallAreaDetectionMappingsAllowTest, testUidNotInTheMappings) { + const uid_t uid = 10222; + ASSERT_EQ(mMappings.getThresholdForUid(uid), std::nullopt); +} + +} // namespace +} // namespace android::scheduler diff --git a/services/surfaceflinger/tests/unittests/SurfaceFlinger_DisplayModeSwitching.cpp b/services/surfaceflinger/tests/unittests/SurfaceFlinger_DisplayModeSwitching.cpp index e17654602b..9ef3e9e825 100644 --- a/services/surfaceflinger/tests/unittests/SurfaceFlinger_DisplayModeSwitching.cpp +++ b/services/surfaceflinger/tests/unittests/SurfaceFlinger_DisplayModeSwitching.cpp @@ -59,10 +59,36 @@ public: .WillByDefault(Return(true)); } + static constexpr HWDisplayId kInnerDisplayHwcId = PrimaryDisplayVariant::HWC_DISPLAY_ID; + static constexpr HWDisplayId kOuterDisplayHwcId = kInnerDisplayHwcId + 1; + + auto injectOuterDisplay() { + constexpr PhysicalDisplayId kOuterDisplayId = PhysicalDisplayId::fromPort(254u); + + constexpr bool kIsPrimary = false; + TestableSurfaceFlinger::FakeHwcDisplayInjector(kOuterDisplayId, hal::DisplayType::PHYSICAL, + kIsPrimary) + .setHwcDisplayId(kOuterDisplayHwcId) + .setPowerMode(hal::PowerMode::OFF) + .inject(&mFlinger, mComposer); + + mOuterDisplay = mFakeDisplayInjector.injectInternalDisplay( + [&](FakeDisplayDeviceInjector& injector) { + injector.setPowerMode(hal::PowerMode::OFF); + injector.setDisplayModes(mock::cloneForDisplay(kOuterDisplayId, kModes), + kModeId120); + }, + {.displayId = kOuterDisplayId, + .hwcDisplayId = kOuterDisplayHwcId, + .isPrimary = kIsPrimary}); + + return std::forward_as_tuple(mDisplay, mOuterDisplay); + } + protected: void setupScheduler(std::shared_ptr<scheduler::RefreshRateSelector>); - sp<DisplayDevice> mDisplay; + sp<DisplayDevice> mDisplay, mOuterDisplay; mock::EventThread* mAppEventThread; static constexpr DisplayModeId kModeId60{0}; @@ -320,32 +346,16 @@ MATCHER_P(ModeSettledTo, modeId, "") { return true; } -TEST_F(DisplayModeSwitchingTest, multiDisplay) { - constexpr HWDisplayId kInnerDisplayHwcId = PrimaryDisplayVariant::HWC_DISPLAY_ID; - constexpr HWDisplayId kOuterDisplayHwcId = kInnerDisplayHwcId + 1; - - constexpr PhysicalDisplayId kOuterDisplayId = PhysicalDisplayId::fromPort(254u); - - constexpr bool kIsPrimary = false; - TestableSurfaceFlinger::FakeHwcDisplayInjector(kOuterDisplayId, hal::DisplayType::PHYSICAL, - kIsPrimary) - .setHwcDisplayId(kOuterDisplayHwcId) - .inject(&mFlinger, mComposer); - - const auto outerDisplay = mFakeDisplayInjector.injectInternalDisplay( - [&](FakeDisplayDeviceInjector& injector) { - injector.setDisplayModes(mock::cloneForDisplay(kOuterDisplayId, kModes), - kModeId120); - }, - {.displayId = kOuterDisplayId, - .hwcDisplayId = kOuterDisplayHwcId, - .isPrimary = kIsPrimary}); +TEST_F(DisplayModeSwitchingTest, innerXorOuterDisplay) { + const auto [innerDisplay, outerDisplay] = injectOuterDisplay(); - const auto& innerDisplay = mDisplay; + EXPECT_TRUE(innerDisplay->isPoweredOn()); + EXPECT_FALSE(outerDisplay->isPoweredOn()); EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId60)); EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId120)); + // Only the inner display is powered on. mFlinger.onActiveDisplayChanged(nullptr, *innerDisplay); EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId60)); @@ -380,6 +390,10 @@ TEST_F(DisplayModeSwitchingTest, multiDisplay) { EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId90)); EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId120)); + innerDisplay->setPowerMode(hal::PowerMode::OFF); + outerDisplay->setPowerMode(hal::PowerMode::ON); + + // Only the outer display is powered on. mFlinger.onActiveDisplayChanged(innerDisplay.get(), *outerDisplay); EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId90)); @@ -401,5 +415,161 @@ TEST_F(DisplayModeSwitchingTest, multiDisplay) { EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId60)); } +TEST_F(DisplayModeSwitchingTest, innerAndOuterDisplay) { + const auto [innerDisplay, outerDisplay] = injectOuterDisplay(); + + EXPECT_TRUE(innerDisplay->isPoweredOn()); + EXPECT_FALSE(outerDisplay->isPoweredOn()); + + EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId60)); + EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId120)); + + outerDisplay->setPowerMode(hal::PowerMode::ON); + + // Both displays are powered on. + mFlinger.onActiveDisplayChanged(nullptr, *innerDisplay); + + EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId60)); + EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId120)); + + EXPECT_EQ(NO_ERROR, + mFlinger.setDesiredDisplayModeSpecs(innerDisplay->getDisplayToken().promote(), + mock::createDisplayModeSpecs(kModeId90.value(), + false, 0.f, 120.f))); + + EXPECT_EQ(NO_ERROR, + mFlinger.setDesiredDisplayModeSpecs(outerDisplay->getDisplayToken().promote(), + mock::createDisplayModeSpecs(kModeId60.value(), + false, 0.f, 120.f))); + + EXPECT_THAT(innerDisplay, ModeSwitchingTo(&mFlinger, kModeId90)); + EXPECT_THAT(outerDisplay, ModeSwitchingTo(&mFlinger, kModeId60)); + + const VsyncPeriodChangeTimeline timeline{.refreshRequired = true}; + EXPECT_CALL(*mComposer, + setActiveConfigWithConstraints(kInnerDisplayHwcId, + hal::HWConfigId(kModeId90.value()), _, _)) + .WillOnce(DoAll(SetArgPointee<3>(timeline), Return(Error::NONE))); + + EXPECT_CALL(*mComposer, + setActiveConfigWithConstraints(kOuterDisplayHwcId, + hal::HWConfigId(kModeId60.value()), _, _)) + .WillOnce(DoAll(SetArgPointee<3>(timeline), Return(Error::NONE))); + + mFlinger.commit(); + + EXPECT_THAT(innerDisplay, ModeSwitchingTo(&mFlinger, kModeId90)); + EXPECT_THAT(outerDisplay, ModeSwitchingTo(&mFlinger, kModeId60)); + + mFlinger.commit(); + + EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId90)); + EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId60)); +} + +TEST_F(DisplayModeSwitchingTest, powerOffDuringModeSet) { + EXPECT_TRUE(mDisplay->isPoweredOn()); + EXPECT_THAT(mDisplay, ModeSettledTo(kModeId60)); + + EXPECT_EQ(NO_ERROR, + mFlinger.setDesiredDisplayModeSpecs(mDisplay->getDisplayToken().promote(), + mock::createDisplayModeSpecs(kModeId90.value(), + false, 0.f, 120.f))); + + EXPECT_THAT(mDisplay, ModeSwitchingTo(&mFlinger, kModeId90)); + + // Power off the display before the mode has been set. + mDisplay->setPowerMode(hal::PowerMode::OFF); + + const VsyncPeriodChangeTimeline timeline{.refreshRequired = true}; + EXPECT_CALL(*mComposer, + setActiveConfigWithConstraints(kInnerDisplayHwcId, + hal::HWConfigId(kModeId90.value()), _, _)) + .WillOnce(DoAll(SetArgPointee<3>(timeline), Return(Error::NONE))); + + mFlinger.commit(); + + // Powering off should not abort the mode set. + EXPECT_FALSE(mDisplay->isPoweredOn()); + EXPECT_THAT(mDisplay, ModeSwitchingTo(&mFlinger, kModeId90)); + + mFlinger.commit(); + + EXPECT_THAT(mDisplay, ModeSettledTo(kModeId90)); +} + +TEST_F(DisplayModeSwitchingTest, powerOffDuringConcurrentModeSet) { + const auto [innerDisplay, outerDisplay] = injectOuterDisplay(); + + EXPECT_TRUE(innerDisplay->isPoweredOn()); + EXPECT_FALSE(outerDisplay->isPoweredOn()); + + EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId60)); + EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId120)); + + outerDisplay->setPowerMode(hal::PowerMode::ON); + + // Both displays are powered on. + mFlinger.onActiveDisplayChanged(nullptr, *innerDisplay); + + EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId60)); + EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId120)); + + EXPECT_EQ(NO_ERROR, + mFlinger.setDesiredDisplayModeSpecs(innerDisplay->getDisplayToken().promote(), + mock::createDisplayModeSpecs(kModeId90.value(), + false, 0.f, 120.f))); + + EXPECT_EQ(NO_ERROR, + mFlinger.setDesiredDisplayModeSpecs(outerDisplay->getDisplayToken().promote(), + mock::createDisplayModeSpecs(kModeId60.value(), + false, 0.f, 120.f))); + + EXPECT_THAT(innerDisplay, ModeSwitchingTo(&mFlinger, kModeId90)); + EXPECT_THAT(outerDisplay, ModeSwitchingTo(&mFlinger, kModeId60)); + + // Power off the outer display before the mode has been set. + outerDisplay->setPowerMode(hal::PowerMode::OFF); + + const VsyncPeriodChangeTimeline timeline{.refreshRequired = true}; + EXPECT_CALL(*mComposer, + setActiveConfigWithConstraints(kInnerDisplayHwcId, + hal::HWConfigId(kModeId90.value()), _, _)) + .WillOnce(DoAll(SetArgPointee<3>(timeline), Return(Error::NONE))); + + mFlinger.commit(); + + // Powering off the inactive display should abort the mode set. + EXPECT_THAT(innerDisplay, ModeSwitchingTo(&mFlinger, kModeId90)); + EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId120)); + + mFlinger.commit(); + + EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId90)); + EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId120)); + + innerDisplay->setPowerMode(hal::PowerMode::OFF); + outerDisplay->setPowerMode(hal::PowerMode::ON); + + // Only the outer display is powered on. + mFlinger.onActiveDisplayChanged(innerDisplay.get(), *outerDisplay); + + EXPECT_CALL(*mComposer, + setActiveConfigWithConstraints(kOuterDisplayHwcId, + hal::HWConfigId(kModeId60.value()), _, _)) + .WillOnce(DoAll(SetArgPointee<3>(timeline), Return(Error::NONE))); + + mFlinger.commit(); + + // The mode set should resume once the display becomes active. + EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId90)); + EXPECT_THAT(outerDisplay, ModeSwitchingTo(&mFlinger, kModeId60)); + + mFlinger.commit(); + + EXPECT_THAT(innerDisplay, ModeSettledTo(kModeId90)); + EXPECT_THAT(outerDisplay, ModeSettledTo(kModeId60)); +} + } // namespace } // namespace android diff --git a/services/surfaceflinger/tests/unittests/SurfaceFlinger_FoldableTest.cpp b/services/surfaceflinger/tests/unittests/SurfaceFlinger_FoldableTest.cpp new file mode 100644 index 0000000000..ed8d909e63 --- /dev/null +++ b/services/surfaceflinger/tests/unittests/SurfaceFlinger_FoldableTest.cpp @@ -0,0 +1,192 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#undef LOG_TAG +#define LOG_TAG "LibSurfaceFlingerUnittests" + +#include "DisplayTransactionTestHelpers.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +namespace android { +namespace { + +struct FoldableTest : DisplayTransactionTest { + static constexpr bool kWithMockScheduler = false; + FoldableTest() : DisplayTransactionTest(kWithMockScheduler) {} + + void SetUp() override { + injectMockScheduler(kInnerDisplayId); + + // Inject inner and outer displays with uninitialized power modes. + constexpr bool kInitPowerMode = false; + { + InnerDisplayVariant::injectHwcDisplay<kInitPowerMode>(this); + auto injector = InnerDisplayVariant::makeFakeExistingDisplayInjector(this); + injector.setPowerMode(std::nullopt); + injector.setRefreshRateSelector(mFlinger.scheduler()->refreshRateSelector()); + mInnerDisplay = injector.inject(); + } + { + OuterDisplayVariant::injectHwcDisplay<kInitPowerMode>(this); + auto injector = OuterDisplayVariant::makeFakeExistingDisplayInjector(this); + injector.setPowerMode(std::nullopt); + mOuterDisplay = injector.inject(); + } + } + + static inline PhysicalDisplayId kInnerDisplayId = InnerDisplayVariant::DISPLAY_ID::get(); + static inline PhysicalDisplayId kOuterDisplayId = OuterDisplayVariant::DISPLAY_ID::get(); + + sp<DisplayDevice> mInnerDisplay, mOuterDisplay; +}; + +TEST_F(FoldableTest, promotesPacesetterOnBoot) { + // When the device boots, the inner display should be the pacesetter. + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kInnerDisplayId); + + // ...and should still be after powering on. + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kInnerDisplayId); +} + +TEST_F(FoldableTest, promotesPacesetterOnFoldUnfold) { + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + + // The outer display should become the pacesetter after folding. + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::OFF); + mFlinger.setPowerModeInternal(mOuterDisplay, PowerMode::ON); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kOuterDisplayId); + + // The inner display should become the pacesetter after unfolding. + mFlinger.setPowerModeInternal(mOuterDisplay, PowerMode::OFF); + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kInnerDisplayId); +} + +TEST_F(FoldableTest, promotesPacesetterOnConcurrentPowerOn) { + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + + // The inner display should stay the pacesetter if both are powered on. + // TODO(b/255635821): The pacesetter should depend on the displays' refresh rates. + mFlinger.setPowerModeInternal(mOuterDisplay, PowerMode::ON); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kInnerDisplayId); + + // The outer display should become the pacesetter if designated. + mFlinger.scheduler()->setPacesetterDisplay(kOuterDisplayId); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kOuterDisplayId); + + // The inner display should become the pacesetter if designated. + mFlinger.scheduler()->setPacesetterDisplay(kInnerDisplayId); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kInnerDisplayId); +} + +TEST_F(FoldableTest, promotesPacesetterOnConcurrentPowerOff) { + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + mFlinger.setPowerModeInternal(mOuterDisplay, PowerMode::ON); + + // The outer display should become the pacesetter if the inner display powers off. + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::OFF); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kOuterDisplayId); + + // The outer display should stay the pacesetter if both are powered on. + // TODO(b/255635821): The pacesetter should depend on the displays' refresh rates. + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kOuterDisplayId); + + // The inner display should become the pacesetter if the outer display powers off. + mFlinger.setPowerModeInternal(mOuterDisplay, PowerMode::OFF); + ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), kInnerDisplayId); +} + +TEST_F(FoldableTest, doesNotRequestHardwareVsyncIfPoweredOff) { + // Both displays are powered off. + EXPECT_CALL(mFlinger.mockSchedulerCallback(), requestHardwareVsync(kInnerDisplayId, _)) + .Times(0); + EXPECT_CALL(mFlinger.mockSchedulerCallback(), requestHardwareVsync(kOuterDisplayId, _)) + .Times(0); + + EXPECT_FALSE(mInnerDisplay->isPoweredOn()); + EXPECT_FALSE(mOuterDisplay->isPoweredOn()); + + auto& scheduler = *mFlinger.scheduler(); + scheduler.onHardwareVsyncRequest(kInnerDisplayId, true); + scheduler.onHardwareVsyncRequest(kOuterDisplayId, true); +} + +TEST_F(FoldableTest, requestsHardwareVsyncForInnerDisplay) { + // Only inner display is powered on. + EXPECT_CALL(mFlinger.mockSchedulerCallback(), requestHardwareVsync(kInnerDisplayId, true)) + .Times(1); + EXPECT_CALL(mFlinger.mockSchedulerCallback(), requestHardwareVsync(kOuterDisplayId, _)) + .Times(0); + + // The injected VsyncSchedule uses TestableScheduler::mockRequestHardwareVsync, so no calls to + // ISchedulerCallback::requestHardwareVsync are expected during setPowerModeInternal. + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + + EXPECT_TRUE(mInnerDisplay->isPoweredOn()); + EXPECT_FALSE(mOuterDisplay->isPoweredOn()); + + auto& scheduler = *mFlinger.scheduler(); + scheduler.onHardwareVsyncRequest(kInnerDisplayId, true); + scheduler.onHardwareVsyncRequest(kOuterDisplayId, true); +} + +TEST_F(FoldableTest, requestsHardwareVsyncForOuterDisplay) { + // Only outer display is powered on. + EXPECT_CALL(mFlinger.mockSchedulerCallback(), requestHardwareVsync(kInnerDisplayId, _)) + .Times(0); + EXPECT_CALL(mFlinger.mockSchedulerCallback(), requestHardwareVsync(kOuterDisplayId, true)) + .Times(1); + + // The injected VsyncSchedule uses TestableScheduler::mockRequestHardwareVsync, so no calls to + // ISchedulerCallback::requestHardwareVsync are expected during setPowerModeInternal. + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::OFF); + mFlinger.setPowerModeInternal(mOuterDisplay, PowerMode::ON); + + EXPECT_FALSE(mInnerDisplay->isPoweredOn()); + EXPECT_TRUE(mOuterDisplay->isPoweredOn()); + + auto& scheduler = *mFlinger.scheduler(); + scheduler.onHardwareVsyncRequest(kInnerDisplayId, true); + scheduler.onHardwareVsyncRequest(kOuterDisplayId, true); +} + +TEST_F(FoldableTest, requestsHardwareVsyncForBothDisplays) { + // Both displays are powered on. + EXPECT_CALL(mFlinger.mockSchedulerCallback(), requestHardwareVsync(kInnerDisplayId, true)) + .Times(1); + EXPECT_CALL(mFlinger.mockSchedulerCallback(), requestHardwareVsync(kOuterDisplayId, true)) + .Times(1); + + // The injected VsyncSchedule uses TestableScheduler::mockRequestHardwareVsync, so no calls to + // ISchedulerCallback::requestHardwareVsync are expected during setPowerModeInternal. + mFlinger.setPowerModeInternal(mInnerDisplay, PowerMode::ON); + mFlinger.setPowerModeInternal(mOuterDisplay, PowerMode::ON); + + EXPECT_TRUE(mInnerDisplay->isPoweredOn()); + EXPECT_TRUE(mOuterDisplay->isPoweredOn()); + + auto& scheduler = *mFlinger.scheduler(); + scheduler.onHardwareVsyncRequest(mInnerDisplay->getPhysicalId(), true); + scheduler.onHardwareVsyncRequest(mOuterDisplay->getPhysicalId(), true); +} + +} // namespace +} // namespace android diff --git a/services/surfaceflinger/tests/unittests/SurfaceFlinger_MultiDisplayPacesetterTest.cpp b/services/surfaceflinger/tests/unittests/SurfaceFlinger_MultiDisplayPacesetterTest.cpp deleted file mode 100644 index e38f56e65f..0000000000 --- a/services/surfaceflinger/tests/unittests/SurfaceFlinger_MultiDisplayPacesetterTest.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef LOG_TAG -#define LOG_TAG "LibSurfaceFlingerUnittests" - -#include "DisplayTransactionTestHelpers.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> - -namespace android { -namespace { - -struct MultiDisplayPacesetterTest : DisplayTransactionTest { - static constexpr bool kWithMockScheduler = false; - MultiDisplayPacesetterTest() : DisplayTransactionTest(kWithMockScheduler) {} -}; - -TEST_F(MultiDisplayPacesetterTest, foldable) { - injectMockScheduler(InnerDisplayVariant::DISPLAY_ID::get()); - - // Inject inner and outer displays with uninitialized power modes. - sp<DisplayDevice> innerDisplay, outerDisplay; - constexpr bool kInitPowerMode = false; - { - InnerDisplayVariant::injectHwcDisplay<kInitPowerMode>(this); - auto injector = InnerDisplayVariant::makeFakeExistingDisplayInjector(this); - injector.setPowerMode(std::nullopt); - injector.setRefreshRateSelector(mFlinger.scheduler()->refreshRateSelector()); - innerDisplay = injector.inject(); - } - { - OuterDisplayVariant::injectHwcDisplay<kInitPowerMode>(this); - auto injector = OuterDisplayVariant::makeFakeExistingDisplayInjector(this); - injector.setPowerMode(std::nullopt); - outerDisplay = injector.inject(); - } - - // When the device boots, the inner display should be the pacesetter. - ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), innerDisplay->getPhysicalId()); - - // ...and should still be after powering on. - mFlinger.setPowerModeInternal(innerDisplay, PowerMode::ON); - ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), innerDisplay->getPhysicalId()); - - // The outer display should become the pacesetter after folding. - mFlinger.setPowerModeInternal(innerDisplay, PowerMode::OFF); - mFlinger.setPowerModeInternal(outerDisplay, PowerMode::ON); - ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), outerDisplay->getPhysicalId()); - - // The inner display should become the pacesetter after unfolding. - mFlinger.setPowerModeInternal(outerDisplay, PowerMode::OFF); - mFlinger.setPowerModeInternal(innerDisplay, PowerMode::ON); - ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), innerDisplay->getPhysicalId()); - - // The inner display should stay the pacesetter if both are powered on. - // TODO(b/255635821): The pacesetter should depend on the displays' refresh rates. - mFlinger.setPowerModeInternal(outerDisplay, PowerMode::ON); - ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), innerDisplay->getPhysicalId()); - - // The outer display should become the pacesetter if designated. - mFlinger.scheduler()->setPacesetterDisplay(outerDisplay->getPhysicalId()); - ASSERT_EQ(mFlinger.scheduler()->pacesetterDisplayId(), outerDisplay->getPhysicalId()); -} - -} // namespace -} // namespace android diff --git a/services/surfaceflinger/tests/unittests/SurfaceFlinger_SetPowerModeInternalTest.cpp b/services/surfaceflinger/tests/unittests/SurfaceFlinger_SetPowerModeInternalTest.cpp index 7754c21805..cf3fab3aa3 100644 --- a/services/surfaceflinger/tests/unittests/SurfaceFlinger_SetPowerModeInternalTest.cpp +++ b/services/surfaceflinger/tests/unittests/SurfaceFlinger_SetPowerModeInternalTest.cpp @@ -61,7 +61,7 @@ struct DozeNotSupportedVariant { struct EventThreadBaseSupportedVariant { static void setupVsyncNoCallExpectations(DisplayTransactionTest* test) { // Expect no change to hardware nor synthetic VSYNC. - EXPECT_CALL(test->mFlinger.mockSchedulerCallback(), setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(test->mFlinger.scheduler()->mockRequestHardwareVsync, Call(_, _)).Times(0); EXPECT_CALL(*test->mEventThread, enableSyntheticVsync(_)).Times(0); } }; @@ -79,13 +79,13 @@ struct EventThreadNotSupportedVariant : public EventThreadBaseSupportedVariant { struct EventThreadIsSupportedVariant : public EventThreadBaseSupportedVariant { static void setupEnableVsyncCallExpectations(DisplayTransactionTest* test) { // Expect to enable hardware VSYNC and disable synthetic VSYNC. - EXPECT_CALL(test->mFlinger.mockSchedulerCallback(), setVsyncEnabled(_, true)).Times(1); + EXPECT_CALL(test->mFlinger.scheduler()->mockRequestHardwareVsync, Call(_, true)).Times(1); EXPECT_CALL(*test->mEventThread, enableSyntheticVsync(false)).Times(1); } static void setupDisableVsyncCallExpectations(DisplayTransactionTest* test) { // Expect to disable hardware VSYNC and enable synthetic VSYNC. - EXPECT_CALL(test->mFlinger.mockSchedulerCallback(), setVsyncEnabled(_, false)).Times(1); + EXPECT_CALL(test->mFlinger.scheduler()->mockRequestHardwareVsync, Call(_, false)).Times(1); EXPECT_CALL(*test->mEventThread, enableSyntheticVsync(true)).Times(1); } }; diff --git a/services/surfaceflinger/tests/unittests/TestableScheduler.h b/services/surfaceflinger/tests/unittests/TestableScheduler.h index 3b6a987796..ffe8c1024d 100644 --- a/services/surfaceflinger/tests/unittests/TestableScheduler.h +++ b/services/surfaceflinger/tests/unittests/TestableScheduler.h @@ -27,6 +27,7 @@ #include "Scheduler/Scheduler.h" #include "Scheduler/VSyncTracker.h" #include "Scheduler/VsyncController.h" +#include "Scheduler/VsyncSchedule.h" #include "mock/MockVSyncDispatch.h" #include "mock/MockVSyncTracker.h" #include "mock/MockVsyncController.h" @@ -80,9 +81,13 @@ public: new VsyncSchedule(displayId, std::move(tracker), std::make_shared< mock::VSyncDispatch>(), - std::move(controller)))); + std::move(controller), + mockRequestHardwareVsync + .AsStdFunction()))); } + testing::MockFunction<void(PhysicalDisplayId, bool)> mockRequestHardwareVsync; + void unregisterDisplay(PhysicalDisplayId displayId) { ftl::FakeGuard guard(kMainThreadContext); Scheduler::unregisterDisplay(displayId); @@ -163,11 +168,16 @@ public: : VsyncSchedule::HwVsyncState::Disabled; } + using Scheduler::onHardwareVsyncRequest; + private: // ICompositor overrides: void configure() override {} - bool commit(TimePoint, VsyncId, TimePoint) override { return false; } - void composite(TimePoint, VsyncId) override {} + bool commit(PhysicalDisplayId, const scheduler::FrameTargets&) override { return false; } + CompositeResultsPerDisplay composite(PhysicalDisplayId, + const scheduler::FrameTargeters&) override { + return {}; + } void sample() override {} }; diff --git a/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h b/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h index 945e48842d..6b13c0ddaf 100644 --- a/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h +++ b/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h @@ -20,6 +20,11 @@ #include <chrono> #include <variant> +#include <ftl/fake_guard.h> +#include <ftl/match.h> +#include <gui/ScreenCaptureResults.h> +#include <ui/DynamicDisplayInfo.h> + #include <compositionengine/Display.h> #include <compositionengine/LayerFECompositionState.h> #include <compositionengine/OutputLayer.h> @@ -27,11 +32,7 @@ #include <compositionengine/impl/Display.h> #include <compositionengine/impl/OutputLayerCompositionState.h> #include <compositionengine/mock/DisplaySurface.h> -#include <ftl/fake_guard.h> -#include <ftl/match.h> -#include <gui/ScreenCaptureResults.h> -#include <ui/DynamicDisplayInfo.h> #include "DisplayDevice.h" #include "FakeVsyncConfiguration.h" #include "FrameTracer/FrameTracer.h" @@ -44,7 +45,6 @@ #include "Scheduler/RefreshRateSelector.h" #include "StartPropertySetThread.h" #include "SurfaceFlinger.h" -#include "SurfaceFlingerDefaultFactory.h" #include "TestableScheduler.h" #include "mock/DisplayHardware/MockComposer.h" #include "mock/DisplayHardware/MockDisplayMode.h" @@ -353,32 +353,67 @@ public: * Forwarding for functions being tested */ - void configure() { mFlinger->configure(); } + void configure() { + ftl::FakeGuard guard(kMainThreadContext); + mFlinger->configure(); + } void configureAndCommit() { configure(); commitTransactionsLocked(eDisplayTransactionNeeded); } - TimePoint commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expectedVsyncTime) { - mFlinger->commit(frameTime, vsyncId, expectedVsyncTime); - return frameTime; + void commit(TimePoint frameTime, VsyncId vsyncId, TimePoint expectedVsyncTime, + bool composite = false) { + ftl::FakeGuard guard(kMainThreadContext); + + const auto displayIdOpt = mScheduler->pacesetterDisplayId(); + LOG_ALWAYS_FATAL_IF(!displayIdOpt); + const auto displayId = *displayIdOpt; + + constexpr bool kBackpressureGpuComposition = true; + scheduler::FrameTargeter frameTargeter(displayId, kBackpressureGpuComposition); + + frameTargeter.beginFrame({.frameBeginTime = frameTime, + .vsyncId = vsyncId, + .expectedVsyncTime = expectedVsyncTime, + .sfWorkDuration = 10ms}, + *mScheduler->getVsyncSchedule()); + + scheduler::FrameTargets targets; + scheduler::FrameTargeters targeters; + + for (const auto& [id, display] : + FTL_FAKE_GUARD(mFlinger->mStateLock, mFlinger->mPhysicalDisplays)) { + targets.try_emplace(id, &frameTargeter.target()); + targeters.try_emplace(id, &frameTargeter); + } + + mFlinger->commit(displayId, targets); + + if (composite) { + mFlinger->composite(displayId, targeters); + } } - TimePoint commit(TimePoint frameTime, VsyncId vsyncId) { - return commit(frameTime, vsyncId, frameTime + Period(10ms)); + void commit(TimePoint frameTime, VsyncId vsyncId, bool composite = false) { + return commit(frameTime, vsyncId, frameTime + Period(10ms), composite); } - TimePoint commit() { + void commit(bool composite = false) { const TimePoint frameTime = scheduler::SchedulerClock::now(); - return commit(frameTime, kVsyncId); + commit(frameTime, kVsyncId, composite); } void commitAndComposite(TimePoint frameTime, VsyncId vsyncId, TimePoint expectedVsyncTime) { - mFlinger->composite(commit(frameTime, vsyncId, expectedVsyncTime), vsyncId); + constexpr bool kComposite = true; + commit(frameTime, vsyncId, expectedVsyncTime, kComposite); } - void commitAndComposite() { mFlinger->composite(commit(), kVsyncId); } + void commitAndComposite() { + constexpr bool kComposite = true; + commit(kComposite); + } auto createDisplay(const String8& displayName, bool secure, float requestedRefreshRate = 0.0f) { return mFlinger->createDisplay(displayName, secure, requestedRefreshRate); diff --git a/services/surfaceflinger/tests/unittests/TransactionProtoParserTest.cpp b/services/surfaceflinger/tests/unittests/TransactionProtoParserTest.cpp index dd72174a37..a95a6453d5 100644 --- a/services/surfaceflinger/tests/unittests/TransactionProtoParserTest.cpp +++ b/services/surfaceflinger/tests/unittests/TransactionProtoParserTest.cpp @@ -122,7 +122,7 @@ TEST(TransactionProtoParserTest, parseDisplayInfo) { google::protobuf::RepeatedPtrField<proto::DisplayInfo> displayProtos; auto displayInfoProto = displayProtos.Add(); *displayInfoProto = TransactionProtoParser::toProto(d1, layerStack); - display::DisplayMap<ui::LayerStack, frontend::DisplayInfo> displayInfos; + frontend::DisplayInfos displayInfos; TransactionProtoParser::fromProto(displayProtos, displayInfos); ASSERT_TRUE(displayInfos.contains(ui::LayerStack::fromValue(layerStack))); diff --git a/services/surfaceflinger/tests/unittests/TransactionTracingTest.cpp b/services/surfaceflinger/tests/unittests/TransactionTracingTest.cpp index 92411a7ba9..809966f9d6 100644 --- a/services/surfaceflinger/tests/unittests/TransactionTracingTest.cpp +++ b/services/surfaceflinger/tests/unittests/TransactionTracingTest.cpp @@ -37,7 +37,7 @@ protected: static constexpr size_t SMALL_BUFFER_SIZE = 1024; TransactionTracing mTracing; - void flush(int64_t vsyncId) { mTracing.flush(vsyncId); } + void flush() { mTracing.flush(); } proto::TransactionTraceFile writeToProto() { return mTracing.writeToProto(); } proto::TransactionTraceEntry bufferFront() { @@ -57,7 +57,7 @@ protected: std::vector<TransactionState> transactions; update.transactions.emplace_back(transaction); mTracing.addCommittedTransactions(vsyncId, 0, update, {}, false); - flush(vsyncId); + flush(); } void verifyEntry(const proto::TransactionTraceEntry& actualProto, @@ -116,7 +116,7 @@ TEST_F(TransactionTracingTest, addTransactions) { secondUpdate.transactions = std::vector<TransactionState>(transactions.begin(), transactions.begin() + 50); mTracing.addCommittedTransactions(secondTransactionSetVsyncId, 0, secondUpdate, {}, false); - flush(secondTransactionSetVsyncId); + flush(); proto::TransactionTraceFile proto = writeToProto(); ASSERT_EQ(proto.entry().size(), 2); @@ -158,7 +158,7 @@ protected: VSYNC_ID_FIRST_LAYER_CHANGE = ++mVsyncId; mTracing.addCommittedTransactions(VSYNC_ID_FIRST_LAYER_CHANGE, 0, update, {}, false); - flush(VSYNC_ID_FIRST_LAYER_CHANGE); + flush(); } // add transactions that modify the layer state further so we can test that layer state @@ -178,7 +178,7 @@ protected: update.transactions.emplace_back(transaction); VSYNC_ID_SECOND_LAYER_CHANGE = ++mVsyncId; mTracing.addCommittedTransactions(VSYNC_ID_SECOND_LAYER_CHANGE, 0, update, {}, false); - flush(VSYNC_ID_SECOND_LAYER_CHANGE); + flush(); } // remove child layer @@ -290,7 +290,7 @@ protected: update.transactions.emplace_back(transaction); mTracing.addCommittedTransactions(mVsyncId, 0, update, {}, false); - flush(mVsyncId); + flush(); } } diff --git a/services/surfaceflinger/tests/unittests/VsyncScheduleTest.cpp b/services/surfaceflinger/tests/unittests/VsyncScheduleTest.cpp index 4010fa6a5b..a8a3cd0293 100644 --- a/services/surfaceflinger/tests/unittests/VsyncScheduleTest.cpp +++ b/services/surfaceflinger/tests/unittests/VsyncScheduleTest.cpp @@ -25,7 +25,6 @@ #include <scheduler/Fps.h> #include "Scheduler/VsyncSchedule.h" #include "ThreadContext.h" -#include "mock/MockSchedulerCallback.h" #include "mock/MockVSyncDispatch.h" #include "mock/MockVSyncTracker.h" #include "mock/MockVsyncController.h" @@ -34,20 +33,21 @@ using testing::_; namespace android { -constexpr PhysicalDisplayId DEFAULT_DISPLAY_ID = PhysicalDisplayId::fromPort(42u); +constexpr PhysicalDisplayId kDisplayId = PhysicalDisplayId::fromPort(42u); class VsyncScheduleTest : public testing::Test { protected: VsyncScheduleTest(); ~VsyncScheduleTest() override; - scheduler::mock::SchedulerCallback mCallback; + testing::MockFunction<void(PhysicalDisplayId, bool)> mRequestHardwareVsync; + const std::unique_ptr<scheduler::VsyncSchedule> mVsyncSchedule = std::unique_ptr<scheduler::VsyncSchedule>( - new scheduler::VsyncSchedule(DEFAULT_DISPLAY_ID, - std::make_shared<mock::VSyncTracker>(), + new scheduler::VsyncSchedule(kDisplayId, std::make_shared<mock::VSyncTracker>(), std::make_shared<mock::VSyncDispatch>(), - std::make_unique<mock::VsyncController>())); + std::make_unique<mock::VsyncController>(), + mRequestHardwareVsync.AsStdFunction())); mock::VsyncController& getController() { return *static_cast<mock::VsyncController*>(&mVsyncSchedule->getController()); @@ -75,21 +75,21 @@ TEST_F(VsyncScheduleTest, InitiallyDisallowed) { } TEST_F(VsyncScheduleTest, EnableDoesNothingWhenDisallowed) { - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); - mVsyncSchedule->enableHardwareVsync(mCallback); + mVsyncSchedule->enableHardwareVsync(); } TEST_F(VsyncScheduleTest, DisableDoesNothingWhenDisallowed) { - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); - mVsyncSchedule->disableHardwareVsync(mCallback, false /* disallow */); + mVsyncSchedule->disableHardwareVsync(false /* disallow */); } TEST_F(VsyncScheduleTest, DisableDoesNothingWhenDisallowed2) { - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); - mVsyncSchedule->disableHardwareVsync(mCallback, true /* disallow */); + mVsyncSchedule->disableHardwareVsync(true /* disallow */); } TEST_F(VsyncScheduleTest, MakeAllowed) { @@ -98,33 +98,33 @@ TEST_F(VsyncScheduleTest, MakeAllowed) { TEST_F(VsyncScheduleTest, DisableDoesNothingWhenDisabled) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); - mVsyncSchedule->disableHardwareVsync(mCallback, false /* disallow */); + mVsyncSchedule->disableHardwareVsync(false /* disallow */); } TEST_F(VsyncScheduleTest, DisableDoesNothingWhenDisabled2) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); - mVsyncSchedule->disableHardwareVsync(mCallback, true /* disallow */); + mVsyncSchedule->disableHardwareVsync(true /* disallow */); } TEST_F(VsyncScheduleTest, EnableWorksWhenDisabled) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, true)); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, true)); - mVsyncSchedule->enableHardwareVsync(mCallback); + mVsyncSchedule->enableHardwareVsync(); } TEST_F(VsyncScheduleTest, EnableWorksOnce) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, true)); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, true)); - mVsyncSchedule->enableHardwareVsync(mCallback); + mVsyncSchedule->enableHardwareVsync(); - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); - mVsyncSchedule->enableHardwareVsync(mCallback); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); + mVsyncSchedule->enableHardwareVsync(); } TEST_F(VsyncScheduleTest, AllowedIsSticky) { @@ -134,22 +134,22 @@ TEST_F(VsyncScheduleTest, AllowedIsSticky) { TEST_F(VsyncScheduleTest, EnableDisable) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, true)); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, true)); - mVsyncSchedule->enableHardwareVsync(mCallback); + mVsyncSchedule->enableHardwareVsync(); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, false)); - mVsyncSchedule->disableHardwareVsync(mCallback, false /* disallow */); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, false)); + mVsyncSchedule->disableHardwareVsync(false /* disallow */); } TEST_F(VsyncScheduleTest, EnableDisable2) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, true)); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, true)); - mVsyncSchedule->enableHardwareVsync(mCallback); + mVsyncSchedule->enableHardwareVsync(); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, false)); - mVsyncSchedule->disableHardwareVsync(mCallback, true /* disallow */); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, false)); + mVsyncSchedule->disableHardwareVsync(true /* disallow */); } TEST_F(VsyncScheduleTest, StartPeriodTransition) { @@ -159,22 +159,22 @@ TEST_F(VsyncScheduleTest, StartPeriodTransition) { const Period period = (60_Hz).getPeriod(); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, true)); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, true)); EXPECT_CALL(getController(), startPeriodTransition(period.ns(), false)); - mVsyncSchedule->startPeriodTransition(mCallback, period, false); + mVsyncSchedule->startPeriodTransition(period, false); } TEST_F(VsyncScheduleTest, StartPeriodTransitionAlreadyEnabled) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - mVsyncSchedule->enableHardwareVsync(mCallback); + mVsyncSchedule->enableHardwareVsync(); const Period period = (60_Hz).getPeriod(); - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); EXPECT_CALL(getController(), startPeriodTransition(period.ns(), false)); - mVsyncSchedule->startPeriodTransition(mCallback, period, false); + mVsyncSchedule->startPeriodTransition(period, false); } TEST_F(VsyncScheduleTest, StartPeriodTransitionForce) { @@ -182,20 +182,20 @@ TEST_F(VsyncScheduleTest, StartPeriodTransitionForce) { const Period period = (60_Hz).getPeriod(); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, true)); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, true)); EXPECT_CALL(getController(), startPeriodTransition(period.ns(), true)); - mVsyncSchedule->startPeriodTransition(mCallback, period, true); + mVsyncSchedule->startPeriodTransition(period, true); } TEST_F(VsyncScheduleTest, AddResyncSampleDisallowed) { const Period period = (60_Hz).getPeriod(); const auto timestamp = TimePoint::now(); - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); EXPECT_CALL(getController(), addHwVsyncTimestamp(_, _, _)).Times(0); - mVsyncSchedule->addResyncSample(mCallback, timestamp, period); + mVsyncSchedule->addResyncSample(timestamp, period); } TEST_F(VsyncScheduleTest, AddResyncSampleDisabled) { @@ -203,40 +203,40 @@ TEST_F(VsyncScheduleTest, AddResyncSampleDisabled) { const Period period = (60_Hz).getPeriod(); const auto timestamp = TimePoint::now(); - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); EXPECT_CALL(getController(), addHwVsyncTimestamp(_, _, _)).Times(0); - mVsyncSchedule->addResyncSample(mCallback, timestamp, period); + mVsyncSchedule->addResyncSample(timestamp, period); } TEST_F(VsyncScheduleTest, AddResyncSampleReturnsTrue) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - mVsyncSchedule->enableHardwareVsync(mCallback); + mVsyncSchedule->enableHardwareVsync(); const Period period = (60_Hz).getPeriod(); const auto timestamp = TimePoint::now(); - EXPECT_CALL(mCallback, setVsyncEnabled(_, _)).Times(0); + EXPECT_CALL(mRequestHardwareVsync, Call(_, _)).Times(0); EXPECT_CALL(getController(), addHwVsyncTimestamp(timestamp.ns(), std::optional<nsecs_t>(period.ns()), _)) .WillOnce(Return(true)); - mVsyncSchedule->addResyncSample(mCallback, timestamp, period); + mVsyncSchedule->addResyncSample(timestamp, period); } TEST_F(VsyncScheduleTest, AddResyncSampleReturnsFalse) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - mVsyncSchedule->enableHardwareVsync(mCallback); + mVsyncSchedule->enableHardwareVsync(); const Period period = (60_Hz).getPeriod(); const auto timestamp = TimePoint::now(); - EXPECT_CALL(mCallback, setVsyncEnabled(DEFAULT_DISPLAY_ID, false)); + EXPECT_CALL(mRequestHardwareVsync, Call(kDisplayId, false)); EXPECT_CALL(getController(), addHwVsyncTimestamp(timestamp.ns(), std::optional<nsecs_t>(period.ns()), _)) .WillOnce(Return(false)); - mVsyncSchedule->addResyncSample(mCallback, timestamp, period); + mVsyncSchedule->addResyncSample(timestamp, period); } TEST_F(VsyncScheduleTest, PendingState) FTL_FAKE_GUARD(kMainThreadContext) { @@ -250,19 +250,19 @@ TEST_F(VsyncScheduleTest, PendingState) FTL_FAKE_GUARD(kMainThreadContext) { TEST_F(VsyncScheduleTest, DisableDoesNotMakeAllowed) { ASSERT_FALSE(mVsyncSchedule->isHardwareVsyncAllowed(false /* makeAllowed */)); - mVsyncSchedule->disableHardwareVsync(mCallback, false /* disallow */); + mVsyncSchedule->disableHardwareVsync(false /* disallow */); ASSERT_FALSE(mVsyncSchedule->isHardwareVsyncAllowed(false /* makeAllowed */)); } TEST_F(VsyncScheduleTest, DisallowMakesNotAllowed) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - mVsyncSchedule->disableHardwareVsync(mCallback, true /* disallow */); + mVsyncSchedule->disableHardwareVsync(true /* disallow */); ASSERT_FALSE(mVsyncSchedule->isHardwareVsyncAllowed(false /* makeAllowed */)); } TEST_F(VsyncScheduleTest, StillAllowedAfterDisable) { ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(true /* makeAllowed */)); - mVsyncSchedule->disableHardwareVsync(mCallback, false /* disallow */); + mVsyncSchedule->disableHardwareVsync(false /* disallow */); ASSERT_TRUE(mVsyncSchedule->isHardwareVsyncAllowed(false /* makeAllowed */)); } diff --git a/services/surfaceflinger/tests/unittests/mock/DisplayHardware/MockPowerAdvisor.h b/services/surfaceflinger/tests/unittests/mock/DisplayHardware/MockPowerAdvisor.h index 3caa2b9847..d635508b5e 100644 --- a/services/surfaceflinger/tests/unittests/mock/DisplayHardware/MockPowerAdvisor.h +++ b/services/surfaceflinger/tests/unittests/mock/DisplayHardware/MockPowerAdvisor.h @@ -32,6 +32,7 @@ public: MOCK_METHOD(void, setExpensiveRenderingExpected, (DisplayId displayId, bool expected), (override)); MOCK_METHOD(bool, isUsingExpensiveRendering, (), (override)); + MOCK_METHOD(void, notifyCpuLoadUp, (), (override)); MOCK_METHOD(void, notifyDisplayUpdateImminentAndCpuReset, (), (override)); MOCK_METHOD(bool, usePowerHintSession, (), (override)); MOCK_METHOD(bool, supportsPowerHintSession, (), (override)); diff --git a/services/surfaceflinger/tests/unittests/mock/DisplayHardware/MockPowerHalController.h b/services/surfaceflinger/tests/unittests/mock/DisplayHardware/MockPowerHalController.h index 358395d323..8e22f43b76 100644 --- a/services/surfaceflinger/tests/unittests/mock/DisplayHardware/MockPowerHalController.h +++ b/services/surfaceflinger/tests/unittests/mock/DisplayHardware/MockPowerHalController.h @@ -39,6 +39,7 @@ class MockPowerHalController : public power::PowerHalController { public: MockPowerHalController(); ~MockPowerHalController() override; + MOCK_METHOD(void, init, (), (override)); MOCK_METHOD(HalResult<void>, setBoost, (Boost, int32_t), (override)); MOCK_METHOD(HalResult<void>, setMode, (Mode, bool), (override)); MOCK_METHOD(HalResult<sp<hardware::power::IPowerHintSession>>, createHintSession, diff --git a/services/surfaceflinger/tests/unittests/mock/MockFrameRateMode.h b/services/surfaceflinger/tests/unittests/mock/MockFrameRateMode.h index ef9cd9bc43..4cfdd58c70 100644 --- a/services/surfaceflinger/tests/unittests/mock/MockFrameRateMode.h +++ b/services/surfaceflinger/tests/unittests/mock/MockFrameRateMode.h @@ -19,5 +19,7 @@ #include <scheduler/FrameRateMode.h> // Use a C style macro to keep the line numbers printed in gtest -#define EXPECT_FRAME_RATE_MODE(modePtr, fps, mode) \ - EXPECT_EQ((scheduler::FrameRateMode{(fps), (modePtr)}), (mode)) +#define EXPECT_FRAME_RATE_MODE(_modePtr, _fps, _mode) \ + EXPECT_EQ((scheduler::FrameRateMode{(_fps), (_modePtr)}), (_mode)) \ + << "Expected " << (_fps) << " (" << (_modePtr)->getFps() << ") but was " \ + << (_mode).fps << " (" << (_mode).modePtr->getFps() << ")" diff --git a/services/surfaceflinger/tests/unittests/mock/MockSchedulerCallback.h b/services/surfaceflinger/tests/unittests/mock/MockSchedulerCallback.h index a8eca2192f..306eb4d845 100644 --- a/services/surfaceflinger/tests/unittests/mock/MockSchedulerCallback.h +++ b/services/surfaceflinger/tests/unittests/mock/MockSchedulerCallback.h @@ -23,14 +23,14 @@ namespace android::scheduler::mock { struct SchedulerCallback final : ISchedulerCallback { - MOCK_METHOD(void, setVsyncEnabled, (PhysicalDisplayId, bool), (override)); + MOCK_METHOD(void, requestHardwareVsync, (PhysicalDisplayId, bool), (override)); MOCK_METHOD(void, requestDisplayModes, (std::vector<display::DisplayModeRequest>), (override)); MOCK_METHOD(void, kernelTimerChanged, (bool), (override)); MOCK_METHOD(void, triggerOnFrameRateOverridesChanged, (), (override)); }; struct NoOpSchedulerCallback final : ISchedulerCallback { - void setVsyncEnabled(PhysicalDisplayId, bool) override {} + void requestHardwareVsync(PhysicalDisplayId, bool) override {} void requestDisplayModes(std::vector<display::DisplayModeRequest>) override {} void kernelTimerChanged(bool) override {} void triggerOnFrameRateOverridesChanged() override {} diff --git a/services/surfaceflinger/tests/utils/WindowInfosListenerUtils.h b/services/surfaceflinger/tests/utils/WindowInfosListenerUtils.h index 8e28a75ed4..11723c7509 100644 --- a/services/surfaceflinger/tests/utils/WindowInfosListenerUtils.h +++ b/services/surfaceflinger/tests/utils/WindowInfosListenerUtils.h @@ -14,6 +14,7 @@ * limitations under the License. */ +#include <android-base/properties.h> #include <gtest/gtest.h> #include <gui/SurfaceComposerClient.h> #include <private/android_filesystem_config.h> @@ -21,7 +22,8 @@ #include <future> namespace android { -using Transaction = SurfaceComposerClient::Transaction; + +using base::HwTimeoutMultiplier; using gui::DisplayInfo; using gui::WindowInfo; @@ -36,7 +38,8 @@ public: auto listener = sp<WindowInfosListener>::make(std::move(predicate), promise); mClient->addWindowInfosListener(listener); auto future = promise.get_future(); - bool satisfied = future.wait_for(std::chrono::seconds{1}) == std::future_status::ready; + bool satisfied = future.wait_for(std::chrono::seconds{5 * HwTimeoutMultiplier()}) == + std::future_status::ready; mClient->removeWindowInfosListener(listener); return satisfied; } diff --git a/services/vibratorservice/test/VibratorCallbackSchedulerTest.cpp b/services/vibratorservice/test/VibratorCallbackSchedulerTest.cpp index aaeb8f990a..4c0910a75e 100644 --- a/services/vibratorservice/test/VibratorCallbackSchedulerTest.cpp +++ b/services/vibratorservice/test/VibratorCallbackSchedulerTest.cpp @@ -102,18 +102,6 @@ TEST_F(VibratorCallbackSchedulerTest, TestScheduleMultipleCallbacksRunsInDelayOr ASSERT_THAT(getExpiredCallbacks(), ElementsAre(3, 2, 1)); } -TEST_F(VibratorCallbackSchedulerTest, TestScheduleInParallelRunsInDelayOrder) { - std::vector<std::thread> threads; - for (int i = 0; i < 5; i++) { - threads.push_back(std::thread( - [=]() { mScheduler->schedule(createCallback(i), milliseconds(10 + 2 * i)); })); - } - std::for_each(threads.begin(), threads.end(), [](std::thread& t) { t.join(); }); - - ASSERT_TRUE(waitForCallbacks(5, 25ms)); - ASSERT_THAT(getExpiredCallbacks(), ElementsAre(0, 1, 2, 3, 4)); -} - TEST_F(VibratorCallbackSchedulerTest, TestDestructorDropsPendingCallbacksAndKillsThread) { mScheduler->schedule(createCallback(1), 5ms); mScheduler.reset(nullptr); diff --git a/vulkan/include/vulkan/vk_android_native_buffer.h b/vulkan/include/vulkan/vk_android_native_buffer.h index 40cf9fbd67..e78f47003b 100644 --- a/vulkan/include/vulkan/vk_android_native_buffer.h +++ b/vulkan/include/vulkan/vk_android_native_buffer.h @@ -55,7 +55,12 @@ extern "C" { * This version of the extension is largely designed to clean up the mix of * GrallocUsage and GrallocUsage2 */ -#define VK_ANDROID_NATIVE_BUFFER_SPEC_VERSION 9 +/* + * NOTE ON VK_ANDROID_NATIVE_BUFFER_SPEC_VERSION 10 + * + * This version of the extension cleans up a bug introduced in version 9 + */ +#define VK_ANDROID_NATIVE_BUFFER_SPEC_VERSION 10 #define VK_ANDROID_NATIVE_BUFFER_EXTENSION_NAME "VK_ANDROID_native_buffer" #define VK_ANDROID_NATIVE_BUFFER_ENUM(type, id) \ @@ -69,6 +74,8 @@ extern "C" { VK_ANDROID_NATIVE_BUFFER_ENUM(VkStructureType, 2) #define VK_STRUCTURE_TYPE_GRALLOC_USAGE_INFO_ANDROID \ VK_ANDROID_NATIVE_BUFFER_ENUM(VkStructureType, 3) +#define VK_STRUCTURE_TYPE_GRALLOC_USAGE_INFO_2_ANDROID \ + VK_ANDROID_NATIVE_BUFFER_ENUM(VkStructureType, 4) /* clang-format off */ typedef enum VkSwapchainImageUsageFlagBitsANDROID { @@ -152,6 +159,23 @@ typedef struct { VkImageUsageFlags imageUsage; } VkGrallocUsageInfoANDROID; +/* + * struct VkGrallocUsageInfo2ANDROID + * + * sType: VK_STRUCTURE_TYPE_GRALLOC_USAGE_INFO_2_ANDROID + * pNext: NULL or a pointer to a structure extending this structure + * format: value specifying the format the image will be created with + * imageUsage: bitmask of VkImageUsageFlagBits describing intended usage + * swapchainImageUsage: is a bitmask of VkSwapchainImageUsageFlagsANDROID + */ +typedef struct { + VkStructureType sType; + const void* pNext; + VkFormat format; + VkImageUsageFlags imageUsage; + VkSwapchainImageUsageFlagsANDROID swapchainImageUsage; +} VkGrallocUsageInfo2ANDROID; + /* DEPRECATED in SPEC_VERSION 6 */ typedef VkResult (VKAPI_PTR *PFN_vkGetSwapchainGrallocUsageANDROID)( VkDevice device, @@ -168,12 +192,18 @@ typedef VkResult (VKAPI_PTR *PFN_vkGetSwapchainGrallocUsage2ANDROID)( uint64_t* grallocConsumerUsage, uint64_t* grallocProducerUsage); -/* ADDED in SPEC_VERSION 9 */ +/* DEPRECATED in SPEC_VERSION 10 */ typedef VkResult (VKAPI_PTR *PFN_vkGetSwapchainGrallocUsage3ANDROID)( VkDevice device, const VkGrallocUsageInfoANDROID* grallocUsageInfo, uint64_t* grallocUsage); +/* ADDED in SPEC_VERSION 10 */ +typedef VkResult (VKAPI_PTR *PFN_vkGetSwapchainGrallocUsage4ANDROID)( + VkDevice device, + const VkGrallocUsageInfo2ANDROID* grallocUsageInfo, + uint64_t* grallocUsage); + typedef VkResult (VKAPI_PTR *PFN_vkAcquireImageANDROID)( VkDevice device, VkImage image, @@ -208,13 +238,20 @@ VKAPI_ATTR VkResult VKAPI_CALL vkGetSwapchainGrallocUsage2ANDROID( uint64_t* grallocProducerUsage ); -/* ADDED in SPEC_VERSION 9 */ +/* DEPRECATED in SPEC_VERSION 10 */ VKAPI_ATTR VkResult VKAPI_CALL vkGetSwapchainGrallocUsage3ANDROID( VkDevice device, const VkGrallocUsageInfoANDROID* grallocUsageInfo, uint64_t* grallocUsage ); +/* ADDED in SPEC_VERSION 10 */ +VKAPI_ATTR VkResult VKAPI_CALL vkGetSwapchainGrallocUsage4ANDROID( + VkDevice device, + const VkGrallocUsageInfo2ANDROID* grallocUsageInfo, + uint64_t* grallocUsage +); + VKAPI_ATTR VkResult VKAPI_CALL vkAcquireImageANDROID( VkDevice device, VkImage image, diff --git a/vulkan/libvulkan/driver.cpp b/vulkan/libvulkan/driver.cpp index a16ab48a18..1cf30ae67b 100644 --- a/vulkan/libvulkan/driver.cpp +++ b/vulkan/libvulkan/driver.cpp @@ -1460,13 +1460,15 @@ VkResult CreateDevice(VkPhysicalDevice physicalDevice, if ((wrapper.GetHalExtensions()[ProcHook::ANDROID_native_buffer]) && !data->driver.GetSwapchainGrallocUsageANDROID && !data->driver.GetSwapchainGrallocUsage2ANDROID && - !data->driver.GetSwapchainGrallocUsage3ANDROID) { + !data->driver.GetSwapchainGrallocUsage3ANDROID && + !data->driver.GetSwapchainGrallocUsage4ANDROID) { ALOGE( "Driver's implementation of ANDROID_native_buffer is broken;" " must expose at least one of " "vkGetSwapchainGrallocUsageANDROID or " "vkGetSwapchainGrallocUsage2ANDROID or " - "vkGetSwapchainGrallocUsage3ANDROID"); + "vkGetSwapchainGrallocUsage3ANDROID or " + "vkGetSwapchainGrallocUsage4ANDROID"); data->driver.DestroyDevice(dev, pAllocator); FreeDeviceData(data, data_allocator); diff --git a/vulkan/libvulkan/driver_gen.cpp b/vulkan/libvulkan/driver_gen.cpp index 798af5c6a6..8f090083f8 100644 --- a/vulkan/libvulkan/driver_gen.cpp +++ b/vulkan/libvulkan/driver_gen.cpp @@ -512,6 +512,13 @@ const ProcHook g_proc_hooks[] = { nullptr, }, { + "vkGetSwapchainGrallocUsage4ANDROID", + ProcHook::DEVICE, + ProcHook::ANDROID_native_buffer, + nullptr, + nullptr, + }, + { "vkGetSwapchainGrallocUsageANDROID", ProcHook::DEVICE, ProcHook::ANDROID_native_buffer, @@ -692,6 +699,7 @@ bool InitDriverTable(VkDevice dev, INIT_PROC_EXT(ANDROID_native_buffer, false, dev, GetSwapchainGrallocUsageANDROID); INIT_PROC_EXT(ANDROID_native_buffer, false, dev, GetSwapchainGrallocUsage2ANDROID); INIT_PROC_EXT(ANDROID_native_buffer, false, dev, GetSwapchainGrallocUsage3ANDROID); + INIT_PROC_EXT(ANDROID_native_buffer, false, dev, GetSwapchainGrallocUsage4ANDROID); INIT_PROC_EXT(ANDROID_native_buffer, true, dev, AcquireImageANDROID); INIT_PROC_EXT(ANDROID_native_buffer, true, dev, QueueSignalReleaseImageANDROID); // clang-format on diff --git a/vulkan/libvulkan/driver_gen.h b/vulkan/libvulkan/driver_gen.h index 31ba04ba1f..4527214c3f 100644 --- a/vulkan/libvulkan/driver_gen.h +++ b/vulkan/libvulkan/driver_gen.h @@ -128,6 +128,7 @@ struct DeviceDriverTable { PFN_vkGetSwapchainGrallocUsageANDROID GetSwapchainGrallocUsageANDROID; PFN_vkGetSwapchainGrallocUsage2ANDROID GetSwapchainGrallocUsage2ANDROID; PFN_vkGetSwapchainGrallocUsage3ANDROID GetSwapchainGrallocUsage3ANDROID; + PFN_vkGetSwapchainGrallocUsage4ANDROID GetSwapchainGrallocUsage4ANDROID; PFN_vkAcquireImageANDROID AcquireImageANDROID; PFN_vkQueueSignalReleaseImageANDROID QueueSignalReleaseImageANDROID; // clang-format on diff --git a/vulkan/libvulkan/swapchain.cpp b/vulkan/libvulkan/swapchain.cpp index a257f06755..bffbe9d8d5 100644 --- a/vulkan/libvulkan/swapchain.cpp +++ b/vulkan/libvulkan/swapchain.cpp @@ -1118,12 +1118,17 @@ VkResult GetPhysicalDeviceSurfaceFormats2KHR( VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2; imageFormatInfo.format = pSurfaceFormats[i].surfaceFormat.format; + imageFormatInfo.type = VK_IMAGE_TYPE_2D; + imageFormatInfo.usage = + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; imageFormatInfo.pNext = nullptr; VkImageCompressionControlEXT compressionControl = {}; compressionControl.sType = VK_STRUCTURE_TYPE_IMAGE_COMPRESSION_CONTROL_EXT; compressionControl.pNext = imageFormatInfo.pNext; + compressionControl.flags = + VK_IMAGE_COMPRESSION_FIXED_RATE_DEFAULT_EXT; imageFormatInfo.pNext = &compressionControl; @@ -1571,7 +1576,47 @@ VkResult CreateSwapchainKHR(VkDevice device, void* usage_info_pNext = nullptr; VkImageCompressionControlEXT image_compression = {}; uint64_t native_usage = 0; - if (dispatch.GetSwapchainGrallocUsage3ANDROID) { + if (dispatch.GetSwapchainGrallocUsage4ANDROID) { + ATRACE_BEGIN("GetSwapchainGrallocUsage4ANDROID"); + VkGrallocUsageInfo2ANDROID gralloc_usage_info = {}; + gralloc_usage_info.sType = + VK_STRUCTURE_TYPE_GRALLOC_USAGE_INFO_2_ANDROID; + gralloc_usage_info.format = create_info->imageFormat; + gralloc_usage_info.imageUsage = create_info->imageUsage; + gralloc_usage_info.swapchainImageUsage = swapchain_image_usage; + + // Look through the pNext chain for an image compression control struct + // if one is found AND the appropriate extensions are enabled, + // append it to be the gralloc usage pNext chain + const VkSwapchainCreateInfoKHR* create_infos = create_info; + while (create_infos->pNext) { + create_infos = reinterpret_cast<const VkSwapchainCreateInfoKHR*>( + create_infos->pNext); + switch (create_infos->sType) { + case VK_STRUCTURE_TYPE_IMAGE_COMPRESSION_CONTROL_EXT: { + const VkImageCompressionControlEXT* compression_infos = + reinterpret_cast<const VkImageCompressionControlEXT*>( + create_infos); + image_compression = *compression_infos; + image_compression.pNext = nullptr; + usage_info_pNext = &image_compression; + } break; + + default: + // Ignore all other info structs + break; + } + } + gralloc_usage_info.pNext = usage_info_pNext; + + result = dispatch.GetSwapchainGrallocUsage4ANDROID( + device, &gralloc_usage_info, &native_usage); + ATRACE_END(); + if (result != VK_SUCCESS) { + ALOGE("vkGetSwapchainGrallocUsage4ANDROID failed: %d", result); + return VK_ERROR_SURFACE_LOST_KHR; + } + } else if (dispatch.GetSwapchainGrallocUsage3ANDROID) { ATRACE_BEGIN("GetSwapchainGrallocUsage3ANDROID"); VkGrallocUsageInfoANDROID gralloc_usage_info = {}; gralloc_usage_info.sType = VK_STRUCTURE_TYPE_GRALLOC_USAGE_INFO_ANDROID; diff --git a/vulkan/nulldrv/null_driver.cpp b/vulkan/nulldrv/null_driver.cpp index f998b1ad18..2e87f1718b 100644 --- a/vulkan/nulldrv/null_driver.cpp +++ b/vulkan/nulldrv/null_driver.cpp @@ -959,6 +959,17 @@ VkResult GetSwapchainGrallocUsage3ANDROID( return VK_SUCCESS; } +VkResult GetSwapchainGrallocUsage4ANDROID( + VkDevice, + const VkGrallocUsageInfo2ANDROID* grallocUsageInfo, + uint64_t* grallocUsage) { + // The null driver never reads or writes the gralloc buffer + ALOGV("TODO: vk%s - grallocUsageInfo->format:%i", __FUNCTION__, + grallocUsageInfo->format); + *grallocUsage = 0; + return VK_SUCCESS; +} + VkResult AcquireImageANDROID(VkDevice, VkImage, int fence, diff --git a/vulkan/nulldrv/null_driver_gen.cpp b/vulkan/nulldrv/null_driver_gen.cpp index 0cb7bd3185..935535f5bf 100644 --- a/vulkan/nulldrv/null_driver_gen.cpp +++ b/vulkan/nulldrv/null_driver_gen.cpp @@ -262,6 +262,7 @@ const NameProc kInstanceProcs[] = { {"vkGetSemaphoreCounterValue", reinterpret_cast<PFN_vkVoidFunction>(static_cast<PFN_vkGetSemaphoreCounterValue>(GetSemaphoreCounterValue))}, {"vkGetSwapchainGrallocUsage2ANDROID", reinterpret_cast<PFN_vkVoidFunction>(static_cast<PFN_vkGetSwapchainGrallocUsage2ANDROID>(GetSwapchainGrallocUsage2ANDROID))}, {"vkGetSwapchainGrallocUsage3ANDROID", reinterpret_cast<PFN_vkVoidFunction>(static_cast<PFN_vkGetSwapchainGrallocUsage3ANDROID>(GetSwapchainGrallocUsage3ANDROID))}, + {"vkGetSwapchainGrallocUsage4ANDROID", reinterpret_cast<PFN_vkVoidFunction>(static_cast<PFN_vkGetSwapchainGrallocUsage4ANDROID>(GetSwapchainGrallocUsage4ANDROID))}, {"vkGetSwapchainGrallocUsageANDROID", reinterpret_cast<PFN_vkVoidFunction>(static_cast<PFN_vkGetSwapchainGrallocUsageANDROID>(GetSwapchainGrallocUsageANDROID))}, {"vkInvalidateMappedMemoryRanges", reinterpret_cast<PFN_vkVoidFunction>(static_cast<PFN_vkInvalidateMappedMemoryRanges>(InvalidateMappedMemoryRanges))}, {"vkMapMemory", reinterpret_cast<PFN_vkVoidFunction>(static_cast<PFN_vkMapMemory>(MapMemory))}, diff --git a/vulkan/nulldrv/null_driver_gen.h b/vulkan/nulldrv/null_driver_gen.h index 5c7fea0fa8..fb3bd05e07 100644 --- a/vulkan/nulldrv/null_driver_gen.h +++ b/vulkan/nulldrv/null_driver_gen.h @@ -210,6 +210,7 @@ VKAPI_ATTR void GetDescriptorSetLayoutSupport(VkDevice device, const VkDescripto VKAPI_ATTR VkResult GetSwapchainGrallocUsageANDROID(VkDevice device, VkFormat format, VkImageUsageFlags imageUsage, int* grallocUsage); VKAPI_ATTR VkResult GetSwapchainGrallocUsage2ANDROID(VkDevice device, VkFormat format, VkImageUsageFlags imageUsage, VkSwapchainImageUsageFlagsANDROID swapchainImageUsage, uint64_t* grallocConsumerUsage, uint64_t* grallocProducerUsage); VKAPI_ATTR VkResult GetSwapchainGrallocUsage3ANDROID(VkDevice device, const VkGrallocUsageInfoANDROID* grallocUsageInfo, uint64_t* grallocUsage); +VKAPI_ATTR VkResult GetSwapchainGrallocUsage4ANDROID(VkDevice device, const VkGrallocUsageInfo2ANDROID* grallocUsageInfo, uint64_t* grallocUsage); VKAPI_ATTR VkResult AcquireImageANDROID(VkDevice device, VkImage image, int nativeFenceFd, VkSemaphore semaphore, VkFence fence); VKAPI_ATTR VkResult QueueSignalReleaseImageANDROID(VkQueue queue, uint32_t waitSemaphoreCount, const VkSemaphore* pWaitSemaphores, VkImage image, int* pNativeFenceFd); VKAPI_ATTR VkResult CreateRenderPass2(VkDevice device, const VkRenderPassCreateInfo2* pCreateInfo, const VkAllocationCallbacks* pAllocator, VkRenderPass* pRenderPass); diff --git a/vulkan/scripts/generator_common.py b/vulkan/scripts/generator_common.py index c25c6cbda0..866c1b7b75 100644 --- a/vulkan/scripts/generator_common.py +++ b/vulkan/scripts/generator_common.py @@ -70,6 +70,7 @@ _OPTIONAL_COMMANDS = [ 'vkGetSwapchainGrallocUsageANDROID', 'vkGetSwapchainGrallocUsage2ANDROID', 'vkGetSwapchainGrallocUsage3ANDROID', + 'vkGetSwapchainGrallocUsage4ANDROID', ] # Dict for mapping dispatch table to a type. |