diff options
| author | 2024-10-25 20:59:24 +0000 | |
|---|---|---|
| committer | 2024-12-02 08:26:47 +0000 | |
| commit | e33845cf67a462b10f1894642e052b97f27c722c (patch) | |
| tree | 0cfe89870514e231fe063c30448eeb8bc2dbad6d | |
| parent | 8a3f409623e965f466a7153974766366201c6431 (diff) | |
Enable cursor to transition across multiple displays
This CL enables cursor to move between displays. It uses a fake
topology that assumes all available displays are connected in the
following order:
default-display (top-edge) -> next-display (right-edge)
-> next-display (right-edge) ...
Test: atest inputflinger_tests
Test: verify cursor can move between displays as expected
Bug: 367659738
Bug: 367660694
Flag: com.android.input.flags.connected_displays_cursor
Change-Id: I8b3b7c3c0e68dca1e7ce281cb7ff6efab263a500
6 files changed, 340 insertions, 24 deletions
diff --git a/services/inputflinger/PointerChoreographer.cpp b/services/inputflinger/PointerChoreographer.cpp index 38e5974c3f..2434d2badb 100644 --- a/services/inputflinger/PointerChoreographer.cpp +++ b/services/inputflinger/PointerChoreographer.cpp @@ -205,20 +205,30 @@ void PointerChoreographer::fadeMouseCursorOnKeyPress(const android::NotifyKeyArg } NotifyMotionArgs PointerChoreographer::processMotion(const NotifyMotionArgs& args) { - std::scoped_lock _l(getLock()); + NotifyMotionArgs newArgs(args); + PointerDisplayChange pointerDisplayChange; + { // acquire lock + std::scoped_lock _l(getLock()); + if (isFromMouse(args)) { + newArgs = processMouseEventLocked(args); + pointerDisplayChange = calculatePointerDisplayChangeToNotify(); + } else if (isFromTouchpad(args)) { + newArgs = processTouchpadEventLocked(args); + pointerDisplayChange = calculatePointerDisplayChangeToNotify(); + } else if (isFromDrawingTablet(args)) { + processDrawingTabletEventLocked(args); + } else if (mStylusPointerIconEnabled && isStylusHoverEvent(args)) { + processStylusHoverEventLocked(args); + } else if (isFromSource(args.source, AINPUT_SOURCE_TOUCHSCREEN)) { + processTouchscreenAndStylusEventLocked(args); + } + } // release lock - if (isFromMouse(args)) { - return processMouseEventLocked(args); - } else if (isFromTouchpad(args)) { - return processTouchpadEventLocked(args); - } else if (isFromDrawingTablet(args)) { - processDrawingTabletEventLocked(args); - } else if (mStylusPointerIconEnabled && isStylusHoverEvent(args)) { - processStylusHoverEventLocked(args); - } else if (isFromSource(args.source, AINPUT_SOURCE_TOUCHSCREEN)) { - processTouchscreenAndStylusEventLocked(args); + if (pointerDisplayChange) { + // pointer display may have changed if mouse crossed display boundary + notifyPointerDisplayChange(pointerDisplayChange, mPolicy); } - return args; + return newArgs; } NotifyMotionArgs PointerChoreographer::processMouseEventLocked(const NotifyMotionArgs& args) { @@ -245,7 +255,8 @@ NotifyMotionArgs PointerChoreographer::processMouseEventLocked(const NotifyMotio // This is a relative mouse, so move the cursor by the specified amount. processPointerDeviceMotionEventLocked(/*byref*/ newArgs, /*byref*/ pc); } - if (canUnfadeOnDisplay(displayId)) { + // Note displayId may have changed if the cursor moved to a different display + if (canUnfadeOnDisplay(newArgs.displayId)) { pc.unfade(PointerControllerInterface::Transition::IMMEDIATE); } return newArgs; @@ -272,7 +283,9 @@ NotifyMotionArgs PointerChoreographer::processTouchpadEventLocked(const NotifyMo newArgs.xCursorPosition = x; newArgs.yCursorPosition = y; } - if (canUnfadeOnDisplay(displayId)) { + + // Note displayId may have changed if the cursor moved to a different display + if (canUnfadeOnDisplay(newArgs.displayId)) { pc.unfade(PointerControllerInterface::Transition::IMMEDIATE); } return newArgs; @@ -283,7 +296,14 @@ void PointerChoreographer::processPointerDeviceMotionEventLocked(NotifyMotionArg const float deltaX = newArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X); const float deltaY = newArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y); - pc.move(deltaX, deltaY); + FloatPoint unconsumedDelta = pc.move(deltaX, deltaY); + if (com::android::input::flags::connected_displays_cursor() && + (std::abs(unconsumedDelta.x) > 0 || std::abs(unconsumedDelta.y) > 0)) { + handleUnconsumedDeltaLocked(pc, unconsumedDelta); + // pointer may have moved to a different viewport + newArgs.displayId = pc.getDisplayId(); + } + const auto [x, y] = pc.getPosition(); newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_X, x); newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_Y, y); @@ -291,6 +311,34 @@ void PointerChoreographer::processPointerDeviceMotionEventLocked(NotifyMotionArg newArgs.yCursorPosition = y; } +void PointerChoreographer::handleUnconsumedDeltaLocked(PointerControllerInterface& pc, + const FloatPoint& unconsumedDelta) { + const ui::LogicalDisplayId sourceDisplayId = pc.getDisplayId(); + const auto& sourceViewport = *findViewportByIdLocked(sourceDisplayId); + std::optional<const DisplayViewport*> destination = + findDestinationDisplayLocked(sourceViewport, unconsumedDelta); + if (!destination) { + // no adjacent display + return; + } + + const DisplayViewport* destinationViewport = *destination; + + if (mMousePointersByDisplay.find(destinationViewport->displayId) != + mMousePointersByDisplay.end()) { + LOG(FATAL) << "A cursor already exists on destination display" + << destinationViewport->displayId; + } + mDefaultMouseDisplayId = destinationViewport->displayId; + auto pcNode = mMousePointersByDisplay.extract(sourceDisplayId); + pcNode.key() = destinationViewport->displayId; + mMousePointersByDisplay.insert(std::move(pcNode)); + + // This will place cursor at the center of the target display for now + // TODO (b/367660694) place the cursor at the appropriate position in destination display + pc.setDisplayViewport(*destinationViewport); +} + void PointerChoreographer::processDrawingTabletEventLocked(const android::NotifyMotionArgs& args) { if (args.displayId == ui::LogicalDisplayId::INVALID) { return; @@ -436,7 +484,8 @@ void PointerChoreographer::processDeviceReset(const NotifyDeviceResetArgs& args) } void PointerChoreographer::onControllerAddedOrRemovedLocked() { - if (!com::android::input::flags::hide_pointer_indicators_for_secure_windows()) { + if (!com::android::input::flags::hide_pointer_indicators_for_secure_windows() && + !com::android::input::flags::connected_displays_cursor()) { return; } bool requireListener = !mTouchPointersByDevice.empty() || !mMousePointersByDisplay.empty() || @@ -502,6 +551,13 @@ void PointerChoreographer::notifyPointerCaptureChanged( mNextListener.notify(args); } +void PointerChoreographer::setDisplayTopology( + const std::unordered_map<ui::LogicalDisplayId, std::vector<AdjacentDisplay>>& + displayTopology) { + std::scoped_lock _l(getLock()); + mTopology = displayTopology; +} + void PointerChoreographer::dump(std::string& dump) { std::scoped_lock _l(getLock()); @@ -873,6 +929,104 @@ PointerChoreographer::ControllerConstructor PointerChoreographer::getStylusContr return ConstructorDelegate(std::move(ctor)); } +void PointerChoreographer::populateFakeDisplayTopologyLocked( + const std::vector<gui::DisplayInfo>& displayInfos) { + if (!com::android::input::flags::connected_displays_cursor()) { + return; + } + + if (displayInfos.size() == mTopology.size()) { + bool displaysChanged = false; + for (const auto& displayInfo : displayInfos) { + if (mTopology.find(displayInfo.displayId) == mTopology.end()) { + displaysChanged = true; + break; + } + } + + if (!displaysChanged) { + return; + } + } + + // create a fake topology assuming following order + // default-display (top-edge) -> next-display (right-edge) -> next-display (right-edge) ... + // ┌─────────┬─────────┐ + // │ next │ next 2 │ ... + // ├─────────┼─────────┘ + // │ default │ + // └─────────┘ + mTopology.clear(); + + // treat default display as base, in real topology it should be the primary-display + ui::LogicalDisplayId previousDisplay = ui::LogicalDisplayId::DEFAULT; + for (const auto& displayInfo : displayInfos) { + if (displayInfo.displayId == ui::LogicalDisplayId::DEFAULT) { + continue; + } + if (previousDisplay == ui::LogicalDisplayId::DEFAULT) { + mTopology[previousDisplay].push_back({displayInfo.displayId, DisplayPosition::TOP, 0}); + mTopology[displayInfo.displayId].push_back( + {previousDisplay, DisplayPosition::BOTTOM, 0}); + } else { + mTopology[previousDisplay].push_back( + {displayInfo.displayId, DisplayPosition::RIGHT, 0}); + mTopology[displayInfo.displayId].push_back({previousDisplay, DisplayPosition::LEFT, 0}); + } + previousDisplay = displayInfo.displayId; + } + + // update default pointer display. In real topology it should be the primary-display + if (mTopology.find(mDefaultMouseDisplayId) == mTopology.end()) { + mDefaultMouseDisplayId = ui::LogicalDisplayId::DEFAULT; + } +} + +std::optional<const DisplayViewport*> PointerChoreographer::findDestinationDisplayLocked( + const DisplayViewport& sourceViewport, const FloatPoint& unconsumedDelta) const { + DisplayPosition sourceBoundary; + if (unconsumedDelta.x > 0) { + sourceBoundary = DisplayPosition::RIGHT; + } else if (unconsumedDelta.x < 0) { + sourceBoundary = DisplayPosition::LEFT; + } else if (unconsumedDelta.y > 0) { + sourceBoundary = DisplayPosition::BOTTOM; + } else { + sourceBoundary = DisplayPosition::TOP; + } + + // Choreographer works in un-rotate coordinate space so we need to rotate boundary by viewport + // orientation to find the rotated boundary + constexpr int MOD = ftl::to_underlying(ui::Rotation::ftl_last) + 1; + sourceBoundary = static_cast<DisplayPosition>( + (ftl::to_underlying(sourceBoundary) + ftl::to_underlying(sourceViewport.orientation)) % + MOD); + + const auto& destination = mTopology.find(sourceViewport.displayId); + if (destination == mTopology.end()) { + // Topology is likely out of sync with viewport info, wait for it to be updated + LOG(WARNING) << "Source display missing from topology " << sourceViewport.displayId; + return std::nullopt; + } + + for (const auto& adjacentDisplay : destination->second) { + if (adjacentDisplay.position != sourceBoundary) { + continue; + } + const DisplayViewport* destinationViewport = + findViewportByIdLocked(adjacentDisplay.displayId); + if (destinationViewport == nullptr) { + // Topology is likely out of sync with viewport info, wait for them to be updated + LOG(WARNING) << "Cannot find viewport for adjacent display " + << adjacentDisplay.displayId << "of source display " + << sourceViewport.displayId; + break; + } + return destinationViewport; + } + return std::nullopt; +} + // --- PointerChoreographer::PointerChoreographerDisplayInfoListener --- void PointerChoreographer::PointerChoreographerDisplayInfoListener::onWindowInfosChanged( @@ -883,12 +1037,14 @@ void PointerChoreographer::PointerChoreographerDisplayInfoListener::onWindowInfo } auto newPrivacySensitiveDisplays = getPrivacySensitiveDisplaysFromWindowInfos(windowInfosUpdate.windowInfos); + + // PointerChoreographer uses Listener's lock. + base::ScopedLockAssertion assumeLocked(mPointerChoreographer->getLock()); if (newPrivacySensitiveDisplays != mPrivacySensitiveDisplays) { mPrivacySensitiveDisplays = std::move(newPrivacySensitiveDisplays); - // PointerChoreographer uses Listener's lock. - base::ScopedLockAssertion assumeLocked(mPointerChoreographer->getLock()); mPointerChoreographer->onPrivacySensitiveDisplaysChangedLocked(mPrivacySensitiveDisplays); } + mPointerChoreographer->populateFakeDisplayTopologyLocked(windowInfosUpdate.displayInfos); } void PointerChoreographer::PointerChoreographerDisplayInfoListener::setInitialDisplayInfosLocked( diff --git a/services/inputflinger/PointerChoreographer.h b/services/inputflinger/PointerChoreographer.h index fba1aef260..a9699351f4 100644 --- a/services/inputflinger/PointerChoreographer.h +++ b/services/inputflinger/PointerChoreographer.h @@ -113,6 +113,24 @@ public: void notifyDeviceReset(const NotifyDeviceResetArgs& args) override; void notifyPointerCaptureChanged(const NotifyPointerCaptureChangedArgs& args) override; + // TODO(b/362719483) remove these when real topology is available + enum class DisplayPosition : int32_t { + RIGHT = 0, + TOP = 1, + LEFT = 2, + BOTTOM = 3, + ftl_last = BOTTOM, + }; + + struct AdjacentDisplay { + ui::LogicalDisplayId displayId; + DisplayPosition position; + float offsetPx; + }; + void setDisplayTopology( + const std::unordered_map<ui::LogicalDisplayId, std::vector<AdjacentDisplay>>& + displayTopology); + void dump(std::string& dump) override; private: @@ -153,6 +171,19 @@ private: const std::unordered_set<ui::LogicalDisplayId>& privacySensitiveDisplays) REQUIRES(getLock()); + void handleUnconsumedDeltaLocked(PointerControllerInterface& pc, + const FloatPoint& unconsumedDelta) REQUIRES(getLock()); + + void populateFakeDisplayTopologyLocked(const std::vector<gui::DisplayInfo>& displayInfos) + REQUIRES(getLock()); + + std::optional<const DisplayViewport*> findDestinationDisplayLocked( + const DisplayViewport& sourceViewport, const FloatPoint& unconsumedDelta) const + REQUIRES(getLock()); + + std::unordered_map<ui::LogicalDisplayId, std::vector<AdjacentDisplay>> mTopology + GUARDED_BY(getLock()); + /* This listener keeps tracks of visible privacy sensitive displays and updates the * choreographer if there are any changes. * diff --git a/services/inputflinger/include/PointerControllerInterface.h b/services/inputflinger/include/PointerControllerInterface.h index 8f3d9ca778..3391f794b1 100644 --- a/services/inputflinger/include/PointerControllerInterface.h +++ b/services/inputflinger/include/PointerControllerInterface.h @@ -72,8 +72,12 @@ public: /* Dumps the state of the pointer controller. */ virtual std::string dump() = 0; - /* Move the pointer. */ - virtual void move(float deltaX, float deltaY) = 0; + /* Move the pointer and return unconsumed delta if the pointer has crossed the current + * viewport bounds . + * + * Return value may be used to move pointer to corresponding adjacent display, if it exists in + * the display-topology */ + [[nodiscard]] virtual FloatPoint move(float deltaX, float deltaY) = 0; /* Sets the absolute location of the pointer. */ virtual void setPosition(float x, float y) = 0; diff --git a/services/inputflinger/tests/FakePointerController.cpp b/services/inputflinger/tests/FakePointerController.cpp index 887a939e09..c1606a7ca5 100644 --- a/services/inputflinger/tests/FakePointerController.cpp +++ b/services/inputflinger/tests/FakePointerController.cpp @@ -148,15 +148,20 @@ bool FakePointerController::isPointerShown() { return mIsPointerShown; } -void FakePointerController::move(float deltaX, float deltaY) { - if (!mEnabled) return; +FloatPoint FakePointerController::move(float deltaX, float deltaY) { + if (!mEnabled) return {0, 0}; mX += deltaX; + mY += deltaY; + + const FloatPoint position(mX, mY); + if (mX < mMinX) mX = mMinX; if (mX > mMaxX) mX = mMaxX; - mY += deltaY; if (mY < mMinY) mY = mMinY; if (mY > mMaxY) mY = mMaxY; + + return {position.x - mX, position.y - mY}; } void FakePointerController::fade(Transition) { diff --git a/services/inputflinger/tests/FakePointerController.h b/services/inputflinger/tests/FakePointerController.h index 9b773a7715..04adff8234 100644 --- a/services/inputflinger/tests/FakePointerController.h +++ b/services/inputflinger/tests/FakePointerController.h @@ -65,7 +65,7 @@ public: private: std::string dump() override { return ""; } - void move(float deltaX, float deltaY) override; + FloatPoint move(float deltaX, float deltaY) override; void unfade(Transition) override; void setPresentation(Presentation) override {} void setSpots(const PointerCoords*, const uint32_t*, BitSet32 spotIdBits, diff --git a/services/inputflinger/tests/PointerChoreographer_test.cpp b/services/inputflinger/tests/PointerChoreographer_test.cpp index 411c7baf77..f427658d75 100644 --- a/services/inputflinger/tests/PointerChoreographer_test.cpp +++ b/services/inputflinger/tests/PointerChoreographer_test.cpp @@ -2601,6 +2601,126 @@ TEST_P(PointerVisibilityAndTouchpadTapStateOnKeyPressTestFixture, TestMetaKeyCom metaKeyCombinationDoesNotHidePointer(*pc, AKEYCODE_A, AKEYCODE_META_RIGHT); } +using PointerChoreographerDisplayTopologyTestFixtureParam = + std::tuple<std::string_view /*name*/, int32_t /*source device*/, + ControllerType /*PointerController*/, ToolType /*pointer tool type*/, + FloatPoint /*source position */, FloatPoint /*hover move X/Y */, + ui::LogicalDisplayId /*destination display*/>; + +class PointerChoreographerDisplayTopologyTestFixture + : public PointerChoreographerTest, + public testing::WithParamInterface<PointerChoreographerDisplayTopologyTestFixtureParam> { +public: + static constexpr ui::LogicalDisplayId DISPLAY_CENTER_ID = ui::LogicalDisplayId{10}; + static constexpr ui::LogicalDisplayId DISPLAY_TOP_ID = ui::LogicalDisplayId{20}; + static constexpr ui::LogicalDisplayId DISPLAY_RIGHT_ID = ui::LogicalDisplayId{30}; + static constexpr ui::LogicalDisplayId DISPLAY_BOTTOM_ID = ui::LogicalDisplayId{40}; + static constexpr ui::LogicalDisplayId DISPLAY_LEFT_ID = ui::LogicalDisplayId{50}; + + PointerChoreographerDisplayTopologyTestFixture() { + com::android::input::flags::connected_displays_cursor(true); + } + +protected: + std::vector<DisplayViewport> mViewports{ + createViewport(DISPLAY_CENTER_ID, /*width*/ 100, /*height*/ 100), + createViewport(DISPLAY_TOP_ID, /*width*/ 90, /*height*/ 90), + createViewport(DISPLAY_RIGHT_ID, /*width*/ 90, /*height*/ 90), + createViewport(DISPLAY_BOTTOM_ID, /*width*/ 90, /*height*/ 90), + createViewport(DISPLAY_LEFT_ID, /*width*/ 90, /*height*/ 90), + }; + + std::unordered_map<ui::LogicalDisplayId, std::vector<PointerChoreographer::AdjacentDisplay>> + mTopology{ + {DISPLAY_CENTER_ID, + {{DISPLAY_TOP_ID, PointerChoreographer::DisplayPosition::TOP, 0.0f}, + {DISPLAY_RIGHT_ID, PointerChoreographer::DisplayPosition::RIGHT, 0.0f}, + {DISPLAY_BOTTOM_ID, PointerChoreographer::DisplayPosition::BOTTOM, 0.0f}, + {DISPLAY_LEFT_ID, PointerChoreographer::DisplayPosition::LEFT, 0.0f}}}, + }; + +private: + DisplayViewport createViewport(ui::LogicalDisplayId displayId, int32_t width, int32_t height) { + DisplayViewport viewport; + viewport.displayId = displayId; + viewport.logicalRight = width; + viewport.logicalBottom = height; + return viewport; + } +}; + +TEST_P(PointerChoreographerDisplayTopologyTestFixture, PointerChoreographerDisplayTopologyTest) { + const auto& [_, device, pointerControllerType, pointerToolType, initialPosition, hoverMove, + destinationDisplay] = GetParam(); + + mChoreographer.setDisplayViewports(mViewports); + mChoreographer.setDefaultMouseDisplayId( + PointerChoreographerDisplayTopologyTestFixture::DISPLAY_CENTER_ID); + mChoreographer.setDisplayTopology(mTopology); + + mChoreographer.notifyInputDevicesChanged( + {/*id=*/0, {generateTestDeviceInfo(DEVICE_ID, device, ui::LogicalDisplayId::INVALID)}}); + + auto pc = assertPointerControllerCreated(pointerControllerType); + ASSERT_EQ(PointerChoreographerDisplayTopologyTestFixture::DISPLAY_CENTER_ID, + pc->getDisplayId()); + + // Set initial position of the PointerController. + pc->setPosition(initialPosition.x, initialPosition.y); + ASSERT_TRUE(pc->isPointerShown()); + + // Make NotifyMotionArgs and notify Choreographer. + auto pointerBuilder = PointerBuilder(/*id=*/0, pointerToolType) + .axis(AMOTION_EVENT_AXIS_RELATIVE_X, hoverMove.x) + .axis(AMOTION_EVENT_AXIS_RELATIVE_Y, hoverMove.y); + + mChoreographer.notifyMotion(MotionArgsBuilder(AMOTION_EVENT_ACTION_HOVER_MOVE, device) + .pointer(pointerBuilder) + .deviceId(DEVICE_ID) + .displayId(ui::LogicalDisplayId::INVALID) + .build()); + + // Check that the PointerController updated the position and the pointer is shown. + // TODO(b/362719483) assert pointer controller position here + ASSERT_TRUE(pc->isPointerShown()); + ASSERT_EQ(pc->getDisplayId(), destinationDisplay); + + // Check that x-y coordinates, displayId and cursor position are correctly updated. + // TODO(b/362719483) assert Coords and cursor position here + mTestListener.assertNotifyMotionWasCalled(WithDisplayId(destinationDisplay)); +} + +INSTANTIATE_TEST_SUITE_P( + PointerChoreographerTest, PointerChoreographerDisplayTopologyTestFixture, + testing::Values( + std::make_tuple("UnchangedDisplay", AINPUT_SOURCE_MOUSE, ControllerType::MOUSE, + ToolType::MOUSE, FloatPoint(50, 50) /* initial x/y */, + FloatPoint(25, 25) /* delta x/y */, + PointerChoreographerDisplayTopologyTestFixture::DISPLAY_CENTER_ID), + std::make_tuple("TransitionToRightDisplay", AINPUT_SOURCE_MOUSE, + ControllerType::MOUSE, ToolType::MOUSE, + FloatPoint(50, 50) /* initial x/y */, + FloatPoint(100, 25) /* delta x/y */, + PointerChoreographerDisplayTopologyTestFixture::DISPLAY_RIGHT_ID), + std::make_tuple("TransitionToLeftDisplay", AINPUT_SOURCE_MOUSE, + ControllerType::MOUSE, ToolType::MOUSE, + FloatPoint(50, 50) /* initial x/y */, + FloatPoint(-100, 25) /* delta x/y */, + PointerChoreographerDisplayTopologyTestFixture::DISPLAY_LEFT_ID), + std::make_tuple("TransitionToTopDisplay", + AINPUT_SOURCE_MOUSE | AINPUT_SOURCE_TOUCHPAD, ControllerType::MOUSE, + ToolType::FINGER, FloatPoint(50, 50) /* initial x/y */, + FloatPoint(25, -100) /* delta x/y */, + PointerChoreographerDisplayTopologyTestFixture::DISPLAY_TOP_ID), + std::make_tuple("TransitionToBottomDisplay", + AINPUT_SOURCE_MOUSE | AINPUT_SOURCE_TOUCHPAD, ControllerType::MOUSE, + ToolType::FINGER, FloatPoint(50, 50) /* initial x/y */, + FloatPoint(25, 100) /* delta x/y */, + PointerChoreographerDisplayTopologyTestFixture::DISPLAY_BOTTOM_ID)), + [](const testing::TestParamInfo<PointerChoreographerDisplayTopologyTestFixtureParam>& p) { + return std::string{std::get<0>(p.param)}; + }); + class PointerChoreographerWindowInfoListenerTest : public testing::Test {}; TEST_F_WITH_FLAGS( |