From e64bae455ed9230b68afa6fbbfd825b8d2988894 Mon Sep 17 00:00:00 2001 From: Chavi Weingarten Date: Thu, 30 Jun 2022 16:15:12 +0000 Subject: Use layer mirroring when screen recording via VD When a Display has the flag VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, the content on the display is expected to mirror another display. This was previously done by setting the same layerStack. This change removes that behavior and instead will create a mirror hierarchy of the other display and place it at the root of the new Display. This is leveraging the work done for ContentRecorder, but using it as default for all displays instead of only for VD created by MediaProjection. Test: Screen record via MediaProjection Test: Screen record via DisplayManager Test: ContentRecorderTests Test: DisplayContentTests Bug: 237664947 Change-Id: I18b36dadc43ba1c37f9516ff10d24bb8683c04ac --- .../hardware/display/DisplayManagerInternal.java | 9 +++ .../android/internal/protolog/ProtoLogGroup.java | 2 +- data/etc/services.core.protolog.json | 24 ++++++++ .../com/android/server/display/DisplayDevice.java | 7 ++- .../server/display/DisplayManagerService.java | 57 ++++++++++--------- .../com/android/server/wm/ContentRecorder.java | 7 ++- .../java/com/android/server/wm/DisplayContent.java | 64 +++++++++++++++++++++- .../server/display/DisplayManagerServiceTest.java | 52 +++++++++++++++++- .../android/server/wm/ContentRecorderTests.java | 3 + .../com/android/server/wm/DisplayContentTests.java | 38 +++++++++++++ 10 files changed, 228 insertions(+), 35 deletions(-) diff --git a/core/java/android/hardware/display/DisplayManagerInternal.java b/core/java/android/hardware/display/DisplayManagerInternal.java index b0b3e9e743cf..52cef0f1efd0 100644 --- a/core/java/android/hardware/display/DisplayManagerInternal.java +++ b/core/java/android/hardware/display/DisplayManagerInternal.java @@ -373,6 +373,15 @@ public abstract class DisplayManagerInternal { */ public abstract Point getDisplaySurfaceDefaultSize(int displayId); + /** + * Get a new displayId which represents the display you want to mirror. If mirroring is not + * enabled on the display, {@link Display#INVALID_DISPLAY} will be returned. + * + * @param displayId The id of the display. + * @return The displayId that should be mirrored or INVALID_DISPLAY if mirroring is not enabled. + */ + public abstract int getDisplayIdToMirror(int displayId); + /** * Receives early interactivity changes from power manager. * diff --git a/core/java/com/android/internal/protolog/ProtoLogGroup.java b/core/java/com/android/internal/protolog/ProtoLogGroup.java index 7f36c79591b3..0e8dc071cbdc 100644 --- a/core/java/com/android/internal/protolog/ProtoLogGroup.java +++ b/core/java/com/android/internal/protolog/ProtoLogGroup.java @@ -83,7 +83,7 @@ public enum ProtoLogGroup implements IProtoLogGroup { Consts.TAG_WM), WM_DEBUG_WINDOW_INSETS(Consts.ENABLE_DEBUG, Consts.ENABLE_LOG_TO_PROTO_DEBUG, false, Consts.TAG_WM), - WM_DEBUG_CONTENT_RECORDING(Consts.ENABLE_DEBUG, Consts.ENABLE_LOG_TO_PROTO_DEBUG, true, + WM_DEBUG_CONTENT_RECORDING(Consts.ENABLE_DEBUG, Consts.ENABLE_LOG_TO_PROTO_DEBUG, false, Consts.TAG_WM), WM_DEBUG_WALLPAPER(Consts.ENABLE_DEBUG, Consts.ENABLE_LOG_TO_PROTO_DEBUG, false, Consts.TAG_WM), WM_DEBUG_BACK_PREVIEW(Consts.ENABLE_DEBUG, Consts.ENABLE_LOG_TO_PROTO_DEBUG, true, diff --git a/data/etc/services.core.protolog.json b/data/etc/services.core.protolog.json index ae1c94611355..b327bf01f7de 100644 --- a/data/etc/services.core.protolog.json +++ b/data/etc/services.core.protolog.json @@ -13,12 +13,24 @@ "group": "WM_DEBUG_STARTING_WINDOW", "at": "com\/android\/server\/wm\/ActivityRecord.java" }, + "-2123789565": { + "message": "Found no matching mirror display for id=%d for DEFAULT_DISPLAY. Nothing to mirror.", + "level": "WARN", + "group": "WM_DEBUG_CONTENT_RECORDING", + "at": "com\/android\/server\/wm\/DisplayContent.java" + }, "-2121056984": { "message": "%s", "level": "WARN", "group": "WM_DEBUG_LOCKTASK", "at": "com\/android\/server\/wm\/LockTaskController.java" }, + "-2113780196": { + "message": "Successfully created a ContentRecordingSession for displayId=%d to mirror content from displayId=%d", + "level": "VERBOSE", + "group": "WM_DEBUG_CONTENT_RECORDING", + "at": "com\/android\/server\/wm\/DisplayContent.java" + }, "-2111539867": { "message": "remove IME snapshot, caller=%s", "level": "INFO", @@ -1297,6 +1309,12 @@ "group": "WM_DEBUG_CONFIGURATION", "at": "com\/android\/server\/wm\/ActivityRecord.java" }, + "-838378223": { + "message": "Attempting to mirror self on %d", + "level": "WARN", + "group": "WM_DEBUG_CONTENT_RECORDING", + "at": "com\/android\/server\/wm\/DisplayContent.java" + }, "-814760297": { "message": "Looking for task of %s in %s", "level": "DEBUG", @@ -1405,6 +1423,12 @@ "group": "WM_DEBUG_CONTENT_RECORDING", "at": "com\/android\/server\/wm\/ContentRecorder.java" }, + "-729864558": { + "message": "Attempting to mirror %d from %d but no DisplayContent associated. Changing to mirror default display.", + "level": "WARN", + "group": "WM_DEBUG_CONTENT_RECORDING", + "at": "com\/android\/server\/wm\/DisplayContent.java" + }, "-729530161": { "message": "Moving to DESTROYED: %s (no app)", "level": "VERBOSE", diff --git a/services/core/java/com/android/server/display/DisplayDevice.java b/services/core/java/com/android/server/display/DisplayDevice.java index e53aef733293..99e709ea3fd8 100644 --- a/services/core/java/com/android/server/display/DisplayDevice.java +++ b/services/core/java/com/android/server/display/DisplayDevice.java @@ -127,12 +127,13 @@ abstract class DisplayDevice { /** * Returns the default size of the surface associated with the display, or null if the surface - * is not provided for layer mirroring by SurfaceFlinger. - * Only used for mirroring started from MediaProjection. + * is not provided for layer mirroring by SurfaceFlinger. For non virtual displays, this will + * be the actual display device's size. */ @Nullable public Point getDisplaySurfaceDefaultSizeLocked() { - return null; + DisplayDeviceInfo displayDeviceInfo = getDisplayDeviceInfoLocked(); + return new Point(displayDeviceInfo.width, displayDeviceInfo.height); } /** diff --git a/services/core/java/com/android/server/display/DisplayManagerService.java b/services/core/java/com/android/server/display/DisplayManagerService.java index b3434805ee41..6108b3f5d8be 100644 --- a/services/core/java/com/android/server/display/DisplayManagerService.java +++ b/services/core/java/com/android/server/display/DisplayManagerService.java @@ -2214,24 +2214,10 @@ public final class DisplayManagerService extends SystemService { private void configureDisplayLocked(SurfaceControl.Transaction t, DisplayDevice device) { final DisplayDeviceInfo info = device.getDisplayDeviceInfoLocked(); - final boolean ownContent = (info.flags & DisplayDeviceInfo.FLAG_OWN_CONTENT_ONLY) != 0; // Find the logical display that the display device is showing. // Certain displays only ever show their own content. LogicalDisplay display = mLogicalDisplayMapper.getDisplayLocked(device); - // Proceed with display-managed mirroring only if window manager will not be handling it. - if (!ownContent && !device.isWindowManagerMirroringLocked()) { - // Only mirror the display if content recording is not taking place in WM. - if (display != null && !display.hasContentLocked()) { - // If the display does not have any content of its own, then - // automatically mirror the requested logical display contents if possible. - display = mLogicalDisplayMapper.getDisplayLocked( - device.getDisplayIdToMirrorLocked()); - } - if (display == null) { - display = mLogicalDisplayMapper.getDisplayLocked(Display.DEFAULT_DISPLAY); - } - } // Apply the logical display configuration to the display device. if (display == null) { @@ -2545,18 +2531,6 @@ public final class DisplayManagerService extends SystemService { } } - @VisibleForTesting - int getDisplayIdToMirrorInternal(int displayId) { - synchronized (mSyncRoot) { - final LogicalDisplay display = mLogicalDisplayMapper.getDisplayLocked(displayId); - if (display != null) { - final DisplayDevice displayDevice = display.getPrimaryDisplayDeviceLocked(); - return displayDevice.getDisplayIdToMirrorLocked(); - } - return Display.INVALID_DISPLAY; - } - } - @VisibleForTesting Surface getVirtualDisplaySurfaceInternal(IBinder appToken) { synchronized (mSyncRoot) { @@ -3853,6 +3827,37 @@ public final class DisplayManagerService extends SystemService { return null; } } + + @Override + public int getDisplayIdToMirror(int displayId) { + synchronized (mSyncRoot) { + final LogicalDisplay display = mLogicalDisplayMapper.getDisplayLocked(displayId); + if (display == null) { + return Display.INVALID_DISPLAY; + } + + final DisplayDevice displayDevice = display.getPrimaryDisplayDeviceLocked(); + final boolean ownContent = (displayDevice.getDisplayDeviceInfoLocked().flags + & DisplayDeviceInfo.FLAG_OWN_CONTENT_ONLY) != 0; + // If the display has enabled mirroring, but specified that it will be managed by + // WindowManager, return an invalid display id. This is to ensure we don't + // accidentally select the display id to mirror based on DM logic and instead allow + // the caller to specify what area to mirror. + if (ownContent || displayDevice.isWindowManagerMirroringLocked()) { + return Display.INVALID_DISPLAY; + } + + int displayIdToMirror = displayDevice.getDisplayIdToMirrorLocked(); + LogicalDisplay displayToMirror = mLogicalDisplayMapper.getDisplayLocked( + displayIdToMirror); + // If the displayId for the requested mirror doesn't exist, fallback to mirroring + // default display. + if (displayToMirror == null) { + displayIdToMirror = Display.DEFAULT_DISPLAY; + } + return displayIdToMirror; + } + } } class DesiredDisplayModeSpecsObserver diff --git a/services/core/java/com/android/server/wm/ContentRecorder.java b/services/core/java/com/android/server/wm/ContentRecorder.java index 5d2d5826f227..7731f2851de7 100644 --- a/services/core/java/com/android/server/wm/ContentRecorder.java +++ b/services/core/java/com/android/server/wm/ContentRecorder.java @@ -87,6 +87,10 @@ final class ContentRecorder { mContentRecordingSession = session; } + boolean isContentRecordingSessionSet() { + return mContentRecordingSession != null; + } + /** * Returns {@code true} if this DisplayContent is currently recording. */ @@ -299,8 +303,7 @@ final class ContentRecorder { mDisplayContent.mWmService.mWindowContextListenerController.getContainer( tokenToRecord); if (wc == null) { - // Un-set the window token to record for this VirtualDisplay. Fall back to - // Display stack capture for the entire display. + // Fall back to screenrecording using the data sent to DisplayManager mDisplayContent.mWmService.mDisplayManagerInternal.setWindowManagerMirroring( mDisplayContent.getDisplayId(), false); handleStartRecordingFailed(); diff --git a/services/core/java/com/android/server/wm/DisplayContent.java b/services/core/java/com/android/server/wm/DisplayContent.java index b54cd415d8df..cb486d45700d 100644 --- a/services/core/java/com/android/server/wm/DisplayContent.java +++ b/services/core/java/com/android/server/wm/DisplayContent.java @@ -6365,12 +6365,74 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp getContentRecorder().setContentRecordingSession(session); } + /** + * This is to enable mirroring on virtual displays that specify the + * {@link android.hardware.display.DisplayManager#VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR} but don't + * mirror using MediaProjection. When done through MediaProjection API, the + * ContentRecordingSession will be created automatically. + * + * This should only be called when there's no ContentRecordingSession already set for this + * display. The code will ask DMS if this display should enable display mirroring and which + * displayId to mirror from. + * + * @return true if the {@link ContentRecordingSession} was set for display mirroring using data + * from DMS, false if there was no ContentRecordingSession created. + */ + boolean setDisplayMirroring() { + int mirrorDisplayId = mWmService.mDisplayManagerInternal.getDisplayIdToMirror(mDisplayId); + if (mirrorDisplayId == INVALID_DISPLAY) { + return false; + } + + if (mirrorDisplayId == mDisplayId) { + if (mDisplayId != DEFAULT_DISPLAY) { + ProtoLog.w(WM_DEBUG_CONTENT_RECORDING, + "Attempting to mirror self on %d", mirrorDisplayId); + } + return false; + } + + // This is very unlikely, and probably impossible, but if the current display is + // DEFAULT_DISPLAY and the displayId to mirror results in an invalid display, we don't want + // to mirror the DEFAULT_DISPLAY so instead we just return + DisplayContent mirrorDc = mRootWindowContainer.getDisplayContentOrCreate(mirrorDisplayId); + if (mirrorDc == null && mDisplayId == DEFAULT_DISPLAY) { + ProtoLog.w(WM_DEBUG_CONTENT_RECORDING, "Found no matching mirror display for id=%d for" + + " DEFAULT_DISPLAY. Nothing to mirror.", mirrorDisplayId); + return false; + } + + if (mirrorDc == null) { + mirrorDc = mRootWindowContainer.getDefaultDisplay(); + ProtoLog.w(WM_DEBUG_CONTENT_RECORDING, + "Attempting to mirror %d from %d but no DisplayContent associated. Changing " + + "to mirror default display.", + mirrorDisplayId, mDisplayId); + } + + ContentRecordingSession session = ContentRecordingSession + .createDisplaySession(mirrorDc.getDisplayUiContext().getWindowContextToken()) + .setDisplayId(mDisplayId); + setContentRecordingSession(session); + ProtoLog.v(WM_DEBUG_CONTENT_RECORDING, + "Successfully created a ContentRecordingSession for displayId=%d to mirror " + + "content from displayId=%d", + mDisplayId, mirrorDisplayId); + return true; + } + /** * Start recording if this DisplayContent no longer has content. Stop recording if it now * has content or the display is not on. */ @VisibleForTesting void updateRecording() { - getContentRecorder().updateRecording(); + if (mContentRecorder == null || !mContentRecorder.isContentRecordingSessionSet()) { + if (!setDisplayMirroring()) { + return; + } + } + + mContentRecorder.updateRecording(); } /** diff --git a/services/tests/servicestests/src/com/android/server/display/DisplayManagerServiceTest.java b/services/tests/servicestests/src/com/android/server/display/DisplayManagerServiceTest.java index c11e2b02cd97..3eb1dea80c7f 100644 --- a/services/tests/servicestests/src/com/android/server/display/DisplayManagerServiceTest.java +++ b/services/tests/servicestests/src/com/android/server/display/DisplayManagerServiceTest.java @@ -17,6 +17,7 @@ package com.android.server.display; import static android.Manifest.permission.ADD_TRUSTED_DISPLAY; +import static android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY; import static com.android.server.display.VirtualDisplayAdapter.UNIQUE_ID_PREFIX; @@ -577,6 +578,7 @@ public class DisplayManagerServiceTest { // This is effectively the DisplayManager service published to ServiceManager. DisplayManagerService.BinderService binderService = displayManager.new BinderService(); + DisplayManagerService.LocalService localDisplayManager = displayManager.new LocalService(); final String uniqueId = "uniqueId --- displayIdToMirrorTest"; final int width = 600; @@ -606,12 +608,58 @@ public class DisplayManagerServiceTest { displayManager.getDisplayHandler().runWithScissors(() -> {}, 0 /* now */); // The displayId to mirror should be a default display if there is none initially. - assertEquals(displayManager.getDisplayIdToMirrorInternal(firstDisplayId), + assertEquals(localDisplayManager.getDisplayIdToMirror(firstDisplayId), Display.DEFAULT_DISPLAY); - assertEquals(displayManager.getDisplayIdToMirrorInternal(secondDisplayId), + assertEquals(localDisplayManager.getDisplayIdToMirror(secondDisplayId), firstDisplayId); } + @Test + public void testGetDisplayIdToMirror() throws Exception { + DisplayManagerService displayManager = new DisplayManagerService(mContext, mBasicInjector); + registerDefaultDisplays(displayManager); + + // This is effectively the DisplayManager service published to ServiceManager. + DisplayManagerService.BinderService binderService = displayManager.new BinderService(); + DisplayManagerService.LocalService localDisplayManager = displayManager.new LocalService(); + + final String uniqueId = "uniqueId --- displayIdToMirrorTest"; + final int width = 600; + final int height = 800; + final int dpi = 320; + + when(mMockAppToken.asBinder()).thenReturn(mMockAppToken); + final VirtualDisplayConfig.Builder builder = new VirtualDisplayConfig.Builder( + VIRTUAL_DISPLAY_NAME, width, height, dpi) + .setUniqueId(uniqueId) + .setFlags(VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY); + final int firstDisplayId = binderService.createVirtualDisplay(builder.build(), + mMockAppToken /* callback */, null /* projection */, PACKAGE_NAME); + + // The second virtual display requests to mirror the first virtual display. + final String uniqueId2 = "uniqueId --- displayIdToMirrorTest #2"; + when(mMockAppToken2.asBinder()).thenReturn(mMockAppToken2); + final VirtualDisplayConfig.Builder builder2 = new VirtualDisplayConfig.Builder( + VIRTUAL_DISPLAY_NAME, width, height, dpi) + .setUniqueId(uniqueId2) + .setWindowManagerMirroring(true); + final int secondDisplayId = binderService.createVirtualDisplay(builder2.build(), + mMockAppToken2 /* callback */, null /* projection */, + PACKAGE_NAME); + displayManager.performTraversalInternal(mock(SurfaceControl.Transaction.class)); + + // flush the handler + displayManager.getDisplayHandler().runWithScissors(() -> {}, 0 /* now */); + + // The displayId to mirror should be a invalid since the display had flag OWN_CONTENT_ONLY + assertEquals(localDisplayManager.getDisplayIdToMirror(firstDisplayId), + Display.INVALID_DISPLAY); + // The second display has mirroring managed by WindowManager so the mirror displayId should + // be invalid. + assertEquals(localDisplayManager.getDisplayIdToMirror(secondDisplayId), + Display.INVALID_DISPLAY); + } + /** * Tests that the virtual display is created with * {@link VirtualDisplayConfig.Builder#setSurface(Surface)} diff --git a/services/tests/wmtests/src/com/android/server/wm/ContentRecorderTests.java b/services/tests/wmtests/src/com/android/server/wm/ContentRecorderTests.java index c5117bb83976..69ba8ad58c7c 100644 --- a/services/tests/wmtests/src/com/android/server/wm/ContentRecorderTests.java +++ b/services/tests/wmtests/src/com/android/server/wm/ContentRecorderTests.java @@ -19,6 +19,7 @@ package com.android.server.wm; import static android.content.res.Configuration.ORIENTATION_PORTRAIT; import static android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR; +import static android.view.Display.INVALID_DISPLAY; import static com.android.dx.mockito.inline.extended.ExtendedMockito.any; import static com.android.dx.mockito.inline.extended.ExtendedMockito.doReturn; @@ -85,6 +86,8 @@ public class ContentRecorderTests extends WindowTestsBase { mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().height()); mRecordedSurface = surfaceControlMirrors(sSurfaceSize); + doReturn(INVALID_DISPLAY).when(mWm.mDisplayManagerInternal).getDisplayIdToMirror(anyInt()); + // GIVEN the VirtualDisplay associated with the session (so the display has state ON). VirtualDisplay virtualDisplay = mWm.mDisplayManager.createVirtualDisplay("VirtualDisplay", sSurfaceSize.x, sSurfaceSize.y, diff --git a/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java b/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java index 641a3adf337a..2cf9c01a4476 100644 --- a/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java +++ b/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java @@ -31,6 +31,7 @@ import static android.os.Build.VERSION_CODES.P; import static android.os.Build.VERSION_CODES.Q; import static android.view.Display.DEFAULT_DISPLAY; import static android.view.Display.FLAG_PRIVATE; +import static android.view.Display.INVALID_DISPLAY; import static android.view.DisplayCutout.BOUNDS_POSITION_TOP; import static android.view.DisplayCutout.fromBoundingRect; import static android.view.InsetsState.ITYPE_IME; @@ -2588,6 +2589,43 @@ public class DisplayContentTests extends WindowTestsBase { display.release(); } + @Test + public void testVirtualDisplayContent_displayMirroring() { + // GIVEN MediaProjection has already initialized the WindowToken of the DisplayArea to + // mirror. + final IBinder tokenToMirror = setUpDefaultTaskDisplayAreaWindowToken(); + + // GIVEN SurfaceControl can successfully mirror the provided surface. + Point surfaceSize = new Point( + mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().width(), + mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().height()); + surfaceControlMirrors(surfaceSize); + // Initially disable getDisplayIdToMirror since the DMS may create the DC outside the direct + // call in the test. We need to spy on the DC before updateRecording is called or we can't + // verify setDisplayMirroring is called + doReturn(INVALID_DISPLAY).when(mWm.mDisplayManagerInternal).getDisplayIdToMirror(anyInt()); + + // GIVEN a new VirtualDisplay with an associated surface. + final VirtualDisplay display = createVirtualDisplay(surfaceSize, new Surface()); + final int displayId = display.getDisplay().getDisplayId(); + + // GIVEN a session for this display. + mWm.mRoot.onDisplayAdded(displayId); + + // WHEN getting the DisplayContent for the new virtual display. + DisplayContent actualDC = mWm.mRoot.getDisplayContent(displayId); + spyOn(actualDC); + // Return the default display as the value to mirror to ensure the VD with flag mirroring + // creates a ContentRecordingSession automatically. + doReturn(DEFAULT_DISPLAY).when(mWm.mDisplayManagerInternal).getDisplayIdToMirror(anyInt()); + actualDC.updateRecording(); + + // THEN mirroring is initiated for the default display's DisplayArea. + verify(actualDC).setDisplayMirroring(); + assertThat(actualDC.isCurrentlyRecording()).isTrue(); + display.release(); + } + private static class MirroringTestToken extends Binder { } -- cgit v1.2.3-59-g8ed1b