diff options
8 files changed, 164 insertions, 71 deletions
diff --git a/api/current.txt b/api/current.txt index 3fe7d74a37fc..a5a51d66375d 100644 --- a/api/current.txt +++ b/api/current.txt @@ -2873,7 +2873,7 @@ package android.accessibilityservice { method public void onSystemActionsChanged(); method public final boolean performGlobalAction(int); method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo); - method public boolean takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<android.graphics.Bitmap>); + method public boolean takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<android.accessibilityservice.AccessibilityService.ScreenshotResult>); field public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20; // 0x14 field public static final int GESTURE_2_FINGER_SINGLE_TAP = 19; // 0x13 field public static final int GESTURE_2_FINGER_SWIPE_DOWN = 26; // 0x1a @@ -2952,6 +2952,12 @@ package android.accessibilityservice { method public void onMagnificationChanged(@NonNull android.accessibilityservice.AccessibilityService.MagnificationController, @NonNull android.graphics.Region, float, float, float); } + public static final class AccessibilityService.ScreenshotResult { + method @Nullable public android.graphics.ColorSpace getColorSpace(); + method @NonNull public android.hardware.HardwareBuffer getHardwareBuffer(); + method public long getTimestamp(); + } + public static final class AccessibilityService.SoftKeyboardController { method public void addOnShowModeChangedListener(@NonNull android.accessibilityservice.AccessibilityService.SoftKeyboardController.OnShowModeChangedListener); method public void addOnShowModeChangedListener(@NonNull android.accessibilityservice.AccessibilityService.SoftKeyboardController.OnShowModeChangedListener, @Nullable android.os.Handler); diff --git a/core/java/android/accessibilityservice/AccessibilityService.java b/core/java/android/accessibilityservice/AccessibilityService.java index 11f7f465cca5..b65f68e177ca 100644 --- a/core/java/android/accessibilityservice/AccessibilityService.java +++ b/core/java/android/accessibilityservice/AccessibilityService.java @@ -28,7 +28,9 @@ import android.content.Context; import android.content.Intent; import android.content.pm.ParceledListSlice; import android.graphics.Bitmap; +import android.graphics.ColorSpace; import android.graphics.Region; +import android.hardware.HardwareBuffer; import android.os.Binder; import android.os.Build; import android.os.Handler; @@ -585,7 +587,12 @@ public abstract class AccessibilityService extends Service { private FingerprintGestureController mFingerprintGestureController; /** @hide */ - public static final String KEY_ACCESSIBILITY_SCREENSHOT = "screenshot"; + public static final String KEY_ACCESSIBILITY_SCREENSHOT_HARDWAREBUFFER = + "screenshot_hardwareBuffer"; + + /** @hide */ + public static final String KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE_ID = + "screenshot_colorSpaceId"; /** * Callback for {@link android.view.accessibility.AccessibilityEvent}s. @@ -1888,8 +1895,9 @@ public abstract class AccessibilityService extends Service { } /** - * Takes a screenshot of the specified display and returns it by {@link Bitmap.Config#HARDWARE} - * format. + * Takes a screenshot of the specified display and returns it via an + * {@link AccessibilityService.ScreenshotResult}. You can use {@link Bitmap#wrapHardwareBuffer} + * to construct the bitmap from the ScreenshotResult's payload. * <p> * <strong>Note:</strong> In order to take screenshot your service has * to declare the capability to take screenshot by setting the @@ -1907,7 +1915,7 @@ public abstract class AccessibilityService extends Service { * @return {@code true} if the taking screenshot accepted, {@code false} if not. */ public boolean takeScreenshot(int displayId, @NonNull @CallbackExecutor Executor executor, - @NonNull Consumer<Bitmap> callback) { + @NonNull Consumer<ScreenshotResult> callback) { Preconditions.checkNotNull(executor, "executor cannot be null"); Preconditions.checkNotNull(callback, "callback cannot be null"); final IAccessibilityServiceConnection connection = @@ -1917,14 +1925,22 @@ public abstract class AccessibilityService extends Service { return false; } try { - connection.takeScreenshotWithCallback(displayId, new RemoteCallback((result) -> { - final Bitmap screenshot = result.getParcelable(KEY_ACCESSIBILITY_SCREENSHOT); - final long identity = Binder.clearCallingIdentity(); - try { - executor.execute(() -> callback.accept(screenshot)); - } finally { - Binder.restoreCallingIdentity(identity); + connection.takeScreenshot(displayId, new RemoteCallback((result) -> { + if (result == null) { + sendScreenshotResult(executor, callback, null); + return; } + final HardwareBuffer hardwareBuffer = + result.getParcelable(KEY_ACCESSIBILITY_SCREENSHOT_HARDWAREBUFFER); + final int colorSpaceId = + result.getInt(KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE_ID); + ColorSpace colorSpace = null; + if (colorSpaceId >= 0 && colorSpaceId < ColorSpace.Named.values().length) { + colorSpace = ColorSpace.get(ColorSpace.Named.values()[colorSpaceId]); + } + ScreenshotResult screenshot = new ScreenshotResult(hardwareBuffer, + colorSpace, System.currentTimeMillis()); + sendScreenshotResult(executor, callback, screenshot); })); } catch (RemoteException re) { throw new RuntimeException(re); @@ -2323,4 +2339,67 @@ public abstract class AccessibilityService extends Service { this.handler = handler; } } + + private void sendScreenshotResult(Executor executor, Consumer<ScreenshotResult> callback, + ScreenshotResult screenshot) { + final ScreenshotResult result = screenshot; + final long identity = Binder.clearCallingIdentity(); + try { + executor.execute(() -> callback.accept(result)); + } finally { + Binder.restoreCallingIdentity(identity); + } + } + + /** + * Class including hardwareBuffer, colorSpace, and timestamp to be the result for + * {@link AccessibilityService#takeScreenshot} API. + * <p> + * <strong>Note:</strong> colorSpace would be null if the name of this colorSpace isn't at + * {@link ColorSpace.Named}. + * </p> + */ + public static final class ScreenshotResult { + private final @NonNull HardwareBuffer mHardwareBuffer; + private final @Nullable ColorSpace mColorSpace; + private final long mTimestamp; + + private ScreenshotResult(@NonNull HardwareBuffer hardwareBuffer, + @Nullable ColorSpace colorSpace, long timestamp) { + Preconditions.checkNotNull(hardwareBuffer, "hardwareBuffer cannot be null"); + mHardwareBuffer = hardwareBuffer; + mColorSpace = colorSpace; + mTimestamp = timestamp; + } + + /** + * Gets the colorSpace identifying a specific organization of colors of the screenshot. + * + * @return the colorSpace or {@code null} if the name of colorSpace isn't at + * {@link ColorSpace.Named} + */ + @Nullable + public ColorSpace getColorSpace() { + return mColorSpace; + } + + /** + * Gets the hardwareBuffer representing a memory buffer of the screenshot. + * + * @return the hardwareBuffer + */ + @NonNull + public HardwareBuffer getHardwareBuffer() { + return mHardwareBuffer; + } + + /** + * Gets the timestamp of taking the screenshot. + * + * @return the timestamp from {@link System#currentTimeMillis()} + */ + public long getTimestamp() { + return mTimestamp; + }; + } } diff --git a/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl b/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl index 5db4dd7470d8..9177d4d27491 100644 --- a/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl +++ b/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl @@ -110,7 +110,5 @@ interface IAccessibilityServiceConnection { int getWindowIdForLeashToken(IBinder token); - Bitmap takeScreenshot(int displayId); - - void takeScreenshotWithCallback(int displayId, in RemoteCallback callback); + void takeScreenshot(int displayId, in RemoteCallback callback); } diff --git a/core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java b/core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java index 8b8e9ea4c6ee..b71c5800161e 100644 --- a/core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java +++ b/core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java @@ -19,7 +19,6 @@ package android.view.accessibility; import android.accessibilityservice.AccessibilityServiceInfo; import android.accessibilityservice.IAccessibilityServiceConnection; import android.content.pm.ParceledListSlice; -import android.graphics.Bitmap; import android.graphics.Region; import android.os.Bundle; import android.os.IBinder; @@ -157,9 +156,5 @@ public class AccessibilityServiceConnectionImpl extends IAccessibilityServiceCon return -1; } - public Bitmap takeScreenshot(int displayId) { - return null; - } - - public void takeScreenshotWithCallback(int displayId, RemoteCallback callback) {} + public void takeScreenshot(int displayId, RemoteCallback callback) {} } diff --git a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java index a5877ccbde7c..565ee63d89ab 100644 --- a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java @@ -16,6 +16,8 @@ package com.android.server.accessibility; +import static android.accessibilityservice.AccessibilityService.KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE_ID; +import static android.accessibilityservice.AccessibilityService.KEY_ACCESSIBILITY_SCREENSHOT_HARDWAREBUFFER; import static android.accessibilityservice.AccessibilityServiceInfo.DEFAULT; import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY; import static android.view.accessibility.AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS; @@ -36,10 +38,11 @@ import android.content.Intent; import android.content.ServiceConnection; import android.content.pm.PackageManager; import android.content.pm.ParceledListSlice; -import android.graphics.Bitmap; +import android.graphics.GraphicBuffer; import android.graphics.Point; import android.graphics.Rect; import android.graphics.Region; +import android.hardware.HardwareBuffer; import android.hardware.display.DisplayManager; import android.hardware.display.DisplayManagerGlobal; import android.os.Binder; @@ -50,6 +53,7 @@ import android.os.IBinder; import android.os.Looper; import android.os.Message; import android.os.PowerManager; +import android.os.RemoteCallback; import android.os.RemoteException; import android.os.ServiceManager; import android.os.SystemClock; @@ -71,6 +75,7 @@ import com.android.internal.annotations.GuardedBy; import com.android.internal.compat.IPlatformCompat; import com.android.internal.os.SomeArgs; import com.android.internal.util.DumpUtils; +import com.android.internal.util.function.pooled.PooledLambda; import com.android.server.LocalServices; import com.android.server.accessibility.AccessibilityWindowManager.RemoteAccessibilityConnection; import com.android.server.wm.ActivityTaskManagerInternal; @@ -106,6 +111,8 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ private final PowerManager mPowerManager; private final IPlatformCompat mIPlatformCompat; + private final Handler mMainHandler; + // Handler for scheduling method invocations on the main thread. public final InvocationHandler mInvocationHandler; @@ -238,6 +245,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mSecurityPolicy = securityPolicy; mSystemActionPerformer = systemActionPerfomer; mSystemSupport = systemSupport; + mMainHandler = mainHandler; mInvocationHandler = new InvocationHandler(mainHandler.getLooper()); mA11yWindowManager = a11yWindowManager; mDisplayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); @@ -959,52 +967,72 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mInvocationHandler.setSoftKeyboardCallbackEnabled(enabled); } - @Nullable @Override - public Bitmap takeScreenshot(int displayId) { + public void takeScreenshot(int displayId, RemoteCallback callback) { synchronized (mLock) { if (!hasRightsToCurrentUserLocked()) { - return null; + sendScreenshotResult(true, null, callback); + return; } if (!mSecurityPolicy.canTakeScreenshotLocked(this)) { - return null; + sendScreenshotResult(true, null, callback); + throw new SecurityException("Services don't have the capability of taking" + + " the screenshot."); } } if (!mSecurityPolicy.checkAccessibilityAccess(this)) { - return null; + sendScreenshotResult(true, null, callback); + return; } final Display display = DisplayManagerGlobal.getInstance() .getRealDisplay(displayId); if (display == null) { - return null; + sendScreenshotResult(true, null, callback); + return; } - final Point displaySize = new Point(); - display.getRealSize(displaySize); - final int rotation = display.getRotation(); - Bitmap screenShot = null; + sendScreenshotResult(false, display, callback); + } + private void sendScreenshotResult(boolean noResult, Display display, RemoteCallback callback) { + final boolean noScreenshot = noResult; final long identity = Binder.clearCallingIdentity(); try { - final Rect crop = new Rect(0, 0, displaySize.x, displaySize.y); - // TODO (b/145893483): calling new API with the display as a parameter - // when surface control supported. - screenShot = SurfaceControl.screenshot(crop, displaySize.x, displaySize.y, - rotation); - if (screenShot != null) { - // Optimization for telling the bitmap that all of the pixels are known to be - // opaque (false). This is meant as a drawing hint, as in some cases a bitmap - // that is known to be opaque can take a faster drawing case than one that may - // have non-opaque per-pixel alpha values. - screenShot.setHasAlpha(false); - } + mMainHandler.post(PooledLambda.obtainRunnable((nonArg) -> { + if (noScreenshot) { + callback.sendResult(null); + return; + } + final Point displaySize = new Point(); + // TODO (b/145893483): calling new API with the display as a parameter + // when surface control supported. + final IBinder token = SurfaceControl.getInternalDisplayToken(); + final Rect crop = new Rect(0, 0, displaySize.x, displaySize.y); + final int rotation = display.getRotation(); + display.getRealSize(displaySize); + + final SurfaceControl.ScreenshotGraphicBuffer screenshotBuffer = + SurfaceControl.screenshotToBufferWithSecureLayersUnsafe(token, crop, + displaySize.x, displaySize.y, false, + rotation); + final GraphicBuffer graphicBuffer = screenshotBuffer.getGraphicBuffer(); + final HardwareBuffer hardwareBuffer = + HardwareBuffer.createFromGraphicBuffer(graphicBuffer); + final int colorSpaceId = screenshotBuffer.getColorSpace().getId(); + + // Send back the result. + final Bundle payload = new Bundle(); + payload.putParcelable(KEY_ACCESSIBILITY_SCREENSHOT_HARDWAREBUFFER, + hardwareBuffer); + payload.putInt(KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE_ID, colorSpaceId); + callback.sendResult(payload); + }, null).recycleOnUse()); } finally { Binder.restoreCallingIdentity(identity); } - return screenShot; } @Override diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java index 25911a7ed0ea..edb4445151d5 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java @@ -16,8 +16,6 @@ package com.android.server.accessibility; -import static android.accessibilityservice.AccessibilityService.KEY_ACCESSIBILITY_SCREENSHOT; - import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; import android.Manifest; @@ -27,20 +25,16 @@ import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.pm.ParceledListSlice; -import android.graphics.Bitmap; import android.os.Binder; -import android.os.Bundle; import android.os.Handler; import android.os.IBinder; import android.os.Process; -import android.os.RemoteCallback; import android.os.RemoteException; import android.os.UserHandle; import android.provider.Settings; import android.util.Slog; import android.view.Display; -import com.android.internal.util.function.pooled.PooledLambda; import com.android.server.inputmethod.InputMethodManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.WindowManagerInternal; @@ -393,15 +387,4 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect } } } - - @Override - public void takeScreenshotWithCallback(int displayId, RemoteCallback callback) { - mMainHandler.post(PooledLambda.obtainRunnable((nonArg) -> { - final Bitmap screenshot = super.takeScreenshot(displayId); - // Send back the result. - final Bundle payload = new Bundle(); - payload.putParcelable(KEY_ACCESSIBILITY_SCREENSHOT, screenshot); - callback.sendResult(payload); - }, null).recycleOnUse()); - } } diff --git a/services/accessibility/java/com/android/server/accessibility/UiAutomationManager.java b/services/accessibility/java/com/android/server/accessibility/UiAutomationManager.java index 5d9af26a8339..d1c3a02c6761 100644 --- a/services/accessibility/java/com/android/server/accessibility/UiAutomationManager.java +++ b/services/accessibility/java/com/android/server/accessibility/UiAutomationManager.java @@ -328,6 +328,6 @@ class UiAutomationManager { public void onFingerprintGesture(int gesture) {} @Override - public void takeScreenshotWithCallback(int displayId, RemoteCallback callback) {} + public void takeScreenshot(int displayId, RemoteCallback callback) {} } } diff --git a/services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java b/services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java index cf10559c8198..dfe950ea93d6 100644 --- a/services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java +++ b/services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java @@ -47,6 +47,7 @@ import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -703,14 +704,20 @@ public class AbstractAccessibilityServiceConnectionTest { @Test public void takeScreenshot_returnNull() { - // no canTakeScreenshot, should return null. - when(mMockSecurityPolicy.canTakeScreenshotLocked(mServiceConnection)).thenReturn(false); - assertThat(mServiceConnection.takeScreenshot(Display.DEFAULT_DISPLAY), is(nullValue())); - // no checkAccessibilityAccess, should return null. when(mMockSecurityPolicy.canTakeScreenshotLocked(mServiceConnection)).thenReturn(true); when(mMockSecurityPolicy.checkAccessibilityAccess(mServiceConnection)).thenReturn(false); - assertThat(mServiceConnection.takeScreenshot(Display.DEFAULT_DISPLAY), is(nullValue())); + mServiceConnection.takeScreenshot(Display.DEFAULT_DISPLAY, new RemoteCallback((result) -> { + assertNull(result); + })); + } + + @Test (expected = SecurityException.class) + public void takeScreenshot_throwSecurityException() { + // no canTakeScreenshot, should throw security exception. + when(mMockSecurityPolicy.canTakeScreenshotLocked(mServiceConnection)).thenReturn(false); + mServiceConnection.takeScreenshot(Display.DEFAULT_DISPLAY, new RemoteCallback((result) -> { + })); } private void updateServiceInfo(AccessibilityServiceInfo serviceInfo, int eventType, @@ -852,8 +859,5 @@ public class AbstractAccessibilityServiceConnectionTest { @Override public void onFingerprintGesture(int gesture) {} - - @Override - public void takeScreenshotWithCallback(int displayId, RemoteCallback callback) {} } } |