diff options
| author | 2022-03-03 08:32:03 +0000 | |
|---|---|---|
| committer | 2022-03-03 08:32:03 +0000 | |
| commit | 3de3b76c4b515dbb6bdc6818a764e35ff1f7f8f7 (patch) | |
| tree | 06d16ae2df28be84bf42e49802d98ff547156296 | |
| parent | 81eff37a33f79f1ce4c871db1239b8b08e60abf6 (diff) | |
| parent | ce4c02258b75bcfd7c4581373da3407c30f54f9e (diff) | |
Merge changes from topic "A11y_Bound" into tm-dev
* changes:
Fixs the ANR issue
Descreses the counts of windows_change_bounds events
| -rw-r--r-- | services/core/java/com/android/server/wm/AccessibilityController.java | 34 | ||||
| -rw-r--r-- | services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java | 317 |
2 files changed, 281 insertions, 70 deletions
diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java index d3f3abe1085d..e37b08f417f2 100644 --- a/services/core/java/com/android/server/wm/AccessibilityController.java +++ b/services/core/java/com/android/server/wm/AccessibilityController.java @@ -267,6 +267,8 @@ final class AccessibilityController { FLAGS_MAGNIFICATION_CALLBACK | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK, "displayId=" + displayId + "; spec={" + spec + "}"); } + mAccessibilityWindowsPopulator.setMagnificationSpec(displayId, spec); + final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId); if (displayMagnifier != null) { displayMagnifier.setMagnificationSpec(spec); @@ -458,19 +460,6 @@ final class AccessibilityController { return null; } - boolean getMagnificationSpecForDisplay(int displayId, MagnificationSpec outSpec) { - if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { - mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForDisplay", - FLAGS_MAGNIFICATION_CALLBACK, "displayId=" + displayId); - } - final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId); - if (displayMagnifier == null) { - return false; - } - - return displayMagnifier.getMagnificationSpec(outSpec); - } - boolean hasCallbacks() { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) { @@ -775,25 +764,6 @@ final class AccessibilityController { return spec; } - boolean getMagnificationSpec(MagnificationSpec outSpec) { - if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { - mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec", - FLAGS_MAGNIFICATION_CALLBACK); - } - MagnificationSpec spec = mMagnifedViewport.getMagnificationSpec(); - if (spec == null) { - return false; - } - - outSpec.setTo(spec); - if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { - mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec", - FLAGS_MAGNIFICATION_CALLBACK, "outSpec={" + outSpec + "}"); - } - - return true; - } - void getMagnificationRegion(Region outMagnificationRegion) { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationRegion", diff --git a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java index c0fb83ba294c..f9689a8a2bca 100644 --- a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java +++ b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java @@ -41,7 +41,9 @@ import android.window.WindowInfosListener; import com.android.internal.annotations.GuardedBy; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * This class is the accessibility windows population adapter. @@ -68,13 +70,24 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { private final SparseArray<Matrix> mMagnificationSpecInverseMatrix = new SparseArray<>(); @GuardedBy("mLock") private final SparseArray<DisplayInfo> mDisplayInfos = new SparseArray<>(); + private final SparseArray<MagnificationSpec> mCurrentMagnificationSpec = new SparseArray<>(); + @GuardedBy("mLock") + private final SparseArray<MagnificationSpec> mPreviousMagnificationSpec = new SparseArray<>(); @GuardedBy("mLock") private final List<InputWindowHandle> mVisibleWindows = new ArrayList<>(); @GuardedBy("mLock") private boolean mWindowsNotificationEnabled = false; + @GuardedBy("mLock") + private final Map<IBinder, Matrix> mWindowsTransformMatrixMap = new HashMap<>(); private final Object mLock = new Object(); private final Handler mHandler; + private final Matrix mTempMatrix1 = new Matrix(); + private final Matrix mTempMatrix2 = new Matrix(); + private final float[] mTempFloat1 = new float[9]; + private final float[] mTempFloat2 = new float[9]; + private final float[] mTempFloat3 = new float[9]; + AccessibilityWindowsPopulator(WindowManagerService service, AccessibilityController accessibilityController) { mService = service; @@ -132,13 +145,27 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { @Override public void onWindowInfosChanged(InputWindowHandle[] windowHandles, DisplayInfo[] displayInfos) { + mHandler.post(() -> onWindowInfosChangedInternal(windowHandles, displayInfos)); + } + + private void onWindowInfosChangedInternal(InputWindowHandle[] windowHandles, + DisplayInfo[] displayInfos) { + final List<InputWindowHandle> tempVisibleWindows = new ArrayList<>(); + + for (InputWindowHandle window : windowHandles) { + if (window.visible && window.getWindow() != null) { + tempVisibleWindows.add(window); + } + } + final HashMap<IBinder, Matrix> windowsTransformMatrixMap = + getWindowsTransformMatrix(tempVisibleWindows); + synchronized (mLock) { + mWindowsTransformMatrixMap.clear(); + mWindowsTransformMatrixMap.putAll(windowsTransformMatrixMap); + mVisibleWindows.clear(); - for (InputWindowHandle window : windowHandles) { - if (window.visible && window.getWindow() != null) { - mVisibleWindows.add(window); - } - } + mVisibleWindows.addAll(tempVisibleWindows); mDisplayInfos.clear(); for (final DisplayInfo displayInfo : displayInfos) { @@ -146,17 +173,36 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { } if (mWindowsNotificationEnabled) { - if (!mHandler.hasMessages( - MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { + if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { mHandler.sendEmptyMessageDelayed( MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT, WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS); } - populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); + populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded(); } } } + private HashMap<IBinder, Matrix> getWindowsTransformMatrix(List<InputWindowHandle> windows) { + synchronized (mService.mGlobalLock) { + final HashMap<IBinder, Matrix> windowsTransformMatrixMap = new HashMap<>(); + + for (InputWindowHandle inputWindowHandle : windows) { + final IWindow iWindow = inputWindowHandle.getWindow(); + final WindowState windowState = iWindow != null ? mService.mWindowMap.get( + iWindow.asBinder()) : null; + + if (windowState != null && windowState.shouldMagnify()) { + final Matrix transformMatrix = new Matrix(); + windowState.getTransformationMatrix(sTempFloats, transformMatrix); + windowsTransformMatrixMap.put(iWindow.asBinder(), transformMatrix); + } + } + + return windowsTransformMatrixMap; + } + } + /** * Sets to notify the accessibilityController to compute changed windows on * the display after populating the visible windows if the windows reported @@ -171,14 +217,43 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { } mWindowsNotificationEnabled = register; if (mWindowsNotificationEnabled) { - populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); + populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded(); } else { releaseResources(); } } } - private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked() { + /** + * Sets the magnification spec for calculating the window bounds of all windows + * reported from the surface flinger in the magnifying. + * + * @param displayId The display Id. + * @param spec THe magnification spec. + */ + public void setMagnificationSpec(int displayId, MagnificationSpec spec) { + synchronized (mLock) { + MagnificationSpec currentMagnificationSpec = mCurrentMagnificationSpec.get(displayId); + if (currentMagnificationSpec == null) { + currentMagnificationSpec = new MagnificationSpec(); + currentMagnificationSpec.setTo(spec); + mCurrentMagnificationSpec.put(displayId, currentMagnificationSpec); + + return; + } + + MagnificationSpec previousMagnificationSpec = mPreviousMagnificationSpec.get(displayId); + if (previousMagnificationSpec == null) { + previousMagnificationSpec = new MagnificationSpec(); + mPreviousMagnificationSpec.put(displayId, previousMagnificationSpec); + } + previousMagnificationSpec.setTo(currentMagnificationSpec); + currentMagnificationSpec.setTo(spec); + } + } + + @GuardedBy("mLock") + private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded() { final SparseArray<List<InputWindowHandle>> tempWindowHandleList = new SparseArray<>(); for (final InputWindowHandle windowHandle : mVisibleWindows) { @@ -188,15 +263,15 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { if (inputWindowHandles == null) { inputWindowHandles = new ArrayList<>(); tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles); - generateMagnificationSpecInverseMatrixLocked(windowHandle.displayId); } inputWindowHandles.add(windowHandle); } + findMagnificationSpecInverseMatrixIfNeeded(tempWindowHandleList); final List<Integer> displayIdsForWindowsChanged = new ArrayList<>(); - - getDisplaysForWindowsChangedLocked(displayIdsForWindowsChanged, tempWindowHandleList, + getDisplaysForWindowsChanged(displayIdsForWindowsChanged, tempWindowHandleList, mInputWindowHandlesOnDisplays); + // Clones all windows from the callback of the surface flinger. mInputWindowHandlesOnDisplays.clear(); for (int i = 0; i < tempWindowHandleList.size(); i++) { @@ -204,7 +279,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId)); } - if (displayIdsForWindowsChanged.size() > 0) { + if (!displayIdsForWindowsChanged.isEmpty()) { if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, displayIdsForWindowsChanged).sendToTarget(); @@ -217,7 +292,8 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS); } - private void getDisplaysForWindowsChangedLocked(List<Integer> outDisplayIdsForWindowsChanged, + @GuardedBy("mLock") + private static void getDisplaysForWindowsChanged(List<Integer> outDisplayIdsForWindowsChanged, SparseArray<List<InputWindowHandle>> newWindowsList, SparseArray<List<InputWindowHandle>> oldWindowsList) { for (int i = 0; i < newWindowsList.size(); i++) { @@ -225,13 +301,14 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { final List<InputWindowHandle> newWindows = newWindowsList.get(displayId); final List<InputWindowHandle> oldWindows = oldWindowsList.get(displayId); - if (hasWindowsChangedLocked(newWindows, oldWindows)) { + if (hasWindowsChanged(newWindows, oldWindows)) { outDisplayIdsForWindowsChanged.add(displayId); } } } - private boolean hasWindowsChangedLocked(List<InputWindowHandle> newWindows, + @GuardedBy("mLock") + private static boolean hasWindowsChanged(List<InputWindowHandle> newWindows, List<InputWindowHandle> oldWindows) { if (oldWindows == null || oldWindows.size() != newWindows.size()) { return true; @@ -253,34 +330,195 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { return false; } - private void generateMagnificationSpecInverseMatrixLocked(int displayId) { - MagnificationSpec spec = new MagnificationSpec(); - if (!mAccessibilityController.getMagnificationSpecForDisplay(displayId, spec)) { - mMagnificationSpecInverseMatrix.remove(displayId); - return; + @GuardedBy("mLock") + private void findMagnificationSpecInverseMatrixIfNeeded(SparseArray<List<InputWindowHandle>> + windowHandleList) { + MagnificationSpec currentMagnificationSpec; + MagnificationSpec previousMagnificationSpec; + for (int i = 0; i < windowHandleList.size(); i++) { + final int displayId = windowHandleList.keyAt(i); + List<InputWindowHandle> inputWindowHandles = windowHandleList.get(displayId); + + final MagnificationSpec currentSpec = mCurrentMagnificationSpec.get(displayId); + if (currentSpec == null) { + continue; + } + currentMagnificationSpec = new MagnificationSpec(); + currentMagnificationSpec.setTo(currentSpec); + + final MagnificationSpec previousSpec = mPreviousMagnificationSpec.get(displayId); + + if (previousSpec == null) { + final Matrix inverseMatrixForCurrentSpec = new Matrix(); + generateInverseMatrix(currentMagnificationSpec, inverseMatrixForCurrentSpec); + mMagnificationSpecInverseMatrix.put(displayId, inverseMatrixForCurrentSpec); + continue; + } + previousMagnificationSpec = new MagnificationSpec(); + previousMagnificationSpec.setTo(previousSpec); + + generateInverseMatrixBasedOnProperMagnificationSpecForDisplay(inputWindowHandles, + currentMagnificationSpec, previousMagnificationSpec); } - sTempFloats[Matrix.MSCALE_X] = spec.scale; - sTempFloats[Matrix.MSKEW_Y] = 0; - sTempFloats[Matrix.MSKEW_X] = 0; - sTempFloats[Matrix.MSCALE_Y] = spec.scale; - sTempFloats[Matrix.MTRANS_X] = spec.offsetX; - sTempFloats[Matrix.MTRANS_Y] = spec.offsetY; - sTempFloats[Matrix.MPERSP_0] = 0; - sTempFloats[Matrix.MPERSP_1] = 0; - sTempFloats[Matrix.MPERSP_2] = 1; + } - final Matrix tempMatrix = new Matrix(); - tempMatrix.setValues(sTempFloats); + @GuardedBy("mLock") + private void generateInverseMatrixBasedOnProperMagnificationSpecForDisplay( + List<InputWindowHandle> inputWindowHandles, MagnificationSpec currentMagnificationSpec, + MagnificationSpec previousMagnificationSpec) { + // To decrease the counts of holding the WindowManagerService#mGlogalLock in + // the method, getWindowTransformMatrix(), this for loop begins from the bottom + // to top of the z-order windows. + for (int index = inputWindowHandles.size() - 1; index >= 0; index--) { + final Matrix windowTransformMatrix = mTempMatrix2; + final InputWindowHandle windowHandle = inputWindowHandles.get(index); + final IBinder iBinder = windowHandle.getWindow().asBinder(); + + if (getWindowTransformMatrix(iBinder, windowTransformMatrix)) { + generateMagnificationSpecInverseMatrix(windowHandle, currentMagnificationSpec, + previousMagnificationSpec, windowTransformMatrix); + + break; + } + } + } - final Matrix inverseMatrix = new Matrix(); - final boolean result = tempMatrix.invert(inverseMatrix); + @GuardedBy("mLock") + private boolean getWindowTransformMatrix(IBinder iBinder, Matrix outTransform) { + final Matrix windowMatrix = iBinder != null + ? mWindowsTransformMatrixMap.get(iBinder) : null; + + if (windowMatrix == null) { + return false; + } + outTransform.set(windowMatrix); + return true; + } + + /** + * Generates the inverse matrix based on the proper magnification spec. + * The magnification spec associated with the InputWindowHandle might not the current + * spec set by WM, which might be the previous one. To find the appropriate spec, + * we store two consecutive magnification specs, and found out which one is the proper + * one closing the identity matrix for generating the inverse matrix. + * + * @param inputWindowHandle The window from the surface flinger. + * @param currentMagnificationSpec The current magnification spec. + * @param previousMagnificationSpec The previous magnification spec. + * @param transformMatrix The transform matrix of the window doesn't consider the + * magnifying effect. + */ + @GuardedBy("mLock") + private void generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle, + @NonNull MagnificationSpec currentMagnificationSpec, + @NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix) { + + final float[] identityMatrixFloatsForCurrentSpec = mTempFloat1; + computeIdentityMatrix(inputWindowHandle, currentMagnificationSpec, + transformMatrix, identityMatrixFloatsForCurrentSpec); + final float[] identityMatrixFloatsForPreviousSpec = mTempFloat2; + computeIdentityMatrix(inputWindowHandle, previousMagnificationSpec, + transformMatrix, identityMatrixFloatsForPreviousSpec); + + Matrix inverseMatrixForMagnificationSpec = new Matrix(); + if (selectProperMagnificationSpecByComparingIdentityDegree( + identityMatrixFloatsForCurrentSpec, identityMatrixFloatsForPreviousSpec)) { + generateInverseMatrix(currentMagnificationSpec, + inverseMatrixForMagnificationSpec); + + // Choosing the current spec means the previous spec is out of date, + // so removing it. And if the current spec is no magnifying, meaning + // the magnifying is done so removing the inverse matrix of this display. + mPreviousMagnificationSpec.remove(inputWindowHandle.displayId); + if (currentMagnificationSpec.isNop()) { + mCurrentMagnificationSpec.remove(inputWindowHandle.displayId); + mMagnificationSpecInverseMatrix.remove(inputWindowHandle.displayId); + return; + } + } else { + generateInverseMatrix(previousMagnificationSpec, + inverseMatrixForMagnificationSpec); + } + + mMagnificationSpecInverseMatrix.put(inputWindowHandle.displayId, + inverseMatrixForMagnificationSpec); + } + + /** + * Computes the identity matrix for generating the + * inverse matrix based on below formula under window is at the stable state: + * inputWindowHandle#transform * MagnificationSpecMatrix * WindowState#transform + * = IdentityMatrix + */ + @GuardedBy("mLock") + private void computeIdentityMatrix(InputWindowHandle inputWindowHandle, + @NonNull MagnificationSpec magnificationSpec, + Matrix transformMatrix, float[] magnifyMatrixFloats) { + final Matrix specMatrix = mTempMatrix1; + transformMagnificationSpecToMatrix(magnificationSpec, specMatrix); + + final Matrix resultMatrix = new Matrix(inputWindowHandle.transform); + resultMatrix.preConcat(specMatrix); + resultMatrix.preConcat(transformMatrix); + + resultMatrix.getValues(magnifyMatrixFloats); + } + + /** + * @return true if selecting the magnification spec one, otherwise selecting the + * magnification spec two. + */ + @GuardedBy("mLock") + private boolean selectProperMagnificationSpecByComparingIdentityDegree( + float[] magnifyMatrixFloatsForSpecOne, + float[] magnifyMatrixFloatsForSpecTwo) { + final float[] IdentityMatrixValues = mTempFloat3; + Matrix.IDENTITY_MATRIX.getValues(IdentityMatrixValues); + + final float scaleDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] + - magnifyMatrixFloatsForSpecOne[Matrix.MSCALE_X]); + final float scaleDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] + - magnifyMatrixFloatsForSpecTwo[Matrix.MSCALE_X]); + final float offsetXDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] + - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_X]); + final float offsetXDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] + - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_X]); + final float offsetYDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] + - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_Y]); + final float offsetYDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] + - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_Y]); + final float offsetDiffForSpecOne = offsetXDiffForSpecOne + + offsetYDiffForSpecOne; + final float offsetDiffForSpecTwo = offsetXDiffForSpecTwo + + offsetYDiffForSpecTwo; + + return Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) > 0 + || (Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) == 0 + && Float.compare(offsetDiffForSpecTwo, offsetDiffForSpecOne) > 0); + } + + @GuardedBy("mLock") + private static void generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix) { + outMatrix.reset(); + + final Matrix tempMatrix = new Matrix(); + transformMagnificationSpecToMatrix(spec, tempMatrix); + + final boolean result = tempMatrix.invert(outMatrix); if (!result) { Slog.e(TAG, "Can't inverse the magnification spec matrix with the " - + "magnification spec = " + spec + " on the displayId = " + displayId); - return; + + "magnification spec = " + spec); + outMatrix.reset(); } - mMagnificationSpecInverseMatrix.set(displayId, inverseMatrix); + } + + @GuardedBy("mLock") + private static void transformMagnificationSpecToMatrix(MagnificationSpec spec, + Matrix outMatrix) { + outMatrix.reset(); + outMatrix.postScale(spec.scale, spec.scale); + outMatrix.postTranslate(spec.offsetX, spec.offsetY); } private void notifyWindowsChanged(@NonNull List<Integer> displayIdsForWindowsChanged) { @@ -310,6 +548,9 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { mMagnificationSpecInverseMatrix.clear(); mVisibleWindows.clear(); mDisplayInfos.clear(); + mCurrentMagnificationSpec.clear(); + mPreviousMagnificationSpec.clear(); + mWindowsTransformMatrixMap.clear(); mWindowsNotificationEnabled = false; mHandler.removeCallbacksAndMessages(null); } |