diff options
4 files changed, 208 insertions, 17 deletions
diff --git a/core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java b/core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java index b4cd145ca374..3c5eba78174f 100644 --- a/core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java +++ b/core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java @@ -428,6 +428,17 @@ public final class SystemUiDeviceConfigFlags { */ public static final String SCREENSHOT_KEYCHORD_DELAY = "screenshot_keychord_delay"; + /** + * (boolean) Whether to use an ML model for the Back Gesture. + */ + public static final String USE_BACK_GESTURE_ML_MODEL = "use_back_gesture_ml_model"; + + /** + * (float) Threshold for Back Gesture ML model prediction. + */ + public static final String BACK_GESTURE_ML_MODEL_THRESHOLD = "back_gesture_ml_model_threshold"; + + private SystemUiDeviceConfigFlags() { } } diff --git a/packages/SystemUI/src/com/android/systemui/SystemUIFactory.java b/packages/SystemUI/src/com/android/systemui/SystemUIFactory.java index 5674fdd3bb36..4a77de2850dd 100644 --- a/packages/SystemUI/src/com/android/systemui/SystemUIFactory.java +++ b/packages/SystemUI/src/com/android/systemui/SystemUIFactory.java @@ -18,6 +18,7 @@ package com.android.systemui; import android.annotation.NonNull; import android.content.Context; +import android.content.res.AssetManager; import android.content.res.Resources; import android.os.Handler; import android.os.Looper; @@ -39,6 +40,7 @@ import com.android.systemui.screenshot.ScreenshotNotificationSmartActionsProvide import com.android.systemui.statusbar.NotificationListener; import com.android.systemui.statusbar.NotificationMediaManager; import com.android.systemui.statusbar.notification.NotificationWakeUpCoordinator; +import com.android.systemui.statusbar.phone.BackGestureTfClassifierProvider; import com.android.systemui.statusbar.phone.DozeParameters; import com.android.systemui.statusbar.phone.KeyguardBouncer; import com.android.systemui.statusbar.phone.KeyguardBypassController; @@ -182,4 +184,13 @@ public class SystemUIFactory { return mContext; } } + + /** + * Creates an instance of BackGestureTfClassifierProvider. + * This method is overridden in vendor specific implementation of Sys UI. + */ + public BackGestureTfClassifierProvider createBackGestureTfClassifierProvider( + AssetManager am) { + return new BackGestureTfClassifierProvider(); + } } diff --git a/packages/SystemUI/src/com/android/systemui/statusbar/phone/BackGestureTfClassifierProvider.java b/packages/SystemUI/src/com/android/systemui/statusbar/phone/BackGestureTfClassifierProvider.java new file mode 100644 index 000000000000..0af79c3886b9 --- /dev/null +++ b/packages/SystemUI/src/com/android/systemui/statusbar/phone/BackGestureTfClassifierProvider.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.systemui.statusbar.phone; + +import android.content.res.AssetManager; + +import java.util.HashMap; +import java.util.Map; + +/** + * This class can be overridden by a vendor-specific sys UI implementation, + * in order to provide classification models for the Back Gesture. + */ +public class BackGestureTfClassifierProvider { + private static final String TAG = "BackGestureTfClassifierProvider"; + + /** + * Default implementation that returns an empty map. + * This method is overridden in vendor-specific Sys UI implementation. + * + * @param am An AssetManager to get the vocab file. + */ + public Map<String, Integer> loadVocab(AssetManager am) { + return new HashMap<String, Integer>(); + } + + /** + * This method is overridden in vendor-specific Sys UI implementation. + * + * @param featuresVector List of input features. + * + */ + public float predict(Object[] featuresVector) { + return -1; + } + + /** + * Interpreter owns resources. This method releases the resources after + * use to avoid memory leak. + * This method is overridden in vendor-specific Sys UI implementation. + * + */ + public void release() {} + + /** + * Returns whether to use the ML model for Back Gesture. + * This method is overridden in vendor-specific Sys UI implementation. + * + */ + public boolean isActive() { + return false; + } +} diff --git a/packages/SystemUI/src/com/android/systemui/statusbar/phone/EdgeBackGestureHandler.java b/packages/SystemUI/src/com/android/systemui/statusbar/phone/EdgeBackGestureHandler.java index 00a932cb1e8a..2641bca04aea 100644 --- a/packages/SystemUI/src/com/android/systemui/statusbar/phone/EdgeBackGestureHandler.java +++ b/packages/SystemUI/src/com/android/systemui/statusbar/phone/EdgeBackGestureHandler.java @@ -56,6 +56,7 @@ import com.android.internal.config.sysui.SystemUiDeviceConfigFlags; import com.android.internal.policy.GestureNavigationSettingsObserver; import com.android.systemui.Dependency; import com.android.systemui.R; +import com.android.systemui.SystemUIFactory; import com.android.systemui.broadcast.BroadcastDispatcher; import com.android.systemui.bubbles.BubbleController; import com.android.systemui.model.SysUiState; @@ -76,6 +77,7 @@ import com.android.systemui.tracing.nano.SystemUiTraceProto; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.Executor; /** @@ -117,8 +119,31 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa public void onTaskStackChanged() { mGestureBlockingActivityRunning = isGestureBlockingActivityRunning(); } + @Override + public void onTaskCreated(int taskId, ComponentName componentName) { + if (componentName != null) { + mPackageName = componentName.getPackageName(); + } else { + mPackageName = "_UNKNOWN"; + } + } }; + private DeviceConfig.OnPropertiesChangedListener mOnPropertiesChangedListener = + new DeviceConfig.OnPropertiesChangedListener() { + @Override + public void onPropertiesChanged(DeviceConfig.Properties properties) { + if (DeviceConfig.NAMESPACE_SYSTEMUI.equals(properties.getNamespace()) + && (properties.getKeyset().contains( + SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_THRESHOLD) + || properties.getKeyset().contains( + SystemUiDeviceConfigFlags.USE_BACK_GESTURE_ML_MODEL))) { + updateMLModelState(); + } + } + }; + + private final Context mContext; private final OverviewProxyService mOverviewProxyService; private final Runnable mStateChangeCallback; @@ -173,6 +198,13 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa private int mRightInset; private int mSysUiFlags; + // For Tf-Lite model. + private BackGestureTfClassifierProvider mBackGestureTfClassifierProvider; + private Map<String, Integer> mVocab; + private boolean mUseMLModel; + private float mMLModelThreshold; + private String mPackageName; + private final GestureNavigationSettingsObserver mGestureNavigationSettingsObserver; private final NavigationEdgeBackPlugin.BackCallback mBackCallback = @@ -230,7 +262,6 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa Log.e(TAG, "Failed to add gesture blocking activities", e); } } - mLongPressTimeout = Math.min(MAX_LONG_PRESS_TIMEOUT, ViewConfiguration.getLongPressTimeout()); @@ -344,6 +375,7 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa mContext.getSystemService(DisplayManager.class).unregisterDisplayListener(this); mPluginManager.removePluginListener(this); ActivityManagerWrapper.getInstance().unregisterTaskStackListener(mTaskStackListener); + DeviceConfig.removeOnPropertiesChangedListener(mOnPropertiesChangedListener); try { WindowManagerGlobal.getWindowManagerService() @@ -359,6 +391,9 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa mContext.getSystemService(DisplayManager.class).registerDisplayListener(this, mContext.getMainThreadHandler()); ActivityManagerWrapper.getInstance().registerTaskStackListener(mTaskStackListener); + DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_SYSTEMUI, + runnable -> (mContext.getMainThreadHandler()).post(runnable), + mOnPropertiesChangedListener); try { WindowManagerGlobal.getWindowManagerService() @@ -379,6 +414,8 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa mPluginManager.addPluginListener( this, NavigationEdgeBackPlugin.class, /*allowMultiple=*/ false); } + // Update the ML model resources. + updateMLModelState(); } @Override @@ -431,26 +468,87 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa } } - private boolean isWithinTouchRegion(int x, int y) { - // Disallow if we are in the bottom gesture area - if (y >= (mDisplaySize.y - mBottomGestureHeight)) { - return false; + private void updateMLModelState() { + boolean newState = mIsEnabled && DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_SYSTEMUI, + SystemUiDeviceConfigFlags.USE_BACK_GESTURE_ML_MODEL, false); + + if (newState == mUseMLModel) { + return; } - // If the point is way too far (twice the margin), it is - // not interesting to us for logging purposes, nor we - // should process it. Simply return false and keep - // mLogGesture = false. - if (x > 2 * (mEdgeWidthLeft + mLeftInset) - && x < (mDisplaySize.x - 2 * (mEdgeWidthRight + mRightInset))) { - return false; + if (newState) { + mBackGestureTfClassifierProvider = SystemUIFactory.getInstance() + .createBackGestureTfClassifierProvider(mContext.getAssets()); + mMLModelThreshold = DeviceConfig.getFloat(DeviceConfig.NAMESPACE_SYSTEMUI, + SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_THRESHOLD, 0.9f); + if (mBackGestureTfClassifierProvider.isActive()) { + mVocab = mBackGestureTfClassifierProvider.loadVocab(mContext.getAssets()); + mUseMLModel = true; + return; + } } - // Denotes whether we should proceed with the gesture. - // Even if it is false, we may want to log it assuming - // it is not invalid due to exclusion. - boolean withinRange = x <= mEdgeWidthLeft + mLeftInset - || x >= (mDisplaySize.x - mEdgeWidthRight - mRightInset); + mUseMLModel = false; + if (mBackGestureTfClassifierProvider != null) { + mBackGestureTfClassifierProvider.release(); + mBackGestureTfClassifierProvider = null; + } + } + + private float getBackGesturePredictionsCategory(int x, int y) { + if (!mVocab.containsKey(mPackageName)) { + return -1; + } + + int distanceFromEdge; + int location; + if (x <= mDisplaySize.x / 2.0) { + location = 1; // left + distanceFromEdge = x; + } else { + location = 2; // right + distanceFromEdge = mDisplaySize.x - x; + } + + Object[] featuresVector = { + new long[]{(long) mDisplaySize.x}, + new long[]{(long) distanceFromEdge}, + new long[]{(long) location}, + new long[]{(long) mVocab.get(mPackageName)}, + new long[]{(long) y}, + }; + + final float results = mBackGestureTfClassifierProvider.predict(featuresVector); + if (results == -1) return -1; + + return results >= mMLModelThreshold ? 1 : 0; + } + + private boolean isWithinTouchRegion(int x, int y) { + boolean withinRange = false; + float results = -1; + + if (mUseMLModel && (results = getBackGesturePredictionsCategory(x, y)) != -1) { + withinRange = results == 1 ? true : false; + } else { + // Disallow if we are in the bottom gesture area + if (y >= (mDisplaySize.y - mBottomGestureHeight)) { + return false; + } + // If the point is way too far (twice the margin), it is + // not interesting to us for logging purposes, nor we + // should process it. Simply return false and keep + // mLogGesture = false. + if (x > 2 * (mEdgeWidthLeft + mLeftInset) + && x < (mDisplaySize.x - 2 * (mEdgeWidthRight + mRightInset))) { + return false; + } + // Denotes whether we should proceed with the gesture. + // Even if it is false, we may want to log it assuming + // it is not invalid due to exclusion. + withinRange = x <= mEdgeWidthLeft + mLeftInset + || x >= (mDisplaySize.x - mEdgeWidthRight - mRightInset); + } // Always allow if the user is in a transient sticky immersive state if (mIsNavBarShownTransiently) { @@ -648,6 +746,11 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa ActivityManager.RunningTaskInfo runningTask = ActivityManagerWrapper.getInstance().getRunningTask(); ComponentName topActivity = runningTask == null ? null : runningTask.topActivity; + if (topActivity != null) { + mPackageName = topActivity.getPackageName(); + } else { + mPackageName = "_UNKNOWN"; + } return topActivity != null && mGestureBlockingActivities.contains(topActivity); } |