diff options
6 files changed, 196 insertions, 31 deletions
diff --git a/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayController.java b/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayController.java index 63c20651fcd5..f97d6af632b0 100644 --- a/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayController.java +++ b/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayController.java @@ -22,6 +22,7 @@ import static android.view.WindowManager.LayoutParams.TYPE_SCREENSHOT; import static com.android.internal.config.sysui.SystemUiDeviceConfigFlags.CLIPBOARD_OVERLAY_SHOW_ACTIONS; import static com.android.internal.config.sysui.SystemUiDeviceConfigFlags.CLIPBOARD_OVERLAY_SHOW_EDIT_BUTTON; +import static com.android.systemui.clipboardoverlay.ClipboardOverlayEvent.CLIPBOARD_OVERLAY_ACTION_SHOWN; import static com.android.systemui.clipboardoverlay.ClipboardOverlayEvent.CLIPBOARD_OVERLAY_ACTION_TAPPED; import static com.android.systemui.clipboardoverlay.ClipboardOverlayEvent.CLIPBOARD_OVERLAY_DISMISSED_OTHER; import static com.android.systemui.clipboardoverlay.ClipboardOverlayEvent.CLIPBOARD_OVERLAY_DISMISS_TAPPED; @@ -41,7 +42,6 @@ import android.app.RemoteAction; import android.content.BroadcastReceiver; import android.content.ClipData; import android.content.ClipDescription; -import android.content.ComponentName; import android.content.ContentResolver; import android.content.Context; import android.content.Intent; @@ -51,7 +51,6 @@ import android.graphics.Bitmap; import android.hardware.display.DisplayManager; import android.hardware.input.InputManager; import android.net.Uri; -import android.os.AsyncTask; import android.os.Looper; import android.provider.DeviceConfig; import android.text.TextUtils; @@ -62,10 +61,6 @@ import android.view.InputEvent; import android.view.InputEventReceiver; import android.view.InputMonitor; import android.view.MotionEvent; -import android.view.textclassifier.TextClassification; -import android.view.textclassifier.TextClassificationManager; -import android.view.textclassifier.TextClassifier; -import android.view.textclassifier.TextLinks; import androidx.annotation.NonNull; @@ -74,12 +69,13 @@ import com.android.systemui.R; import com.android.systemui.broadcast.BroadcastDispatcher; import com.android.systemui.broadcast.BroadcastSender; import com.android.systemui.clipboardoverlay.dagger.ClipboardOverlayModule.OverlayWindowContext; +import com.android.systemui.dagger.qualifiers.Background; import com.android.systemui.flags.FeatureFlags; import com.android.systemui.screenshot.TimeoutHandler; import java.io.IOException; -import java.util.ArrayList; import java.util.Optional; +import java.util.concurrent.Executor; import javax.inject.Inject; @@ -102,9 +98,9 @@ public class ClipboardOverlayController implements ClipboardListener.ClipboardOv private final DisplayManager mDisplayManager; private final ClipboardOverlayWindow mWindow; private final TimeoutHandler mTimeoutHandler; - private final TextClassifier mTextClassifier; private final ClipboardOverlayUtils mClipboardUtils; private final FeatureFlags mFeatureFlags; + private final Executor mBgExecutor; private final ClipboardOverlayView mView; @@ -189,6 +185,7 @@ public class ClipboardOverlayController implements ClipboardListener.ClipboardOv TimeoutHandler timeoutHandler, FeatureFlags featureFlags, ClipboardOverlayUtils clipboardUtils, + @Background Executor bgExecutor, UiEventLogger uiEventLogger) { mBroadcastDispatcher = broadcastDispatcher; mDisplayManager = requireNonNull(context.getSystemService(DisplayManager.class)); @@ -204,14 +201,12 @@ public class ClipboardOverlayController implements ClipboardListener.ClipboardOv hideImmediate(); }); - mTextClassifier = requireNonNull(context.getSystemService(TextClassificationManager.class)) - .getTextClassifier(); - mTimeoutHandler = timeoutHandler; mTimeoutHandler.setDefaultTimeoutMillis(CLIPBOARD_DEFAULT_TIMEOUT_MILLIS); mFeatureFlags = featureFlags; mClipboardUtils = clipboardUtils; + mBgExecutor = bgExecutor; mView.setCallbacks(mClipboardCallbacks); @@ -281,7 +276,7 @@ public class ClipboardOverlayController implements ClipboardListener.ClipboardOv if (isRemote || DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_SYSTEMUI, CLIPBOARD_OVERLAY_SHOW_ACTIONS, false)) { if (item.getTextLinks() != null) { - AsyncTask.execute(() -> classifyText(clipData.getItemAt(0), clipSource)); + classifyText(clipData.getItemAt(0), clipSource); } } if (isSensitive) { @@ -338,22 +333,18 @@ public class ClipboardOverlayController implements ClipboardListener.ClipboardOv } private void classifyText(ClipData.Item item, String source) { - ArrayList<RemoteAction> actions = new ArrayList<>(); - for (TextLinks.TextLink link : item.getTextLinks().getLinks()) { - TextClassification classification = mTextClassifier.classifyText( - item.getText(), link.getStart(), link.getEnd(), null); - actions.addAll(classification.getActions()); - } - mView.post(() -> { - Optional<RemoteAction> action = actions.stream().filter(remoteAction -> { - ComponentName component = remoteAction.getActionIntent().getIntent().getComponent(); - return component != null && !TextUtils.equals(source, component.getPackageName()); - }).findFirst(); - mView.resetActionChips(); - action.ifPresent(remoteAction -> mView.setActionChip(remoteAction, () -> { - mClipboardLogger.logSessionComplete(CLIPBOARD_OVERLAY_ACTION_TAPPED); - animateOut(); - })); + mBgExecutor.execute(() -> { + Optional<RemoteAction> action = mClipboardUtils.getAction(item, source); + mView.post(() -> { + mView.resetActionChips(); + action.ifPresent(remoteAction -> { + mView.setActionChip(remoteAction, () -> { + mClipboardLogger.logSessionComplete(CLIPBOARD_OVERLAY_ACTION_TAPPED); + animateOut(); + }); + mClipboardLogger.logUnguarded(CLIPBOARD_OVERLAY_ACTION_SHOWN); + }); + }); }); } @@ -539,6 +530,10 @@ public class ClipboardOverlayController implements ClipboardListener.ClipboardOv mClipSource = clipSource; } + void logUnguarded(@NonNull UiEventLogger.UiEventEnum event) { + mUiEventLogger.log(event, 0, mClipSource); + } + void logSessionComplete(@NonNull UiEventLogger.UiEventEnum event) { if (!mGuarded) { mGuarded = true; diff --git a/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayEvent.java b/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayEvent.java index a0b2ab99e240..9917507ec3bf 100644 --- a/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayEvent.java +++ b/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayEvent.java @@ -28,6 +28,8 @@ public enum ClipboardOverlayEvent implements UiEventLogger.UiEventEnum { CLIPBOARD_OVERLAY_EDIT_TAPPED(951), @UiEvent(doc = "clipboard share tapped") CLIPBOARD_OVERLAY_SHARE_TAPPED(1067), + @UiEvent(doc = "clipboard smart action shown") + CLIPBOARD_OVERLAY_ACTION_SHOWN(1260), @UiEvent(doc = "clipboard action tapped") CLIPBOARD_OVERLAY_ACTION_TAPPED(952), @UiEvent(doc = "clipboard remote copy tapped") diff --git a/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayUtils.java b/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayUtils.java index c194e66f8e92..785e4a0743e4 100644 --- a/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayUtils.java +++ b/packages/SystemUI/src/com/android/systemui/clipboardoverlay/ClipboardOverlayUtils.java @@ -16,22 +16,34 @@ package com.android.systemui.clipboardoverlay; +import android.app.RemoteAction; import android.content.ClipData; import android.content.ClipDescription; import android.content.ComponentName; import android.content.Context; import android.os.Build; import android.provider.DeviceConfig; +import android.text.TextUtils; +import android.view.textclassifier.TextClassification; +import android.view.textclassifier.TextClassificationManager; +import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextLinks; import com.android.internal.config.sysui.SystemUiDeviceConfigFlags; import com.android.systemui.R; +import java.util.ArrayList; +import java.util.Optional; + import javax.inject.Inject; class ClipboardOverlayUtils { + private final TextClassifier mTextClassifier; + @Inject - ClipboardOverlayUtils() { + ClipboardOverlayUtils(TextClassificationManager textClassificationManager) { + mTextClassifier = textClassificationManager.getTextClassifier(); } boolean isRemoteCopy(Context context, ClipData clipData, String clipSource) { @@ -52,4 +64,21 @@ class ClipboardOverlayUtils { } return false; } + + public Optional<RemoteAction> getAction(ClipData.Item item, String source) { + return getActions(item).stream().filter(remoteAction -> { + ComponentName component = remoteAction.getActionIntent().getIntent().getComponent(); + return component != null && !TextUtils.equals(source, component.getPackageName()); + }).findFirst(); + } + + private ArrayList<RemoteAction> getActions(ClipData.Item item) { + ArrayList<RemoteAction> actions = new ArrayList<>(); + for (TextLinks.TextLink link : item.getTextLinks().getLinks()) { + TextClassification classification = mTextClassifier.classifyText( + item.getText(), link.getStart(), link.getEnd(), null); + actions.addAll(classification.getActions()); + } + return actions; + } } diff --git a/packages/SystemUI/src/com/android/systemui/dagger/FrameworkServicesModule.java b/packages/SystemUI/src/com/android/systemui/dagger/FrameworkServicesModule.java index ccb9557d757b..f902252fa2e6 100644 --- a/packages/SystemUI/src/com/android/systemui/dagger/FrameworkServicesModule.java +++ b/packages/SystemUI/src/com/android/systemui/dagger/FrameworkServicesModule.java @@ -92,6 +92,7 @@ import android.view.WindowManagerGlobal; import android.view.accessibility.AccessibilityManager; import android.view.accessibility.CaptioningManager; import android.view.inputmethod.InputMethodManager; +import android.view.textclassifier.TextClassificationManager; import androidx.core.app.NotificationManagerCompat; @@ -631,4 +632,10 @@ public class FrameworkServicesModule { static BluetoothAdapter provideBluetoothAdapter(BluetoothManager bluetoothManager) { return bluetoothManager.getAdapter(); } + + @Provides + @Singleton + static TextClassificationManager provideTextClassificationManager(Context context) { + return context.getSystemService(TextClassificationManager.class); + } } diff --git a/packages/SystemUI/tests/src/com/android/systemui/clipboardoverlay/ClipboardOverlayControllerTest.java b/packages/SystemUI/tests/src/com/android/systemui/clipboardoverlay/ClipboardOverlayControllerTest.java index d6e621f1f135..b4e85c06933a 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/clipboardoverlay/ClipboardOverlayControllerTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/clipboardoverlay/ClipboardOverlayControllerTest.java @@ -16,12 +16,14 @@ package com.android.systemui.clipboardoverlay; +import static com.android.systemui.clipboardoverlay.ClipboardOverlayEvent.CLIPBOARD_OVERLAY_ACTION_SHOWN; import static com.android.systemui.clipboardoverlay.ClipboardOverlayEvent.CLIPBOARD_OVERLAY_DISMISS_TAPPED; import static com.android.systemui.clipboardoverlay.ClipboardOverlayEvent.CLIPBOARD_OVERLAY_SHARE_TAPPED; import static com.android.systemui.clipboardoverlay.ClipboardOverlayEvent.CLIPBOARD_OVERLAY_SWIPE_DISMISSED; import static com.android.systemui.flags.Flags.CLIPBOARD_REMOTE_BEHAVIOR; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -29,10 +31,13 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import android.animation.Animator; +import android.app.RemoteAction; import android.content.ClipData; import android.content.ClipDescription; +import android.content.Context; import android.net.Uri; import android.os.PersistableBundle; +import android.view.textclassifier.TextLinks; import androidx.test.filters.SmallTest; import androidx.test.runner.AndroidJUnit4; @@ -42,6 +47,8 @@ import com.android.systemui.SysuiTestCase; import com.android.systemui.broadcast.BroadcastSender; import com.android.systemui.flags.FakeFeatureFlags; import com.android.systemui.screenshot.TimeoutHandler; +import com.android.systemui.util.concurrency.FakeExecutor; +import com.android.systemui.util.time.FakeSystemClock; import org.junit.After; import org.junit.Before; @@ -50,7 +57,12 @@ import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.util.Optional; @SmallTest @RunWith(AndroidJUnit4.class) @@ -80,6 +92,8 @@ public class ClipboardOverlayControllerTest extends SysuiTestCase { private ArgumentCaptor<ClipboardOverlayView.ClipboardOverlayCallbacks> mOverlayCallbacksCaptor; private ClipboardOverlayView.ClipboardOverlayCallbacks mCallbacks; + private FakeExecutor mExecutor = new FakeExecutor(new FakeSystemClock()); + @Before public void setup() { MockitoAnnotations.initMocks(this); @@ -101,6 +115,7 @@ public class ClipboardOverlayControllerTest extends SysuiTestCase { mTimeoutHandler, mFeatureFlags, mClipboardUtils, + mExecutor, mUiEventLogger); verify(mClipboardOverlayView).setCallbacks(mOverlayCallbacksCaptor.capture()); mCallbacks = mOverlayCallbacksCaptor.getValue(); @@ -237,4 +252,29 @@ public class ClipboardOverlayControllerTest extends SysuiTestCase { verify(mUiEventLogger).log(CLIPBOARD_OVERLAY_DISMISS_TAPPED, 0, "second.package"); verifyNoMoreInteractions(mUiEventLogger); } + + @Test + public void test_logOnClipboardActionsShown() { + ClipData.Item item = mSampleClipData.getItemAt(0); + item.setTextLinks(Mockito.mock(TextLinks.class)); + mFeatureFlags.set(CLIPBOARD_REMOTE_BEHAVIOR, true); + when(mClipboardUtils.isRemoteCopy(any(Context.class), any(ClipData.class), anyString())) + .thenReturn(true); + when(mClipboardUtils.getAction(any(ClipData.Item.class), anyString())) + .thenReturn(Optional.of(Mockito.mock(RemoteAction.class))); + when(mClipboardOverlayView.post(any(Runnable.class))).thenAnswer(new Answer<Object>() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + ((Runnable) invocation.getArgument(0)).run(); + return null; + } + }); + + mOverlayController.setClipData( + new ClipData(mSampleClipData.getDescription(), item), "actionShownSource"); + mExecutor.runAllReady(); + + verify(mUiEventLogger).log(CLIPBOARD_OVERLAY_ACTION_SHOWN, 0, "actionShownSource"); + verifyNoMoreInteractions(mUiEventLogger); + } } diff --git a/packages/SystemUI/tests/src/com/android/systemui/clipboardoverlay/ClipboardOverlayUtilsTest.java b/packages/SystemUI/tests/src/com/android/systemui/clipboardoverlay/ClipboardOverlayUtilsTest.java index 09b1699d3ffc..aea6be3d468b 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/clipboardoverlay/ClipboardOverlayUtilsTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/clipboardoverlay/ClipboardOverlayUtilsTest.java @@ -16,13 +16,24 @@ package com.android.systemui.clipboardoverlay; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.when; +import android.app.RemoteAction; import android.content.ClipData; import android.content.ClipDescription; import android.os.PersistableBundle; import android.testing.TestableResources; +import android.util.ArrayMap; +import android.view.textclassifier.TextClassification; +import android.view.textclassifier.TextClassificationManager; +import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextLinks; import androidx.test.filters.SmallTest; import androidx.test.runner.AndroidJUnit4; @@ -30,19 +41,84 @@ import androidx.test.runner.AndroidJUnit4; import com.android.systemui.R; import com.android.systemui.SysuiTestCase; +import com.google.android.collect.Lists; + import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.util.Map; +import java.util.Optional; @SmallTest @RunWith(AndroidJUnit4.class) public class ClipboardOverlayUtilsTest extends SysuiTestCase { private ClipboardOverlayUtils mClipboardUtils; + @Mock + private TextClassificationManager mTextClassificationManager; + @Mock + private TextClassifier mTextClassifier; + + @Mock + private ClipData.Item mClipDataItem; @Before public void setUp() { - mClipboardUtils = new ClipboardOverlayUtils(); + MockitoAnnotations.initMocks(this); + + when(mTextClassificationManager.getTextClassifier()).thenReturn(mTextClassifier); + mClipboardUtils = new ClipboardOverlayUtils(mTextClassificationManager); + } + + @Test + public void test_getAction_noLinks_returnsEmptyOptional() { + ClipData.Item item = new ClipData.Item("no text links"); + item.setTextLinks(Mockito.mock(TextLinks.class)); + + Optional<RemoteAction> action = mClipboardUtils.getAction(item, ""); + + assertTrue(action.isEmpty()); + } + + @Test + public void test_getAction_returnsFirstLink() { + when(mClipDataItem.getTextLinks()).thenReturn(getFakeTextLinks()); + when(mClipDataItem.getText()).thenReturn(""); + RemoteAction actionA = constructRemoteAction("abc"); + RemoteAction actionB = constructRemoteAction("def"); + TextClassification classificationA = Mockito.mock(TextClassification.class); + when(classificationA.getActions()).thenReturn(Lists.newArrayList(actionA)); + TextClassification classificationB = Mockito.mock(TextClassification.class); + when(classificationB.getActions()).thenReturn(Lists.newArrayList(actionB)); + when(mTextClassifier.classifyText(anyString(), anyInt(), anyInt(), isNull())).thenReturn( + classificationA, classificationB); + + RemoteAction result = mClipboardUtils.getAction(mClipDataItem, "def").orElse(null); + + assertEquals(actionA, result); + } + + @Test + public void test_getAction_skipsMatchingComponent() { + when(mClipDataItem.getTextLinks()).thenReturn(getFakeTextLinks()); + when(mClipDataItem.getText()).thenReturn(""); + RemoteAction actionA = constructRemoteAction("abc"); + RemoteAction actionB = constructRemoteAction("def"); + TextClassification classificationA = Mockito.mock(TextClassification.class); + when(classificationA.getActions()).thenReturn(Lists.newArrayList(actionA)); + TextClassification classificationB = Mockito.mock(TextClassification.class); + when(classificationB.getActions()).thenReturn(Lists.newArrayList(actionB)); + when(mTextClassifier.classifyText(anyString(), anyInt(), anyInt(), isNull())).thenReturn( + classificationA, classificationB); + + RemoteAction result = mClipboardUtils.getAction(mClipDataItem, "abc").orElse(null); + + assertEquals(actionB, result); } @Test @@ -92,7 +168,7 @@ public class ClipboardOverlayUtilsTest extends SysuiTestCase { assertFalse(mClipboardUtils.isRemoteCopy(mContext, data, "")); } - static ClipData constructClipData(String[] mimeTypes, ClipData.Item item, + private static ClipData constructClipData(String[] mimeTypes, ClipData.Item item, PersistableBundle extras) { ClipDescription description = new ClipDescription("Test", mimeTypes); if (extras != null) { @@ -100,4 +176,20 @@ public class ClipboardOverlayUtilsTest extends SysuiTestCase { } return new ClipData(description, item); } + + private static RemoteAction constructRemoteAction(String packageName) { + RemoteAction action = Mockito.mock(RemoteAction.class, Answers.RETURNS_DEEP_STUBS); + when(action.getActionIntent().getIntent().getComponent().getPackageName()) + .thenReturn(packageName); + return action; + } + + private static TextLinks getFakeTextLinks() { + TextLinks.Builder textLinks = new TextLinks.Builder("test"); + final Map<String, Float> scores = new ArrayMap<>(); + scores.put(TextClassifier.TYPE_EMAIL, 1f); + textLinks.addLink(0, 0, scores); + textLinks.addLink(0, 0, scores); + return textLinks.build(); + } } |