From e94e0787db1b5b8ce5fe533634bd116a063faef1 Mon Sep 17 00:00:00 2001 From: Tony Mak Date: Fri, 14 Dec 2018 11:57:54 +0800 Subject: Send logs to TextClassifier by calling onTextClassifierEvent in NAS Whenever NAS called TextClassifier.suggestConversationActions, it will cache the notification key to result id mapping. The result id will be used to log subsequent events related to these suggestions. This change should allow us to collect CTR. TODO: Log the coverage, i.e. among all suggestConversationActions request, how many of them actually contains some suggestions. BUG: 120803809 Test: atest SmartActionHelperTest Test: Manual, add a log in TextClassifierImpl.onTextClassifierEvent 1. Send a message notification 2. Expand the notification, observe event is logged. 4. Clicked on one of the replies, observe event is logged. 5. Send another message to myself 6. Inline reply it, observe event is logged. Change-Id: I590d9bfcdb7ae7ee7976740d71bf7f1204683939 --- .../notification/NotificationAssistantService.java | 13 +- .../textclassifier/ActionsSuggestionsHelper.java | 27 +++ .../view/textclassifier/TextClassifierEvent.java | 22 +++ .../view/textclassifier/TextClassifierImpl.java | 18 +- .../ext/services/notification/Assistant.java | 39 +++-- .../services/notification/NotificationEntry.java | 18 ++ .../services/notification/SmartActionsHelper.java | 193 +++++++++++++++++---- .../notification/SmartActionHelperTest.java | 145 +++++++++++++++- 8 files changed, 412 insertions(+), 63 deletions(-) diff --git a/core/java/android/service/notification/NotificationAssistantService.java b/core/java/android/service/notification/NotificationAssistantService.java index c850a4e0f815..dd0242ae2dc8 100644 --- a/core/java/android/service/notification/NotificationAssistantService.java +++ b/core/java/android/service/notification/NotificationAssistantService.java @@ -19,7 +19,7 @@ package android.service.notification; import static java.lang.annotation.RetentionPolicy.SOURCE; import android.annotation.IntDef; -import android.annotation.Nullable; +import android.annotation.NonNull; import android.annotation.SdkConstant; import android.annotation.SystemApi; import android.annotation.TestApi; @@ -179,13 +179,13 @@ public abstract class NotificationAssistantService extends NotificationListenerS * @param isExpanded whether the notification is expanded. */ public void onNotificationExpansionChanged( - String key, boolean isUserAction, boolean isExpanded) {} + @NonNull String key, boolean isUserAction, boolean isExpanded) {} /** * Implement this to know when a direct reply is sent from a notification. * @param key the notification key */ - public void onNotificationDirectReply(String key) {} + public void onNotificationDirectReply(@NonNull String key) {} /** * Implement this to know when a suggested reply is sent. @@ -193,7 +193,9 @@ public abstract class NotificationAssistantService extends NotificationListenerS * @param reply the reply that is just sent * @param source the source that provided the reply, e.g. SOURCE_FROM_APP */ - public void onSuggestedReplySent(String key, CharSequence reply, @Source int source) {} + public void onSuggestedReplySent(@NonNull String key, @NonNull CharSequence reply, + @Source int source) { + } /** * Implement this to know when an action is clicked. @@ -201,7 +203,8 @@ public abstract class NotificationAssistantService extends NotificationListenerS * @param action the action that is just clicked * @param source the source that provided the action, e.g. SOURCE_FROM_APP */ - public void onActionClicked(String key, @Nullable Notification.Action action, int source) { + public void onActionClicked(@NonNull String key, @NonNull Notification.Action action, + @Source int source) { } /** diff --git a/core/java/android/view/textclassifier/ActionsSuggestionsHelper.java b/core/java/android/view/textclassifier/ActionsSuggestionsHelper.java index b41096c74bf7..77cb4cd28763 100644 --- a/core/java/android/view/textclassifier/ActionsSuggestionsHelper.java +++ b/core/java/android/view/textclassifier/ActionsSuggestionsHelper.java @@ -17,6 +17,7 @@ package android.view.textclassifier; import android.app.Person; +import android.content.Context; import android.text.TextUtils; import android.util.ArrayMap; @@ -28,7 +29,10 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; import java.util.function.Function; import java.util.stream.Collectors; @@ -84,6 +88,29 @@ public final class ActionsSuggestionsHelper { new ActionsSuggestionsModel.ConversationMessage[nativeMessages.size()]); } + /** + * Returns the result id for logging. + */ + public static String createResultId( + Context context, + List messages, + int modelVersion, + List modelLocales) { + final StringJoiner localesJoiner = new StringJoiner(","); + for (Locale locale : modelLocales) { + localesJoiner.add(locale.toLanguageTag()); + } + final String modelName = String.format( + Locale.US, "%s_v%d", localesJoiner.toString(), modelVersion); + final int hash = Objects.hash( + messages.stream() + .map(ConversationActions.Message::getText) + .collect(Collectors.toList()), + context.getPackageName()); + return SelectionSessionLogger.SignatureParser.createSignature( + SelectionSessionLogger.CLASSIFIER_ID, modelName, hash); + } + private static final class PersonEncoder { private final Map mMapping = new ArrayMap<>(); private int mNextUserId = FIRST_NON_LOCAL_USER; diff --git a/core/java/android/view/textclassifier/TextClassifierEvent.java b/core/java/android/view/textclassifier/TextClassifierEvent.java index 3bb9ee88dae9..f2fea02171f9 100644 --- a/core/java/android/view/textclassifier/TextClassifierEvent.java +++ b/core/java/android/view/textclassifier/TextClassifierEvent.java @@ -27,6 +27,7 @@ import com.android.internal.util.Preconditions; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.Arrays; /** * A text classifier event. @@ -498,4 +499,25 @@ public final class TextClassifierEvent implements Parcelable { } // TODO: Add build(boolean validate). } + + @Override + public String toString() { + StringBuilder out = new StringBuilder(128); + out.append("TextClassifierEvent{"); + out.append("mEventCategory=").append(mEventCategory); + out.append(", mEventType=").append(mEventType); + out.append(", mEventContext=").append(mEventContext); + out.append(", mResultId=").append(mResultId); + out.append(", mEventIndex=").append(mEventIndex); + out.append(", mEventTime=").append(mEventTime); + out.append(", mExtras=").append(mExtras); + out.append(", mRelativeWordStartIndex=").append(mRelativeWordStartIndex); + out.append(", mRelativeWordEndIndex=").append(mRelativeWordEndIndex); + out.append(", mRelativeSuggestedWordStartIndex=").append(mRelativeSuggestedWordStartIndex); + out.append(", mRelativeSuggestedWordEndIndex=").append(mRelativeSuggestedWordEndIndex); + out.append(", mActionIndices=").append(Arrays.toString(mActionIndices)); + out.append(", mLanguage=").append(mLanguage); + out.append("}"); + return out.toString(); + } } diff --git a/core/java/android/view/textclassifier/TextClassifierImpl.java b/core/java/android/view/textclassifier/TextClassifierImpl.java index 9b0f9c6ee5e8..fcd06c384921 100644 --- a/core/java/android/view/textclassifier/TextClassifierImpl.java +++ b/core/java/android/view/textclassifier/TextClassifierImpl.java @@ -80,6 +80,8 @@ public final class TextClassifierImpl implements TextClassifier { private static final String LOG_TAG = DEFAULT_LOG_TAG; + private static final boolean DEBUG = false; + private static final File FACTORY_MODEL_DIR = new File("/etc/textclassifier/"); // Annotator private static final String ANNOTATOR_FACTORY_MODEL_FILENAME_REGEX = @@ -109,6 +111,8 @@ public final class TextClassifierImpl implements TextClassifier { @GuardedBy("mLock") // Do not access outside this lock. private LangIdModel mLangIdImpl; @GuardedBy("mLock") // Do not access outside this lock. + private ModelFileManager.ModelFile mActionModelInUse; + @GuardedBy("mLock") // Do not access outside this lock. private ActionsSuggestionsModel mActionsImpl; private final Object mLoggerLock = new Object(); @@ -342,8 +346,10 @@ public final class TextClassifierImpl implements TextClassifier { } @Override - public void onTextClassifierEvent(@NonNull TextClassifierEvent event) { - // TODO: Implement. + public void onTextClassifierEvent(TextClassifierEvent event) { + if (DEBUG) { + Log.d(DEFAULT_LOG_TAG, "onTextClassifierEvent() called with: event = [" + event + "]"); + } } /** @inheritDoc */ @@ -408,7 +414,12 @@ public final class TextClassifierImpl implements TextClassifier { .setConfidenceScore(nativeSuggestion.getScore()) .build()); } - return new ConversationActions(conversationActions, /*id*/ null); + String resultId = ActionsSuggestionsHelper.createResultId( + mContext, + request.getConversation(), + mActionModelInUse.getVersion(), + mActionModelInUse.getSupportedLocales()); + return new ConversationActions(conversationActions, resultId); } catch (Throwable t) { // Avoid throwing from this method. Log the error. Log.e(LOG_TAG, "Error suggesting conversation actions.", t); @@ -517,6 +528,7 @@ public final class TextClassifierImpl implements TextClassifier { try { if (pfd != null) { mActionsImpl = new ActionsSuggestionsModel(pfd.getFd()); + mActionModelInUse = bestModel; } } finally { maybeCloseAndLogError(pfd); diff --git a/packages/ExtServices/src/android/ext/services/notification/Assistant.java b/packages/ExtServices/src/android/ext/services/notification/Assistant.java index 133d8ba8357b..0628e6d3e7f2 100644 --- a/packages/ExtServices/src/android/ext/services/notification/Assistant.java +++ b/packages/ExtServices/src/android/ext/services/notification/Assistant.java @@ -117,7 +117,7 @@ public class Assistant extends NotificationAssistantService { mPackageManager = ActivityThread.getPackageManager(); mSettings = mSettingsFactory.createAndRegister(mHandler, getApplicationContext().getContentResolver(), getUserId(), this::updateThresholds); - mSmartActionsHelper = new SmartActionsHelper(); + mSmartActionsHelper = new SmartActionsHelper(getContext(), mSettings); mNotificationCategorizer = new NotificationCategorizer(); mAgingHelper = new AgingHelper(getContext(), mNotificationCategorizer, @@ -215,10 +215,8 @@ public class Assistant extends NotificationAssistantService { return null; } NotificationEntry entry = new NotificationEntry(mPackageManager, sbn, channel); - ArrayList actions = - mSmartActionsHelper.suggestActions(this, entry, mSettings); - ArrayList replies = - mSmartActionsHelper.suggestReplies(this, entry, mSettings); + ArrayList actions = mSmartActionsHelper.suggestActions(entry); + ArrayList replies = mSmartActionsHelper.suggestReplies(entry); return createEnqueuedNotificationAdjustment(entry, actions, replies); } @@ -343,6 +341,7 @@ public class Assistant extends NotificationAssistantService { if (entry != null) { entry.setSeen(); mAgingHelper.onNotificationSeen(entry); + mSmartActionsHelper.onNotificationSeen(entry); } } } catch (Throwable e) { @@ -351,34 +350,46 @@ public class Assistant extends NotificationAssistantService { } @Override - public void onNotificationExpansionChanged(String key, boolean isUserAction, + public void onNotificationExpansionChanged(@NonNull String key, boolean isUserAction, boolean isExpanded) { if (DEBUG) { - Log.i(TAG, - "onNotificationExpansionChanged " + key + ", isUserAction =" + isUserAction - + ", isExpanded = isExpanded"); + Log.d(TAG, "onNotificationExpansionChanged() called with: key = [" + key + + "], isUserAction = [" + isUserAction + "], isExpanded = [" + isExpanded + + "]"); + } + NotificationEntry entry = mLiveNotifications.get(key); + + if (entry != null) { + entry.setExpanded(isExpanded); + mSmartActionsHelper.onNotificationExpansionChanged(entry, isUserAction, isExpanded); } } @Override - public void onNotificationDirectReply(String key) { + public void onNotificationDirectReply(@NonNull String key) { if (DEBUG) Log.i(TAG, "onNotificationDirectReply " + key); + mSmartActionsHelper.onNotificationDirectReply(key); } @Override - public void onSuggestedReplySent(String key, CharSequence reply, int source) { + public void onSuggestedReplySent(@NonNull String key, @NonNull CharSequence reply, + @Source int source) { if (DEBUG) { Log.d(TAG, "onSuggestedReplySent() called with: key = [" + key + "], reply = [" + reply + "], source = [" + source + "]"); } + mSmartActionsHelper.onSuggestedReplySent(key, reply, source); } @Override - public void onActionClicked(String key, Notification.Action action, int source) { + public void onActionClicked(@NonNull String key, @NonNull Notification.Action action, + @Source int source) { if (DEBUG) { - Log.d(TAG, "onActionClicked() called with: key = [" + key + "], action = [" + action.title - + "], source = [" + source + "]"); + Log.d(TAG, + "onActionClicked() called with: key = [" + key + "], action = [" + action.title + + "], source = [" + source + "]"); } + mSmartActionsHelper.onActionClicked(key, action, source); } @Override diff --git a/packages/ExtServices/src/android/ext/services/notification/NotificationEntry.java b/packages/ExtServices/src/android/ext/services/notification/NotificationEntry.java index 6f437bd5d96f..71fd9ce6e4af 100644 --- a/packages/ExtServices/src/android/ext/services/notification/NotificationEntry.java +++ b/packages/ExtServices/src/android/ext/services/notification/NotificationEntry.java @@ -52,6 +52,8 @@ public class NotificationEntry { private NotificationChannel mChannel; private int mImportance; private boolean mSeen; + private boolean mExpanded; + private boolean mIsShowActionEventLogged; public NotificationEntry(IPackageManager packageManager, StatusBarNotification sbn, NotificationChannel channel) { @@ -216,10 +218,26 @@ public class NotificationEntry { mSeen = true; } + public void setExpanded(boolean expanded) { + mExpanded = expanded; + } + + public void setShowActionEventLogged() { + mIsShowActionEventLogged = true; + } + public boolean hasSeen() { return mSeen; } + public boolean isExpanded() { + return mExpanded; + } + + public boolean isShowActionEventLogged() { + return mIsShowActionEventLogged; + } + public StatusBarNotification getSbn() { return mSbn; } diff --git a/packages/ExtServices/src/android/ext/services/notification/SmartActionsHelper.java b/packages/ExtServices/src/android/ext/services/notification/SmartActionsHelper.java index 38df9b0a6fdc..b041842c45b9 100644 --- a/packages/ExtServices/src/android/ext/services/notification/SmartActionsHelper.java +++ b/packages/ExtServices/src/android/ext/services/notification/SmartActionsHelper.java @@ -24,12 +24,16 @@ import android.content.Context; import android.os.Bundle; import android.os.Parcelable; import android.os.Process; +import android.service.notification.NotificationAssistantService; import android.text.TextUtils; import android.util.ArrayMap; +import android.util.LruCache; import android.view.textclassifier.ConversationActions; import android.view.textclassifier.TextClassification; +import android.view.textclassifier.TextClassificationContext; import android.view.textclassifier.TextClassificationManager; import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextClassifierEvent; import android.view.textclassifier.TextLinks; import java.time.Instant; @@ -47,6 +51,7 @@ public class SmartActionsHelper { private static final ArrayList EMPTY_ACTION_LIST = new ArrayList<>(); private static final ArrayList EMPTY_REPLY_LIST = new ArrayList<>(); + private static final String KEY_ACTION_TYPE = "action_type"; // If a notification has any of these flags set, it's inelgibile for actions being added. private static final int FLAG_MASK_INELGIBILE_FOR_ACTIONS = Notification.FLAG_ONGOING_EVENT @@ -59,6 +64,7 @@ public class SmartActionsHelper { private static final int MAX_SUGGESTED_REPLIES = 3; // TODO: Make this configurable. private static final int MAX_MESSAGES_TO_EXTRACT = 5; + private static final int MAX_RESULT_ID_TO_CACHE = 20; private static final ConversationActions.TypeConfig TYPE_CONFIG = new ConversationActions.TypeConfig.Builder().setIncludedTypes( @@ -68,26 +74,36 @@ public class SmartActionsHelper { private static final List HINTS = Collections.singletonList(ConversationActions.HINT_FOR_NOTIFICATION); - SmartActionsHelper() { + private Context mContext; + @Nullable + private TextClassifier mTextClassifier; + @NonNull + private AssistantSettings mSettings; + private LruCache mNotificationKeyToResultIdCache = + new LruCache<>(MAX_RESULT_ID_TO_CACHE); + + SmartActionsHelper(Context context, AssistantSettings settings) { + mContext = context; + TextClassificationManager textClassificationManager = + mContext.getSystemService(TextClassificationManager.class); + if (textClassificationManager != null) { + mTextClassifier = textClassificationManager.getTextClassifier(); + } + mSettings = settings; } /** * Adds action adjustments based on the notification contents. */ @NonNull - ArrayList suggestActions(@Nullable Context context, - @NonNull NotificationEntry entry, @NonNull AssistantSettings settings) { - if (!settings.mGenerateActions) { + ArrayList suggestActions(@NonNull NotificationEntry entry) { + if (!mSettings.mGenerateActions) { return EMPTY_ACTION_LIST; } if (!isEligibleForActionAdjustment(entry)) { return EMPTY_ACTION_LIST; } - if (context == null) { - return EMPTY_ACTION_LIST; - } - TextClassificationManager tcm = context.getSystemService(TextClassificationManager.class); - if (tcm == null) { + if (mTextClassifier == null) { return EMPTY_ACTION_LIST; } List messages = extractMessages(entry.getNotification()); @@ -96,22 +112,17 @@ public class SmartActionsHelper { } // TODO: Move to TextClassifier.suggestConversationActions once it is ready. return suggestActionsFromText( - tcm, messages.get(messages.size() - 1).getText(), MAX_SMART_ACTIONS); + messages.get(messages.size() - 1).getText(), MAX_SMART_ACTIONS); } - ArrayList suggestReplies(@Nullable Context context, - @NonNull NotificationEntry entry, @NonNull AssistantSettings settings) { - if (!settings.mGenerateReplies) { + ArrayList suggestReplies(@NonNull NotificationEntry entry) { + if (!mSettings.mGenerateReplies) { return EMPTY_REPLY_LIST; } if (!isEligibleForReplyAdjustment(entry)) { return EMPTY_REPLY_LIST; } - if (context == null) { - return EMPTY_REPLY_LIST; - } - TextClassificationManager tcm = context.getSystemService(TextClassificationManager.class); - if (tcm == null) { + if (mTextClassifier == null) { return EMPTY_REPLY_LIST; } List messages = extractMessages(entry.getNotification()); @@ -125,14 +136,122 @@ public class SmartActionsHelper { .setTypeConfig(TYPE_CONFIG) .build(); - TextClassifier textClassifier = tcm.getTextClassifier(); + ConversationActions conversationActionsResult = + mTextClassifier.suggestConversationActions(request); List conversationActions = - textClassifier.suggestConversationActions(request).getConversationActions(); - - return conversationActions.stream() + conversationActionsResult.getConversationActions(); + ArrayList replies = conversationActions.stream() .map(conversationAction -> conversationAction.getTextReply()) .filter(textReply -> !TextUtils.isEmpty(textReply)) .collect(Collectors.toCollection(ArrayList::new)); + + String resultId = conversationActionsResult.getId(); + if (resultId != null && !replies.isEmpty()) { + mNotificationKeyToResultIdCache.put(entry.getSbn().getKey(), resultId); + } + return replies; + } + + void onNotificationSeen(@NonNull NotificationEntry entry) { + if (entry.isExpanded()) { + maybeSendActionShownEvent(entry); + } + } + + void onNotificationExpansionChanged(@NonNull NotificationEntry entry, boolean isUserAction, + boolean isExpanded) { + // Notification can be expanded in the background, and thus the isUserAction check. + if (isUserAction && isExpanded) { + maybeSendActionShownEvent(entry); + } + } + + void onNotificationDirectReply(@NonNull String key) { + if (mTextClassifier == null) { + return; + } + String resultId = mNotificationKeyToResultIdCache.get(key); + if (resultId == null) { + return; + } + TextClassifierEvent textClassifierEvent = + createTextClassifierEventBuilder(TextClassifierEvent.TYPE_MANUAL_REPLY, resultId) + .build(); + mTextClassifier.onTextClassifierEvent(textClassifierEvent); + } + + void onSuggestedReplySent(@NonNull String key, @NonNull CharSequence reply, + @NotificationAssistantService.Source int source) { + if (mTextClassifier == null) { + return; + } + if (source != NotificationAssistantService.SOURCE_FROM_ASSISTANT) { + return; + } + String resultId = mNotificationKeyToResultIdCache.get(key); + if (resultId == null) { + return; + } + TextClassifierEvent textClassifierEvent = + createTextClassifierEventBuilder(TextClassifierEvent.TYPE_SMART_ACTION, resultId) + .setEntityType(ConversationActions.TYPE_TEXT_REPLY) + .build(); + mTextClassifier.onTextClassifierEvent(textClassifierEvent); + } + + void onActionClicked(@NonNull String key, @NonNull Notification.Action action, + @NotificationAssistantService.Source int source) { + if (mTextClassifier == null) { + return; + } + if (source != NotificationAssistantService.SOURCE_FROM_ASSISTANT) { + return; + } + String resultId = mNotificationKeyToResultIdCache.get(key); + if (resultId == null) { + return; + } + String actionType = action.getExtras().getString(KEY_ACTION_TYPE); + if (actionType == null) { + return; + } + TextClassifierEvent textClassifierEvent = + createTextClassifierEventBuilder(TextClassifierEvent.TYPE_SMART_ACTION, resultId) + .setEntityType(actionType) + .build(); + mTextClassifier.onTextClassifierEvent(textClassifierEvent); + } + + private TextClassifierEvent.Builder createTextClassifierEventBuilder( + int eventType, @NonNull String resultId) { + return new TextClassifierEvent.Builder( + TextClassifierEvent.CATEGORY_CONVERSATION_ACTIONS, eventType) + .setEventTime(System.currentTimeMillis()) + .setEventContext( + new TextClassificationContext.Builder( + mContext.getPackageName(), TextClassifier.WIDGET_TYPE_NOTIFICATION) + .build()) + .setResultId(resultId); + } + + private void maybeSendActionShownEvent(@NonNull NotificationEntry entry) { + if (mTextClassifier == null) { + return; + } + String resultId = mNotificationKeyToResultIdCache.get(entry.getSbn().getKey()); + if (resultId == null) { + return; + } + // Only report if this is the first time the user sees these suggestions. + if (entry.isShowActionEventLogged()) { + return; + } + entry.setShowActionEventLogged(); + TextClassifierEvent textClassifierEvent = + createTextClassifierEventBuilder(TextClassifierEvent.TYPE_ACTIONS_SHOWN, resultId) + .build(); + // TODO: If possible, report which replies / actions are actually seen by user. + mTextClassifier.onTextClassifierEvent(textClassifierEvent); } /** @@ -220,13 +339,10 @@ public class SmartActionsHelper { /** Returns a list of actions to act on entities in a given piece of text. */ @NonNull private ArrayList suggestActionsFromText( - @NonNull TextClassificationManager tcm, @Nullable CharSequence text, - int maxSmartActions) { + @Nullable CharSequence text, int maxSmartActions) { if (TextUtils.isEmpty(text)) { return EMPTY_ACTION_LIST; } - TextClassifier textClassifier = tcm.getTextClassifier(); - // We want to process only text visible to the user to avoid confusing suggestions, so we // truncate the text to a reasonable length. This is particularly important for e.g. // email apps that sometimes include the text for the entire thread. @@ -239,7 +355,7 @@ public class SmartActionsHelper { Collections.singletonList( TextClassifier.HINT_TEXT_IS_NOT_EDITABLE))) .build(); - TextLinks links = textClassifier.generateLinks(textLinksRequest); + TextLinks links = mTextClassifier.generateLinks(textLinksRequest); ArrayMap entityTypeFrequency = getEntityTypeFrequency(links); ArrayList actions = new ArrayList<>(); @@ -254,19 +370,26 @@ public class SmartActionsHelper { // Generate the actions, and add the most prominent ones to the action bar. TextClassification classification = - textClassifier.classifyText( + mTextClassifier.classifyText( new TextClassification.Request.Builder( text, link.getStart(), link.getEnd()).build()); + if (classification.getEntityCount() == 0) { + continue; + } int numOfActions = Math.min( MAX_ACTIONS_PER_LINK, classification.getActions().size()); for (int i = 0; i < numOfActions; ++i) { - RemoteAction action = classification.getActions().get(i); - actions.add( - new Notification.Action.Builder( - action.getIcon(), - action.getTitle(), - action.getActionIntent()) - .build()); + RemoteAction remoteAction = classification.getActions().get(i); + Notification.Action action = new Notification.Action.Builder( + remoteAction.getIcon(), + remoteAction.getTitle(), + remoteAction.getActionIntent()) + .setSemanticAction( + Notification.Action.SEMANTIC_ACTION_CONTEXTUAL_SUGGESTION) + .addExtras(Bundle.forPair(KEY_ACTION_TYPE, classification.getEntity(0))) + .build(); + actions.add(action); + // We have enough smart actions. if (actions.size() >= maxSmartActions) { return actions; diff --git a/packages/ExtServices/tests/src/android/ext/services/notification/SmartActionHelperTest.java b/packages/ExtServices/tests/src/android/ext/services/notification/SmartActionHelperTest.java index 0352ebcec8b3..da382a003621 100644 --- a/packages/ExtServices/tests/src/android/ext/services/notification/SmartActionHelperTest.java +++ b/packages/ExtServices/tests/src/android/ext/services/notification/SmartActionHelperTest.java @@ -28,12 +28,14 @@ import android.app.Notification; import android.app.Person; import android.content.Context; import android.os.Process; +import android.service.notification.NotificationAssistantService; import android.service.notification.StatusBarNotification; import android.support.test.InstrumentationRegistry; import android.support.test.runner.AndroidJUnit4; import android.view.textclassifier.ConversationActions; import android.view.textclassifier.TextClassificationManager; import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextClassifierEvent; import com.google.common.truth.FailureStrategy; import com.google.common.truth.Subject; @@ -44,12 +46,14 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -58,8 +62,14 @@ import javax.annotation.Nullable; @RunWith(AndroidJUnit4.class) public class SmartActionHelperTest { + private static final String NOTIFICATION_KEY = "key"; + private static final String RESULT_ID = "id"; - private SmartActionsHelper mSmartActionsHelper = new SmartActionsHelper(); + private static final ConversationActions.ConversationAction REPLY_ACTION = + new ConversationActions.ConversationAction.Builder( + ConversationActions.TYPE_TEXT_REPLY).setTextReply("Home").build(); + + private SmartActionsHelper mSmartActionsHelper; private Context mContext; @Mock private TextClassifier mTextClassifier; @Mock private NotificationEntry mNotificationEntry; @@ -75,7 +85,7 @@ public class SmartActionHelperTest { mContext.getSystemService(TextClassificationManager.class) .setTextClassifier(mTextClassifier); when(mTextClassifier.suggestConversationActions(any(ConversationActions.Request.class))) - .thenReturn(new ConversationActions(Collections.emptyList(), null)); + .thenReturn(new ConversationActions(Arrays.asList(REPLY_ACTION), RESULT_ID)); when(mNotificationEntry.getSbn()).thenReturn(mStatusBarNotification); // The notification is eligible to have smart suggestions. @@ -83,18 +93,20 @@ public class SmartActionHelperTest { when(mNotificationEntry.isMessaging()).thenReturn(true); when(mStatusBarNotification.getPackageName()).thenReturn("random.app"); when(mStatusBarNotification.getUser()).thenReturn(Process.myUserHandle()); + when(mStatusBarNotification.getKey()).thenReturn(NOTIFICATION_KEY); mNotificationBuilder = new Notification.Builder(mContext, "channel"); mSettings = AssistantSettings.createForTesting( null, null, Process.myUserHandle().getIdentifier(), null); mSettings.mGenerateActions = true; mSettings.mGenerateReplies = true; + mSmartActionsHelper = new SmartActionsHelper(mContext, mSettings); } @Test public void testSuggestReplies_notMessagingApp() { when(mNotificationEntry.isMessaging()).thenReturn(false); ArrayList textReplies = - mSmartActionsHelper.suggestReplies(mContext, mNotificationEntry, mSettings); + mSmartActionsHelper.suggestReplies(mNotificationEntry); assertThat(textReplies).isEmpty(); } @@ -102,7 +114,7 @@ public class SmartActionHelperTest { public void testSuggestReplies_noInlineReply() { when(mNotificationEntry.hasInlineReply()).thenReturn(false); ArrayList textReplies = - mSmartActionsHelper.suggestReplies(mContext, mNotificationEntry, mSettings); + mSmartActionsHelper.suggestReplies(mNotificationEntry); assertThat(textReplies).isEmpty(); } @@ -169,18 +181,128 @@ public class SmartActionHelperTest { .build(); when(mNotificationEntry.getNotification()).thenReturn(notification); - mSmartActionsHelper.suggestReplies(mContext, mNotificationEntry, mSettings); + mSmartActionsHelper.suggestReplies(mNotificationEntry); verify(mTextClassifier, never()) .suggestConversationActions(any(ConversationActions.Request.class)); } + @Test + public void testOnSuggestedReplySent() { + final String message = "Where are you?"; + Notification notification = mNotificationBuilder.setContentText(message).build(); + when(mNotificationEntry.getNotification()).thenReturn(notification); + + mSmartActionsHelper.suggestReplies(mNotificationEntry); + mSmartActionsHelper.onSuggestedReplySent( + NOTIFICATION_KEY, message, NotificationAssistantService.SOURCE_FROM_ASSISTANT); + + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(TextClassifierEvent.class); + verify(mTextClassifier).onTextClassifierEvent(argumentCaptor.capture()); + TextClassifierEvent textClassifierEvent = argumentCaptor.getValue(); + assertTextClassifierEvent(textClassifierEvent, TextClassifierEvent.TYPE_SMART_ACTION); + } + + @Test + public void testOnSuggestedReplySent_anotherNotification() { + final String message = "Where are you?"; + Notification notification = mNotificationBuilder.setContentText(message).build(); + when(mNotificationEntry.getNotification()).thenReturn(notification); + + mSmartActionsHelper.suggestReplies(mNotificationEntry); + mSmartActionsHelper.onSuggestedReplySent( + "something_else", message, NotificationAssistantService.SOURCE_FROM_ASSISTANT); + + verify(mTextClassifier, never()) + .onTextClassifierEvent(Mockito.any(TextClassifierEvent.class)); + } + + @Test + public void testOnSuggestedReplySent_missingResultId() { + when(mTextClassifier.suggestConversationActions(any(ConversationActions.Request.class))) + .thenReturn(new ConversationActions(Collections.emptyList(), null)); + + final String message = "Where are you?"; + Notification notification = mNotificationBuilder.setContentText(message).build(); + when(mNotificationEntry.getNotification()).thenReturn(notification); + + mSmartActionsHelper.suggestReplies(mNotificationEntry); + mSmartActionsHelper.onSuggestedReplySent( + "something_else", message, NotificationAssistantService.SOURCE_FROM_ASSISTANT); + + verify(mTextClassifier, never()) + .onTextClassifierEvent(Mockito.any(TextClassifierEvent.class)); + } + + @Test + public void testOnNotificationDirectReply() { + Notification notification = mNotificationBuilder.setContentText("Where are you?").build(); + when(mNotificationEntry.getNotification()).thenReturn(notification); + + mSmartActionsHelper.suggestReplies(mNotificationEntry); + mSmartActionsHelper.onNotificationDirectReply(NOTIFICATION_KEY); + + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(TextClassifierEvent.class); + verify(mTextClassifier).onTextClassifierEvent(argumentCaptor.capture()); + TextClassifierEvent textClassifierEvent = argumentCaptor.getValue(); + assertTextClassifierEvent(textClassifierEvent, TextClassifierEvent.TYPE_MANUAL_REPLY); + } + + @Test + public void testOnNotificationExpansionChanged() { + final String message = "Where are you?"; + Notification notification = mNotificationBuilder.setContentText(message).build(); + when(mNotificationEntry.getNotification()).thenReturn(notification); + + mSmartActionsHelper.suggestReplies(mNotificationEntry); + mSmartActionsHelper.onNotificationExpansionChanged(mNotificationEntry, true, true); + + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(TextClassifierEvent.class); + verify(mTextClassifier).onTextClassifierEvent(argumentCaptor.capture()); + TextClassifierEvent textClassifierEvent = argumentCaptor.getValue(); + assertTextClassifierEvent(textClassifierEvent, TextClassifierEvent.TYPE_ACTIONS_SHOWN); + } + + @Test + public void testOnNotificationsSeen_notExpanded() { + final String message = "Where are you?"; + Notification notification = mNotificationBuilder.setContentText(message).build(); + when(mNotificationEntry.getNotification()).thenReturn(notification); + when(mNotificationEntry.isExpanded()).thenReturn(false); + + mSmartActionsHelper.suggestReplies(mNotificationEntry); + mSmartActionsHelper.onNotificationSeen(mNotificationEntry); + + verify(mTextClassifier, never()).onTextClassifierEvent( + Mockito.any(TextClassifierEvent.class)); + } + + @Test + public void testOnNotificationsSeen_expanded() { + final String message = "Where are you?"; + Notification notification = mNotificationBuilder.setContentText(message).build(); + when(mNotificationEntry.getNotification()).thenReturn(notification); + when(mNotificationEntry.isExpanded()).thenReturn(true); + + mSmartActionsHelper.suggestReplies(mNotificationEntry); + mSmartActionsHelper.onNotificationSeen(mNotificationEntry); + + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(TextClassifierEvent.class); + verify(mTextClassifier).onTextClassifierEvent(argumentCaptor.capture()); + TextClassifierEvent textClassifierEvent = argumentCaptor.getValue(); + assertTextClassifierEvent(textClassifierEvent, TextClassifierEvent.TYPE_ACTIONS_SHOWN); + } + private ZonedDateTime createZonedDateTimeFromMsUtc(long msUtc) { return ZonedDateTime.ofInstant(Instant.ofEpochMilli(msUtc), ZoneOffset.systemDefault()); } private List getMessagesInRequest() { - mSmartActionsHelper.suggestReplies(mContext, mNotificationEntry, mSettings); + mSmartActionsHelper.suggestReplies(mNotificationEntry); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(ConversationActions.Request.class); @@ -189,6 +311,17 @@ public class SmartActionHelperTest { return request.getConversation(); } + private void assertTextClassifierEvent( + TextClassifierEvent textClassifierEvent, int expectedEventType) { + assertThat(textClassifierEvent.getEventCategory()) + .isEqualTo(TextClassifierEvent.CATEGORY_CONVERSATION_ACTIONS); + assertThat(textClassifierEvent.getEventContext().getPackageName()) + .isEqualTo(InstrumentationRegistry.getTargetContext().getPackageName()); + assertThat(textClassifierEvent.getEventContext().getWidgetType()) + .isEqualTo(TextClassifier.WIDGET_TYPE_NOTIFICATION); + assertThat(textClassifierEvent.getEventType()).isEqualTo(expectedEventType); + } + private static final class MessageSubject extends Subject { -- cgit v1.2.3-59-g8ed1b