Implemented autofill field classification on multiple fields and user data.
Test: atest CtsAutoFillServiceTestCases:FieldsClassificationTest
Test: atest CtsAutoFillServiceTestCases:FieldsClassificationScorerTest
Test: atest CtsAutoFillServiceTestCases:UserDataTest
Bug: 68045531
Change-Id: Ia9252cb5b84236a76a1419f4a2669b2e933f5177
diff --git a/core/java/android/service/autofill/FieldsClassificationScorer.java b/core/java/android/service/autofill/FieldsClassificationScorer.java
index fea8ebf..b95a7ed 100644
--- a/core/java/android/service/autofill/FieldsClassificationScorer.java
+++ b/core/java/android/service/autofill/FieldsClassificationScorer.java
@@ -38,9 +38,24 @@
* partial mathces are something in between, typically using edit-distance algorithms.
*/
public static int getScore(@NonNull AutofillValue actualValue, @NonNull String userData) {
- // TODO(b/67867469): implement edit distance - currently it's returning either 0 or 100%
if (actualValue == null || !actualValue.isText() || userData == null) return 0;
- return actualValue.getTextValue().toString().equalsIgnoreCase(userData) ? MAX_VALUE : 0;
+ // TODO(b/67867469): implement edit distance - currently it's returning either 0, 100%, or
+ // partial match when number of chars match
+ final String textValue = actualValue.getTextValue().toString();
+ final int total = textValue.length();
+ if (total != userData.length()) return 0;
+
+ int matches = 0;
+ for (int i = 0; i < total; i++) {
+ if (Character.toLowerCase(textValue.charAt(i)) == Character
+ .toLowerCase(userData.charAt(i))) {
+ matches++;
+ }
+ }
+
+ final float percentage = ((float) matches) / total;
+ final int rounded = (int) (percentage * MAX_VALUE);
+ return rounded;
}
private FieldsClassificationScorer() {
diff --git a/core/java/android/service/autofill/FillEventHistory.java b/core/java/android/service/autofill/FillEventHistory.java
index cf6f296..facad2d 100644
--- a/core/java/android/service/autofill/FillEventHistory.java
+++ b/core/java/android/service/autofill/FillEventHistory.java
@@ -16,6 +16,8 @@
package android.service.autofill;
+import static android.view.autofill.Helper.sVerbose;
+
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
@@ -24,8 +26,10 @@
import android.os.Bundle;
import android.os.Parcel;
import android.os.Parcelable;
+import android.service.autofill.FieldClassification.Match;
import android.util.ArrayMap;
import android.util.ArraySet;
+import android.util.Log;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillManager;
@@ -35,10 +39,10 @@
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
/**
@@ -58,6 +62,8 @@
* the history will clear out after some pre-defined time).
*/
public final class FillEventHistory implements Parcelable {
+ private static final String TAG = "FillEventHistory";
+
/**
* Not in parcel. The ID of the autofill session that created the {@link FillResponse}.
*/
@@ -149,10 +155,10 @@
parcel.writeStringList(event.mManuallyFilledDatasetIds.get(j));
}
}
- parcel.writeParcelable(event.mDetectedFieldId, flags);
- if (event.mDetectedRemoteId != null) {
- parcel.writeString(event.mDetectedRemoteId);
- parcel.writeInt(event.mDetectedFieldScore);
+ final AutofillId[] detectedFields = event.mDetectedFieldIds;
+ parcel.writeParcelableArray(detectedFields, flags);
+ if (detectedFields != null) {
+ parcel.writeParcelableArray(event.mDetectedMatches, flags);
}
}
}
@@ -244,10 +250,8 @@
@Nullable private final ArrayList<AutofillId> mManuallyFilledFieldIds;
@Nullable private final ArrayList<ArrayList<String>> mManuallyFilledDatasetIds;
- // TODO(b/67867469): store list of fields instead of hardcoding just one
- @Nullable private final AutofillId mDetectedFieldId;
- @Nullable private final String mDetectedRemoteId;
- private final int mDetectedFieldScore;
+ @Nullable private final AutofillId[] mDetectedFieldIds;
+ @Nullable private final Match[] mDetectedMatches;
/**
* Returns the type of the event.
@@ -365,15 +369,19 @@
*/
@TestApi
@NonNull public Map<AutofillId, FieldClassification> getFieldsClassification() {
- if (mDetectedFieldId == null || mDetectedRemoteId == null
- || mDetectedFieldScore == -1) {
+ if (mDetectedFieldIds == null) {
return Collections.emptyMap();
}
-
- final ArrayMap<AutofillId, FieldClassification> map = new ArrayMap<>(1);
- map.put(mDetectedFieldId,
- new FieldClassification(new FieldClassification.Match(
- mDetectedRemoteId, mDetectedFieldScore)));
+ final int size = mDetectedFieldIds.length;
+ final ArrayMap<AutofillId, FieldClassification> map = new ArrayMap<>(size);
+ for (int i = 0; i < size; i++) {
+ final AutofillId id = mDetectedFieldIds[i];
+ final Match match = mDetectedMatches[i];
+ if (sVerbose) {
+ Log.v(TAG, "getFieldsClassification[" + i + "]: id=" + id + ", match=" + match);
+ }
+ map.put(id, new FieldClassification(match));
+ }
return map;
}
@@ -468,7 +476,7 @@
@Nullable ArrayList<String> changedDatasetIds,
@Nullable ArrayList<AutofillId> manuallyFilledFieldIds,
@Nullable ArrayList<ArrayList<String>> manuallyFilledDatasetIds,
- @Nullable Map<AutofillId, FieldClassification> fieldsClassification) {
+ @Nullable AutofillId[] detectedFieldIds, @Nullable Match[] detectedMaches) {
mEventType = Preconditions.checkArgumentInRange(eventType, 0, TYPE_CONTEXT_COMMITTED,
"eventType");
mDatasetId = datasetId;
@@ -492,20 +500,8 @@
mManuallyFilledFieldIds = manuallyFilledFieldIds;
mManuallyFilledDatasetIds = manuallyFilledDatasetIds;
- // TODO(b/67867469): store list of fields instead of hardcoding just one
- if (fieldsClassification == null) {
- mDetectedFieldId = null;
- mDetectedRemoteId = null;
- mDetectedFieldScore = 0;
-
- } else {
- final Entry<AutofillId, FieldClassification> tmpEntry = fieldsClassification
- .entrySet().iterator().next();
- final FieldClassification.Match tmpMatch = tmpEntry.getValue().getTopMatch();
- mDetectedFieldId = tmpEntry.getKey();
- mDetectedRemoteId = tmpMatch.getRemoteId();
- mDetectedFieldScore = tmpMatch.getScore();
- }
+ mDetectedFieldIds = detectedFieldIds;
+ mDetectedMatches = detectedMaches;
}
@Override
@@ -518,9 +514,8 @@
+ ", changedDatasetsIds=" + mChangedDatasetIds
+ ", manuallyFilledFieldIds=" + mManuallyFilledFieldIds
+ ", manuallyFilledDatasetIds=" + mManuallyFilledDatasetIds
- + ", detectedFieldId=" + mDetectedFieldId
- + ", detectedRemoteId=" + mDetectedRemoteId
- + ", detectedFieldScore=" + mDetectedFieldScore
+ + ", detectedFieldIds=" + Arrays.toString(mDetectedFieldIds)
+ + ", detectedMaches =" + Arrays.toString(mDetectedMatches)
+ "]";
}
}
@@ -556,23 +551,17 @@
} else {
manuallyFilledDatasetIds = null;
}
- // TODO(b/67867469): store list of fields instead of hardcoding just one
- ArrayMap<AutofillId, FieldClassification> fieldsClassification = null;
- final AutofillId detectedFieldId = parcel.readParcelable(null);
- if (detectedFieldId == null) {
- fieldsClassification = null;
- } else {
- fieldsClassification = new ArrayMap<AutofillId, FieldClassification>(1);
- fieldsClassification.put(detectedFieldId,
- new FieldClassification(new FieldClassification.Match(
- parcel.readString(), parcel.readInt())));
- }
+ final AutofillId[] detectedFieldIds = parcel.readParcelableArray(null,
+ AutofillId.class);
+ final Match[] detectedMatches = (detectedFieldIds != null)
+ ? parcel.readParcelableArray(null, Match.class)
+ : null;
selection.addEvent(new Event(eventType, datasetId, clientState,
selectedDatasetIds, ignoredDatasets,
changedFieldIds, changedDatasetIds,
manuallyFilledFieldIds, manuallyFilledDatasetIds,
- fieldsClassification));
+ detectedFieldIds, detectedMatches));
}
return selection;
}
diff --git a/core/java/android/service/autofill/UserData.java b/core/java/android/service/autofill/UserData.java
index 16d8d4a..ea5660d 100644
--- a/core/java/android/service/autofill/UserData.java
+++ b/core/java/android/service/autofill/UserData.java
@@ -29,7 +29,6 @@
import android.os.Parcel;
import android.os.Parcelable;
import android.provider.Settings;
-import android.util.ArraySet;
import android.util.Log;
import android.view.autofill.Helper;
@@ -79,10 +78,16 @@
/** @hide */
public void dump(String prefix, PrintWriter pw) {
- // Cannot disclose remote ids because they could contain PII
+ // Cannot disclose remote ids or values because they could contain PII
pw.print(prefix); pw.print("Remote ids size: "); pw.println(mRemoteIds.length);
+ for (int i = 0; i < mRemoteIds.length; i++) {
+ pw.print(prefix); pw.print(prefix); pw.print(i); pw.print(": ");
+ pw.println(Helper.getRedacted(mRemoteIds[i]));
+ }
+ pw.print(prefix); pw.print("Values size: "); pw.println(mValues.length);
for (int i = 0; i < mValues.length; i++) {
- pw.print(prefix); pw.print(prefix); pw.print(i); pw.print(": "); pw.println(mValues[i]);
+ pw.print(prefix); pw.print(prefix); pw.print(i); pw.print(": ");
+ pw.println(Helper.getRedacted(mValues[i]));
}
}
@@ -104,7 +109,7 @@
*/
@TestApi
public static final class Builder {
- private final ArraySet<String> mRemoteIds;
+ private final ArrayList<String> mRemoteIds;
private final ArrayList<String> mValues;
private boolean mDestroyed;
@@ -120,7 +125,7 @@
checkValidRemoteId(remoteId);
checkValidValue(value);
final int capacity = getMaxUserDataSize();
- mRemoteIds = new ArraySet<>(capacity);
+ mRemoteIds = new ArrayList<>(capacity);
mValues = new ArrayList<>(capacity);
mRemoteIds.add(remoteId);
mValues.add(value);
@@ -149,10 +154,14 @@
Preconditions.checkState(!mRemoteIds.contains(remoteId),
// Don't include remoteId on message because it could contain PII
"already has entry with same remoteId");
+ Preconditions.checkState(!mValues.contains(value),
+ // Don't include remoteId on message because it could contain PII
+ "already has entry with same value");
Preconditions.checkState(mRemoteIds.size() < getMaxUserDataSize(),
"already added " + mRemoteIds.size() + " elements");
mRemoteIds.add(remoteId);
mValues.add(value);
+
return this;
}
diff --git a/core/java/android/view/autofill/Helper.java b/core/java/android/view/autofill/Helper.java
index b95704a..4b2c53c 100644
--- a/core/java/android/view/autofill/Helper.java
+++ b/core/java/android/view/autofill/Helper.java
@@ -18,11 +18,6 @@
import android.annotation.NonNull;
import android.annotation.Nullable;
-import android.os.Bundle;
-
-import java.util.Arrays;
-import java.util.Objects;
-import java.util.Set;
/** @hide */
public final class Helper {
@@ -31,37 +26,20 @@
public static boolean sDebug = false;
public static boolean sVerbose = false;
- public static final String REDACTED = "[REDACTED]";
-
- static StringBuilder append(StringBuilder builder, Bundle bundle) {
- if (bundle == null || !sDebug) {
- builder.append("N/A");
- } else if (!sVerbose) {
- builder.append(REDACTED);
- } else {
- final Set<String> keySet = bundle.keySet();
- builder.append("[Bundle with ").append(keySet.size()).append(" extras:");
- for (String key : keySet) {
- final Object value = bundle.get(key);
- builder.append(' ').append(key).append('=');
- builder.append((value instanceof Object[])
- ? Arrays.toString((Objects[]) value) : value);
- }
- builder.append(']');
- }
- return builder;
- }
-
/**
* Appends {@code value} to the {@code builder} redacting its contents.
*/
public static void appendRedacted(@NonNull StringBuilder builder,
@Nullable CharSequence value) {
- if (value == null) {
- builder.append("null");
- } else {
- builder.append(value.length()).append("_chars");
- }
+ builder.append(getRedacted(value));
+ }
+
+ /**
+ * Gets the redacted version of a value.
+ */
+ @NonNull
+ public static String getRedacted(@Nullable CharSequence value) {
+ return (value == null) ? "null" : value.length() + "_chars";
}
/**
diff --git a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java
index 77e8907..a465495 100644
--- a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java
+++ b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java
@@ -47,7 +47,7 @@
import android.provider.Settings;
import android.service.autofill.AutofillService;
import android.service.autofill.AutofillServiceInfo;
-import android.service.autofill.FieldClassification;
+import android.service.autofill.FieldClassification.Match;
import android.service.autofill.FillEventHistory;
import android.service.autofill.FillEventHistory.Event;
import android.service.autofill.FillResponse;
@@ -75,7 +75,7 @@
import java.io.PrintWriter;
import java.util.ArrayList;
-import java.util.Map;
+import java.util.Arrays;
import java.util.Random;
/**
@@ -628,7 +628,7 @@
if (isValidEventLocked("setAuthenticationSelected()", sessionId)) {
mEventHistory.addEvent(
new Event(Event.TYPE_AUTHENTICATION_SELECTED, null, clientState, null, null,
- null, null, null, null, null));
+ null, null, null, null, null, null));
}
}
}
@@ -642,7 +642,7 @@
if (isValidEventLocked("logDatasetAuthenticationSelected()", sessionId)) {
mEventHistory.addEvent(
new Event(Event.TYPE_DATASET_AUTHENTICATION_SELECTED, selectedDataset,
- clientState, null, null, null, null, null, null, null));
+ clientState, null, null, null, null, null, null, null, null));
}
}
}
@@ -654,7 +654,7 @@
synchronized (mLock) {
if (isValidEventLocked("logSaveShown()", sessionId)) {
mEventHistory.addEvent(new Event(Event.TYPE_SAVE_SHOWN, null, clientState, null,
- null, null, null, null, null, null));
+ null, null, null, null, null, null, null));
}
}
}
@@ -668,7 +668,7 @@
if (isValidEventLocked("logDatasetSelected()", sessionId)) {
mEventHistory.addEvent(
new Event(Event.TYPE_DATASET_SELECTED, selectedDataset, clientState, null,
- null, null, null, null, null, null));
+ null, null, null, null, null, null, null));
}
}
}
@@ -683,15 +683,24 @@
@Nullable ArrayList<String> changedDatasetIds,
@Nullable ArrayList<AutofillId> manuallyFilledFieldIds,
@Nullable ArrayList<ArrayList<String>> manuallyFilledDatasetIds,
- @Nullable Map<AutofillId, FieldClassification> fieldsClassification) {
+ @NonNull ArrayList<AutofillId> detectedFieldIdsList,
+ @NonNull ArrayList<Match> detectedMatchesList) {
synchronized (mLock) {
if (isValidEventLocked("logDatasetNotSelected()", sessionId)) {
+ AutofillId[] detectedFieldsIds = null;
+ Match[] detectedMatches = null;
+ if (!detectedFieldIdsList.isEmpty()) {
+ detectedFieldsIds = new AutofillId[detectedFieldIdsList.size()];
+ detectedFieldIdsList.toArray(detectedFieldsIds);
+ detectedMatches = new Match[detectedMatchesList.size()];
+ detectedMatchesList.toArray(detectedMatches);
+ }
mEventHistory.addEvent(new Event(Event.TYPE_CONTEXT_COMMITTED, null,
clientState, selectedDatasets, ignoredDatasets,
changedFieldIds, changedDatasetIds,
manuallyFilledFieldIds, manuallyFilledDatasetIds,
- fieldsClassification));
+ detectedFieldsIds, detectedMatches));
}
}
}
diff --git a/services/autofill/java/com/android/server/autofill/Session.java b/services/autofill/java/com/android/server/autofill/Session.java
index 106ac8f..b6d8869 100644
--- a/services/autofill/java/com/android/server/autofill/Session.java
+++ b/services/autofill/java/com/android/server/autofill/Session.java
@@ -55,7 +55,7 @@
import android.os.SystemClock;
import android.service.autofill.AutofillService;
import android.service.autofill.Dataset;
-import android.service.autofill.FieldClassification;
+import android.service.autofill.FieldClassification.Match;
import android.service.autofill.FillContext;
import android.service.autofill.FillRequest;
import android.service.autofill.FillResponse;
@@ -944,23 +944,19 @@
}
final UserData userData = mService.getUserData();
- // TODO(b/67867469): support multiple fields / merge redundant variables below
- final AutofillId detectableFieldId;
- final String detectableRemoteId;
- String detectedRemoteId = null;
- Map<AutofillId, FieldClassification> fieldsClassification = null;
- if (userData == null) {
- detectableFieldId = null;
- detectableRemoteId = null;
+ final ArrayList<AutofillId> detectedFieldIds;
+ final ArrayList<Match> detectedMatches;
+
+ if (userData != null) {
+ final int maxFieldsSize = UserData.getMaxFieldClassificationIdsSize();
+ detectedFieldIds = new ArrayList<>(maxFieldsSize);
+ detectedMatches = new ArrayList<>(maxFieldsSize);
} else {
- // TODO(b/67867469): hardcoded to just first entry on initial refactoring.
- detectableFieldId = fieldClassificationIds[0];
- detectableRemoteId = userData.getRemoteIds()[0];
+ detectedFieldIds = null;
+ detectedMatches = null;
}
- int detectedFieldScore = -1;
-
for (int i = 0; i < mViewStates.size(); i++) {
final ViewState viewState = mViewStates.valueAt(i);
final int state = viewState.getState();
@@ -1003,8 +999,8 @@
final AutofillValue currentValue = viewState.getCurrentValue();
if (currentValue == null) {
if (sDebug) {
- Slog.d(TAG, "logContextCommitted(): skipping view witout current value "
- + "( " + viewState + ")");
+ Slog.d(TAG, "logContextCommitted(): skipping view without current "
+ + "value ( " + viewState + ")");
}
continue;
}
@@ -1065,22 +1061,11 @@
} // for j
}
- // Check if detectable field changed.
- if (detectableFieldId != null && detectableFieldId.equals(viewState.id)
- && currentValue.isText() && currentValue.getTextValue() != null) {
- // TODO(b/67867469): hardcoded to just first entry on initial refactoring.
- detectedFieldScore = FieldsClassificationScorer.getScore(currentValue,
- userData.getValues()[0]);
- if (detectedFieldScore > 0) {
- detectedRemoteId = detectableRemoteId;
- fieldsClassification = new ArrayMap<AutofillId, FieldClassification>(1);
- fieldsClassification.put(detectableFieldId,
- new FieldClassification(new FieldClassification.Match(
- detectedRemoteId, detectedFieldScore)));
- } else if (sVerbose) {
- Slog.v(TAG, "Detection mismatch for field " + detectableFieldId);
- }
- } // if
+ // Sets field classification score for field
+ if (userData!= null) {
+ setScore(detectedFieldIds, detectedMatches, userData, viewState.id,
+ currentValue);
+ }
} // else
} // else
}
@@ -1092,9 +1077,8 @@
+ ", changedAutofillIds=" + changedFieldIds
+ ", changedDatasetIds=" + changedDatasetIds
+ ", manuallyFilledIds=" + manuallyFilledIds
- + ", detectableFieldId=" + detectableFieldId
- + ", detectedFieldScore=" + detectedFieldScore
- + ", fieldsClassification=" + fieldsClassification
+ + ", detectedFieldIds=" + detectedFieldIds
+ + ", detectedMatches=" + detectedMatches
);
}
@@ -1116,7 +1100,47 @@
mService.logContextCommitted(id, mClientState, mSelectedDatasetIds, ignoredDatasets,
changedFieldIds, changedDatasetIds,
- manuallyFilledFieldIds, manuallyFilledDatasetIds, fieldsClassification);
+ manuallyFilledFieldIds, manuallyFilledDatasetIds,
+ detectedFieldIds, detectedMatches);
+ }
+
+ /**
+ * Adds the top score match to {@code detectedFieldsIds} and {@code detectedMatches} for
+ * {@code fieldId} based on its {@code currentValue} and {@code userData}.
+ */
+ private static void setScore(@NonNull ArrayList<AutofillId> detectedFieldIds,
+ @NonNull ArrayList<Match> detectedMatches, @NonNull UserData userData,
+ @NonNull AutofillId fieldId, @NonNull AutofillValue currentValue) {
+
+ final String[] userValues = userData.getValues();
+ final String[] remoteIds = userData.getRemoteIds();
+
+ // Sanity check
+ if (userValues == null || remoteIds == null || userValues.length != remoteIds.length) {
+ final int valuesLength = userValues == null ? -1 : userValues.length;
+ final int idsLength = remoteIds == null ? -1 : remoteIds.length;
+ Slog.w(TAG, "setScores(): user data mismatch: values.length = "
+ + valuesLength + ", ids.length = " + idsLength);
+ return;
+ }
+ String remoteId = null;
+ int topScore = 0;
+ for (int i = 0; i < userValues.length; i++) {
+ final String value = userValues[i];
+ final int score = FieldsClassificationScorer.getScore(currentValue, value);
+ if (score > topScore) {
+ topScore = score;
+ remoteId = remoteIds[i];
+ }
+ }
+
+ if (remoteId != null && topScore > 0) {
+ if (sVerbose) Slog.v(TAG, "setScores(): top score for #" + fieldId + " is " + topScore);
+ detectedFieldIds.add(fieldId);
+ detectedMatches.add(new Match(remoteId, topScore));
+ } else if (sVerbose) {
+ Slog.v(TAG, "setScores(): no top score for #" + fieldId + ": " + topScore);
+ }
}
/**