Add translation APIs for virtual views.

Bug: 178046780
Test: manual
Test: atest CtsTranslationTestCases
CTS-Coverage-Bug: 177960696

Change-Id: I99caed0d42b3a164a25c2707eba7057c42c19140

6 files changed, 167 insertions, 20 deletions

diff --git a/core/api/current.txt b/core/api/current.txt
index 3773d3b09299..4584b69809ba 100644
--- a/core/api/current.txt
+++ b/core/api/current.txt
@@ -48771,6 +48771,7 @@ package android.view {
     method protected int[] onCreateDrawableState(int);
     method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
     method @Nullable public android.view.translation.ViewTranslationRequest onCreateTranslationRequest(@NonNull int[]);
+    method public void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer<android.view.translation.ViewTranslationRequest>);
     method @CallSuper protected void onDetachedFromWindow();
     method protected void onDisplayHint(int);
     method public boolean onDragEvent(android.view.DragEvent);
@@ -48816,6 +48817,7 @@ package android.view {
     method public boolean onTouchEvent(android.view.MotionEvent);
     method public boolean onTrackballEvent(android.view.MotionEvent);
     method public void onTranslationResponse(@NonNull android.view.translation.ViewTranslationResponse);
+    method public void onTranslationResponse(@NonNull android.util.LongSparseArray<android.view.translation.ViewTranslationResponse>);
     method @CallSuper public void onVisibilityAggregated(boolean);
     method protected void onVisibilityChanged(@NonNull android.view.View, int);
     method public void onWindowFocusChanged(boolean);
diff --git a/core/api/system-current.txt b/core/api/system-current.txt
index 49b8a901bcbb..0220c02ab1a4 100644
--- a/core/api/system-current.txt
+++ b/core/api/system-current.txt
@@ -14719,6 +14719,7 @@ package android.webkit {
     method public default boolean onCheckIsTextEditor();
     method public void onConfigurationChanged(android.content.res.Configuration);
     method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
+    method @Nullable public default void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer<android.view.translation.ViewTranslationRequest>);
     method public void onDetachedFromWindow();
     method public boolean onDragEvent(android.view.DragEvent);
     method public void onDraw(android.graphics.Canvas);
@@ -14743,6 +14744,7 @@ package android.webkit {
     method public void onStartTemporaryDetach();
     method public boolean onTouchEvent(android.view.MotionEvent);
     method public boolean onTrackballEvent(android.view.MotionEvent);
+    method public default void onTranslationResponse(@NonNull android.util.LongSparseArray<android.view.translation.ViewTranslationResponse>);
     method public void onVisibilityChanged(android.view.View, int);
     method public void onWindowFocusChanged(boolean);
     method public void onWindowVisibilityChanged(int);
diff --git a/core/java/android/view/View.java b/core/java/android/view/View.java
index fb528995b2f7..615dd82fb848 100644
--- a/core/java/android/view/View.java
+++ b/core/java/android/view/View.java
@@ -107,6 +107,7 @@ import android.util.AttributeSet;
 import android.util.FloatProperty;
 import android.util.LayoutDirection;
 import android.util.Log;
+import android.util.LongSparseArray;
 import android.util.LongSparseLongArray;
 import android.util.Pair;
 import android.util.Pools.SynchronizedPool;
@@ -30741,6 +30742,30 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
     }
 
     /**
+     * Returns a {@link ViewTranslationRequest} list which represents the content to be translated
+     * in the virtual view. This is called if this view returned a virtual view structure
+     * from {@link #onProvideContentCaptureStructure} and the system determined that those virtual
+     * views were relevant for translation.
+     *
+     * <p>The default implementation does nothing.</p>
+     *
+     * @param virtualChildIds the virtual child ids which represents the child views in the virtual
+     * view.
+     * @param supportedFormats the supported translation formats. For now, the only possible value
+     * is the {@link android.view.translation.TranslationSpec#DATA_FORMAT_TEXT}.
+     * @param requestsCollector a {@link ViewTranslationRequest} collector that will be called
+     * multiple times to collect the information to be translated in the virtual view. One
+     * {@link ViewTranslationRequest} per virtual child. The {@link ViewTranslationRequest} must
+     * contains the {@link AutofillId} corresponding to the virtualChildIds.
+     */
+    @SuppressLint("NullableCollection")
+    public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
+            @NonNull @DataFormat int[] supportedFormats,
+            @NonNull Consumer<ViewTranslationRequest> requestsCollector) {
+        // no-op
+    }
+
+    /**
      * Returns a {@link ViewTranslationCallback} that is used to display/hide the translated
      * information. If the View supports displaying translated content, it should implement
      * {@link ViewTranslationCallback}.
@@ -30782,13 +30807,29 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
     }
 
     /**
+     * Called when the content from {@link View#onCreateTranslationRequest} had been translated by
+     * the TranslationService.
+     *
+     * <p> The default implementation does nothing.</p>
+     *
+     * @param response a {@link ViewTranslationResponse} SparseArray for the request that send by
+     * {@link View#onCreateTranslationRequests} that contains the translated information which can
+     * be shown in the view. The key of SparseArray is
+     * the virtual child ids.
+     */
+    public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
+        // no-op
+    }
+
+    /**
      * Dispatch to collect the {@link ViewTranslationRequest}s for translation purpose by traversing
      * the hierarchy when the app requests ui translation. Typically, this method should only be
      * overridden by subclasses that provide a view hierarchy (such as {@link ViewGroup}). Other
      * classes should override {@link View#onCreateTranslationRequest}. When requested to start the
      * ui translation, the system will call this method to traverse the view hierarchy to call
      * {@link View#onCreateTranslationRequest} to build {@link ViewTranslationRequest}s and create a
-     * {@link android.view.translation.Translator} to translate the requests.
+     * {@link android.view.translation.Translator} to translate the requests. All the
+     * {@link ViewTranslationRequest}s will be added when the traversal is done.
      *
      * <p> The default implementation will call {@link View#onCreateTranslationRequest} to build
      * {@link ViewTranslationRequest} if the view should be translated. </p>
@@ -30808,14 +30849,15 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
             @NonNull List<ViewTranslationRequest> requests) {
         AutofillId autofillId = getAutofillId();
         if (viewIds.containsKey(autofillId)) {
-            ViewTranslationRequest request = null;
             if (viewIds.get(autofillId) == null) {
-                request = onCreateTranslationRequest(supportedFormats);
+                ViewTranslationRequest request = onCreateTranslationRequest(supportedFormats);
                 if (request != null && request.getKeys().size() > 0) {
                     requests.add(request);
                 }
             } else {
-                // TODO: handle virtual view
+                onCreateTranslationRequests(viewIds.get(autofillId), supportedFormats, request -> {
+                    requests.add(request);
+                });
             }
         }
     }
diff --git a/core/java/android/view/translation/UiTranslationController.java b/core/java/android/view/translation/UiTranslationController.java
index cfe892f8c8c9..6ba40c3cc5d9 100644
--- a/core/java/android/view/translation/UiTranslationController.java
+++ b/core/java/android/view/translation/UiTranslationController.java
@@ -31,6 +31,7 @@ import android.os.HandlerThread;
 import android.os.Process;
 import android.util.ArrayMap;
 import android.util.Log;
+import android.util.LongSparseArray;
 import android.util.Pair;
 import android.util.SparseArray;
 import android.util.SparseIntArray;
@@ -238,29 +239,90 @@ public class UiTranslationController {
         final SparseArray<ViewTranslationResponse> translatedResult =
                 response.getViewTranslationResponses();
         final SparseArray<ViewTranslationResponse> viewsResult = new SparseArray<>();
-        final SparseArray<ViewTranslationResponse> virtualViewsResult = new SparseArray<>();
-        final List<AutofillId> viewIds = new ArrayList<>();
+        final SparseArray<LongSparseArray<ViewTranslationResponse>> virtualViewsResult =
+                new SparseArray<>();
+        // TODO: use another structure to prevent autoboxing?
+        final List<Integer> viewIds = new ArrayList<>();
+
         for (int i = 0; i < translatedResult.size(); i++) {
             final ViewTranslationResponse result = translatedResult.valueAt(i);
             final AutofillId autofillId = result.getAutofillId();
+            if (!viewIds.contains(autofillId.getViewId())) {
+                viewIds.add(autofillId.getViewId());
+            }
             if (autofillId.isNonVirtual()) {
                 viewsResult.put(translatedResult.keyAt(i), result);
-                viewIds.add(autofillId);
             } else {
-                virtualViewsResult.put(translatedResult.keyAt(i), result);
+                final boolean isVirtualViewAdded =
+                        virtualViewsResult.indexOfKey(autofillId.getViewId()) > 0;
+                final LongSparseArray<ViewTranslationResponse> childIds =
+                        isVirtualViewAdded ? virtualViewsResult.get(autofillId.getViewId())
+                                : new LongSparseArray<>();
+                childIds.put(autofillId.getVirtualChildLongId(), result);
+                if (!isVirtualViewAdded) {
+                    virtualViewsResult.put(autofillId.getViewId(), childIds);
+                }
             }
         }
+        // Traverse tree and get views by the responsed AutofillId
+        findViewsTraversalByAutofillIds(viewIds);
+
         if (viewsResult.size() > 0) {
-            onTranslationCompleted(viewsResult, viewIds);
+            onTranslationCompleted(viewsResult);
+        }
+        if (virtualViewsResult.size() > 0) {
+            onVirtualViewTranslationCompleted(virtualViewsResult);
+        }
+    }
+
+    /**
+     * The method is used to handle the translation result for the vertual views.
+     */
+    private void onVirtualViewTranslationCompleted(
+            SparseArray<LongSparseArray<ViewTranslationResponse>> translatedResult) {
+        if (!mActivity.isResumed()) {
+            if (DEBUG) {
+                Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
+            }
+            return;
+        }
+        synchronized (mLock) {
+            if (mCurrentState == STATE_UI_TRANSLATION_FINISHED) {
+                Log.w(TAG, "onTranslationCompleted: the translation state is finished now. "
+                        + "Skip to show the translated text.");
+                return;
+            }
+            for (int i = 0; i < translatedResult.size(); i++) {
+                final AutofillId autofillId = new AutofillId(translatedResult.keyAt(i));
+                final View view = mViews.get(autofillId).get();
+                if (view == null) {
+                    Log.w(TAG, "onTranslationCompleted: the view for autofill id " + autofillId
+                            + " may be gone.");
+                    continue;
+                }
+                final LongSparseArray<ViewTranslationResponse> virtualChildResponse =
+                        translatedResult.valueAt(i);
+                mActivity.runOnUiThread(() -> {
+                    if (view.getViewTranslationCallback() == null) {
+                        if (DEBUG) {
+                            Log.d(TAG, view + " doesn't support showing translation because of "
+                                    + "null ViewTranslationCallback.");
+                        }
+                        return;
+                    }
+                    view.onTranslationResponse(virtualChildResponse);
+                    if (view.getViewTranslationCallback() != null) {
+                        view.getViewTranslationCallback().onShowTranslation(view);
+                    }
+                });
+            }
         }
-        //TODO(b/177960696): call virtual views onTranslationCompleted()
     }
 
     /**
      * The method is used to handle the translation result for non-vertual views.
      */
-    private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult,
-            List<AutofillId> viewIds) {
+    private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult) {
         if (!mActivity.isResumed()) {
             if (DEBUG) {
                 Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
@@ -277,8 +339,6 @@ public class UiTranslationController {
                         + "Skip to show the translated text.");
                 return;
             }
-            // Traverse tree and get views by the responsed AutofillId
-            findViewsTraversalByAutofillIds(viewIds);
             for (int i = 0; i < resultCount; i++) {
                 final ViewTranslationResponse response = translatedResult.valueAt(i);
                 final AutofillId autofillId = response.getAutofillId();
@@ -300,7 +360,9 @@ public class UiTranslationController {
                         return;
                     }
                     view.onTranslationResponse(response);
-                    view.getViewTranslationCallback().onShowTranslation(view);
+                    if (view.getViewTranslationCallback() != null) {
+                        view.getViewTranslationCallback().onShowTranslation(view);
+                    }
                 });
             }
         }
@@ -359,7 +421,7 @@ public class UiTranslationController {
                         childs = new long[childCount];
                         viewIds.put(virtualViewAutofillId, childs);
                     }
-                    int end = childs.length;
+                    int end = childs.length - 1;
                     childs[end] = autofillId.getVirtualChildLongId();
                 }
             }
@@ -403,7 +465,7 @@ public class UiTranslationController {
         return new int[] {TranslationSpec.DATA_FORMAT_TEXT};
     }
 
-    private void findViewsTraversalByAutofillIds(List<AutofillId> sourceViewIds) {
+    private void findViewsTraversalByAutofillIds(List<Integer> sourceViewIds) {
         final ArrayList<ViewRootImpl> roots =
                 WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
         for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
@@ -417,7 +479,7 @@ public class UiTranslationController {
     }
 
     private void findViewsTraversalByAutofillIds(ViewGroup viewGroup,
-            List<AutofillId> sourceViewIds) {
+            List<Integer> sourceViewIds) {
         final int childCount = viewGroup.getChildCount();
         for (int i = 0; i < childCount; ++i) {
             final View child = viewGroup.getChildAt(i);
@@ -429,9 +491,9 @@ public class UiTranslationController {
         }
     }
 
-    private void addViewIfNeeded(List<AutofillId> sourceViewIds, View view) {
+    private void addViewIfNeeded(List<Integer> sourceViewIds, View view) {
         final AutofillId autofillId = view.getAutofillId();
-        if (sourceViewIds.contains(autofillId)) {
+        if (sourceViewIds.contains(autofillId.getViewId()) && !mViews.containsKey(autofillId)) {
             mViews.put(autofillId, new WeakReference<>(view));
         }
     }
diff --git a/core/java/android/webkit/WebView.java b/core/java/android/webkit/WebView.java
index 487d13e40bf3..6d4fa658b3e9 100644
--- a/core/java/android/webkit/WebView.java
+++ b/core/java/android/webkit/WebView.java
@@ -45,6 +45,7 @@ import android.os.StrictMode;
 import android.print.PrintDocumentAdapter;
 import android.util.AttributeSet;
 import android.util.Log;
+import android.util.LongSparseArray;
 import android.util.SparseArray;
 import android.view.DragEvent;
 import android.view.KeyEvent;
@@ -64,6 +65,9 @@ import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
 import android.view.inspector.InspectableProperty;
 import android.view.textclassifier.TextClassifier;
+import android.view.translation.TranslationSpec.DataFormat;
+import android.view.translation.ViewTranslationRequest;
+import android.view.translation.ViewTranslationResponse;
 import android.widget.AbsoluteLayout;
 
 import java.io.BufferedWriter;
@@ -73,6 +77,7 @@ import java.lang.annotation.RetentionPolicy;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
+import java.util.function.Consumer;
 
 /**
  * A View that displays web pages.
@@ -2854,6 +2859,20 @@ public class WebView extends AbsoluteLayout
         return mProvider.getViewDelegate().isVisibleToUserForAutofill(virtualId);
     }
 
+    @Override
+    @Nullable
+    public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
+            @NonNull @DataFormat int[] supportedFormats,
+            @NonNull Consumer<ViewTranslationRequest> requestsCollector) {
+        mProvider.getViewDelegate().onCreateTranslationRequests(virtualChildIds, supportedFormats,
+                requestsCollector);
+    }
+
+    @Override
+    public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
+        mProvider.getViewDelegate().onTranslationResponse(response);
+    }
+
     /** @hide */
     @Override
     public void onInitializeAccessibilityNodeInfoInternal(AccessibilityNodeInfo info) {
diff --git a/core/java/android/webkit/WebViewProvider.java b/core/java/android/webkit/WebViewProvider.java
index 18a110b2be0f..264736008782 100644
--- a/core/java/android/webkit/WebViewProvider.java
+++ b/core/java/android/webkit/WebViewProvider.java
@@ -18,6 +18,7 @@ package android.webkit;
 
 import android.annotation.NonNull;
 import android.annotation.Nullable;
+import android.annotation.SuppressLint;
 import android.annotation.SystemApi;
 import android.content.Intent;
 import android.content.res.Configuration;
@@ -33,6 +34,7 @@ import android.os.Bundle;
 import android.os.Handler;
 import android.os.Message;
 import android.print.PrintDocumentAdapter;
+import android.util.LongSparseArray;
 import android.util.SparseArray;
 import android.view.DragEvent;
 import android.view.KeyEvent;
@@ -47,6 +49,9 @@ import android.view.autofill.AutofillValue;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
 import android.view.textclassifier.TextClassifier;
+import android.view.translation.TranslationSpec.DataFormat;
+import android.view.translation.ViewTranslationRequest;
+import android.view.translation.ViewTranslationResponse;
 import android.webkit.WebView.HitTestResult;
 import android.webkit.WebView.PictureListener;
 import android.webkit.WebView.VisualStateCallback;
@@ -56,6 +61,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.util.Map;
 import java.util.concurrent.Executor;
+import java.util.function.Consumer;
 
 /**
  * WebView backend provider interface: this interface is the abstract backend to a WebView
@@ -358,6 +364,20 @@ public interface WebViewProvider {
                 @SuppressWarnings("unused") int flags) {
         }
 
+        @SuppressLint("NullableCollection")
+        @Nullable
+        default void onCreateTranslationRequests(
+                @NonNull @SuppressWarnings("unused") long[] virtualChildIds,
+                @NonNull @SuppressWarnings("unused") @DataFormat int[] supportedFormats,
+                @NonNull @SuppressWarnings("unused")
+                        Consumer<ViewTranslationRequest> requestsCollector) {
+        }
+
+        default void onTranslationResponse(
+                @NonNull @SuppressWarnings("unused")
+                        LongSparseArray<ViewTranslationResponse> response) {
+        }
+
         public AccessibilityNodeProvider getAccessibilityNodeProvider();
 
         public void onInitializeAccessibilityNodeInfo(AccessibilityNodeInfo info);