1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.view.translation;
18 
19 import static android.view.translation.Helper.ANIMATION_DURATION_MILLIS;
20 import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_FINISHED;
21 import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_PAUSED;
22 import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_RESUMED;
23 import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_STARTED;
24 
25 import android.annotation.NonNull;
26 import android.annotation.WorkerThread;
27 import android.app.Activity;
28 import android.app.assist.ActivityId;
29 import android.content.Context;
30 import android.os.Bundle;
31 import android.os.Handler;
32 import android.os.HandlerThread;
33 import android.os.Process;
34 import android.util.ArrayMap;
35 import android.util.ArraySet;
36 import android.util.Dumpable;
37 import android.util.IntArray;
38 import android.util.Log;
39 import android.util.LongSparseArray;
40 import android.util.Pair;
41 import android.util.SparseArray;
42 import android.util.SparseIntArray;
43 import android.view.View;
44 import android.view.ViewGroup;
45 import android.view.ViewRootImpl;
46 import android.view.WindowManagerGlobal;
47 import android.view.autofill.AutofillId;
48 import android.view.translation.UiTranslationManager.UiTranslationState;
49 import android.widget.TextView;
50 import android.widget.TextViewTranslationCallback;
51 
52 import com.android.internal.util.function.pooled.PooledLambda;
53 
54 import java.io.PrintWriter;
55 import java.lang.ref.WeakReference;
56 import java.util.ArrayList;
57 import java.util.List;
58 import java.util.Map;
59 import java.util.function.BiConsumer;
60 
61 /**
62  * A controller to manage the ui translation requests for the {@link Activity}.
63  *
64  * @hide
65  */
66 public class UiTranslationController implements Dumpable {
67 
68     public static final boolean DEBUG = Log.isLoggable(UiTranslationManager.LOG_TAG, Log.DEBUG);
69 
70     /** @hide */
71     public static final String DUMPABLE_NAME = "UiTranslationController";
72 
73     private static final String TAG = "UiTranslationController";
74 
75     @NonNull
76     private final Activity mActivity;
77     @NonNull
78     private final Context mContext;
79     @NonNull
80     private final Object mLock = new Object();
81 
82     // Each Translator is distinguished by sourceSpec and desSepc.
83     @NonNull
84     private final ArrayMap<Pair<TranslationSpec, TranslationSpec>, Translator> mTranslators;
85     @NonNull
86     private final ArrayMap<AutofillId, WeakReference<View>> mViews;
87     /**
88      * Views for which {@link UiTranslationSpec#shouldPadContentForCompat()} is true.
89      */
90     @NonNull
91     private final ArraySet<AutofillId> mViewsToPadContent;
92     @NonNull
93     private final HandlerThread mWorkerThread;
94     @NonNull
95     private final Handler mWorkerHandler;
96     private int mCurrentState;
97     @NonNull
98     private ArraySet<AutofillId> mLastRequestAutofillIds;
99 
UiTranslationController(Activity activity, Context context)100     public UiTranslationController(Activity activity, Context context) {
101         mActivity = activity;
102         mContext = context;
103         mViews = new ArrayMap<>();
104         mTranslators = new ArrayMap<>();
105         mViewsToPadContent = new ArraySet<>();
106 
107         mWorkerThread =
108                 new HandlerThread("UiTranslationController_" + mActivity.getComponentName(),
109                         Process.THREAD_PRIORITY_FOREGROUND);
110         mWorkerThread.start();
111         mWorkerHandler = mWorkerThread.getThreadHandler();
112         activity.addDumpable(this);
113     }
114 
115     /**
116      * Update the Ui translation state.
117      */
updateUiTranslationState(@iTranslationState int state, TranslationSpec sourceSpec, TranslationSpec targetSpec, List<AutofillId> views, UiTranslationSpec uiTranslationSpec)118     public void updateUiTranslationState(@UiTranslationState int state, TranslationSpec sourceSpec,
119             TranslationSpec targetSpec, List<AutofillId> views,
120             UiTranslationSpec uiTranslationSpec) {
121         if (mActivity.isDestroyed()) {
122             Log.i(TAG, "Cannot update " + stateToString(state) + " for destroyed " + mActivity);
123             return;
124         }
125         boolean isLoggable = Log.isLoggable(UiTranslationManager.LOG_TAG, Log.DEBUG);
126         Log.i(TAG, "updateUiTranslationState state: " + stateToString(state)
127                 + (isLoggable ? (", views: " + views + ", spec: " + uiTranslationSpec) : ""));
128         synchronized (mLock) {
129             mCurrentState = state;
130             if (views != null) {
131                 setLastRequestAutofillIdsLocked(views);
132             }
133         }
134         switch (state) {
135             case STATE_UI_TRANSLATION_STARTED:
136                 if (uiTranslationSpec != null && uiTranslationSpec.shouldPadContentForCompat()) {
137                     synchronized (mLock) {
138                         mViewsToPadContent.addAll(views);
139                         // TODO: Cleanup disappeared views from mViews and mViewsToPadContent at
140                         //  some appropriate place.
141                     }
142                 }
143                 final Pair<TranslationSpec, TranslationSpec> specs =
144                         new Pair<>(sourceSpec, targetSpec);
145                 if (!mTranslators.containsKey(specs)) {
146                     mWorkerHandler.sendMessage(PooledLambda.obtainMessage(
147                             UiTranslationController::createTranslatorAndStart,
148                             UiTranslationController.this, sourceSpec, targetSpec, views));
149                 } else {
150                     onUiTranslationStarted(mTranslators.get(specs), views);
151                 }
152                 break;
153             case STATE_UI_TRANSLATION_PAUSED:
154                 runForEachView((view, callback) -> callback.onHideTranslation(view));
155                 break;
156             case STATE_UI_TRANSLATION_RESUMED:
157                 runForEachView((view, callback) -> callback.onShowTranslation(view));
158                 break;
159             case STATE_UI_TRANSLATION_FINISHED:
160                 destroyTranslators();
161                 runForEachView((view, callback) -> {
162                     view.clearTranslationState();
163                 });
164                 notifyTranslationFinished(/* activityDestroyed= */ false);
165                 synchronized (mLock) {
166                     mViews.clear();
167                 }
168                 break;
169             default:
170                 Log.w(TAG, "onAutoTranslationStateChange(): unknown state: " + state);
171         }
172     }
173 
174     /**
175      * Called when the Activity is destroyed.
176      */
onActivityDestroyed()177     public void onActivityDestroyed() {
178         synchronized (mLock) {
179             Log.i(TAG, "onActivityDestroyed(): mCurrentState is " + stateToString(mCurrentState));
180             if (mCurrentState != STATE_UI_TRANSLATION_FINISHED) {
181                 notifyTranslationFinished(/* activityDestroyed= */ true);
182             }
183             mViews.clear();
184             destroyTranslators();
185             mWorkerThread.quitSafely();
186         }
187     }
188 
notifyTranslationFinished(boolean activityDestroyed)189     private void notifyTranslationFinished(boolean activityDestroyed) {
190         UiTranslationManager manager = mContext.getSystemService(UiTranslationManager.class);
191         if (manager != null) {
192             manager.onTranslationFinished(activityDestroyed,
193                     new ActivityId(mActivity.getTaskId(), mActivity.getShareableActivityToken()),
194                     mActivity.getComponentName());
195         }
196     }
197 
setLastRequestAutofillIdsLocked(List<AutofillId> views)198     private void setLastRequestAutofillIdsLocked(List<AutofillId> views) {
199         if (mLastRequestAutofillIds == null) {
200             mLastRequestAutofillIds = new ArraySet<>();
201         }
202         if (mLastRequestAutofillIds.size() > 0) {
203             mLastRequestAutofillIds.clear();
204         }
205         mLastRequestAutofillIds.addAll(views);
206     }
207 
208     @Override
getDumpableName()209     public String getDumpableName() {
210         return DUMPABLE_NAME;
211     }
212 
213     @Override
dump(PrintWriter pw, String[] args)214     public void dump(PrintWriter pw, String[] args) {
215         String outerPrefix = "";
216         pw.print(outerPrefix); pw.println("UiTranslationController:");
217         final String pfx = outerPrefix + "  ";
218         pw.print(pfx); pw.print("activity: "); pw.print(mActivity);
219         pw.print(pfx); pw.print("resumed: "); pw.println(mActivity.isResumed());
220         pw.print(pfx); pw.print("current state: "); pw.println(mCurrentState);
221         final int translatorSize = mTranslators.size();
222         pw.print(outerPrefix); pw.print("number translator: "); pw.println(translatorSize);
223         for (int i = 0; i < translatorSize; i++) {
224             pw.print(outerPrefix); pw.print("#"); pw.println(i);
225             final Translator translator = mTranslators.valueAt(i);
226             translator.dump(outerPrefix, pw);
227             pw.println();
228         }
229         synchronized (mLock) {
230             final int viewSize = mViews.size();
231             pw.print(outerPrefix); pw.print("number views: "); pw.println(viewSize);
232             for (int i = 0; i < viewSize; i++) {
233                 pw.print(outerPrefix); pw.print("#"); pw.println(i);
234                 final AutofillId autofillId = mViews.keyAt(i);
235                 final View view = mViews.valueAt(i).get();
236                 pw.print(pfx); pw.print("autofillId: "); pw.println(autofillId);
237                 pw.print(pfx); pw.print("view:"); pw.println(view);
238             }
239             pw.print(outerPrefix); pw.print("padded views: "); pw.println(mViewsToPadContent);
240         }
241         if (Log.isLoggable(UiTranslationManager.LOG_TAG, Log.DEBUG)) {
242             dumpViewByTraversal(outerPrefix, pw);
243         }
244     }
245 
dumpViewByTraversal(String outerPrefix, PrintWriter pw)246     private void dumpViewByTraversal(String outerPrefix, PrintWriter pw) {
247         final ArrayList<ViewRootImpl> roots =
248                 WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
249         pw.print(outerPrefix); pw.println("Dump views:");
250         for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
251             final View rootView = roots.get(rootNum).getView();
252             if (rootView instanceof ViewGroup) {
253                 dumpChildren((ViewGroup) rootView, outerPrefix, pw);
254             } else {
255                 dumpViewInfo(rootView, outerPrefix, pw);
256             }
257         }
258     }
259 
dumpChildren(ViewGroup viewGroup, String outerPrefix, PrintWriter pw)260     private void dumpChildren(ViewGroup viewGroup, String outerPrefix, PrintWriter pw) {
261         final int childCount = viewGroup.getChildCount();
262         for (int i = 0; i < childCount; ++i) {
263             final View child = viewGroup.getChildAt(i);
264             if (child instanceof ViewGroup) {
265                 pw.print(outerPrefix); pw.println("Children: ");
266                 pw.print(outerPrefix); pw.print(outerPrefix); pw.println(child);
267                 dumpChildren((ViewGroup) child, outerPrefix, pw);
268             } else {
269                 pw.print(outerPrefix); pw.println("End Children: ");
270                 pw.print(outerPrefix); pw.print(outerPrefix); pw.print(child);
271                 dumpViewInfo(child, outerPrefix, pw);
272             }
273         }
274     }
275 
dumpViewInfo(View view, String outerPrefix, PrintWriter pw)276     private void dumpViewInfo(View view, String outerPrefix, PrintWriter pw) {
277         final AutofillId autofillId = view.getAutofillId();
278         pw.print(outerPrefix); pw.print("autofillId: "); pw.print(autofillId);
279         // TODO: print TranslationTransformation
280         boolean isContainsView = false;
281         boolean isRequestedView = false;
282         synchronized (mLock) {
283             if (mLastRequestAutofillIds.contains(autofillId)) {
284                 isRequestedView = true;
285             }
286             final WeakReference<View> viewRef = mViews.get(autofillId);
287             if (viewRef != null && viewRef.get() != null) {
288                 isContainsView = true;
289             }
290         }
291         pw.print(outerPrefix); pw.print("isContainsView: "); pw.print(isContainsView);
292         pw.print(outerPrefix); pw.print("isRequestedView: "); pw.println(isRequestedView);
293     }
294 
295     /**
296      * The method is used by {@link Translator}, it will be called when the translation is done. The
297      * translation result can be get from here.
298      */
onTranslationCompleted(TranslationResponse response)299     public void onTranslationCompleted(TranslationResponse response) {
300         if (response == null || response.getTranslationStatus()
301                 != TranslationResponse.TRANSLATION_STATUS_SUCCESS) {
302             Log.w(TAG, "Fail result from TranslationService, status=" + (response == null
303                     ? "null"
304                     : response.getTranslationStatus()));
305             return;
306         }
307         final SparseArray<ViewTranslationResponse> translatedResult =
308                 response.getViewTranslationResponses();
309         final SparseArray<ViewTranslationResponse> viewsResult = new SparseArray<>();
310         final SparseArray<LongSparseArray<ViewTranslationResponse>> virtualViewsResult =
311                 new SparseArray<>();
312         final IntArray viewIds = new IntArray(1);
313         for (int i = 0; i < translatedResult.size(); i++) {
314             final ViewTranslationResponse result = translatedResult.valueAt(i);
315             final AutofillId autofillId = result.getAutofillId();
316             if (viewIds.indexOf(autofillId.getViewId()) < 0) {
317                 viewIds.add(autofillId.getViewId());
318             }
319             if (autofillId.isNonVirtual()) {
320                 viewsResult.put(translatedResult.keyAt(i), result);
321             } else {
322                 final boolean isVirtualViewAdded =
323                         virtualViewsResult.indexOfKey(autofillId.getViewId()) >= 0;
324                 final LongSparseArray<ViewTranslationResponse> childIds =
325                         isVirtualViewAdded ? virtualViewsResult.get(autofillId.getViewId())
326                                 : new LongSparseArray<>();
327                 childIds.put(autofillId.getVirtualChildLongId(), result);
328                 if (!isVirtualViewAdded) {
329                     virtualViewsResult.put(autofillId.getViewId(), childIds);
330                 }
331             }
332         }
333         // Traverse tree and get views by the responsed AutofillId
334         findViewsTraversalByAutofillIds(viewIds);
335 
336         if (viewsResult.size() > 0) {
337             onTranslationCompleted(viewsResult);
338         }
339         if (virtualViewsResult.size() > 0) {
340             onVirtualViewTranslationCompleted(virtualViewsResult);
341         }
342     }
343 
344     /**
345      * The method is used to handle the translation result for the vertual views.
346      */
onVirtualViewTranslationCompleted( SparseArray<LongSparseArray<ViewTranslationResponse>> translatedResult)347     private void onVirtualViewTranslationCompleted(
348             SparseArray<LongSparseArray<ViewTranslationResponse>> translatedResult) {
349         boolean isLoggable = Log.isLoggable(UiTranslationManager.LOG_TAG, Log.DEBUG);
350         if (mActivity.isDestroyed()) {
351             Log.v(TAG, "onTranslationCompleted:" + mActivity + "is destroyed.");
352             return;
353         }
354         synchronized (mLock) {
355             if (mCurrentState == STATE_UI_TRANSLATION_FINISHED) {
356                 Log.w(TAG, "onTranslationCompleted: the translation state is finished now. "
357                         + "Skip to show the translated text.");
358                 return;
359             }
360             for (int i = 0; i < translatedResult.size(); i++) {
361                 final AutofillId autofillId = new AutofillId(translatedResult.keyAt(i));
362                 final WeakReference<View> viewRef = mViews.get(autofillId);
363                 if (viewRef == null) {
364                     continue;
365                 }
366                 final View view = viewRef.get();
367                 if (view == null) {
368                     Log.w(TAG, "onTranslationCompleted: the view for autofill id " + autofillId
369                             + " may be gone.");
370                     continue;
371                 }
372                 final LongSparseArray<ViewTranslationResponse> virtualChildResponse =
373                         translatedResult.valueAt(i);
374                 if (isLoggable) {
375                     Log.v(TAG, "onVirtualViewTranslationCompleted: received response for "
376                             + "AutofillId " + autofillId);
377                 }
378                 view.onVirtualViewTranslationResponses(virtualChildResponse);
379                 if (mCurrentState == STATE_UI_TRANSLATION_PAUSED) {
380                     return;
381                 }
382                 mActivity.runOnUiThread(() -> {
383                     if (view.getViewTranslationCallback() == null) {
384                         if (isLoggable) {
385                             Log.d(TAG, view + " doesn't support showing translation because of "
386                                     + "null ViewTranslationCallback.");
387                         }
388                         return;
389                     }
390                     if (view.getViewTranslationCallback() != null) {
391                         view.getViewTranslationCallback().onShowTranslation(view);
392                     }
393                 });
394             }
395         }
396     }
397 
398     /**
399      * The method is used to handle the translation result for non-vertual views.
400      */
onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult)401     private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult) {
402         boolean isLoggable = Log.isLoggable(UiTranslationManager.LOG_TAG, Log.DEBUG);
403         if (mActivity.isDestroyed()) {
404             Log.v(TAG, "onTranslationCompleted:" + mActivity + "is destroyed.");
405             return;
406         }
407         final int resultCount = translatedResult.size();
408         if (isLoggable) {
409             Log.v(TAG, "onTranslationCompleted: receive " + resultCount + " responses.");
410         }
411         synchronized (mLock) {
412             if (mCurrentState == STATE_UI_TRANSLATION_FINISHED) {
413                 Log.w(TAG, "onTranslationCompleted: the translation state is finished now. "
414                         + "Skip to show the translated text.");
415                 return;
416             }
417             for (int i = 0; i < resultCount; i++) {
418                 final ViewTranslationResponse response = translatedResult.valueAt(i);
419                 if (isLoggable) {
420                     Log.v(TAG, "onTranslationCompleted: "
421                             + sanitizedViewTranslationResponse(response));
422                 }
423                 final AutofillId autofillId = response.getAutofillId();
424                 if (autofillId == null) {
425                     Log.w(TAG, "No AutofillId is set in ViewTranslationResponse");
426                     continue;
427                 }
428                 final WeakReference<View> viewRef = mViews.get(autofillId);
429                 if (viewRef == null) {
430                     continue;
431                 }
432                 final View view = viewRef.get();
433                 if (view == null) {
434                     Log.w(TAG, "onTranslationCompleted: the view for autofill id " + autofillId
435                             + " may be gone.");
436                     continue;
437                 }
438                 int currentState;
439                 currentState = mCurrentState;
440                 mActivity.runOnUiThread(() -> {
441                     ViewTranslationCallback callback = view.getViewTranslationCallback();
442                     if (view.getViewTranslationResponse() != null
443                             && view.getViewTranslationResponse().equals(response)) {
444                         if (callback instanceof TextViewTranslationCallback) {
445                             TextViewTranslationCallback textViewCallback =
446                                     (TextViewTranslationCallback) callback;
447                             if (textViewCallback.isShowingTranslation()
448                                     || textViewCallback.isAnimationRunning()) {
449                                 if (isLoggable) {
450                                     Log.d(TAG, "Duplicate ViewTranslationResponse for " + autofillId
451                                             + ". Ignoring.");
452                                 }
453                                 return;
454                             }
455                         }
456                     }
457                     if (callback == null) {
458                         if (view instanceof TextView) {
459                             // developer doesn't provide their override, we set the default TextView
460                             // implementation.
461                             callback = new TextViewTranslationCallback();
462                             view.setViewTranslationCallback(callback);
463                         } else {
464                             if (isLoggable) {
465                                 Log.d(TAG, view + " doesn't support showing translation because of "
466                                         + "null ViewTranslationCallback.");
467                             }
468                             return;
469                         }
470                     }
471                     callback.setAnimationDurationMillis(ANIMATION_DURATION_MILLIS);
472                     if (mViewsToPadContent.contains(autofillId)) {
473                         callback.enableContentPadding();
474                     }
475                     view.onViewTranslationResponse(response);
476                     if (currentState == STATE_UI_TRANSLATION_PAUSED) {
477                         return;
478                     }
479                     callback.onShowTranslation(view);
480                 });
481             }
482         }
483     }
484 
485     /**
486      * Creates a Translator for the given source and target translation specs and start the ui
487      * translation when the Translator is created successfully.
488      */
489     @WorkerThread
createTranslatorAndStart(TranslationSpec sourceSpec, TranslationSpec targetSpec, List<AutofillId> views)490     private void createTranslatorAndStart(TranslationSpec sourceSpec, TranslationSpec targetSpec,
491             List<AutofillId> views) {
492         // Create Translator
493         final Translator translator = createTranslatorIfNeeded(sourceSpec, targetSpec);
494         if (translator == null) {
495             Log.w(TAG, "Can not create Translator for sourceSpec:" + sourceSpec + " targetSpec:"
496                     + targetSpec);
497             return;
498         }
499         onUiTranslationStarted(translator, views);
500     }
501 
502     @WorkerThread
sendTranslationRequest(Translator translator, List<ViewTranslationRequest> requests)503     private void sendTranslationRequest(Translator translator,
504             List<ViewTranslationRequest> requests) {
505         if (requests.size() == 0) {
506             Log.w(TAG, "No ViewTranslationRequest was collected.");
507             return;
508         }
509         final TranslationRequest request = new TranslationRequest.Builder()
510                 .setViewTranslationRequests(requests)
511                 .build();
512         if (Log.isLoggable(UiTranslationManager.LOG_TAG, Log.DEBUG)) {
513             StringBuilder msg = new StringBuilder("sendTranslationRequest:{requests=[");
514             for (ViewTranslationRequest viewRequest: requests) {
515                 msg.append("{request=")
516                         .append(sanitizedViewTranslationRequest(viewRequest))
517                         .append("}, ");
518             }
519             Log.d(TAG, "sendTranslationRequest: " + msg.toString());
520         }
521         translator.requestUiTranslate(request, (r) -> r.run(), this::onTranslationCompleted);
522     }
523 
524     /**
525      * Called when there is an ui translation request comes to request view translation.
526      */
onUiTranslationStarted(Translator translator, List<AutofillId> views)527     private void onUiTranslationStarted(Translator translator, List<AutofillId> views) {
528         synchronized (mLock) {
529             // Filter the request views' AutofillId
530             SparseIntArray virtualViewChildCount = getRequestVirtualViewChildCount(views);
531             Map<AutofillId, long[]> viewIds = new ArrayMap<>();
532             Map<AutofillId, Integer> unusedIndices = null;
533             for (int i = 0; i < views.size(); i++) {
534                 AutofillId autofillId = views.get(i);
535                 if (autofillId.isNonVirtual()) {
536                     viewIds.put(autofillId, null);
537                 } else {
538                     if (unusedIndices == null) {
539                         unusedIndices = new ArrayMap<>();
540                     }
541                     // The virtual id get from content capture is long, see getVirtualChildLongId()
542                     // e.g. 1001, 1001:2, 1002:1 -> 1001, <1,2>; 1002, <1>
543                     AutofillId virtualViewAutofillId = new AutofillId(autofillId.getViewId());
544                     long[] childs;
545                     int end = 0;
546                     if (viewIds.containsKey(virtualViewAutofillId)) {
547                         childs = viewIds.get(virtualViewAutofillId);
548                         end = unusedIndices.get(virtualViewAutofillId);
549                     } else {
550                         int childCount = virtualViewChildCount.get(autofillId.getViewId());
551                         childs = new long[childCount];
552                         viewIds.put(virtualViewAutofillId, childs);
553                     }
554                     unusedIndices.put(virtualViewAutofillId, end + 1);
555                     childs[end] = autofillId.getVirtualChildLongId();
556                 }
557             }
558             ArrayList<ViewTranslationRequest> requests = new ArrayList<>();
559             int[] supportedFormats = getSupportedFormatsLocked();
560             ArrayList<ViewRootImpl> roots =
561                     WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
562             TranslationCapability capability =
563                     getTranslationCapability(translator.getTranslationContext());
564             mActivity.runOnUiThread(() -> {
565                 // traverse the hierarchy to collect ViewTranslationRequests
566                 for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
567                     View rootView = roots.get(rootNum).getView();
568                     rootView.dispatchCreateViewTranslationRequest(viewIds, supportedFormats,
569                             capability, requests);
570                 }
571                 mWorkerHandler.sendMessage(PooledLambda.obtainMessage(
572                         UiTranslationController::sendTranslationRequest,
573                         UiTranslationController.this, translator, requests));
574             });
575         }
576     }
577 
getRequestVirtualViewChildCount(List<AutofillId> views)578     private SparseIntArray getRequestVirtualViewChildCount(List<AutofillId> views) {
579         SparseIntArray virtualViewCount = new SparseIntArray();
580         for (int i = 0; i < views.size(); i++) {
581             AutofillId autofillId = views.get(i);
582             if (!autofillId.isNonVirtual()) {
583                 int virtualViewId = autofillId.getViewId();
584                 if (virtualViewCount.indexOfKey(virtualViewId) < 0) {
585                     virtualViewCount.put(virtualViewId, 1);
586                 } else {
587                     virtualViewCount.put(virtualViewId, (virtualViewCount.get(virtualViewId) + 1));
588                 }
589             }
590         }
591         return virtualViewCount;
592     }
593 
getSupportedFormatsLocked()594     private int[] getSupportedFormatsLocked() {
595         // We only support text now
596         return new int[] {TranslationSpec.DATA_FORMAT_TEXT};
597     }
598 
getTranslationCapability(TranslationContext translationContext)599     private TranslationCapability getTranslationCapability(TranslationContext translationContext) {
600         // We only support text to text capability now, we will query real status from service when
601         // we support more translation capabilities.
602         return new TranslationCapability(TranslationCapability.STATE_ON_DEVICE,
603                 translationContext.getSourceSpec(),
604                 translationContext.getTargetSpec(), /* uiTranslationEnabled= */ true,
605                 /* supportedTranslationFlags= */ 0);
606     }
607 
findViewsTraversalByAutofillIds(IntArray sourceViewIds)608     private void findViewsTraversalByAutofillIds(IntArray sourceViewIds) {
609         final ArrayList<ViewRootImpl> roots =
610                 WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
611         for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
612             final View rootView = roots.get(rootNum).getView();
613             if (rootView instanceof ViewGroup) {
614                 findViewsTraversalByAutofillIds((ViewGroup) rootView, sourceViewIds);
615             }
616             addViewIfNeeded(sourceViewIds, rootView);
617         }
618     }
619 
findViewsTraversalByAutofillIds(ViewGroup viewGroup, IntArray sourceViewIds)620     private void findViewsTraversalByAutofillIds(ViewGroup viewGroup,
621             IntArray sourceViewIds) {
622         final int childCount = viewGroup.getChildCount();
623         for (int i = 0; i < childCount; ++i) {
624             final View child = viewGroup.getChildAt(i);
625             if (child instanceof ViewGroup) {
626                 findViewsTraversalByAutofillIds((ViewGroup) child, sourceViewIds);
627             }
628             addViewIfNeeded(sourceViewIds, child);
629         }
630     }
631 
addViewIfNeeded(IntArray sourceViewIds, View view)632     private void addViewIfNeeded(IntArray sourceViewIds, View view) {
633         final AutofillId autofillId = view.getAutofillId();
634         if (autofillId != null && (sourceViewIds.indexOf(autofillId.getViewId()) >= 0)
635                 && !mViews.containsKey(autofillId)) {
636             mViews.put(autofillId, new WeakReference<>(view));
637         }
638     }
639 
runForEachView(BiConsumer<View, ViewTranslationCallback> action)640     private void runForEachView(BiConsumer<View, ViewTranslationCallback> action) {
641         synchronized (mLock) {
642             boolean isLoggable = Log.isLoggable(UiTranslationManager.LOG_TAG, Log.DEBUG);
643             final ArrayMap<AutofillId, WeakReference<View>> views = new ArrayMap<>(mViews);
644             if (views.size() == 0) {
645                 Log.w(TAG, "No views can be excuted for runForEachView.");
646             }
647             mActivity.runOnUiThread(() -> {
648                 final int viewCounts = views.size();
649                 for (int i = 0; i < viewCounts; i++) {
650                     final View view = views.valueAt(i).get();
651                     if (isLoggable) {
652                         Log.d(TAG, "runForEachView for autofillId = " + (view != null
653                                 ? view.getAutofillId() : " null"));
654                     }
655                     if (view == null || view.getViewTranslationCallback() == null) {
656                         if (isLoggable) {
657                             Log.d(TAG, "View was gone or ViewTranslationCallback for autofillId "
658                                     + "= " + views.keyAt(i));
659                         }
660                         continue;
661                     }
662                     action.accept(view, view.getViewTranslationCallback());
663                 }
664             });
665         }
666     }
667 
createTranslatorIfNeeded( TranslationSpec sourceSpec, TranslationSpec targetSpec)668     private Translator createTranslatorIfNeeded(
669             TranslationSpec sourceSpec, TranslationSpec targetSpec) {
670         final TranslationManager tm = mContext.getSystemService(TranslationManager.class);
671         if (tm == null) {
672             Log.e(TAG, "Can not find TranslationManager when trying to create translator.");
673             return null;
674         }
675         final TranslationContext translationContext =
676                 new TranslationContext.Builder(sourceSpec, targetSpec)
677                         .setActivityId(
678                                 new ActivityId(
679                                         mActivity.getTaskId(),
680                                         mActivity.getShareableActivityToken()))
681                         .build();
682         final Translator translator = tm.createTranslator(translationContext);
683         if (translator != null) {
684             final Pair<TranslationSpec, TranslationSpec> specs = new Pair<>(sourceSpec, targetSpec);
685             mTranslators.put(specs, translator);
686         }
687         return translator;
688     }
689 
destroyTranslators()690     private void destroyTranslators() {
691         synchronized (mLock) {
692             final int count = mTranslators.size();
693             for (int i = 0; i < count; i++) {
694                 Translator translator = mTranslators.valueAt(i);
695                 translator.destroy();
696             }
697             mTranslators.clear();
698         }
699     }
700 
701     /**
702      * Returns a string representation of the state.
703      */
stateToString(@iTranslationState int state)704     public static String stateToString(@UiTranslationState int state) {
705         switch (state) {
706             case STATE_UI_TRANSLATION_STARTED:
707                 return "UI_TRANSLATION_STARTED";
708             case STATE_UI_TRANSLATION_PAUSED:
709                 return "UI_TRANSLATION_PAUSED";
710             case STATE_UI_TRANSLATION_RESUMED:
711                 return "UI_TRANSLATION_RESUMED";
712             case STATE_UI_TRANSLATION_FINISHED:
713                 return "UI_TRANSLATION_FINISHED";
714             default:
715                 return "Unknown state (" + state + ")";
716         }
717     }
718 
719     /**
720      * Returns a sanitized string representation of {@link ViewTranslationRequest};
721      */
sanitizedViewTranslationRequest(@onNull ViewTranslationRequest request)722     private static String sanitizedViewTranslationRequest(@NonNull ViewTranslationRequest request) {
723         StringBuilder msg = new StringBuilder("ViewTranslationRequest:{values=[");
724         for (String key: request.getKeys()) {
725             final TranslationRequestValue value = request.getValue(key);
726             msg.append("{text=").append(value.getText() == null
727                     ? "null"
728                     : "string[" + value.getText().length() + "]}, ");
729         }
730         return msg.toString();
731     }
732 
733     /**
734      * Returns a sanitized string representation of {@link ViewTranslationResponse};
735      */
sanitizedViewTranslationResponse( @onNull ViewTranslationResponse response)736     private static String sanitizedViewTranslationResponse(
737             @NonNull ViewTranslationResponse response) {
738         StringBuilder msg = new StringBuilder("ViewTranslationResponse:{values=[");
739         for (String key: response.getKeys()) {
740             final TranslationResponseValue value = response.getValue(key);
741             msg.append("{status=").append(value.getStatusCode()).append(", ");
742             msg.append("text=").append(value.getText() == null
743                     ? "null"
744                     : "string[" + value.getText().length() + "], ");
745             final Bundle definitions =
746                     (Bundle) value.getExtras().get(TranslationResponseValue.EXTRA_DEFINITIONS);
747             if (definitions != null) {
748                 msg.append("definitions={");
749                 for (String partOfSpeech : definitions.keySet()) {
750                     msg.append(partOfSpeech).append(":[");
751                     for (CharSequence definition : definitions.getCharSequenceArray(partOfSpeech)) {
752                         msg.append(definition == null
753                                 ? "null, "
754                                 : "string[" + definition.length() + "], ");
755                     }
756                     msg.append("], ");
757                 }
758                 msg.append("}");
759             }
760             msg.append("transliteration=").append(value.getTransliteration() == null
761                     ? "null"
762                     : "string[" + value.getTransliteration().length() + "]}, ");
763         }
764         return msg.toString();
765     }
766 }
767