1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server.autofill;
18 
19 import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
20 import static com.android.server.autofill.Helper.sDebug;
21 import static com.android.server.autofill.Helper.sVerbose;
22 
23 import android.annotation.BinderThread;
24 import android.annotation.NonNull;
25 import android.annotation.Nullable;
26 import android.content.ComponentName;
27 import android.os.Bundle;
28 import android.os.Handler;
29 import android.os.RemoteException;
30 import android.util.Slog;
31 import android.view.autofill.AutofillId;
32 import android.view.inputmethod.InlineSuggestion;
33 import android.view.inputmethod.InlineSuggestionsRequest;
34 import android.view.inputmethod.InlineSuggestionsResponse;
35 
36 import com.android.internal.annotations.GuardedBy;
37 import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback;
38 import com.android.internal.inputmethod.IInlineSuggestionsResponseCallback;
39 import com.android.internal.inputmethod.InlineSuggestionsRequestInfo;
40 import com.android.server.autofill.ui.InlineFillUi;
41 import com.android.server.inputmethod.InputMethodManagerInternal;
42 
43 import java.lang.ref.WeakReference;
44 import java.util.List;
45 import java.util.Optional;
46 import java.util.function.Consumer;
47 
48 /**
49  * Maintains an inline suggestion session with the IME.
50  *
51  * <p> Each session corresponds to one request from the Autofill manager service to create an
52  * {@link InlineSuggestionsRequest}. It's responsible for receiving callbacks from the IME and
53  * sending {@link android.view.inputmethod.InlineSuggestionsResponse} to IME.
54  */
55 final class AutofillInlineSuggestionsRequestSession {
56 
57     private static final String TAG = AutofillInlineSuggestionsRequestSession.class.getSimpleName();
58 
59     @NonNull
60     private final InputMethodManagerInternal mInputMethodManagerInternal;
61     private final int mUserId;
62     @NonNull
63     private final ComponentName mComponentName;
64     @NonNull
65     private final Object mLock;
66     @NonNull
67     private final Handler mHandler;
68     @NonNull
69     private final Bundle mUiExtras;
70     @NonNull
71     private final InlineFillUi.InlineUiEventCallback mUiCallback;
72 
73     @GuardedBy("mLock")
74     @NonNull
75     private AutofillId mAutofillId;
76     @GuardedBy("mLock")
77     @Nullable
78     private Consumer<InlineSuggestionsRequest> mImeRequestConsumer;
79 
80     @GuardedBy("mLock")
81     private boolean mImeRequestReceived;
82     @GuardedBy("mLock")
83     @Nullable
84     private InlineSuggestionsRequest mImeRequest;
85     @GuardedBy("mLock")
86     @Nullable
87     private IInlineSuggestionsResponseCallback mResponseCallback;
88 
89     @GuardedBy("mLock")
90     @Nullable
91     private AutofillId mImeCurrentFieldId;
92     @GuardedBy("mLock")
93     private boolean mImeInputStarted;
94     @GuardedBy("mLock")
95     private boolean mImeInputViewStarted;
96     @GuardedBy("mLock")
97     @Nullable
98     private InlineFillUi mInlineFillUi;
99     @GuardedBy("mLock")
100     private Boolean mPreviousResponseIsNotEmpty = null;
101 
102     @GuardedBy("mLock")
103     private boolean mDestroyed = false;
104     @GuardedBy("mLock")
105     private boolean mPreviousHasNonPinSuggestionShow;
106     @GuardedBy("mLock")
107     private boolean mImeSessionInvalidated = false;
108 
109     private boolean mImeShowing = false;
110 
AutofillInlineSuggestionsRequestSession( @onNull InputMethodManagerInternal inputMethodManagerInternal, int userId, @NonNull ComponentName componentName, @NonNull Handler handler, @NonNull Object lock, @NonNull AutofillId autofillId, @NonNull Consumer<InlineSuggestionsRequest> requestConsumer, @NonNull Bundle uiExtras, @NonNull InlineFillUi.InlineUiEventCallback callback)111     AutofillInlineSuggestionsRequestSession(
112             @NonNull InputMethodManagerInternal inputMethodManagerInternal, int userId,
113             @NonNull ComponentName componentName, @NonNull Handler handler, @NonNull Object lock,
114             @NonNull AutofillId autofillId,
115             @NonNull Consumer<InlineSuggestionsRequest> requestConsumer, @NonNull Bundle uiExtras,
116             @NonNull InlineFillUi.InlineUiEventCallback callback) {
117         mInputMethodManagerInternal = inputMethodManagerInternal;
118         mUserId = userId;
119         mComponentName = componentName;
120         mHandler = handler;
121         mLock = lock;
122         mUiExtras = uiExtras;
123         mUiCallback = callback;
124 
125         mAutofillId = autofillId;
126         mImeRequestConsumer = requestConsumer;
127     }
128 
129     @GuardedBy("mLock")
130     @NonNull
getAutofillIdLocked()131     AutofillId getAutofillIdLocked() {
132         return mAutofillId;
133     }
134 
135     /**
136      * Returns the {@link InlineSuggestionsRequest} provided by IME.
137      *
138      * <p> The caller is responsible for making sure Autofill hears back from IME before calling
139      * this method, using the {@link #mImeRequestConsumer}.
140      */
141     @GuardedBy("mLock")
getInlineSuggestionsRequestLocked()142     Optional<InlineSuggestionsRequest> getInlineSuggestionsRequestLocked() {
143         if (mDestroyed) {
144             return Optional.empty();
145         }
146         return Optional.ofNullable(mImeRequest);
147     }
148 
149     /**
150      * Requests showing the inline suggestion in the IME when the IME becomes visible and is focused
151      * on the {@code autofillId}.
152      *
153      * @return false if the IME callback is not available.
154      */
155     @GuardedBy("mLock")
onInlineSuggestionsResponseLocked(@onNull InlineFillUi inlineFillUi)156     boolean onInlineSuggestionsResponseLocked(@NonNull InlineFillUi inlineFillUi) {
157         if (mDestroyed) {
158             return false;
159         }
160         if (sDebug) {
161             Slog.d(TAG,
162                     "onInlineSuggestionsResponseLocked called for:" + inlineFillUi.getAutofillId());
163         }
164         if (mImeRequest == null || mResponseCallback == null || mImeSessionInvalidated) {
165             return false;
166         }
167         // TODO(b/151123764): each session should only correspond to one field.
168         mAutofillId = inlineFillUi.getAutofillId();
169         mInlineFillUi = inlineFillUi;
170         maybeUpdateResponseToImeLocked();
171         return true;
172     }
173 
174     /**
175      * Prevents further interaction with the IME. Must be called before starting a new request
176      * session to avoid unwanted behavior from two overlapping requests.
177      */
178     @GuardedBy("mLock")
destroySessionLocked()179     void destroySessionLocked() {
180         mDestroyed = true;
181 
182         if (!mImeRequestReceived) {
183             Slog.w(TAG,
184                     "Never received an InlineSuggestionsRequest from the IME for " + mAutofillId);
185         }
186     }
187 
188     /**
189      * Requests the IME to create an {@link InlineSuggestionsRequest}.
190      *
191      * <p> This method should only be called once per session.
192      */
193     @GuardedBy("mLock")
onCreateInlineSuggestionsRequestLocked()194     void onCreateInlineSuggestionsRequestLocked() {
195         if (mDestroyed) {
196             return;
197         }
198         mImeSessionInvalidated = false;
199         if (sDebug) Slog.d(TAG, "onCreateInlineSuggestionsRequestLocked called: " + mAutofillId);
200         mInputMethodManagerInternal.onCreateInlineSuggestionsRequest(mUserId,
201                 new InlineSuggestionsRequestInfo(mComponentName, mAutofillId, mUiExtras),
202                 new InlineSuggestionsRequestCallbackImpl(this));
203     }
204 
205     /**
206      * Clear the locally cached inline fill UI, but don't clear the suggestion in IME.
207      *
208      * See also {@link AutofillInlineSessionController#resetInlineFillUiLocked()}
209      */
210     @GuardedBy("mLock")
resetInlineFillUiLocked()211     void resetInlineFillUiLocked() {
212         mInlineFillUi = null;
213     }
214 
215     /**
216      * Optionally sends inline response to the IME, depending on the current state.
217      */
218     @GuardedBy("mLock")
maybeUpdateResponseToImeLocked()219     private void maybeUpdateResponseToImeLocked() {
220         if (sVerbose) Slog.v(TAG, "maybeUpdateResponseToImeLocked called");
221         if (mDestroyed || mResponseCallback == null) {
222             return;
223         }
224         if (mImeInputViewStarted && mInlineFillUi != null && match(mAutofillId,
225                 mImeCurrentFieldId)) {
226             // if IME is visible, and response is not null, send the response
227             InlineSuggestionsResponse response = mInlineFillUi.getInlineSuggestionsResponse();
228             boolean isEmptyResponse = response.getInlineSuggestions().isEmpty();
229             if (isEmptyResponse && Boolean.FALSE.equals(mPreviousResponseIsNotEmpty)) {
230                 // No-op if both the previous response and current response are empty.
231                 return;
232             }
233             maybeNotifyFillUiEventLocked(response.getInlineSuggestions());
234             updateResponseToImeUncheckLocked(response);
235             mPreviousResponseIsNotEmpty = !isEmptyResponse;
236         }
237     }
238 
239     /**
240      * Sends the {@code response} to the IME, assuming all the relevant checks are already done.
241      */
242     @GuardedBy("mLock")
updateResponseToImeUncheckLocked(InlineSuggestionsResponse response)243     private void updateResponseToImeUncheckLocked(InlineSuggestionsResponse response) {
244         if (mDestroyed) {
245             return;
246         }
247         if (sDebug) Slog.d(TAG, "Send inline response: " + response.getInlineSuggestions().size());
248         try {
249             mResponseCallback.onInlineSuggestionsResponse(mAutofillId, response);
250         } catch (RemoteException e) {
251             Slog.e(TAG, "RemoteException sending InlineSuggestionsResponse to IME");
252         }
253     }
254 
255     @GuardedBy("mLock")
maybeNotifyFillUiEventLocked(@onNull List<InlineSuggestion> suggestions)256     private void maybeNotifyFillUiEventLocked(@NonNull List<InlineSuggestion> suggestions) {
257         if (mDestroyed) {
258             return;
259         }
260         boolean hasSuggestionToShow = false;
261         for (int i = 0; i < suggestions.size(); i++) {
262             InlineSuggestion suggestion = suggestions.get(i);
263             // It is possible we don't have any match result but we still have pinned
264             // suggestions. Only notify we have non-pinned suggestions to show
265             if (!suggestion.getInfo().isPinned()) {
266                 hasSuggestionToShow = true;
267                 break;
268             }
269         }
270         if (sDebug) {
271             Slog.d(TAG, "maybeNotifyFillUiEventLoked(): hasSuggestionToShow=" + hasSuggestionToShow
272                     + ", mPreviousHasNonPinSuggestionShow=" + mPreviousHasNonPinSuggestionShow);
273         }
274         // Use mPreviousHasNonPinSuggestionShow to save previous status, if the display status
275         // change, we can notify the event.
276         if (hasSuggestionToShow && !mPreviousHasNonPinSuggestionShow) {
277             // From no suggestion to has suggestions to show
278             mUiCallback.notifyInlineUiShown(mAutofillId);
279         } else if (!hasSuggestionToShow && mPreviousHasNonPinSuggestionShow) {
280             // From has suggestions to no suggestions to show
281             mUiCallback.notifyInlineUiHidden(mAutofillId);
282         }
283         // Update the latest status
284         mPreviousHasNonPinSuggestionShow = hasSuggestionToShow;
285     }
286 
287     /**
288      * Handles the {@code request} and {@code callback} received from the IME.
289      *
290      * <p> Should only invoked in the {@link #mHandler} thread.
291      */
handleOnReceiveImeRequest(@ullable InlineSuggestionsRequest request, @Nullable IInlineSuggestionsResponseCallback callback)292     private void handleOnReceiveImeRequest(@Nullable InlineSuggestionsRequest request,
293             @Nullable IInlineSuggestionsResponseCallback callback) {
294         synchronized (mLock) {
295             if (mDestroyed || mImeRequestReceived) {
296                 return;
297             }
298             mImeRequestReceived = true;
299             mImeSessionInvalidated = false;
300 
301             if (request != null && callback != null) {
302                 mImeRequest = request;
303                 mResponseCallback = callback;
304                 handleOnReceiveImeStatusUpdated(mAutofillId, true, false);
305             }
306             if (mImeRequestConsumer != null) {
307                 // Note that mImeRequest is only set if both request and callback are non-null.
308                 mImeRequestConsumer.accept(mImeRequest);
309                 mImeRequestConsumer = null;
310             }
311         }
312     }
313 
314     /**
315      * Handles the IME status updates received from the IME.
316      *
317      * <p> Should only be invoked in the {@link #mHandler} thread.
318      */
handleOnReceiveImeStatusUpdated(boolean imeInputStarted, boolean imeInputViewStarted)319     private void handleOnReceiveImeStatusUpdated(boolean imeInputStarted,
320             boolean imeInputViewStarted) {
321         synchronized (mLock) {
322             if (mDestroyed) {
323                 return;
324             }
325             mImeShowing = imeInputViewStarted;
326             if (mImeCurrentFieldId != null) {
327                 boolean imeInputStartedChanged = (mImeInputStarted != imeInputStarted);
328                 boolean imeInputViewStartedChanged = (mImeInputViewStarted != imeInputViewStarted);
329                 mImeInputStarted = imeInputStarted;
330                 mImeInputViewStarted = imeInputViewStarted;
331                 if (imeInputStartedChanged || imeInputViewStartedChanged) {
332                     maybeUpdateResponseToImeLocked();
333                 }
334             }
335         }
336     }
337 
338     /**
339      * Handles the IME status updates received from the IME.
340      *
341      * <p> Should only be invoked in the {@link #mHandler} thread.
342      */
handleOnReceiveImeStatusUpdated(@ullable AutofillId imeFieldId, boolean imeInputStarted, boolean imeInputViewStarted)343     private void handleOnReceiveImeStatusUpdated(@Nullable AutofillId imeFieldId,
344             boolean imeInputStarted, boolean imeInputViewStarted) {
345         synchronized (mLock) {
346             if (mDestroyed) {
347                 return;
348             }
349             if (imeFieldId != null) {
350                 mImeCurrentFieldId = imeFieldId;
351             }
352             handleOnReceiveImeStatusUpdated(imeInputStarted, imeInputViewStarted);
353         }
354     }
355 
356     /**
357      * Handles the IME session status received from the IME.
358      *
359      * <p> Should only be invoked in the {@link #mHandler} thread.
360      */
handleOnReceiveImeSessionInvalidated()361     private void handleOnReceiveImeSessionInvalidated() {
362         synchronized (mLock) {
363             if (mDestroyed) {
364                 return;
365             }
366             mImeSessionInvalidated = true;
367         }
368     }
369 
isImeShowing()370     boolean isImeShowing() {
371         synchronized (mLock) {
372             return !mDestroyed && mImeShowing;
373         }
374     }
375 
376     /**
377      * Internal implementation of {@link IInlineSuggestionsRequestCallback}.
378      */
379     private static final class InlineSuggestionsRequestCallbackImpl extends
380             IInlineSuggestionsRequestCallback.Stub {
381 
382         private final WeakReference<AutofillInlineSuggestionsRequestSession> mSession;
383 
InlineSuggestionsRequestCallbackImpl( AutofillInlineSuggestionsRequestSession session)384         private InlineSuggestionsRequestCallbackImpl(
385                 AutofillInlineSuggestionsRequestSession session) {
386             mSession = new WeakReference<>(session);
387         }
388 
389         @BinderThread
390         @Override
onInlineSuggestionsUnsupported()391         public void onInlineSuggestionsUnsupported() throws RemoteException {
392             if (sDebug) Slog.d(TAG, "onInlineSuggestionsUnsupported() called.");
393             final AutofillInlineSuggestionsRequestSession session = mSession.get();
394             if (session != null) {
395                 session.mHandler.sendMessage(obtainMessage(
396                         AutofillInlineSuggestionsRequestSession::handleOnReceiveImeRequest, session,
397                         null, null));
398             }
399         }
400 
401         @BinderThread
402         @Override
onInlineSuggestionsRequest(InlineSuggestionsRequest request, IInlineSuggestionsResponseCallback callback)403         public void onInlineSuggestionsRequest(InlineSuggestionsRequest request,
404                 IInlineSuggestionsResponseCallback callback) {
405             if (sDebug) Slog.d(TAG, "onInlineSuggestionsRequest() received: " + request);
406             final AutofillInlineSuggestionsRequestSession session = mSession.get();
407             if (session != null) {
408                 session.mHandler.sendMessage(obtainMessage(
409                         AutofillInlineSuggestionsRequestSession::handleOnReceiveImeRequest, session,
410                         request, callback));
411             }
412         }
413 
414         @Override
onInputMethodStartInput(AutofillId imeFieldId)415         public void onInputMethodStartInput(AutofillId imeFieldId) throws RemoteException {
416             if (sVerbose) Slog.v(TAG, "onInputMethodStartInput() received on " + imeFieldId);
417             final AutofillInlineSuggestionsRequestSession session = mSession.get();
418             if (session != null) {
419                 session.mHandler.sendMessage(obtainMessage(
420                         AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated,
421                         session, imeFieldId, true, false));
422             }
423         }
424 
425         @Override
onInputMethodShowInputRequested(boolean requestResult)426         public void onInputMethodShowInputRequested(boolean requestResult) throws RemoteException {
427             if (sVerbose) {
428                 Slog.v(TAG, "onInputMethodShowInputRequested() received: " + requestResult);
429             }
430         }
431 
432         @BinderThread
433         @Override
onInputMethodStartInputView()434         public void onInputMethodStartInputView() {
435             if (sVerbose) Slog.v(TAG, "onInputMethodStartInputView() received");
436             final AutofillInlineSuggestionsRequestSession session = mSession.get();
437             if (session != null) {
438                 session.mHandler.sendMessage(obtainMessage(
439                         AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated,
440                         session, true, true));
441             }
442         }
443 
444         @BinderThread
445         @Override
onInputMethodFinishInputView()446         public void onInputMethodFinishInputView() {
447             if (sVerbose) Slog.v(TAG, "onInputMethodFinishInputView() received");
448             final AutofillInlineSuggestionsRequestSession session = mSession.get();
449             if (session != null) {
450                 session.mHandler.sendMessage(obtainMessage(
451                         AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated,
452                         session, true, false));
453             }
454         }
455 
456         @Override
onInputMethodFinishInput()457         public void onInputMethodFinishInput() throws RemoteException {
458             if (sVerbose) Slog.v(TAG, "onInputMethodFinishInput() received");
459             final AutofillInlineSuggestionsRequestSession session = mSession.get();
460             if (session != null) {
461                 session.mHandler.sendMessage(obtainMessage(
462                         AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated,
463                         session, false, false));
464             }
465         }
466 
467         @BinderThread
468         @Override
onInlineSuggestionsSessionInvalidated()469         public void onInlineSuggestionsSessionInvalidated() throws RemoteException {
470             if (sDebug) Slog.d(TAG, "onInlineSuggestionsSessionInvalidated() called.");
471             final AutofillInlineSuggestionsRequestSession session = mSession.get();
472             if (session != null) {
473                 session.mHandler.sendMessage(obtainMessage(
474                         AutofillInlineSuggestionsRequestSession
475                                 ::handleOnReceiveImeSessionInvalidated, session));
476             }
477         }
478     }
479 
match(@ullable AutofillId autofillId, @Nullable AutofillId imeClientFieldId)480     private static boolean match(@Nullable AutofillId autofillId,
481             @Nullable AutofillId imeClientFieldId) {
482         // The IME doesn't have information about the virtual view id for the child views in the
483         // web view, so we are only comparing the parent view id here. This means that for cases
484         // where there are two input fields in the web view, they will have the same view id
485         // (although different virtual child id), and we will not be able to distinguish them.
486         return autofillId != null && imeClientFieldId != null
487                 && autofillId.getViewId() == imeClientFieldId.getViewId();
488     }
489 }
490