1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server.inputmethod;
18 
19 import static android.view.InputDevice.SOURCE_STYLUS;
20 
21 import android.Manifest;
22 import android.annotation.AnyThread;
23 import android.annotation.NonNull;
24 import android.annotation.Nullable;
25 import android.annotation.RequiresPermission;
26 import android.annotation.UiThread;
27 import android.hardware.input.InputManagerGlobal;
28 import android.os.Handler;
29 import android.os.IBinder;
30 import android.os.Looper;
31 import android.text.TextUtils;
32 import android.util.Slog;
33 import android.view.BatchedInputEventReceiver;
34 import android.view.Choreographer;
35 import android.view.Display;
36 import android.view.InputChannel;
37 import android.view.InputEvent;
38 import android.view.InputEventReceiver;
39 import android.view.MotionEvent;
40 import android.view.PointerIcon;
41 import android.view.SurfaceControl;
42 import android.view.View;
43 import android.view.inputmethod.InputMethodManager;
44 
45 import com.android.server.LocalServices;
46 import com.android.server.input.InputManagerInternal;
47 import com.android.server.wm.WindowManagerInternal;
48 
49 import java.util.ArrayList;
50 import java.util.List;
51 import java.util.Objects;
52 import java.util.OptionalInt;
53 
54 // TODO(b/210039666): See if we can make this class thread-safe.
55 final class HandwritingModeController {
56 
57     public static final String TAG = HandwritingModeController.class.getSimpleName();
58     static final boolean DEBUG = false;
59     // Use getHandwritingBufferSize() and not this value directly.
60     private static final int EVENT_BUFFER_SIZE = 100;
61     // A longer event buffer used for handwriting delegation
62     // TODO(b/210039666): make this device touch sampling rate dependent.
63     // Use getHandwritingBufferSize() and not this value directly.
64     private static final int LONG_EVENT_BUFFER_SIZE = EVENT_BUFFER_SIZE * 20;
65     private static final long HANDWRITING_DELEGATION_IDLE_TIMEOUT_MS = 3000;
66 
67     // This must be the looper for the UiThread.
68     private final Looper mLooper;
69     private final InputManagerInternal mInputManagerInternal;
70     private final WindowManagerInternal mWindowManagerInternal;
71 
72     private ArrayList<MotionEvent> mHandwritingBuffer;
73     private InputEventReceiver mHandwritingEventReceiver;
74     private Runnable mInkWindowInitRunnable;
75     private boolean mRecordingGesture;
76     private int mCurrentDisplayId;
77     // when set, package names are used for handwriting delegation.
78     private @Nullable String mDelegatePackageName;
79     private @Nullable String mDelegatorPackageName;
80     private Runnable mDelegationIdleTimeoutRunnable;
81     private Handler mDelegationIdleTimeoutHandler;
82 
83     private HandwritingEventReceiverSurface mHandwritingSurface;
84 
85     private int mCurrentRequestId;
86 
87     @AnyThread
HandwritingModeController(Looper uiThreadLooper, Runnable inkWindowInitRunnable)88     HandwritingModeController(Looper uiThreadLooper, Runnable inkWindowInitRunnable) {
89         mLooper = uiThreadLooper;
90         mCurrentDisplayId = Display.INVALID_DISPLAY;
91         mInputManagerInternal = LocalServices.getService(InputManagerInternal.class);
92         mWindowManagerInternal = LocalServices.getService(WindowManagerInternal.class);
93         mCurrentRequestId = 0;
94         mInkWindowInitRunnable = inkWindowInitRunnable;
95     }
96 
97     // TODO(b/210039666): Consider moving this to MotionEvent
isStylusEvent(MotionEvent event)98     private static boolean isStylusEvent(MotionEvent event) {
99         if (!event.isFromSource(SOURCE_STYLUS)) {
100             return false;
101         }
102         final int tool = event.getToolType(0);
103         return tool == MotionEvent.TOOL_TYPE_STYLUS || tool == MotionEvent.TOOL_TYPE_ERASER;
104     }
105 
106     /**
107      * Initializes the handwriting spy on the given displayId.
108      *
109      * This must be called from the UI Thread because it will start processing events using an
110      * InputEventReceiver that batches events according to the current thread's Choreographer.
111      */
112     @UiThread
initializeHandwritingSpy(int displayId)113     void initializeHandwritingSpy(int displayId) {
114         // When resetting, reuse resources if we are reinitializing on the same display.
115         reset(displayId == mCurrentDisplayId);
116         mCurrentDisplayId = displayId;
117 
118         if (mHandwritingBuffer == null) {
119             mHandwritingBuffer = new ArrayList<>(getHandwritingBufferSize());
120         }
121 
122         if (DEBUG) Slog.d(TAG, "Initializing handwriting spy monitor for display: " + displayId);
123         final String name = "stylus-handwriting-event-receiver-" + displayId;
124         final InputChannel channel = mInputManagerInternal.createInputChannel(name);
125         Objects.requireNonNull(channel, "Failed to create input channel");
126         final SurfaceControl surface =
127                 mHandwritingSurface != null ? mHandwritingSurface.getSurface()
128                         : mWindowManagerInternal.getHandwritingSurfaceForDisplay(displayId);
129         if (surface == null) {
130             Slog.e(TAG, "Failed to create input surface");
131             return;
132         }
133 
134         mHandwritingSurface = new HandwritingEventReceiverSurface(
135                 name, displayId, surface, channel);
136 
137         // Use a dup of the input channel so that event processing can be paused by disposing the
138         // event receiver without causing a fd hangup.
139         mHandwritingEventReceiver = new BatchedInputEventReceiver.SimpleBatchedInputEventReceiver(
140                 channel.dup(), mLooper, Choreographer.getInstance(), this::onInputEvent);
141         mCurrentRequestId++;
142     }
143 
getCurrentRequestId()144     OptionalInt getCurrentRequestId() {
145         if (mHandwritingSurface == null) {
146             Slog.e(TAG, "Cannot get requestId: Handwriting was not initialized.");
147             return OptionalInt.empty();
148         }
149         return OptionalInt.of(mCurrentRequestId);
150     }
151 
isStylusGestureOngoing()152     boolean isStylusGestureOngoing() {
153         return mRecordingGesture;
154     }
155 
hasOngoingStylusHandwritingSession()156     boolean hasOngoingStylusHandwritingSession() {
157         return mHandwritingSurface != null && mHandwritingSurface.isIntercepting();
158     }
159 
160     /**
161      * Prepare delegation of stylus handwriting to a different editor
162      * @see InputMethodManager#prepareStylusHandwritingDelegation(View, String)
163      */
prepareStylusHandwritingDelegation( @onNull String delegatePackageName, @NonNull String delegatorPackageName)164     void prepareStylusHandwritingDelegation(
165             @NonNull String delegatePackageName, @NonNull String delegatorPackageName) {
166         mDelegatePackageName = delegatePackageName;
167         mDelegatorPackageName = delegatorPackageName;
168         if (mHandwritingBuffer == null) {
169             mHandwritingBuffer = new ArrayList<>(getHandwritingBufferSize());
170         } else {
171             mHandwritingBuffer.ensureCapacity(getHandwritingBufferSize());
172         }
173         scheduleHandwritingDelegationTimeout();
174     }
175 
getDelegatePackageName()176     @Nullable String getDelegatePackageName() {
177         return mDelegatePackageName;
178     }
179 
getDelegatorPackageName()180     @Nullable String getDelegatorPackageName() {
181         return mDelegatorPackageName;
182     }
183 
scheduleHandwritingDelegationTimeout()184     private void scheduleHandwritingDelegationTimeout() {
185         if (mDelegationIdleTimeoutHandler == null) {
186             mDelegationIdleTimeoutHandler = new Handler(mLooper);
187         } else {
188             mDelegationIdleTimeoutHandler.removeCallbacks(mDelegationIdleTimeoutRunnable);
189         }
190         mDelegationIdleTimeoutRunnable =  () -> {
191             Slog.d(TAG, "Stylus handwriting delegation idle timed-out.");
192             clearPendingHandwritingDelegation();
193             if (mHandwritingBuffer != null) {
194                 mHandwritingBuffer.forEach(MotionEvent::recycle);
195                 mHandwritingBuffer.clear();
196                 mHandwritingBuffer.trimToSize();
197                 mHandwritingBuffer.ensureCapacity(getHandwritingBufferSize());
198             }
199         };
200         mDelegationIdleTimeoutHandler.postDelayed(
201                 mDelegationIdleTimeoutRunnable, HANDWRITING_DELEGATION_IDLE_TIMEOUT_MS);
202     }
203 
getHandwritingBufferSize()204     private int getHandwritingBufferSize() {
205         if (mDelegatePackageName != null && mDelegatorPackageName != null) {
206             return LONG_EVENT_BUFFER_SIZE;
207         }
208         return EVENT_BUFFER_SIZE;
209     }
210     /**
211      * Clear any pending handwriting delegation info.
212      */
clearPendingHandwritingDelegation()213     void clearPendingHandwritingDelegation() {
214         if (DEBUG) {
215             Slog.d(TAG, "clearPendingHandwritingDelegation");
216         }
217         if (mDelegationIdleTimeoutHandler != null) {
218             mDelegationIdleTimeoutHandler.removeCallbacks(mDelegationIdleTimeoutRunnable);
219             mDelegationIdleTimeoutHandler = null;
220         }
221         mDelegationIdleTimeoutRunnable = null;
222         mDelegatorPackageName = null;
223         mDelegatePackageName = null;
224     }
225 
226     /**
227      * Starts a {@link HandwritingSession} to transfer to the IME.
228      *
229      * This must be called from the UI Thread to avoid race conditions between processing more
230      * input events and disposing the input event receiver.
231      * @return the handwriting session to send to the IME, or null if the request was invalid.
232      */
233     @RequiresPermission(Manifest.permission.MONITOR_INPUT)
234     @UiThread
235     @Nullable
startHandwritingSession( int requestId, int imePid, int imeUid, IBinder focusedWindowToken)236     HandwritingSession startHandwritingSession(
237             int requestId, int imePid, int imeUid, IBinder focusedWindowToken) {
238         clearPendingHandwritingDelegation();
239         if (mHandwritingSurface == null) {
240             Slog.e(TAG, "Cannot start handwriting session: Handwriting was not initialized.");
241             return null;
242         }
243         if (requestId != mCurrentRequestId) {
244             Slog.e(TAG, "Cannot start handwriting session: Invalid request id: " + requestId);
245             return null;
246         }
247         if (!mRecordingGesture || mHandwritingBuffer.isEmpty()) {
248             Slog.e(TAG, "Cannot start handwriting session: No stylus gesture is being recorded.");
249             return null;
250         }
251         Objects.requireNonNull(mHandwritingEventReceiver,
252                 "Handwriting session was already transferred to IME.");
253         final MotionEvent downEvent = mHandwritingBuffer.get(0);
254         assert (downEvent.getActionMasked() == MotionEvent.ACTION_DOWN);
255         if (!mWindowManagerInternal.isPointInsideWindow(
256                 focusedWindowToken, mCurrentDisplayId, downEvent.getRawX(), downEvent.getRawY())) {
257             Slog.e(TAG, "Cannot start handwriting session: "
258                     + "Stylus gesture did not start inside the focused window.");
259             return null;
260         }
261         if (DEBUG) Slog.d(TAG, "Starting handwriting session in display: " + mCurrentDisplayId);
262 
263         InputManagerGlobal.getInstance()
264                 .pilferPointers(mHandwritingSurface.getInputChannel().getToken());
265 
266         // Stop processing more events.
267         mHandwritingEventReceiver.dispose();
268         mHandwritingEventReceiver = null;
269         mRecordingGesture = false;
270 
271         if (mHandwritingSurface.isIntercepting()) {
272             throw new IllegalStateException(
273                     "Handwriting surface should not be already intercepting.");
274         }
275         mHandwritingSurface.startIntercepting(imePid, imeUid);
276 
277         // Unset the pointer icon for the stylus in case the app had set it.
278         InputManagerGlobal.getInstance().setPointerIconType(PointerIcon.TYPE_NOT_SPECIFIED);
279 
280         return new HandwritingSession(mCurrentRequestId, mHandwritingSurface.getInputChannel(),
281                 mHandwritingBuffer);
282     }
283 
284     /**
285      * Reset the current handwriting session without initializing another session.
286      *
287      * This must be called from UI Thread to avoid race conditions between processing more input
288      * events and disposing the input event receiver.
289      */
290     @UiThread
reset()291     void reset() {
292         reset(false /* reinitializing */);
293     }
294 
setInkWindowInitializer(Runnable inkWindowInitializer)295     void setInkWindowInitializer(Runnable inkWindowInitializer) {
296         mInkWindowInitRunnable = inkWindowInitializer;
297     }
298 
reset(boolean reinitializing)299     private void reset(boolean reinitializing) {
300         if (mHandwritingEventReceiver != null) {
301             mHandwritingEventReceiver.dispose();
302             mHandwritingEventReceiver = null;
303         }
304 
305         if (mHandwritingBuffer != null) {
306             mHandwritingBuffer.forEach(MotionEvent::recycle);
307             mHandwritingBuffer.clear();
308             if (!reinitializing) {
309                 mHandwritingBuffer = null;
310             }
311         }
312 
313         if (mHandwritingSurface != null) {
314             mHandwritingSurface.getInputChannel().dispose();
315             if (!reinitializing) {
316                 mHandwritingSurface.remove();
317                 mHandwritingSurface = null;
318             }
319         }
320 
321         clearPendingHandwritingDelegation();
322         mRecordingGesture = false;
323     }
324 
onInputEvent(InputEvent ev)325     private boolean onInputEvent(InputEvent ev) {
326         if (mHandwritingEventReceiver == null) {
327             throw new IllegalStateException(
328                     "Input Event should not be processed when IME has the spy channel.");
329         }
330 
331         if (!(ev instanceof MotionEvent)) {
332             Slog.wtf(TAG, "Received non-motion event in stylus monitor.");
333             return false;
334         }
335         final MotionEvent event = (MotionEvent) ev;
336         if (!isStylusEvent(event)) {
337             return false;
338         }
339         if (event.getDisplayId() != mCurrentDisplayId) {
340             Slog.wtf(TAG, "Received stylus event associated with the incorrect display.");
341             return false;
342         }
343 
344         onStylusEvent(event);
345         return true;
346     }
347 
onStylusEvent(MotionEvent event)348     private void onStylusEvent(MotionEvent event) {
349         final int action = event.getActionMasked();
350 
351         if (mInkWindowInitRunnable != null && (action == MotionEvent.ACTION_HOVER_ENTER
352                 || event.getAction() == MotionEvent.ACTION_HOVER_ENTER)) {
353             // Ask IMMS to make ink window ready.
354             mInkWindowInitRunnable.run();
355             mInkWindowInitRunnable = null;
356         }
357 
358         // If handwriting delegation is ongoing, don't clear the buffer so that multiple strokes
359         // can be buffered across windows.
360         if (TextUtils.isEmpty(mDelegatePackageName)
361                 && (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL)) {
362             mRecordingGesture = false;
363             mHandwritingBuffer.clear();
364             return;
365         }
366 
367         if (action == MotionEvent.ACTION_DOWN) {
368             mRecordingGesture = true;
369         }
370 
371         if (!mRecordingGesture) {
372             return;
373         }
374 
375         if (mHandwritingBuffer.size() >= getHandwritingBufferSize()) {
376             if (DEBUG) {
377                 Slog.w(TAG, "Current gesture exceeds the buffer capacity."
378                         + " The rest of the gesture will not be recorded.");
379             }
380             mRecordingGesture = false;
381             return;
382         }
383 
384         mHandwritingBuffer.add(MotionEvent.obtain(event));
385     }
386 
387     static final class HandwritingSession {
388         private final int mRequestId;
389         private final InputChannel mHandwritingChannel;
390         private final List<MotionEvent> mRecordedEvents;
391 
HandwritingSession(int requestId, InputChannel handwritingChannel, List<MotionEvent> recordedEvents)392         private HandwritingSession(int requestId, InputChannel handwritingChannel,
393                 List<MotionEvent> recordedEvents) {
394             mRequestId = requestId;
395             mHandwritingChannel = handwritingChannel;
396             mRecordedEvents = recordedEvents;
397         }
398 
getRequestId()399         int getRequestId() {
400             return mRequestId;
401         }
402 
getHandwritingChannel()403         InputChannel getHandwritingChannel() {
404             return mHandwritingChannel;
405         }
406 
getRecordedEvents()407         List<MotionEvent> getRecordedEvents() {
408             return mRecordedEvents;
409         }
410     }
411 }
412