1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.android.test.handwritingime; 17 18 import android.R; 19 import android.annotation.Nullable; 20 import android.graphics.PointF; 21 import android.graphics.RectF; 22 import android.inputmethodservice.InputMethodService; 23 import android.util.Log; 24 import android.view.MotionEvent; 25 import android.view.View; 26 import android.view.ViewGroup; 27 import android.view.Window; 28 import android.view.inputmethod.CursorAnchorInfo; 29 import android.view.inputmethod.DeleteGesture; 30 import android.view.inputmethod.EditorInfo; 31 import android.view.inputmethod.HandwritingGesture; 32 import android.view.inputmethod.InputConnection; 33 import android.view.inputmethod.InsertGesture; 34 import android.view.inputmethod.JoinOrSplitGesture; 35 import android.view.inputmethod.RemoveSpaceGesture; 36 import android.view.inputmethod.SelectGesture; 37 import android.widget.AdapterView; 38 import android.widget.ArrayAdapter; 39 import android.widget.CheckBox; 40 import android.widget.FrameLayout; 41 import android.widget.LinearLayout; 42 import android.widget.Spinner; 43 import android.widget.Toast; 44 45 import java.util.Random; 46 import java.util.function.IntConsumer; 47 48 public class HandwritingIme extends InputMethodService { 49 private static final int OP_NONE = 0; 50 private static final int OP_SELECT = 1; 51 private static final int OP_DELETE = 2; 52 private static final int OP_INSERT = 3; 53 private static final int OP_REMOVE_SPACE = 4; 54 private static final int OP_JOIN_OR_SPLIT = 5; 55 56 private InkView mInk; 57 58 static final String TAG = "HandwritingIme"; 59 private int mRichGestureMode = OP_NONE; 60 private int mRichGestureGranularity = -1; 61 private Spinner mRichGestureModeSpinner; 62 private Spinner mRichGestureGranularitySpinner; 63 private PointF mRichGestureStartPoint; 64 65 static final int BOUNDS_INFO_NONE = 0; 66 static final int BOUNDS_INFO_VISIBLE_LINE_BOUNDS = 1; 67 static final int BOUNDS_INFO_EDITOR_BOUNDS = 2; 68 private int mBoundsInfoMode = BOUNDS_INFO_NONE; 69 private LinearLayout mBoundsInfoCheckBoxes; 70 71 private final IntConsumer mResultConsumer = value -> Log.d(TAG, "Gesture result: " + value); 72 73 interface HandwritingFinisher { finish()74 void finish(); 75 } 76 77 interface StylusListener { onStylusEvent(MotionEvent me)78 void onStylusEvent(MotionEvent me); 79 } 80 81 final class StylusConsumer implements StylusListener { 82 @Override onStylusEvent(MotionEvent me)83 public void onStylusEvent(MotionEvent me) { 84 HandwritingIme.this.onStylusEvent(me); 85 } 86 } 87 88 final class HandwritingFinisherImpl implements HandwritingFinisher { 89 HandwritingFinisherImpl()90 HandwritingFinisherImpl() {} 91 92 @Override finish()93 public void finish() { 94 finishStylusHandwriting(); 95 Log.d(TAG, "HandwritingIme called finishStylusHandwriting() "); 96 } 97 } 98 onStylusEvent(@ullable MotionEvent event)99 private void onStylusEvent(@Nullable MotionEvent event) { 100 // TODO Hookup recognizer here 101 switch (event.getAction()) { 102 case MotionEvent.ACTION_UP: { 103 if (areRichGesturesEnabled()) { 104 HandwritingGesture gesture = null; 105 switch (mRichGestureMode) { 106 case OP_SELECT: 107 gesture = new SelectGesture.Builder() 108 .setGranularity(mRichGestureGranularity) 109 .setSelectionArea(getSanitizedRectF(mRichGestureStartPoint.x, 110 mRichGestureStartPoint.y, event.getX(), event.getY())) 111 .setFallbackText("fallback text") 112 .build(); 113 break; 114 case OP_DELETE: 115 gesture = new DeleteGesture.Builder() 116 .setGranularity(mRichGestureGranularity) 117 .setDeletionArea(getSanitizedRectF(mRichGestureStartPoint.x, 118 mRichGestureStartPoint.y, event.getX(), event.getY())) 119 .setFallbackText("fallback text") 120 .build(); 121 break; 122 case OP_INSERT: 123 gesture = new InsertGesture.Builder() 124 .setInsertionPoint(new PointF( 125 mRichGestureStartPoint.x, mRichGestureStartPoint.y)) 126 .setTextToInsert(" ") 127 .setFallbackText("fallback text") 128 .build(); 129 break; 130 case OP_REMOVE_SPACE: 131 gesture = new RemoveSpaceGesture.Builder() 132 .setPoints( 133 new PointF(mRichGestureStartPoint.x, 134 mRichGestureStartPoint.y), 135 new PointF(event.getX(), event.getY())) 136 .setFallbackText("fallback text") 137 .build(); 138 break; 139 case OP_JOIN_OR_SPLIT: 140 gesture = new JoinOrSplitGesture.Builder() 141 .setJoinOrSplitPoint(new PointF( 142 mRichGestureStartPoint.x, mRichGestureStartPoint.y)) 143 .setFallbackText("fallback text") 144 .build(); 145 break; 146 } 147 if (gesture == null) { 148 // This shouldn't happen 149 Log.e(TAG, "Unrecognized gesture mode: " + mRichGestureMode); 150 return; 151 } 152 performGesture(gesture); 153 } else { 154 // insert random ASCII char 155 sendKeyChar((char) (56 + new Random().nextInt(66))); 156 } 157 return; 158 } 159 case MotionEvent.ACTION_DOWN: { 160 if (areRichGesturesEnabled()) { 161 mRichGestureStartPoint = new PointF(event.getX(), event.getY()); 162 } 163 return; 164 } 165 } 166 } 167 168 /** 169 * sanitize values to support rectangles in all cases. 170 */ getSanitizedRectF(float left, float top, float right, float bottom)171 private RectF getSanitizedRectF(float left, float top, float right, float bottom) { 172 // swap values when left > right OR top > bottom. 173 if (left > right) { 174 float temp = left; 175 left = right; 176 right = temp; 177 } 178 if (top > bottom) { 179 float temp = top; 180 top = bottom; 181 bottom = temp; 182 } 183 // increment by a pixel so that RectF.isEmpty() isn't true. 184 if (left == right) { 185 right++; 186 } 187 if (top == bottom) { 188 bottom++; 189 } 190 191 RectF rectF = new RectF(left, top, right, bottom); 192 Log.d(TAG, "Sending RichGesture " + rectF.toShortString()); 193 return rectF; 194 } 195 performGesture(HandwritingGesture gesture)196 private void performGesture(HandwritingGesture gesture) { 197 InputConnection ic = getCurrentInputConnection(); 198 if (getCurrentInputStarted() && ic != null) { 199 ic.performHandwritingGesture(gesture, Runnable::run, mResultConsumer); 200 } else { 201 // This shouldn't happen 202 Log.e(TAG, "No active InputConnection"); 203 } 204 } 205 206 @Override onCreateInputView()207 public View onCreateInputView() { 208 Log.d(TAG, "onCreateInputView"); 209 final ViewGroup view = new FrameLayout(this); 210 view.setPadding(0, 0, 0, 0); 211 212 LinearLayout layout = new LinearLayout(this); 213 layout.setLayoutParams(new LinearLayout.LayoutParams( 214 ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT)); 215 layout.setOrientation(LinearLayout.VERTICAL); 216 layout.addView(getRichGestureActionsSpinner()); 217 layout.addView(getRichGestureGranularitySpinner()); 218 layout.addView(getBoundsInfoCheckBoxes()); 219 layout.setBackgroundColor(getColor(R.color.holo_green_light)); 220 view.addView(layout); 221 222 return view; 223 } 224 getRichGestureActionsSpinner()225 private View getRichGestureActionsSpinner() { 226 if (mRichGestureModeSpinner != null) { 227 return mRichGestureModeSpinner; 228 } 229 mRichGestureModeSpinner = new Spinner(this); 230 mRichGestureModeSpinner.setPadding(100, 0, 100, 0); 231 mRichGestureModeSpinner.setTooltipText("Handwriting IME mode"); 232 String[] items = new String[]{ 233 "Handwriting IME - Rich gesture disabled", 234 "Rich gesture SELECT", 235 "Rich gesture DELETE", 236 "Rich gesture INSERT", 237 "Rich gesture REMOVE SPACE", 238 "Rich gesture JOIN OR SPLIT", 239 }; 240 ArrayAdapter<String> adapter = new ArrayAdapter<>(this, 241 android.R.layout.simple_spinner_dropdown_item, items); 242 adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); 243 mRichGestureModeSpinner.setAdapter(adapter); 244 mRichGestureModeSpinner.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() { 245 @Override 246 public void onItemSelected(AdapterView<?> parent, View view, int position, long id) { 247 mRichGestureMode = position; 248 mRichGestureGranularitySpinner.setEnabled( 249 mRichGestureMode == OP_SELECT || mRichGestureMode == OP_DELETE); 250 Log.d(TAG, "Setting RichGesture Mode " + mRichGestureMode); 251 } 252 253 @Override 254 public void onNothingSelected(AdapterView<?> parent) { 255 mRichGestureMode = OP_NONE; 256 mRichGestureGranularitySpinner.setEnabled(false); 257 } 258 }); 259 mRichGestureModeSpinner.setSelection(0); // default disabled 260 return mRichGestureModeSpinner; 261 } 262 updateCursorAnchorInfo(int boundsInfoMode)263 private void updateCursorAnchorInfo(int boundsInfoMode) { 264 final InputConnection ic = getCurrentInputConnection(); 265 if (ic == null) return; 266 267 if (boundsInfoMode == BOUNDS_INFO_NONE) { 268 ic.requestCursorUpdates(0); 269 return; 270 } 271 272 final int cursorUpdateMode = InputConnection.CURSOR_UPDATE_MONITOR; 273 int cursorUpdateFilter = 0; 274 if ((boundsInfoMode & BOUNDS_INFO_EDITOR_BOUNDS) != 0) { 275 cursorUpdateFilter |= InputConnection.CURSOR_UPDATE_FILTER_EDITOR_BOUNDS; 276 } 277 278 if ((boundsInfoMode & BOUNDS_INFO_VISIBLE_LINE_BOUNDS) != 0) { 279 cursorUpdateFilter |= InputConnection.CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS; 280 } 281 ic.requestCursorUpdates(cursorUpdateMode | cursorUpdateFilter); 282 } 283 updateBoundsInfoMode()284 private void updateBoundsInfoMode() { 285 if (mInk != null) { 286 mInk.setBoundsInfoMode(mBoundsInfoMode); 287 } 288 updateCursorAnchorInfo(mBoundsInfoMode); 289 } 290 getBoundsInfoCheckBoxes()291 private View getBoundsInfoCheckBoxes() { 292 if (mBoundsInfoCheckBoxes != null) { 293 return mBoundsInfoCheckBoxes; 294 } 295 mBoundsInfoCheckBoxes = new LinearLayout(this); 296 mBoundsInfoCheckBoxes.setPadding(100, 0, 100, 0); 297 mBoundsInfoCheckBoxes.setOrientation(LinearLayout.HORIZONTAL); 298 299 final CheckBox editorBoundsInfoCheckBox = new CheckBox(this); 300 editorBoundsInfoCheckBox.setText("EditorBoundsInfo"); 301 editorBoundsInfoCheckBox.setOnCheckedChangeListener((buttonView, isChecked) -> { 302 if (isChecked) { 303 mBoundsInfoMode |= BOUNDS_INFO_EDITOR_BOUNDS; 304 } else { 305 mBoundsInfoMode &= ~BOUNDS_INFO_EDITOR_BOUNDS; 306 } 307 updateBoundsInfoMode(); 308 }); 309 310 final CheckBox visibleLineBoundsInfoCheckBox = new CheckBox(this); 311 visibleLineBoundsInfoCheckBox.setText("VisibleLineBounds"); 312 visibleLineBoundsInfoCheckBox.setOnCheckedChangeListener((buttonView, isChecked) -> { 313 if (isChecked) { 314 mBoundsInfoMode |= BOUNDS_INFO_VISIBLE_LINE_BOUNDS; 315 } else { 316 mBoundsInfoMode &= ~BOUNDS_INFO_VISIBLE_LINE_BOUNDS; 317 } 318 updateBoundsInfoMode(); 319 }); 320 321 mBoundsInfoCheckBoxes.addView(editorBoundsInfoCheckBox); 322 mBoundsInfoCheckBoxes.addView(visibleLineBoundsInfoCheckBox); 323 return mBoundsInfoCheckBoxes; 324 } 325 getRichGestureGranularitySpinner()326 private View getRichGestureGranularitySpinner() { 327 if (mRichGestureGranularitySpinner != null) { 328 return mRichGestureGranularitySpinner; 329 } 330 mRichGestureGranularitySpinner = new Spinner(this); 331 mRichGestureGranularitySpinner.setPadding(100, 0, 100, 0); 332 mRichGestureGranularitySpinner.setTooltipText(" Granularity"); 333 String[] items = 334 new String[] { "Granularity - UNDEFINED", 335 "Granularity - WORD", "Granularity - CHARACTER"}; 336 ArrayAdapter<String> adapter = new ArrayAdapter<>(this, 337 android.R.layout.simple_spinner_dropdown_item, items); 338 adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); 339 mRichGestureGranularitySpinner.setAdapter(adapter); 340 mRichGestureGranularitySpinner.setOnItemSelectedListener( 341 new AdapterView.OnItemSelectedListener() { 342 @Override 343 public void onItemSelected(AdapterView<?> parent, View view, int position, long id) { 344 mRichGestureGranularity = position; 345 Log.d(TAG, "Setting RichGesture Granularity " + mRichGestureGranularity); 346 } 347 348 @Override 349 public void onNothingSelected(AdapterView<?> parent) { 350 mRichGestureGranularity = 0; 351 } 352 }); 353 mRichGestureGranularitySpinner.setSelection(1); 354 return mRichGestureGranularitySpinner; 355 } 356 onPrepareStylusHandwriting()357 public void onPrepareStylusHandwriting() { 358 Log.d(TAG, "onPrepareStylusHandwriting "); 359 if (mInk == null) { 360 mInk = new InkView(this, new HandwritingFinisherImpl(), new StylusConsumer()); 361 mInk.setBoundsInfoMode(mBoundsInfoMode); 362 } 363 } 364 365 @Override onStartStylusHandwriting()366 public boolean onStartStylusHandwriting() { 367 Log.d(TAG, "onStartStylusHandwriting "); 368 Toast.makeText(this, "START HW", Toast.LENGTH_SHORT).show(); 369 Window inkWindow = getStylusHandwritingWindow(); 370 inkWindow.setContentView(mInk, mInk.getLayoutParams()); 371 return true; 372 } 373 374 @Override onFinishStylusHandwriting()375 public void onFinishStylusHandwriting() { 376 Log.d(TAG, "onFinishStylusHandwriting "); 377 Toast.makeText(this, "Finish HW", Toast.LENGTH_SHORT).show(); 378 // Free-up 379 ((ViewGroup) mInk.getParent()).removeView(mInk); 380 mInk = null; 381 } 382 383 @Override onEvaluateFullscreenMode()384 public boolean onEvaluateFullscreenMode() { 385 return false; 386 } 387 areRichGesturesEnabled()388 private boolean areRichGesturesEnabled() { 389 return mRichGestureMode != OP_NONE; 390 } 391 392 @Override onUpdateCursorAnchorInfo(CursorAnchorInfo cursorAnchorInfo)393 public void onUpdateCursorAnchorInfo(CursorAnchorInfo cursorAnchorInfo) { 394 if (mInk != null) { 395 mInk.setCursorAnchorInfo(cursorAnchorInfo); 396 } 397 } 398 399 @Override onStartInput(EditorInfo attribute, boolean restarting)400 public void onStartInput(EditorInfo attribute, boolean restarting) { 401 updateCursorAnchorInfo(mBoundsInfoMode); 402 } 403 } 404