1 /* 2 ** Copyright 2011, The Android Open Source Project 3 ** 4 ** Licensed under the Apache License, Version 2.0 (the "License"); 5 ** you may not use this file except in compliance with the License. 6 ** You may obtain a copy of the License at 7 ** 8 ** http://www.apache.org/licenses/LICENSE-2.0 9 ** 10 ** Unless required by applicable law or agreed to in writing, software 11 ** distributed under the License is distributed on an "AS IS" BASIS, 12 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 ** See the License for the specific language governing permissions and 14 ** limitations under the License. 15 */ 16 17 package android.view.accessibility; 18 19 import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_INTERACTION_CLIENT; 20 import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_INTERACTION_CONNECTION_CALLBACK; 21 import static android.os.Build.VERSION_CODES.S; 22 import static android.view.accessibility.AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_MASK; 23 import static android.view.accessibility.AccessibilityNodeInfo.FLAG_PREFETCH_MASK; 24 25 import android.accessibilityservice.AccessibilityService; 26 import android.accessibilityservice.IAccessibilityServiceConnection; 27 import android.annotation.CallbackExecutor; 28 import android.annotation.NonNull; 29 import android.annotation.Nullable; 30 import android.annotation.SuppressLint; 31 import android.compat.annotation.UnsupportedAppUsage; 32 import android.content.Context; 33 import android.os.Binder; 34 import android.os.Build; 35 import android.os.Bundle; 36 import android.os.Handler; 37 import android.os.IBinder; 38 import android.os.Looper; 39 import android.os.Message; 40 import android.os.Process; 41 import android.os.RemoteException; 42 import android.os.SystemClock; 43 import android.util.Log; 44 import android.util.LongSparseArray; 45 import android.util.Pair; 46 import android.util.SparseArray; 47 import android.util.SparseLongArray; 48 import android.view.Display; 49 import android.view.SurfaceControl; 50 import android.view.ViewConfiguration; 51 import android.window.ScreenCapture; 52 53 import com.android.internal.annotations.VisibleForTesting; 54 import com.android.internal.util.ArrayUtils; 55 56 import java.util.ArrayDeque; 57 import java.util.ArrayList; 58 import java.util.Arrays; 59 import java.util.Collections; 60 import java.util.HashSet; 61 import java.util.List; 62 import java.util.Queue; 63 import java.util.concurrent.Executor; 64 import java.util.concurrent.atomic.AtomicInteger; 65 66 /** 67 * This class is a singleton that performs accessibility interaction 68 * which is it queries remote view hierarchies about snapshots of their 69 * views as well requests from these hierarchies to perform certain 70 * actions on their views. 71 * 72 * Rationale: The content retrieval APIs are synchronous from a client's 73 * perspective but internally they are asynchronous. The client thread 74 * calls into the system requesting an action and providing a callback 75 * to receive the result after which it waits up to a timeout for that 76 * result. The system enforces security and the delegates the request 77 * to a given view hierarchy where a message is posted (from a binder 78 * thread) describing what to be performed by the main UI thread the 79 * result of which it delivered via the mentioned callback. However, 80 * the blocked client thread and the main UI thread of the target view 81 * hierarchy can be the same thread, for example an accessibility service 82 * and an activity run in the same process, thus they are executed on the 83 * same main thread. In such a case the retrieval will fail since the UI 84 * thread that has to process the message describing the work to be done 85 * is blocked waiting for a result is has to compute! To avoid this scenario 86 * when making a call the client also passes its process and thread ids so 87 * the accessed view hierarchy can detect if the client making the request 88 * is running in its main UI thread. In such a case the view hierarchy, 89 * specifically the binder thread performing the IPC to it, does not post a 90 * message to be run on the UI thread but passes it to the singleton 91 * interaction client through which all interactions occur and the latter is 92 * responsible to execute the message before starting to wait for the 93 * asynchronous result delivered via the callback. In this case the expected 94 * result is already received so no waiting is performed. 95 * 96 * @hide 97 */ 98 public final class AccessibilityInteractionClient 99 extends IAccessibilityInteractionConnectionCallback.Stub { 100 101 public static final int NO_ID = -1; 102 103 public static final String CALL_STACK = "call_stack"; 104 public static final String IGNORE_CALL_STACK = "ignore_call_stack"; 105 106 private static final String LOG_TAG = "AccessibilityInteractionClient"; 107 108 private static final boolean DEBUG = false; 109 110 private static final boolean CHECK_INTEGRITY = true; 111 112 private static final long TIMEOUT_INTERACTION_MILLIS = 5000; 113 114 private static final long DISABLE_PREFETCHING_FOR_SCROLLING_MILLIS = 115 (long) (ViewConfiguration.getSendRecurringAccessibilityEventsInterval() * 1.5); 116 117 private static final Object sStaticLock = new Object(); 118 119 private static final LongSparseArray<AccessibilityInteractionClient> sClients = 120 new LongSparseArray<>(); 121 122 private static final SparseArray<IAccessibilityServiceConnection> sConnectionCache = 123 new SparseArray<>(); 124 125 // Used to generate connection ids for direct app-process connections. Start sufficiently far 126 // enough from the connection ids generated by AccessibilityManagerService. 127 private static int sDirectConnectionIdCounter = 1 << 30; 128 private static int sDirectConnectionCount = 0; 129 130 /** List of timestamps which indicate the latest time an a11y service receives a scroll event 131 from a window, mapping from windowId -> timestamp. */ 132 private static final SparseLongArray sScrollingWindows = new SparseLongArray(); 133 134 private static SparseArray<AccessibilityCache> sCaches = new SparseArray<>(); 135 136 private final AtomicInteger mInteractionIdCounter = new AtomicInteger(); 137 138 private final Object mInstanceLock = new Object(); 139 140 private final AccessibilityManager mAccessibilityManager; 141 142 private volatile int mInteractionId = -1; 143 private volatile int mCallingUid = Process.INVALID_UID; 144 // call stack for IAccessibilityInteractionConnectionCallback APIs. These callback APIs are 145 // shared by multiple requests APIs in IAccessibilityServiceConnection. To correctly log the 146 // request API which triggers the callback, we log trace entries for callback after the 147 // request API thread waiting for the callback returns. To log the correct callback stack in 148 // the request API thread, we save the callback stack in this member variables. 149 private List<StackTraceElement> mCallStackOfCallback; 150 151 private AccessibilityNodeInfo mFindAccessibilityNodeInfoResult; 152 153 private List<AccessibilityNodeInfo> mFindAccessibilityNodeInfosResult; 154 155 private boolean mPerformAccessibilityActionResult; 156 157 // SparseArray of interaction ID -> screenshot executor+callback. 158 private final SparseArray<Pair<Executor, AccessibilityService.TakeScreenshotCallback>> 159 mTakeScreenshotOfWindowCallbacks = new SparseArray<>(); 160 161 private Message mSameThreadMessage; 162 163 private int mInteractionIdWaitingForPrefetchResult = -1; 164 private int mConnectionIdWaitingForPrefetchResult; 165 private String[] mPackageNamesForNextPrefetchResult; 166 167 /** 168 * @return The client for the current thread. 169 */ 170 @UnsupportedAppUsage() getInstance()171 public static AccessibilityInteractionClient getInstance() { 172 final long threadId = Thread.currentThread().getId(); 173 return getInstanceForThread(threadId); 174 } 175 176 /** 177 * <strong>Note:</strong> We keep one instance per interrogating thread since 178 * the instance contains state which can lead to undesired thread interleavings. 179 * We do not have a thread local variable since other threads should be able to 180 * look up the correct client knowing a thread id. See ViewRootImpl for details. 181 * 182 * @return The client for a given <code>threadId</code>. 183 */ getInstanceForThread(long threadId)184 public static AccessibilityInteractionClient getInstanceForThread(long threadId) { 185 synchronized (sStaticLock) { 186 AccessibilityInteractionClient client = sClients.get(threadId); 187 if (client == null) { 188 client = new AccessibilityInteractionClient(); 189 sClients.put(threadId, client); 190 } 191 return client; 192 } 193 } 194 195 /** 196 * @return The client for the current thread. 197 */ getInstance(Context context)198 public static AccessibilityInteractionClient getInstance(Context context) { 199 final long threadId = Thread.currentThread().getId(); 200 if (context != null) { 201 return getInstanceForThread(threadId, context); 202 } 203 return getInstanceForThread(threadId); 204 } 205 206 /** 207 * <strong>Note:</strong> We keep one instance per interrogating thread since 208 * the instance contains state which can lead to undesired thread interleavings. 209 * We do not have a thread local variable since other threads should be able to 210 * look up the correct client knowing a thread id. See ViewRootImpl for details. 211 * 212 * @return The client for a given <code>threadId</code>. 213 */ getInstanceForThread(long threadId, Context context)214 public static AccessibilityInteractionClient getInstanceForThread(long threadId, 215 Context context) { 216 synchronized (sStaticLock) { 217 AccessibilityInteractionClient client = sClients.get(threadId); 218 if (client == null) { 219 client = new AccessibilityInteractionClient(context); 220 sClients.put(threadId, client); 221 } 222 return client; 223 } 224 } 225 226 /** 227 * Gets a cached accessibility service connection. 228 * 229 * @param connectionId The connection id. 230 * @return The cached connection if such. 231 */ getConnection(int connectionId)232 public static IAccessibilityServiceConnection getConnection(int connectionId) { 233 synchronized (sConnectionCache) { 234 return sConnectionCache.get(connectionId); 235 } 236 } 237 238 /** 239 * Adds a cached accessibility service connection. 240 * 241 * Adds a cache if {@code initializeCache} is true 242 * @param connectionId The connection id. 243 * @param connection The connection. 244 * @param initializeCache whether to initialize a cache 245 */ addConnection(int connectionId, IAccessibilityServiceConnection connection, boolean initializeCache)246 public static void addConnection(int connectionId, IAccessibilityServiceConnection connection, 247 boolean initializeCache) { 248 if (connectionId == NO_ID) { 249 return; 250 } 251 synchronized (sConnectionCache) { 252 IAccessibilityServiceConnection existingConnection = getConnection(connectionId); 253 if (existingConnection instanceof DirectAccessibilityConnection) { 254 throw new IllegalArgumentException( 255 "Cannot add service connection with id " + connectionId 256 + " which conflicts with existing direct connection."); 257 } 258 sConnectionCache.put(connectionId, connection); 259 if (!initializeCache) { 260 return; 261 } 262 sCaches.put(connectionId, new AccessibilityCache( 263 new AccessibilityCache.AccessibilityNodeRefresher())); 264 } 265 } 266 267 /** 268 * Adds a new {@link DirectAccessibilityConnection} using the provided 269 * {@link IAccessibilityInteractionConnection} to create a direct connection between 270 * this client and the {@link android.view.ViewRootImpl} for queries inside the app process. 271 * 272 * <p> 273 * See {@link DirectAccessibilityConnection} for supported methods. 274 * </p> 275 * 276 * @param connection The ViewRootImpl's {@link IAccessibilityInteractionConnection}. 277 */ addDirectConnection(IAccessibilityInteractionConnection connection, AccessibilityManager accessibilityManager)278 public static int addDirectConnection(IAccessibilityInteractionConnection connection, 279 AccessibilityManager accessibilityManager) { 280 synchronized (sConnectionCache) { 281 int connectionId = sDirectConnectionIdCounter++; 282 if (getConnection(connectionId) != null) { 283 throw new IllegalArgumentException( 284 "Cannot add direct connection with existing id " + connectionId); 285 } 286 DirectAccessibilityConnection directAccessibilityConnection = 287 new DirectAccessibilityConnection(connection, accessibilityManager); 288 sConnectionCache.put(connectionId, directAccessibilityConnection); 289 sDirectConnectionCount++; 290 // Do not use AccessibilityCache for this connection, since there is no corresponding 291 // AccessibilityService to handle cache invalidation events. 292 return connectionId; 293 } 294 } 295 296 /** Check if any {@link DirectAccessibilityConnection} is currently in the connection cache. */ hasAnyDirectConnection()297 public static boolean hasAnyDirectConnection() { 298 return sDirectConnectionCount > 0; 299 } 300 301 /** 302 * Gets a cached associated with the connection id if available. 303 * 304 */ getCache(int connectionId)305 public static AccessibilityCache getCache(int connectionId) { 306 synchronized (sConnectionCache) { 307 return sCaches.get(connectionId); 308 } 309 } 310 311 /** 312 * Removes a cached accessibility service connection. 313 * 314 * @param connectionId The connection id. 315 */ removeConnection(int connectionId)316 public static void removeConnection(int connectionId) { 317 synchronized (sConnectionCache) { 318 if (getConnection(connectionId) instanceof DirectAccessibilityConnection) { 319 sDirectConnectionCount--; 320 } 321 sConnectionCache.remove(connectionId); 322 sCaches.remove(connectionId); 323 } 324 } 325 326 /** 327 * This method is only for testing. Replacing the cache is a generally terrible idea, but 328 * tests need to be able to verify this class's interactions with the cache 329 */ 330 @VisibleForTesting setCache(int connectionId, AccessibilityCache cache)331 public static void setCache(int connectionId, AccessibilityCache cache) { 332 synchronized (sConnectionCache) { 333 sCaches.put(connectionId, cache); 334 } 335 } 336 AccessibilityInteractionClient()337 private AccessibilityInteractionClient() { 338 /* reducing constructor visibility */ 339 mAccessibilityManager = null; 340 } 341 AccessibilityInteractionClient(Context context)342 private AccessibilityInteractionClient(Context context) { 343 mAccessibilityManager = context.getSystemService(AccessibilityManager.class); 344 } 345 346 /** 347 * Sets the message to be processed if the interacted view hierarchy 348 * and the interacting client are running in the same thread. 349 * 350 * @param message The message. 351 */ 352 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) setSameThreadMessage(Message message)353 public void setSameThreadMessage(Message message) { 354 synchronized (mInstanceLock) { 355 mSameThreadMessage = message; 356 mInstanceLock.notifyAll(); 357 } 358 } 359 360 /** 361 * Gets the root {@link AccessibilityNodeInfo} in the currently active window. 362 * 363 * @param connectionId The id of a connection for interacting with the system. 364 * @return The root {@link AccessibilityNodeInfo} if found, null otherwise. 365 */ getRootInActiveWindow(int connectionId, @AccessibilityNodeInfo.PrefetchingStrategy int strategy)366 public AccessibilityNodeInfo getRootInActiveWindow(int connectionId, 367 @AccessibilityNodeInfo.PrefetchingStrategy int strategy) { 368 return findAccessibilityNodeInfoByAccessibilityId(connectionId, 369 AccessibilityWindowInfo.ACTIVE_WINDOW_ID, AccessibilityNodeInfo.ROOT_NODE_ID, 370 false, strategy, null); 371 } 372 373 /** 374 * Gets the info for a window. 375 * 376 * @param connectionId The id of a connection for interacting with the system. 377 * @param accessibilityWindowId A unique window id. Use 378 * {@link android.view.accessibility.AccessibilityWindowInfo#ACTIVE_WINDOW_ID} 379 * to query the currently active window. 380 * @return The {@link AccessibilityWindowInfo}. 381 */ getWindow(int connectionId, int accessibilityWindowId)382 public AccessibilityWindowInfo getWindow(int connectionId, int accessibilityWindowId) { 383 return getWindow(connectionId, accessibilityWindowId, /* bypassCache */ false); 384 } 385 386 /** 387 * Gets the info for a window. 388 * 389 * @param connectionId The id of a connection for interacting with the system. 390 * @param accessibilityWindowId A unique window id. Use 391 * {@link AccessibilityWindowInfo#ACTIVE_WINDOW_ID} 392 * to query the currently active window. 393 * @param bypassCache Whether to bypass the cache. 394 * @return The {@link AccessibilityWindowInfo}. 395 */ getWindow(int connectionId, int accessibilityWindowId, boolean bypassCache)396 public AccessibilityWindowInfo getWindow(int connectionId, int accessibilityWindowId, 397 boolean bypassCache) { 398 try { 399 IAccessibilityServiceConnection connection = getConnection(connectionId); 400 if (connection != null) { 401 AccessibilityWindowInfo window; 402 AccessibilityCache cache = getCache(connectionId); 403 if (cache != null) { 404 if (!bypassCache) { 405 window = cache.getWindow(accessibilityWindowId); 406 if (window != null) { 407 if (DEBUG) { 408 Log.i(LOG_TAG, "Window cache hit"); 409 } 410 if (shouldTraceClient()) { 411 logTraceClient(connection, "getWindow cache", 412 "connectionId=" + connectionId + ";accessibilityWindowId=" 413 + accessibilityWindowId + ";bypassCache=false"); 414 } 415 return window; 416 } 417 if (DEBUG) { 418 Log.i(LOG_TAG, "Window cache miss"); 419 } 420 } 421 } else { 422 if (DEBUG) { 423 Log.w(LOG_TAG, "Cache is null for connection id: " + connectionId); 424 } 425 } 426 427 final long identityToken = Binder.clearCallingIdentity(); 428 try { 429 window = connection.getWindow(accessibilityWindowId); 430 } finally { 431 Binder.restoreCallingIdentity(identityToken); 432 } 433 if (shouldTraceClient()) { 434 logTraceClient(connection, "getWindow", "connectionId=" + connectionId 435 + ";accessibilityWindowId=" + accessibilityWindowId + ";bypassCache=" 436 + bypassCache); 437 } 438 439 if (window != null) { 440 if (!bypassCache && cache != null) { 441 cache.addWindow(window); 442 } 443 return window; 444 } 445 } else { 446 if (DEBUG) { 447 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 448 } 449 } 450 } catch (RemoteException re) { 451 Log.e(LOG_TAG, "Error while calling remote getWindow", re); 452 } 453 return null; 454 } 455 456 /** 457 * Gets the info for all windows of the default display. 458 * 459 * @param connectionId The id of a connection for interacting with the system. 460 * @return The {@link AccessibilityWindowInfo} list. 461 */ getWindows(int connectionId)462 public List<AccessibilityWindowInfo> getWindows(int connectionId) { 463 return getWindowsOnDisplay(connectionId, Display.DEFAULT_DISPLAY); 464 } 465 466 /** 467 * Gets the info for all windows of the specified display. 468 * 469 * @param connectionId The id of a connection for interacting with the system. 470 * @return The {@link AccessibilityWindowInfo} list belonging to {@code displayId}. 471 */ getWindowsOnDisplay(int connectionId, int displayId)472 public List<AccessibilityWindowInfo> getWindowsOnDisplay(int connectionId, int displayId) { 473 final SparseArray<List<AccessibilityWindowInfo>> windows = 474 getWindowsOnAllDisplays(connectionId); 475 return windows.get(displayId, Collections.emptyList()); 476 } 477 /** 478 * Gets the info for all windows of all displays. 479 * 480 * @param connectionId The id of a connection for interacting with the system. 481 * @return The SparseArray of {@link AccessibilityWindowInfo} list. 482 * The key of SparseArray is display ID. 483 */ getWindowsOnAllDisplays(int connectionId)484 public SparseArray<List<AccessibilityWindowInfo>> getWindowsOnAllDisplays(int connectionId) { 485 try { 486 IAccessibilityServiceConnection connection = getConnection(connectionId); 487 if (connection != null) { 488 SparseArray<List<AccessibilityWindowInfo>> windows; 489 AccessibilityCache cache = getCache(connectionId); 490 if (cache != null) { 491 windows = cache.getWindowsOnAllDisplays(); 492 if (windows != null) { 493 if (DEBUG) { 494 Log.i(LOG_TAG, "Windows cache hit"); 495 } 496 if (shouldTraceClient()) { 497 logTraceClient( 498 connection, "getWindows cache", "connectionId=" + connectionId); 499 } 500 return windows; 501 } 502 if (DEBUG) { 503 Log.i(LOG_TAG, "Windows cache miss"); 504 } 505 } else { 506 if (DEBUG) { 507 Log.w(LOG_TAG, "Cache is null for connection id: " + connectionId); 508 } 509 } 510 511 long populationTimeStamp; 512 final long identityToken = Binder.clearCallingIdentity(); 513 try { 514 populationTimeStamp = SystemClock.uptimeMillis(); 515 windows = connection.getWindows(); 516 } finally { 517 Binder.restoreCallingIdentity(identityToken); 518 } 519 if (shouldTraceClient()) { 520 logTraceClient(connection, "getWindows", "connectionId=" + connectionId); 521 } 522 if (windows != null) { 523 if (cache != null) { 524 cache.setWindowsOnAllDisplays(windows, populationTimeStamp); 525 } 526 return windows; 527 } 528 } else { 529 if (DEBUG) { 530 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 531 } 532 } 533 } catch (RemoteException re) { 534 Log.e(LOG_TAG, "Error while calling remote getWindowsOnAllDisplays", re); 535 } 536 537 final SparseArray<List<AccessibilityWindowInfo>> emptyWindows = new SparseArray<>(); 538 return emptyWindows; 539 } 540 541 542 /** 543 * Finds an {@link AccessibilityNodeInfo} by accessibility id and given leash token instead of 544 * window id. This method is used to find the leashed node on the embedded view hierarchy. 545 * 546 * @param connectionId The id of a connection for interacting with the system. 547 * @param leashToken The token of the embedded hierarchy. 548 * @param accessibilityNodeId A unique view id or virtual descendant id from 549 * where to start the search. Use 550 * {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID} 551 * to start from the root. 552 * @param bypassCache Whether to bypass the cache while looking for the node. 553 * @param prefetchFlags flags to guide prefetching. 554 * @param arguments Optional action arguments. 555 * @return An {@link AccessibilityNodeInfo} if found, null otherwise. 556 */ findAccessibilityNodeInfoByAccessibilityId( int connectionId, @NonNull IBinder leashToken, long accessibilityNodeId, boolean bypassCache, int prefetchFlags, Bundle arguments)557 public @Nullable AccessibilityNodeInfo findAccessibilityNodeInfoByAccessibilityId( 558 int connectionId, @NonNull IBinder leashToken, long accessibilityNodeId, 559 boolean bypassCache, int prefetchFlags, Bundle arguments) { 560 if (leashToken == null) { 561 return null; 562 } 563 int windowId = -1; 564 try { 565 IAccessibilityServiceConnection connection = getConnection(connectionId); 566 if (connection != null) { 567 windowId = connection.getWindowIdForLeashToken(leashToken); 568 } else { 569 if (DEBUG) { 570 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 571 } 572 } 573 } catch (RemoteException re) { 574 Log.e(LOG_TAG, "Error while calling remote getWindowIdForLeashToken", re); 575 } 576 if (windowId == -1) { 577 return null; 578 } 579 return findAccessibilityNodeInfoByAccessibilityId(connectionId, windowId, 580 accessibilityNodeId, bypassCache, prefetchFlags, arguments); 581 } 582 583 /** 584 * Finds an {@link AccessibilityNodeInfo} by accessibility id. 585 * 586 * @param connectionId The id of a connection for interacting with the system. 587 * @param accessibilityWindowId A unique window id. Use 588 * {@link android.view.accessibility.AccessibilityWindowInfo#ACTIVE_WINDOW_ID} 589 * to query the currently active window. 590 * @param accessibilityNodeId A unique view id or virtual descendant id from 591 * where to start the search. Use 592 * {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID} 593 * to start from the root. 594 * @param bypassCache Whether to bypass the cache while looking for the node. 595 * @param prefetchFlags flags to guide prefetching. 596 * @return An {@link AccessibilityNodeInfo} if found, null otherwise. 597 */ findAccessibilityNodeInfoByAccessibilityId( int connectionId, int accessibilityWindowId, long accessibilityNodeId, boolean bypassCache, int prefetchFlags, Bundle arguments)598 public @Nullable AccessibilityNodeInfo findAccessibilityNodeInfoByAccessibilityId( 599 int connectionId, int accessibilityWindowId, long accessibilityNodeId, 600 boolean bypassCache, int prefetchFlags, Bundle arguments) { 601 try { 602 IAccessibilityServiceConnection connection = getConnection(connectionId); 603 if (connection != null) { 604 if (!bypassCache) { 605 AccessibilityCache cache = getCache(connectionId); 606 if (cache != null) { 607 AccessibilityNodeInfo cachedInfo = cache.getNode( 608 accessibilityWindowId, accessibilityNodeId); 609 if (cachedInfo != null) { 610 if (DEBUG) { 611 Log.i(LOG_TAG, "Node cache hit for " 612 + idToString(accessibilityWindowId, accessibilityNodeId)); 613 } 614 if (shouldTraceClient()) { 615 logTraceClient(connection, 616 "findAccessibilityNodeInfoByAccessibilityId cache", 617 "connectionId=" + connectionId + ";accessibilityWindowId=" 618 + accessibilityWindowId + ";accessibilityNodeId=" 619 + accessibilityNodeId + ";bypassCache=" 620 + bypassCache + ";prefetchFlags=" + prefetchFlags 621 + ";arguments=" + arguments); 622 } 623 return cachedInfo; 624 } 625 if (!cache.isEnabled()) { 626 // Skip prefetching if cache is disabled. 627 prefetchFlags &= ~FLAG_PREFETCH_MASK; 628 } 629 if (DEBUG) { 630 Log.i(LOG_TAG, "Node cache miss for " 631 + idToString(accessibilityWindowId, accessibilityNodeId)); 632 } 633 } else { 634 if (DEBUG) { 635 Log.w(LOG_TAG, "Cache is null for connection id: " + connectionId); 636 } 637 } 638 } else { 639 // No need to prefech nodes in bypass cache case. 640 prefetchFlags &= ~FLAG_PREFETCH_MASK; 641 } 642 // Skip prefetching if window is scrolling. 643 if ((prefetchFlags & FLAG_PREFETCH_MASK) != 0 644 && isWindowScrolling(accessibilityWindowId)) { 645 prefetchFlags &= ~FLAG_PREFETCH_MASK; 646 } 647 648 final int descendantPrefetchFlags = prefetchFlags & FLAG_PREFETCH_DESCENDANTS_MASK; 649 if ((descendantPrefetchFlags & (descendantPrefetchFlags - 1)) != 0) { 650 throw new IllegalArgumentException("There can be no more than one descendant" 651 + " prefetching strategy"); 652 } 653 final int interactionId = mInteractionIdCounter.getAndIncrement(); 654 if (shouldTraceClient()) { 655 logTraceClient(connection, "findAccessibilityNodeInfoByAccessibilityId", 656 "InteractionId:" + interactionId + "connectionId=" + connectionId 657 + ";accessibilityWindowId=" + accessibilityWindowId 658 + ";accessibilityNodeId=" + accessibilityNodeId + ";bypassCache=" 659 + bypassCache + ";prefetchFlags=" + prefetchFlags + ";arguments=" 660 + arguments); 661 } 662 final String[] packageNames; 663 final long identityToken = Binder.clearCallingIdentity(); 664 try { 665 packageNames = connection.findAccessibilityNodeInfoByAccessibilityId( 666 accessibilityWindowId, accessibilityNodeId, interactionId, this, 667 prefetchFlags, Thread.currentThread().getId(), arguments); 668 } finally { 669 Binder.restoreCallingIdentity(identityToken); 670 } 671 if (packageNames != null) { 672 if ((prefetchFlags 673 & AccessibilityNodeInfo.FLAG_PREFETCH_UNINTERRUPTIBLE) != 0) { 674 List<AccessibilityNodeInfo> infos = 675 getFindAccessibilityNodeInfosResultAndClear( 676 interactionId); 677 if (shouldTraceCallback()) { 678 logTraceCallback(connection, 679 "findAccessibilityNodeInfoByAccessibilityId", 680 "InteractionId:" + interactionId + ";connectionId=" 681 + connectionId + ";Result: " + infos); 682 } 683 finalizeAndCacheAccessibilityNodeInfos(infos, connectionId, 684 bypassCache, packageNames); 685 if (infos != null && !infos.isEmpty()) { 686 return infos.get(0); 687 } 688 } else { 689 AccessibilityNodeInfo info = 690 getFindAccessibilityNodeInfoResultAndClear(interactionId); 691 if (shouldTraceCallback()) { 692 logTraceCallback(connection, 693 "findAccessibilityNodeInfoByAccessibilityId", 694 "InteractionId:" + interactionId + ";connectionId=" 695 + connectionId + ";Result: " + info); 696 } 697 if ((prefetchFlags & FLAG_PREFETCH_MASK) != 0 698 && info != null) { 699 setInteractionWaitingForPrefetchResult(interactionId, connectionId, 700 packageNames); 701 } 702 finalizeAndCacheAccessibilityNodeInfo(info, connectionId, 703 bypassCache, packageNames); 704 return info; 705 } 706 707 } 708 } else { 709 if (DEBUG) { 710 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 711 } 712 } 713 } catch (RemoteException re) { 714 Log.e(LOG_TAG, "Error while calling remote" 715 + " findAccessibilityNodeInfoByAccessibilityId", re); 716 } 717 return null; 718 } 719 setInteractionWaitingForPrefetchResult(int interactionId, int connectionId, String[] packageNames)720 private void setInteractionWaitingForPrefetchResult(int interactionId, int connectionId, 721 String[] packageNames) { 722 synchronized (mInstanceLock) { 723 mInteractionIdWaitingForPrefetchResult = interactionId; 724 mConnectionIdWaitingForPrefetchResult = connectionId; 725 mPackageNamesForNextPrefetchResult = packageNames; 726 } 727 } 728 idToString(int accessibilityWindowId, long accessibilityNodeId)729 private static String idToString(int accessibilityWindowId, long accessibilityNodeId) { 730 return accessibilityWindowId + "/" 731 + AccessibilityNodeInfo.idToString(accessibilityNodeId); 732 } 733 734 /** 735 * Finds an {@link AccessibilityNodeInfo} by View id. The search is performed in 736 * the window whose id is specified and starts from the node whose accessibility 737 * id is specified. 738 * 739 * @param connectionId The id of a connection for interacting with the system. 740 * @param accessibilityWindowId A unique window id. Use 741 * {@link android.view.accessibility.AccessibilityWindowInfo#ACTIVE_WINDOW_ID} 742 * to query the currently active window. 743 * @param accessibilityNodeId A unique view id or virtual descendant id from 744 * where to start the search. Use 745 * {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID} 746 * to start from the root. 747 * @param viewId The fully qualified resource name of the view id to find. 748 * @return An list of {@link AccessibilityNodeInfo} if found, empty list otherwise. 749 */ findAccessibilityNodeInfosByViewId(int connectionId, int accessibilityWindowId, long accessibilityNodeId, String viewId)750 public List<AccessibilityNodeInfo> findAccessibilityNodeInfosByViewId(int connectionId, 751 int accessibilityWindowId, long accessibilityNodeId, String viewId) { 752 try { 753 IAccessibilityServiceConnection connection = getConnection(connectionId); 754 if (connection != null) { 755 final int interactionId = mInteractionIdCounter.getAndIncrement(); 756 final String[] packageNames; 757 final long identityToken = Binder.clearCallingIdentity(); 758 try { 759 if (shouldTraceClient()) { 760 logTraceClient(connection, "findAccessibilityNodeInfosByViewId", 761 "InteractionId=" + interactionId + ";connectionId=" + connectionId 762 + ";accessibilityWindowId=" + accessibilityWindowId 763 + ";accessibilityNodeId=" + accessibilityNodeId + ";viewId=" 764 + viewId); 765 } 766 767 packageNames = connection.findAccessibilityNodeInfosByViewId( 768 accessibilityWindowId, accessibilityNodeId, viewId, interactionId, this, 769 Thread.currentThread().getId()); 770 } finally { 771 Binder.restoreCallingIdentity(identityToken); 772 } 773 774 if (packageNames != null) { 775 List<AccessibilityNodeInfo> infos = getFindAccessibilityNodeInfosResultAndClear( 776 interactionId); 777 if (shouldTraceCallback()) { 778 logTraceCallback(connection, "findAccessibilityNodeInfosByViewId", 779 "InteractionId=" + interactionId + ";connectionId=" + connectionId 780 + ":Result: " + infos); 781 } 782 if (infos != null) { 783 finalizeAndCacheAccessibilityNodeInfos(infos, connectionId, 784 false, packageNames); 785 return infos; 786 } 787 } 788 } else { 789 if (DEBUG) { 790 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 791 } 792 } 793 } catch (RemoteException re) { 794 Log.w(LOG_TAG, "Error while calling remote" 795 + " findAccessibilityNodeInfoByViewIdInActiveWindow", re); 796 } 797 return Collections.emptyList(); 798 } 799 800 /** 801 * Takes a screenshot of the window with the provided {@code accessibilityWindowId} and 802 * returns the answer asynchronously. This async behavior is similar to {@link 803 * AccessibilityService#takeScreenshot} but unlike other methods in this class which perform 804 * synchronous waiting in the AccessibilityService client. 805 * 806 * @see AccessibilityService#takeScreenshotOfWindow 807 */ takeScreenshotOfWindow(int connectionId, int accessibilityWindowId, @NonNull @CallbackExecutor Executor executor, @NonNull AccessibilityService.TakeScreenshotCallback callback)808 public void takeScreenshotOfWindow(int connectionId, int accessibilityWindowId, 809 @NonNull @CallbackExecutor Executor executor, 810 @NonNull AccessibilityService.TakeScreenshotCallback callback) { 811 synchronized (mInstanceLock) { 812 try { 813 IAccessibilityServiceConnection connection = getConnection(connectionId); 814 if (connection == null) { 815 executor.execute(() -> callback.onFailure( 816 AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR)); 817 return; 818 } 819 final long identityToken = Binder.clearCallingIdentity(); 820 try { 821 final int interactionId = mInteractionIdCounter.getAndIncrement(); 822 mTakeScreenshotOfWindowCallbacks.put(interactionId, 823 Pair.create(executor, callback)); 824 // Create a ScreenCaptureListener to receive the screenshot directly from 825 // SurfaceFlinger instead of requiring an extra IPC from the app: 826 // A11yService -> App -> SurfaceFlinger -> A11yService 827 ScreenCapture.ScreenCaptureListener listener = 828 new ScreenCapture.ScreenCaptureListener( 829 screenshot -> sendWindowScreenshotSuccess(screenshot, 830 interactionId)); 831 connection.takeScreenshotOfWindow(accessibilityWindowId, interactionId, 832 listener, this); 833 new Handler(Looper.getMainLooper()).postDelayed(() -> { 834 synchronized (mInstanceLock) { 835 // Notify failure if we still haven't sent a response after timeout. 836 if (mTakeScreenshotOfWindowCallbacks.contains(interactionId)) { 837 sendTakeScreenshotOfWindowError( 838 AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR, 839 interactionId); 840 } 841 } 842 }, TIMEOUT_INTERACTION_MILLIS); 843 } finally { 844 Binder.restoreCallingIdentity(identityToken); 845 } 846 } catch (RemoteException re) { 847 executor.execute(() -> callback.onFailure( 848 AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR)); 849 } 850 } 851 } 852 853 /** 854 * Finds {@link AccessibilityNodeInfo}s by View text. The match is case 855 * insensitive containment. The search is performed in the window whose 856 * id is specified and starts from the node whose accessibility id is 857 * specified. 858 * 859 * @param connectionId The id of a connection for interacting with the system. 860 * @param accessibilityWindowId A unique window id. Use 861 * {@link android.view.accessibility.AccessibilityWindowInfo#ACTIVE_WINDOW_ID} 862 * to query the currently active window. 863 * @param accessibilityNodeId A unique view id or virtual descendant id from 864 * where to start the search. Use 865 * {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID} 866 * to start from the root. 867 * @param text The searched text. 868 * @return A list of found {@link AccessibilityNodeInfo}s. 869 */ findAccessibilityNodeInfosByText(int connectionId, int accessibilityWindowId, long accessibilityNodeId, String text)870 public List<AccessibilityNodeInfo> findAccessibilityNodeInfosByText(int connectionId, 871 int accessibilityWindowId, long accessibilityNodeId, String text) { 872 try { 873 IAccessibilityServiceConnection connection = getConnection(connectionId); 874 if (connection != null) { 875 final int interactionId = mInteractionIdCounter.getAndIncrement(); 876 if (shouldTraceClient()) { 877 logTraceClient(connection, "findAccessibilityNodeInfosByText", 878 "InteractionId:" + interactionId + "connectionId=" + connectionId 879 + ";accessibilityWindowId=" + accessibilityWindowId 880 + ";accessibilityNodeId=" + accessibilityNodeId + ";text=" + text); 881 } 882 final String[] packageNames; 883 final long identityToken = Binder.clearCallingIdentity(); 884 try { 885 packageNames = connection.findAccessibilityNodeInfosByText( 886 accessibilityWindowId, accessibilityNodeId, text, interactionId, this, 887 Thread.currentThread().getId()); 888 } finally { 889 Binder.restoreCallingIdentity(identityToken); 890 } 891 892 if (packageNames != null) { 893 List<AccessibilityNodeInfo> infos = getFindAccessibilityNodeInfosResultAndClear( 894 interactionId); 895 if (shouldTraceCallback()) { 896 logTraceCallback(connection, "findAccessibilityNodeInfosByText", 897 "InteractionId=" + interactionId + ";connectionId=" + connectionId 898 + ";Result: " + infos); 899 } 900 if (infos != null) { 901 finalizeAndCacheAccessibilityNodeInfos(infos, connectionId, 902 false, packageNames); 903 return infos; 904 } 905 } 906 } else { 907 if (DEBUG) { 908 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 909 } 910 } 911 } catch (RemoteException re) { 912 Log.w(LOG_TAG, "Error while calling remote" 913 + " findAccessibilityNodeInfosByViewText", re); 914 } 915 return Collections.emptyList(); 916 } 917 918 /** 919 * Finds the {@link AccessibilityNodeInfo} that has the 920 * specified focus type. The search is performed in the window whose id is specified 921 * and starts from the node whose accessibility id is specified. 922 * 923 * @param connectionId The id of a connection for interacting with the system. 924 * @param accessibilityWindowId A unique window id. Use 925 * {@link AccessibilityWindowInfo#ANY_WINDOW_ID} to query all 926 * windows 927 * @param accessibilityNodeId A unique view id or virtual descendant id from 928 * where to start the search. Use 929 * {@link AccessibilityNodeInfo#ROOT_NODE_ID} 930 * to start from the root. 931 * @param focusType The focus type. 932 * @return The accessibility focused {@link AccessibilityNodeInfo}. 933 */ 934 @SuppressLint("LongLogTag") findFocus(int connectionId, int accessibilityWindowId, long accessibilityNodeId, int focusType)935 public AccessibilityNodeInfo findFocus(int connectionId, int accessibilityWindowId, 936 long accessibilityNodeId, int focusType) { 937 try { 938 IAccessibilityServiceConnection connection = getConnection(connectionId); 939 if (connection != null) { 940 AccessibilityCache cache = getCache(connectionId); 941 if (cache != null) { 942 AccessibilityNodeInfo cachedInfo = cache.getFocus(focusType, 943 accessibilityNodeId, accessibilityWindowId); 944 if (cachedInfo != null) { 945 if (DEBUG) { 946 Log.i(LOG_TAG, "Focused node cache hit retrieved" 947 + idToString(cachedInfo.getWindowId(), 948 cachedInfo.getSourceNodeId())); 949 } 950 return cachedInfo; 951 } 952 if (DEBUG) { 953 Log.i(LOG_TAG, "Focused node cache miss with " 954 + idToString(accessibilityWindowId, accessibilityNodeId)); 955 } 956 } else { 957 if (DEBUG) { 958 Log.w(LOG_TAG, "Cache is null for connection id: " + connectionId); 959 } 960 } 961 final int interactionId = mInteractionIdCounter.getAndIncrement(); 962 if (shouldTraceClient()) { 963 logTraceClient(connection, "findFocus", 964 "InteractionId:" + interactionId + "connectionId=" + connectionId 965 + ";accessibilityWindowId=" + accessibilityWindowId 966 + ";accessibilityNodeId=" + accessibilityNodeId + ";focusType=" 967 + focusType); 968 } 969 final String[] packageNames; 970 final long identityToken = Binder.clearCallingIdentity(); 971 try { 972 packageNames = connection.findFocus(accessibilityWindowId, 973 accessibilityNodeId, focusType, interactionId, this, 974 Thread.currentThread().getId()); 975 } finally { 976 Binder.restoreCallingIdentity(identityToken); 977 } 978 979 if (packageNames != null) { 980 AccessibilityNodeInfo info = getFindAccessibilityNodeInfoResultAndClear( 981 interactionId); 982 if (shouldTraceCallback()) { 983 logTraceCallback(connection, "findFocus", "InteractionId=" + interactionId 984 + ";connectionId=" + connectionId + ";Result:" + info); 985 } 986 finalizeAndCacheAccessibilityNodeInfo(info, connectionId, false, packageNames); 987 return info; 988 } 989 } else { 990 if (DEBUG) { 991 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 992 } 993 } 994 } catch (RemoteException re) { 995 Log.w(LOG_TAG, "Error while calling remote findFocus", re); 996 } 997 return null; 998 } 999 1000 /** 1001 * Finds the accessibility focused {@link android.view.accessibility.AccessibilityNodeInfo}. 1002 * The search is performed in the window whose id is specified and starts from the 1003 * node whose accessibility id is specified. 1004 * 1005 * @param connectionId The id of a connection for interacting with the system. 1006 * @param accessibilityWindowId A unique window id. Use 1007 * {@link android.view.accessibility.AccessibilityWindowInfo#ACTIVE_WINDOW_ID} 1008 * to query the currently active window. 1009 * @param accessibilityNodeId A unique view id or virtual descendant id from 1010 * where to start the search. Use 1011 * {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID} 1012 * to start from the root. 1013 * @param direction The direction in which to search for focusable. 1014 * @return The accessibility focused {@link AccessibilityNodeInfo}. 1015 */ focusSearch(int connectionId, int accessibilityWindowId, long accessibilityNodeId, int direction)1016 public AccessibilityNodeInfo focusSearch(int connectionId, int accessibilityWindowId, 1017 long accessibilityNodeId, int direction) { 1018 try { 1019 IAccessibilityServiceConnection connection = getConnection(connectionId); 1020 if (connection != null) { 1021 final int interactionId = mInteractionIdCounter.getAndIncrement(); 1022 if (shouldTraceClient()) { 1023 logTraceClient(connection, "focusSearch", 1024 "InteractionId:" + interactionId + "connectionId=" + connectionId 1025 + ";accessibilityWindowId=" + accessibilityWindowId 1026 + ";accessibilityNodeId=" + accessibilityNodeId + ";direction=" 1027 + direction); 1028 } 1029 final String[] packageNames; 1030 final long identityToken = Binder.clearCallingIdentity(); 1031 try { 1032 packageNames = connection.focusSearch(accessibilityWindowId, 1033 accessibilityNodeId, direction, interactionId, this, 1034 Thread.currentThread().getId()); 1035 } finally { 1036 Binder.restoreCallingIdentity(identityToken); 1037 } 1038 1039 if (packageNames != null) { 1040 AccessibilityNodeInfo info = getFindAccessibilityNodeInfoResultAndClear( 1041 interactionId); 1042 finalizeAndCacheAccessibilityNodeInfo(info, connectionId, false, packageNames); 1043 if (shouldTraceCallback()) { 1044 logTraceCallback(connection, "focusSearch", "InteractionId=" + interactionId 1045 + ";connectionId=" + connectionId + ";Result:" + info); 1046 } 1047 return info; 1048 } 1049 } else { 1050 if (DEBUG) { 1051 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 1052 } 1053 } 1054 } catch (RemoteException re) { 1055 Log.w(LOG_TAG, "Error while calling remote accessibilityFocusSearch", re); 1056 } 1057 return null; 1058 } 1059 1060 /** 1061 * Performs an accessibility action on an {@link AccessibilityNodeInfo}. 1062 * 1063 * @param connectionId The id of a connection for interacting with the system. 1064 * @param accessibilityWindowId A unique window id. Use 1065 * {@link android.view.accessibility.AccessibilityWindowInfo#ACTIVE_WINDOW_ID} 1066 * to query the currently active window. 1067 * @param accessibilityNodeId A unique view id or virtual descendant id from 1068 * where to start the search. Use 1069 * {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID} 1070 * to start from the root. 1071 * @param action The action to perform. 1072 * @param arguments Optional action arguments. 1073 * @return Whether the action was performed. 1074 */ performAccessibilityAction(int connectionId, int accessibilityWindowId, long accessibilityNodeId, int action, Bundle arguments)1075 public boolean performAccessibilityAction(int connectionId, int accessibilityWindowId, 1076 long accessibilityNodeId, int action, Bundle arguments) { 1077 try { 1078 IAccessibilityServiceConnection connection = getConnection(connectionId); 1079 if (connection != null) { 1080 final int interactionId = mInteractionIdCounter.getAndIncrement(); 1081 if (shouldTraceClient()) { 1082 logTraceClient(connection, "performAccessibilityAction", 1083 "InteractionId:" + interactionId + "connectionId=" + connectionId 1084 + ";accessibilityWindowId=" + accessibilityWindowId 1085 + ";accessibilityNodeId=" + accessibilityNodeId + ";action=" + action 1086 + ";arguments=" + arguments); 1087 } 1088 final boolean success; 1089 final long identityToken = Binder.clearCallingIdentity(); 1090 try { 1091 success = connection.performAccessibilityAction( 1092 accessibilityWindowId, accessibilityNodeId, action, arguments, 1093 interactionId, this, Thread.currentThread().getId()); 1094 } finally { 1095 Binder.restoreCallingIdentity(identityToken); 1096 } 1097 1098 if (success) { 1099 final boolean result = 1100 getPerformAccessibilityActionResultAndClear(interactionId); 1101 if (shouldTraceCallback()) { 1102 logTraceCallback(connection, "performAccessibilityAction", 1103 "InteractionId=" + interactionId + ";connectionId=" + connectionId 1104 + ";Result: " + result); 1105 } 1106 return result; 1107 } 1108 } else { 1109 if (DEBUG) { 1110 Log.w(LOG_TAG, "No connection for connection id: " + connectionId); 1111 } 1112 } 1113 } catch (RemoteException re) { 1114 Log.w(LOG_TAG, "Error while calling remote performAccessibilityAction", re); 1115 } 1116 return false; 1117 } 1118 1119 /** 1120 * Clears the cache associated with {@code connectionId} 1121 * @param connectionId the connection id 1122 */ 1123 @UnsupportedAppUsage(maxTargetSdk = S, publicAlternatives = 1124 "{@link android.accessibilityservice.AccessibilityService#clearCache()}") clearCache(int connectionId)1125 public void clearCache(int connectionId) { 1126 AccessibilityCache cache = getCache(connectionId); 1127 if (cache == null) { 1128 return; 1129 } 1130 cache.clear(); 1131 } 1132 1133 /** 1134 * Informs the cache associated with {@code connectionId} of {@code event} 1135 * @param event the event 1136 * @param connectionId the connection id 1137 */ onAccessibilityEvent(AccessibilityEvent event, int connectionId)1138 public void onAccessibilityEvent(AccessibilityEvent event, int connectionId) { 1139 switch (event.getEventType()) { 1140 case AccessibilityEvent.TYPE_VIEW_SCROLLED: 1141 updateScrollingWindow(event.getWindowId(), SystemClock.uptimeMillis()); 1142 break; 1143 case AccessibilityEvent.TYPE_WINDOWS_CHANGED: 1144 if (event.getWindowChanges() == AccessibilityEvent.WINDOWS_CHANGE_REMOVED) { 1145 deleteScrollingWindow(event.getWindowId()); 1146 } 1147 break; 1148 default: 1149 break; 1150 } 1151 AccessibilityCache cache = getCache(connectionId); 1152 if (cache == null) { 1153 if (DEBUG) { 1154 Log.w(LOG_TAG, "Cache is null for connection id: " + connectionId); 1155 } 1156 return; 1157 } 1158 cache.onAccessibilityEvent(event); 1159 } 1160 1161 /** 1162 * Gets the the result of an async request that returns an {@link AccessibilityNodeInfo}. 1163 * 1164 * @param interactionId The interaction id to match the result with the request. 1165 * @return The result {@link AccessibilityNodeInfo}. 1166 */ getFindAccessibilityNodeInfoResultAndClear(int interactionId)1167 private AccessibilityNodeInfo getFindAccessibilityNodeInfoResultAndClear(int interactionId) { 1168 synchronized (mInstanceLock) { 1169 final boolean success = waitForResultTimedLocked(interactionId); 1170 AccessibilityNodeInfo result = success ? mFindAccessibilityNodeInfoResult : null; 1171 clearResultLocked(); 1172 return result; 1173 } 1174 } 1175 1176 /** 1177 * {@inheritDoc} 1178 */ setFindAccessibilityNodeInfoResult(AccessibilityNodeInfo info, int interactionId)1179 public void setFindAccessibilityNodeInfoResult(AccessibilityNodeInfo info, 1180 int interactionId) { 1181 synchronized (mInstanceLock) { 1182 if (interactionId > mInteractionId) { 1183 mFindAccessibilityNodeInfoResult = info; 1184 mInteractionId = interactionId; 1185 mCallingUid = Binder.getCallingUid(); 1186 mCallStackOfCallback = new ArrayList<StackTraceElement>( 1187 Arrays.asList(Thread.currentThread().getStackTrace())); 1188 } 1189 mInstanceLock.notifyAll(); 1190 } 1191 } 1192 1193 /** 1194 * Gets the the result of an async request that returns {@link AccessibilityNodeInfo}s. 1195 * 1196 * @param interactionId The interaction id to match the result with the request. 1197 * @return The result {@link AccessibilityNodeInfo}s. 1198 */ getFindAccessibilityNodeInfosResultAndClear( int interactionId)1199 private List<AccessibilityNodeInfo> getFindAccessibilityNodeInfosResultAndClear( 1200 int interactionId) { 1201 synchronized (mInstanceLock) { 1202 final boolean success = waitForResultTimedLocked(interactionId); 1203 final List<AccessibilityNodeInfo> result; 1204 if (success) { 1205 result = mFindAccessibilityNodeInfosResult; 1206 } else { 1207 result = Collections.emptyList(); 1208 } 1209 clearResultLocked(); 1210 if (Build.IS_DEBUGGABLE && CHECK_INTEGRITY) { 1211 checkFindAccessibilityNodeInfoResultIntegrity(result); 1212 } 1213 return result; 1214 } 1215 } 1216 1217 /** 1218 * {@inheritDoc} 1219 */ setFindAccessibilityNodeInfosResult(List<AccessibilityNodeInfo> infos, int interactionId)1220 public void setFindAccessibilityNodeInfosResult(List<AccessibilityNodeInfo> infos, 1221 int interactionId) { 1222 synchronized (mInstanceLock) { 1223 if (interactionId > mInteractionId) { 1224 if (infos != null) { 1225 // If the call is not an IPC, i.e. it is made from the same process, we need to 1226 // instantiate new result list to avoid passing internal instances to clients. 1227 final boolean isIpcCall = (Binder.getCallingPid() != Process.myPid()); 1228 if (!isIpcCall) { 1229 mFindAccessibilityNodeInfosResult = new ArrayList<>(infos); 1230 } else { 1231 mFindAccessibilityNodeInfosResult = infos; 1232 } 1233 } else { 1234 mFindAccessibilityNodeInfosResult = Collections.emptyList(); 1235 } 1236 mInteractionId = interactionId; 1237 mCallingUid = Binder.getCallingUid(); 1238 mCallStackOfCallback = new ArrayList<StackTraceElement>( 1239 Arrays.asList(Thread.currentThread().getStackTrace())); 1240 } 1241 mInstanceLock.notifyAll(); 1242 } 1243 } 1244 1245 /** 1246 * {@inheritDoc} 1247 */ 1248 @Override setPrefetchAccessibilityNodeInfoResult(@onNull List<AccessibilityNodeInfo> infos, int interactionId)1249 public void setPrefetchAccessibilityNodeInfoResult(@NonNull List<AccessibilityNodeInfo> infos, 1250 int interactionId) { 1251 int interactionIdWaitingForPrefetchResultCopy = -1; 1252 int connectionIdWaitingForPrefetchResultCopy = -1; 1253 String[] packageNamesForNextPrefetchResultCopy = null; 1254 1255 if (infos.isEmpty()) { 1256 return; 1257 } 1258 1259 synchronized (mInstanceLock) { 1260 if (mInteractionIdWaitingForPrefetchResult == interactionId) { 1261 interactionIdWaitingForPrefetchResultCopy = mInteractionIdWaitingForPrefetchResult; 1262 connectionIdWaitingForPrefetchResultCopy = 1263 mConnectionIdWaitingForPrefetchResult; 1264 if (mPackageNamesForNextPrefetchResult != null) { 1265 packageNamesForNextPrefetchResultCopy = 1266 new String[mPackageNamesForNextPrefetchResult.length]; 1267 for (int i = 0; i < mPackageNamesForNextPrefetchResult.length; i++) { 1268 packageNamesForNextPrefetchResultCopy[i] = 1269 mPackageNamesForNextPrefetchResult[i]; 1270 } 1271 } 1272 } 1273 } 1274 1275 if (interactionIdWaitingForPrefetchResultCopy == interactionId) { 1276 finalizeAndCacheAccessibilityNodeInfos( 1277 infos, connectionIdWaitingForPrefetchResultCopy, false, 1278 packageNamesForNextPrefetchResultCopy); 1279 if (shouldTraceCallback()) { 1280 logTrace(getConnection(connectionIdWaitingForPrefetchResultCopy), 1281 "setPrefetchAccessibilityNodeInfoResult", 1282 "InteractionId:" + interactionId + ";connectionId=" 1283 + connectionIdWaitingForPrefetchResultCopy + ";Result: " + infos, 1284 Binder.getCallingUid(), 1285 Arrays.asList(Thread.currentThread().getStackTrace()), 1286 new HashSet<>(Collections.singletonList("getStackTrace")), 1287 FLAGS_ACCESSIBILITY_INTERACTION_CONNECTION_CALLBACK); 1288 } 1289 } else if (DEBUG) { 1290 Log.w(LOG_TAG, "Prefetching for interaction with id " + interactionId + " dropped " 1291 + infos.size() + " nodes"); 1292 } 1293 } 1294 1295 /** 1296 * Gets the result of a request to perform an accessibility action. 1297 * 1298 * @param interactionId The interaction id to match the result with the request. 1299 * @return Whether the action was performed. 1300 */ getPerformAccessibilityActionResultAndClear(int interactionId)1301 private boolean getPerformAccessibilityActionResultAndClear(int interactionId) { 1302 synchronized (mInstanceLock) { 1303 final boolean success = waitForResultTimedLocked(interactionId); 1304 final boolean result = success ? mPerformAccessibilityActionResult : false; 1305 clearResultLocked(); 1306 return result; 1307 } 1308 } 1309 1310 /** 1311 * {@inheritDoc} 1312 */ setPerformAccessibilityActionResult(boolean succeeded, int interactionId)1313 public void setPerformAccessibilityActionResult(boolean succeeded, int interactionId) { 1314 synchronized (mInstanceLock) { 1315 if (interactionId > mInteractionId) { 1316 mPerformAccessibilityActionResult = succeeded; 1317 mInteractionId = interactionId; 1318 mCallingUid = Binder.getCallingUid(); 1319 mCallStackOfCallback = new ArrayList<StackTraceElement>( 1320 Arrays.asList(Thread.currentThread().getStackTrace())); 1321 } 1322 mInstanceLock.notifyAll(); 1323 } 1324 } 1325 1326 /** 1327 * Sends the result of a window screenshot request to the requesting client. 1328 * 1329 * {@link #takeScreenshotOfWindow} does not perform synchronous waiting, so this method 1330 * does not notify any wait lock. 1331 */ sendWindowScreenshotSuccess(ScreenCapture.ScreenshotHardwareBuffer screenshot, int interactionId)1332 private void sendWindowScreenshotSuccess(ScreenCapture.ScreenshotHardwareBuffer screenshot, 1333 int interactionId) { 1334 if (screenshot == null) { 1335 sendTakeScreenshotOfWindowError( 1336 AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR, interactionId); 1337 return; 1338 } 1339 synchronized (mInstanceLock) { 1340 if (mTakeScreenshotOfWindowCallbacks.contains(interactionId)) { 1341 final AccessibilityService.ScreenshotResult result = 1342 new AccessibilityService.ScreenshotResult(screenshot.getHardwareBuffer(), 1343 screenshot.getColorSpace(), SystemClock.uptimeMillis()); 1344 final Pair<Executor, AccessibilityService.TakeScreenshotCallback> pair = 1345 mTakeScreenshotOfWindowCallbacks.get(interactionId); 1346 final Executor executor = pair.first; 1347 final AccessibilityService.TakeScreenshotCallback callback = pair.second; 1348 executor.execute(() -> callback.onSuccess(result)); 1349 mTakeScreenshotOfWindowCallbacks.remove(interactionId); 1350 } 1351 } 1352 } 1353 1354 /** 1355 * Sends an error code for a window screenshot request to the requesting client. 1356 * 1357 * @param errorCode The error code from {@link AccessibilityService.ScreenshotErrorCode}. 1358 * @param interactionId The interaction id of the request. 1359 */ 1360 @Override sendTakeScreenshotOfWindowError( @ccessibilityService.ScreenshotErrorCode int errorCode, int interactionId)1361 public void sendTakeScreenshotOfWindowError( 1362 @AccessibilityService.ScreenshotErrorCode int errorCode, int interactionId) { 1363 synchronized (mInstanceLock) { 1364 if (mTakeScreenshotOfWindowCallbacks.contains(interactionId)) { 1365 final Pair<Executor, AccessibilityService.TakeScreenshotCallback> pair = 1366 mTakeScreenshotOfWindowCallbacks.get(interactionId); 1367 final Executor executor = pair.first; 1368 final AccessibilityService.TakeScreenshotCallback callback = pair.second; 1369 executor.execute(() -> callback.onFailure(errorCode)); 1370 mTakeScreenshotOfWindowCallbacks.remove(interactionId); 1371 } 1372 } 1373 } 1374 1375 /** 1376 * Clears the result state. 1377 */ clearResultLocked()1378 private void clearResultLocked() { 1379 mInteractionId = -1; 1380 mFindAccessibilityNodeInfoResult = null; 1381 mFindAccessibilityNodeInfosResult = null; 1382 mPerformAccessibilityActionResult = false; 1383 } 1384 1385 /** 1386 * Waits up to a given bound for a result of a request and returns it. 1387 * 1388 * @param interactionId The interaction id to match the result with the request. 1389 * @return Whether the result was received. 1390 */ waitForResultTimedLocked(int interactionId)1391 private boolean waitForResultTimedLocked(int interactionId) { 1392 long waitTimeMillis = TIMEOUT_INTERACTION_MILLIS; 1393 final long startTimeMillis = SystemClock.uptimeMillis(); 1394 while (true) { 1395 try { 1396 Message sameProcessMessage = getSameProcessMessageAndClear(); 1397 if (sameProcessMessage != null) { 1398 sameProcessMessage.getTarget().handleMessage(sameProcessMessage); 1399 } 1400 1401 if (mInteractionId == interactionId) { 1402 return true; 1403 } 1404 if (mInteractionId > interactionId) { 1405 return false; 1406 } 1407 final long elapsedTimeMillis = SystemClock.uptimeMillis() - startTimeMillis; 1408 waitTimeMillis = TIMEOUT_INTERACTION_MILLIS - elapsedTimeMillis; 1409 if (waitTimeMillis <= 0) { 1410 return false; 1411 } 1412 mInstanceLock.wait(waitTimeMillis); 1413 } catch (InterruptedException ie) { 1414 /* ignore */ 1415 } 1416 } 1417 } 1418 1419 /** 1420 * Finalize an {@link AccessibilityNodeInfo} before passing it to the client. 1421 * 1422 * @param info The info. 1423 * @param connectionId The id of the connection to the system. 1424 * @param bypassCache Whether or not to bypass the cache. The node is added to the cache if 1425 * this value is {@code false} 1426 * @param packageNames The valid package names a node can come from. 1427 */ finalizeAndCacheAccessibilityNodeInfo(AccessibilityNodeInfo info, int connectionId, boolean bypassCache, String[] packageNames)1428 private void finalizeAndCacheAccessibilityNodeInfo(AccessibilityNodeInfo info, 1429 int connectionId, boolean bypassCache, String[] packageNames) { 1430 if (info != null) { 1431 info.setConnectionId(connectionId); 1432 // Empty array means any package name is Okay 1433 if (!ArrayUtils.isEmpty(packageNames)) { 1434 CharSequence packageName = info.getPackageName(); 1435 if (packageName == null 1436 || !ArrayUtils.contains(packageNames, packageName.toString())) { 1437 // If the node package not one of the valid ones, pick the top one - this 1438 // is one of the packages running in the introspected UID. 1439 info.setPackageName(packageNames[0]); 1440 } 1441 } 1442 info.setSealed(true); 1443 if (!bypassCache) { 1444 AccessibilityCache cache = getCache(connectionId); 1445 if (cache == null) { 1446 if (DEBUG) { 1447 Log.w(LOG_TAG, "Cache is null for connection id: " + connectionId); 1448 } 1449 return; 1450 } 1451 cache.add(info); 1452 } 1453 } 1454 } 1455 1456 /** 1457 * Finalize {@link AccessibilityNodeInfo}s before passing them to the client. 1458 * 1459 * @param infos The {@link AccessibilityNodeInfo}s. 1460 * @param connectionId The id of the connection to the system. 1461 * @param bypassCache Whether or not to bypass the cache. The nodes are added to the cache if 1462 * this value is {@code false} 1463 * @param packageNames The valid package names a node can come from. 1464 */ finalizeAndCacheAccessibilityNodeInfos(List<AccessibilityNodeInfo> infos, int connectionId, boolean bypassCache, String[] packageNames)1465 private void finalizeAndCacheAccessibilityNodeInfos(List<AccessibilityNodeInfo> infos, 1466 int connectionId, boolean bypassCache, String[] packageNames) { 1467 if (infos != null) { 1468 final int infosCount = infos.size(); 1469 for (int i = 0; i < infosCount; i++) { 1470 AccessibilityNodeInfo info = infos.get(i); 1471 finalizeAndCacheAccessibilityNodeInfo(info, connectionId, 1472 bypassCache, packageNames); 1473 } 1474 } 1475 } 1476 1477 /** 1478 * Gets the message stored if the interacted and interacting 1479 * threads are the same. 1480 * 1481 * @return The message. 1482 */ getSameProcessMessageAndClear()1483 private Message getSameProcessMessageAndClear() { 1484 synchronized (mInstanceLock) { 1485 Message result = mSameThreadMessage; 1486 mSameThreadMessage = null; 1487 return result; 1488 } 1489 } 1490 1491 /** 1492 * Checks whether the infos are a fully connected tree with no duplicates. 1493 * 1494 * @param infos The result list to check. 1495 */ checkFindAccessibilityNodeInfoResultIntegrity(List<AccessibilityNodeInfo> infos)1496 private void checkFindAccessibilityNodeInfoResultIntegrity(List<AccessibilityNodeInfo> infos) { 1497 if (infos.size() == 0) { 1498 return; 1499 } 1500 // Find the root node. 1501 AccessibilityNodeInfo root = infos.get(0); 1502 final int infoCount = infos.size(); 1503 for (int i = 1; i < infoCount; i++) { 1504 for (int j = i; j < infoCount; j++) { 1505 AccessibilityNodeInfo candidate = infos.get(j); 1506 if (root.getParentNodeId() == candidate.getSourceNodeId()) { 1507 root = candidate; 1508 break; 1509 } 1510 } 1511 } 1512 if (root == null) { 1513 Log.e(LOG_TAG, "No root."); 1514 } 1515 // Check for duplicates. 1516 HashSet<AccessibilityNodeInfo> seen = new HashSet<>(); 1517 Queue<AccessibilityNodeInfo> fringe = new ArrayDeque<>(); 1518 fringe.add(root); 1519 while (!fringe.isEmpty()) { 1520 AccessibilityNodeInfo current = fringe.poll(); 1521 if (!seen.add(current)) { 1522 Log.e(LOG_TAG, "Duplicate node."); 1523 return; 1524 } 1525 final int childCount = current.getChildCount(); 1526 for (int i = 0; i < childCount; i++) { 1527 final long childId = current.getChildId(i); 1528 for (int j = 0; j < infoCount; j++) { 1529 AccessibilityNodeInfo child = infos.get(j); 1530 if (child.getSourceNodeId() == childId) { 1531 fringe.add(child); 1532 } 1533 } 1534 } 1535 } 1536 final int disconnectedCount = infos.size() - seen.size(); 1537 if (disconnectedCount > 0) { 1538 Log.e(LOG_TAG, disconnectedCount + " Disconnected nodes."); 1539 } 1540 } 1541 1542 /** 1543 * Update scroll event timestamp of a given window. 1544 * 1545 * @param windowId The window id. 1546 * @param uptimeMillis Device uptime millis. 1547 */ updateScrollingWindow(int windowId, long uptimeMillis)1548 private void updateScrollingWindow(int windowId, long uptimeMillis) { 1549 synchronized (sScrollingWindows) { 1550 sScrollingWindows.put(windowId, uptimeMillis); 1551 } 1552 } 1553 1554 /** 1555 * Remove a window from the scrolling windows list. 1556 * 1557 * @param windowId The window id. 1558 */ deleteScrollingWindow(int windowId)1559 private void deleteScrollingWindow(int windowId) { 1560 synchronized (sScrollingWindows) { 1561 sScrollingWindows.delete(windowId); 1562 } 1563 } 1564 1565 /** 1566 * Whether or not the window is scrolling. 1567 * 1568 * @param windowId 1569 * @return true if it's scrolling. 1570 */ isWindowScrolling(int windowId)1571 private boolean isWindowScrolling(int windowId) { 1572 synchronized (sScrollingWindows) { 1573 final long latestScrollingTime = sScrollingWindows.get(windowId); 1574 if (latestScrollingTime == 0) { 1575 return false; 1576 } 1577 final long currentUptime = SystemClock.uptimeMillis(); 1578 if (currentUptime > (latestScrollingTime + DISABLE_PREFETCHING_FOR_SCROLLING_MILLIS)) { 1579 sScrollingWindows.delete(windowId); 1580 return false; 1581 } 1582 } 1583 return true; 1584 } 1585 shouldTraceClient()1586 private boolean shouldTraceClient() { 1587 return (mAccessibilityManager != null) 1588 && mAccessibilityManager.isA11yInteractionClientTraceEnabled(); 1589 } 1590 shouldTraceCallback()1591 private boolean shouldTraceCallback() { 1592 return (mAccessibilityManager != null) 1593 && mAccessibilityManager.isA11yInteractionConnectionCBTraceEnabled(); 1594 } 1595 logTrace( IAccessibilityServiceConnection connection, String method, String params, int callingUid, List<StackTraceElement> callStack, HashSet<String> ignoreSet, long logTypes)1596 private void logTrace( 1597 IAccessibilityServiceConnection connection, String method, String params, 1598 int callingUid, List<StackTraceElement> callStack, HashSet<String> ignoreSet, 1599 long logTypes) { 1600 try { 1601 Bundle b = new Bundle(); 1602 b.putSerializable(CALL_STACK, new ArrayList<StackTraceElement>(callStack)); 1603 if (ignoreSet != null) { 1604 b.putSerializable(IGNORE_CALL_STACK, ignoreSet); 1605 } 1606 connection.logTrace(SystemClock.elapsedRealtimeNanos(), 1607 LOG_TAG + "." + method, 1608 logTypes, params, Process.myPid(), Thread.currentThread().getId(), 1609 callingUid, b); 1610 } catch (RemoteException e) { 1611 Log.e(LOG_TAG, "Failed to log trace. " + e); 1612 } 1613 } 1614 logTraceCallback( IAccessibilityServiceConnection connection, String method, String params)1615 private void logTraceCallback( 1616 IAccessibilityServiceConnection connection, String method, String params) { 1617 logTrace(connection, method + " callback", params, mCallingUid, mCallStackOfCallback, 1618 new HashSet<String>(Arrays.asList("getStackTrace")), 1619 FLAGS_ACCESSIBILITY_INTERACTION_CONNECTION_CALLBACK); 1620 } 1621 logTraceClient( IAccessibilityServiceConnection connection, String method, String params)1622 private void logTraceClient( 1623 IAccessibilityServiceConnection connection, String method, String params) { 1624 logTrace(connection, method, params, Binder.getCallingUid(), 1625 Arrays.asList(Thread.currentThread().getStackTrace()), 1626 new HashSet<String>(Arrays.asList("getStackTrace", "logTraceClient")), 1627 FLAGS_ACCESSIBILITY_INTERACTION_CLIENT); 1628 } 1629 1630 /** Attaches an accessibility overlay to the specified window. */ attachAccessibilityOverlayToWindow( int connectionId, int accessibilityWindowId, SurfaceControl sc)1631 public void attachAccessibilityOverlayToWindow( 1632 int connectionId, int accessibilityWindowId, SurfaceControl sc) { 1633 synchronized (mInstanceLock) { 1634 try { 1635 IAccessibilityServiceConnection connection = getConnection(connectionId); 1636 if (connection == null) { 1637 Log.e(LOG_TAG, "Error while getting service connection."); 1638 return; 1639 } 1640 connection.attachAccessibilityOverlayToWindow(accessibilityWindowId, sc); 1641 } catch (RemoteException re) { 1642 re.rethrowFromSystemServer(); 1643 } 1644 } 1645 } 1646 } 1647