1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.voiceinteraction; 18 19 import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; 20 import static android.Manifest.permission.LOG_COMPAT_CHANGE; 21 import static android.Manifest.permission.READ_COMPAT_CHANGE_CONFIG; 22 import static android.Manifest.permission.RECORD_AUDIO; 23 import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN; 24 import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL; 25 import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT; 26 import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS; 27 import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN; 28 import static android.service.voice.HotwordDetectionService.KEY_INITIALIZATION_STATUS; 29 import static android.service.voice.HotwordDetectionServiceFailure.ERROR_CODE_COPY_AUDIO_DATA_FAILURE; 30 31 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; 32 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; 33 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; 34 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; 35 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; 36 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; 37 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; 38 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; 39 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT; 40 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION; 41 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; 42 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; 43 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; 44 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; 45 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; 46 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; 47 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; 48 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; 49 import static com.android.server.voiceinteraction.HotwordDetectionConnection.ENFORCE_HOTWORD_PHRASE_ID; 50 51 import android.annotation.NonNull; 52 import android.annotation.Nullable; 53 import android.annotation.RequiresPermission; 54 import android.app.AppOpsManager; 55 import android.app.compat.CompatChanges; 56 import android.attention.AttentionManagerInternal; 57 import android.content.Context; 58 import android.content.PermissionChecker; 59 import android.hardware.soundtrigger.SoundTrigger; 60 import android.media.AudioFormat; 61 import android.media.permission.Identity; 62 import android.media.permission.PermissionUtil; 63 import android.os.Binder; 64 import android.os.Bundle; 65 import android.os.IBinder; 66 import android.os.IRemoteCallback; 67 import android.os.ParcelFileDescriptor; 68 import android.os.PersistableBundle; 69 import android.os.RemoteException; 70 import android.os.SharedMemory; 71 import android.service.voice.HotwordDetectedResult; 72 import android.service.voice.HotwordDetectionService; 73 import android.service.voice.HotwordDetectionServiceFailure; 74 import android.service.voice.HotwordDetector; 75 import android.service.voice.HotwordRejectedResult; 76 import android.service.voice.IDspHotwordDetectionCallback; 77 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; 78 import android.service.voice.VisualQueryDetectionServiceFailure; 79 import android.text.TextUtils; 80 import android.util.Pair; 81 import android.util.Slog; 82 83 import com.android.internal.annotations.GuardedBy; 84 import com.android.internal.app.IHotwordRecognitionStatusCallback; 85 import com.android.internal.infra.AndroidFuture; 86 import com.android.server.LocalServices; 87 import com.android.server.policy.AppOpsPolicy; 88 import com.android.server.voiceinteraction.VoiceInteractionManagerServiceImpl.DetectorRemoteExceptionListener; 89 90 import java.io.Closeable; 91 import java.io.IOException; 92 import java.io.InputStream; 93 import java.io.OutputStream; 94 import java.io.PrintWriter; 95 import java.time.Duration; 96 import java.time.Instant; 97 import java.util.concurrent.Executor; 98 import java.util.concurrent.Executors; 99 import java.util.concurrent.ScheduledExecutorService; 100 import java.util.concurrent.TimeUnit; 101 import java.util.concurrent.TimeoutException; 102 import java.util.concurrent.atomic.AtomicBoolean; 103 104 /** 105 * A class that provides sandboxed detector to communicate with the {@link 106 * HotwordDetectionService} and {@link VisualQueryDetectionService}. 107 * 108 * Trusted hotword detectors such as {@link SoftwareHotwordDetector} and 109 * {@link AlwaysOnHotwordDetector} will leverage this class to communitcate with 110 * {@link HotwordDetectionService}; similarly, {@link VisualQueryDetector} will communicate with 111 * {@link VisualQueryDetectionService}. 112 * 113 * This class provides the methods to do initialization with the {@link HotwordDetectionService} and 114 * {@link VisualQueryDetectionService} handles external source detection for 115 * {@link HotwordDetectionService}. It also provides the methods to check if we can egress the data 116 * from the {@link HotwordDetectionService} and {@link VisualQueryDetectionService}. 117 * 118 * The subclass should override the {@link #informRestartProcessLocked()} to handle the trusted 119 * process restart. 120 */ 121 abstract class DetectorSession { 122 private static final String TAG = "DetectorSession"; 123 static final boolean DEBUG = false; 124 125 private static final String HOTWORD_DETECTION_OP_MESSAGE = 126 "Providing hotword detection result to VoiceInteractionService"; 127 128 // The error codes are used for onHotwordDetectionServiceFailure callback. 129 // Define these due to lines longer than 100 characters. 130 static final int ONDETECTED_GOT_SECURITY_EXCEPTION = 131 HotwordDetectionServiceFailure.ERROR_CODE_ON_DETECTED_SECURITY_EXCEPTION; 132 static final int ONDETECTED_STREAM_COPY_ERROR = 133 HotwordDetectionServiceFailure.ERROR_CODE_ON_DETECTED_STREAM_COPY_FAILURE; 134 135 // TODO: These constants need to be refined. 136 private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000; 137 private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; 138 private static final Duration MAX_UPDATE_TIMEOUT_DURATION = 139 Duration.ofMillis(MAX_UPDATE_TIMEOUT_MILLIS); 140 141 // Hotword metrics 142 private static final int METRICS_INIT_UNKNOWN_TIMEOUT = 143 HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; 144 private static final int METRICS_INIT_UNKNOWN_NO_VALUE = 145 HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; 146 private static final int METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE = 147 HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; 148 private static final int METRICS_INIT_CALLBACK_STATE_ERROR = 149 HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; 150 private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS = 151 HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; 152 153 static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = 154 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; 155 static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = 156 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; 157 static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = 158 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; 159 160 private static final int METRICS_EXTERNAL_SOURCE_DETECTED = 161 HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; 162 private static final int METRICS_EXTERNAL_SOURCE_REJECTED = 163 HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; 164 private static final int EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = 165 HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; 166 private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION = 167 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; 168 169 private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); 170 // TODO: This may need to be a Handler(looper) 171 final ScheduledExecutorService mScheduledExecutorService; 172 private final AppOpsManager mAppOpsManager; 173 final HotwordAudioStreamCopier mHotwordAudioStreamCopier; 174 final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); 175 final IHotwordRecognitionStatusCallback mCallback; 176 177 final Object mLock; 178 final int mVoiceInteractionServiceUid; 179 final Context mContext; 180 181 @Nullable AttentionManagerInternal mAttentionManagerInternal = null; 182 183 final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = 184 this::setProximityValue; 185 186 /** Identity used for attributing app ops when delivering data to the Interactor. */ 187 @Nullable 188 private final Identity mVoiceInteractorIdentity; 189 @GuardedBy("mLock") 190 ParcelFileDescriptor mCurrentAudioSink; 191 @GuardedBy("mLock") 192 @NonNull HotwordDetectionConnection.ServiceConnection mRemoteDetectionService; 193 boolean mDebugHotwordLogging = false; 194 @GuardedBy("mLock") 195 private double mProximityMeters = PROXIMITY_UNKNOWN; 196 @GuardedBy("mLock") 197 private boolean mInitialized = false; 198 @GuardedBy("mLock") 199 private boolean mDestroyed = false; 200 @GuardedBy("mLock") 201 boolean mPerformingExternalSourceHotwordDetection; 202 @NonNull final IBinder mToken; 203 204 @NonNull DetectorRemoteExceptionListener mRemoteExceptionListener; 205 DetectorSession( @onNull HotwordDetectionConnection.ServiceConnection remoteDetectionService, @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, @NonNull DetectorRemoteExceptionListener listener)206 DetectorSession( 207 @NonNull HotwordDetectionConnection.ServiceConnection remoteDetectionService, 208 @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, 209 @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, 210 Identity voiceInteractorIdentity, 211 @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, 212 @NonNull DetectorRemoteExceptionListener listener) { 213 mRemoteExceptionListener = listener; 214 mRemoteDetectionService = remoteDetectionService; 215 mLock = lock; 216 mContext = context; 217 mToken = token; 218 mCallback = callback; 219 mVoiceInteractionServiceUid = voiceInteractionServiceUid; 220 mVoiceInteractorIdentity = voiceInteractorIdentity; 221 mAppOpsManager = mContext.getSystemService(AppOpsManager.class); 222 if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 223 mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, 224 getDetectorType(), 225 mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, 226 mVoiceInteractorIdentity.attributionTag); 227 } else { 228 mHotwordAudioStreamCopier = null; 229 } 230 231 mScheduledExecutorService = scheduledExecutorService; 232 mDebugHotwordLogging = logging; 233 234 if (ENABLE_PROXIMITY_RESULT) { 235 mAttentionManagerInternal = LocalServices.getService(AttentionManagerInternal.class); 236 if (mAttentionManagerInternal != null) { 237 mAttentionManagerInternal.onStartProximityUpdates(mProximityCallbackInternal); 238 } 239 } 240 } 241 notifyOnDetectorRemoteException()242 void notifyOnDetectorRemoteException() { 243 Slog.d(TAG, "notifyOnDetectorRemoteException: mRemoteExceptionListener=" 244 + mRemoteExceptionListener); 245 if (mRemoteExceptionListener != null) { 246 mRemoteExceptionListener.onDetectorRemoteException(mToken, getDetectorType()); 247 } 248 } 249 250 @SuppressWarnings("GuardedBy") updateStateAfterProcessStartLocked(PersistableBundle options, SharedMemory sharedMemory)251 private void updateStateAfterProcessStartLocked(PersistableBundle options, 252 SharedMemory sharedMemory) { 253 if (DEBUG) { 254 Slog.d(TAG, "updateStateAfterProcessStartLocked"); 255 } 256 AndroidFuture<Void> voidFuture = mRemoteDetectionService.postAsync(service -> { 257 AndroidFuture<Void> future = new AndroidFuture<>(); 258 IRemoteCallback statusCallback = new IRemoteCallback.Stub() { 259 @Override 260 public void sendResult(Bundle bundle) throws RemoteException { 261 if (DEBUG) { 262 Slog.d(TAG, "updateState finish"); 263 } 264 future.complete(null); 265 if (mUpdateStateAfterStartFinished.getAndSet(true)) { 266 Slog.w(TAG, "call callback after timeout"); 267 if (getDetectorType() 268 != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 269 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 270 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, 271 mVoiceInteractionServiceUid); 272 } 273 return; 274 } 275 Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle); 276 int status = statusResultPair.first; 277 int initResultMetricsResult = statusResultPair.second; 278 try { 279 mCallback.onStatusReported(status); 280 if (getDetectorType() 281 != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 282 HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), 283 initResultMetricsResult, mVoiceInteractionServiceUid); 284 } 285 } catch (RemoteException e) { 286 Slog.w(TAG, "Failed to report initialization status: " + e); 287 if (getDetectorType() 288 != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 289 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 290 METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, 291 mVoiceInteractionServiceUid); 292 } 293 notifyOnDetectorRemoteException(); 294 } 295 } 296 }; 297 try { 298 service.updateState(options, sharedMemory, statusCallback); 299 if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 300 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 301 HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, 302 mVoiceInteractionServiceUid); 303 } 304 } catch (RemoteException e) { 305 // TODO: (b/181842909) Report an error to voice interactor 306 Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); 307 if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 308 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 309 HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, 310 mVoiceInteractionServiceUid); 311 } 312 } 313 return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); 314 }).whenComplete((res, err) -> { 315 if (err instanceof TimeoutException) { 316 Slog.w(TAG, "updateState timed out"); 317 if (mUpdateStateAfterStartFinished.getAndSet(true)) { 318 return; 319 } 320 try { 321 mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); 322 if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 323 HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), 324 METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid); 325 } 326 } catch (RemoteException e) { 327 Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); 328 if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 329 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 330 METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, 331 mVoiceInteractionServiceUid); 332 } 333 notifyOnDetectorRemoteException(); 334 } 335 } else if (err != null) { 336 Slog.w(TAG, "Failed to update state: " + err); 337 } 338 }); 339 if (voidFuture == null) { 340 Slog.w(TAG, "Failed to create AndroidFuture"); 341 } 342 } 343 getInitStatusAndMetricsResult(Bundle bundle)344 private static Pair<Integer, Integer> getInitStatusAndMetricsResult(Bundle bundle) { 345 if (bundle == null) { 346 return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, METRICS_INIT_UNKNOWN_NO_VALUE); 347 } 348 int status = bundle.getInt(KEY_INITIALIZATION_STATUS, INITIALIZATION_STATUS_UNKNOWN); 349 if (status > HotwordDetectionService.getMaxCustomInitializationStatus()) { 350 return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, 351 status == INITIALIZATION_STATUS_UNKNOWN 352 ? METRICS_INIT_UNKNOWN_NO_VALUE 353 : METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE); 354 } 355 // TODO: should guard against negative here 356 int metricsResult = status == INITIALIZATION_STATUS_SUCCESS 357 ? METRICS_INIT_CALLBACK_STATE_SUCCESS 358 : METRICS_INIT_CALLBACK_STATE_ERROR; 359 return new Pair<>(status, metricsResult); 360 } 361 362 @SuppressWarnings("GuardedBy") updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, Instant lastRestartInstant)363 void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, 364 Instant lastRestartInstant) { 365 if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 366 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 367 HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, 368 mVoiceInteractionServiceUid); 369 } 370 // Prevent doing the init late, so restart is handled equally to a clean process start. 371 // TODO(b/191742511): this logic needs a test 372 if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus( 373 MAX_UPDATE_TIMEOUT_DURATION).isBefore(lastRestartInstant)) { 374 Slog.v(TAG, "call updateStateAfterProcessStartLocked"); 375 updateStateAfterProcessStartLocked(options, sharedMemory); 376 } else { 377 mRemoteDetectionService.run( 378 service -> service.updateState(options, sharedMemory, /* callback= */ null)); 379 } 380 } 381 startListeningFromExternalSourceLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, IMicrophoneHotwordDetectionVoiceInteractionCallback callback)382 void startListeningFromExternalSourceLocked( 383 ParcelFileDescriptor audioStream, 384 AudioFormat audioFormat, 385 @Nullable PersistableBundle options, 386 IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { 387 if (DEBUG) { 388 Slog.d(TAG, "startListeningFromExternalSourceLocked"); 389 } 390 391 handleExternalSourceHotwordDetectionLocked( 392 audioStream, 393 audioFormat, 394 options, 395 callback); 396 } 397 398 @SuppressWarnings("GuardedBy") handleExternalSourceHotwordDetectionLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, IMicrophoneHotwordDetectionVoiceInteractionCallback callback)399 private void handleExternalSourceHotwordDetectionLocked( 400 ParcelFileDescriptor audioStream, 401 AudioFormat audioFormat, 402 @Nullable PersistableBundle options, 403 IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { 404 if (DEBUG) { 405 Slog.d(TAG, "#handleExternalSourceHotwordDetectionLocked"); 406 } 407 if (mPerformingExternalSourceHotwordDetection) { 408 Slog.i(TAG, "Hotword validation is already in progress for external source."); 409 return; 410 } 411 412 InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); 413 414 Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe(); 415 if (clientPipe == null) { 416 // TODO: Need to propagate as unknown error or something? 417 return; 418 } 419 ParcelFileDescriptor serviceAudioSink = clientPipe.second; 420 ParcelFileDescriptor serviceAudioSource = clientPipe.first; 421 422 mCurrentAudioSink = serviceAudioSink; 423 mPerformingExternalSourceHotwordDetection = true; 424 425 mAudioCopyExecutor.execute(() -> { 426 try (InputStream source = audioSource; 427 OutputStream fos = 428 new ParcelFileDescriptor.AutoCloseOutputStream(serviceAudioSink)) { 429 430 byte[] buffer = new byte[1024]; 431 while (true) { 432 int bytesRead = source.read(buffer, 0, 1024); 433 434 if (bytesRead < 0) { 435 Slog.i(TAG, "Reached end of stream for external hotword"); 436 break; 437 } 438 439 // TODO: First write to ring buffer to make sure we don't lose data if the next 440 // statement fails. 441 // ringBuffer.append(buffer, bytesRead); 442 fos.write(buffer, 0, bytesRead); 443 } 444 } catch (IOException e) { 445 Slog.w(TAG, "Failed supplying audio data to validator", e); 446 447 try { 448 callback.onHotwordDetectionServiceFailure( 449 new HotwordDetectionServiceFailure(ERROR_CODE_COPY_AUDIO_DATA_FAILURE, 450 "Copy audio data failure for external source detection.")); 451 } catch (RemoteException ex) { 452 Slog.w(TAG, "Failed to report onHotwordDetectionServiceFailure status: " + ex); 453 if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 454 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 455 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, 456 mVoiceInteractionServiceUid); 457 } 458 notifyOnDetectorRemoteException(); 459 } 460 } finally { 461 synchronized (mLock) { 462 mPerformingExternalSourceHotwordDetection = false; 463 closeExternalAudioStreamLocked("start external source"); 464 } 465 } 466 }); 467 468 // TODO: handle cancellations well 469 // TODO: what if we cancelled and started a new one? 470 mRemoteDetectionService.run( 471 service -> { 472 service.detectFromMicrophoneSource( 473 serviceAudioSource, 474 // TODO: consider making a proxy callback + copy of audio format 475 AUDIO_SOURCE_EXTERNAL, 476 audioFormat, 477 options, 478 new IDspHotwordDetectionCallback.Stub() { 479 @Override 480 public void onRejected(HotwordRejectedResult result) 481 throws RemoteException { 482 synchronized (mLock) { 483 mPerformingExternalSourceHotwordDetection = false; 484 HotwordMetricsLogger.writeDetectorEvent( 485 getDetectorType(), 486 METRICS_EXTERNAL_SOURCE_REJECTED, 487 mVoiceInteractionServiceUid); 488 mScheduledExecutorService.schedule( 489 () -> { 490 bestEffortClose(serviceAudioSink, audioSource); 491 }, 492 EXTERNAL_HOTWORD_CLEANUP_MILLIS, 493 TimeUnit.MILLISECONDS); 494 495 try { 496 callback.onRejected(result); 497 } catch (RemoteException e) { 498 notifyOnDetectorRemoteException(); 499 throw e; 500 } 501 if (result != null) { 502 Slog.i(TAG, "Egressed 'hotword rejected result' " 503 + "from hotword trusted process"); 504 if (mDebugHotwordLogging) { 505 Slog.i(TAG, "Egressed detected result: " + result); 506 } 507 } 508 } 509 } 510 511 @Override 512 public void onDetected(HotwordDetectedResult triggerResult) 513 throws RemoteException { 514 synchronized (mLock) { 515 mPerformingExternalSourceHotwordDetection = false; 516 HotwordMetricsLogger.writeDetectorEvent( 517 getDetectorType(), 518 METRICS_EXTERNAL_SOURCE_DETECTED, 519 mVoiceInteractionServiceUid); 520 mScheduledExecutorService.schedule( 521 () -> { 522 bestEffortClose(serviceAudioSink, audioSource); 523 }, 524 EXTERNAL_HOTWORD_CLEANUP_MILLIS, 525 TimeUnit.MILLISECONDS); 526 527 try { 528 enforcePermissionsForDataDelivery(); 529 } catch (SecurityException e) { 530 Slog.w(TAG, "Ignoring #onDetected due to a " 531 + "SecurityException", e); 532 HotwordMetricsLogger.writeDetectorEvent( 533 getDetectorType(), 534 EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, 535 mVoiceInteractionServiceUid); 536 try { 537 callback.onHotwordDetectionServiceFailure( 538 new HotwordDetectionServiceFailure( 539 ONDETECTED_GOT_SECURITY_EXCEPTION, 540 "Security exception occurs in " 541 + "#onDetected method")); 542 } catch (RemoteException e1) { 543 notifyOnDetectorRemoteException(); 544 throw e1; 545 } 546 return; 547 } 548 HotwordDetectedResult newResult; 549 try { 550 newResult = mHotwordAudioStreamCopier 551 .startCopyingAudioStreams(triggerResult); 552 } catch (IOException e) { 553 Slog.w(TAG, "Ignoring #onDetected due to a " 554 + "IOException", e); 555 // TODO: Write event 556 try { 557 callback.onHotwordDetectionServiceFailure( 558 new HotwordDetectionServiceFailure( 559 ONDETECTED_STREAM_COPY_ERROR, 560 "Copy audio stream failure.")); 561 } catch (RemoteException e1) { 562 notifyOnDetectorRemoteException(); 563 throw e1; 564 } 565 return; 566 } 567 try { 568 callback.onDetected(newResult, /* audioFormat= */ null, 569 /* audioStream= */ null); 570 } catch (RemoteException e) { 571 notifyOnDetectorRemoteException(); 572 throw e; 573 } 574 Slog.i(TAG, "Egressed " 575 + HotwordDetectedResult.getUsageSize(newResult) 576 + " bits from hotword trusted process"); 577 if (mDebugHotwordLogging) { 578 Slog.i(TAG, 579 "Egressed detected result: " + newResult); 580 } 581 } 582 } 583 }); 584 585 // A copy of this has been created and passed to the hotword validator 586 bestEffortClose(serviceAudioSource); 587 }); 588 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 589 HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION, 590 mVoiceInteractionServiceUid); 591 } 592 initialize(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)593 void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) { 594 synchronized (mLock) { 595 if (mInitialized || mDestroyed) { 596 return; 597 } 598 updateStateAfterProcessStartLocked(options, sharedMemory); 599 mInitialized = true; 600 } 601 } 602 603 @SuppressWarnings("GuardedBy") destroyLocked()604 void destroyLocked() { 605 mDestroyed = true; 606 mDebugHotwordLogging = false; 607 mRemoteDetectionService = null; 608 mRemoteExceptionListener = null; 609 if (mAttentionManagerInternal != null) { 610 mAttentionManagerInternal.onStopProximityUpdates(mProximityCallbackInternal); 611 } 612 } 613 setDebugHotwordLoggingLocked(boolean logging)614 void setDebugHotwordLoggingLocked(boolean logging) { 615 Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); 616 mDebugHotwordLogging = logging; 617 } 618 619 @SuppressWarnings("GuardedBy") updateRemoteSandboxedDetectionServiceLocked( @onNull HotwordDetectionConnection.ServiceConnection remoteDetectionService)620 void updateRemoteSandboxedDetectionServiceLocked( 621 @NonNull HotwordDetectionConnection.ServiceConnection remoteDetectionService) { 622 mRemoteDetectionService = remoteDetectionService; 623 } 624 reportErrorGetRemoteException()625 private void reportErrorGetRemoteException() { 626 if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 627 HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), 628 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, 629 mVoiceInteractionServiceUid); 630 } 631 notifyOnDetectorRemoteException(); 632 } 633 reportErrorLocked(@onNull HotwordDetectionServiceFailure hotwordDetectionServiceFailure)634 void reportErrorLocked(@NonNull HotwordDetectionServiceFailure hotwordDetectionServiceFailure) { 635 try { 636 mCallback.onHotwordDetectionServiceFailure(hotwordDetectionServiceFailure); 637 } catch (RemoteException e) { 638 Slog.w(TAG, "Failed to call onHotwordDetectionServiceFailure: " + e); 639 reportErrorGetRemoteException(); 640 } 641 } 642 reportErrorLocked( @onNull VisualQueryDetectionServiceFailure visualQueryDetectionServiceFailure)643 void reportErrorLocked( 644 @NonNull VisualQueryDetectionServiceFailure visualQueryDetectionServiceFailure) { 645 try { 646 mCallback.onVisualQueryDetectionServiceFailure(visualQueryDetectionServiceFailure); 647 } catch (RemoteException e) { 648 Slog.w(TAG, "Failed to call onVisualQueryDetectionServiceFailure: " + e); 649 reportErrorGetRemoteException(); 650 } 651 } 652 reportErrorLocked(@onNull String errorMessage)653 void reportErrorLocked(@NonNull String errorMessage) { 654 try { 655 mCallback.onUnknownFailure(errorMessage); 656 } catch (RemoteException e) { 657 Slog.w(TAG, "Failed to call onUnknownFailure: " + e); 658 reportErrorGetRemoteException(); 659 } 660 } 661 662 /** 663 * Called when the trusted process is restarted. 664 */ informRestartProcessLocked()665 abstract void informRestartProcessLocked(); 666 isSameCallback(@ullable IHotwordRecognitionStatusCallback callback)667 boolean isSameCallback(@Nullable IHotwordRecognitionStatusCallback callback) { 668 synchronized (mLock) { 669 if (callback == null) { 670 return false; 671 } 672 return mCallback.asBinder().equals(callback.asBinder()); 673 } 674 } 675 isSameToken(@onNull IBinder token)676 boolean isSameToken(@NonNull IBinder token) { 677 synchronized (mLock) { 678 if (token == null) { 679 return false; 680 } 681 return mToken == token; 682 } 683 } 684 isDestroyed()685 boolean isDestroyed() { 686 synchronized (mLock) { 687 return mDestroyed; 688 } 689 } 690 createPipe()691 private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() { 692 ParcelFileDescriptor[] fileDescriptors; 693 try { 694 fileDescriptors = ParcelFileDescriptor.createPipe(); 695 } catch (IOException e) { 696 Slog.e(TAG, "Failed to create audio stream pipe", e); 697 return null; 698 } 699 700 return Pair.create(fileDescriptors[0], fileDescriptors[1]); 701 } 702 saveProximityValueToBundle(HotwordDetectedResult result)703 void saveProximityValueToBundle(HotwordDetectedResult result) { 704 synchronized (mLock) { 705 if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { 706 result.setProximity(mProximityMeters); 707 } 708 } 709 } 710 setProximityValue(double proximityMeters)711 private void setProximityValue(double proximityMeters) { 712 synchronized (mLock) { 713 mProximityMeters = proximityMeters; 714 } 715 } 716 717 @SuppressWarnings("GuardedBy") closeExternalAudioStreamLocked(String reason)718 void closeExternalAudioStreamLocked(String reason) { 719 if (mCurrentAudioSink != null) { 720 Slog.i(TAG, "Closing external audio stream to hotword detector: " + reason); 721 bestEffortClose(mCurrentAudioSink); 722 mCurrentAudioSink = null; 723 } 724 } 725 bestEffortClose(Closeable... closeables)726 private static void bestEffortClose(Closeable... closeables) { 727 for (Closeable closeable : closeables) { 728 bestEffortClose(closeable); 729 } 730 } 731 bestEffortClose(Closeable closeable)732 private static void bestEffortClose(Closeable closeable) { 733 try { 734 closeable.close(); 735 } catch (IOException e) { 736 if (DEBUG) { 737 Slog.w(TAG, "Failed closing", e); 738 } 739 } 740 } 741 742 // TODO: Share this code with SoundTriggerMiddlewarePermission. enforcePermissionsForDataDelivery()743 void enforcePermissionsForDataDelivery() { 744 Binder.withCleanCallingIdentity(() -> { 745 synchronized (mLock) { 746 if (AppOpsPolicy.isHotwordDetectionServiceRequired(mContext.getPackageManager())) { 747 int result = PermissionChecker.checkPermissionForPreflight( 748 mContext, RECORD_AUDIO, /* pid */ -1, mVoiceInteractorIdentity.uid, 749 mVoiceInteractorIdentity.packageName); 750 if (result != PermissionChecker.PERMISSION_GRANTED) { 751 throw new SecurityException( 752 "Failed to obtain permission RECORD_AUDIO for identity " 753 + mVoiceInteractorIdentity); 754 } 755 int hotwordOp = AppOpsManager.strOpToOp( 756 AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD); 757 mAppOpsManager.noteOpNoThrow(hotwordOp, 758 mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, 759 mVoiceInteractorIdentity.attributionTag, HOTWORD_DETECTION_OP_MESSAGE); 760 } else { 761 enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, 762 RECORD_AUDIO, HOTWORD_DETECTION_OP_MESSAGE); 763 } 764 enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, 765 CAPTURE_AUDIO_HOTWORD, HOTWORD_DETECTION_OP_MESSAGE); 766 } 767 }); 768 } 769 770 /** 771 * Throws a {@link SecurityException} if the given identity has no permission to receive data. 772 * 773 * @param context A {@link Context}, used for permission checks. 774 * @param identity The identity to check. 775 * @param permission The identifier of the permission we want to check. 776 * @param reason The reason why we're requesting the permission, for auditing purposes. 777 */ enforcePermissionForDataDelivery(@onNull Context context, @NonNull Identity identity, @NonNull String permission, @NonNull String reason)778 private static void enforcePermissionForDataDelivery(@NonNull Context context, 779 @NonNull Identity identity, @NonNull String permission, @NonNull String reason) { 780 final int status = PermissionUtil.checkPermissionForDataDelivery(context, identity, 781 permission, reason); 782 if (status != PermissionChecker.PERMISSION_GRANTED) { 783 throw new SecurityException( 784 TextUtils.formatSimple("Failed to obtain permission %s for identity %s", 785 permission, 786 identity)); 787 } 788 } 789 790 @RequiresPermission(allOf = {READ_COMPAT_CHANGE_CONFIG, LOG_COMPAT_CHANGE}) enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, SoundTrigger.KeyphraseRecognitionEvent recognitionEvent)791 void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, 792 SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { 793 if (!CompatChanges.isChangeEnabled(ENFORCE_HOTWORD_PHRASE_ID, 794 mVoiceInteractionServiceUid)) { 795 return; 796 } 797 // verify the phrase ID in HotwordDetectedResult is not exposing extra phrases 798 // the DSP did not detect 799 for (SoundTrigger.KeyphraseRecognitionExtra keyphrase : recognitionEvent.keyphraseExtras) { 800 if (keyphrase.getKeyphraseId() == result.getHotwordPhraseId()) { 801 return; 802 } 803 } 804 throw new SecurityException("Ignoring #onDetected due to trusted service " 805 + "sharing a keyphrase ID which the DSP did not detect"); 806 } 807 getDetectorType()808 private int getDetectorType() { 809 if (this instanceof DspTrustedHotwordDetectorSession) { 810 return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP; 811 } else if (this instanceof SoftwareTrustedHotwordDetectorSession) { 812 return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE; 813 } else if (this instanceof VisualQueryDetectorSession) { 814 return HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR; 815 } 816 Slog.v(TAG, "Unexpected detector type"); 817 return -1; 818 } 819 820 @SuppressWarnings("GuardedBy") dumpLocked(String prefix, PrintWriter pw)821 public void dumpLocked(String prefix, PrintWriter pw) { 822 pw.print(prefix); pw.print("mCallback="); pw.println(mCallback); 823 pw.print(prefix); pw.print("mUpdateStateAfterStartFinished="); 824 pw.println(mUpdateStateAfterStartFinished); 825 pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized); 826 pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed); 827 pw.print(prefix); pw.print("DetectorType="); 828 pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); 829 pw.print(prefix); pw.print("mPerformingExternalSourceHotwordDetection="); 830 pw.println(mPerformingExternalSourceHotwordDetection); 831 } 832 } 833