1 /** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; 20 import static android.Manifest.permission.RECORD_AUDIO; 21 22 import android.annotation.IntDef; 23 import android.annotation.NonNull; 24 import android.annotation.Nullable; 25 import android.annotation.RequiresPermission; 26 import android.annotation.SystemApi; 27 import android.annotation.TestApi; 28 import android.app.ActivityThread; 29 import android.compat.annotation.UnsupportedAppUsage; 30 import android.content.Context; 31 import android.content.Intent; 32 import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; 33 import android.hardware.soundtrigger.KeyphraseMetadata; 34 import android.hardware.soundtrigger.SoundTrigger; 35 import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; 36 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 37 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 38 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 39 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 40 import android.media.AudioFormat; 41 import android.media.permission.Identity; 42 import android.os.AsyncTask; 43 import android.os.Binder; 44 import android.os.Build; 45 import android.os.Handler; 46 import android.os.IBinder; 47 import android.os.Message; 48 import android.os.ParcelFileDescriptor; 49 import android.os.PersistableBundle; 50 import android.os.RemoteException; 51 import android.os.SharedMemory; 52 import android.util.Log; 53 import android.util.Slog; 54 55 import com.android.internal.app.IHotwordRecognitionStatusCallback; 56 import com.android.internal.app.IVoiceInteractionManagerService; 57 import com.android.internal.app.IVoiceInteractionSoundTriggerSession; 58 59 import java.io.PrintWriter; 60 import java.lang.annotation.Retention; 61 import java.lang.annotation.RetentionPolicy; 62 import java.util.Locale; 63 64 /** 65 * A class that lets a VoiceInteractionService implementation interact with 66 * always-on keyphrase detection APIs. 67 * 68 * @hide 69 * TODO(b/168605867): Once Metalava supports expressing a removed public, but current system API, 70 * mark and track it as such. 71 */ 72 @SystemApi 73 public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { 74 //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----// 75 /** 76 * Indicates that this hotword detector is no longer valid for any recognition 77 * and should not be used anymore. 78 */ 79 private static final int STATE_INVALID = -3; 80 81 /** 82 * Indicates that recognition for the given keyphrase is not available on the system 83 * because of the hardware configuration. 84 * No further interaction should be performed with the detector that returns this availability. 85 */ 86 public static final int STATE_HARDWARE_UNAVAILABLE = -2; 87 88 /** 89 * Indicates that recognition for the given keyphrase is not supported. 90 * No further interaction should be performed with the detector that returns this availability. 91 * 92 * @deprecated This is no longer a valid state. Enrollment can occur outside of 93 * {@link KeyphraseEnrollmentInfo} through another privileged application. We can no longer 94 * determine ahead of time if the keyphrase and locale are unsupported by the system. 95 */ 96 @Deprecated 97 public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; 98 99 /** 100 * Indicates that the given keyphrase is not enrolled. 101 * The caller may choose to begin an enrollment flow for the keyphrase. 102 */ 103 public static final int STATE_KEYPHRASE_UNENROLLED = 1; 104 105 /** 106 * Indicates that the given keyphrase is currently enrolled and it's possible to start 107 * recognition for it. 108 */ 109 public static final int STATE_KEYPHRASE_ENROLLED = 2; 110 111 /** 112 * Indicates that the availability state of the active keyphrase can't be known due to an error. 113 * 114 * <p>NOTE: No further interaction should be performed with the detector that returns this 115 * state, it would be better to create {@link AlwaysOnHotwordDetector} again. 116 */ 117 public static final int STATE_ERROR = 3; 118 119 /** 120 * Indicates that the detector isn't ready currently. 121 */ 122 private static final int STATE_NOT_READY = 0; 123 124 //-- Flags for startRecognition ----// 125 /** @hide */ 126 @Retention(RetentionPolicy.SOURCE) 127 @IntDef(flag = true, prefix = { "RECOGNITION_FLAG_" }, value = { 128 RECOGNITION_FLAG_NONE, 129 RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO, 130 RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS, 131 RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION, 132 RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION, 133 RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER, 134 }) 135 public @interface RecognitionFlags {} 136 137 /** 138 * Empty flag for {@link #startRecognition(int)}. 139 * 140 * @hide 141 */ 142 public static final int RECOGNITION_FLAG_NONE = 0; 143 144 /** 145 * Recognition flag for {@link #startRecognition(int)} that indicates 146 * whether the trigger audio for hotword needs to be captured. 147 */ 148 public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; 149 150 /** 151 * Recognition flag for {@link #startRecognition(int)} that indicates 152 * whether the recognition should keep going on even after the keyphrase triggers. 153 * If this flag is specified, it's possible to get multiple triggers after a 154 * call to {@link #startRecognition(int)} if the user speaks the keyphrase multiple times. 155 * When this isn't specified, the default behavior is to stop recognition once the 156 * keyphrase is spoken, till the caller starts recognition again. 157 */ 158 public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2; 159 160 /** 161 * Audio capabilities flag for {@link #startRecognition(int)} that indicates 162 * if the underlying recognition should use AEC. 163 * This capability may or may not be supported by the system, and support can be queried 164 * by calling {@link #getSupportedAudioCapabilities()}. The corresponding capabilities field for 165 * this flag is {@link #AUDIO_CAPABILITY_ECHO_CANCELLATION}. If this flag is passed without the 166 * audio capability supported, there will be no audio effect applied. 167 */ 168 public static final int RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION = 0x4; 169 170 /** 171 * Audio capabilities flag for {@link #startRecognition(int)} that indicates 172 * if the underlying recognition should use noise suppression. 173 * This capability may or may not be supported by the system, and support can be queried 174 * by calling {@link #getSupportedAudioCapabilities()}. The corresponding capabilities field for 175 * this flag is {@link #AUDIO_CAPABILITY_NOISE_SUPPRESSION}. If this flag is passed without the 176 * audio capability supported, there will be no audio effect applied. 177 */ 178 public static final int RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION = 0x8; 179 180 /** 181 * Recognition flag for {@link #startRecognition(int)} that indicates whether the recognition 182 * should continue after battery saver mode is enabled. 183 * When this flag is specified, the caller will be checked for 184 * {@link android.Manifest.permission#SOUND_TRIGGER_RUN_IN_BATTERY_SAVER} permission granted. 185 */ 186 public static final int RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER = 0x10; 187 188 //---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----// 189 // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags. 190 191 /** @hide */ 192 @Retention(RetentionPolicy.SOURCE) 193 @IntDef(flag = true, prefix = { "RECOGNITION_MODE_" }, value = { 194 RECOGNITION_MODE_VOICE_TRIGGER, 195 RECOGNITION_MODE_USER_IDENTIFICATION, 196 }) 197 public @interface RecognitionModes {} 198 199 /** 200 * Simple recognition of the key phrase. 201 * Returned by {@link #getSupportedRecognitionModes()} 202 */ 203 public static final int RECOGNITION_MODE_VOICE_TRIGGER 204 = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; 205 206 /** 207 * User identification performed with the keyphrase recognition. 208 * Returned by {@link #getSupportedRecognitionModes()} 209 */ 210 public static final int RECOGNITION_MODE_USER_IDENTIFICATION 211 = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; 212 213 //-- Audio capabilities. Values in returned bit field for getSupportedAudioCapabilities() --// 214 215 /** @hide */ 216 @Retention(RetentionPolicy.SOURCE) 217 @IntDef(flag = true, prefix = { "AUDIO_CAPABILITY_" }, value = { 218 AUDIO_CAPABILITY_ECHO_CANCELLATION, 219 AUDIO_CAPABILITY_NOISE_SUPPRESSION, 220 }) 221 public @interface AudioCapabilities {} 222 223 /** 224 * If set the underlying module supports AEC. 225 * Returned by {@link #getSupportedAudioCapabilities()} 226 */ 227 public static final int AUDIO_CAPABILITY_ECHO_CANCELLATION = 228 SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_ECHO_CANCELLATION; 229 230 /** 231 * If set, the underlying module supports noise suppression. 232 * Returned by {@link #getSupportedAudioCapabilities()} 233 */ 234 public static final int AUDIO_CAPABILITY_NOISE_SUPPRESSION = 235 SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_NOISE_SUPPRESSION; 236 237 /** @hide */ 238 @Retention(RetentionPolicy.SOURCE) 239 @IntDef(flag = true, prefix = { "MODEL_PARAM_" }, value = { 240 MODEL_PARAM_THRESHOLD_FACTOR, 241 }) 242 public @interface ModelParams {} 243 244 /** 245 * Controls the sensitivity threshold adjustment factor for a given model. 246 * Negative value corresponds to less sensitive model (high threshold) and 247 * a positive value corresponds to a more sensitive model (low threshold). 248 * Default value is 0. 249 */ 250 public static final int MODEL_PARAM_THRESHOLD_FACTOR = 251 android.hardware.soundtrigger.ModelParams.THRESHOLD_FACTOR; 252 253 static final String TAG = "AlwaysOnHotwordDetector"; 254 static final boolean DBG = false; 255 256 private static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 257 private static final int STATUS_OK = SoundTrigger.STATUS_OK; 258 259 private static final int MSG_AVAILABILITY_CHANGED = 1; 260 private static final int MSG_HOTWORD_DETECTED = 2; 261 private static final int MSG_DETECTION_ERROR = 3; 262 private static final int MSG_DETECTION_PAUSE = 4; 263 private static final int MSG_DETECTION_RESUME = 5; 264 private static final int MSG_HOTWORD_REJECTED = 6; 265 private static final int MSG_HOTWORD_STATUS_REPORTED = 7; 266 private static final int MSG_PROCESS_RESTARTED = 8; 267 268 private final String mText; 269 private final Locale mLocale; 270 /** 271 * The metadata of the Keyphrase, derived from the enrollment application. 272 * This may be null if this keyphrase isn't supported by the enrollment application. 273 */ 274 @Nullable 275 private KeyphraseMetadata mKeyphraseMetadata; 276 private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; 277 private final IVoiceInteractionManagerService mModelManagementService; 278 private final IVoiceInteractionSoundTriggerSession mSoundTriggerSession; 279 private final SoundTriggerListener mInternalCallback; 280 private final Callback mExternalCallback; 281 private final Handler mHandler; 282 private final IBinder mBinder = new Binder(); 283 private final int mTargetSdkVersion; 284 private final boolean mSupportHotwordDetectionService; 285 286 private int mAvailability = STATE_NOT_READY; 287 288 /** 289 * A ModelParamRange is a representation of supported parameter range for a 290 * given loaded model. 291 */ 292 public static final class ModelParamRange { 293 private final SoundTrigger.ModelParamRange mModelParamRange; 294 295 /** @hide */ ModelParamRange(SoundTrigger.ModelParamRange modelParamRange)296 ModelParamRange(SoundTrigger.ModelParamRange modelParamRange) { 297 mModelParamRange = modelParamRange; 298 } 299 300 /** 301 * Get the beginning of the param range 302 * 303 * @return The inclusive start of the supported range. 304 */ getStart()305 public int getStart() { 306 return mModelParamRange.getStart(); 307 } 308 309 /** 310 * Get the end of the param range 311 * 312 * @return The inclusive end of the supported range. 313 */ getEnd()314 public int getEnd() { 315 return mModelParamRange.getEnd(); 316 } 317 318 @Override 319 @NonNull toString()320 public String toString() { 321 return mModelParamRange.toString(); 322 } 323 324 @Override equals(@ullable Object obj)325 public boolean equals(@Nullable Object obj) { 326 return mModelParamRange.equals(obj); 327 } 328 329 @Override hashCode()330 public int hashCode() { 331 return mModelParamRange.hashCode(); 332 } 333 } 334 335 /** 336 * Additional payload for {@link Callback#onDetected}. 337 */ 338 public static class EventPayload { 339 private final boolean mTriggerAvailable; 340 // Indicates if {@code captureSession} can be used to continue capturing more audio 341 // from the DSP hardware. 342 private final boolean mCaptureAvailable; 343 // The session to use when attempting to capture more audio from the DSP hardware. 344 private final int mCaptureSession; 345 private final AudioFormat mAudioFormat; 346 // Raw data associated with the event. 347 // This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true. 348 private final byte[] mData; 349 private final HotwordDetectedResult mHotwordDetectedResult; 350 private final ParcelFileDescriptor mAudioStream; 351 EventPayload(boolean triggerAvailable, boolean captureAvailable, AudioFormat audioFormat, int captureSession, byte[] data)352 EventPayload(boolean triggerAvailable, boolean captureAvailable, 353 AudioFormat audioFormat, int captureSession, byte[] data) { 354 this(triggerAvailable, captureAvailable, audioFormat, captureSession, data, null, 355 null); 356 } 357 EventPayload(boolean triggerAvailable, boolean captureAvailable, AudioFormat audioFormat, int captureSession, byte[] data, HotwordDetectedResult hotwordDetectedResult)358 EventPayload(boolean triggerAvailable, boolean captureAvailable, 359 AudioFormat audioFormat, int captureSession, byte[] data, 360 HotwordDetectedResult hotwordDetectedResult) { 361 this(triggerAvailable, captureAvailable, audioFormat, captureSession, data, 362 hotwordDetectedResult, null); 363 } 364 EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult)365 EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult) { 366 this(false, false, audioFormat, -1, null, hotwordDetectedResult, null); 367 } 368 EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult, ParcelFileDescriptor audioStream)369 EventPayload(AudioFormat audioFormat, 370 HotwordDetectedResult hotwordDetectedResult, 371 ParcelFileDescriptor audioStream) { 372 this(false, false, audioFormat, -1, null, hotwordDetectedResult, audioStream); 373 } 374 EventPayload(boolean triggerAvailable, boolean captureAvailable, AudioFormat audioFormat, int captureSession, byte[] data, HotwordDetectedResult hotwordDetectedResult, ParcelFileDescriptor audioStream)375 private EventPayload(boolean triggerAvailable, boolean captureAvailable, 376 AudioFormat audioFormat, int captureSession, byte[] data, 377 HotwordDetectedResult hotwordDetectedResult, ParcelFileDescriptor audioStream) { 378 mTriggerAvailable = triggerAvailable; 379 mCaptureAvailable = captureAvailable; 380 mCaptureSession = captureSession; 381 mAudioFormat = audioFormat; 382 mData = data; 383 mHotwordDetectedResult = hotwordDetectedResult; 384 mAudioStream = audioStream; 385 } 386 387 /** 388 * Gets the format of the audio obtained using {@link #getTriggerAudio()}. 389 * May be null if there's no audio present. 390 */ 391 @Nullable getCaptureAudioFormat()392 public AudioFormat getCaptureAudioFormat() { 393 return mAudioFormat; 394 } 395 396 /** 397 * Gets the raw audio that triggered the keyphrase. 398 * This may be null if the trigger audio isn't available. 399 * If non-null, the format of the audio can be obtained by calling 400 * {@link #getCaptureAudioFormat()}. 401 * 402 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 403 */ 404 @Nullable getTriggerAudio()405 public byte[] getTriggerAudio() { 406 if (mTriggerAvailable) { 407 return mData; 408 } else { 409 return null; 410 } 411 } 412 413 /** 414 * Gets the session ID to start a capture from the DSP. 415 * This may be null if streaming capture isn't possible. 416 * If non-null, the format of the audio that can be captured can be 417 * obtained using {@link #getCaptureAudioFormat()}. 418 * 419 * TODO: Candidate for Public API when the API to start capture with a session ID 420 * is made public. 421 * 422 * TODO: Add this to {@link #getCaptureAudioFormat()}: 423 * "Gets the format of the audio obtained using {@link #getTriggerAudio()} 424 * or {@link #getCaptureSession()}. May be null if no audio can be obtained 425 * for either the trigger or a streaming session." 426 * 427 * TODO: Should this return a known invalid value instead? 428 * 429 * @hide 430 */ 431 @Nullable 432 @UnsupportedAppUsage getCaptureSession()433 public Integer getCaptureSession() { 434 if (mCaptureAvailable) { 435 return mCaptureSession; 436 } else { 437 return null; 438 } 439 } 440 441 /** 442 * Returns {@link HotwordDetectedResult} associated with the hotword event, passed from 443 * {@link HotwordDetectionService}. 444 */ 445 @Nullable getHotwordDetectedResult()446 public HotwordDetectedResult getHotwordDetectedResult() { 447 return mHotwordDetectedResult; 448 } 449 450 /** 451 * Returns a stream with bytes corresponding to the open audio stream with hotword data. 452 * 453 * <p>This data represents an audio stream in the format returned by 454 * {@link #getCaptureAudioFormat}. 455 * 456 * <p>Clients are expected to start consuming the stream within 1 second of receiving the 457 * event. 458 * 459 * <p>When this method returns a non-null, clients must close this stream when it's no 460 * longer needed. Failing to do so will result in microphone being open for longer periods 461 * of time, and app being attributed for microphone usage. 462 */ 463 @Nullable getAudioStream()464 public ParcelFileDescriptor getAudioStream() { 465 return mAudioStream; 466 } 467 } 468 469 /** 470 * Callbacks for always-on hotword detection. 471 */ 472 public abstract static class Callback implements HotwordDetector.Callback { 473 474 /** 475 * Updates the availability state of the active keyphrase and locale on every keyphrase 476 * sound model change. 477 * 478 * <p>This API is called whenever there's a possibility that the keyphrase associated 479 * with this detector has been updated. It is not guaranteed that there is in fact any 480 * change, as it may be called for other reasons.</p> 481 * 482 * <p>This API is also guaranteed to be called right after an AlwaysOnHotwordDetector 483 * instance is created to updated the current availability state.</p> 484 * 485 * <p>Availability implies the current enrollment state of the given keyphrase. If the 486 * hardware on this system is not capable of listening for the given keyphrase, 487 * {@link AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE} will be returned. 488 * 489 * @see AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE 490 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNENROLLED 491 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_ENROLLED 492 * @see AlwaysOnHotwordDetector#STATE_ERROR 493 */ onAvailabilityChanged(int status)494 public abstract void onAvailabilityChanged(int status); 495 496 /** 497 * Called when the keyphrase is spoken. 498 * This implicitly stops listening for the keyphrase once it's detected. 499 * Clients should start a recognition again once they are done handling this 500 * detection. 501 * 502 * @param eventPayload Payload data for the detection event. 503 * This may contain the trigger audio, if requested when calling 504 * {@link AlwaysOnHotwordDetector#startRecognition(int)}. 505 */ onDetected(@onNull EventPayload eventPayload)506 public abstract void onDetected(@NonNull EventPayload eventPayload); 507 508 /** 509 * Called when the detection fails due to an error. 510 */ onError()511 public abstract void onError(); 512 513 /** 514 * Called when the recognition is paused temporarily for some reason. 515 * This is an informational callback, and the clients shouldn't be doing anything here 516 * except showing an indication on their UI if they have to. 517 */ onRecognitionPaused()518 public abstract void onRecognitionPaused(); 519 520 /** 521 * Called when the recognition is resumed after it was temporarily paused. 522 * This is an informational callback, and the clients shouldn't be doing anything here 523 * except showing an indication on their UI if they have to. 524 */ onRecognitionResumed()525 public abstract void onRecognitionResumed(); 526 527 /** 528 * Called when the {@link HotwordDetectionService second stage detection} did not detect the 529 * keyphrase. 530 * 531 * @param result Info about the second stage detection result, provided by the 532 * {@link HotwordDetectionService}. 533 */ onRejected(@onNull HotwordRejectedResult result)534 public void onRejected(@NonNull HotwordRejectedResult result) { 535 } 536 537 /** 538 * Called when the {@link HotwordDetectionService} is created by the system and given a 539 * short amount of time to report it's initialization state. 540 * 541 * @param status Info about initialization state of {@link HotwordDetectionService}; the 542 * allowed values are {@link HotwordDetectionService#INITIALIZATION_STATUS_SUCCESS}, 543 * 1<->{@link HotwordDetectionService#getMaxCustomInitializationStatus()}, 544 * {@link HotwordDetectionService#INITIALIZATION_STATUS_UNKNOWN}. 545 */ onHotwordDetectionServiceInitialized(int status)546 public void onHotwordDetectionServiceInitialized(int status) { 547 } 548 549 /** 550 * Called with the {@link HotwordDetectionService} is restarted. 551 * 552 * Clients are expected to call {@link HotwordDetector#updateState} to share the state with 553 * the newly created service. 554 */ onHotwordDetectionServiceRestarted()555 public void onHotwordDetectionServiceRestarted() { 556 } 557 } 558 559 /** 560 * @param text The keyphrase text to get the detector for. 561 * @param locale The java locale for the detector. 562 * @param callback A non-null Callback for receiving the recognition events. 563 * @param modelManagementService A service that allows management of sound models. 564 * @param targetSdkVersion The target SDK version. 565 * @param supportHotwordDetectionService {@code true} if hotword detection service should be 566 * triggered, otherwise {@code false}. 567 * @param options Application configuration data provided by the 568 * {@link VoiceInteractionService}. PersistableBundle does not allow any remotable objects or 569 * other contents that can be used to communicate with other processes. 570 * @param sharedMemory The unrestricted data blob provided by the 571 * {@link VoiceInteractionService}. Use this to provide the hotword models data or other 572 * such data to the trusted process. 573 * 574 * @hide 575 */ AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, IVoiceInteractionManagerService modelManagementService, int targetSdkVersion, boolean supportHotwordDetectionService, @Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory)576 public AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, 577 KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, 578 IVoiceInteractionManagerService modelManagementService, int targetSdkVersion, 579 boolean supportHotwordDetectionService, @Nullable PersistableBundle options, 580 @Nullable SharedMemory sharedMemory) { 581 super(modelManagementService, callback); 582 583 mHandler = new MyHandler(); 584 mText = text; 585 mLocale = locale; 586 mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; 587 mExternalCallback = callback; 588 mInternalCallback = new SoundTriggerListener(mHandler); 589 mModelManagementService = modelManagementService; 590 mTargetSdkVersion = targetSdkVersion; 591 mSupportHotwordDetectionService = supportHotwordDetectionService; 592 if (mSupportHotwordDetectionService) { 593 updateStateLocked(options, sharedMemory, mInternalCallback); 594 } 595 try { 596 Identity identity = new Identity(); 597 identity.packageName = ActivityThread.currentOpPackageName(); 598 mSoundTriggerSession = mModelManagementService.createSoundTriggerSessionAsOriginator( 599 identity, mBinder); 600 } catch (RemoteException e) { 601 throw e.rethrowAsRuntimeException(); 602 } 603 new RefreshAvailabiltyTask().execute(); 604 } 605 606 /** 607 * {@inheritDoc} 608 * 609 * @throws IllegalStateException if this AlwaysOnHotwordDetector wasn't specified to use a 610 * {@link HotwordDetectionService} when it was created. In addition, if this 611 * AlwaysOnHotwordDetector is in an invalid or error state. 612 */ 613 @Override updateState(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)614 public final void updateState(@Nullable PersistableBundle options, 615 @Nullable SharedMemory sharedMemory) { 616 synchronized (mLock) { 617 if (!mSupportHotwordDetectionService) { 618 throw new IllegalStateException( 619 "updateState called, but it doesn't support hotword detection service"); 620 } 621 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 622 throw new IllegalStateException( 623 "updateState called on an invalid detector or error state"); 624 } 625 } 626 627 super.updateState(options, sharedMemory); 628 } 629 630 /** 631 * Test API to simulate to trigger hardware recognition event for test. 632 * 633 * @hide 634 */ 635 @TestApi 636 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) triggerHardwareRecognitionEventForTest(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data)637 public void triggerHardwareRecognitionEventForTest(int status, int soundModelHandle, 638 boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, 639 boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data) { 640 Log.d(TAG, "triggerHardwareRecognitionEventForTest()"); 641 synchronized (mLock) { 642 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 643 throw new IllegalStateException("triggerHardwareRecognitionEventForTest called on" 644 + " an invalid detector or error state"); 645 } 646 try { 647 mModelManagementService.triggerHardwareRecognitionEventForTest( 648 new KeyphraseRecognitionEvent(status, soundModelHandle, captureAvailable, 649 captureSession, captureDelayMs, capturePreambleMs, triggerInData, 650 captureFormat, data, null /* keyphraseExtras */), 651 mInternalCallback); 652 } catch (RemoteException e) { 653 throw e.rethrowFromSystemServer(); 654 } 655 } 656 } 657 658 /** 659 * Gets the recognition modes supported by the associated keyphrase. 660 * 661 * @see #RECOGNITION_MODE_USER_IDENTIFICATION 662 * @see #RECOGNITION_MODE_VOICE_TRIGGER 663 * 664 * @throws UnsupportedOperationException if the keyphrase itself isn't supported. 665 * Callers should only call this method after a supported state callback on 666 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 667 * @throws IllegalStateException if the detector is in an invalid or error state. 668 * This may happen if another detector has been instantiated or the 669 * {@link VoiceInteractionService} hosting this detector has been shut down. 670 */ getSupportedRecognitionModes()671 public @RecognitionModes int getSupportedRecognitionModes() { 672 if (DBG) Slog.d(TAG, "getSupportedRecognitionModes()"); 673 synchronized (mLock) { 674 return getSupportedRecognitionModesLocked(); 675 } 676 } 677 getSupportedRecognitionModesLocked()678 private int getSupportedRecognitionModesLocked() { 679 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 680 throw new IllegalStateException( 681 "getSupportedRecognitionModes called on an invalid detector or error state"); 682 } 683 684 // This method only makes sense if we can actually support a recognition. 685 if (mAvailability != STATE_KEYPHRASE_ENROLLED || mKeyphraseMetadata == null) { 686 throw new UnsupportedOperationException( 687 "Getting supported recognition modes for the keyphrase is not supported"); 688 } 689 690 return mKeyphraseMetadata.getRecognitionModeFlags(); 691 } 692 693 /** 694 * Get the audio capabilities supported by the platform which can be enabled when 695 * starting a recognition. 696 * Caller must be the active voice interaction service via 697 * Settings.Secure.VOICE_INTERACTION_SERVICE. 698 * 699 * @see #AUDIO_CAPABILITY_ECHO_CANCELLATION 700 * @see #AUDIO_CAPABILITY_NOISE_SUPPRESSION 701 * 702 * @return Bit field encoding of the AudioCapabilities supported. 703 */ 704 @AudioCapabilities getSupportedAudioCapabilities()705 public int getSupportedAudioCapabilities() { 706 if (DBG) Slog.d(TAG, "getSupportedAudioCapabilities()"); 707 synchronized (mLock) { 708 return getSupportedAudioCapabilitiesLocked(); 709 } 710 } 711 getSupportedAudioCapabilitiesLocked()712 private int getSupportedAudioCapabilitiesLocked() { 713 try { 714 ModuleProperties properties = 715 mSoundTriggerSession.getDspModuleProperties(); 716 if (properties != null) { 717 return properties.getAudioCapabilities(); 718 } 719 720 return 0; 721 } catch (RemoteException e) { 722 throw e.rethrowFromSystemServer(); 723 } 724 } 725 726 /** 727 * Starts recognition for the associated keyphrase. 728 * Caller must be the active voice interaction service via 729 * Settings.Secure.VOICE_INTERACTION_SERVICE. 730 * 731 * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 732 * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 733 * 734 * @param recognitionFlags The flags to control the recognition properties. 735 * @return Indicates whether the call succeeded or not. 736 * @throws UnsupportedOperationException if the recognition isn't supported. 737 * Callers should only call this method after a supported state callback on 738 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 739 * @throws IllegalStateException if the detector is in an invalid or error state. 740 * This may happen if another detector has been instantiated or the 741 * {@link VoiceInteractionService} hosting this detector has been shut down. 742 */ 743 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) startRecognition(@ecognitionFlags int recognitionFlags)744 public boolean startRecognition(@RecognitionFlags int recognitionFlags) { 745 if (DBG) Slog.d(TAG, "startRecognition(" + recognitionFlags + ")"); 746 synchronized (mLock) { 747 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 748 throw new IllegalStateException( 749 "startRecognition called on an invalid detector or error state"); 750 } 751 752 // Check if we can start/stop a recognition. 753 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 754 throw new UnsupportedOperationException( 755 "Recognition for the given keyphrase is not supported"); 756 } 757 758 return startRecognitionLocked(recognitionFlags) == STATUS_OK; 759 } 760 } 761 762 /** 763 * Starts recognition for the associated keyphrase. 764 * 765 * @see #startRecognition(int) 766 */ 767 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 768 @Override startRecognition()769 public boolean startRecognition() { 770 return startRecognition(0); 771 } 772 773 /** 774 * Stops recognition for the associated keyphrase. 775 * Caller must be the active voice interaction service via 776 * Settings.Secure.VOICE_INTERACTION_SERVICE. 777 * 778 * @return Indicates whether the call succeeded or not. 779 * @throws UnsupportedOperationException if the recognition isn't supported. 780 * Callers should only call this method after a supported state callback on 781 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 782 * @throws IllegalStateException if the detector is in an invalid or error state. 783 * This may happen if another detector has been instantiated or the 784 * {@link VoiceInteractionService} hosting this detector has been shut down. 785 */ 786 // TODO: Remove this RequiresPermission since it isn't actually enforced. Also fix the javadoc 787 // about permissions enforcement (when it throws vs when it just returns false) for other 788 // methods in this class. 789 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 790 @Override stopRecognition()791 public boolean stopRecognition() { 792 if (DBG) Slog.d(TAG, "stopRecognition()"); 793 synchronized (mLock) { 794 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 795 throw new IllegalStateException( 796 "stopRecognition called on an invalid detector or error state"); 797 } 798 799 // Check if we can start/stop a recognition. 800 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 801 throw new UnsupportedOperationException( 802 "Recognition for the given keyphrase is not supported"); 803 } 804 805 return stopRecognitionLocked() == STATUS_OK; 806 } 807 } 808 809 /** 810 * Set a model specific {@link ModelParams} with the given value. This 811 * parameter will keep its value for the duration the model is loaded regardless of starting and 812 * stopping recognition. Once the model is unloaded, the value will be lost. 813 * {@link AlwaysOnHotwordDetector#queryParameter} should be checked first before calling this 814 * method. 815 * Caller must be the active voice interaction service via 816 * Settings.Secure.VOICE_INTERACTION_SERVICE. 817 * 818 * @param modelParam {@link ModelParams} 819 * @param value Value to set 820 * @return - {@link SoundTrigger#STATUS_OK} in case of success 821 * - {@link SoundTrigger#STATUS_NO_INIT} if the native service cannot be reached 822 * - {@link SoundTrigger#STATUS_BAD_VALUE} invalid input parameter 823 * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence or 824 * if API is not supported by HAL 825 * @throws IllegalStateException if the detector is in an invalid or error state. 826 * This may happen if another detector has been instantiated or the 827 * {@link VoiceInteractionService} hosting this detector has been shut down. 828 */ 829 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) setParameter(@odelParams int modelParam, int value)830 public int setParameter(@ModelParams int modelParam, int value) { 831 if (DBG) { 832 Slog.d(TAG, "setParameter(" + modelParam + ", " + value + ")"); 833 } 834 835 synchronized (mLock) { 836 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 837 throw new IllegalStateException( 838 "setParameter called on an invalid detector or error state"); 839 } 840 841 return setParameterLocked(modelParam, value); 842 } 843 } 844 845 /** 846 * Get a model specific {@link ModelParams}. This parameter will keep its value 847 * for the duration the model is loaded regardless of starting and stopping recognition. 848 * Once the model is unloaded, the value will be lost. If the value is not set, a default 849 * value is returned. See {@link ModelParams} for parameter default values. 850 * {@link AlwaysOnHotwordDetector#queryParameter} should be checked first before 851 * calling this method. 852 * Caller must be the active voice interaction service via 853 * Settings.Secure.VOICE_INTERACTION_SERVICE. 854 * 855 * @param modelParam {@link ModelParams} 856 * @return value of parameter 857 * @throws IllegalStateException if the detector is in an invalid or error state. 858 * This may happen if another detector has been instantiated or the 859 * {@link VoiceInteractionService} hosting this detector has been shut down. 860 */ 861 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) getParameter(@odelParams int modelParam)862 public int getParameter(@ModelParams int modelParam) { 863 if (DBG) { 864 Slog.d(TAG, "getParameter(" + modelParam + ")"); 865 } 866 867 synchronized (mLock) { 868 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 869 throw new IllegalStateException( 870 "getParameter called on an invalid detector or error state"); 871 } 872 873 return getParameterLocked(modelParam); 874 } 875 } 876 877 /** 878 * Determine if parameter control is supported for the given model handle. 879 * This method should be checked prior to calling {@link AlwaysOnHotwordDetector#setParameter} 880 * or {@link AlwaysOnHotwordDetector#getParameter}. 881 * Caller must be the active voice interaction service via 882 * Settings.Secure.VOICE_INTERACTION_SERVICE. 883 * 884 * @param modelParam {@link ModelParams} 885 * @return supported range of parameter, null if not supported 886 * @throws IllegalStateException if the detector is in an invalid or error state. 887 * This may happen if another detector has been instantiated or the 888 * {@link VoiceInteractionService} hosting this detector has been shut down. 889 */ 890 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 891 @Nullable queryParameter(@odelParams int modelParam)892 public ModelParamRange queryParameter(@ModelParams int modelParam) { 893 if (DBG) { 894 Slog.d(TAG, "queryParameter(" + modelParam + ")"); 895 } 896 897 synchronized (mLock) { 898 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 899 throw new IllegalStateException( 900 "queryParameter called on an invalid detector or error state"); 901 } 902 903 return queryParameterLocked(modelParam); 904 } 905 } 906 907 /** 908 * Creates an intent to start the enrollment for the associated keyphrase. 909 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 910 * Starting re-enrollment is only valid if the keyphrase is un-enrolled, 911 * i.e. {@link #STATE_KEYPHRASE_UNENROLLED}, 912 * otherwise {@link #createReEnrollIntent()} should be preferred. 913 * 914 * @return An {@link Intent} to start enrollment for the given keyphrase. 915 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 916 * Callers should only call this method after a supported state callback on 917 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 918 * @throws IllegalStateException if the detector is in an invalid state. 919 * This may happen if another detector has been instantiated or the 920 * {@link VoiceInteractionService} hosting this detector has been shut down. 921 */ 922 @Nullable createEnrollIntent()923 public Intent createEnrollIntent() { 924 if (DBG) Slog.d(TAG, "createEnrollIntent"); 925 synchronized (mLock) { 926 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_ENROLL); 927 } 928 } 929 930 /** 931 * Creates an intent to start the un-enrollment for the associated keyphrase. 932 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 933 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 934 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 935 * 936 * @return An {@link Intent} to start un-enrollment for the given keyphrase. 937 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 938 * Callers should only call this method after a supported state callback on 939 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 940 * @throws IllegalStateException if the detector is in an invalid state. 941 * This may happen if another detector has been instantiated or the 942 * {@link VoiceInteractionService} hosting this detector has been shut down. 943 */ 944 @Nullable createUnEnrollIntent()945 public Intent createUnEnrollIntent() { 946 if (DBG) Slog.d(TAG, "createUnEnrollIntent"); 947 synchronized (mLock) { 948 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_UN_ENROLL); 949 } 950 } 951 952 /** 953 * Creates an intent to start the re-enrollment for the associated keyphrase. 954 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 955 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 956 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 957 * 958 * @return An {@link Intent} to start re-enrollment for the given keyphrase. 959 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 960 * Callers should only call this method after a supported state callback on 961 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 962 * @throws IllegalStateException if the detector is in an invalid or error state. 963 * This may happen if another detector has been instantiated or the 964 * {@link VoiceInteractionService} hosting this detector has been shut down. 965 */ 966 @Nullable createReEnrollIntent()967 public Intent createReEnrollIntent() { 968 if (DBG) Slog.d(TAG, "createReEnrollIntent"); 969 synchronized (mLock) { 970 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_RE_ENROLL); 971 } 972 } 973 getManageIntentLocked(@eyphraseEnrollmentInfo.ManageActions int action)974 private Intent getManageIntentLocked(@KeyphraseEnrollmentInfo.ManageActions int action) { 975 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 976 throw new IllegalStateException( 977 "getManageIntent called on an invalid detector or error state"); 978 } 979 980 // This method only makes sense if we can actually support a recognition. 981 if (mAvailability != STATE_KEYPHRASE_ENROLLED 982 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 983 throw new UnsupportedOperationException( 984 "Managing the given keyphrase is not supported"); 985 } 986 987 return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); 988 } 989 990 /** 991 * Invalidates this hotword detector so that any future calls to this result 992 * in an IllegalStateException. 993 * 994 * @hide 995 */ invalidate()996 void invalidate() { 997 synchronized (mLock) { 998 mAvailability = STATE_INVALID; 999 notifyStateChangedLocked(); 1000 1001 if (mSupportHotwordDetectionService) { 1002 try { 1003 mModelManagementService.shutdownHotwordDetectionService(); 1004 } catch (RemoteException e) { 1005 throw e.rethrowFromSystemServer(); 1006 } 1007 } 1008 } 1009 } 1010 1011 /** 1012 * Reloads the sound models from the service. 1013 * 1014 * @hide 1015 */ onSoundModelsChanged()1016 void onSoundModelsChanged() { 1017 synchronized (mLock) { 1018 if (mAvailability == STATE_INVALID 1019 || mAvailability == STATE_HARDWARE_UNAVAILABLE 1020 || mAvailability == STATE_ERROR) { 1021 Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config" 1022 + " or in the error state"); 1023 return; 1024 } 1025 1026 // Stop the recognition before proceeding. 1027 // This is done because we want to stop the recognition on an older model if it changed 1028 // or was deleted. 1029 // The availability change callback should ensure that the client starts recognition 1030 // again if needed. 1031 if (mAvailability == STATE_KEYPHRASE_ENROLLED) { 1032 try { 1033 stopRecognitionLocked(); 1034 } catch (SecurityException e) { 1035 Slog.w(TAG, "Failed to Stop the recognition", e); 1036 if (mTargetSdkVersion <= Build.VERSION_CODES.R) { 1037 throw e; 1038 } 1039 updateAndNotifyStateChangedLocked(STATE_ERROR); 1040 return; 1041 } 1042 } 1043 1044 // Execute a refresh availability task - which should then notify of a change. 1045 new RefreshAvailabiltyTask().execute(); 1046 } 1047 } 1048 startRecognitionLocked(int recognitionFlags)1049 private int startRecognitionLocked(int recognitionFlags) { 1050 KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1]; 1051 // TODO: Do we need to do something about the confidence level here? 1052 recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.getId(), 1053 mKeyphraseMetadata.getRecognitionModeFlags(), 0, new ConfidenceLevel[0]); 1054 boolean captureTriggerAudio = 1055 (recognitionFlags&RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; 1056 boolean allowMultipleTriggers = 1057 (recognitionFlags&RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0; 1058 boolean runInBatterySaver = (recognitionFlags&RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER) != 0; 1059 1060 int audioCapabilities = 0; 1061 if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION) != 0) { 1062 audioCapabilities |= AUDIO_CAPABILITY_ECHO_CANCELLATION; 1063 } 1064 if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION) != 0) { 1065 audioCapabilities |= AUDIO_CAPABILITY_NOISE_SUPPRESSION; 1066 } 1067 1068 int code; 1069 try { 1070 code = mSoundTriggerSession.startRecognition( 1071 mKeyphraseMetadata.getId(), mLocale.toLanguageTag(), mInternalCallback, 1072 new RecognitionConfig(captureTriggerAudio, allowMultipleTriggers, 1073 recognitionExtra, null /* additional data */, audioCapabilities), 1074 runInBatterySaver); 1075 } catch (RemoteException e) { 1076 throw e.rethrowFromSystemServer(); 1077 } 1078 1079 if (code != STATUS_OK) { 1080 Slog.w(TAG, "startRecognition() failed with error code " + code); 1081 } 1082 return code; 1083 } 1084 stopRecognitionLocked()1085 private int stopRecognitionLocked() { 1086 int code; 1087 try { 1088 code = mSoundTriggerSession.stopRecognition(mKeyphraseMetadata.getId(), 1089 mInternalCallback); 1090 } catch (RemoteException e) { 1091 throw e.rethrowFromSystemServer(); 1092 } 1093 1094 if (code != STATUS_OK) { 1095 Slog.w(TAG, "stopRecognition() failed with error code " + code); 1096 } 1097 return code; 1098 } 1099 setParameterLocked(@odelParams int modelParam, int value)1100 private int setParameterLocked(@ModelParams int modelParam, int value) { 1101 try { 1102 int code = mSoundTriggerSession.setParameter(mKeyphraseMetadata.getId(), modelParam, 1103 value); 1104 1105 if (code != STATUS_OK) { 1106 Slog.w(TAG, "setParameter failed with error code " + code); 1107 } 1108 1109 return code; 1110 } catch (RemoteException e) { 1111 throw e.rethrowFromSystemServer(); 1112 } 1113 } 1114 getParameterLocked(@odelParams int modelParam)1115 private int getParameterLocked(@ModelParams int modelParam) { 1116 try { 1117 return mSoundTriggerSession.getParameter(mKeyphraseMetadata.getId(), modelParam); 1118 } catch (RemoteException e) { 1119 throw e.rethrowFromSystemServer(); 1120 } 1121 } 1122 1123 @Nullable queryParameterLocked(@odelParams int modelParam)1124 private ModelParamRange queryParameterLocked(@ModelParams int modelParam) { 1125 try { 1126 SoundTrigger.ModelParamRange modelParamRange = 1127 mSoundTriggerSession.queryParameter(mKeyphraseMetadata.getId(), modelParam); 1128 1129 if (modelParamRange == null) { 1130 return null; 1131 } 1132 1133 return new ModelParamRange(modelParamRange); 1134 } catch (RemoteException e) { 1135 throw e.rethrowFromSystemServer(); 1136 } 1137 } 1138 updateAndNotifyStateChangedLocked(int availability)1139 private void updateAndNotifyStateChangedLocked(int availability) { 1140 if (DBG) { 1141 Slog.d(TAG, "Hotword availability changed from " + mAvailability 1142 + " -> " + availability); 1143 } 1144 mAvailability = availability; 1145 notifyStateChangedLocked(); 1146 } 1147 notifyStateChangedLocked()1148 private void notifyStateChangedLocked() { 1149 Message message = Message.obtain(mHandler, MSG_AVAILABILITY_CHANGED); 1150 message.arg1 = mAvailability; 1151 message.sendToTarget(); 1152 } 1153 1154 /** @hide */ 1155 static final class SoundTriggerListener extends IHotwordRecognitionStatusCallback.Stub { 1156 private final Handler mHandler; 1157 SoundTriggerListener(Handler handler)1158 public SoundTriggerListener(Handler handler) { 1159 mHandler = handler; 1160 } 1161 1162 @Override onKeyphraseDetected( KeyphraseRecognitionEvent event, HotwordDetectedResult result)1163 public void onKeyphraseDetected( 1164 KeyphraseRecognitionEvent event, HotwordDetectedResult result) { 1165 if (DBG) { 1166 Slog.d(TAG, "onDetected(" + event + ")"); 1167 } else { 1168 Slog.i(TAG, "onDetected"); 1169 } 1170 Message.obtain(mHandler, MSG_HOTWORD_DETECTED, 1171 new EventPayload(event.triggerInData, event.captureAvailable, 1172 event.captureFormat, event.captureSession, event.data, result)) 1173 .sendToTarget(); 1174 } 1175 @Override onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event)1176 public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) { 1177 Slog.w(TAG, "Generic sound trigger event detected at AOHD: " + event); 1178 } 1179 1180 @Override onRejected(@onNull HotwordRejectedResult result)1181 public void onRejected(@NonNull HotwordRejectedResult result) { 1182 if (DBG) { 1183 Slog.d(TAG, "onRejected(" + result + ")"); 1184 } else { 1185 Slog.i(TAG, "onRejected"); 1186 } 1187 Message.obtain(mHandler, MSG_HOTWORD_REJECTED, result).sendToTarget(); 1188 } 1189 1190 @Override onError(int status)1191 public void onError(int status) { 1192 Slog.i(TAG, "onError: " + status); 1193 mHandler.sendEmptyMessage(MSG_DETECTION_ERROR); 1194 } 1195 1196 @Override onRecognitionPaused()1197 public void onRecognitionPaused() { 1198 Slog.i(TAG, "onRecognitionPaused"); 1199 mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE); 1200 } 1201 1202 @Override onRecognitionResumed()1203 public void onRecognitionResumed() { 1204 Slog.i(TAG, "onRecognitionResumed"); 1205 mHandler.sendEmptyMessage(MSG_DETECTION_RESUME); 1206 } 1207 1208 @Override onStatusReported(int status)1209 public void onStatusReported(int status) { 1210 if (DBG) { 1211 Slog.d(TAG, "onStatusReported(" + status + ")"); 1212 } else { 1213 Slog.i(TAG, "onStatusReported"); 1214 } 1215 Message message = Message.obtain(mHandler, MSG_HOTWORD_STATUS_REPORTED); 1216 message.arg1 = status; 1217 message.sendToTarget(); 1218 } 1219 1220 @Override onProcessRestarted()1221 public void onProcessRestarted() { 1222 Slog.i(TAG, "onProcessRestarted"); 1223 mHandler.sendEmptyMessage(MSG_PROCESS_RESTARTED); 1224 } 1225 } 1226 1227 class MyHandler extends Handler { 1228 @Override handleMessage(Message msg)1229 public void handleMessage(Message msg) { 1230 synchronized (mLock) { 1231 if (mAvailability == STATE_INVALID) { 1232 Slog.w(TAG, "Received message: " + msg.what + " for an invalid detector"); 1233 return; 1234 } 1235 } 1236 1237 switch (msg.what) { 1238 case MSG_AVAILABILITY_CHANGED: 1239 mExternalCallback.onAvailabilityChanged(msg.arg1); 1240 break; 1241 case MSG_HOTWORD_DETECTED: 1242 mExternalCallback.onDetected((EventPayload) msg.obj); 1243 break; 1244 case MSG_DETECTION_ERROR: 1245 mExternalCallback.onError(); 1246 break; 1247 case MSG_DETECTION_PAUSE: 1248 mExternalCallback.onRecognitionPaused(); 1249 break; 1250 case MSG_DETECTION_RESUME: 1251 mExternalCallback.onRecognitionResumed(); 1252 break; 1253 case MSG_HOTWORD_REJECTED: 1254 mExternalCallback.onRejected((HotwordRejectedResult) msg.obj); 1255 break; 1256 case MSG_HOTWORD_STATUS_REPORTED: 1257 mExternalCallback.onHotwordDetectionServiceInitialized(msg.arg1); 1258 break; 1259 case MSG_PROCESS_RESTARTED: 1260 mExternalCallback.onHotwordDetectionServiceRestarted(); 1261 break; 1262 default: 1263 super.handleMessage(msg); 1264 } 1265 } 1266 } 1267 1268 class RefreshAvailabiltyTask extends AsyncTask<Void, Void, Void> { 1269 1270 @Override doInBackground(Void... params)1271 public Void doInBackground(Void... params) { 1272 try { 1273 int availability = internalGetInitialAvailability(); 1274 1275 synchronized (mLock) { 1276 if (availability == STATE_NOT_READY) { 1277 internalUpdateEnrolledKeyphraseMetadata(); 1278 if (mKeyphraseMetadata != null) { 1279 availability = STATE_KEYPHRASE_ENROLLED; 1280 } else { 1281 availability = STATE_KEYPHRASE_UNENROLLED; 1282 } 1283 } 1284 updateAndNotifyStateChangedLocked(availability); 1285 } 1286 } catch (SecurityException e) { 1287 Slog.w(TAG, "Failed to refresh availability", e); 1288 if (mTargetSdkVersion <= Build.VERSION_CODES.R) { 1289 throw e; 1290 } 1291 synchronized (mLock) { 1292 updateAndNotifyStateChangedLocked(STATE_ERROR); 1293 } 1294 } 1295 1296 return null; 1297 } 1298 1299 /** 1300 * @return The initial availability without checking the enrollment status. 1301 */ internalGetInitialAvailability()1302 private int internalGetInitialAvailability() { 1303 synchronized (mLock) { 1304 // This detector has already been invalidated. 1305 if (mAvailability == STATE_INVALID) { 1306 return STATE_INVALID; 1307 } 1308 } 1309 1310 ModuleProperties dspModuleProperties; 1311 try { 1312 dspModuleProperties = 1313 mSoundTriggerSession.getDspModuleProperties(); 1314 } catch (RemoteException e) { 1315 throw e.rethrowFromSystemServer(); 1316 } 1317 1318 // No DSP available 1319 if (dspModuleProperties == null) { 1320 return STATE_HARDWARE_UNAVAILABLE; 1321 } 1322 1323 return STATE_NOT_READY; 1324 } 1325 internalUpdateEnrolledKeyphraseMetadata()1326 private void internalUpdateEnrolledKeyphraseMetadata() { 1327 try { 1328 mKeyphraseMetadata = mModelManagementService.getEnrolledKeyphraseMetadata( 1329 mText, mLocale.toLanguageTag()); 1330 } catch (RemoteException e) { 1331 throw e.rethrowFromSystemServer(); 1332 } 1333 } 1334 } 1335 1336 /** @hide */ dump(String prefix, PrintWriter pw)1337 public void dump(String prefix, PrintWriter pw) { 1338 synchronized (mLock) { 1339 pw.print(prefix); pw.print("Text="); pw.println(mText); 1340 pw.print(prefix); pw.print("Locale="); pw.println(mLocale); 1341 pw.print(prefix); pw.print("Availability="); pw.println(mAvailability); 1342 pw.print(prefix); pw.print("KeyphraseMetadata="); pw.println(mKeyphraseMetadata); 1343 pw.print(prefix); pw.print("EnrollmentInfo="); pw.println(mKeyphraseEnrollmentInfo); 1344 } 1345 } 1346 } 1347