1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import static java.util.Objects.requireNonNull; 20 21 import android.annotation.DurationMillisLong; 22 import android.annotation.IntDef; 23 import android.annotation.NonNull; 24 import android.annotation.Nullable; 25 import android.annotation.SdkConstant; 26 import android.annotation.SuppressLint; 27 import android.annotation.SystemApi; 28 import android.annotation.TestApi; 29 import android.app.Service; 30 import android.content.ContentCaptureOptions; 31 import android.content.Context; 32 import android.content.Intent; 33 import android.hardware.soundtrigger.SoundTrigger; 34 import android.media.AudioFormat; 35 import android.media.AudioSystem; 36 import android.os.IBinder; 37 import android.os.IRemoteCallback; 38 import android.os.ParcelFileDescriptor; 39 import android.os.PersistableBundle; 40 import android.os.RemoteException; 41 import android.os.SharedMemory; 42 import android.speech.IRecognitionServiceManager; 43 import android.util.Log; 44 import android.view.contentcapture.ContentCaptureManager; 45 import android.view.contentcapture.IContentCaptureManager; 46 47 import java.lang.annotation.Documented; 48 import java.lang.annotation.Retention; 49 import java.lang.annotation.RetentionPolicy; 50 import java.util.Locale; 51 import java.util.function.IntConsumer; 52 53 /** 54 * Implemented by an application that wants to offer detection for hotword. The service can be used 55 * for both DSP and non-DSP detectors. 56 * 57 * The system will bind an application's {@link VoiceInteractionService} first. When {@link 58 * VoiceInteractionService#createHotwordDetector(PersistableBundle, SharedMemory, 59 * HotwordDetector.Callback)} or {@link VoiceInteractionService#createAlwaysOnHotwordDetector( 60 * String, Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} is called, 61 * the system will bind application's {@link HotwordDetectionService}. Either on a hardware 62 * trigger or on request from the {@link VoiceInteractionService}, the system calls into the 63 * {@link HotwordDetectionService} to request detection. The {@link HotwordDetectionService} then 64 * uses {@link Callback#onDetected(HotwordDetectedResult)} to inform the system that a relevant 65 * keyphrase was detected, or if applicable uses {@link Callback#onRejected(HotwordRejectedResult)} 66 * to inform the system that a keyphrase was not detected. The system then relays this result to 67 * the {@link VoiceInteractionService} through {@link HotwordDetector.Callback}. 68 * 69 * Note: Methods in this class may be called concurrently 70 * 71 * @hide 72 */ 73 @SystemApi 74 public abstract class HotwordDetectionService extends Service 75 implements SandboxedDetectionInitializer { 76 private static final String TAG = "HotwordDetectionService"; 77 private static final boolean DBG = false; 78 79 private static final long UPDATE_TIMEOUT_MILLIS = 20000; 80 81 /** 82 * Feature flag for Attention Service. 83 * 84 * @hide 85 */ 86 @TestApi 87 public static final boolean ENABLE_PROXIMITY_RESULT = true; 88 89 /** 90 * Indicates that the updated status is successful. 91 * 92 * @deprecated Replaced with 93 * {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_SUCCESS} 94 */ 95 @Deprecated 96 public static final int INITIALIZATION_STATUS_SUCCESS = 97 SandboxedDetectionInitializer.INITIALIZATION_STATUS_SUCCESS; 98 99 /** 100 * Indicates that the callback wasn’t invoked within the timeout. 101 * This is used by system. 102 * 103 * @deprecated Replaced with 104 * {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_UNKNOWN} 105 */ 106 @Deprecated 107 public static final int INITIALIZATION_STATUS_UNKNOWN = 108 SandboxedDetectionInitializer.INITIALIZATION_STATUS_UNKNOWN; 109 110 /** 111 * Source for the given audio stream. 112 * 113 * @hide 114 */ 115 @Documented 116 @Retention(RetentionPolicy.SOURCE) 117 @IntDef({ 118 AUDIO_SOURCE_MICROPHONE, 119 AUDIO_SOURCE_EXTERNAL 120 }) 121 @interface AudioSource {} 122 123 /** @hide */ 124 public static final int AUDIO_SOURCE_MICROPHONE = 1; 125 /** @hide */ 126 public static final int AUDIO_SOURCE_EXTERNAL = 2; 127 128 /** 129 * The {@link Intent} that must be declared as handled by the service. 130 * To be supported, the service must also require the 131 * {@link android.Manifest.permission#BIND_HOTWORD_DETECTION_SERVICE} permission so 132 * that other applications can not abuse it. 133 */ 134 @SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION) 135 public static final String SERVICE_INTERFACE = 136 "android.service.voice.HotwordDetectionService"; 137 138 @Nullable 139 private ContentCaptureManager mContentCaptureManager; 140 @Nullable 141 private IRecognitionServiceManager mIRecognitionServiceManager; 142 143 private final ISandboxedDetectionService mInterface = new ISandboxedDetectionService.Stub() { 144 @Override 145 public void detectFromDspSource( 146 SoundTrigger.KeyphraseRecognitionEvent event, 147 AudioFormat audioFormat, 148 long timeoutMillis, 149 IDspHotwordDetectionCallback callback) 150 throws RemoteException { 151 if (DBG) { 152 Log.d(TAG, "#detectFromDspSource"); 153 } 154 HotwordDetectionService.this.onDetect( 155 new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(), 156 timeoutMillis, 157 new Callback(callback)); 158 } 159 160 @Override 161 public void updateState(PersistableBundle options, SharedMemory sharedMemory, 162 IRemoteCallback callback) throws RemoteException { 163 Log.v(TAG, "#updateState" + (callback != null ? " with callback" : "")); 164 HotwordDetectionService.this.onUpdateStateInternal( 165 options, 166 sharedMemory, 167 callback); 168 } 169 170 @Override 171 public void detectFromMicrophoneSource( 172 ParcelFileDescriptor audioStream, 173 @AudioSource int audioSource, 174 AudioFormat audioFormat, 175 PersistableBundle options, 176 IDspHotwordDetectionCallback callback) 177 throws RemoteException { 178 if (DBG) { 179 Log.d(TAG, "#detectFromMicrophoneSource"); 180 } 181 switch (audioSource) { 182 case AUDIO_SOURCE_MICROPHONE: 183 HotwordDetectionService.this.onDetect( 184 new Callback(callback)); 185 break; 186 case AUDIO_SOURCE_EXTERNAL: 187 HotwordDetectionService.this.onDetect( 188 audioStream, 189 audioFormat, 190 options, 191 new Callback(callback)); 192 break; 193 default: 194 Log.i(TAG, "Unsupported audio source " + audioSource); 195 } 196 } 197 198 @Override 199 public void detectWithVisualSignals( 200 IDetectorSessionVisualQueryDetectionCallback callback) { 201 throw new UnsupportedOperationException("Not supported by HotwordDetectionService"); 202 } 203 204 @Override 205 public void updateAudioFlinger(IBinder audioFlinger) { 206 AudioSystem.setAudioFlingerBinder(audioFlinger); 207 } 208 209 @Override 210 public void updateContentCaptureManager(IContentCaptureManager manager, 211 ContentCaptureOptions options) { 212 mContentCaptureManager = new ContentCaptureManager( 213 HotwordDetectionService.this, manager, options); 214 } 215 216 @Override 217 public void updateRecognitionServiceManager(IRecognitionServiceManager manager) { 218 mIRecognitionServiceManager = manager; 219 } 220 221 @Override 222 public void ping(IRemoteCallback callback) throws RemoteException { 223 callback.sendResult(null); 224 } 225 226 @Override 227 public void stopDetection() { 228 HotwordDetectionService.this.onStopDetection(); 229 } 230 231 @Override 232 public void registerRemoteStorageService(IDetectorSessionStorageService 233 detectorSessionStorageService) { 234 throw new UnsupportedOperationException("Hotword cannot access files from the disk."); 235 } 236 }; 237 238 @Override 239 @Nullable onBind(@onNull Intent intent)240 public final IBinder onBind(@NonNull Intent intent) { 241 if (SERVICE_INTERFACE.equals(intent.getAction())) { 242 return mInterface.asBinder(); 243 } 244 Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": " 245 + intent); 246 return null; 247 } 248 249 @Override 250 @SuppressLint("OnNameExpected") getSystemService(@erviceName @onNull String name)251 public @Nullable Object getSystemService(@ServiceName @NonNull String name) { 252 if (Context.CONTENT_CAPTURE_MANAGER_SERVICE.equals(name)) { 253 return mContentCaptureManager; 254 } else if (Context.SPEECH_RECOGNITION_SERVICE.equals(name) 255 && mIRecognitionServiceManager != null) { 256 return mIRecognitionServiceManager.asBinder(); 257 } else { 258 return super.getSystemService(name); 259 } 260 } 261 262 /** 263 * Returns the maximum number of initialization status for some application specific failed 264 * reasons. 265 * 266 * Note: The value 0 is reserved for success. 267 * 268 * @hide 269 * @deprecated Replaced with 270 * {@link SandboxedDetectionInitializer#getMaxCustomInitializationStatus()} 271 */ 272 @SystemApi 273 @Deprecated getMaxCustomInitializationStatus()274 public static int getMaxCustomInitializationStatus() { 275 return MAXIMUM_NUMBER_OF_INITIALIZATION_STATUS_CUSTOM_ERROR; 276 } 277 278 /** 279 * Called when the device hardware (such as a DSP) detected the hotword, to request second stage 280 * validation before handing over the audio to the {@link AlwaysOnHotwordDetector}. 281 * 282 * <p>After {@code callback} is invoked or {@code timeoutMillis} has passed, and invokes the 283 * appropriate {@link AlwaysOnHotwordDetector.Callback callback}. 284 * 285 * <p>When responding to a detection event, the 286 * {@link HotwordDetectedResult#getHotwordPhraseId()} must match a keyphrase ID listed 287 * in the eventPayload's 288 * {@link AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras()} list. This is 289 * forcing the intention of the {@link HotwordDetectionService} to validate an event from the 290 * voice engine and not augment its result. 291 * 292 * @param eventPayload Payload data for the hardware detection event. This may contain the 293 * trigger audio, if requested when calling 294 * {@link AlwaysOnHotwordDetector#startRecognition(int)}. 295 * Each {@link AlwaysOnHotwordDetector} will be associated with at minimum a unique 296 * keyphrase ID indicated by 297 * {@link AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras()}[0]. 298 * Any extra 299 * {@link android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra}'s 300 * in the eventPayload represent additional phrases detected by the voice engine. 301 * @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If 302 * the application fails to abide by the timeout, system will close the 303 * microphone and cancel the operation. 304 * @param callback The callback to use for responding to the detection request. 305 * 306 * @hide 307 */ 308 @SystemApi onDetect( @onNull AlwaysOnHotwordDetector.EventPayload eventPayload, @DurationMillisLong long timeoutMillis, @NonNull Callback callback)309 public void onDetect( 310 @NonNull AlwaysOnHotwordDetector.EventPayload eventPayload, 311 @DurationMillisLong long timeoutMillis, 312 @NonNull Callback callback) { 313 // TODO: Add a helpful error message. 314 throw new UnsupportedOperationException(); 315 } 316 317 /** 318 * Called when the {@link VoiceInteractionService#createAlwaysOnHotwordDetector(String, Locale, 319 * PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} or 320 * {@link AlwaysOnHotwordDetector#updateState(PersistableBundle, SharedMemory)} requests an 321 * update of the hotword detection parameters. 322 * 323 * {@inheritDoc} 324 * @hide 325 */ 326 @Override 327 @SystemApi onUpdateState( @ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @DurationMillisLong long callbackTimeoutMillis, @Nullable IntConsumer statusCallback)328 public void onUpdateState( 329 @Nullable PersistableBundle options, 330 @Nullable SharedMemory sharedMemory, 331 @DurationMillisLong long callbackTimeoutMillis, 332 @Nullable IntConsumer statusCallback) {} 333 334 /** 335 * Called when the {@link VoiceInteractionService} requests that this service 336 * {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly 337 * from the device microphone. 338 * <p> 339 * On successful detection of a hotword, call 340 * {@link Callback#onDetected(HotwordDetectedResult)}. 341 * 342 * @param callback The callback to use for responding to the detection request. 343 * {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here. 344 */ onDetect(@onNull Callback callback)345 public void onDetect(@NonNull Callback callback) { 346 // TODO: Add a helpful error message. 347 throw new UnsupportedOperationException(); 348 } 349 350 /** 351 * Called when the {@link VoiceInteractionService} requests that this service 352 * {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat, 353 * PersistableBundle)} run} hotword recognition on audio coming from an external connected 354 * microphone. 355 * <p> 356 * Upon invoking the {@code callback}, the system closes {@code audioStream} and sends the 357 * detection result to the {@link HotwordDetector.Callback hotword detector}. 358 * 359 * @param audioStream Stream containing audio bytes returned from a microphone 360 * @param audioFormat Format of the supplied audio 361 * @param options Options supporting detection, such as configuration specific to the source of 362 * the audio, provided through 363 * {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat, 364 * PersistableBundle)}. 365 * @param callback The callback to use for responding to the detection request. 366 */ onDetect( @onNull ParcelFileDescriptor audioStream, @NonNull AudioFormat audioFormat, @Nullable PersistableBundle options, @NonNull Callback callback)367 public void onDetect( 368 @NonNull ParcelFileDescriptor audioStream, 369 @NonNull AudioFormat audioFormat, 370 @Nullable PersistableBundle options, 371 @NonNull Callback callback) { 372 // TODO: Add a helpful error message. 373 throw new UnsupportedOperationException(); 374 } 375 onUpdateStateInternal(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, IRemoteCallback callback)376 private void onUpdateStateInternal(@Nullable PersistableBundle options, 377 @Nullable SharedMemory sharedMemory, IRemoteCallback callback) { 378 IntConsumer intConsumer = 379 SandboxedDetectionInitializer.createInitializationStatusConsumer(callback); 380 onUpdateState(options, sharedMemory, UPDATE_TIMEOUT_MILLIS, intConsumer); 381 } 382 383 /** 384 * Called when the {@link VoiceInteractionService} 385 * {@link HotwordDetector#stopRecognition() requests} that hotword recognition be stopped. 386 * <p> 387 * Any open {@link android.media.AudioRecord} should be closed here. 388 */ onStopDetection()389 public void onStopDetection() { 390 } 391 392 /** 393 * Callback for returning the detection result. 394 * 395 * @hide 396 */ 397 @SystemApi 398 public static final class Callback { 399 // TODO: consider making the constructor a test api for testing purpose 400 private final IDspHotwordDetectionCallback mRemoteCallback; 401 Callback(IDspHotwordDetectionCallback remoteCallback)402 private Callback(IDspHotwordDetectionCallback remoteCallback) { 403 mRemoteCallback = remoteCallback; 404 } 405 406 /** 407 * Informs the {@link HotwordDetector} that the keyphrase was detected. 408 * 409 * @param result Info about the detection result. This is provided to the 410 * {@link HotwordDetector}. 411 */ onDetected(@onNull HotwordDetectedResult result)412 public void onDetected(@NonNull HotwordDetectedResult result) { 413 requireNonNull(result); 414 final PersistableBundle persistableBundle = result.getExtras(); 415 if (!persistableBundle.isEmpty() && HotwordDetectedResult.getParcelableSize( 416 persistableBundle) > HotwordDetectedResult.getMaxBundleSize()) { 417 throw new IllegalArgumentException( 418 "The bundle size of result is larger than max bundle size (" 419 + HotwordDetectedResult.getMaxBundleSize() 420 + ") of HotwordDetectedResult"); 421 } 422 try { 423 mRemoteCallback.onDetected(result); 424 } catch (RemoteException e) { 425 throw e.rethrowFromSystemServer(); 426 } 427 } 428 429 /** 430 * Informs the {@link HotwordDetector} that the keyphrase was not detected. 431 * <p> 432 * This cannot not be used when recognition is done through 433 * {@link #onDetect(ParcelFileDescriptor, AudioFormat, Callback)}. 434 * 435 * @param result Info about the second stage detection result. This is provided to 436 * the {@link HotwordDetector}. 437 */ onRejected(@onNull HotwordRejectedResult result)438 public void onRejected(@NonNull HotwordRejectedResult result) { 439 requireNonNull(result); 440 try { 441 mRemoteCallback.onRejected(result); 442 } catch (RemoteException e) { 443 throw e.rethrowFromSystemServer(); 444 } 445 } 446 } 447 } 448