1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import static java.util.Objects.requireNonNull; 20 21 import android.annotation.DurationMillisLong; 22 import android.annotation.IntDef; 23 import android.annotation.NonNull; 24 import android.annotation.Nullable; 25 import android.annotation.SdkConstant; 26 import android.annotation.SuppressLint; 27 import android.annotation.SystemApi; 28 import android.app.Service; 29 import android.content.ContentCaptureOptions; 30 import android.content.Context; 31 import android.content.Intent; 32 import android.hardware.soundtrigger.SoundTrigger; 33 import android.media.AudioFormat; 34 import android.media.AudioSystem; 35 import android.os.Bundle; 36 import android.os.IBinder; 37 import android.os.IRemoteCallback; 38 import android.os.ParcelFileDescriptor; 39 import android.os.PersistableBundle; 40 import android.os.RemoteException; 41 import android.os.SharedMemory; 42 import android.util.Log; 43 import android.view.contentcapture.ContentCaptureManager; 44 import android.view.contentcapture.IContentCaptureManager; 45 46 import java.lang.annotation.Documented; 47 import java.lang.annotation.Retention; 48 import java.lang.annotation.RetentionPolicy; 49 import java.util.Locale; 50 import java.util.function.IntConsumer; 51 52 /** 53 * Implemented by an application that wants to offer detection for hotword. The service can be used 54 * for both DSP and non-DSP detectors. 55 * 56 * The system will bind an application's {@link VoiceInteractionService} first. When {@link 57 * VoiceInteractionService#createHotwordDetector(PersistableBundle, SharedMemory, 58 * HotwordDetector.Callback)} or {@link VoiceInteractionService#createAlwaysOnHotwordDetector( 59 * String, Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} is called, 60 * the system will bind application's {@link HotwordDetectionService}. Either on a hardware 61 * trigger or on request from the {@link VoiceInteractionService}, the system calls into the 62 * {@link HotwordDetectionService} to request detection. The {@link HotwordDetectionService} then 63 * uses {@link Callback#onDetected(HotwordDetectedResult)} to inform the system that a relevant 64 * keyphrase was detected, or if applicable uses {@link Callback#onRejected(HotwordRejectedResult)} 65 * to inform the system that a keyphrase was not detected. The system then relays this result to 66 * the {@link VoiceInteractionService} through {@link HotwordDetector.Callback}. 67 * 68 * Note: Methods in this class may be called concurrently 69 * 70 * @hide 71 */ 72 @SystemApi 73 public abstract class HotwordDetectionService extends Service { 74 private static final String TAG = "HotwordDetectionService"; 75 private static final boolean DBG = false; 76 77 private static final long UPDATE_TIMEOUT_MILLIS = 5000; 78 79 /** @hide */ 80 public static final String KEY_INITIALIZATION_STATUS = "initialization_status"; 81 82 /** 83 * The maximum number of initialization status for some application specific failed reasons. 84 * 85 * @hide 86 */ 87 public static final int MAXIMUM_NUMBER_OF_INITIALIZATION_STATUS_CUSTOM_ERROR = 2; 88 89 /** 90 * Indicates that the updated status is successful. 91 */ 92 public static final int INITIALIZATION_STATUS_SUCCESS = 0; 93 94 /** 95 * Indicates that the callback wasn’t invoked within the timeout. 96 * This is used by system. 97 */ 98 public static final int INITIALIZATION_STATUS_UNKNOWN = 100; 99 100 /** 101 * Source for the given audio stream. 102 * 103 * @hide 104 */ 105 @Documented 106 @Retention(RetentionPolicy.SOURCE) 107 @IntDef({ 108 AUDIO_SOURCE_MICROPHONE, 109 AUDIO_SOURCE_EXTERNAL 110 }) 111 @interface AudioSource {} 112 113 /** @hide */ 114 public static final int AUDIO_SOURCE_MICROPHONE = 1; 115 /** @hide */ 116 public static final int AUDIO_SOURCE_EXTERNAL = 2; 117 118 /** 119 * The {@link Intent} that must be declared as handled by the service. 120 * To be supported, the service must also require the 121 * {@link android.Manifest.permission#BIND_HOTWORD_DETECTION_SERVICE} permission so 122 * that other applications can not abuse it. 123 */ 124 @SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION) 125 public static final String SERVICE_INTERFACE = 126 "android.service.voice.HotwordDetectionService"; 127 128 @Nullable 129 private ContentCaptureManager mContentCaptureManager; 130 131 private final IHotwordDetectionService mInterface = new IHotwordDetectionService.Stub() { 132 @Override 133 public void detectFromDspSource( 134 SoundTrigger.KeyphraseRecognitionEvent event, 135 AudioFormat audioFormat, 136 long timeoutMillis, 137 IDspHotwordDetectionCallback callback) 138 throws RemoteException { 139 if (DBG) { 140 Log.d(TAG, "#detectFromDspSource"); 141 } 142 HotwordDetectionService.this.onDetect( 143 new AlwaysOnHotwordDetector.EventPayload( 144 event.triggerInData, event.captureAvailable, 145 event.captureFormat, event.captureSession, event.data), 146 timeoutMillis, 147 new Callback(callback)); 148 } 149 150 @Override 151 public void updateState(PersistableBundle options, SharedMemory sharedMemory, 152 IRemoteCallback callback) throws RemoteException { 153 Log.v(TAG, "#updateState" + (callback != null ? " with callback" : "")); 154 HotwordDetectionService.this.onUpdateStateInternal( 155 options, 156 sharedMemory, 157 callback); 158 } 159 160 @Override 161 public void detectFromMicrophoneSource( 162 ParcelFileDescriptor audioStream, 163 @AudioSource int audioSource, 164 AudioFormat audioFormat, 165 PersistableBundle options, 166 IDspHotwordDetectionCallback callback) 167 throws RemoteException { 168 if (DBG) { 169 Log.d(TAG, "#detectFromMicrophoneSource"); 170 } 171 switch (audioSource) { 172 case AUDIO_SOURCE_MICROPHONE: 173 HotwordDetectionService.this.onDetect( 174 new Callback(callback)); 175 break; 176 case AUDIO_SOURCE_EXTERNAL: 177 HotwordDetectionService.this.onDetect( 178 audioStream, 179 audioFormat, 180 options, 181 new Callback(callback)); 182 break; 183 default: 184 Log.i(TAG, "Unsupported audio source " + audioSource); 185 } 186 } 187 188 @Override 189 public void updateAudioFlinger(IBinder audioFlinger) { 190 AudioSystem.setAudioFlingerBinder(audioFlinger); 191 } 192 193 @Override 194 public void updateContentCaptureManager(IContentCaptureManager manager, 195 ContentCaptureOptions options) { 196 mContentCaptureManager = new ContentCaptureManager( 197 HotwordDetectionService.this, manager, options); 198 } 199 200 @Override 201 public void ping(IRemoteCallback callback) throws RemoteException { 202 callback.sendResult(null); 203 } 204 205 @Override 206 public void stopDetection() { 207 HotwordDetectionService.this.onStopDetection(); 208 } 209 }; 210 211 @Override 212 @Nullable onBind(@onNull Intent intent)213 public final IBinder onBind(@NonNull Intent intent) { 214 if (SERVICE_INTERFACE.equals(intent.getAction())) { 215 return mInterface.asBinder(); 216 } 217 Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": " 218 + intent); 219 return null; 220 } 221 222 @Override 223 @SuppressLint("OnNameExpected") getSystemService(@erviceName @onNull String name)224 public @Nullable Object getSystemService(@ServiceName @NonNull String name) { 225 if (Context.CONTENT_CAPTURE_MANAGER_SERVICE.equals(name)) { 226 return mContentCaptureManager; 227 } else { 228 return super.getSystemService(name); 229 } 230 } 231 232 /** 233 * Returns the maximum number of initialization status for some application specific failed 234 * reasons. 235 * 236 * Note: The value 0 is reserved for success. 237 * 238 * @hide 239 */ 240 @SystemApi getMaxCustomInitializationStatus()241 public static int getMaxCustomInitializationStatus() { 242 return MAXIMUM_NUMBER_OF_INITIALIZATION_STATUS_CUSTOM_ERROR; 243 } 244 245 /** 246 * Called when the device hardware (such as a DSP) detected the hotword, to request second stage 247 * validation before handing over the audio to the {@link AlwaysOnHotwordDetector}. 248 * <p> 249 * After {@code callback} is invoked or {@code timeoutMillis} has passed, and invokes the 250 * appropriate {@link AlwaysOnHotwordDetector.Callback callback}. 251 * 252 * @param eventPayload Payload data for the hardware detection event. This may contain the 253 * trigger audio, if requested when calling 254 * {@link AlwaysOnHotwordDetector#startRecognition(int)}. 255 * @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If 256 * the application fails to abide by the timeout, system will close the 257 * microphone and cancel the operation. 258 * @param callback The callback to use for responding to the detection request. 259 * 260 * @hide 261 */ 262 @SystemApi onDetect( @onNull AlwaysOnHotwordDetector.EventPayload eventPayload, @DurationMillisLong long timeoutMillis, @NonNull Callback callback)263 public void onDetect( 264 @NonNull AlwaysOnHotwordDetector.EventPayload eventPayload, 265 @DurationMillisLong long timeoutMillis, 266 @NonNull Callback callback) { 267 // TODO: Add a helpful error message. 268 throw new UnsupportedOperationException(); 269 } 270 271 /** 272 * Called when the {@link VoiceInteractionService#createAlwaysOnHotwordDetector(String, Locale, 273 * PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} or 274 * {@link AlwaysOnHotwordDetector#updateState(PersistableBundle, SharedMemory)} requests an 275 * update of the hotword detection parameters. 276 * 277 * @param options Application configuration data to provide to the 278 * {@link HotwordDetectionService}. PersistableBundle does not allow any remotable objects or 279 * other contents that can be used to communicate with other processes. 280 * @param sharedMemory The unrestricted data blob to provide to the 281 * {@link HotwordDetectionService}. Use this to provide the hotword models data or other 282 * such data to the trusted process. 283 * @param callbackTimeoutMillis Timeout in milliseconds for the operation to invoke the 284 * statusCallback. 285 * @param statusCallback Use this to return the updated result; the allowed values are 286 * {@link #INITIALIZATION_STATUS_SUCCESS}, 1<->{@link #getMaxCustomInitializationStatus()}. 287 * This is non-null only when the {@link HotwordDetectionService} is being initialized; and it 288 * is null if the state is updated after that. 289 * 290 * @hide 291 */ 292 @SystemApi onUpdateState( @ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @DurationMillisLong long callbackTimeoutMillis, @Nullable IntConsumer statusCallback)293 public void onUpdateState( 294 @Nullable PersistableBundle options, 295 @Nullable SharedMemory sharedMemory, 296 @DurationMillisLong long callbackTimeoutMillis, 297 @Nullable IntConsumer statusCallback) {} 298 299 /** 300 * Called when the {@link VoiceInteractionService} requests that this service 301 * {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly 302 * from the device microphone. 303 * <p> 304 * On successful detection of a hotword, call 305 * {@link Callback#onDetected(HotwordDetectedResult)}. 306 * 307 * @param callback The callback to use for responding to the detection request. 308 * {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here. 309 */ onDetect(@onNull Callback callback)310 public void onDetect(@NonNull Callback callback) { 311 // TODO: Add a helpful error message. 312 throw new UnsupportedOperationException(); 313 } 314 315 /** 316 * Called when the {@link VoiceInteractionService} requests that this service 317 * {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat, 318 * PersistableBundle)} run} hotword recognition on audio coming from an external connected 319 * microphone. 320 * <p> 321 * Upon invoking the {@code callback}, the system closes {@code audioStream} and sends the 322 * detection result to the {@link HotwordDetector.Callback hotword detector}. 323 * 324 * @param audioStream Stream containing audio bytes returned from a microphone 325 * @param audioFormat Format of the supplied audio 326 * @param options Options supporting detection, such as configuration specific to the source of 327 * the audio, provided through 328 * {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat, 329 * PersistableBundle)}. 330 * @param callback The callback to use for responding to the detection request. 331 */ onDetect( @onNull ParcelFileDescriptor audioStream, @NonNull AudioFormat audioFormat, @Nullable PersistableBundle options, @NonNull Callback callback)332 public void onDetect( 333 @NonNull ParcelFileDescriptor audioStream, 334 @NonNull AudioFormat audioFormat, 335 @Nullable PersistableBundle options, 336 @NonNull Callback callback) { 337 // TODO: Add a helpful error message. 338 throw new UnsupportedOperationException(); 339 } 340 onUpdateStateInternal(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, IRemoteCallback callback)341 private void onUpdateStateInternal(@Nullable PersistableBundle options, 342 @Nullable SharedMemory sharedMemory, IRemoteCallback callback) { 343 IntConsumer intConsumer = null; 344 if (callback != null) { 345 intConsumer = 346 value -> { 347 if (value > getMaxCustomInitializationStatus()) { 348 throw new IllegalArgumentException( 349 "The initialization status is invalid for " + value); 350 } 351 try { 352 Bundle status = new Bundle(); 353 status.putInt(KEY_INITIALIZATION_STATUS, value); 354 callback.sendResult(status); 355 } catch (RemoteException e) { 356 throw e.rethrowFromSystemServer(); 357 } 358 }; 359 } 360 onUpdateState(options, sharedMemory, UPDATE_TIMEOUT_MILLIS, intConsumer); 361 } 362 363 /** 364 * Called when the {@link VoiceInteractionService} 365 * {@link HotwordDetector#stopRecognition() requests} that hotword recognition be stopped. 366 * <p> 367 * Any open {@link android.media.AudioRecord} should be closed here. 368 */ onStopDetection()369 public void onStopDetection() { 370 } 371 372 /** 373 * Callback for returning the detection result. 374 * 375 * @hide 376 */ 377 @SystemApi 378 public static final class Callback { 379 // TODO: need to make sure we don't store remote references, but not a high priority. 380 private final IDspHotwordDetectionCallback mRemoteCallback; 381 Callback(IDspHotwordDetectionCallback remoteCallback)382 private Callback(IDspHotwordDetectionCallback remoteCallback) { 383 mRemoteCallback = remoteCallback; 384 } 385 386 /** 387 * Informs the {@link HotwordDetector} that the keyphrase was detected. 388 * 389 * @param result Info about the detection result. This is provided to the 390 * {@link HotwordDetector}. 391 */ onDetected(@onNull HotwordDetectedResult result)392 public void onDetected(@NonNull HotwordDetectedResult result) { 393 requireNonNull(result); 394 final PersistableBundle persistableBundle = result.getExtras(); 395 if (!persistableBundle.isEmpty() && HotwordDetectedResult.getParcelableSize( 396 persistableBundle) > HotwordDetectedResult.getMaxBundleSize()) { 397 throw new IllegalArgumentException( 398 "The bundle size of result is larger than max bundle size (" 399 + HotwordDetectedResult.getMaxBundleSize() 400 + ") of HotwordDetectedResult"); 401 } 402 try { 403 mRemoteCallback.onDetected(result); 404 } catch (RemoteException e) { 405 throw e.rethrowFromSystemServer(); 406 } 407 } 408 409 /** 410 * Informs the {@link HotwordDetector} that the keyphrase was not detected. 411 * <p> 412 * This cannot not be used when recognition is done through 413 * {@link #onDetect(ParcelFileDescriptor, AudioFormat, Callback)}. 414 * 415 * @param result Info about the second stage detection result. This is provided to 416 * the {@link HotwordDetector}. 417 */ onRejected(@onNull HotwordRejectedResult result)418 public void onRejected(@NonNull HotwordRejectedResult result) { 419 requireNonNull(result); 420 try { 421 mRemoteCallback.onRejected(result); 422 } catch (RemoteException e) { 423 throw e.rethrowFromSystemServer(); 424 } 425 } 426 } 427 } 428