/* * Copyright (C) 2019 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.server.soundtrigger_middleware; import android.annotation.NonNull; import android.annotation.Nullable; import android.media.soundtrigger.ModelParameterRange; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; import android.media.soundtrigger_middleware.ISoundTriggerCallback; import android.media.soundtrigger_middleware.ISoundTriggerModule; import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.Binder; import android.os.IBinder; import android.os.RemoteException; import android.util.Slog; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; /** * This is an implementation of a single module of the ISoundTriggerMiddlewareService interface, * exposing itself through the {@link ISoundTriggerModule} interface, possibly to multiple separate * clients. *

* Typical usage is to query the module capabilities using {@link #getProperties()} and then to use * the module through an {@link ISoundTriggerModule} instance, obtained via {@link * #attach(ISoundTriggerCallback)}. Every such interface is its own session and state is not shared * between sessions (i.e. cannot use a handle obtained from one session through another). *

* Important conventions: *

* * @hide */ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.GlobalCallback { static private final String TAG = "SoundTriggerModule"; @NonNull private final HalFactory mHalFactory; @NonNull private ISoundTriggerHal mHalService; @NonNull private final SoundTriggerMiddlewareImpl.AudioSessionProvider mAudioSessionProvider; private final Set mActiveSessions = new HashSet<>(); private Properties mProperties; /** * Ctor. * * @param halFactory - A factory for the underlying HAL driver. * @param audioSessionProvider - Creates a session token + device id/port pair used to * associate recognition events with the audio stream used to access data. */ SoundTriggerModule(@NonNull HalFactory halFactory, @NonNull SoundTriggerMiddlewareImpl.AudioSessionProvider audioSessionProvider) { mHalFactory = Objects.requireNonNull(halFactory); mAudioSessionProvider = Objects.requireNonNull(audioSessionProvider); attachToHal(); } /** * Establish a client session with this module. * * This module may be shared by multiple clients, each will get its own session. While resources * are shared between the clients, each session has its own state and data should not be shared * across sessions. * * @param callback The client callback, which will be used for all messages. This is a oneway * callback, so will never block, throw an unchecked exception or return a * value. * @return The interface through which this module can be controlled. */ synchronized @NonNull ISoundTriggerModule attach(@NonNull ISoundTriggerCallback callback) { Session session = new Session(callback); mActiveSessions.add(session); return session; } /** * Query the module's properties. * * @return The properties structure. */ synchronized @NonNull Properties getProperties() { return mProperties; } @Override public void binderDied() { Slog.w(TAG, "Underlying HAL driver died."); List callbacks; synchronized (this) { callbacks = new ArrayList<>(mActiveSessions.size()); for (Session session : mActiveSessions) { callbacks.add(session.moduleDied()); } mActiveSessions.clear(); reset(); } // Trigger the callbacks outside of the lock to avoid deadlocks. for (ISoundTriggerCallback callback : callbacks) { try { callback.onModuleDied(); } catch (RemoteException e) { throw e.rethrowAsRuntimeException(); } } } /** * Resets the transient state of this object. */ private void reset() { mHalService.detach(); attachToHal(); } /** * Attached to the HAL service via factory. */ private void attachToHal() { mHalService = null; while (mHalService == null) { try { mHalService = new SoundTriggerHalEnforcer( new SoundTriggerHalWatchdog( new SoundTriggerDuplicateModelHandler(mHalFactory.create()))); } catch (RuntimeException e) { if (!(e.getCause() instanceof RemoteException)) { throw e; } } } mHalService.linkToDeath(this); mHalService.registerCallback(this); mProperties = mHalService.getProperties(); } /** * Remove session from the list of active sessions. * * @param session The session to remove. */ private void removeSession(@NonNull Session session) { mActiveSessions.remove(session); } @Override public void onResourcesAvailable() { List callbacks; synchronized (this) { callbacks = new ArrayList<>(mActiveSessions.size()); for (Session session : mActiveSessions) { callbacks.add(session.mCallback); } } // Trigger the callbacks outside of the lock to avoid deadlocks. for (ISoundTriggerCallback callback : callbacks) { try { callback.onResourcesAvailable(); } catch (RemoteException e) { throw e.rethrowAsRuntimeException(); } } } /** State of a single sound model. */ private enum ModelState { /** Initial state, until load() is called. */ INIT, /** Model is loaded, but recognition is not active. */ LOADED, /** Model is loaded and recognition is active. */ ACTIVE } /** * A single client session with this module. * * This is the main interface used to interact with this module. */ private class Session implements ISoundTriggerModule { private ISoundTriggerCallback mCallback; private final IBinder mToken = new Binder(); private final Map mLoadedModels = new HashMap<>(); /** * Ctor. * * @param callback The client callback interface. */ private Session(@NonNull ISoundTriggerCallback callback) { mCallback = callback; mHalService.clientAttached(mToken); } @Override public void detach() { synchronized (SoundTriggerModule.this) { if (mCallback == null) { return; } removeSession(this); mCallback = null; mHalService.clientDetached(mToken); } } @Override public int loadModel(@NonNull SoundModel model) { synchronized (SoundTriggerModule.this) { SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession audioSession = mAudioSessionProvider.acquireSession(); try { checkValid(); Model loadedModel = new Model(); return loadedModel.load(model, audioSession); } catch (Exception e) { // We must do this outside the lock, to avoid possible deadlocks with the remote // process that provides the audio sessions, which may also be calling into us. try { mAudioSessionProvider.releaseSession(audioSession.mSessionHandle); } catch (Exception ee) { Slog.e(TAG, "Failed to release session.", ee); } throw e; } } } @Override public int loadPhraseModel(@NonNull PhraseSoundModel model) { synchronized (SoundTriggerModule.this) { SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession audioSession = mAudioSessionProvider.acquireSession(); try { checkValid(); Model loadedModel = new Model(); int result = loadedModel.load(model, audioSession); Slog.d(TAG, String.format("loadPhraseModel()->%d", result)); return result; } catch (Exception e) { // We must do this outside the lock, to avoid possible deadlocks with the remote // process that provides the audio sessions, which may also be calling into us. try { mAudioSessionProvider.releaseSession(audioSession.mSessionHandle); } catch (Exception ee) { Slog.e(TAG, "Failed to release session.", ee); } throw e; } } } @Override public void unloadModel(int modelHandle) { synchronized (SoundTriggerModule.this) { int sessionId; checkValid(); sessionId = mLoadedModels.get(modelHandle).unload(); mAudioSessionProvider.releaseSession(sessionId); } } @Override public IBinder startRecognition(int modelHandle, @NonNull RecognitionConfig config) { synchronized (SoundTriggerModule.this) { checkValid(); return mLoadedModels.get(modelHandle).startRecognition(config); } } @Override public void stopRecognition(int modelHandle) { Model model; synchronized (SoundTriggerModule.this) { checkValid(); model = mLoadedModels.get(modelHandle); } model.stopRecognition(); } @Override public void forceRecognitionEvent(int modelHandle) { synchronized (SoundTriggerModule.this) { checkValid(); mLoadedModels.get(modelHandle).forceRecognitionEvent(); } } @Override public void setModelParameter(int modelHandle, int modelParam, int value) { synchronized (SoundTriggerModule.this) { checkValid(); mLoadedModels.get(modelHandle).setParameter(modelParam, value); } } @Override public int getModelParameter(int modelHandle, int modelParam) { synchronized (SoundTriggerModule.this) { checkValid(); return mLoadedModels.get(modelHandle).getParameter(modelParam); } } @Override @Nullable public ModelParameterRange queryModelParameterSupport(int modelHandle, int modelParam) { synchronized (SoundTriggerModule.this) { checkValid(); return mLoadedModels.get(modelHandle).queryModelParameterSupport(modelParam); } } /** * The underlying module HAL is dead. * @return The client callback that needs to be invoked to notify the client. */ private ISoundTriggerCallback moduleDied() { ISoundTriggerCallback callback = mCallback; mCallback = null; return callback; } private void checkValid() { if (mCallback == null) { throw new RecoverableException(Status.DEAD_OBJECT); } } @Override public @NonNull IBinder asBinder() { throw new UnsupportedOperationException( "This implementation is not intended to be used directly with Binder."); } /** * A single sound model in the system. * * All model-based operations are delegated to this class and implemented here. */ private class Model implements ISoundTriggerHal.ModelCallback { public int mHandle; private ModelState mState = ModelState.INIT; private SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession mSession; private IBinder mRecognitionToken = null; private boolean mIsStopping = false; private @NonNull ModelState getState() { return mState; } private void setState(@NonNull ModelState state) { mState = state; SoundTriggerModule.this.notifyAll(); } private int load(@NonNull SoundModel model, SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession audioSession) { mSession = audioSession; mHandle = mHalService.loadSoundModel(model, this); setState(ModelState.LOADED); mLoadedModels.put(mHandle, this); return mHandle; } private int load(@NonNull PhraseSoundModel model, SoundTriggerMiddlewareImpl.AudioSessionProvider.AudioSession audioSession) { mSession = audioSession; mHandle = mHalService.loadPhraseSoundModel(model, this); setState(ModelState.LOADED); mLoadedModels.put(mHandle, this); return mHandle; } /** * Unloads the model. * @return The audio session handle. */ private int unload() { mHalService.unloadSoundModel(mHandle); mLoadedModels.remove(mHandle); return mSession.mSessionHandle; } private IBinder startRecognition(@NonNull RecognitionConfig config) { if (mIsStopping == true) { throw new RecoverableException(Status.INTERNAL_ERROR, "Race occurred"); } mHalService.startRecognition(mHandle, mSession.mDeviceHandle, mSession.mIoHandle, config); mRecognitionToken = new Binder(); setState(ModelState.ACTIVE); return mRecognitionToken; } private void stopRecognition() { synchronized (SoundTriggerModule.this) { if (getState() == ModelState.LOADED) { // This call is idempotent in order to avoid races. return; } mRecognitionToken = null; mIsStopping = true; } mHalService.stopRecognition(mHandle); synchronized (SoundTriggerModule.this) { mIsStopping = false; setState(ModelState.LOADED); } } /** Request a forced recognition event. Will do nothing if recognition is inactive. */ private void forceRecognitionEvent() { if (getState() != ModelState.ACTIVE) { // This call is idempotent in order to avoid races. return; } mHalService.forceRecognitionEvent(mHandle); } private void setParameter(int modelParam, int value) { mHalService.setModelParameter(mHandle, ConversionUtil.aidl2hidlModelParameter(modelParam), value); } private int getParameter(int modelParam) { return mHalService.getModelParameter(mHandle, ConversionUtil.aidl2hidlModelParameter(modelParam)); } @Nullable private ModelParameterRange queryModelParameterSupport(int modelParam) { return mHalService.queryParameter(mHandle, modelParam); } @Override public void recognitionCallback(int modelHandle, @NonNull RecognitionEventSys event) { ISoundTriggerCallback callback; synchronized (SoundTriggerModule.this) { if (mRecognitionToken == null) { return; } event.token = mRecognitionToken; if (!event.recognitionEvent.recognitionStillActive) { setState(ModelState.LOADED); mRecognitionToken = null; } callback = mCallback; } // The callback must be invoked outside of the lock. try { if (callback != null) { callback.onRecognition(mHandle, event, mSession.mSessionHandle); } } catch (RemoteException e) { // We're not expecting any exceptions here. throw e.rethrowAsRuntimeException(); } } @Override public void phraseRecognitionCallback(int modelHandle, @NonNull PhraseRecognitionEventSys event) { ISoundTriggerCallback callback; synchronized (SoundTriggerModule.this) { if (mRecognitionToken == null) { return; } event.token = mRecognitionToken; if (!event.phraseRecognitionEvent.common.recognitionStillActive) { setState(ModelState.LOADED); mRecognitionToken = null; } callback = mCallback; } // The callback must be invoked outside of the lock. try { if (callback != null) { mCallback.onPhraseRecognition(mHandle, event, mSession.mSessionHandle); } } catch (RemoteException e) { // We're not expecting any exceptions here. throw e.rethrowAsRuntimeException(); } } @Override public void modelUnloaded(int modelHandle) { ISoundTriggerCallback callback; synchronized (SoundTriggerModule.this) { callback = mCallback; } // The callback must be invoked outside of the lock. try { if (callback != null) { callback.onModelUnloaded(modelHandle); } } catch (RemoteException e) { // We're not expecting any exceptions here. throw e.rethrowAsRuntimeException(); } } } } }