1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server.voiceinteraction;
18 
19 import static android.service.voice.VisualQueryDetectionServiceFailure.ERROR_CODE_ILLEGAL_ATTENTION_STATE;
20 import static android.service.voice.VisualQueryDetectionServiceFailure.ERROR_CODE_ILLEGAL_STREAMING_STATE;
21 
22 import android.annotation.NonNull;
23 import android.annotation.Nullable;
24 import android.content.Context;
25 import android.media.AudioFormat;
26 import android.media.permission.Identity;
27 import android.os.IBinder;
28 import android.os.ParcelFileDescriptor;
29 import android.os.PersistableBundle;
30 import android.os.RemoteException;
31 import android.os.SharedMemory;
32 import android.service.voice.IDetectorSessionVisualQueryDetectionCallback;
33 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
34 import android.service.voice.ISandboxedDetectionService;
35 import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
36 import android.service.voice.VisualQueryDetectionServiceFailure;
37 import android.util.Slog;
38 
39 import com.android.internal.app.IHotwordRecognitionStatusCallback;
40 import com.android.internal.app.IVisualQueryDetectionAttentionListener;
41 import com.android.server.voiceinteraction.VoiceInteractionManagerServiceImpl.DetectorRemoteExceptionListener;
42 
43 import java.io.PrintWriter;
44 import java.util.Objects;
45 import java.util.concurrent.ScheduledExecutorService;
46 
47 /**
48  * A class that provides visual query detector to communicate with the {@link
49  * android.service.voice.VisualQueryDetectionService}.
50  *
51  * This class can handle the visual query detection whose detector is created by using
52  * {@link android.service.voice.VoiceInteractionService#createVisualQueryDetector(PersistableBundle
53  * ,SharedMemory, HotwordDetector.Callback)}.
54  */
55 final class VisualQueryDetectorSession extends DetectorSession {
56 
57     private static final String TAG = "VisualQueryDetectorSession";
58     private IVisualQueryDetectionAttentionListener mAttentionListener;
59     private boolean mEgressingData;
60     private boolean mQueryStreaming;
61 
62     //TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.
VisualQueryDetectorSession( @onNull HotwordDetectionConnection.ServiceConnection remoteService, @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, @NonNull DetectorRemoteExceptionListener listener)63     VisualQueryDetectorSession(
64             @NonNull HotwordDetectionConnection.ServiceConnection remoteService,
65             @NonNull Object lock, @NonNull Context context, @NonNull IBinder token,
66             @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid,
67             Identity voiceInteractorIdentity,
68             @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging,
69             @NonNull DetectorRemoteExceptionListener listener) {
70         super(remoteService, lock, context, token, callback,
71                 voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,
72                 logging, listener);
73         mEgressingData = false;
74         mQueryStreaming = false;
75         mAttentionListener = null;
76         // TODO: handle notify RemoteException to client
77     }
78 
79     @Override
80     @SuppressWarnings("GuardedBy")
informRestartProcessLocked()81     void informRestartProcessLocked() {
82         Slog.v(TAG, "informRestartProcessLocked");
83         mUpdateStateAfterStartFinished.set(false);
84         try {
85             mCallback.onProcessRestarted();
86         } catch (RemoteException e) {
87             Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);
88             notifyOnDetectorRemoteException();
89         }
90     }
91 
setVisualQueryDetectionAttentionListenerLocked( @ullable IVisualQueryDetectionAttentionListener listener)92     void setVisualQueryDetectionAttentionListenerLocked(
93             @Nullable IVisualQueryDetectionAttentionListener listener) {
94         mAttentionListener = listener;
95     }
96 
97     @SuppressWarnings("GuardedBy")
startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback)98     boolean startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
99         if (DEBUG) {
100             Slog.d(TAG, "startPerceivingLocked");
101         }
102 
103         IDetectorSessionVisualQueryDetectionCallback internalCallback =
104                 new IDetectorSessionVisualQueryDetectionCallback.Stub(){
105 
106             @Override
107             public void onAttentionGained() {
108                 Slog.v(TAG, "BinderCallback#onAttentionGained");
109                 mEgressingData = true;
110                 if (mAttentionListener == null) {
111                     return;
112                 }
113                 try {
114                     mAttentionListener.onAttentionGained();
115                 } catch (RemoteException e) {
116                     Slog.e(TAG, "Error delivering attention gained event.", e);
117                     try {
118                         callback.onVisualQueryDetectionServiceFailure(
119                                 new VisualQueryDetectionServiceFailure(
120                                         ERROR_CODE_ILLEGAL_ATTENTION_STATE,
121                                         "Attention listener failed to switch to GAINED state."));
122                     } catch (RemoteException ex) {
123                         Slog.v(TAG, "Fail to call onVisualQueryDetectionServiceFailure");
124                     }
125                     return;
126                 }
127             }
128 
129             @Override
130             public void onAttentionLost() {
131                 Slog.v(TAG, "BinderCallback#onAttentionLost");
132                 mEgressingData = false;
133                 if (mAttentionListener == null) {
134                     return;
135                 }
136                 try {
137                     mAttentionListener.onAttentionLost();
138                 } catch (RemoteException e) {
139                     Slog.e(TAG, "Error delivering attention lost event.", e);
140                     try {
141                         callback.onVisualQueryDetectionServiceFailure(
142                                 new VisualQueryDetectionServiceFailure(
143                                         ERROR_CODE_ILLEGAL_ATTENTION_STATE,
144                                         "Attention listener failed to switch to LOST state."));
145                     } catch (RemoteException ex) {
146                         Slog.v(TAG, "Fail to call onVisualQueryDetectionServiceFailure");
147                     }
148                     return;
149                 }
150             }
151 
152             @Override
153             public void onQueryDetected(@NonNull String partialQuery) throws RemoteException {
154                 Objects.requireNonNull(partialQuery);
155                 Slog.v(TAG, "BinderCallback#onQueryDetected");
156                 if (!mEgressingData) {
157                     Slog.v(TAG, "Query should not be egressed within the unattention state.");
158                     callback.onVisualQueryDetectionServiceFailure(
159                             new VisualQueryDetectionServiceFailure(
160                                     ERROR_CODE_ILLEGAL_STREAMING_STATE,
161                                     "Cannot stream queries without attention signals."));
162                     return;
163                 }
164                 mQueryStreaming = true;
165                 callback.onQueryDetected(partialQuery);
166                 Slog.i(TAG, "Egressed from visual query detection process.");
167             }
168 
169             @Override
170             public void onQueryFinished() throws RemoteException {
171                 Slog.v(TAG, "BinderCallback#onQueryFinished");
172                 if (!mQueryStreaming) {
173                     Slog.v(TAG, "Query streaming state signal FINISHED is block since there is"
174                             + " no active query being streamed.");
175                     callback.onVisualQueryDetectionServiceFailure(
176                             new VisualQueryDetectionServiceFailure(
177                                     ERROR_CODE_ILLEGAL_STREAMING_STATE,
178                                     "Cannot send FINISHED signal with no query streamed."));
179                     return;
180                 }
181                 callback.onQueryFinished();
182                 mQueryStreaming = false;
183             }
184 
185             @Override
186             public void onQueryRejected() throws RemoteException {
187                 Slog.v(TAG, "BinderCallback#onQueryRejected");
188                 if (!mQueryStreaming) {
189                     Slog.v(TAG, "Query streaming state signal REJECTED is block since there is"
190                             + " no active query being streamed.");
191                     callback.onVisualQueryDetectionServiceFailure(
192                             new VisualQueryDetectionServiceFailure(
193                                     ERROR_CODE_ILLEGAL_STREAMING_STATE,
194                                     "Cannot send REJECTED signal with no query streamed."));
195                     return;
196                 }
197                 callback.onQueryRejected();
198                 mQueryStreaming = false;
199             }
200         };
201         return mRemoteDetectionService.run(
202                 service -> service.detectWithVisualSignals(internalCallback));
203     }
204 
205     @SuppressWarnings("GuardedBy")
stopPerceivingLocked()206     boolean stopPerceivingLocked() {
207         if (DEBUG) {
208             Slog.d(TAG, "stopPerceivingLocked");
209         }
210         return mRemoteDetectionService.run(ISandboxedDetectionService::stopDetection);
211     }
212 
213     @Override
startListeningFromExternalSourceLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, IMicrophoneHotwordDetectionVoiceInteractionCallback callback)214      void startListeningFromExternalSourceLocked(
215             ParcelFileDescriptor audioStream,
216             AudioFormat audioFormat,
217             @Nullable PersistableBundle options,
218             IMicrophoneHotwordDetectionVoiceInteractionCallback callback)
219              throws UnsupportedOperationException {
220         throw new UnsupportedOperationException("HotwordDetectionService method"
221                 + " should not be called from VisualQueryDetectorSession.");
222     }
223 
224 
225     @SuppressWarnings("GuardedBy")
dumpLocked(String prefix, PrintWriter pw)226     public void dumpLocked(String prefix, PrintWriter pw) {
227         super.dumpLocked(prefix, pw);
228         pw.print(prefix);
229     }
230 }
231 
232 
233