1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.test.voiceinteraction;
18 
19 import android.app.ActivityManager;
20 import android.app.VoiceInteractor;
21 import android.app.assist.AssistContent;
22 import android.app.assist.AssistStructure;
23 import android.content.Context;
24 import android.content.Intent;
25 import android.graphics.Bitmap;
26 import android.net.Uri;
27 import android.os.Bundle;
28 import android.service.voice.VoiceInteractionSession;
29 import android.util.Log;
30 import android.view.View;
31 import android.widget.Button;
32 import android.widget.CheckBox;
33 import android.widget.ImageView;
34 import android.widget.TextView;
35 
36 public class MainInteractionSession extends VoiceInteractionSession
37         implements View.OnClickListener {
38     static final String TAG = "MainInteractionSession";
39 
40     Intent mStartIntent;
41     View mContentView;
42     AssistVisualizer mAssistVisualizer;
43     View mTopContent;
44     View mBottomContent;
45     TextView mText;
46     Button mTreeButton;
47     Button mTextButton;
48     Button mStartButton;
49     CheckBox mOptionsCheck;
50     View mOptionsContainer;
51     CheckBox mDisallowAssist;
52     CheckBox mDisallowScreenshot;
53     TextView mOptionsText;
54     ImageView mScreenshot;
55     ImageView mFullScreenshot;
56     Button mConfirmButton;
57     Button mCompleteButton;
58     Button mAbortButton;
59 
60     Bundle mAssistData;
61     AssistStructure mAssistStructure;
62     AssistContent mAssistContent;
63 
64     static final int STATE_IDLE = 0;
65     static final int STATE_LAUNCHING = 1;
66     static final int STATE_CONFIRM = 2;
67     static final int STATE_PICK_OPTION = 3;
68     static final int STATE_COMMAND = 4;
69     static final int STATE_ABORT_VOICE = 5;
70     static final int STATE_COMPLETE_VOICE = 6;
71     static final int STATE_DONE = 7;
72 
73     int mState = STATE_IDLE;
74     VoiceInteractor.PickOptionRequest.Option[] mPendingOptions;
75     CharSequence mPendingPrompt;
76     Request mPendingRequest;
77     int mCurrentTask = -1;
78     int mShowFlags;
79 
MainInteractionSession(Context context)80     MainInteractionSession(Context context) {
81         super(context);
82     }
83 
84     @Override
onCreate()85     public void onCreate() {
86         super.onCreate();
87         ActivityManager am = getContext().getSystemService(ActivityManager.class);
88         am.setWatchHeapLimit(40 * 1024 * 1024);
89     }
90 
91     @Override
onShow(Bundle args, int showFlags)92     public void onShow(Bundle args, int showFlags) {
93         super.onShow(args, showFlags);
94         mShowFlags = showFlags;
95         Log.i(TAG, "onShow: flags=0x" + Integer.toHexString(showFlags) + " args=" + args);
96         mState = STATE_IDLE;
97         mStartIntent = args != null ? (Intent)args.getParcelable("intent") : null;
98         if (mStartIntent == null) {
99             mStartIntent = new Intent(getContext(), TestInteractionActivity.class);
100         }
101         if (mAssistVisualizer != null) {
102             mAssistVisualizer.clearAssistData();
103         }
104         onHandleScreenshot(null);
105         updateState();
106         refreshOptions();
107     }
108 
109     @Override
onHide()110     public void onHide() {
111         super.onHide();
112         if (mAssistVisualizer != null) {
113             mAssistVisualizer.clearAssistData();
114         }
115         mState = STATE_DONE;
116         updateState();
117     }
118 
119     @Override
onCreateContentView()120     public View onCreateContentView() {
121         mContentView = getLayoutInflater().inflate(R.layout.voice_interaction_session, null);
122         mAssistVisualizer = (AssistVisualizer)mContentView.findViewById(R.id.assist_visualizer);
123         if (mAssistStructure != null) {
124             mAssistVisualizer.setAssistStructure(mAssistStructure);
125         }
126         mTopContent = mContentView.findViewById(R.id.top_content);
127         mBottomContent = mContentView.findViewById(R.id.bottom_content);
128         mText = (TextView)mContentView.findViewById(R.id.text);
129         mTreeButton = (Button)mContentView.findViewById(R.id.do_tree);
130         mTreeButton.setOnClickListener(this);
131         mTextButton = (Button)mContentView.findViewById(R.id.do_text);
132         mTextButton.setOnClickListener(this);
133         mStartButton = (Button)mContentView.findViewById(R.id.start);
134         mStartButton.setOnClickListener(this);
135         mScreenshot = (ImageView)mContentView.findViewById(R.id.screenshot);
136         mScreenshot.setOnClickListener(this);
137         mFullScreenshot = (ImageView)mContentView.findViewById(R.id.full_screenshot);
138         mOptionsCheck = (CheckBox)mContentView.findViewById(R.id.show_options);
139         mOptionsCheck.setOnClickListener(this);
140         mOptionsContainer = mContentView.findViewById(R.id.options);
141         mDisallowAssist = (CheckBox)mContentView.findViewById(R.id.disallow_structure);
142         mDisallowAssist.setOnClickListener(this);
143         mDisallowScreenshot = (CheckBox)mContentView.findViewById(R.id.disallow_screenshot);
144         mDisallowScreenshot.setOnClickListener(this);
145         mOptionsText = (TextView)mContentView.findViewById(R.id.options_text);
146         mConfirmButton = (Button)mContentView.findViewById(R.id.confirm);
147         mConfirmButton.setOnClickListener(this);
148         mCompleteButton = (Button)mContentView.findViewById(R.id.complete);
149         mCompleteButton.setOnClickListener(this);
150         mAbortButton = (Button)mContentView.findViewById(R.id.abort);
151         mAbortButton.setOnClickListener(this);
152         refreshOptions();
153         return mContentView;
154     }
155 
refreshOptions()156     void refreshOptions() {
157         if (mOptionsContainer != null) {
158             if (mOptionsCheck.isChecked()) {
159                 mOptionsContainer.setVisibility(View.VISIBLE);
160                 int flags = getDisabledShowContext();
161                 mDisallowAssist.setChecked((flags & SHOW_WITH_ASSIST) != 0);
162                 mDisallowScreenshot.setChecked((flags & SHOW_WITH_SCREENSHOT) != 0);
163                 int disabled = getUserDisabledShowContext();
164                 mOptionsText.setText("Disabled: 0x" + Integer.toHexString(disabled));
165             } else {
166                 mOptionsContainer.setVisibility(View.GONE);
167             }
168         }
169     }
170 
onHandleAssist(Bundle assistBundle)171     public void onHandleAssist(Bundle assistBundle) {
172     }
173 
logAssistContentAndData(AssistContent content, Bundle data)174     private void logAssistContentAndData(AssistContent content, Bundle data) {
175         if (content != null) {
176             Log.i(TAG, "Assist intent: " + content.getIntent());
177             Log.i(TAG, "Assist intent from app: " + content.isAppProvidedIntent());
178             Log.i(TAG, "Assist clipdata: " + content.getClipData());
179             Log.i(TAG, "Assist structured data: " + content.getStructuredData());
180             Log.i(TAG, "Assist web uri: " + content.getWebUri());
181             Log.i(TAG, "Assist web uri from app: " + content.isAppProvidedWebUri());
182             Log.i(TAG, "Assist extras: " + content.getExtras());
183         }
184         if (data != null) {
185             Uri referrer = data.getParcelable(Intent.EXTRA_REFERRER);
186             if (referrer != null) {
187                 Log.i(TAG, "Referrer: " + referrer);
188             }
189         }
190     }
191 
192     @Override
onHandleAssist(Bundle data, AssistStructure structure, AssistContent content)193     public void onHandleAssist(Bundle data, AssistStructure structure, AssistContent content) {
194         mAssistData = data;
195         mAssistStructure = structure;
196         mAssistContent = content;
197         if (mAssistVisualizer != null) {
198             if (mAssistStructure != null) {
199                 mAssistVisualizer.setAssistStructure(mAssistStructure);
200             } else {
201                 mAssistVisualizer.clearAssistData();
202             }
203         }
204         logAssistContentAndData(content, data);
205     }
206 
207     @Override
onHandleAssistSecondary(final Bundle data, final AssistStructure structure, final AssistContent content, int index, int count)208     public void onHandleAssistSecondary(final Bundle data, final AssistStructure structure,
209             final AssistContent content, int index, int count) {
210         Log.i(TAG, "Got secondary activity assist data " + index + " of " + count);
211         Log.i(TAG, "Showing assist structure after a few seconds...");
212         mContentView.postDelayed(new Runnable() {
213             public void run() {
214                 onHandleAssist(data, structure, content);
215             }
216         }, 2000 * index);
217     }
218 
219     @Override
onHandleScreenshot(Bitmap screenshot)220     public void onHandleScreenshot(Bitmap screenshot) {
221         if (mScreenshot != null) {
222             if (screenshot != null) {
223                 mScreenshot.setImageBitmap(screenshot);
224                 mScreenshot.setAdjustViewBounds(true);
225                 mScreenshot.setMaxWidth(screenshot.getWidth() / 3);
226                 mScreenshot.setMaxHeight(screenshot.getHeight() / 3);
227                 mFullScreenshot.setImageBitmap(screenshot);
228             } else {
229                 mScreenshot.setImageDrawable(null);
230                 mFullScreenshot.setImageDrawable(null);
231             }
232         }
233     }
234 
updateState()235     void updateState() {
236         if (mTopContent == null) {
237             return;
238         }
239         if (mState == STATE_IDLE) {
240             mTopContent.setVisibility(View.VISIBLE);
241             mBottomContent.setVisibility(View.GONE);
242             mAssistVisualizer.setVisibility(View.VISIBLE);
243         } else if (mState == STATE_DONE) {
244             mTopContent.setVisibility(View.GONE);
245             mBottomContent.setVisibility(View.GONE);
246             mAssistVisualizer.setVisibility(View.GONE);
247         } else {
248             mTopContent.setVisibility(View.GONE);
249             mBottomContent.setVisibility(View.VISIBLE);
250             mAssistVisualizer.setVisibility(View.GONE);
251         }
252         mStartButton.setEnabled(mState == STATE_IDLE);
253         mConfirmButton.setEnabled(mState == STATE_CONFIRM || mState == STATE_PICK_OPTION
254                 || mState == STATE_COMMAND);
255         mAbortButton.setEnabled(mState == STATE_ABORT_VOICE);
256         mCompleteButton.setEnabled(mState == STATE_COMPLETE_VOICE);
257     }
258 
onClick(View v)259     public void onClick(View v) {
260         if (v == mTreeButton) {
261             if (mAssistVisualizer != null) {
262                 logAssistContentAndData(mAssistContent, mAssistData);
263                 mAssistVisualizer.logTree();
264             }
265         } else if (v == mTextButton) {
266             if (mAssistVisualizer != null) {
267                 mAssistVisualizer.logText();
268             }
269         } else if (v == mOptionsCheck) {
270             refreshOptions();
271         } else if (v == mDisallowAssist) {
272             int flags = getDisabledShowContext();
273             if (mDisallowAssist.isChecked()) {
274                 flags |= SHOW_WITH_ASSIST;
275             } else {
276                 flags &= ~SHOW_WITH_ASSIST;
277             }
278             setDisabledShowContext(flags);
279         } else if (v == mDisallowScreenshot) {
280             int flags = getDisabledShowContext();
281             if (mDisallowScreenshot.isChecked()) {
282                 flags |= SHOW_WITH_SCREENSHOT;
283             } else {
284                 flags &= ~SHOW_WITH_SCREENSHOT;
285             }
286             setDisabledShowContext(flags);
287         } else if (v == mStartButton) {
288             mState = STATE_LAUNCHING;
289             updateState();
290             startVoiceActivity(mStartIntent);
291         } else if (v == mConfirmButton) {
292             if (mPendingRequest instanceof ConfirmationRequest) {
293                 ((ConfirmationRequest)mPendingRequest).sendConfirmationResult(true, null);
294                 mPendingRequest = null;
295                 mState = STATE_LAUNCHING;
296             } else if (mPendingRequest instanceof PickOptionRequest) {
297                 PickOptionRequest pick = (PickOptionRequest)mPendingRequest;
298                 int numReturn = mPendingOptions.length/2;
299                 if (numReturn <= 0) {
300                     numReturn = 1;
301                 }
302                 VoiceInteractor.PickOptionRequest.Option[] picked
303                         = new VoiceInteractor.PickOptionRequest.Option[numReturn];
304                 for (int i=0; i<picked.length; i++) {
305                     picked[i] = mPendingOptions[i*2];
306                 }
307                 mPendingOptions = picked;
308                 if (picked.length <= 1) {
309                     pick.sendPickOptionResult(picked, null);
310                     mPendingRequest = null;
311                     mState = STATE_LAUNCHING;
312                 } else {
313                     pick.sendIntermediatePickOptionResult(picked, null);
314                     updatePickText();
315                 }
316             } else if (mPendingRequest instanceof CommandRequest) {
317                 Bundle result = new Bundle();
318                 result.putString("key", "a result!");
319                 ((CommandRequest)mPendingRequest).sendResult(result);
320                 mPendingRequest = null;
321                 mState = STATE_LAUNCHING;
322             }
323         } else if (v == mAbortButton && mPendingRequest instanceof AbortVoiceRequest) {
324             ((AbortVoiceRequest)mPendingRequest).sendAbortResult(null);
325             mPendingRequest = null;
326         } else if (v == mCompleteButton && mPendingRequest instanceof CompleteVoiceRequest) {
327             ((CompleteVoiceRequest)mPendingRequest).sendCompleteResult(null);
328             mPendingRequest = null;
329         } else if (v == mScreenshot) {
330             if (mFullScreenshot.getVisibility() != View.VISIBLE) {
331                 mFullScreenshot.setVisibility(View.VISIBLE);
332             } else {
333                 mFullScreenshot.setVisibility(View.INVISIBLE);
334             }
335         }
336         updateState();
337     }
338 
339     @Override
onComputeInsets(Insets outInsets)340     public void onComputeInsets(Insets outInsets) {
341         super.onComputeInsets(outInsets);
342         if (mState != STATE_IDLE) {
343             outInsets.contentInsets.top = mBottomContent.getTop();
344             outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT;
345         } else if ((mShowFlags & SHOW_SOURCE_ACTIVITY) != 0) {
346             outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT;
347         }
348     }
349 
350     @Override
onTaskStarted(Intent intent, int taskId)351     public void onTaskStarted(Intent intent, int taskId) {
352         super.onTaskStarted(intent, taskId);
353         mCurrentTask = taskId;
354     }
355 
356     @Override
onTaskFinished(Intent intent, int taskId)357     public void onTaskFinished(Intent intent, int taskId) {
358         super.onTaskFinished(intent, taskId);
359         if (mCurrentTask == taskId) {
360             mCurrentTask = -1;
361         }
362     }
363 
364     @Override
onLockscreenShown()365     public void onLockscreenShown() {
366         if (mCurrentTask < 0) {
367             hide();
368         }
369     }
370 
371     @Override
onGetSupportedCommands(String[] commands)372     public boolean[] onGetSupportedCommands(String[] commands) {
373         boolean[] res = new boolean[commands.length];
374         for (int i=0; i<commands.length; i++) {
375             if ("com.android.test.voiceinteraction.COMMAND".equals(commands[i])) {
376                 res[i] = true;
377             }
378         }
379         return res;
380     }
381 
setPrompt(VoiceInteractor.Prompt prompt)382     void setPrompt(VoiceInteractor.Prompt prompt) {
383         if (prompt == null) {
384             mText.setText("(null)");
385             mPendingPrompt = "";
386         } else {
387             mText.setText(prompt.getVisualPrompt());
388             mPendingPrompt = prompt.getVisualPrompt();
389         }
390     }
391 
392     @Override
onRequestConfirmation(ConfirmationRequest request)393     public void onRequestConfirmation(ConfirmationRequest request) {
394         Log.i(TAG, "onConfirm: prompt=" + request.getVoicePrompt() + " extras="
395                 + request.getExtras());
396         setPrompt(request.getVoicePrompt());
397         mConfirmButton.setText("Confirm");
398         mPendingRequest = request;
399         mState = STATE_CONFIRM;
400         updateState();
401     }
402 
403     @Override
onRequestPickOption(PickOptionRequest request)404     public void onRequestPickOption(PickOptionRequest request) {
405         Log.i(TAG, "onPickOption: prompt=" + request.getVoicePrompt() + " options="
406                 + request.getOptions() + " extras=" + request.getExtras());
407         mConfirmButton.setText("Pick Option");
408         mPendingRequest = request;
409         setPrompt(request.getVoicePrompt());
410         mPendingOptions = request.getOptions();
411         mState = STATE_PICK_OPTION;
412         updatePickText();
413         updateState();
414     }
415 
updatePickText()416     void updatePickText() {
417         StringBuilder sb = new StringBuilder();
418         sb.append(mPendingPrompt);
419         sb.append(": ");
420         for (int i=0; i<mPendingOptions.length; i++) {
421             if (i > 0) {
422                 sb.append(", ");
423             }
424             sb.append(mPendingOptions[i].getLabel());
425         }
426         mText.setText(sb.toString());
427     }
428 
429     @Override
onRequestCompleteVoice(CompleteVoiceRequest request)430     public void onRequestCompleteVoice(CompleteVoiceRequest request) {
431         Log.i(TAG, "onCompleteVoice: message=" + request.getVoicePrompt() + " extras="
432                 + request.getExtras());
433         setPrompt(request.getVoicePrompt());
434         mPendingRequest = request;
435         mState = STATE_COMPLETE_VOICE;
436         updateState();
437     }
438 
439     @Override
onRequestAbortVoice(AbortVoiceRequest request)440     public void onRequestAbortVoice(AbortVoiceRequest request) {
441         Log.i(TAG, "onAbortVoice: message=" + request.getVoicePrompt() + " extras="
442                 + request.getExtras());
443         setPrompt(request.getVoicePrompt());
444         mPendingRequest = request;
445         mState = STATE_ABORT_VOICE;
446         updateState();
447     }
448 
449     @Override
onRequestCommand(CommandRequest request)450     public void onRequestCommand(CommandRequest request) {
451         Bundle extras = request.getExtras();
452         if (extras != null) {
453             extras.getString("arg");
454         }
455         Log.i(TAG, "onCommand: command=" + request.getCommand() + " extras=" + extras);
456         mText.setText("Command: " + request.getCommand() + ", " + extras);
457         mConfirmButton.setText("Finish Command");
458         mPendingRequest = request;
459         mState = STATE_COMMAND;
460         updateState();
461     }
462 
463     @Override
onCancelRequest(Request request)464     public void onCancelRequest(Request request) {
465         Log.i(TAG, "onCancel");
466         if (mPendingRequest == request) {
467             mPendingRequest = null;
468             mState = STATE_LAUNCHING;
469             updateState();
470         }
471         request.cancel();
472     }
473 }
474