1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.hardware.camera2.impl;
18 
19 import android.annotation.NonNull;
20 import android.annotation.Nullable;
21 import android.annotation.RequiresPermission;
22 import android.content.Context;
23 import android.graphics.ImageFormat;
24 import android.graphics.SurfaceTexture;
25 import android.hardware.HardwareBuffer;
26 import android.hardware.SyncFence;
27 import android.hardware.camera2.CameraAccessException;
28 import android.hardware.camera2.CameraCaptureSession;
29 import android.hardware.camera2.CameraCharacteristics;
30 import android.hardware.camera2.CameraDevice;
31 import android.hardware.camera2.CameraExtensionCharacteristics;
32 import android.hardware.camera2.CameraExtensionSession;
33 import android.hardware.camera2.CaptureFailure;
34 import android.hardware.camera2.CaptureRequest;
35 import android.hardware.camera2.CaptureResult;
36 import android.hardware.camera2.TotalCaptureResult;
37 import android.hardware.camera2.extension.CaptureBundle;
38 import android.hardware.camera2.extension.CaptureStageImpl;
39 import android.hardware.camera2.extension.ICaptureProcessorImpl;
40 import android.hardware.camera2.extension.IImageCaptureExtenderImpl;
41 import android.hardware.camera2.extension.IInitializeSessionCallback;
42 import android.hardware.camera2.extension.IPreviewExtenderImpl;
43 import android.hardware.camera2.extension.IProcessResultImpl;
44 import android.hardware.camera2.extension.IRequestUpdateProcessorImpl;
45 import android.hardware.camera2.extension.LatencyPair;
46 import android.hardware.camera2.extension.ParcelImage;
47 import android.hardware.camera2.params.DynamicRangeProfiles;
48 import android.hardware.camera2.params.ExtensionSessionConfiguration;
49 import android.hardware.camera2.params.OutputConfiguration;
50 import android.hardware.camera2.params.SessionConfiguration;
51 import android.hardware.camera2.utils.ExtensionSessionStatsAggregator;
52 import android.hardware.camera2.utils.SurfaceUtils;
53 import android.media.Image;
54 import android.media.ImageReader;
55 import android.media.ImageWriter;
56 import android.os.Binder;
57 import android.os.Handler;
58 import android.os.HandlerThread;
59 import android.os.IBinder;
60 import android.os.RemoteException;
61 import android.util.Log;
62 import android.util.LongSparseArray;
63 import android.util.Pair;
64 import android.util.Size;
65 import android.view.Surface;
66 
67 import java.io.Closeable;
68 import java.io.IOException;
69 import java.util.ArrayList;
70 import java.util.Arrays;
71 import java.util.HashMap;
72 import java.util.List;
73 import java.util.Map;
74 import java.util.Set;
75 import java.util.concurrent.Executor;
76 
77 public final class CameraExtensionSessionImpl extends CameraExtensionSession {
78     private static final int PREVIEW_QUEUE_SIZE = 10;
79     private static final String TAG = "CameraExtensionSessionImpl";
80 
81     private final Executor mExecutor;
82     private final CameraDevice mCameraDevice;
83     private final IImageCaptureExtenderImpl mImageExtender;
84     private final IPreviewExtenderImpl mPreviewExtender;
85     private final Handler mHandler;
86     private final HandlerThread mHandlerThread;
87     private final StateCallback mCallbacks;
88     private final List<Size> mSupportedPreviewSizes;
89     private final InitializeSessionHandler mInitializeHandler;
90     private final int mSessionId;
91     private final Set<CaptureRequest.Key> mSupportedRequestKeys;
92     private final Set<CaptureResult.Key> mSupportedResultKeys;
93     private final ExtensionSessionStatsAggregator mStatsAggregator;
94     private IBinder mToken = null;
95     private boolean mCaptureResultsSupported;
96 
97     private CameraCaptureSession mCaptureSession = null;
98     private Surface mCameraRepeatingSurface, mClientRepeatingRequestSurface;
99     private Surface mCameraBurstSurface, mClientCaptureSurface;
100     private Surface mClientPostviewSurface;
101     private ImageReader mRepeatingRequestImageReader = null;
102     private ImageReader mBurstCaptureImageReader = null;
103     private ImageReader mStubCaptureImageReader = null;
104     private ImageWriter mRepeatingRequestImageWriter = null;
105     private CameraOutputImageCallback mRepeatingRequestImageCallback = null;
106     private CameraOutputImageCallback mBurstCaptureImageCallback = null;
107 
108     private CameraExtensionJpegProcessor mImageJpegProcessor = null;
109     private ICaptureProcessorImpl mImageProcessor = null;
110     private CameraExtensionForwardProcessor mPreviewImageProcessor = null;
111     private IRequestUpdateProcessorImpl mPreviewRequestUpdateProcessor = null;
112     private int mPreviewProcessorType = IPreviewExtenderImpl.PROCESSOR_TYPE_NONE;
113 
114     private boolean mInitialized;
115     private boolean mSessionClosed;
116     // Enable/Disable internal preview/(repeating request). Extensions expect
117     // that preview/(repeating request) is enabled and active at any point in time.
118     // In case the client doesn't explicitly enable repeating requests, the framework
119     // will do so internally.
120     private boolean mInternalRepeatingRequestEnabled = true;
121 
122     private final Context mContext;
123 
124     // Lock to synchronize cross-thread access to device public interface
125     final Object mInterfaceLock;
126 
nativeGetSurfaceFormat(Surface surface)127     private static int nativeGetSurfaceFormat(Surface surface) {
128         return SurfaceUtils.getSurfaceFormat(surface);
129     }
130 
131     /**
132      * @hide
133      */
134     @RequiresPermission(android.Manifest.permission.CAMERA)
createCameraExtensionSession( @onNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice, @NonNull Map<String, CameraCharacteristics> characteristicsMap, @NonNull Context ctx, @NonNull ExtensionSessionConfiguration config, int sessionId, @NonNull IBinder token)135     public static CameraExtensionSessionImpl createCameraExtensionSession(
136             @NonNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice,
137             @NonNull Map<String, CameraCharacteristics> characteristicsMap,
138             @NonNull Context ctx,
139             @NonNull ExtensionSessionConfiguration config,
140             int sessionId,
141             @NonNull IBinder token)
142             throws CameraAccessException, RemoteException {
143         String cameraId = cameraDevice.getId();
144         CameraExtensionCharacteristics extensionChars = new CameraExtensionCharacteristics(ctx,
145                 cameraId, characteristicsMap);
146 
147         if (!CameraExtensionCharacteristics.isExtensionSupported(cameraDevice.getId(),
148                 config.getExtension(),
149                 CameraExtensionUtils.getCharacteristicsMapNative(characteristicsMap))) {
150             throw new UnsupportedOperationException("Unsupported extension type: " +
151                     config.getExtension());
152         }
153 
154         if (config.getOutputConfigurations().isEmpty() ||
155                 config.getOutputConfigurations().size() > 2) {
156             throw new IllegalArgumentException("Unexpected amount of output surfaces, received: " +
157                     config.getOutputConfigurations().size() + " expected <= 2");
158         }
159 
160         for (OutputConfiguration c : config.getOutputConfigurations()) {
161             if (c.getDynamicRangeProfile() != DynamicRangeProfiles.STANDARD) {
162                 throw new IllegalArgumentException("Unsupported dynamic range profile: " +
163                         c.getDynamicRangeProfile());
164             }
165             if (c.getStreamUseCase() !=
166                     CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT) {
167                 throw new IllegalArgumentException("Unsupported stream use case: " +
168                         c.getStreamUseCase());
169             }
170         }
171 
172         Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders =
173                 CameraExtensionCharacteristics.initializeExtension(config.getExtension());
174 
175         int suitableSurfaceCount = 0;
176         List<Size> supportedPreviewSizes = extensionChars.getExtensionSupportedSizes(
177                 config.getExtension(), SurfaceTexture.class);
178         Surface repeatingRequestSurface = CameraExtensionUtils.getRepeatingRequestSurface(
179                 config.getOutputConfigurations(), supportedPreviewSizes);
180         if (repeatingRequestSurface != null) {
181             suitableSurfaceCount++;
182         }
183 
184         HashMap<Integer, List<Size>> supportedCaptureSizes = new HashMap<>();
185         for (int format : CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
186             List<Size> supportedSizes = extensionChars.getExtensionSupportedSizes(
187                     config.getExtension(), format);
188             if (supportedSizes != null) {
189                 supportedCaptureSizes.put(format, supportedSizes);
190             }
191         }
192         Surface burstCaptureSurface = CameraExtensionUtils.getBurstCaptureSurface(
193                 config.getOutputConfigurations(), supportedCaptureSizes);
194         if (burstCaptureSurface != null) {
195             suitableSurfaceCount++;
196         }
197 
198         if (suitableSurfaceCount != config.getOutputConfigurations().size()) {
199             throw new IllegalArgumentException("One or more unsupported output surfaces found!");
200         }
201 
202         Surface postviewSurface = null;
203         if (burstCaptureSurface != null && config.getPostviewOutputConfiguration() != null) {
204             CameraExtensionUtils.SurfaceInfo burstCaptureSurfaceInfo =
205                     CameraExtensionUtils.querySurface(burstCaptureSurface);
206             Size burstCaptureSurfaceSize =
207                     new Size(burstCaptureSurfaceInfo.mWidth, burstCaptureSurfaceInfo.mHeight);
208             HashMap<Integer, List<Size>> supportedPostviewSizes = new HashMap<>();
209             for (int format : CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
210                 List<Size> supportedSizesPostview = extensionChars.getPostviewSupportedSizes(
211                         config.getExtension(), burstCaptureSurfaceSize, format);
212                 if (supportedSizesPostview != null) {
213                     supportedPostviewSizes.put(format, supportedSizesPostview);
214                 }
215             }
216 
217             postviewSurface = CameraExtensionUtils.getPostviewSurface(
218                         config.getPostviewOutputConfiguration(), supportedPostviewSizes,
219                         burstCaptureSurfaceInfo.mFormat);
220 
221             if (postviewSurface == null) {
222                 throw new IllegalArgumentException("Unsupported output surface for postview!");
223             }
224         }
225 
226         extenders.first.init(cameraId, characteristicsMap.get(cameraId).getNativeMetadata());
227         extenders.first.onInit(token, cameraId,
228                 characteristicsMap.get(cameraId).getNativeMetadata());
229         extenders.second.init(cameraId, characteristicsMap.get(cameraId).getNativeMetadata());
230         extenders.second.onInit(token, cameraId,
231                 characteristicsMap.get(cameraId).getNativeMetadata());
232 
233         CameraExtensionSessionImpl session = new CameraExtensionSessionImpl(
234                 ctx,
235                 extenders.second,
236                 extenders.first,
237                 supportedPreviewSizes,
238                 cameraDevice,
239                 repeatingRequestSurface,
240                 burstCaptureSurface,
241                 postviewSurface,
242                 config.getStateCallback(),
243                 config.getExecutor(),
244                 sessionId,
245                 token,
246                 extensionChars.getAvailableCaptureRequestKeys(config.getExtension()),
247                 extensionChars.getAvailableCaptureResultKeys(config.getExtension()));
248 
249         session.mStatsAggregator.setClientName(ctx.getOpPackageName());
250         session.mStatsAggregator.setExtensionType(config.getExtension());
251 
252         session.initialize();
253 
254         return session;
255     }
256 
CameraExtensionSessionImpl(Context ctx, @NonNull IImageCaptureExtenderImpl imageExtender, @NonNull IPreviewExtenderImpl previewExtender, @NonNull List<Size> previewSizes, @NonNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice, @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, @Nullable Surface postviewSurface, @NonNull StateCallback callback, @NonNull Executor executor, int sessionId, @NonNull IBinder token, @NonNull Set<CaptureRequest.Key> requestKeys, @Nullable Set<CaptureResult.Key> resultKeys)257     public CameraExtensionSessionImpl(Context ctx, @NonNull IImageCaptureExtenderImpl imageExtender,
258             @NonNull IPreviewExtenderImpl previewExtender,
259             @NonNull List<Size> previewSizes,
260             @NonNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice,
261             @Nullable Surface repeatingRequestSurface,
262             @Nullable Surface burstCaptureSurface,
263             @Nullable Surface postviewSurface,
264             @NonNull StateCallback callback,
265             @NonNull Executor executor,
266             int sessionId,
267             @NonNull IBinder token,
268             @NonNull Set<CaptureRequest.Key> requestKeys,
269             @Nullable Set<CaptureResult.Key> resultKeys) {
270         mContext = ctx;
271         mImageExtender = imageExtender;
272         mPreviewExtender = previewExtender;
273         mCameraDevice = cameraDevice;
274         mCallbacks = callback;
275         mExecutor = executor;
276         mClientRepeatingRequestSurface = repeatingRequestSurface;
277         mClientCaptureSurface = burstCaptureSurface;
278         mClientPostviewSurface = postviewSurface;
279         mSupportedPreviewSizes = previewSizes;
280         mHandlerThread = new HandlerThread(TAG);
281         mHandlerThread.start();
282         mHandler = new Handler(mHandlerThread.getLooper());
283         mInitialized = false;
284         mSessionClosed = false;
285         mInitializeHandler = new InitializeSessionHandler();
286         mSessionId = sessionId;
287         mToken = token;
288         mSupportedRequestKeys = requestKeys;
289         mSupportedResultKeys = resultKeys;
290         mCaptureResultsSupported = !resultKeys.isEmpty();
291         mInterfaceLock = cameraDevice.mInterfaceLock;
292 
293         mStatsAggregator = new ExtensionSessionStatsAggregator(mCameraDevice.getId(),
294                 /*isAdvanced=*/false);
295     }
296 
initializeRepeatingRequestPipeline()297     private void initializeRepeatingRequestPipeline() throws RemoteException {
298         CameraExtensionUtils.SurfaceInfo repeatingSurfaceInfo =
299                 new CameraExtensionUtils.SurfaceInfo();
300         mPreviewProcessorType = mPreviewExtender.getProcessorType();
301         if (mClientRepeatingRequestSurface != null) {
302             repeatingSurfaceInfo = CameraExtensionUtils.querySurface(
303                     mClientRepeatingRequestSurface);
304         } else {
305             // Make the intermediate surface behave as any regular 'SurfaceTexture'
306             CameraExtensionUtils.SurfaceInfo captureSurfaceInfo = CameraExtensionUtils.querySurface(
307                     mClientCaptureSurface);
308             Size captureSize = new Size(captureSurfaceInfo.mWidth, captureSurfaceInfo.mHeight);
309             Size previewSize = findSmallestAspectMatchedSize(mSupportedPreviewSizes, captureSize);
310             repeatingSurfaceInfo.mWidth = previewSize.getWidth();
311             repeatingSurfaceInfo.mHeight = previewSize.getHeight();
312             repeatingSurfaceInfo.mUsage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE;
313         }
314 
315         if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
316             try {
317                 mPreviewImageProcessor = new CameraExtensionForwardProcessor(
318                         mPreviewExtender.getPreviewImageProcessor(), repeatingSurfaceInfo.mFormat,
319                         repeatingSurfaceInfo.mUsage, mHandler);
320             } catch (ClassCastException e) {
321                 throw new UnsupportedOperationException("Failed casting preview processor!");
322             }
323             mPreviewImageProcessor.onImageFormatUpdate(
324                     CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT);
325             mPreviewImageProcessor.onResolutionUpdate(new Size(repeatingSurfaceInfo.mWidth,
326                     repeatingSurfaceInfo.mHeight));
327             mPreviewImageProcessor.onOutputSurface(null, -1);
328             mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth,
329                     repeatingSurfaceInfo.mHeight,
330                     CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, PREVIEW_QUEUE_SIZE,
331                     repeatingSurfaceInfo.mUsage);
332             mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface();
333         } else if (mPreviewProcessorType ==
334                 IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
335             try {
336                 mPreviewRequestUpdateProcessor = mPreviewExtender.getRequestUpdateProcessor();
337             } catch (ClassCastException e) {
338                 throw new UnsupportedOperationException("Failed casting preview processor!");
339             }
340             mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth,
341                     repeatingSurfaceInfo.mHeight,
342                     CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT,
343                     PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage);
344             mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface();
345             android.hardware.camera2.extension.Size sz =
346                     new android.hardware.camera2.extension.Size();
347             sz.width = repeatingSurfaceInfo.mWidth;
348             sz.height = repeatingSurfaceInfo.mHeight;
349             mPreviewRequestUpdateProcessor.onResolutionUpdate(sz);
350             mPreviewRequestUpdateProcessor.onImageFormatUpdate(
351                     CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT);
352         } else {
353             mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth,
354                     repeatingSurfaceInfo.mHeight,
355                     CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT,
356                     PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage);
357             mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface();
358         }
359         mRepeatingRequestImageCallback = new CameraOutputImageCallback(
360                 mRepeatingRequestImageReader);
361         mRepeatingRequestImageReader
362                 .setOnImageAvailableListener(mRepeatingRequestImageCallback, mHandler);
363     }
364 
initializeBurstCapturePipeline()365     private void initializeBurstCapturePipeline() throws RemoteException {
366         mImageProcessor = mImageExtender.getCaptureProcessor();
367         if ((mImageProcessor == null) && (mImageExtender.getMaxCaptureStage() != 1)) {
368             throw new UnsupportedOperationException("Multiple stages expected without" +
369                     " a valid capture processor!");
370         }
371 
372         if (mImageProcessor != null) {
373             if (mClientCaptureSurface != null) {
374                 CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface(
375                         mClientCaptureSurface);
376                 if (surfaceInfo.mFormat == ImageFormat.JPEG) {
377                     mImageJpegProcessor = new CameraExtensionJpegProcessor(mImageProcessor);
378                     mImageProcessor = mImageJpegProcessor;
379                 }
380                 mBurstCaptureImageReader = ImageReader.newInstance(surfaceInfo.mWidth,
381                         surfaceInfo.mHeight, CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT,
382                         mImageExtender.getMaxCaptureStage());
383             } else {
384                 // The client doesn't intend to trigger multi-frame capture, however the
385                 // image extender still needs to get initialized and the camera still capture
386                 // stream configured for the repeating request processing to work.
387                 mBurstCaptureImageReader = ImageReader.newInstance(
388                         mRepeatingRequestImageReader.getWidth(),
389                         mRepeatingRequestImageReader.getHeight(),
390                         CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1);
391                 // The still capture output is not going to be used but we still need a valid
392                 // surface to register.
393                 mStubCaptureImageReader = ImageReader.newInstance(
394                         mRepeatingRequestImageReader.getWidth(),
395                         mRepeatingRequestImageReader.getHeight(),
396                         CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1);
397                 mImageProcessor.onOutputSurface(mStubCaptureImageReader.getSurface(),
398                         CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT);
399             }
400 
401             mBurstCaptureImageCallback = new CameraOutputImageCallback(mBurstCaptureImageReader);
402             mBurstCaptureImageReader.setOnImageAvailableListener(mBurstCaptureImageCallback,
403                     mHandler);
404             mCameraBurstSurface = mBurstCaptureImageReader.getSurface();
405             android.hardware.camera2.extension.Size sz =
406                     new android.hardware.camera2.extension.Size();
407             sz.width = mBurstCaptureImageReader.getWidth();
408             sz.height = mBurstCaptureImageReader.getHeight();
409 
410             if (mClientPostviewSurface != null) {
411                 CameraExtensionUtils.SurfaceInfo postviewSurfaceInfo =
412                         CameraExtensionUtils.querySurface(mClientPostviewSurface);
413                 android.hardware.camera2.extension.Size postviewSize =
414                         new android.hardware.camera2.extension.Size();
415                 postviewSize.width = postviewSurfaceInfo.mWidth;
416                 postviewSize.height = postviewSurfaceInfo.mHeight;
417                 mImageProcessor.onResolutionUpdate(sz, postviewSize);
418             } else {
419                 mImageProcessor.onResolutionUpdate(sz, null);
420             }
421 
422             mImageProcessor.onImageFormatUpdate(mBurstCaptureImageReader.getImageFormat());
423         } else {
424             if (mClientCaptureSurface != null) {
425                 // Redirect camera output directly in to client output surface
426                 mCameraBurstSurface = mClientCaptureSurface;
427             } else {
428                 // The client doesn't intend to trigger multi-frame capture, however the
429                 // image extender still needs to get initialized and the camera still capture
430                 // stream configured for the repeating request processing to work.
431                 mBurstCaptureImageReader = ImageReader.newInstance(
432                         mRepeatingRequestImageReader.getWidth(),
433                         mRepeatingRequestImageReader.getHeight(),
434                         // Camera devices accept only Jpeg output if the image processor is null
435                         ImageFormat.JPEG, 1);
436                 mCameraBurstSurface = mBurstCaptureImageReader.getSurface();
437             }
438         }
439     }
440 
finishPipelineInitialization()441     private void finishPipelineInitialization() throws RemoteException {
442         if (mClientRepeatingRequestSurface != null) {
443             if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
444                 mPreviewRequestUpdateProcessor.onOutputSurface(mClientRepeatingRequestSurface,
445                         nativeGetSurfaceFormat(mClientRepeatingRequestSurface));
446                 mRepeatingRequestImageWriter = ImageWriter.newInstance(
447                         mClientRepeatingRequestSurface,
448                         PREVIEW_QUEUE_SIZE,
449                         CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT);
450             } else if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_NONE) {
451                 mRepeatingRequestImageWriter = ImageWriter.newInstance(
452                         mClientRepeatingRequestSurface,
453                         PREVIEW_QUEUE_SIZE,
454                         CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT);
455             }
456         }
457         if ((mImageProcessor != null) && (mClientCaptureSurface != null)) {
458             if (mClientPostviewSurface != null) {
459                 mImageProcessor.onPostviewOutputSurface(mClientPostviewSurface);
460             }
461 
462             CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface(
463                     mClientCaptureSurface);
464             mImageProcessor.onOutputSurface(mClientCaptureSurface, surfaceInfo.mFormat);
465         }
466     }
467 
468     /**
469      * @hide
470      */
initialize()471     public synchronized void initialize() throws CameraAccessException, RemoteException {
472         if (mInitialized) {
473             Log.d(TAG,
474                     "Session already initialized");
475             return;
476         }
477         int previewSessionType = mPreviewExtender.getSessionType();
478         int imageSessionType = mImageExtender.getSessionType();
479         if (previewSessionType != imageSessionType) {
480             throw new IllegalStateException("Preview extender session type: " + previewSessionType +
481                 "and image extender session type: " + imageSessionType + " mismatch!");
482         }
483         int sessionType = SessionConfiguration.SESSION_REGULAR;
484         if ((previewSessionType != -1) &&
485                 (previewSessionType != SessionConfiguration.SESSION_HIGH_SPEED)) {
486             sessionType = previewSessionType;
487             Log.v(TAG, "Using session type: " + sessionType);
488         }
489 
490         ArrayList<CaptureStageImpl> sessionParamsList = new ArrayList<>();
491         ArrayList<OutputConfiguration> outputList = new ArrayList<>();
492         initializeRepeatingRequestPipeline();
493         OutputConfiguration previewOutput = new OutputConfiguration(mCameraRepeatingSurface);
494         // The extension processing logic needs to be able to match images to capture results via
495         // image and result timestamps.
496         previewOutput.setTimestampBase(OutputConfiguration.TIMESTAMP_BASE_SENSOR);
497         previewOutput.setReadoutTimestampEnabled(false);
498         outputList.add(previewOutput);
499         CaptureStageImpl previewSessionParams = mPreviewExtender.onPresetSession();
500         if (previewSessionParams != null) {
501             sessionParamsList.add(previewSessionParams);
502         }
503         initializeBurstCapturePipeline();
504         OutputConfiguration captureOutput = new OutputConfiguration(mCameraBurstSurface);
505         captureOutput.setTimestampBase(OutputConfiguration.TIMESTAMP_BASE_SENSOR);
506         captureOutput.setReadoutTimestampEnabled(false);
507         outputList.add(captureOutput);
508         CaptureStageImpl stillCaptureSessionParams = mImageExtender.onPresetSession();
509         if (stillCaptureSessionParams != null) {
510             sessionParamsList.add(stillCaptureSessionParams);
511         }
512 
513         SessionConfiguration sessionConfig = new SessionConfiguration(
514                 sessionType,
515                 outputList,
516                 new CameraExtensionUtils.HandlerExecutor(mHandler),
517                 new SessionStateHandler());
518 
519         if (!sessionParamsList.isEmpty()) {
520             CaptureRequest sessionParamRequest = createRequest(mCameraDevice, sessionParamsList,
521                     null, CameraDevice.TEMPLATE_PREVIEW);
522             sessionConfig.setSessionParameters(sessionParamRequest);
523         }
524 
525         mCameraDevice.createCaptureSession(sessionConfig);
526     }
527 
528     @Override
getDevice()529     public @NonNull CameraDevice getDevice() {
530         synchronized (mInterfaceLock) {
531             return mCameraDevice;
532         }
533     }
534 
535     @Override
getRealtimeStillCaptureLatency()536     public StillCaptureLatency getRealtimeStillCaptureLatency() throws CameraAccessException {
537         synchronized (mInterfaceLock) {
538             if (!mInitialized) {
539                 throw new IllegalStateException("Uninitialized component");
540             }
541 
542             try {
543                 LatencyPair latency = mImageExtender.getRealtimeCaptureLatency();
544                 if (latency != null) {
545                     return new StillCaptureLatency(latency.first, latency.second);
546                 }
547 
548                 return null;
549             } catch (RemoteException e) {
550                 Log.e(TAG, "Failed to query realtime latency! Extension service does not "
551                         + "respond");
552                 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR);
553             }
554         }
555     }
556 
557     @Override
setRepeatingRequest(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)558     public int setRepeatingRequest(@NonNull CaptureRequest request,
559                                    @NonNull Executor executor,
560                                    @NonNull ExtensionCaptureCallback listener)
561             throws CameraAccessException {
562         synchronized (mInterfaceLock) {
563             if (!mInitialized) {
564                 throw new IllegalStateException("Uninitialized component");
565             }
566 
567             if (mClientRepeatingRequestSurface == null) {
568                 throw new IllegalArgumentException("No registered preview surface");
569             }
570 
571             if (!request.containsTarget(mClientRepeatingRequestSurface) ||
572                     (request.getTargets().size() != 1)) {
573                 throw new IllegalArgumentException("Invalid repeating request output target!");
574             }
575 
576             mInternalRepeatingRequestEnabled = false;
577             try {
578                 return setRepeatingRequest(mPreviewExtender.getCaptureStage(),
579                         new PreviewRequestHandler(request, executor, listener,
580                                 mRepeatingRequestImageCallback), request);
581             } catch (RemoteException e) {
582                 Log.e(TAG, "Failed to set repeating request! Extension service does not "
583                         + "respond");
584                 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR);
585             }
586         }
587     }
588 
compileInitialRequestList()589     private ArrayList<CaptureStageImpl> compileInitialRequestList() {
590         ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>();
591         try {
592             CaptureStageImpl initialPreviewParams = mPreviewExtender.onEnableSession();
593             if (initialPreviewParams != null) {
594                 captureStageList.add(initialPreviewParams);
595             }
596 
597             CaptureStageImpl initialStillCaptureParams = mImageExtender.onEnableSession();
598             if (initialStillCaptureParams != null) {
599                 captureStageList.add(initialStillCaptureParams);
600             }
601         } catch (RemoteException e) {
602             Log.e(TAG, "Failed to initialize session parameters! Extension service does not"
603                     + " respond!");
604         }
605 
606         return captureStageList;
607     }
608 
createBurstRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, CaptureRequest clientRequest, Surface target, int captureTemplate, Map<CaptureRequest, Integer> captureMap)609     private List<CaptureRequest> createBurstRequest(CameraDevice cameraDevice,
610             List<CaptureStageImpl> captureStageList, CaptureRequest clientRequest,
611             Surface target, int captureTemplate, Map<CaptureRequest, Integer> captureMap) {
612         CaptureRequest.Builder requestBuilder;
613         ArrayList<CaptureRequest> ret = new ArrayList<>();
614         for (CaptureStageImpl captureStage : captureStageList) {
615             try {
616                 requestBuilder = cameraDevice.createCaptureRequest(captureTemplate);
617             } catch (CameraAccessException e) {
618                 return null;
619             }
620 
621             // This will guarantee that client configured
622             // parameters always have the highest priority.
623             for (CaptureRequest.Key requestKey : mSupportedRequestKeys){
624                 Object value = clientRequest.get(requestKey);
625                 if (value != null) {
626                     captureStage.parameters.set(requestKey, value);
627                 }
628             }
629 
630             requestBuilder.addTarget(target);
631             CaptureRequest request = requestBuilder.build();
632             CameraMetadataNative.update(request.getNativeMetadata(), captureStage.parameters);
633             ret.add(request);
634             if (captureMap != null) {
635                 captureMap.put(request, captureStage.id);
636             }
637         }
638 
639         return ret;
640     }
641 
createRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate, CaptureRequest clientRequest)642     private CaptureRequest createRequest(CameraDevice cameraDevice,
643             List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate,
644             CaptureRequest clientRequest) throws CameraAccessException {
645         CaptureRequest.Builder requestBuilder;
646         requestBuilder = cameraDevice.createCaptureRequest(captureTemplate);
647         if (target != null) {
648             requestBuilder.addTarget(target);
649         }
650 
651         CaptureRequest ret = requestBuilder.build();
652         CameraMetadataNative nativeMeta = ret.getNativeMetadata();
653         for (CaptureStageImpl captureStage : captureStageList) {
654             if (captureStage != null) {
655                 CameraMetadataNative.update(nativeMeta, captureStage.parameters);
656             }
657         }
658 
659         if (clientRequest != null) {
660             // This will guarantee that client configured
661             // parameters always have the highest priority.
662             for (CaptureRequest.Key requestKey : mSupportedRequestKeys) {
663                 Object value = clientRequest.get(requestKey);
664                 if (value != null) {
665                     nativeMeta.set(requestKey, value);
666                 }
667             }
668         }
669 
670         return ret;
671     }
672 
createRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate)673     private CaptureRequest createRequest(CameraDevice cameraDevice,
674             List<CaptureStageImpl> captureStageList,
675             Surface target,
676             int captureTemplate) throws CameraAccessException {
677         return createRequest(cameraDevice, captureStageList, target, captureTemplate,
678                 /*clientRequest*/ null);
679     }
680 
681     @Override
capture(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)682     public int capture(@NonNull CaptureRequest request,
683                        @NonNull Executor executor,
684                        @NonNull ExtensionCaptureCallback listener) throws CameraAccessException {
685         if (!mInitialized) {
686             throw new IllegalStateException("Uninitialized component");
687         }
688 
689         validateCaptureRequestTargets(request);
690 
691         int seqId = -1;
692         if ((mClientCaptureSurface != null) && request.containsTarget(mClientCaptureSurface)) {
693             HashMap<CaptureRequest, Integer> requestMap = new HashMap<>();
694             List<CaptureRequest> burstRequest;
695             try {
696                 burstRequest = createBurstRequest(mCameraDevice,
697                         mImageExtender.getCaptureStages(), request, mCameraBurstSurface,
698                         CameraDevice.TEMPLATE_STILL_CAPTURE, requestMap);
699             } catch (RemoteException e) {
700                 Log.e(TAG, "Failed to initialize internal burst request! Extension service does"
701                         + " not respond!");
702                 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR);
703             }
704             if (burstRequest == null) {
705                 throw new UnsupportedOperationException(
706                         "Failed to create still capture burst request");
707             }
708 
709             seqId =  mCaptureSession.captureBurstRequests(burstRequest,
710                     new CameraExtensionUtils.HandlerExecutor(mHandler),
711                     new BurstRequestHandler(request, executor, listener, requestMap,
712                             mBurstCaptureImageCallback));
713         } else if ((mClientRepeatingRequestSurface != null) &&
714                 request.containsTarget(mClientRepeatingRequestSurface)) {
715 
716             CaptureRequest captureRequest = null;
717             try {
718                 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>();
719                 captureStageList.add(mPreviewExtender.getCaptureStage());
720 
721                 captureRequest = createRequest(mCameraDevice, captureStageList,
722                         mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW, request);
723             } catch (RemoteException e) {
724                 Log.e(TAG, "Failed to initialize capture request! Extension service does"
725                         + " not respond!");
726                 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR);
727             }
728 
729             seqId = mCaptureSession.capture(captureRequest, new PreviewRequestHandler(request,
730                     executor, listener, mRepeatingRequestImageCallback, true /*singleCapture*/),
731                     mHandler);
732         } else {
733             throw new IllegalArgumentException("Capture request to unknown output surface!");
734         }
735 
736         return seqId;
737     }
738 
validateCaptureRequestTargets(@onNull CaptureRequest request)739     private void validateCaptureRequestTargets(@NonNull CaptureRequest request) {
740         if (request.getTargets().size() == 1) {
741             boolean containsCaptureTarget =
742                     mClientCaptureSurface != null && request.containsTarget(mClientCaptureSurface);
743             boolean containsRepeatingTarget =
744                     mClientRepeatingRequestSurface != null &&
745                     request.containsTarget(mClientRepeatingRequestSurface);
746 
747             if (!containsCaptureTarget && !containsRepeatingTarget) {
748                 throw new IllegalArgumentException("Target output combination requested is " +
749                         "not supported!");
750             }
751         }
752 
753         if ((request.getTargets().size() == 2) &&
754                 (!request.getTargets().containsAll(Arrays.asList(mClientCaptureSurface,
755                 mClientPostviewSurface)))) {
756             throw new IllegalArgumentException("Target output combination requested is " +
757                     "not supported!");
758         }
759 
760         if (request.getTargets().size() > 2) {
761             throw new IllegalArgumentException("Target output combination requested is " +
762                     "not supported!");
763         }
764     }
765 
766     @Override
stopRepeating()767     public void stopRepeating() throws CameraAccessException {
768         synchronized (mInterfaceLock) {
769             if (!mInitialized) {
770                 throw new IllegalStateException("Uninitialized component");
771             }
772 
773             mInternalRepeatingRequestEnabled = true;
774             mCaptureSession.stopRepeating();
775         }
776     }
777 
778     @Override
close()779     public void close() throws CameraAccessException {
780         synchronized (mInterfaceLock) {
781             if (mInitialized) {
782                 mInternalRepeatingRequestEnabled = false;
783                 try {
784                     mCaptureSession.stopRepeating();
785                 } catch (IllegalStateException e) {
786                     // OK: already be closed, nothing else to do
787                     mSessionClosed = true;
788                 }
789 
790                 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>();
791                 try {
792                     CaptureStageImpl disableParams = mPreviewExtender.onDisableSession();
793                     if (disableParams != null) {
794                         captureStageList.add(disableParams);
795                     }
796 
797                     CaptureStageImpl disableStillCaptureParams =
798                             mImageExtender.onDisableSession();
799                     if (disableStillCaptureParams != null) {
800                         captureStageList.add(disableStillCaptureParams);
801                     }
802                 } catch (RemoteException e) {
803                     Log.e(TAG, "Failed to disable extension! Extension service does not "
804                             + "respond!");
805                 }
806                 if (!captureStageList.isEmpty() && !mSessionClosed) {
807                     CaptureRequest disableRequest = createRequest(mCameraDevice, captureStageList,
808                             mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW);
809                     mCaptureSession.capture(disableRequest,
810                             new CloseRequestHandler(mRepeatingRequestImageCallback), mHandler);
811                 }
812 
813                 mSessionClosed = true;
814                 mStatsAggregator.commit(/*isFinal*/true); // Commit stats before closing session
815                 mCaptureSession.close();
816             }
817         }
818     }
819 
820     /**
821      * Called by {@link CameraDeviceImpl} right before the capture session is closed, and before it
822      * calls {@link #release}
823      *
824      * @hide
825      */
commitStats()826     public void commitStats() {
827         synchronized (mInterfaceLock) {
828             if (mInitialized) {
829                 // Only commit stats if a capture session was initialized
830                 mStatsAggregator.commit(/*isFinal*/true);
831             }
832         }
833     }
834 
setInitialCaptureRequest(List<CaptureStageImpl> captureStageList, InitialRequestHandler requestHandler)835     private void setInitialCaptureRequest(List<CaptureStageImpl> captureStageList,
836                                           InitialRequestHandler requestHandler)
837             throws CameraAccessException {
838         CaptureRequest initialRequest = createRequest(mCameraDevice,
839                 captureStageList, mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW);
840         mCaptureSession.capture(initialRequest, requestHandler, mHandler);
841     }
842 
setRepeatingRequest(CaptureStageImpl captureStage, CameraCaptureSession.CaptureCallback requestHandler)843     private int setRepeatingRequest(CaptureStageImpl captureStage,
844             CameraCaptureSession.CaptureCallback requestHandler) throws CameraAccessException {
845         return setRepeatingRequest(captureStage, requestHandler, /*clientRequest*/ null);
846     }
847 
setRepeatingRequest(CaptureStageImpl captureStage, CameraCaptureSession.CaptureCallback requestHandler, CaptureRequest clientRequest)848     private int setRepeatingRequest(CaptureStageImpl captureStage,
849             CameraCaptureSession.CaptureCallback requestHandler, CaptureRequest clientRequest)
850             throws CameraAccessException {
851         ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>();
852         captureStageList.add(captureStage);
853         CaptureRequest repeatingRequest = createRequest(mCameraDevice, captureStageList,
854                 mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW, clientRequest);
855         return mCaptureSession.setSingleRepeatingRequest(repeatingRequest,
856                 new CameraExtensionUtils.HandlerExecutor(mHandler), requestHandler);
857     }
858 
859     /** @hide */
release(boolean skipCloseNotification)860     public void release(boolean skipCloseNotification) {
861         boolean notifyClose = false;
862 
863         synchronized (mInterfaceLock) {
864             mInternalRepeatingRequestEnabled = false;
865             mHandlerThread.quit();
866 
867             try {
868                 if (!mSessionClosed) {
869                     // return value is omitted. nothing can do after session is closed.
870                     mPreviewExtender.onDisableSession();
871                     mImageExtender.onDisableSession();
872                 }
873                 mPreviewExtender.onDeInit(mToken);
874                 mImageExtender.onDeInit(mToken);
875             } catch (RemoteException e) {
876                 Log.e(TAG, "Failed to release extensions! Extension service does not"
877                         + " respond!");
878             }
879 
880             if (mToken != null) {
881                 if (mInitialized || (mCaptureSession != null)) {
882                     notifyClose = true;
883                     CameraExtensionCharacteristics.releaseSession();
884                 }
885                 CameraExtensionCharacteristics.unregisterClient(mContext, mToken);
886             }
887             mInitialized = false;
888             mToken = null;
889 
890             if (mRepeatingRequestImageCallback != null) {
891                 mRepeatingRequestImageCallback.close();
892                 mRepeatingRequestImageCallback = null;
893             }
894 
895             if (mRepeatingRequestImageReader != null) {
896                 mRepeatingRequestImageReader.close();
897                 mRepeatingRequestImageReader = null;
898             }
899 
900             if (mBurstCaptureImageCallback != null) {
901                 mBurstCaptureImageCallback.close();
902                 mBurstCaptureImageCallback = null;
903             }
904 
905             if (mBurstCaptureImageReader != null) {
906                 mBurstCaptureImageReader.close();
907                 mBurstCaptureImageReader = null;
908             }
909 
910             if (mStubCaptureImageReader != null) {
911                 mStubCaptureImageReader.close();
912                 mStubCaptureImageReader = null;
913             }
914 
915             if (mRepeatingRequestImageWriter != null) {
916                 mRepeatingRequestImageWriter.close();
917                 mRepeatingRequestImageWriter = null;
918             }
919 
920             if (mPreviewImageProcessor != null) {
921                 mPreviewImageProcessor.close();
922                 mPreviewImageProcessor = null;
923             }
924 
925             if (mImageJpegProcessor != null) {
926                 mImageJpegProcessor.close();
927                 mImageJpegProcessor = null;
928             }
929 
930             mCaptureSession = null;
931             mImageProcessor = null;
932             mCameraRepeatingSurface = mClientRepeatingRequestSurface = null;
933             mCameraBurstSurface = mClientCaptureSurface = null;
934             mClientPostviewSurface = null;
935         }
936 
937         if (notifyClose && !skipCloseNotification) {
938             final long ident = Binder.clearCallingIdentity();
939             try {
940                 mExecutor.execute(() -> mCallbacks.onClosed(CameraExtensionSessionImpl.this));
941             } finally {
942                 Binder.restoreCallingIdentity(ident);
943             }
944         }
945     }
946 
notifyConfigurationFailure()947     private void notifyConfigurationFailure() {
948         synchronized (mInterfaceLock) {
949             if (mInitialized) {
950                 return;
951             }
952         }
953 
954         release(true /*skipCloseNotification*/);
955 
956         final long ident = Binder.clearCallingIdentity();
957         try {
958             mExecutor.execute(
959                     () -> mCallbacks.onConfigureFailed(CameraExtensionSessionImpl.this));
960         } finally {
961             Binder.restoreCallingIdentity(ident);
962         }
963     }
964 
notifyConfigurationSuccess()965     private void notifyConfigurationSuccess() {
966         synchronized (mInterfaceLock) {
967             if (mInitialized) {
968                 return;
969             } else {
970                 mInitialized = true;
971             }
972         }
973 
974         final long ident = Binder.clearCallingIdentity();
975         try {
976             mExecutor.execute(() -> mCallbacks.onConfigured(CameraExtensionSessionImpl.this));
977         } finally {
978             Binder.restoreCallingIdentity(ident);
979         }
980     }
981 
982     private class SessionStateHandler extends
983             android.hardware.camera2.CameraCaptureSession.StateCallback {
984         @Override
onClosed(@onNull CameraCaptureSession session)985         public void onClosed(@NonNull CameraCaptureSession session) {
986             release(false /*skipCloseNotification*/);
987         }
988 
989         @Override
onConfigureFailed(@onNull CameraCaptureSession session)990         public void onConfigureFailed(@NonNull CameraCaptureSession session) {
991             notifyConfigurationFailure();
992         }
993 
994         @Override
onConfigured(@onNull CameraCaptureSession session)995         public void onConfigured(@NonNull CameraCaptureSession session) {
996             synchronized (mInterfaceLock) {
997                 mCaptureSession = session;
998                 // Commit basic stats as soon as the capture session is created
999                 mStatsAggregator.commit(/*isFinal*/false);
1000                 try {
1001                     finishPipelineInitialization();
1002                     CameraExtensionCharacteristics.initializeSession(mInitializeHandler);
1003                 } catch (RemoteException e) {
1004                     Log.e(TAG, "Failed to initialize session! Extension service does"
1005                             + " not respond!");
1006                     notifyConfigurationFailure();
1007                 }
1008             }
1009         }
1010     }
1011 
1012     private class InitializeSessionHandler extends IInitializeSessionCallback.Stub {
1013         @Override
onSuccess()1014         public void onSuccess() {
1015             mHandler.post(new Runnable() {
1016                 @Override
1017                 public void run() {
1018                     boolean status = true;
1019                     ArrayList<CaptureStageImpl> initialRequestList =
1020                             compileInitialRequestList();
1021                     if (!initialRequestList.isEmpty()) {
1022                         try {
1023                             setInitialCaptureRequest(initialRequestList,
1024                                     new InitialRequestHandler(
1025                                             mRepeatingRequestImageCallback));
1026                         } catch (CameraAccessException e) {
1027                             Log.e(TAG,
1028                                     "Failed to initialize the initial capture "
1029                                             + "request!");
1030                             status = false;
1031                         }
1032                     } else {
1033                         try {
1034                             setRepeatingRequest(mPreviewExtender.getCaptureStage(),
1035                                     new PreviewRequestHandler(null, null, null,
1036                                             mRepeatingRequestImageCallback));
1037                         } catch (CameraAccessException | RemoteException e) {
1038                             Log.e(TAG,
1039                                     "Failed to initialize internal repeating "
1040                                             + "request!");
1041                             status = false;
1042                         }
1043 
1044                     }
1045 
1046                     if (!status) {
1047                         notifyConfigurationFailure();
1048                     }
1049                 }
1050             });
1051         }
1052 
1053         @Override
onFailure()1054         public void onFailure() {
1055             mHandler.post(new Runnable() {
1056                 @Override
1057                 public void run() {
1058                     mCaptureSession.close();
1059                     Log.e(TAG, "Failed to initialize proxy service session!"
1060                             + " This can happen when trying to configure multiple "
1061                             + "concurrent extension sessions!");
1062                     notifyConfigurationFailure();
1063                 }
1064             });
1065         }
1066     }
1067 
1068     private class BurstRequestHandler extends CameraCaptureSession.CaptureCallback {
1069         private final Executor mExecutor;
1070         private final ExtensionCaptureCallback mCallbacks;
1071         private final CaptureRequest mClientRequest;
1072         private final HashMap<CaptureRequest, Integer> mCaptureRequestMap;
1073         private final CameraOutputImageCallback mBurstImageCallback;
1074 
1075         private HashMap<Integer, Pair<Image, TotalCaptureResult>> mCaptureStageMap =
1076                 new HashMap<>();
1077         private LongSparseArray<Pair<Image, Integer>> mCapturePendingMap =
1078                 new LongSparseArray<>();
1079 
1080         private ImageCallback mImageCallback = null;
1081         private boolean mCaptureFailed = false;
1082         private CaptureResultHandler mCaptureResultHandler = null;
1083 
BurstRequestHandler(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback callbacks, @NonNull HashMap<CaptureRequest, Integer> requestMap, @Nullable CameraOutputImageCallback imageCallback)1084         public BurstRequestHandler(@NonNull CaptureRequest request, @NonNull Executor executor,
1085                 @NonNull ExtensionCaptureCallback callbacks,
1086                 @NonNull HashMap<CaptureRequest, Integer> requestMap,
1087                 @Nullable CameraOutputImageCallback imageCallback) {
1088             mClientRequest = request;
1089             mExecutor = executor;
1090             mCallbacks = callbacks;
1091             mCaptureRequestMap = requestMap;
1092             mBurstImageCallback = imageCallback;
1093         }
1094 
notifyCaptureFailed()1095         private void notifyCaptureFailed() {
1096             if (!mCaptureFailed) {
1097                 mCaptureFailed = true;
1098 
1099                 final long ident = Binder.clearCallingIdentity();
1100                 try {
1101                     mExecutor.execute(
1102                             () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this,
1103                                     mClientRequest));
1104                 } finally {
1105                     Binder.restoreCallingIdentity(ident);
1106                 }
1107 
1108                 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap.values()) {
1109                     captureStage.first.close();
1110                 }
1111                 mCaptureStageMap.clear();
1112             }
1113         }
1114 
1115         @Override
onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1116         public void onCaptureStarted(@NonNull CameraCaptureSession session,
1117                                      @NonNull CaptureRequest request,
1118                                      long timestamp,
1119                                      long frameNumber) {
1120             // Trigger the client callback only once in case of burst request
1121             boolean initialCallback = false;
1122             synchronized (mInterfaceLock) {
1123                 if ((mImageProcessor != null) && (mImageCallback == null)) {
1124                     mImageCallback = new ImageCallback();
1125                     initialCallback = true;
1126                 } else if (mImageProcessor == null) {
1127                     // No burst expected in this case
1128                     initialCallback = true;
1129                 }
1130             }
1131 
1132             if (initialCallback) {
1133                 final long ident = Binder.clearCallingIdentity();
1134                 try {
1135                     mExecutor.execute(
1136                             () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this,
1137                                     mClientRequest, timestamp));
1138                 } finally {
1139                     Binder.restoreCallingIdentity(ident);
1140                 }
1141             }
1142 
1143             if ((mBurstImageCallback != null) && (mImageCallback != null)) {
1144                 mBurstImageCallback.registerListener(timestamp, mImageCallback);
1145             }
1146         }
1147 
1148         @Override
onCaptureBufferLost(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull Surface target, long frameNumber)1149         public void onCaptureBufferLost(@NonNull CameraCaptureSession session,
1150                                         @NonNull CaptureRequest request,
1151                                         @NonNull Surface target, long frameNumber) {
1152             notifyCaptureFailed();
1153         }
1154 
1155         @Override
onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1156         public void onCaptureFailed(@NonNull CameraCaptureSession session,
1157                                     @NonNull CaptureRequest request,
1158                                     @NonNull CaptureFailure failure) {
1159             notifyCaptureFailed();
1160         }
1161 
1162         @Override
onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1163         public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session,
1164                                              int sequenceId) {
1165             final long ident = Binder.clearCallingIdentity();
1166             try {
1167                 mExecutor.execute(
1168                         () -> mCallbacks.onCaptureSequenceAborted(CameraExtensionSessionImpl.this,
1169                                 sequenceId));
1170             } finally {
1171                 Binder.restoreCallingIdentity(ident);
1172             }
1173         }
1174 
1175         @Override
onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1176         public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session,
1177                                                int sequenceId,
1178                                                long frameNumber) {
1179             final long ident = Binder.clearCallingIdentity();
1180             try {
1181                 mExecutor.execute(
1182                         () -> mCallbacks
1183                                 .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this,
1184                                         sequenceId));
1185             } finally {
1186                 Binder.restoreCallingIdentity(ident);
1187             }
1188         }
1189 
1190         @Override
onCaptureCompleted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)1191         public void onCaptureCompleted(@NonNull CameraCaptureSession session,
1192                                        @NonNull CaptureRequest request,
1193                                        @NonNull TotalCaptureResult result) {
1194             if (!mCaptureRequestMap.containsKey(request)) {
1195                 Log.e(TAG,
1196                         "Unexpected still capture request received!");
1197                 return;
1198             }
1199             Integer stageId = mCaptureRequestMap.get(request);
1200 
1201             Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
1202             if (timestamp != null) {
1203                 if (mCaptureResultsSupported && (mCaptureResultHandler == null)) {
1204                     mCaptureResultHandler = new CaptureResultHandler(mClientRequest, mExecutor,
1205                             mCallbacks, result.getSequenceId());
1206                 }
1207                 if (mImageProcessor != null) {
1208                     if (mCapturePendingMap.indexOfKey(timestamp) >= 0) {
1209                         Image img = mCapturePendingMap.get(timestamp).first;
1210                         mCaptureStageMap.put(stageId, new Pair<>(img, result));
1211                         checkAndFireBurstProcessing();
1212                     } else {
1213                         mCapturePendingMap.put(timestamp, new Pair<>(null, stageId));
1214                         mCaptureStageMap.put(stageId, new Pair<>(null, result));
1215                     }
1216                 } else {
1217                     mCaptureRequestMap.clear();
1218                     final long ident = Binder.clearCallingIdentity();
1219                     try {
1220                         mExecutor.execute(
1221                                 () -> mCallbacks
1222                                         .onCaptureProcessStarted(CameraExtensionSessionImpl.this,
1223                                                 mClientRequest));
1224 
1225                         if (mCaptureResultHandler != null) {
1226                             mCaptureResultHandler.onCaptureCompleted(timestamp,
1227                                     initializeFilteredResults(result));
1228                         }
1229                     } finally {
1230                         Binder.restoreCallingIdentity(ident);
1231                     }
1232                 }
1233             } else {
1234                 Log.e(TAG,
1235                         "Capture result without valid sensor timestamp!");
1236             }
1237         }
1238 
checkAndFireBurstProcessing()1239         private void checkAndFireBurstProcessing() {
1240             if (mCaptureRequestMap.size() == mCaptureStageMap.size()) {
1241                 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap
1242                         .values()) {
1243                     if ((captureStage.first == null) || (captureStage.second == null)) {
1244                         return;
1245                     }
1246                 }
1247 
1248                 mCaptureRequestMap.clear();
1249                 mCapturePendingMap.clear();
1250                 boolean processStatus = true;
1251                 Byte jpegQuality = mClientRequest.get(CaptureRequest.JPEG_QUALITY);
1252                 Integer jpegOrientation = mClientRequest.get(CaptureRequest.JPEG_ORIENTATION);
1253                 List<CaptureBundle> captureList = initializeParcelable(mCaptureStageMap,
1254                         jpegOrientation, jpegQuality);
1255                 try {
1256                     boolean isPostviewRequested =
1257                             mClientRequest.containsTarget(mClientPostviewSurface);
1258                     mImageProcessor.process(captureList, mCaptureResultHandler,
1259                             isPostviewRequested);
1260                 } catch (RemoteException e) {
1261                     Log.e(TAG, "Failed to process multi-frame request! Extension service "
1262                             + "does not respond!");
1263                     processStatus = false;
1264                 }
1265 
1266                 for (CaptureBundle bundle : captureList) {
1267                     bundle.captureImage.buffer.close();
1268                 }
1269                 captureList.clear();
1270                 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap.values()) {
1271                     captureStage.first.close();
1272                 }
1273                 mCaptureStageMap.clear();
1274 
1275                 final long ident = Binder.clearCallingIdentity();
1276                 try {
1277                     if (processStatus) {
1278                         mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted(
1279                                 CameraExtensionSessionImpl.this, mClientRequest));
1280                     } else {
1281                         mExecutor.execute(() -> mCallbacks.onCaptureFailed(
1282                                 CameraExtensionSessionImpl.this, mClientRequest));
1283                     }
1284                 } finally {
1285                     Binder.restoreCallingIdentity(ident);
1286                 }
1287             }
1288         }
1289 
1290         private class ImageCallback implements OnImageAvailableListener {
1291             @Override
onImageDropped(long timestamp)1292             public void onImageDropped(long timestamp) {
1293                 notifyCaptureFailed();
1294             }
1295 
1296             @Override
onImageAvailable(ImageReader reader, Image img)1297             public void onImageAvailable(ImageReader reader, Image img) {
1298                 if (mCaptureFailed) {
1299                     img.close();
1300                 }
1301 
1302                 long timestamp = img.getTimestamp();
1303                 reader.detachImage(img);
1304                 if (mCapturePendingMap.indexOfKey(timestamp) >= 0) {
1305                     Integer stageId = mCapturePendingMap.get(timestamp).second;
1306                     Pair<Image, TotalCaptureResult> captureStage =
1307                             mCaptureStageMap.get(stageId);
1308                     if (captureStage != null) {
1309                         mCaptureStageMap.put(stageId,
1310                                 new Pair<>(img,
1311                                         captureStage.second));
1312                         checkAndFireBurstProcessing();
1313                     } else {
1314                         Log.e(TAG,
1315                                 "Capture stage: " +
1316                                         mCapturePendingMap.get(timestamp).second +
1317                                         " is absent!");
1318                     }
1319                 } else {
1320                     mCapturePendingMap.put(timestamp,
1321                             new Pair<>(img,
1322                                     -1));
1323                 }
1324             }
1325         }
1326     }
1327 
1328     private class ImageLoopbackCallback implements OnImageAvailableListener {
1329         @Override
onImageDropped(long timestamp)1330         public void onImageDropped(long timestamp) { }
1331 
1332         @Override
onImageAvailable(ImageReader reader, Image img)1333         public void onImageAvailable(ImageReader reader, Image img) {
1334             img.close();
1335         }
1336     }
1337 
1338     private class InitialRequestHandler extends CameraCaptureSession.CaptureCallback {
1339         private final CameraOutputImageCallback mImageCallback;
1340 
InitialRequestHandler(CameraOutputImageCallback imageCallback)1341         public InitialRequestHandler(CameraOutputImageCallback imageCallback) {
1342             mImageCallback = imageCallback;
1343         }
1344 
1345         @Override
onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1346         public void onCaptureStarted(@NonNull CameraCaptureSession session,
1347                 @NonNull CaptureRequest request, long timestamp, long frameNumber) {
1348             mImageCallback.registerListener(timestamp, new ImageLoopbackCallback());
1349         }
1350 
1351         @Override
onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1352         public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session,
1353                 int sequenceId) {
1354             Log.e(TAG, "Initial capture request aborted!");
1355             notifyConfigurationFailure();
1356         }
1357 
1358         @Override
onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1359         public void onCaptureFailed(@NonNull CameraCaptureSession session,
1360                                     @NonNull CaptureRequest request,
1361                                     @NonNull CaptureFailure failure) {
1362             Log.e(TAG, "Initial capture request failed!");
1363             notifyConfigurationFailure();
1364         }
1365 
1366         @Override
onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1367         public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session,
1368                                                int sequenceId,
1369                                                long frameNumber) {
1370             boolean status = true;
1371             synchronized (mInterfaceLock) {
1372                 /**
1373                  * Initialize and set the initial repeating request which will execute in the
1374                  * absence of client repeating requests.
1375                  */
1376                 try {
1377                     setRepeatingRequest(mPreviewExtender.getCaptureStage(),
1378                             new PreviewRequestHandler(null, null, null,
1379                                     mImageCallback));
1380                 } catch (CameraAccessException | RemoteException e) {
1381                     Log.e(TAG, "Failed to start the internal repeating request!");
1382                     status = false;
1383                 }
1384 
1385             }
1386 
1387             if (!status) {
1388                 notifyConfigurationFailure();
1389             }
1390         }
1391     }
1392 
1393     private interface OnImageAvailableListener {
onImageDropped(long timestamp)1394         void onImageDropped(long timestamp);
onImageAvailable(ImageReader reader, Image img)1395         void onImageAvailable (ImageReader reader, Image img);
1396     }
1397 
1398     private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener,
1399             Closeable {
1400         private final ImageReader mImageReader;
1401         // Map timestamp to specific images and listeners
1402         private HashMap<Long, Pair<Image, OnImageAvailableListener>> mImageListenerMap =
1403                 new HashMap<>();
1404         private boolean mOutOfBuffers = false;
1405 
CameraOutputImageCallback(ImageReader imageReader)1406         CameraOutputImageCallback(ImageReader imageReader) {
1407             mImageReader = imageReader;
1408         }
1409 
1410         @Override
onImageAvailable(ImageReader reader)1411         public void onImageAvailable(ImageReader reader) {
1412             Image img;
1413             synchronized (mInterfaceLock) {
1414                 try {
1415                     img = reader.acquireNextImage();
1416                 } catch (IllegalStateException e) {
1417                     Log.e(TAG, "Failed to acquire image, too many images pending!");
1418                     mOutOfBuffers = true;
1419                     return;
1420                 }
1421                 if (img == null) {
1422                     Log.e(TAG, "Invalid image!");
1423                     return;
1424                 }
1425 
1426                 Long timestamp = img.getTimestamp();
1427                 if (mImageListenerMap.containsKey(timestamp)) {
1428                     Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.remove(
1429                             timestamp);
1430                     if (entry.second != null) {
1431                         entry.second.onImageAvailable(reader, img);
1432                     } else {
1433                         Log.w(TAG, "Invalid image listener, dropping frame!");
1434                         img.close();
1435                     }
1436                 } else {
1437                     mImageListenerMap.put(timestamp, new Pair<>(img, null));
1438                 }
1439 
1440                 notifyDroppedImages(timestamp);
1441             }
1442         }
1443 
notifyDroppedImages(long timestamp)1444         private void notifyDroppedImages(long timestamp) {
1445             synchronized (mInterfaceLock) {
1446                 Set<Long> timestamps = mImageListenerMap.keySet();
1447                 ArrayList<Long> removedTs = new ArrayList<>();
1448                 for (long ts : timestamps) {
1449                     if (ts < timestamp) {
1450                         Log.e(TAG, "Dropped image with ts: " + ts);
1451                         Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(ts);
1452                         if (entry.second != null) {
1453                             entry.second.onImageDropped(ts);
1454                         }
1455                         if (entry.first != null) {
1456                             entry.first.close();
1457                         }
1458                         removedTs.add(ts);
1459                     }
1460                 }
1461                 for (long ts : removedTs) {
1462                     mImageListenerMap.remove(ts);
1463                 }
1464             }
1465         }
1466 
registerListener(Long timestamp, OnImageAvailableListener listener)1467         public void registerListener(Long timestamp, OnImageAvailableListener listener) {
1468             synchronized (mInterfaceLock) {
1469                 if (mImageListenerMap.containsKey(timestamp)) {
1470                     Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.remove(
1471                             timestamp);
1472                     if (entry.first != null) {
1473                         listener.onImageAvailable(mImageReader, entry.first);
1474                         if (mOutOfBuffers) {
1475                             mOutOfBuffers = false;
1476                             Log.w(TAG,"Out of buffers, retry!");
1477                             onImageAvailable(mImageReader);
1478                         }
1479                     } else {
1480                         Log.w(TAG, "No valid image for listener with ts: " +
1481                                 timestamp.longValue());
1482                     }
1483                 } else {
1484                     mImageListenerMap.put(timestamp, new Pair<>(null, listener));
1485                 }
1486             }
1487         }
1488 
1489         @Override
close()1490         public void close() {
1491             synchronized (mInterfaceLock) {
1492                 for (Pair<Image, OnImageAvailableListener> entry : mImageListenerMap.values()) {
1493                     if (entry.first != null) {
1494                         entry.first.close();
1495                     }
1496                 }
1497                 for (long timestamp : mImageListenerMap.keySet()) {
1498                     Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(timestamp);
1499                     if (entry.second != null) {
1500                         entry.second.onImageDropped(timestamp);
1501                     }
1502                 }
1503                 mImageListenerMap.clear();
1504             }
1505         }
1506     }
1507 
1508     private class CloseRequestHandler extends CameraCaptureSession.CaptureCallback {
1509         private final CameraOutputImageCallback mImageCallback;
1510 
CloseRequestHandler(CameraOutputImageCallback imageCallback)1511         public CloseRequestHandler(CameraOutputImageCallback imageCallback) {
1512             mImageCallback = imageCallback;
1513         }
1514 
1515         @Override
onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1516         public void onCaptureStarted(@NonNull CameraCaptureSession session,
1517                 @NonNull CaptureRequest request, long timestamp, long frameNumber) {
1518             mImageCallback.registerListener(timestamp, new ImageLoopbackCallback());
1519         }
1520     }
1521 
1522     private class CaptureResultHandler extends IProcessResultImpl.Stub {
1523         private final Executor mExecutor;
1524         private final ExtensionCaptureCallback mCallbacks;
1525         private final CaptureRequest mClientRequest;
1526         private final int mRequestId;
1527 
CaptureResultHandler(@onNull CaptureRequest clientRequest, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener, int requestId)1528         public CaptureResultHandler(@NonNull CaptureRequest clientRequest,
1529                 @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener,
1530                 int requestId) {
1531             mClientRequest = clientRequest;
1532             mExecutor = executor;
1533             mCallbacks = listener;
1534             mRequestId = requestId;
1535         }
1536 
1537         @Override
onCaptureCompleted(long shutterTimestamp, CameraMetadataNative result)1538         public void onCaptureCompleted(long shutterTimestamp, CameraMetadataNative result) {
1539             if (result == null) {
1540                 Log.e(TAG,"Invalid capture result!");
1541                 return;
1542             }
1543 
1544             result.set(CaptureResult.SENSOR_TIMESTAMP, shutterTimestamp);
1545             TotalCaptureResult totalResult = new TotalCaptureResult(mCameraDevice.getId(), result,
1546                     mClientRequest, mRequestId, shutterTimestamp, new ArrayList<CaptureResult>(),
1547                     mSessionId, new PhysicalCaptureResultInfo[0]);
1548             final long ident = Binder.clearCallingIdentity();
1549             try {
1550                 mExecutor.execute(
1551                         () -> mCallbacks.onCaptureResultAvailable(CameraExtensionSessionImpl.this,
1552                                 mClientRequest, totalResult));
1553             } finally {
1554                 Binder.restoreCallingIdentity(ident);
1555             }
1556         }
1557 
1558         @Override
onCaptureProcessProgressed(int progress)1559         public void onCaptureProcessProgressed(int progress) {
1560             final long ident = Binder.clearCallingIdentity();
1561             try {
1562                 mExecutor.execute(
1563                         () -> mCallbacks.onCaptureProcessProgressed(CameraExtensionSessionImpl.this,
1564                                 mClientRequest, progress));
1565             } finally {
1566                 Binder.restoreCallingIdentity(ident);
1567             }
1568         }
1569     }
1570 
1571     // This handler can operate in three modes:
1572     // 1) Using valid client callbacks, which means camera buffers will be propagated the
1573     //    registered output surfaces and clients will be notified accordingly.
1574     // 2) Without any client callbacks where an internal repeating request is kept active
1575     //    to satisfy the extensions continuous preview/(repeating request) requirement.
1576     // 3) Single capture mode, where internal repeating requests are ignored and the preview
1577     //    logic is only triggered for the image processor case.
1578     private class PreviewRequestHandler extends CameraCaptureSession.CaptureCallback {
1579         private final Executor mExecutor;
1580         private final ExtensionCaptureCallback mCallbacks;
1581         private final CaptureRequest mClientRequest;
1582         private final boolean mClientNotificationsEnabled;
1583         private final CameraOutputImageCallback mRepeatingImageCallback;
1584         private final boolean mSingleCapture;
1585         private OnImageAvailableListener mImageCallback = null;
1586         private LongSparseArray<Pair<Image, TotalCaptureResult>> mPendingResultMap =
1587                 new LongSparseArray<>();
1588         private CaptureResultHandler mCaptureResultHandler = null;
1589 
1590         private boolean mRequestUpdatedNeeded = false;
1591 
PreviewRequestHandler(@ullable CaptureRequest clientRequest, @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, @NonNull CameraOutputImageCallback imageCallback)1592         public PreviewRequestHandler(@Nullable CaptureRequest clientRequest,
1593                 @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener,
1594                 @NonNull CameraOutputImageCallback imageCallback) {
1595             this(clientRequest, executor, listener, imageCallback, false /*singleCapture*/);
1596         }
1597 
PreviewRequestHandler(@ullable CaptureRequest clientRequest, @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, @NonNull CameraOutputImageCallback imageCallback, boolean singleCapture)1598         public PreviewRequestHandler(@Nullable CaptureRequest clientRequest,
1599                 @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener,
1600                 @NonNull CameraOutputImageCallback imageCallback, boolean singleCapture) {
1601             mClientRequest = clientRequest;
1602             mExecutor = executor;
1603             mCallbacks = listener;
1604             mClientNotificationsEnabled =
1605                     (mClientRequest != null) && (mExecutor != null) && (mCallbacks != null);
1606             mRepeatingImageCallback = imageCallback;
1607             mSingleCapture = singleCapture;
1608         }
1609 
1610         @Override
onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1611         public void onCaptureStarted(@NonNull CameraCaptureSession session,
1612                                      @NonNull CaptureRequest request,
1613                                      long timestamp,
1614                                      long frameNumber) {
1615             synchronized (mInterfaceLock) {
1616                 // Setup the image callback handler for this repeating request just once
1617                 // after streaming resumes.
1618                 if (mImageCallback == null) {
1619                     if (mPreviewProcessorType ==
1620                             IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
1621                         if (mClientNotificationsEnabled) {
1622                             mPreviewImageProcessor.onOutputSurface(mClientRepeatingRequestSurface,
1623                                     nativeGetSurfaceFormat(mClientRepeatingRequestSurface));
1624                         } else {
1625                             mPreviewImageProcessor.onOutputSurface(null, -1);
1626                         }
1627                         mImageCallback = new ImageProcessCallback();
1628                     } else {
1629                         mImageCallback = mClientNotificationsEnabled ?
1630                                 new ImageForwardCallback(mRepeatingRequestImageWriter) :
1631                                 new ImageLoopbackCallback();
1632                     }
1633                 }
1634             }
1635 
1636             if (mClientNotificationsEnabled) {
1637                 final long ident = Binder.clearCallingIdentity();
1638                 try {
1639                     mExecutor.execute(
1640                             () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this,
1641                                     mClientRequest, timestamp));
1642                 } finally {
1643                     Binder.restoreCallingIdentity(ident);
1644                 }
1645             }
1646 
1647             mRepeatingImageCallback.registerListener(timestamp, mImageCallback);
1648         }
1649 
1650         @Override
onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1651         public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session,
1652                                              int sequenceId) {
1653             synchronized (mInterfaceLock) {
1654                 if (mInternalRepeatingRequestEnabled && !mSingleCapture) {
1655                     resumeInternalRepeatingRequest(true);
1656                 }
1657             }
1658 
1659             if (mClientNotificationsEnabled) {
1660                 final long ident = Binder.clearCallingIdentity();
1661                 try {
1662                     mExecutor.execute(
1663                             () -> mCallbacks
1664                                     .onCaptureSequenceAborted(CameraExtensionSessionImpl.this,
1665                                             sequenceId));
1666                 } finally {
1667                     Binder.restoreCallingIdentity(ident);
1668                 }
1669             } else {
1670                 notifyConfigurationFailure();
1671             }
1672         }
1673 
1674         @Override
onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1675         public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session,
1676                                                int sequenceId,
1677                                                long frameNumber) {
1678 
1679             synchronized (mInterfaceLock) {
1680                 if (mRequestUpdatedNeeded && !mSingleCapture) {
1681                     mRequestUpdatedNeeded = false;
1682                     resumeInternalRepeatingRequest(false);
1683                 } else if (mInternalRepeatingRequestEnabled && !mSingleCapture) {
1684                     resumeInternalRepeatingRequest(true);
1685                 }
1686             }
1687 
1688             if (mClientNotificationsEnabled) {
1689                 final long ident = Binder.clearCallingIdentity();
1690                 try {
1691                     mExecutor.execute(
1692                             () -> mCallbacks
1693                                     .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this,
1694                                             sequenceId));
1695                 } finally {
1696                     Binder.restoreCallingIdentity(ident);
1697                 }
1698             }
1699         }
1700 
1701         @Override
onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1702         public void onCaptureFailed(@NonNull CameraCaptureSession session,
1703                                     @NonNull CaptureRequest request,
1704                                     @NonNull CaptureFailure failure) {
1705 
1706             if (mClientNotificationsEnabled) {
1707                 final long ident = Binder.clearCallingIdentity();
1708                 try {
1709                     mExecutor.execute(
1710                             () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this,
1711                                     mClientRequest));
1712                 } finally {
1713                     Binder.restoreCallingIdentity(ident);
1714                 }
1715             }
1716         }
1717 
1718         @Override
onCaptureCompleted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)1719         public void onCaptureCompleted(@NonNull CameraCaptureSession session,
1720                                        @NonNull CaptureRequest request,
1721                                        @NonNull TotalCaptureResult result) {
1722             boolean notifyClient = mClientNotificationsEnabled;
1723             boolean processStatus = true;
1724 
1725             synchronized (mInterfaceLock) {
1726                 final Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
1727                 if (timestamp != null) {
1728                     if (mCaptureResultsSupported && mClientNotificationsEnabled &&
1729                             (mCaptureResultHandler == null)) {
1730                         mCaptureResultHandler = new CaptureResultHandler(mClientRequest, mExecutor,
1731                                 mCallbacks, result.getSequenceId());
1732                     }
1733                     if ((!mSingleCapture) && (mPreviewProcessorType ==
1734                             IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY)) {
1735                         CaptureStageImpl captureStage = null;
1736                         try {
1737                             captureStage = mPreviewRequestUpdateProcessor.process(
1738                                     result.getNativeMetadata(), result.getSequenceId());
1739                         } catch (RemoteException e) {
1740                             Log.e(TAG, "Extension service does not respond during " +
1741                                     "processing!");
1742                         }
1743                         if (captureStage != null) {
1744                             try {
1745                                 setRepeatingRequest(captureStage, this, request);
1746                                 mRequestUpdatedNeeded = true;
1747                             } catch (IllegalStateException e) {
1748                                 // This is possible in case the camera device closes and the
1749                                 // and the callback here is executed before the onClosed
1750                                 // notification.
1751                             } catch (CameraAccessException e) {
1752                                 Log.e(TAG, "Failed to update repeating request settings!");
1753                             }
1754                         } else {
1755                             mRequestUpdatedNeeded = false;
1756                         }
1757                     } else if (mPreviewProcessorType ==
1758                             IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
1759                         int idx = mPendingResultMap.indexOfKey(timestamp);
1760 
1761                         if ((idx >= 0) && (mPendingResultMap.get(timestamp).first == null)) {
1762                             // Image was dropped before we can receive the capture results
1763                             if ((mCaptureResultHandler != null)) {
1764                                 mCaptureResultHandler.onCaptureCompleted(timestamp,
1765                                         initializeFilteredResults(result));
1766                             }
1767                             discardPendingRepeatingResults(idx, mPendingResultMap, false);
1768                         } else  if (idx >= 0) {
1769                             // Image came before the capture results
1770                             ParcelImage parcelImage = initializeParcelImage(
1771                                     mPendingResultMap.get(timestamp).first);
1772                             try {
1773                                 mPreviewImageProcessor.process(parcelImage, result,
1774                                         mCaptureResultHandler);
1775                             } catch (RemoteException e) {
1776                                 processStatus = false;
1777                                 Log.e(TAG, "Extension service does not respond during " +
1778                                         "processing, dropping frame!");
1779                             } catch (RuntimeException e) {
1780                                 // Runtime exceptions can happen in a few obscure cases where the
1781                                 // client tries to initialize a new capture session while this
1782                                 // session is still ongoing. In such scenario, the camera will
1783                                 // disconnect from the intermediate output surface, which will
1784                                 // invalidate the images that we acquired previously. This can
1785                                 // happen before we get notified via "onClosed" so there aren't
1786                                 // many options to avoid the exception.
1787                                 processStatus = false;
1788                                 Log.e(TAG, "Runtime exception encountered during buffer " +
1789                                         "processing, dropping frame!");
1790                             } finally {
1791                                 parcelImage.buffer.close();
1792                                 mPendingResultMap.get(timestamp).first.close();
1793                             }
1794                             discardPendingRepeatingResults(idx, mPendingResultMap, false);
1795                         } else {
1796                             // Image not yet available
1797                             notifyClient = false;
1798                             mPendingResultMap.put(timestamp,
1799                                     new Pair<>(null,
1800                                             result));
1801                         }
1802                     } else {
1803                         // No special handling for PROCESSOR_TYPE_NONE
1804                     }
1805                     if (notifyClient) {
1806                         final long ident = Binder.clearCallingIdentity();
1807                         try {
1808                             if (processStatus) {
1809                                 mExecutor.execute(() -> mCallbacks
1810                                         .onCaptureProcessStarted(
1811                                                 CameraExtensionSessionImpl.this,
1812                                                 mClientRequest));
1813                                 if ((mCaptureResultHandler != null) && (mPreviewProcessorType !=
1814                                         IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) {
1815                                     mCaptureResultHandler.onCaptureCompleted(timestamp,
1816                                             initializeFilteredResults(result));
1817                                 }
1818                             } else {
1819                                 mExecutor.execute(
1820                                         () -> mCallbacks
1821                                                 .onCaptureFailed(
1822                                                         CameraExtensionSessionImpl.this,
1823                                                         mClientRequest));
1824                             }
1825                         } finally {
1826                             Binder.restoreCallingIdentity(ident);
1827                         }
1828                     }
1829                 } else {
1830                     Log.e(TAG,
1831                             "Result without valid sensor timestamp!");
1832                 }
1833             }
1834 
1835             if (!notifyClient) {
1836                 notifyConfigurationSuccess();
1837             }
1838         }
1839 
resumeInternalRepeatingRequest(boolean internal)1840         private void resumeInternalRepeatingRequest(boolean internal) {
1841             try {
1842                 if (internal) {
1843                     setRepeatingRequest(mPreviewExtender.getCaptureStage(),
1844                             new PreviewRequestHandler(null, null, null,
1845                                     mRepeatingImageCallback));
1846                 } else {
1847                     setRepeatingRequest(mPreviewExtender.getCaptureStage(), this, mClientRequest);
1848                 }
1849             } catch (RemoteException e) {
1850                 Log.e(TAG, "Failed to resume internal repeating request, extension service"
1851                         + " fails to respond!");
1852             } catch (IllegalStateException e) {
1853                 // This is possible in case we try to resume before the state "onClosed"
1854                 // notification is able to reach us.
1855                 Log.w(TAG, "Failed to resume internal repeating request!");
1856             } catch (CameraAccessException e) {
1857                 Log.e(TAG, "Failed to resume internal repeating request!");
1858             }
1859         }
1860 
1861         // Find the timestamp of the oldest pending buffer
calculatePruneThreshold( LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap)1862         private Long calculatePruneThreshold(
1863                 LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap) {
1864             long oldestTimestamp = Long.MAX_VALUE;
1865             for (int idx = 0; idx < previewMap.size(); idx++) {
1866                 Pair<Image, TotalCaptureResult> entry = previewMap.valueAt(idx);
1867                 long timestamp = previewMap.keyAt(idx);
1868                 if ((entry.first != null) && (timestamp < oldestTimestamp)) {
1869                     oldestTimestamp = timestamp;
1870                 }
1871             }
1872             return (oldestTimestamp == Long.MAX_VALUE) ? 0 : oldestTimestamp;
1873         }
1874 
discardPendingRepeatingResults(int idx, LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap, boolean notifyCurrentIndex)1875         private void discardPendingRepeatingResults(int idx, LongSparseArray<Pair<Image,
1876                 TotalCaptureResult>> previewMap, boolean notifyCurrentIndex) {
1877             if (idx < 0) {
1878                 return;
1879             }
1880             for (int i = idx; i >= 0; i--) {
1881                 if (previewMap.valueAt(i).first != null) {
1882                     previewMap.valueAt(i).first.close();
1883                 } else if (mClientNotificationsEnabled && (previewMap.valueAt(i).second != null) &&
1884                         ((i != idx) || notifyCurrentIndex)) {
1885                     TotalCaptureResult result = previewMap.valueAt(i).second;
1886                     Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
1887                     if (mCaptureResultHandler != null) {
1888                         mCaptureResultHandler.onCaptureCompleted(timestamp,
1889                                 initializeFilteredResults(result));
1890                     }
1891 
1892                     Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i));
1893                     final long ident = Binder.clearCallingIdentity();
1894                     try {
1895                         mExecutor.execute(
1896                                 () -> mCallbacks
1897                                         .onCaptureFailed(CameraExtensionSessionImpl.this,
1898                                                 mClientRequest));
1899                     } finally {
1900                         Binder.restoreCallingIdentity(ident);
1901                     }
1902 
1903                 }
1904                 previewMap.removeAt(i);
1905             }
1906         }
1907 
1908         private class ImageForwardCallback implements OnImageAvailableListener {
1909             private final ImageWriter mOutputWriter;
1910 
ImageForwardCallback(@onNull ImageWriter imageWriter)1911             public ImageForwardCallback(@NonNull ImageWriter imageWriter) {
1912                 mOutputWriter = imageWriter;
1913             }
1914 
1915             @Override
onImageDropped(long timestamp)1916             public void onImageDropped(long timestamp) {
1917                 discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),
1918                         mPendingResultMap, true);
1919             }
1920 
1921             @Override
onImageAvailable(ImageReader reader, Image img)1922             public void onImageAvailable(ImageReader reader, Image img) {
1923                 if (img == null) {
1924                     Log.e(TAG, "Invalid image!");
1925                     return;
1926                 }
1927 
1928                 try {
1929                     mOutputWriter.queueInputImage(img);
1930                 } catch (IllegalStateException e) {
1931                     // This is possible in case the client disconnects from the output surface
1932                     // abruptly.
1933                     Log.w(TAG, "Output surface likely abandoned, dropping buffer!");
1934                     img.close();
1935                 } catch (RuntimeException e) {
1936                     // NOTE: This is intended to catch RuntimeException from ImageReader.detachImage
1937                     // ImageReader.detachImage is not supposed to throw RuntimeExceptions but the
1938                     // bug went unchecked for a few years and now its behavior cannot be changed
1939                     // without breaking backwards compatibility.
1940 
1941                     if (!e.getClass().equals(RuntimeException.class)) {
1942                         // re-throw any exceptions that aren't base RuntimeException since they are
1943                         // coming from elsewhere, and we shouldn't silently drop those.
1944                         throw e;
1945                     }
1946 
1947                     Log.w(TAG, "Output surface likely abandoned, dropping buffer!");
1948                     img.close();
1949                 }
1950             }
1951         }
1952 
1953         private class ImageProcessCallback implements OnImageAvailableListener {
1954 
1955             @Override
onImageDropped(long timestamp)1956             public void onImageDropped(long timestamp) {
1957                 discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),
1958                         mPendingResultMap, true);
1959                 // Add an empty frame&results entry to flag that we dropped a frame
1960                 // and valid capture results can immediately return to client.
1961                 mPendingResultMap.put(timestamp, new Pair<>(null, null));
1962             }
1963 
1964             @Override
onImageAvailable(ImageReader reader, Image img)1965             public void onImageAvailable(ImageReader reader, Image img) {
1966                 if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) {
1967                     // We reached the maximum acquired images limit. This is possible in case we
1968                     // have capture failures that result in absent or missing capture results. In
1969                     // such scenario we can prune the oldest pending buffer.
1970                     discardPendingRepeatingResults(
1971                             mPendingResultMap
1972                                     .indexOfKey(calculatePruneThreshold(mPendingResultMap)),
1973                             mPendingResultMap, true);
1974                 }
1975 
1976                 if (img == null) {
1977                     Log.e(TAG,
1978                             "Invalid preview buffer!");
1979                     return;
1980                 }
1981                 try {
1982                     reader.detachImage(img);
1983                 } catch (IllegalStateException e) {
1984                     Log.e(TAG, "Failed to detach image!");
1985                     img.close();
1986                     return;
1987                 } catch (RuntimeException e) {
1988                     // NOTE: This is intended to catch RuntimeException from ImageReader.detachImage
1989                     // ImageReader.detachImage is not supposed to throw RuntimeExceptions but the
1990                     // bug went unchecked for a few years and now its behavior cannot be changed
1991                     // without breaking backwards compatibility.
1992 
1993                     if (!e.getClass().equals(RuntimeException.class)) {
1994                         // re-throw any exceptions that aren't base RuntimeException since they are
1995                         // coming from elsewhere, and we shouldn't silently drop those.
1996                         throw e;
1997                     }
1998 
1999                     Log.e(TAG, "Failed to detach image!");
2000                     img.close();
2001                     return;
2002                 }
2003 
2004                 long timestamp = img.getTimestamp();
2005                 int idx = mPendingResultMap.indexOfKey(timestamp);
2006                 if (idx >= 0) {
2007                     boolean processStatus = true;
2008                     ParcelImage parcelImage = initializeParcelImage(img);
2009                     try {
2010                         mPreviewImageProcessor.process(parcelImage,
2011                                 mPendingResultMap.get(timestamp).second, mCaptureResultHandler);
2012                     } catch (RemoteException e) {
2013                         processStatus = false;
2014                         Log.e(TAG, "Extension service does not respond during " +
2015                                 "processing, dropping frame!");
2016                     } finally {
2017                         parcelImage.buffer.close();
2018                         img.close();
2019                     }
2020                     discardPendingRepeatingResults(idx, mPendingResultMap, false);
2021                     if (mClientNotificationsEnabled) {
2022                         final long ident = Binder.clearCallingIdentity();
2023                         try {
2024                             if (processStatus) {
2025                                 mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted(
2026                                         CameraExtensionSessionImpl.this,
2027                                         mClientRequest));
2028                             } else {
2029                                 mExecutor.execute(() -> mCallbacks.onCaptureFailed(
2030                                         CameraExtensionSessionImpl.this,
2031                                         mClientRequest));
2032                             }
2033                         } finally {
2034                             Binder.restoreCallingIdentity(ident);
2035                         }
2036                     }
2037                 } else {
2038                     mPendingResultMap.put(timestamp, new Pair<>(img, null));
2039                 }
2040             }
2041         }
2042     }
2043 
initializeFilteredResults(TotalCaptureResult result)2044     private CameraMetadataNative initializeFilteredResults(TotalCaptureResult result) {
2045         CameraMetadataNative captureResults = new CameraMetadataNative();
2046         for (CaptureResult.Key key : mSupportedResultKeys) {
2047             Object value = result.get(key);
2048             if (value != null) {
2049                 captureResults.set(key, value);
2050             }
2051         }
2052         return captureResults;
2053     }
2054 
findSmallestAspectMatchedSize(@onNull List<Size> sizes, @NonNull Size arSize)2055     private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes,
2056             @NonNull Size arSize) {
2057         final float TOLL = .01f;
2058 
2059         if (arSize.getHeight() == 0) {
2060             throw new IllegalArgumentException("Invalid input aspect ratio");
2061         }
2062 
2063         float targetAR = ((float) arSize.getWidth()) / arSize.getHeight();
2064         Size ret = null;
2065         Size fallbackSize = null;
2066         for (Size sz : sizes) {
2067             if (fallbackSize == null) {
2068                 fallbackSize = sz;
2069             }
2070             if ((sz.getHeight() > 0) &&
2071                     ((ret == null) ||
2072                             (ret.getWidth() * ret.getHeight()) <
2073                                     (sz.getWidth() * sz.getHeight()))) {
2074                 float currentAR = ((float) sz.getWidth()) / sz.getHeight();
2075                 if (Math.abs(currentAR - targetAR) <= TOLL) {
2076                     ret = sz;
2077                 }
2078             }
2079         }
2080         if (ret == null) {
2081             Log.e(TAG, "AR matched size not found returning first size in list");
2082             ret = fallbackSize;
2083         }
2084 
2085         return ret;
2086     }
2087 
initializeParcelImage(Image img)2088     private static ParcelImage initializeParcelImage(Image img) {
2089         ParcelImage parcelImage = new ParcelImage();
2090         parcelImage.buffer = img.getHardwareBuffer();
2091         try {
2092             SyncFence fd = img.getFence();
2093             if (fd.isValid()) {
2094                 parcelImage.fence = fd.getFdDup();
2095             }
2096         } catch (IOException e) {
2097             Log.e(TAG, "Failed to parcel buffer fence!");
2098         }
2099         parcelImage.width = img.getWidth();
2100         parcelImage.height = img.getHeight();
2101         parcelImage.format = img.getFormat();
2102         parcelImage.timestamp = img.getTimestamp();
2103         parcelImage.transform = img.getTransform();
2104         parcelImage.scalingMode = img.getScalingMode();
2105         parcelImage.planeCount = img.getPlaneCount();
2106         parcelImage.crop = img.getCropRect();
2107 
2108         return parcelImage;
2109     }
2110 
initializeParcelable( HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap, Integer jpegOrientation, Byte jpegQuality)2111     private static List<CaptureBundle> initializeParcelable(
2112             HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap, Integer jpegOrientation,
2113             Byte jpegQuality) {
2114         ArrayList<CaptureBundle> ret = new ArrayList<>();
2115         for (Integer stagetId : captureMap.keySet()) {
2116             Pair<Image, TotalCaptureResult> entry = captureMap.get(stagetId);
2117             CaptureBundle bundle = new CaptureBundle();
2118             bundle.stage = stagetId;
2119             bundle.captureImage = initializeParcelImage(entry.first);
2120             bundle.sequenceId = entry.second.getSequenceId();
2121             bundle.captureResult = entry.second.getNativeMetadata();
2122             if (jpegOrientation != null) {
2123                 bundle.captureResult.set(CaptureResult.JPEG_ORIENTATION, jpegOrientation);
2124             }
2125             if (jpegQuality != null) {
2126                 bundle.captureResult.set(CaptureResult.JPEG_QUALITY, jpegQuality);
2127             }
2128             ret.add(bundle);
2129         }
2130 
2131         return ret;
2132     }
2133 }
2134