1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.hardware.camera2.impl;
18 
19 import android.content.Context;
20 import android.graphics.ImageFormat;
21 import android.graphics.SurfaceTexture;
22 import android.hardware.HardwareBuffer;
23 import android.hardware.camera2.CameraAccessException;
24 import android.hardware.camera2.CameraCaptureSession;
25 import android.hardware.camera2.CameraCharacteristics;
26 import android.hardware.camera2.CameraDevice;
27 import android.hardware.camera2.CameraExtensionCharacteristics;
28 import android.hardware.camera2.CameraExtensionSession;
29 import android.hardware.camera2.CameraManager;
30 import android.hardware.camera2.CaptureFailure;
31 import android.hardware.camera2.CaptureRequest;
32 import android.hardware.camera2.CaptureResult;
33 import android.hardware.camera2.extension.CaptureBundle;
34 import android.hardware.camera2.extension.CaptureStageImpl;
35 import android.hardware.camera2.extension.ICaptureProcessorImpl;
36 import android.hardware.camera2.extension.IImageCaptureExtenderImpl;
37 import android.hardware.camera2.extension.IInitializeSessionCallback;
38 import android.hardware.camera2.extension.IPreviewExtenderImpl;
39 import android.hardware.camera2.extension.IRequestUpdateProcessorImpl;
40 import android.hardware.camera2.extension.ParcelImage;
41 import android.hardware.camera2.TotalCaptureResult;
42 import android.hardware.camera2.params.ExtensionSessionConfiguration;
43 import android.hardware.camera2.params.OutputConfiguration;
44 import android.hardware.camera2.params.SessionConfiguration;
45 import android.hardware.camera2.utils.SurfaceUtils;
46 import android.media.Image;
47 import android.media.ImageReader;
48 import android.media.ImageWriter;
49 import android.os.Binder;
50 import android.os.Handler;
51 import android.os.HandlerThread;
52 import android.os.ParcelFileDescriptor;
53 import android.os.RemoteException;
54 import android.annotation.NonNull;
55 import android.annotation.Nullable;
56 import android.annotation.RequiresPermission;
57 import android.util.Log;
58 import android.util.LongSparseArray;
59 import android.util.Pair;
60 import android.util.Size;
61 import android.view.Surface;
62 
63 import java.io.Closeable;
64 import java.io.IOException;
65 import java.util.ArrayList;
66 import java.util.HashMap;
67 import java.util.List;
68 import java.util.Map;
69 import java.util.concurrent.Executor;
70 
71 public final class CameraExtensionSessionImpl extends CameraExtensionSession {
72     private static final int PREVIEW_QUEUE_SIZE = 3;
73     private static final String TAG = "CameraExtensionSessionImpl";
74 
75     private final Executor mExecutor;
76     private final CameraDevice mCameraDevice;
77     private final long mExtensionClientId;
78     private final IImageCaptureExtenderImpl mImageExtender;
79     private final IPreviewExtenderImpl mPreviewExtender;
80     private final Handler mHandler;
81     private final HandlerThread mHandlerThread;
82     private final StateCallback mCallbacks;
83     private final List<Size> mSupportedPreviewSizes;
84     private final InitializeSessionHandler mInitializeHandler;
85 
86     private CameraCaptureSession mCaptureSession = null;
87     private Surface mCameraRepeatingSurface, mClientRepeatingRequestSurface;
88     private Surface mCameraBurstSurface, mClientCaptureSurface;
89     private ImageReader mRepeatingRequestImageReader = null;
90     private ImageReader mBurstCaptureImageReader = null;
91     private ImageReader mStubCaptureImageReader = null;
92     private ImageWriter mRepeatingRequestImageWriter = null;
93     private CameraOutputImageCallback mRepeatingRequestImageCallback = null;
94     private CameraOutputImageCallback mBurstCaptureImageCallback = null;
95 
96     private CameraExtensionJpegProcessor mImageJpegProcessor = null;
97     private ICaptureProcessorImpl mImageProcessor = null;
98     private CameraExtensionForwardProcessor mPreviewImageProcessor = null;
99     private IRequestUpdateProcessorImpl mPreviewRequestUpdateProcessor = null;
100     private int mPreviewProcessorType = IPreviewExtenderImpl.PROCESSOR_TYPE_NONE;
101 
102     private boolean mInitialized;
103     // Enable/Disable internal preview/(repeating request). Extensions expect
104     // that preview/(repeating request) is enabled and active at any point in time.
105     // In case the client doesn't explicitly enable repeating requests, the framework
106     // will do so internally.
107     private boolean mInternalRepeatingRequestEnabled = true;
108 
109     // Lock to synchronize cross-thread access to device public interface
110     final Object mInterfaceLock = new Object(); // access from this class and Session only!
111 
nativeGetSurfaceFormat(Surface surface)112     private static int nativeGetSurfaceFormat(Surface surface) {
113         return SurfaceUtils.getSurfaceFormat(surface);
114     }
115 
116     /**
117      * @hide
118      */
119     @RequiresPermission(android.Manifest.permission.CAMERA)
createCameraExtensionSession( @onNull CameraDevice cameraDevice, @NonNull Context ctx, @NonNull ExtensionSessionConfiguration config)120     public static CameraExtensionSessionImpl createCameraExtensionSession(
121             @NonNull CameraDevice cameraDevice,
122             @NonNull Context ctx,
123             @NonNull ExtensionSessionConfiguration config)
124             throws CameraAccessException, RemoteException {
125         long clientId = CameraExtensionCharacteristics.registerClient(ctx);
126         if (clientId < 0) {
127             throw new UnsupportedOperationException("Unsupported extension!");
128         }
129 
130         String cameraId = cameraDevice.getId();
131         CameraManager manager = ctx.getSystemService(CameraManager.class);
132         CameraCharacteristics chars = manager.getCameraCharacteristics(cameraId);
133         CameraExtensionCharacteristics extensionChars = new CameraExtensionCharacteristics(ctx,
134                 cameraId, chars);
135 
136         if (!CameraExtensionCharacteristics.isExtensionSupported(cameraDevice.getId(),
137                 config.getExtension(), chars)) {
138             throw new UnsupportedOperationException("Unsupported extension type: " +
139                     config.getExtension());
140         }
141 
142         if (config.getOutputConfigurations().isEmpty() ||
143                 config.getOutputConfigurations().size() > 2) {
144             throw new IllegalArgumentException("Unexpected amount of output surfaces, received: " +
145                     config.getOutputConfigurations().size() + " expected <= 2");
146         }
147 
148         Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders =
149                 CameraExtensionCharacteristics.initializeExtension(config.getExtension());
150 
151         int suitableSurfaceCount = 0;
152         List<Size> supportedPreviewSizes = extensionChars.getExtensionSupportedSizes(
153                 config.getExtension(), SurfaceTexture.class);
154         Surface repeatingRequestSurface = CameraExtensionUtils.getRepeatingRequestSurface(
155                 config.getOutputConfigurations(), supportedPreviewSizes);
156         if (repeatingRequestSurface != null) {
157             suitableSurfaceCount++;
158         }
159 
160         HashMap<Integer, List<Size>> supportedCaptureSizes = new HashMap<>();
161         for (int format : CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
162             List<Size> supportedSizes = extensionChars.getExtensionSupportedSizes(
163                     config.getExtension(), format);
164             if (supportedSizes != null) {
165                 supportedCaptureSizes.put(format, supportedSizes);
166             }
167         }
168         Surface burstCaptureSurface = CameraExtensionUtils.getBurstCaptureSurface(
169                 config.getOutputConfigurations(), supportedCaptureSizes);
170         if (burstCaptureSurface != null) {
171             suitableSurfaceCount++;
172         }
173 
174         if (suitableSurfaceCount != config.getOutputConfigurations().size()) {
175             throw new IllegalArgumentException("One or more unsupported output surfaces found!");
176         }
177 
178         extenders.first.init(cameraId, chars.getNativeMetadata());
179         extenders.first.onInit(cameraId, chars.getNativeMetadata());
180         extenders.second.init(cameraId, chars.getNativeMetadata());
181         extenders.second.onInit(cameraId, chars.getNativeMetadata());
182 
183         CameraExtensionSessionImpl session = new CameraExtensionSessionImpl(
184                 extenders.second,
185                 extenders.first,
186                 supportedPreviewSizes,
187                 clientId,
188                 cameraDevice,
189                 repeatingRequestSurface,
190                 burstCaptureSurface,
191                 config.getStateCallback(),
192                 config.getExecutor());
193 
194         session.initialize();
195 
196         return session;
197     }
198 
CameraExtensionSessionImpl(@onNull IImageCaptureExtenderImpl imageExtender, @NonNull IPreviewExtenderImpl previewExtender, @NonNull List<Size> previewSizes, long extensionClientId, @NonNull CameraDevice cameraDevice, @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, @NonNull StateCallback callback, @NonNull Executor executor)199     public CameraExtensionSessionImpl(@NonNull IImageCaptureExtenderImpl imageExtender,
200             @NonNull IPreviewExtenderImpl previewExtender,
201             @NonNull List<Size> previewSizes,
202             long extensionClientId,
203             @NonNull CameraDevice cameraDevice,
204             @Nullable Surface repeatingRequestSurface,
205             @Nullable Surface burstCaptureSurface,
206             @NonNull StateCallback callback,
207             @NonNull Executor executor) {
208         mExtensionClientId = extensionClientId;
209         mImageExtender = imageExtender;
210         mPreviewExtender = previewExtender;
211         mCameraDevice = cameraDevice;
212         mCallbacks = callback;
213         mExecutor = executor;
214         mClientRepeatingRequestSurface = repeatingRequestSurface;
215         mClientCaptureSurface = burstCaptureSurface;
216         mSupportedPreviewSizes = previewSizes;
217         mHandlerThread = new HandlerThread(TAG);
218         mHandlerThread.start();
219         mHandler = new Handler(mHandlerThread.getLooper());
220         mInitialized = false;
221         mInitializeHandler = new InitializeSessionHandler();
222     }
223 
initializeRepeatingRequestPipeline()224     private void initializeRepeatingRequestPipeline() throws RemoteException {
225         CameraExtensionUtils.SurfaceInfo repeatingSurfaceInfo =
226                 new CameraExtensionUtils.SurfaceInfo();
227         mPreviewProcessorType = mPreviewExtender.getProcessorType();
228         if (mClientRepeatingRequestSurface != null) {
229             repeatingSurfaceInfo = CameraExtensionUtils.querySurface(
230                     mClientRepeatingRequestSurface);
231         } else {
232             // Make the intermediate surface behave as any regular 'SurfaceTexture'
233             CameraExtensionUtils.SurfaceInfo captureSurfaceInfo = CameraExtensionUtils.querySurface(
234                     mClientCaptureSurface);
235             Size captureSize = new Size(captureSurfaceInfo.mWidth, captureSurfaceInfo.mHeight);
236             Size previewSize = findSmallestAspectMatchedSize(mSupportedPreviewSizes, captureSize);
237             repeatingSurfaceInfo.mWidth = previewSize.getWidth();
238             repeatingSurfaceInfo.mHeight = previewSize.getHeight();
239             repeatingSurfaceInfo.mUsage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE;
240         }
241 
242         if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
243             try {
244                 mPreviewImageProcessor = new CameraExtensionForwardProcessor(
245                         mPreviewExtender.getPreviewImageProcessor(), repeatingSurfaceInfo.mFormat,
246                         repeatingSurfaceInfo.mUsage, mHandler);
247             } catch (ClassCastException e) {
248                 throw new UnsupportedOperationException("Failed casting preview processor!");
249             }
250             mPreviewImageProcessor.onImageFormatUpdate(
251                     CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT);
252             mPreviewImageProcessor.onResolutionUpdate(new Size(repeatingSurfaceInfo.mWidth,
253                     repeatingSurfaceInfo.mHeight));
254             mPreviewImageProcessor.onOutputSurface(null, -1);
255             mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth,
256                     repeatingSurfaceInfo.mHeight,
257                     CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, PREVIEW_QUEUE_SIZE,
258                     repeatingSurfaceInfo.mUsage);
259             mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface();
260         } else if (mPreviewProcessorType ==
261                 IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
262             try {
263                 mPreviewRequestUpdateProcessor = mPreviewExtender.getRequestUpdateProcessor();
264             } catch (ClassCastException e) {
265                 throw new UnsupportedOperationException("Failed casting preview processor!");
266             }
267             mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth,
268                     repeatingSurfaceInfo.mHeight,
269                     CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT,
270                     PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage);
271             mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface();
272             android.hardware.camera2.extension.Size sz =
273                     new android.hardware.camera2.extension.Size();
274             sz.width = repeatingSurfaceInfo.mWidth;
275             sz.height = repeatingSurfaceInfo.mHeight;
276             mPreviewRequestUpdateProcessor.onResolutionUpdate(sz);
277             mPreviewRequestUpdateProcessor.onImageFormatUpdate(
278                     CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT);
279         } else {
280             mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth,
281                     repeatingSurfaceInfo.mHeight,
282                     CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT,
283                     PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage);
284             mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface();
285         }
286         mRepeatingRequestImageCallback = new CameraOutputImageCallback(
287                 mRepeatingRequestImageReader);
288         mRepeatingRequestImageReader
289                 .setOnImageAvailableListener(mRepeatingRequestImageCallback, mHandler);
290     }
291 
initializeBurstCapturePipeline()292     private void initializeBurstCapturePipeline() throws RemoteException {
293         mImageProcessor = mImageExtender.getCaptureProcessor();
294         if ((mImageProcessor == null) && (mImageExtender.getMaxCaptureStage() != 1)) {
295             throw new UnsupportedOperationException("Multiple stages expected without" +
296                     " a valid capture processor!");
297         }
298 
299         if (mImageProcessor != null) {
300             if (mClientCaptureSurface != null) {
301                 CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface(
302                         mClientCaptureSurface);
303                 if (surfaceInfo.mFormat == ImageFormat.JPEG) {
304                     mImageJpegProcessor = new CameraExtensionJpegProcessor(mImageProcessor);
305                     mImageProcessor = mImageJpegProcessor;
306                 }
307                 mBurstCaptureImageReader = ImageReader.newInstance(surfaceInfo.mWidth,
308                         surfaceInfo.mHeight, CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT,
309                         mImageExtender.getMaxCaptureStage());
310             } else {
311                 // The client doesn't intend to trigger multi-frame capture, however the
312                 // image extender still needs to get initialized and the camera still capture
313                 // stream configured for the repeating request processing to work.
314                 mBurstCaptureImageReader = ImageReader.newInstance(
315                         mRepeatingRequestImageReader.getWidth(),
316                         mRepeatingRequestImageReader.getHeight(),
317                         CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1);
318                 // The still capture output is not going to be used but we still need a valid
319                 // surface to register.
320                 mStubCaptureImageReader = ImageReader.newInstance(
321                         mRepeatingRequestImageReader.getWidth(),
322                         mRepeatingRequestImageReader.getHeight(),
323                         CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1);
324                 mImageProcessor.onOutputSurface(mStubCaptureImageReader.getSurface(),
325                         CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT);
326             }
327 
328             mBurstCaptureImageCallback = new CameraOutputImageCallback(mBurstCaptureImageReader);
329             mBurstCaptureImageReader.setOnImageAvailableListener(mBurstCaptureImageCallback,
330                     mHandler);
331             mCameraBurstSurface = mBurstCaptureImageReader.getSurface();
332             android.hardware.camera2.extension.Size sz =
333                     new android.hardware.camera2.extension.Size();
334             sz.width = mBurstCaptureImageReader.getWidth();
335             sz.height = mBurstCaptureImageReader.getHeight();
336             mImageProcessor.onResolutionUpdate(sz);
337             mImageProcessor.onImageFormatUpdate(mBurstCaptureImageReader.getImageFormat());
338         } else {
339             if (mClientCaptureSurface != null) {
340                 // Redirect camera output directly in to client output surface
341                 mCameraBurstSurface = mClientCaptureSurface;
342             } else {
343                 // The client doesn't intend to trigger multi-frame capture, however the
344                 // image extender still needs to get initialized and the camera still capture
345                 // stream configured for the repeating request processing to work.
346                 mBurstCaptureImageReader = ImageReader.newInstance(
347                         mRepeatingRequestImageReader.getWidth(),
348                         mRepeatingRequestImageReader.getHeight(),
349                         // Camera devices accept only Jpeg output if the image processor is null
350                         ImageFormat.JPEG, 1);
351                 mCameraBurstSurface = mBurstCaptureImageReader.getSurface();
352             }
353         }
354     }
355 
finishPipelineInitialization()356     private void finishPipelineInitialization() throws RemoteException {
357         if (mClientRepeatingRequestSurface != null) {
358             if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
359                 mPreviewRequestUpdateProcessor.onOutputSurface(mClientRepeatingRequestSurface,
360                         nativeGetSurfaceFormat(mClientRepeatingRequestSurface));
361                 mRepeatingRequestImageWriter = ImageWriter.newInstance(
362                         mClientRepeatingRequestSurface,
363                         PREVIEW_QUEUE_SIZE,
364                         CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT);
365             } else if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_NONE) {
366                 mRepeatingRequestImageWriter = ImageWriter.newInstance(
367                         mClientRepeatingRequestSurface,
368                         PREVIEW_QUEUE_SIZE,
369                         CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT);
370             }
371         }
372         if ((mImageProcessor != null) && (mClientCaptureSurface != null)) {
373             CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface(
374                     mClientCaptureSurface);
375             mImageProcessor.onOutputSurface(mClientCaptureSurface, surfaceInfo.mFormat);
376         }
377     }
378 
379     /**
380      * @hide
381      */
initialize()382     public synchronized void initialize() throws CameraAccessException, RemoteException {
383         if (mInitialized) {
384             Log.d(TAG,
385                     "Session already initialized");
386             return;
387         }
388 
389         ArrayList<CaptureStageImpl> sessionParamsList = new ArrayList<>();
390         ArrayList<OutputConfiguration> outputList = new ArrayList<>();
391         initializeRepeatingRequestPipeline();
392         outputList.add(new OutputConfiguration(mCameraRepeatingSurface));
393         CaptureStageImpl previewSessionParams = mPreviewExtender.onPresetSession();
394         if (previewSessionParams != null) {
395             sessionParamsList.add(previewSessionParams);
396         }
397         initializeBurstCapturePipeline();
398         outputList.add(new OutputConfiguration(mCameraBurstSurface));
399         CaptureStageImpl stillCaptureSessionParams = mImageExtender.onPresetSession();
400         if (stillCaptureSessionParams != null) {
401             sessionParamsList.add(stillCaptureSessionParams);
402         }
403 
404         SessionConfiguration sessionConfig = new SessionConfiguration(
405                 SessionConfiguration.SESSION_REGULAR,
406                 outputList,
407                 new CameraExtensionUtils.HandlerExecutor(mHandler),
408                 new SessionStateHandler());
409 
410         if (!sessionParamsList.isEmpty()) {
411             CaptureRequest sessionParamRequest = createRequest(mCameraDevice, sessionParamsList,
412                     null, CameraDevice.TEMPLATE_PREVIEW);
413             sessionConfig.setSessionParameters(sessionParamRequest);
414         }
415 
416         mCameraDevice.createCaptureSession(sessionConfig);
417     }
418 
419     @Override
getDevice()420     public @NonNull CameraDevice getDevice() {
421         synchronized (mInterfaceLock) {
422             return mCameraDevice;
423         }
424     }
425 
426     @Override
setRepeatingRequest(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)427     public int setRepeatingRequest(@NonNull CaptureRequest request,
428                                    @NonNull Executor executor,
429                                    @NonNull ExtensionCaptureCallback listener)
430             throws CameraAccessException {
431         synchronized (mInterfaceLock) {
432             if (!mInitialized) {
433                 throw new IllegalStateException("Uninitialized component");
434             }
435 
436             if (mClientRepeatingRequestSurface == null) {
437                 throw new IllegalArgumentException("No registered preview surface");
438             }
439 
440             if (!request.containsTarget(mClientRepeatingRequestSurface) ||
441                     (request.getTargets().size() != 1)) {
442                 throw new IllegalArgumentException("Invalid repeating request output target!");
443             }
444 
445             mInternalRepeatingRequestEnabled = false;
446             try {
447                 return setRepeatingRequest(mPreviewExtender.getCaptureStage(),
448                         new RepeatingRequestHandler(request, executor, listener,
449                                 mRepeatingRequestImageCallback));
450             } catch (RemoteException e) {
451                 Log.e(TAG, "Failed to set repeating request! Extension service does not "
452                         + "respond");
453                 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR);
454             }
455         }
456     }
457 
compileInitialRequestList()458     private ArrayList<CaptureStageImpl> compileInitialRequestList() {
459         ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>();
460         try {
461             CaptureStageImpl initialPreviewParams = mPreviewExtender.onEnableSession();
462             if (initialPreviewParams != null) {
463                 captureStageList.add(initialPreviewParams);
464             }
465 
466             CaptureStageImpl initialStillCaptureParams = mImageExtender.onEnableSession();
467             if (initialStillCaptureParams != null) {
468                 captureStageList.add(initialStillCaptureParams);
469             }
470         } catch (RemoteException e) {
471             Log.e(TAG, "Failed to initialize session parameters! Extension service does not"
472                     + " respond!");
473         }
474 
475         return captureStageList;
476     }
477 
createBurstRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, CaptureRequest clientRequest, Surface target, int captureTemplate, Map<CaptureRequest, Integer> captureMap)478     private static List<CaptureRequest> createBurstRequest(CameraDevice cameraDevice,
479             List<CaptureStageImpl> captureStageList, CaptureRequest clientRequest,
480             Surface target, int captureTemplate, Map<CaptureRequest, Integer> captureMap) {
481         CaptureRequest.Builder requestBuilder;
482         ArrayList<CaptureRequest> ret = new ArrayList<>();
483         for (CaptureStageImpl captureStage : captureStageList) {
484             try {
485                 requestBuilder = cameraDevice.createCaptureRequest(captureTemplate);
486             } catch (CameraAccessException e) {
487                 return null;
488             }
489 
490             // This will override the extension capture stage jpeg parameters with the user set
491             // jpeg quality and rotation. This will guarantee that client configured jpeg
492             // parameters always have highest priority.
493             Integer jpegRotation = clientRequest.get(CaptureRequest.JPEG_ORIENTATION);
494             if (jpegRotation != null) {
495                 captureStage.parameters.set(CaptureRequest.JPEG_ORIENTATION, jpegRotation);
496             }
497             Byte jpegQuality = clientRequest.get(CaptureRequest.JPEG_QUALITY);
498             if (jpegQuality != null) {
499                 captureStage.parameters.set(CaptureRequest.JPEG_QUALITY, jpegQuality);
500             }
501 
502             requestBuilder.addTarget(target);
503             CaptureRequest request = requestBuilder.build();
504             CameraMetadataNative.update(request.getNativeMetadata(), captureStage.parameters);
505             ret.add(request);
506             captureMap.put(request, captureStage.id);
507         }
508 
509         return ret;
510     }
511 
createRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate)512     private static CaptureRequest createRequest(CameraDevice cameraDevice,
513                                                 List<CaptureStageImpl> captureStageList,
514                                                 Surface target,
515                                                 int captureTemplate) throws CameraAccessException {
516         CaptureRequest.Builder requestBuilder;
517         requestBuilder = cameraDevice.createCaptureRequest(captureTemplate);
518         if (target != null) {
519             requestBuilder.addTarget(target);
520         }
521 
522         CaptureRequest ret = requestBuilder.build();
523         for (CaptureStageImpl captureStage : captureStageList) {
524             if (captureStage != null) {
525                 CameraMetadataNative.update(ret.getNativeMetadata(), captureStage.parameters);
526             }
527         }
528         return ret;
529     }
530 
531     @Override
capture(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)532     public int capture(@NonNull CaptureRequest request,
533                        @NonNull Executor executor,
534                        @NonNull ExtensionCaptureCallback listener) throws CameraAccessException {
535         if (!mInitialized) {
536             throw new IllegalStateException("Uninitialized component");
537         }
538 
539         if (mClientCaptureSurface == null) {
540             throw new IllegalArgumentException("No output surface registered for single requests!");
541         }
542 
543         if (!request.containsTarget(mClientCaptureSurface) || (request.getTargets().size() != 1)) {
544             throw new IllegalArgumentException("Invalid single capture output target!");
545         }
546 
547         HashMap<CaptureRequest, Integer> requestMap = new HashMap<>();
548         List<CaptureRequest> burstRequest;
549         try {
550             burstRequest = createBurstRequest(mCameraDevice,
551                     mImageExtender.getCaptureStages(), request, mCameraBurstSurface,
552                     CameraDevice.TEMPLATE_STILL_CAPTURE, requestMap);
553         } catch (RemoteException e) {
554             Log.e(TAG, "Failed to initialize internal burst request! Extension service does"
555                     + " not respond!");
556             throw new CameraAccessException(CameraAccessException.CAMERA_ERROR);
557         }
558         if (burstRequest == null) {
559             throw new UnsupportedOperationException("Failed to create still capture burst request");
560         }
561 
562         return mCaptureSession.captureBurstRequests(burstRequest,
563                 new CameraExtensionUtils.HandlerExecutor(mHandler),
564                 new BurstRequestHandler(request, executor, listener, requestMap,
565                         mBurstCaptureImageCallback));
566     }
567 
568     @Override
stopRepeating()569     public void stopRepeating() throws CameraAccessException {
570         synchronized (mInterfaceLock) {
571             if (!mInitialized) {
572                 throw new IllegalStateException("Uninitialized component");
573             }
574 
575             mInternalRepeatingRequestEnabled = true;
576             mCaptureSession.stopRepeating();
577         }
578     }
579 
580     @Override
close()581     public void close() throws CameraAccessException {
582         synchronized (mInterfaceLock) {
583             if (mInitialized) {
584                 mInternalRepeatingRequestEnabled = false;
585                 mCaptureSession.stopRepeating();
586 
587                 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>();
588                 try {
589                     CaptureStageImpl disableParams = mPreviewExtender.onDisableSession();
590                     if (disableParams != null) {
591                         captureStageList.add(disableParams);
592                     }
593 
594                     CaptureStageImpl disableStillCaptureParams =
595                             mImageExtender.onDisableSession();
596                     if (disableStillCaptureParams != null) {
597                         captureStageList.add(disableStillCaptureParams);
598                     }
599                 } catch (RemoteException e) {
600                     Log.e(TAG, "Failed to disable extension! Extension service does not "
601                             + "respond!");
602                 }
603                 if (!captureStageList.isEmpty()) {
604                     CaptureRequest disableRequest = createRequest(mCameraDevice, captureStageList,
605                             mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW);
606                     mCaptureSession.capture(disableRequest,
607                             new CloseRequestHandler(mRepeatingRequestImageCallback), mHandler);
608                 }
609 
610                 mCaptureSession.close();
611             }
612         }
613     }
614 
setInitialCaptureRequest(List<CaptureStageImpl> captureStageList, InitialRequestHandler requestHandler)615     private void setInitialCaptureRequest(List<CaptureStageImpl> captureStageList,
616                                           InitialRequestHandler requestHandler)
617             throws CameraAccessException {
618         CaptureRequest initialRequest = createRequest(mCameraDevice,
619                 captureStageList, mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW);
620         mCaptureSession.capture(initialRequest, requestHandler, mHandler);
621     }
622 
setRepeatingRequest(CaptureStageImpl captureStage, CameraCaptureSession.CaptureCallback requestHandler)623     private int setRepeatingRequest(CaptureStageImpl captureStage,
624                                     CameraCaptureSession.CaptureCallback requestHandler)
625             throws CameraAccessException {
626         ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>();
627         captureStageList.add(captureStage);
628         CaptureRequest repeatingRequest = createRequest(mCameraDevice,
629                 captureStageList, mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW);
630         return mCaptureSession.setSingleRepeatingRequest(repeatingRequest,
631                 new CameraExtensionUtils.HandlerExecutor(mHandler), requestHandler);
632     }
633 
634     /** @hide */
release(boolean skipCloseNotification)635     public void release(boolean skipCloseNotification) {
636         boolean notifyClose = false;
637 
638         synchronized (mInterfaceLock) {
639             mInternalRepeatingRequestEnabled = false;
640             mHandlerThread.quitSafely();
641 
642             try {
643                 mPreviewExtender.onDeInit();
644                 mImageExtender.onDeInit();
645             } catch (RemoteException e) {
646                 Log.e(TAG, "Failed to release extensions! Extension service does not"
647                         + " respond!");
648             }
649 
650             if (mExtensionClientId >= 0) {
651                 CameraExtensionCharacteristics.unregisterClient(mExtensionClientId);
652                 if (mInitialized) {
653                     notifyClose = true;
654                     CameraExtensionCharacteristics.releaseSession();
655                 }
656             }
657             mInitialized = false;
658 
659             if (mRepeatingRequestImageCallback != null) {
660                 mRepeatingRequestImageCallback.close();
661                 mRepeatingRequestImageCallback = null;
662             }
663 
664             if (mRepeatingRequestImageReader != null) {
665                 mRepeatingRequestImageReader.close();
666                 mRepeatingRequestImageReader = null;
667             }
668 
669             if (mBurstCaptureImageCallback != null) {
670                 mBurstCaptureImageCallback.close();
671                 mBurstCaptureImageCallback = null;
672             }
673 
674             if (mBurstCaptureImageReader != null) {
675                 mBurstCaptureImageReader.close();
676                 mBurstCaptureImageReader = null;
677             }
678 
679             if (mStubCaptureImageReader != null) {
680                 mStubCaptureImageReader.close();
681                 mStubCaptureImageReader = null;
682             }
683 
684             if (mRepeatingRequestImageWriter != null) {
685                 mRepeatingRequestImageWriter.close();
686                 mRepeatingRequestImageWriter = null;
687             }
688 
689             if (mPreviewImageProcessor != null) {
690                 mPreviewImageProcessor.close();
691                 mPreviewImageProcessor = null;
692             }
693 
694             if (mImageJpegProcessor != null) {
695                 mImageJpegProcessor.close();
696                 mImageJpegProcessor = null;
697             }
698 
699             mCaptureSession = null;
700             mImageProcessor = null;
701             mCameraRepeatingSurface = mClientRepeatingRequestSurface = null;
702             mCameraBurstSurface = mClientCaptureSurface = null;
703         }
704 
705         if (notifyClose && !skipCloseNotification) {
706             final long ident = Binder.clearCallingIdentity();
707             try {
708                 mExecutor.execute(() -> mCallbacks.onClosed(CameraExtensionSessionImpl.this));
709             } finally {
710                 Binder.restoreCallingIdentity(ident);
711             }
712         }
713     }
714 
notifyConfigurationFailure()715     private void notifyConfigurationFailure() {
716         synchronized (mInterfaceLock) {
717             if (mInitialized) {
718                 return;
719             }
720         }
721 
722         release(true /*skipCloseNotification*/);
723 
724         final long ident = Binder.clearCallingIdentity();
725         try {
726             mExecutor.execute(
727                     () -> mCallbacks.onConfigureFailed(CameraExtensionSessionImpl.this));
728         } finally {
729             Binder.restoreCallingIdentity(ident);
730         }
731     }
732 
notifyConfigurationSuccess()733     private void notifyConfigurationSuccess() {
734         synchronized (mInterfaceLock) {
735             if (mInitialized) {
736                 return;
737             } else {
738                 mInitialized = true;
739             }
740         }
741 
742         final long ident = Binder.clearCallingIdentity();
743         try {
744             mExecutor.execute(() -> mCallbacks.onConfigured(CameraExtensionSessionImpl.this));
745         } finally {
746             Binder.restoreCallingIdentity(ident);
747         }
748     }
749 
750     private class SessionStateHandler extends
751             android.hardware.camera2.CameraCaptureSession.StateCallback {
752         @Override
onClosed(@onNull CameraCaptureSession session)753         public void onClosed(@NonNull CameraCaptureSession session) {
754             release(false /*skipCloseNotification*/);
755         }
756 
757         @Override
onConfigureFailed(@onNull CameraCaptureSession session)758         public void onConfigureFailed(@NonNull CameraCaptureSession session) {
759             notifyConfigurationFailure();
760         }
761 
762         @Override
onConfigured(@onNull CameraCaptureSession session)763         public void onConfigured(@NonNull CameraCaptureSession session) {
764             synchronized (mInterfaceLock) {
765                 mCaptureSession = session;
766                 try {
767                     finishPipelineInitialization();
768                     CameraExtensionCharacteristics.initializeSession(mInitializeHandler);
769                 } catch (RemoteException e) {
770                     Log.e(TAG, "Failed to initialize session! Extension service does"
771                             + " not respond!");
772                     notifyConfigurationFailure();
773                 }
774             }
775         }
776     }
777 
778     private class InitializeSessionHandler extends IInitializeSessionCallback.Stub {
779         @Override
onSuccess()780         public void onSuccess() {
781             boolean status = true;
782             ArrayList<CaptureStageImpl> initialRequestList =
783                     compileInitialRequestList();
784             if (!initialRequestList.isEmpty()) {
785                 try {
786                     setInitialCaptureRequest(initialRequestList,
787                             new InitialRequestHandler(
788                                     mRepeatingRequestImageCallback));
789                 } catch (CameraAccessException e) {
790                     Log.e(TAG,
791                             "Failed to initialize the initial capture "
792                                     + "request!");
793                     status = false;
794                 }
795             } else {
796                 try {
797                     setRepeatingRequest(mPreviewExtender.getCaptureStage(),
798                             new RepeatingRequestHandler(null, null, null,
799                                     mRepeatingRequestImageCallback));
800                 } catch (CameraAccessException | RemoteException e) {
801                     Log.e(TAG,
802                             "Failed to initialize internal repeating "
803                                     + "request!");
804                     status = false;
805                 }
806 
807             }
808 
809             if (!status) {
810                 notifyConfigurationFailure();
811             }
812         }
813 
814         @Override
onFailure()815         public void onFailure() {
816             mCaptureSession.close();
817             Log.e(TAG, "Failed to initialize proxy service session!"
818                     + " This can happen when trying to configure multiple "
819                     + "concurrent extension sessions!");
820             notifyConfigurationFailure();
821         }
822     }
823 
824     private class BurstRequestHandler extends CameraCaptureSession.CaptureCallback {
825         private final Executor mExecutor;
826         private final ExtensionCaptureCallback mCallbacks;
827         private final CaptureRequest mClientRequest;
828         private final HashMap<CaptureRequest, Integer> mCaptureRequestMap;
829         private final CameraOutputImageCallback mBurstImageCallback;
830 
831         private HashMap<Integer, Pair<Image, TotalCaptureResult>> mCaptureStageMap =
832                 new HashMap<>();
833         private LongSparseArray<Pair<Image, Integer>> mCapturePendingMap =
834                 new LongSparseArray<>();
835 
836         private ImageCallback mImageCallback = null;
837         private boolean mCaptureFailed = false;
838 
BurstRequestHandler(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback callbacks, @NonNull HashMap<CaptureRequest, Integer> requestMap, @Nullable CameraOutputImageCallback imageCallback)839         public BurstRequestHandler(@NonNull CaptureRequest request, @NonNull Executor executor,
840                 @NonNull ExtensionCaptureCallback callbacks,
841                 @NonNull HashMap<CaptureRequest, Integer> requestMap,
842                 @Nullable CameraOutputImageCallback imageCallback) {
843             mClientRequest = request;
844             mExecutor = executor;
845             mCallbacks = callbacks;
846             mCaptureRequestMap = requestMap;
847             mBurstImageCallback = imageCallback;
848         }
849 
notifyCaptureFailed()850         private void notifyCaptureFailed() {
851             if (!mCaptureFailed) {
852                 mCaptureFailed = true;
853 
854                 final long ident = Binder.clearCallingIdentity();
855                 try {
856                     mExecutor.execute(
857                             () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this,
858                                     mClientRequest));
859                 } finally {
860                     Binder.restoreCallingIdentity(ident);
861                 }
862 
863                 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap.values()) {
864                     captureStage.first.close();
865                 }
866                 mCaptureStageMap.clear();
867             }
868         }
869 
870         @Override
onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)871         public void onCaptureStarted(@NonNull CameraCaptureSession session,
872                                      @NonNull CaptureRequest request,
873                                      long timestamp,
874                                      long frameNumber) {
875             // Trigger the client callback only once in case of burst request
876             boolean initialCallback = false;
877             synchronized (mInterfaceLock) {
878                 if ((mImageProcessor != null) && (mImageCallback == null)) {
879                     mImageCallback = new ImageCallback();
880                     initialCallback = true;
881                 } else if (mImageProcessor == null) {
882                     // No burst expected in this case
883                     initialCallback = true;
884                 }
885             }
886 
887             if (initialCallback) {
888                 final long ident = Binder.clearCallingIdentity();
889                 try {
890                     mExecutor.execute(
891                             () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this,
892                                     mClientRequest, timestamp));
893                 } finally {
894                     Binder.restoreCallingIdentity(ident);
895                 }
896             }
897 
898             if ((mBurstImageCallback != null) && (mImageCallback != null)) {
899                 mBurstImageCallback.registerListener(timestamp, mImageCallback);
900             }
901         }
902 
903         @Override
onCaptureBufferLost(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull Surface target, long frameNumber)904         public void onCaptureBufferLost(@NonNull CameraCaptureSession session,
905                                         @NonNull CaptureRequest request,
906                                         @NonNull Surface target, long frameNumber) {
907             notifyCaptureFailed();
908         }
909 
910         @Override
onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)911         public void onCaptureFailed(@NonNull CameraCaptureSession session,
912                                     @NonNull CaptureRequest request,
913                                     @NonNull CaptureFailure failure) {
914             notifyCaptureFailed();
915         }
916 
917         @Override
onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)918         public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session,
919                                              int sequenceId) {
920             final long ident = Binder.clearCallingIdentity();
921             try {
922                 mExecutor.execute(
923                         () -> mCallbacks.onCaptureSequenceAborted(CameraExtensionSessionImpl.this,
924                                 sequenceId));
925             } finally {
926                 Binder.restoreCallingIdentity(ident);
927             }
928         }
929 
930         @Override
onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)931         public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session,
932                                                int sequenceId,
933                                                long frameNumber) {
934             final long ident = Binder.clearCallingIdentity();
935             try {
936                 mExecutor.execute(
937                         () -> mCallbacks
938                                 .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this,
939                                         sequenceId));
940             } finally {
941                 Binder.restoreCallingIdentity(ident);
942             }
943         }
944 
945         @Override
onCaptureCompleted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)946         public void onCaptureCompleted(@NonNull CameraCaptureSession session,
947                                        @NonNull CaptureRequest request,
948                                        @NonNull TotalCaptureResult result) {
949             if (!mCaptureRequestMap.containsKey(request)) {
950                 Log.e(TAG,
951                         "Unexpected still capture request received!");
952                 return;
953             }
954             Integer stageId = mCaptureRequestMap.get(request);
955 
956             Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
957             if (timestamp != null) {
958                 if (mImageProcessor != null) {
959                     if (mCapturePendingMap.indexOfKey(timestamp) >= 0) {
960                         Image img = mCapturePendingMap.get(timestamp).first;
961                         mCaptureStageMap.put(stageId,
962                                 new Pair<>(img,
963                                         result));
964                         checkAndFireBurstProcessing();
965                     } else {
966                         mCapturePendingMap.put(timestamp,
967                                 new Pair<>(null,
968                                         stageId));
969                         mCaptureStageMap.put(stageId,
970                                 new Pair<>(null,
971                                         result));
972                     }
973                 } else {
974                     mCaptureRequestMap.clear();
975                     final long ident = Binder.clearCallingIdentity();
976                     try {
977                         mExecutor.execute(
978                                 () -> mCallbacks
979                                         .onCaptureProcessStarted(CameraExtensionSessionImpl.this,
980                                                 mClientRequest));
981                     } finally {
982                         Binder.restoreCallingIdentity(ident);
983                     }
984                 }
985             } else {
986                 Log.e(TAG,
987                         "Capture result without valid sensor timestamp!");
988             }
989         }
990 
checkAndFireBurstProcessing()991         private void checkAndFireBurstProcessing() {
992             if (mCaptureRequestMap.size() == mCaptureStageMap.size()) {
993                 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap
994                         .values()) {
995                     if ((captureStage.first == null) || (captureStage.second == null)) {
996                         return;
997                     }
998                 }
999 
1000                 mCaptureRequestMap.clear();
1001                 mCapturePendingMap.clear();
1002                 boolean processStatus = true;
1003                 Byte jpegQuality = mClientRequest.get(CaptureRequest.JPEG_QUALITY);
1004                 Integer jpegOrientation = mClientRequest.get(CaptureRequest.JPEG_ORIENTATION);
1005                 List<CaptureBundle> captureList = initializeParcelable(mCaptureStageMap,
1006                         jpegOrientation, jpegQuality);
1007                 try {
1008                     mImageProcessor.process(captureList);
1009                 } catch (RemoteException e) {
1010                     Log.e(TAG, "Failed to process multi-frame request! Extension service "
1011                             + "does not respond!");
1012                     processStatus = false;
1013                 }
1014 
1015                 for (CaptureBundle bundle : captureList) {
1016                     bundle.captureImage.buffer.close();
1017                 }
1018                 captureList.clear();
1019                 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap.values()) {
1020                     captureStage.first.close();
1021                 }
1022                 mCaptureStageMap.clear();
1023 
1024                 final long ident = Binder.clearCallingIdentity();
1025                 try {
1026                     if (processStatus) {
1027                         mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted(
1028                                 CameraExtensionSessionImpl.this, mClientRequest));
1029                     } else {
1030                         mExecutor.execute(() -> mCallbacks.onCaptureFailed(
1031                                 CameraExtensionSessionImpl.this, mClientRequest));
1032                     }
1033                 } finally {
1034                     Binder.restoreCallingIdentity(ident);
1035                 }
1036             }
1037         }
1038 
1039         private class ImageCallback implements OnImageAvailableListener {
1040             @Override
onImageAvailable(ImageReader reader, Image img)1041             public void onImageAvailable(ImageReader reader, Image img) {
1042                 if (mCaptureFailed) {
1043                     img.close();
1044                 }
1045 
1046                 long timestamp = img.getTimestamp();
1047                 reader.detachImage(img);
1048                 if (mCapturePendingMap.indexOfKey(timestamp) >= 0) {
1049                     Integer stageId = mCapturePendingMap.get(timestamp).second;
1050                     Pair<Image, TotalCaptureResult> captureStage =
1051                             mCaptureStageMap.get(stageId);
1052                     if (captureStage != null) {
1053                         mCaptureStageMap.put(stageId,
1054                                 new Pair<>(img,
1055                                         captureStage.second));
1056                         checkAndFireBurstProcessing();
1057                     } else {
1058                         Log.e(TAG,
1059                                 "Capture stage: " +
1060                                         mCapturePendingMap.get(timestamp).second +
1061                                         " is absent!");
1062                     }
1063                 } else {
1064                     mCapturePendingMap.put(timestamp,
1065                             new Pair<>(img,
1066                                     -1));
1067                 }
1068             }
1069         }
1070     }
1071 
1072     private class ImageLoopbackCallback implements OnImageAvailableListener {
1073         @Override
onImageAvailable(ImageReader reader, Image img)1074         public void onImageAvailable(ImageReader reader, Image img) {
1075             img.close();
1076         }
1077     }
1078 
1079     private class InitialRequestHandler extends CameraCaptureSession.CaptureCallback {
1080         private final CameraOutputImageCallback mImageCallback;
1081 
InitialRequestHandler(CameraOutputImageCallback imageCallback)1082         public InitialRequestHandler(CameraOutputImageCallback imageCallback) {
1083             mImageCallback = imageCallback;
1084         }
1085 
1086         @Override
onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1087         public void onCaptureStarted(@NonNull CameraCaptureSession session,
1088                 @NonNull CaptureRequest request, long timestamp, long frameNumber) {
1089             mImageCallback.registerListener(timestamp, new ImageLoopbackCallback());
1090         }
1091 
1092         @Override
onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1093         public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session,
1094                 int sequenceId) {
1095             Log.e(TAG, "Initial capture request aborted!");
1096             notifyConfigurationFailure();
1097         }
1098 
1099         @Override
onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1100         public void onCaptureFailed(@NonNull CameraCaptureSession session,
1101                                     @NonNull CaptureRequest request,
1102                                     @NonNull CaptureFailure failure) {
1103             Log.e(TAG, "Initial capture request failed!");
1104             notifyConfigurationFailure();
1105         }
1106 
1107         @Override
onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1108         public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session,
1109                                                int sequenceId,
1110                                                long frameNumber) {
1111             boolean status = true;
1112             synchronized (mInterfaceLock) {
1113                 /**
1114                  * Initialize and set the initial repeating request which will execute in the
1115                  * absence of client repeating requests.
1116                  */
1117                 try {
1118                     setRepeatingRequest(mPreviewExtender.getCaptureStage(),
1119                             new RepeatingRequestHandler(null, null, null,
1120                                     mImageCallback));
1121                 } catch (CameraAccessException | RemoteException e) {
1122                     Log.e(TAG, "Failed to start the internal repeating request!");
1123                     status = false;
1124                 }
1125 
1126             }
1127 
1128             if (!status) {
1129                 notifyConfigurationFailure();
1130             }
1131         }
1132     }
1133 
1134     private interface OnImageAvailableListener {
onImageAvailable(ImageReader reader, Image img)1135         public void onImageAvailable (ImageReader reader, Image img);
1136     }
1137 
1138     private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener,
1139             Closeable {
1140         private final ImageReader mImageReader;
1141         // Map timestamp to specific images and listeners
1142         private HashMap<Long, Pair<Image, OnImageAvailableListener>> mImageListenerMap =
1143                 new HashMap<>();
1144         private boolean mOutOfBuffers = false;
1145 
CameraOutputImageCallback(ImageReader imageReader)1146         CameraOutputImageCallback(ImageReader imageReader) {
1147             mImageReader = imageReader;
1148         }
1149 
1150         @Override
onImageAvailable(ImageReader reader)1151         public void onImageAvailable(ImageReader reader) {
1152             Image img;
1153             try {
1154                 img = reader.acquireNextImage();
1155             } catch (IllegalStateException e) {
1156                 Log.e(TAG, "Failed to acquire image, too many images pending!");
1157                 mOutOfBuffers = true;
1158                 return;
1159             }
1160             if (img == null) {
1161                 Log.e(TAG, "Invalid image!");
1162                 return;
1163             }
1164 
1165             Long timestamp = img.getTimestamp();
1166             if (mImageListenerMap.containsKey(timestamp)) {
1167                 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.remove(timestamp);
1168                 if (entry.second != null) {
1169                     entry.second.onImageAvailable(reader, img);
1170                 } else {
1171                     Log.w(TAG, "Invalid image listener, dropping frame!");
1172                     img.close();
1173                 }
1174             } else {
1175                 mImageListenerMap.put(img.getTimestamp(), new Pair<>(img, null));
1176             }
1177         }
1178 
registerListener(Long timestamp, OnImageAvailableListener listener)1179         public void registerListener(Long timestamp, OnImageAvailableListener listener) {
1180             if (mImageListenerMap.containsKey(timestamp)) {
1181                 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.remove(timestamp);
1182                 if (entry.first != null) {
1183                     listener.onImageAvailable(mImageReader, entry.first);
1184                     if (mOutOfBuffers) {
1185                         mOutOfBuffers = false;
1186                         Log.w(TAG,"Out of buffers, retry!");
1187                         onImageAvailable(mImageReader);
1188                     }
1189                 } else {
1190                     Log.w(TAG, "No valid image for listener with ts: " +
1191                             timestamp.longValue());
1192                 }
1193             } else {
1194                 mImageListenerMap.put(timestamp, new Pair<>(null, listener));
1195             }
1196         }
1197 
1198         @Override
close()1199         public void close() {
1200             for (Pair<Image, OnImageAvailableListener> entry : mImageListenerMap.values()) {
1201                 if (entry.first != null) {
1202                     entry.first.close();
1203                 }
1204             }
1205             mImageListenerMap.clear();
1206         }
1207     }
1208 
1209     private class CloseRequestHandler extends CameraCaptureSession.CaptureCallback {
1210         private final CameraOutputImageCallback mImageCallback;
1211 
CloseRequestHandler(CameraOutputImageCallback imageCallback)1212         public CloseRequestHandler(CameraOutputImageCallback imageCallback) {
1213             mImageCallback = imageCallback;
1214         }
1215 
1216         @Override
onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1217         public void onCaptureStarted(@NonNull CameraCaptureSession session,
1218                 @NonNull CaptureRequest request, long timestamp, long frameNumber) {
1219             mImageCallback.registerListener(timestamp, new ImageLoopbackCallback());
1220         }
1221     }
1222 
1223     // This handler can operate in two modes:
1224     // 1) Using valid client callbacks, which means camera buffers will be propagated the
1225     //    registered output surfaces and clients will be notified accordingly.
1226     // 2) Without any client callbacks where an internal repeating request is kept active
1227     //    to satisfy the extensions continuous preview/(repeating request) requirement.
1228     private class RepeatingRequestHandler extends CameraCaptureSession.CaptureCallback {
1229         private final Executor mExecutor;
1230         private final ExtensionCaptureCallback mCallbacks;
1231         private final CaptureRequest mClientRequest;
1232         private final boolean mClientNotificationsEnabled;
1233         private final CameraOutputImageCallback mRepeatingImageCallback;
1234         private OnImageAvailableListener mImageCallback = null;
1235         private LongSparseArray<Pair<Image, TotalCaptureResult>> mPendingResultMap =
1236                 new LongSparseArray<>();
1237 
1238         private boolean mRequestUpdatedNeeded = false;
1239 
RepeatingRequestHandler(@ullable CaptureRequest clientRequest, @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, @NonNull CameraOutputImageCallback imageCallback)1240         public RepeatingRequestHandler(@Nullable CaptureRequest clientRequest,
1241                 @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener,
1242                 @NonNull CameraOutputImageCallback imageCallback) {
1243             mClientRequest = clientRequest;
1244             mExecutor = executor;
1245             mCallbacks = listener;
1246             mClientNotificationsEnabled =
1247                     (mClientRequest != null) && (mExecutor != null) && (mCallbacks != null);
1248             mRepeatingImageCallback = imageCallback;
1249         }
1250 
1251         @Override
onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1252         public void onCaptureStarted(@NonNull CameraCaptureSession session,
1253                                      @NonNull CaptureRequest request,
1254                                      long timestamp,
1255                                      long frameNumber) {
1256             synchronized (mInterfaceLock) {
1257                 // Setup the image callback handler for this repeating request just once
1258                 // after streaming resumes.
1259                 if (mImageCallback == null) {
1260                     if (mPreviewProcessorType ==
1261                             IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
1262                         if (mClientNotificationsEnabled) {
1263                             mPreviewImageProcessor.onOutputSurface(mClientRepeatingRequestSurface,
1264                                     nativeGetSurfaceFormat(mClientRepeatingRequestSurface));
1265                         } else {
1266                             mPreviewImageProcessor.onOutputSurface(null, -1);
1267                         }
1268                         mImageCallback = new ImageProcessCallback();
1269                     } else {
1270                         mImageCallback = mClientNotificationsEnabled ?
1271                                 new ImageForwardCallback(mRepeatingRequestImageWriter) :
1272                                 new ImageLoopbackCallback();
1273                     }
1274                 }
1275             }
1276 
1277             if (mClientNotificationsEnabled) {
1278                 final long ident = Binder.clearCallingIdentity();
1279                 try {
1280                     mExecutor.execute(
1281                             () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this,
1282                                     mClientRequest, timestamp));
1283                 } finally {
1284                     Binder.restoreCallingIdentity(ident);
1285                 }
1286             }
1287 
1288             mRepeatingImageCallback.registerListener(timestamp, mImageCallback);
1289         }
1290 
1291         @Override
onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1292         public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session,
1293                                              int sequenceId) {
1294             synchronized (mInterfaceLock) {
1295                 if (mInternalRepeatingRequestEnabled) {
1296                     resumeInternalRepeatingRequest(true);
1297                 }
1298             }
1299 
1300             if (mClientNotificationsEnabled) {
1301                 final long ident = Binder.clearCallingIdentity();
1302                 try {
1303                     mExecutor.execute(
1304                             () -> mCallbacks
1305                                     .onCaptureSequenceAborted(CameraExtensionSessionImpl.this,
1306                                             sequenceId));
1307                 } finally {
1308                     Binder.restoreCallingIdentity(ident);
1309                 }
1310             } else {
1311                 notifyConfigurationFailure();
1312             }
1313 
1314         }
1315 
1316         @Override
onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1317         public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session,
1318                                                int sequenceId,
1319                                                long frameNumber) {
1320 
1321             synchronized (mInterfaceLock) {
1322                 if (mRequestUpdatedNeeded) {
1323                     mRequestUpdatedNeeded = false;
1324                     resumeInternalRepeatingRequest(false);
1325                 } else if (mInternalRepeatingRequestEnabled) {
1326                     resumeInternalRepeatingRequest(true);
1327                 }
1328             }
1329 
1330             if (mClientNotificationsEnabled) {
1331                 final long ident = Binder.clearCallingIdentity();
1332                 try {
1333                     mExecutor.execute(
1334                             () -> mCallbacks
1335                                     .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this,
1336                                             sequenceId));
1337                 } finally {
1338                     Binder.restoreCallingIdentity(ident);
1339                 }
1340             }
1341         }
1342 
1343         @Override
onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1344         public void onCaptureFailed(@NonNull CameraCaptureSession session,
1345                                     @NonNull CaptureRequest request,
1346                                     @NonNull CaptureFailure failure) {
1347 
1348             if (mClientNotificationsEnabled) {
1349                 final long ident = Binder.clearCallingIdentity();
1350                 try {
1351                     mExecutor.execute(
1352                             () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this,
1353                                     mClientRequest));
1354                 } finally {
1355                     Binder.restoreCallingIdentity(ident);
1356                 }
1357             }
1358         }
1359 
1360         @Override
onCaptureCompleted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)1361         public void onCaptureCompleted(@NonNull CameraCaptureSession session,
1362                                        @NonNull CaptureRequest request,
1363                                        @NonNull TotalCaptureResult result) {
1364             boolean notifyClient = mClientNotificationsEnabled;
1365             boolean processStatus = true;
1366 
1367             synchronized (mInterfaceLock) {
1368                 final Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
1369                 if (timestamp != null) {
1370                     if (mPreviewProcessorType ==
1371                             IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
1372                         CaptureStageImpl captureStage = null;
1373                         try {
1374                             captureStage = mPreviewRequestUpdateProcessor.process(
1375                                     result.getNativeMetadata(), result.getSequenceId());
1376                         } catch (RemoteException e) {
1377                             Log.e(TAG, "Extension service does not respond during " +
1378                                     "processing!");
1379                         }
1380                         if (captureStage != null) {
1381                             try {
1382                                 setRepeatingRequest(captureStage, this);
1383                                 mRequestUpdatedNeeded = true;
1384                             } catch (IllegalStateException e) {
1385                                 // This is possible in case the camera device closes and the
1386                                 // and the callback here is executed before the onClosed
1387                                 // notification.
1388                             } catch (CameraAccessException e) {
1389                                 Log.e(TAG, "Failed to update repeating request settings!");
1390                             }
1391                         } else {
1392                             mRequestUpdatedNeeded = false;
1393                         }
1394                     } else if (mPreviewProcessorType ==
1395                             IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
1396                         int idx = mPendingResultMap.indexOfKey(timestamp);
1397                         if (idx >= 0) {
1398                             ParcelImage parcelImage = initializeParcelImage(
1399                                     mPendingResultMap.get(timestamp).first);
1400                             try {
1401                                 mPreviewImageProcessor.process(parcelImage, result);
1402                             } catch (RemoteException e) {
1403                                 processStatus = false;
1404                                 Log.e(TAG, "Extension service does not respond during " +
1405                                         "processing, dropping frame!");
1406                             } catch (RuntimeException e) {
1407                                 // Runtime exceptions can happen in a few obscure cases where the
1408                                 // client tries to initialize a new capture session while this
1409                                 // session is still ongoing. In such scenario, the camera will
1410                                 // disconnect from the intermediate output surface, which will
1411                                 // invalidate the images that we acquired previously. This can
1412                                 // happen before we get notified via "onClosed" so there aren't
1413                                 // many options to avoid the exception.
1414                                 processStatus = false;
1415                                 Log.e(TAG, "Runtime exception encountered during buffer " +
1416                                         "processing, dropping frame!");
1417                             } finally {
1418                                 parcelImage.buffer.close();
1419                                 mPendingResultMap.get(timestamp).first.close();
1420                             }
1421                             discardPendingRepeatingResults(idx, mPendingResultMap, false);
1422                         } else {
1423                             notifyClient = false;
1424                             mPendingResultMap.put(timestamp,
1425                                     new Pair<>(null,
1426                                             result));
1427                         }
1428                     } else {
1429                         // No special handling for PROCESSOR_TYPE_NONE
1430                     }
1431                     if (notifyClient) {
1432                         final long ident = Binder.clearCallingIdentity();
1433                         try {
1434                             if (processStatus) {
1435                                 mExecutor.execute(() -> mCallbacks
1436                                         .onCaptureProcessStarted(
1437                                                 CameraExtensionSessionImpl.this,
1438                                                 mClientRequest));
1439                             } else {
1440                                 mExecutor.execute(
1441                                         () -> mCallbacks
1442                                                 .onCaptureFailed(
1443                                                         CameraExtensionSessionImpl.this,
1444                                                         mClientRequest));
1445                             }
1446                         } finally {
1447                             Binder.restoreCallingIdentity(ident);
1448                         }
1449                     }
1450                 } else {
1451                     Log.e(TAG,
1452                             "Result without valid sensor timestamp!");
1453                 }
1454             }
1455 
1456             if (!notifyClient) {
1457                 notifyConfigurationSuccess();
1458             }
1459         }
1460 
resumeInternalRepeatingRequest(boolean internal)1461         private void resumeInternalRepeatingRequest(boolean internal) {
1462             try {
1463                 if (internal) {
1464                     setRepeatingRequest(mPreviewExtender.getCaptureStage(),
1465                             new RepeatingRequestHandler(null, null, null,
1466                                     mRepeatingImageCallback));
1467                 } else {
1468                     setRepeatingRequest(mPreviewExtender.getCaptureStage(), this);
1469                 }
1470             } catch (RemoteException e) {
1471                 Log.e(TAG, "Failed to resume internal repeating request, extension service"
1472                         + " fails to respond!");
1473             } catch (IllegalStateException e) {
1474                 // This is possible in case we try to resume before the state "onClosed"
1475                 // notification is able to reach us.
1476                 Log.w(TAG, "Failed to resume internal repeating request!");
1477             } catch (CameraAccessException e) {
1478                 Log.e(TAG, "Failed to resume internal repeating request!");
1479             }
1480         }
1481 
1482         // Find the timestamp of the oldest pending buffer
calculatePruneThreshold( LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap)1483         private Long calculatePruneThreshold(
1484                 LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap) {
1485             long oldestTimestamp = Long.MAX_VALUE;
1486             for (int idx = 0; idx < previewMap.size(); idx++) {
1487                 Pair<Image, TotalCaptureResult> entry = previewMap.valueAt(idx);
1488                 long timestamp = previewMap.keyAt(idx);
1489                 if ((entry.first != null) && (timestamp < oldestTimestamp)) {
1490                     oldestTimestamp = timestamp;
1491                 }
1492             }
1493             return (oldestTimestamp == Long.MAX_VALUE) ? 0 : oldestTimestamp;
1494         }
1495 
discardPendingRepeatingResults(int idx, LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap, boolean notifyCurrentIndex)1496         private void discardPendingRepeatingResults(int idx, LongSparseArray<Pair<Image,
1497                 TotalCaptureResult>> previewMap, boolean notifyCurrentIndex) {
1498             if (idx < 0) {
1499                 return;
1500             }
1501             for (int i = idx; i >= 0; i--) {
1502                 if (previewMap.valueAt(i).first != null) {
1503                     previewMap.valueAt(i).first.close();
1504                 } else {
1505                     if (mClientNotificationsEnabled && ((i != idx) || notifyCurrentIndex)) {
1506                         Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i));
1507                         final long ident = Binder.clearCallingIdentity();
1508                         try {
1509                             mExecutor.execute(
1510                                     () -> mCallbacks
1511                                             .onCaptureFailed(CameraExtensionSessionImpl.this,
1512                                                     mClientRequest));
1513                         } finally {
1514                             Binder.restoreCallingIdentity(ident);
1515                         }
1516                     }
1517                 }
1518                 previewMap.removeAt(i);
1519             }
1520         }
1521 
1522         private class ImageForwardCallback implements OnImageAvailableListener {
1523             private final ImageWriter mOutputWriter;
1524 
ImageForwardCallback(@onNull ImageWriter imageWriter)1525             public ImageForwardCallback(@NonNull ImageWriter imageWriter) {
1526                 mOutputWriter = imageWriter;
1527             }
1528 
1529             @Override
onImageAvailable(ImageReader reader, Image img)1530             public void onImageAvailable(ImageReader reader, Image img) {
1531                 if (img == null) {
1532                     Log.e(TAG, "Invalid image!");
1533                     return;
1534                 }
1535 
1536                 try {
1537                     mOutputWriter.queueInputImage(img);
1538                 } catch (IllegalStateException e) {
1539                     // This is possible in case the client disconnects from the output surface
1540                     // abruptly.
1541                     Log.w(TAG, "Output surface likely abandoned, dropping buffer!");
1542                     img.close();
1543                 }
1544             }
1545         }
1546 
1547         private class ImageProcessCallback implements OnImageAvailableListener {
1548 
1549             @Override
onImageAvailable(ImageReader reader, Image img)1550             public void onImageAvailable(ImageReader reader, Image img) {
1551                 if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) {
1552                     // We reached the maximum acquired images limit. This is possible in case we
1553                     // have capture failures that result in absent or missing capture results. In
1554                     // such scenario we can prune the oldest pending buffer.
1555                     discardPendingRepeatingResults(
1556                             mPendingResultMap
1557                                     .indexOfKey(calculatePruneThreshold(mPendingResultMap)),
1558                             mPendingResultMap, true);
1559                 }
1560 
1561                 if (img == null) {
1562                     Log.e(TAG,
1563                             "Invalid preview buffer!");
1564                     return;
1565                 }
1566                 try {
1567                     reader.detachImage(img);
1568                 } catch (Exception e) {
1569                     Log.e(TAG,
1570                             "Failed to detach image!");
1571                     img.close();
1572                     return;
1573                 }
1574 
1575                 long timestamp = img.getTimestamp();
1576                 int idx = mPendingResultMap.indexOfKey(timestamp);
1577                 if (idx >= 0) {
1578                     boolean processStatus = true;
1579                     ParcelImage parcelImage = initializeParcelImage(img);
1580                     try {
1581                         mPreviewImageProcessor.process(parcelImage,
1582                                 mPendingResultMap.get(timestamp).second);
1583                     } catch (RemoteException e) {
1584                         processStatus = false;
1585                         Log.e(TAG, "Extension service does not respond during " +
1586                                 "processing, dropping frame!");
1587                     } finally {
1588                         parcelImage.buffer.close();
1589                         img.close();
1590                     }
1591                     discardPendingRepeatingResults(idx, mPendingResultMap, false);
1592                     if (mClientNotificationsEnabled) {
1593                         final long ident = Binder.clearCallingIdentity();
1594                         try {
1595                             if (processStatus) {
1596                                 mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted(
1597                                         CameraExtensionSessionImpl.this,
1598                                         mClientRequest));
1599                             } else {
1600                                 mExecutor.execute(() -> mCallbacks.onCaptureFailed(
1601                                         CameraExtensionSessionImpl.this,
1602                                         mClientRequest));
1603                             }
1604                         } finally {
1605                             Binder.restoreCallingIdentity(ident);
1606                         }
1607                     }
1608                 } else {
1609                     mPendingResultMap.put(timestamp, new Pair<>(img, null));
1610                 }
1611             }
1612         }
1613     }
1614 
findSmallestAspectMatchedSize(@onNull List<Size> sizes, @NonNull Size arSize)1615     private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes,
1616                                                       @NonNull Size arSize) {
1617         final float TOLL = .01f;
1618 
1619         if (arSize.getHeight() == 0) {
1620             throw new IllegalArgumentException("Invalid input aspect ratio");
1621         }
1622 
1623         float targetAR = ((float) arSize.getWidth()) / arSize.getHeight();
1624         Size ret = null;
1625         Size fallbackSize = null;
1626         for (Size sz : sizes) {
1627             if (fallbackSize == null) {
1628                 fallbackSize = sz;
1629             }
1630             if ((sz.getHeight() > 0) &&
1631                     ((ret == null) ||
1632                             (ret.getWidth() * ret.getHeight()) <
1633                                     (sz.getWidth() * sz.getHeight()))) {
1634                 float currentAR = ((float) sz.getWidth()) / sz.getHeight();
1635                 if (Math.abs(currentAR - targetAR) <= TOLL) {
1636                     ret = sz;
1637                 }
1638             }
1639         }
1640         if (ret == null) {
1641             Log.e(TAG, "AR matched size not found returning first size in list");
1642             ret = fallbackSize;
1643         }
1644 
1645         return ret;
1646     }
1647 
initializeParcelImage(Image img)1648     private static ParcelImage initializeParcelImage(Image img) {
1649         ParcelImage parcelImage = new ParcelImage();
1650         parcelImage.buffer = img.getHardwareBuffer();
1651         if (img.getFenceFd() >= 0) {
1652             try {
1653                 parcelImage.fence = ParcelFileDescriptor.fromFd(img.getFenceFd());
1654             } catch (IOException e) {
1655                 Log.e(TAG,"Failed to parcel buffer fence!");
1656             }
1657         }
1658         parcelImage.width = img.getWidth();
1659         parcelImage.height = img.getHeight();
1660         parcelImage.format = img.getFormat();
1661         parcelImage.timestamp = img.getTimestamp();
1662         parcelImage.transform = img.getTransform();
1663         parcelImage.scalingMode = img.getScalingMode();
1664         parcelImage.planeCount = img.getPlaneCount();
1665         parcelImage.crop = img.getCropRect();
1666 
1667         return parcelImage;
1668     }
1669 
initializeParcelable( HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap, Integer jpegOrientation, Byte jpegQuality)1670     private static List<CaptureBundle> initializeParcelable(
1671             HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap, Integer jpegOrientation,
1672             Byte jpegQuality) {
1673         ArrayList<CaptureBundle> ret = new ArrayList<>();
1674         for (Integer stagetId : captureMap.keySet()) {
1675             Pair<Image, TotalCaptureResult> entry = captureMap.get(stagetId);
1676             CaptureBundle bundle = new CaptureBundle();
1677             bundle.stage = stagetId;
1678             bundle.captureImage = initializeParcelImage(entry.first);
1679             bundle.sequenceId = entry.second.getSequenceId();
1680             bundle.captureResult = entry.second.getNativeMetadata();
1681             if (jpegOrientation != null) {
1682                 bundle.captureResult.set(CaptureResult.JPEG_ORIENTATION, jpegOrientation);
1683             }
1684             if (jpegQuality != null) {
1685                 bundle.captureResult.set(CaptureResult.JPEG_QUALITY, jpegQuality);
1686             }
1687             ret.add(bundle);
1688         }
1689 
1690         return ret;
1691     }
1692 }
1693