/** * Copyright (c) 2020, The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.cameraextensions; import android.app.Service; import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; import android.graphics.Camera; import android.graphics.GraphicBuffer; import android.graphics.Rect; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraExtensionCharacteristics; import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; import android.hardware.camera2.extension.CameraOutputConfig; import android.hardware.camera2.extension.CameraSessionConfig; import android.hardware.camera2.extension.CaptureBundle; import android.hardware.camera2.extension.CaptureFailure; import android.hardware.camera2.extension.CaptureStageImpl; import android.hardware.camera2.extension.IAdvancedExtenderImpl; import android.hardware.camera2.extension.ICameraExtensionsProxyService; import android.hardware.camera2.extension.ICaptureCallback; import android.hardware.camera2.extension.ICaptureProcessorImpl; import android.hardware.camera2.extension.IPreviewExtenderImpl; import android.hardware.camera2.extension.IPreviewImageProcessorImpl; import android.hardware.camera2.extension.IRequestCallback; import android.hardware.camera2.extension.IRequestProcessorImpl; import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; import android.hardware.camera2.extension.IImageCaptureExtenderImpl; import android.hardware.camera2.extension.IImageProcessorImpl; import android.hardware.camera2.extension.IInitializeSessionCallback; import android.hardware.camera2.extension.ISessionProcessorImpl; import android.hardware.camera2.extension.LatencyRange; import android.hardware.camera2.extension.OutputConfigId; import android.hardware.camera2.extension.OutputSurface; import android.hardware.camera2.extension.ParcelCaptureResult; import android.hardware.camera2.extension.ParcelImage; import android.hardware.camera2.extension.ParcelTotalCaptureResult; import android.hardware.camera2.extension.Request; import android.hardware.camera2.extension.SizeList; import android.hardware.camera2.impl.CameraMetadataNative; import android.hardware.camera2.TotalCaptureResult; import android.hardware.HardwareBuffer; import android.hardware.camera2.impl.PhysicalCaptureResultInfo; import android.media.Image; import android.media.ImageReader; import android.os.Binder; import android.os.ConditionVariable; import android.os.Handler; import android.os.HandlerExecutor; import android.os.HandlerThread; import android.os.IBinder; import android.os.RemoteException; import android.util.ArraySet; import android.util.Log; import android.util.Pair; import android.util.Range; import android.util.Size; import android.view.Surface; import androidx.camera.extensions.impl.AutoImageCaptureExtenderImpl; import androidx.camera.extensions.impl.AutoPreviewExtenderImpl; import androidx.camera.extensions.impl.BeautyImageCaptureExtenderImpl; import androidx.camera.extensions.impl.BeautyPreviewExtenderImpl; import androidx.camera.extensions.impl.BokehImageCaptureExtenderImpl; import androidx.camera.extensions.impl.BokehPreviewExtenderImpl; import androidx.camera.extensions.impl.CaptureProcessorImpl; import androidx.camera.extensions.impl.ExtensionVersionImpl; import androidx.camera.extensions.impl.HdrImageCaptureExtenderImpl; import androidx.camera.extensions.impl.HdrPreviewExtenderImpl; import androidx.camera.extensions.impl.ImageCaptureExtenderImpl; import androidx.camera.extensions.impl.InitializerImpl; import androidx.camera.extensions.impl.NightImageCaptureExtenderImpl; import androidx.camera.extensions.impl.NightPreviewExtenderImpl; import androidx.camera.extensions.impl.PreviewExtenderImpl; import androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType; import androidx.camera.extensions.impl.PreviewImageProcessorImpl; import androidx.camera.extensions.impl.RequestUpdateProcessorImpl; import androidx.camera.extensions.impl.advanced.AdvancedExtenderImpl; import androidx.camera.extensions.impl.advanced.AutoAdvancedExtenderImpl; import androidx.camera.extensions.impl.advanced.BeautyAdvancedExtenderImpl; import androidx.camera.extensions.impl.advanced.BokehAdvancedExtenderImpl; import androidx.camera.extensions.impl.advanced.Camera2OutputConfigImpl; import androidx.camera.extensions.impl.advanced.Camera2SessionConfigImpl; import androidx.camera.extensions.impl.advanced.HdrAdvancedExtenderImpl; import androidx.camera.extensions.impl.advanced.ImageProcessorImpl; import androidx.camera.extensions.impl.advanced.ImageReaderOutputConfigImpl; import androidx.camera.extensions.impl.advanced.MultiResolutionImageReaderOutputConfigImpl; import androidx.camera.extensions.impl.advanced.NightAdvancedExtenderImpl; import androidx.camera.extensions.impl.advanced.OutputSurfaceImpl; import androidx.camera.extensions.impl.advanced.RequestProcessorImpl; import androidx.camera.extensions.impl.advanced.SessionProcessorImpl; import androidx.camera.extensions.impl.advanced.SurfaceOutputConfigImpl; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.List; public class CameraExtensionsProxyService extends Service { private static final String TAG = "CameraExtensionsProxyService"; private static final String CAMERA_EXTENSION_VERSION_NAME = "androidx.camera.extensions.impl.ExtensionVersionImpl"; private static final String LATEST_VERSION = "1.2.0"; private static final String NON_INIT_VERSION_PREFIX = "1.0"; private static final String ADVANCED_VERSION_PREFIX = "1.2"; private static final String[] SUPPORTED_VERSION_PREFIXES = {ADVANCED_VERSION_PREFIX, "1.1", NON_INIT_VERSION_PREFIX}; private static final boolean EXTENSIONS_PRESENT = checkForExtensions(); private static final String EXTENSIONS_VERSION = EXTENSIONS_PRESENT ? (new ExtensionVersionImpl()).checkApiVersion(LATEST_VERSION) : null; private static final boolean ADVANCED_API_SUPPORTED = checkForAdvancedAPI(); private static final boolean INIT_API_SUPPORTED = EXTENSIONS_PRESENT && (!EXTENSIONS_VERSION.startsWith(NON_INIT_VERSION_PREFIX)); private HashMap mCharacteristicsHashMap = new HashMap<>(); private HashMap mMetadataVendorIdMap = new HashMap<>(); private CameraManager mCameraManager; private static boolean checkForAdvancedAPI() { if (EXTENSIONS_PRESENT && EXTENSIONS_VERSION.startsWith(ADVANCED_VERSION_PREFIX)) { try { return (new ExtensionVersionImpl()).isAdvancedExtenderImplemented(); } catch (NoSuchMethodError e) { // This could happen in case device specific extension implementations are using an // older extension API but incorrectly set the extension version. } } return false; } private static boolean checkForExtensions() { try { Class.forName(CAMERA_EXTENSION_VERSION_NAME); } catch (ClassNotFoundException e) { return false; } String extensionVersion = (new ExtensionVersionImpl()).checkApiVersion(LATEST_VERSION); for (String supportedVersion : SUPPORTED_VERSION_PREFIXES) { if (extensionVersion.startsWith(supportedVersion)) { return true; } } return false; } /** * A per-process global camera extension manager instance, to track and * initialize/release extensions depending on client activity. */ private static final class CameraExtensionManagerGlobal { private static final String TAG = "CameraExtensionManagerGlobal"; private final int EXTENSION_DELAY_MS = 1000; private final Handler mHandler; private final HandlerThread mHandlerThread; private final Object mLock = new Object(); private long mCurrentClientCount = 0; private ArraySet mActiveClients = new ArraySet<>(); private IInitializeSessionCallback mInitializeCb = null; // Singleton instance private static final CameraExtensionManagerGlobal GLOBAL_CAMERA_MANAGER = new CameraExtensionManagerGlobal(); // Singleton, don't allow construction private CameraExtensionManagerGlobal() { mHandlerThread = new HandlerThread(TAG); mHandlerThread.start(); mHandler = new Handler(mHandlerThread.getLooper()); } private final static class InitializeHandler implements InitializerImpl.OnExtensionsInitializedCallback { private final InitializerFuture mStatusFuture; public InitializeHandler(InitializerFuture statusFuture) { mStatusFuture = statusFuture; } @Override public void onSuccess() { mStatusFuture.setStatus(true); } @Override public void onFailure(int error) { mStatusFuture.setStatus(false); } } private final static class ReleaseHandler implements InitializerImpl.OnExtensionsDeinitializedCallback { private final InitializerFuture mStatusFuture; public ReleaseHandler(InitializerFuture statusFuture) { mStatusFuture = statusFuture; } @Override public void onSuccess() { mStatusFuture.setStatus(true); } @Override public void onFailure(int i) { mStatusFuture.setStatus(false); } } private static class InitializerFuture implements Future { private volatile Boolean mStatus; ConditionVariable mCondVar = new ConditionVariable(/*opened*/false); public void setStatus(boolean status) { mStatus = status; mCondVar.open(); } @Override public boolean cancel(boolean mayInterruptIfRunning) { return false; // don't allow canceling this task } @Override public boolean isCancelled() { return false; // can never cancel this task } @Override public boolean isDone() { return mStatus != null; } @Override public Boolean get() { mCondVar.block(); return mStatus; } @Override public Boolean get(long timeout, TimeUnit unit) throws TimeoutException { long timeoutMs = unit.convert(timeout, TimeUnit.MILLISECONDS); if (!mCondVar.block(timeoutMs)) { throw new TimeoutException( "Failed to receive status after " + timeout + " " + unit); } if (mStatus == null) { throw new AssertionError(); } return mStatus; } } public static CameraExtensionManagerGlobal get() { return GLOBAL_CAMERA_MANAGER; } public long registerClient(Context ctx) { synchronized (mLock) { if (INIT_API_SUPPORTED) { if (mActiveClients.isEmpty()) { InitializerFuture status = new InitializerFuture(); InitializerImpl.init(EXTENSIONS_VERSION, ctx, new InitializeHandler(status), new HandlerExecutor(mHandler)); boolean initSuccess; try { initSuccess = status.get(EXTENSION_DELAY_MS, TimeUnit.MILLISECONDS); } catch (TimeoutException e) { Log.e(TAG, "Timed out while initializing camera extensions!"); return -1; } if (!initSuccess) { Log.e(TAG, "Failed while initializing camera extensions!"); return -1; } } } long ret = mCurrentClientCount; mCurrentClientCount++; if (mCurrentClientCount < 0) { mCurrentClientCount = 0; } mActiveClients.add(ret); return ret; } } public void unregisterClient(long clientId) { synchronized (mLock) { if (mActiveClients.remove(clientId) && mActiveClients.isEmpty() && INIT_API_SUPPORTED) { InitializerFuture status = new InitializerFuture(); InitializerImpl.deinit(new ReleaseHandler(status), new HandlerExecutor(mHandler)); boolean releaseSuccess; try { releaseSuccess = status.get(EXTENSION_DELAY_MS, TimeUnit.MILLISECONDS); } catch (TimeoutException e) { Log.e(TAG, "Timed out while releasing camera extensions!"); return; } if (!releaseSuccess) { Log.e(TAG, "Failed while releasing camera extensions!"); } } } } private IBinder.DeathRecipient mDeathRecipient = new IBinder.DeathRecipient() { @Override public void binderDied() { synchronized (mLock) { mInitializeCb = null; } } }; public boolean initializeSession(IInitializeSessionCallback cb) { synchronized (mLock) { if (mInitializeCb == null) { mInitializeCb = cb; try { mInitializeCb.asBinder().linkToDeath(mDeathRecipient, 0); } catch (RemoteException e) { e.printStackTrace(); } } else { return false; } } return true; } public void releaseSession() { synchronized (mLock) { if (mInitializeCb != null) { mInitializeCb.asBinder().unlinkToDeath(mDeathRecipient, 0); mInitializeCb = null; } } } } /** * @hide */ private static long registerClient(Context ctx) { if (!EXTENSIONS_PRESENT) { return -1; } return CameraExtensionManagerGlobal.get().registerClient(ctx); } /** * @hide */ public static void unregisterClient(long clientId) { if (!EXTENSIONS_PRESENT) { return; } CameraExtensionManagerGlobal.get().unregisterClient(clientId); } /** * @hide */ public static boolean initializeSession(IInitializeSessionCallback cb) { if (!EXTENSIONS_PRESENT) { return false; } return CameraExtensionManagerGlobal.get().initializeSession(cb); } /** * @hide */ public static void releaseSession() { if (!EXTENSIONS_PRESENT) { return; } CameraExtensionManagerGlobal.get().releaseSession(); } /** * @hide */ public static Pair initializeExtension( int extensionType) { switch (extensionType) { case CameraExtensionCharacteristics.EXTENSION_AUTOMATIC: return new Pair<>(new AutoPreviewExtenderImpl(), new AutoImageCaptureExtenderImpl()); case CameraExtensionCharacteristics.EXTENSION_BEAUTY: return new Pair<>(new BeautyPreviewExtenderImpl(), new BeautyImageCaptureExtenderImpl()); case CameraExtensionCharacteristics.EXTENSION_BOKEH: return new Pair<>(new BokehPreviewExtenderImpl(), new BokehImageCaptureExtenderImpl()); case CameraExtensionCharacteristics.EXTENSION_HDR: return new Pair<>(new HdrPreviewExtenderImpl(), new HdrImageCaptureExtenderImpl()); case CameraExtensionCharacteristics.EXTENSION_NIGHT: return new Pair<>(new NightPreviewExtenderImpl(), new NightImageCaptureExtenderImpl()); default: throw new IllegalArgumentException("Unknown extension: " + extensionType); } } /** * @hide */ public static AdvancedExtenderImpl initializeAdvancedExtensionImpl(int extensionType) { switch (extensionType) { case CameraExtensionCharacteristics.EXTENSION_AUTOMATIC: return new AutoAdvancedExtenderImpl(); case CameraExtensionCharacteristics.EXTENSION_BEAUTY: return new BeautyAdvancedExtenderImpl(); case CameraExtensionCharacteristics.EXTENSION_BOKEH: return new BokehAdvancedExtenderImpl(); case CameraExtensionCharacteristics.EXTENSION_HDR: return new HdrAdvancedExtenderImpl(); case CameraExtensionCharacteristics.EXTENSION_NIGHT: return new NightAdvancedExtenderImpl(); default: throw new IllegalArgumentException("Unknown extension: " + extensionType); } } @Override public void onCreate() { super.onCreate(); // This will setup the camera vendor tag descriptor in the service process // along with all camera characteristics. try { mCameraManager = getSystemService(CameraManager.class); String [] cameraIds = mCameraManager.getCameraIdListNoLazy(); if (cameraIds != null) { for (String cameraId : cameraIds) { CameraCharacteristics chars = mCameraManager.getCameraCharacteristics(cameraId); mCharacteristicsHashMap.put(cameraId, chars); Object thisClass = CameraCharacteristics.Key.class; Class> keyClass = (Class>)thisClass; ArrayList> vendorKeys = chars.getNativeMetadata().getAllVendorKeys(keyClass); if ((vendorKeys != null) && !vendorKeys.isEmpty()) { mMetadataVendorIdMap.put(cameraId, vendorKeys.get(0).getVendorId()); } } } } catch (CameraAccessException e) { Log.e(TAG, "Failed to query camera characteristics!"); } } @Override public void onDestroy() { super.onDestroy(); } @Override public IBinder onBind(Intent intent) { return new CameraExtensionsProxyServiceStub(); } private static List initializeParcelable( List> sizes) { if (sizes == null) { return null; } ArrayList ret = new ArrayList<>(sizes.size()); for (Pair entry : sizes) { SizeList sizeList = new SizeList(); sizeList.format = entry.first; sizeList.sizes = new ArrayList<>(); for (android.util.Size size : entry.second) { android.hardware.camera2.extension.Size sz = new android.hardware.camera2.extension.Size(); sz.width = size.getWidth(); sz.height = size.getHeight(); sizeList.sizes.add(sz); } } return ret; } private static List initializeParcelable( Map> sizes) { if (sizes == null) { return null; } ArrayList ret = new ArrayList<>(sizes.size()); for (Map.Entry> entry : sizes.entrySet()) { SizeList sizeList = new SizeList(); sizeList.format = entry.getKey(); sizeList.sizes = new ArrayList<>(); for (android.util.Size size : entry.getValue()) { android.hardware.camera2.extension.Size sz = new android.hardware.camera2.extension.Size(); sz.width = size.getWidth(); sz.height = size.getHeight(); sizeList.sizes.add(sz); } ret.add(sizeList); } return ret; } private CameraMetadataNative initializeParcelableMetadata( List> paramList, String cameraId) { if (paramList == null) { return null; } CameraMetadataNative ret = new CameraMetadataNative(); if (mMetadataVendorIdMap.containsKey(cameraId)) { ret.setVendorId(mMetadataVendorIdMap.get(cameraId)); } for (Pair param : paramList) { ret.set(param.first, param.second); } return ret; } private CameraMetadataNative initializeParcelableMetadata( Map, Object> paramMap, String cameraId) { if (paramMap == null) { return null; } CameraMetadataNative ret = new CameraMetadataNative(); if (mMetadataVendorIdMap.containsKey(cameraId)) { ret.setVendorId(mMetadataVendorIdMap.get(cameraId)); } for (Map.Entry, Object> param : paramMap.entrySet()) { ret.set(((CaptureRequest.Key) param.getKey()), param.getValue()); } return ret; } private android.hardware.camera2.extension.CaptureStageImpl initializeParcelable( androidx.camera.extensions.impl.CaptureStageImpl captureStage, String cameraId) { if (captureStage == null) { return null; } android.hardware.camera2.extension.CaptureStageImpl ret = new android.hardware.camera2.extension.CaptureStageImpl(); ret.id = captureStage.getId(); ret.parameters = initializeParcelableMetadata(captureStage.getParameters(), cameraId); return ret; } private Request initializeParcelable(RequestProcessorImpl.Request request, int requestId, String cameraId) { Request ret = new Request(); ret.targetOutputConfigIds = new ArrayList<>(); for (int id : request.getTargetOutputConfigIds()) { OutputConfigId configId = new OutputConfigId(); configId.id = id; ret.targetOutputConfigIds.add(configId); } ret.templateId = request.getTemplateId(); ret.parameters = initializeParcelableMetadata(request.getParameters(), cameraId); ret.requestId = requestId; return ret; } private class CameraExtensionsProxyServiceStub extends ICameraExtensionsProxyService.Stub { @Override public long registerClient() { return CameraExtensionsProxyService.registerClient(CameraExtensionsProxyService.this); } @Override public void unregisterClient(long clientId) { CameraExtensionsProxyService.unregisterClient(clientId); } private boolean checkCameraPermission() { int allowed = CameraExtensionsProxyService.this.checkPermission( android.Manifest.permission.CAMERA, Binder.getCallingPid(), Binder.getCallingUid()); return (PackageManager.PERMISSION_GRANTED == allowed); } @Override public void initializeSession(IInitializeSessionCallback cb) { try { if (!checkCameraPermission()) { Log.i(TAG, "Camera permission required for initializing capture session"); cb.onFailure(); return; } if (CameraExtensionsProxyService.initializeSession(cb)) { cb.onSuccess(); } else { cb.onFailure(); } } catch (RemoteException e) { Log.e(TAG, "Client doesn't respond!"); } } @Override public void releaseSession() { if (checkCameraPermission()) { CameraExtensionsProxyService.releaseSession(); } } @Override public boolean advancedExtensionsSupported() { return ADVANCED_API_SUPPORTED; } @Override public IAdvancedExtenderImpl initializeAdvancedExtension(int extensionType) { AdvancedExtenderImpl extension; try { extension = initializeAdvancedExtensionImpl(extensionType); } catch (IllegalArgumentException e) { return null; } return new AdvancedExtenderImplStub(extension); } @Override public IPreviewExtenderImpl initializePreviewExtension(int extensionType) { Pair extension; try { extension = initializeExtension(extensionType); } catch (IllegalArgumentException e) { return null; } return new PreviewExtenderImplStub(extension.first); } @Override public IImageCaptureExtenderImpl initializeImageExtension(int extensionType) { Pair extension; try { extension = initializeExtension(extensionType); } catch (IllegalArgumentException e) { return null; } return new ImageCaptureExtenderImplStub(extension.second); } } private class AdvancedExtenderImplStub extends IAdvancedExtenderImpl.Stub { private final AdvancedExtenderImpl mAdvancedExtender; public AdvancedExtenderImplStub(AdvancedExtenderImpl advancedExtender) { mAdvancedExtender = advancedExtender; } @Override public boolean isExtensionAvailable(String cameraId) { return mAdvancedExtender.isExtensionAvailable(cameraId, mCharacteristicsHashMap); } @Override public void init(String cameraId) { mAdvancedExtender.init(cameraId, mCharacteristicsHashMap); } @Override public List getSupportedPreviewOutputResolutions(String cameraId) { Map> supportedSizesMap = mAdvancedExtender.getSupportedPreviewOutputResolutions(cameraId); if (supportedSizesMap != null) { return initializeParcelable(supportedSizesMap); } return null; } @Override public List getSupportedCaptureOutputResolutions(String cameraId) { Map> supportedSizesMap = mAdvancedExtender.getSupportedCaptureOutputResolutions(cameraId); if (supportedSizesMap != null) { return initializeParcelable(supportedSizesMap); } return null; } @Override public LatencyRange getEstimatedCaptureLatencyRange(String cameraId, android.hardware.camera2.extension.Size outputSize, int format) { Size sz = new Size(outputSize.width, outputSize.height); Range latencyRange = mAdvancedExtender.getEstimatedCaptureLatencyRange(cameraId, sz, format); if (latencyRange != null) { LatencyRange ret = new LatencyRange(); ret.min = latencyRange.getLower(); ret.max = latencyRange.getUpper(); return ret; } return null; } @Override public ISessionProcessorImpl getSessionProcessor() { return new SessionProcessorImplStub(mAdvancedExtender.createSessionProcessor()); } } private class CaptureCallbackStub implements SessionProcessorImpl.CaptureCallback { private final ICaptureCallback mCaptureCallback; private CaptureCallbackStub(ICaptureCallback captureCallback) { mCaptureCallback = captureCallback; } @Override public void onCaptureStarted(int captureSequenceId, long timestamp) { if (mCaptureCallback != null) { try { mCaptureCallback.onCaptureStarted(captureSequenceId, timestamp); } catch (RemoteException e) { Log.e(TAG, "Failed to notify capture start due to remote " + "exception!"); } } } @Override public void onCaptureProcessStarted(int captureSequenceId) { if (mCaptureCallback != null) { try { mCaptureCallback.onCaptureProcessStarted(captureSequenceId); } catch (RemoteException e) { Log.e(TAG, "Failed to notify capture process start due to remote " + "exception!"); } } } @Override public void onCaptureFailed(int captureSequenceId) { if (mCaptureCallback != null) { try { mCaptureCallback.onCaptureFailed(captureSequenceId); } catch (RemoteException e) { Log.e(TAG, "Failed to notify capture failure due to remote " + "exception!"); } } } @Override public void onCaptureSequenceCompleted(int captureSequenceId) { if (mCaptureCallback != null) { try { mCaptureCallback.onCaptureSequenceCompleted(captureSequenceId); } catch (RemoteException e) { Log.e(TAG, "Failed to notify capture sequence end due to remote " + "exception!"); } } } @Override public void onCaptureSequenceAborted(int captureSequenceId) { if (mCaptureCallback != null) { try { mCaptureCallback.onCaptureSequenceAborted(captureSequenceId); } catch (RemoteException e) { Log.e(TAG, "Failed to notify capture sequence abort due to remote " + "exception!"); } } } } private class RequestCallbackStub extends IRequestCallback.Stub { private final List mRequests; private final RequestProcessorImpl.Callback mCallback; public RequestCallbackStub(List requests, RequestProcessorImpl.Callback callback) { mCallback = callback; if (mCallback != null) { mRequests = requests; } else { Log.w(TAG, "No valid request callbacks!"); mRequests = new ArrayList<>(); } } @Override public void onCaptureStarted(int requestId, long frameNumber, long timestamp) { if (mCallback != null) { if (mRequests.get(requestId) != null) { mCallback.onCaptureStarted(mRequests.get(requestId), frameNumber, timestamp); } else { Log.e(TAG,"Request id: " + requestId + " not found!"); } } } @Override public void onCaptureProgressed(int requestId, ParcelCaptureResult partialResult) { if (mCallback != null) { if (mRequests.get(requestId) != null) { CaptureResult result = new CaptureResult(partialResult.cameraId, partialResult.results, partialResult.parent, partialResult.sequenceId, partialResult.frameNumber); mCallback.onCaptureProgressed(mRequests.get(requestId), result); } else { Log.e(TAG,"Request id: " + requestId + " not found!"); } } } @Override public void onCaptureCompleted(int requestId, ParcelTotalCaptureResult totalCaptureResult) { if (mCallback != null) { if (mRequests.get(requestId) != null) { PhysicalCaptureResultInfo[] physicalResults = new PhysicalCaptureResultInfo[0]; if ((totalCaptureResult.physicalResult != null) && (!totalCaptureResult.physicalResult.isEmpty())) { int count = totalCaptureResult.physicalResult.size(); physicalResults = new PhysicalCaptureResultInfo[count]; physicalResults = totalCaptureResult.physicalResult.toArray( physicalResults); } ArrayList partials = new ArrayList<>( totalCaptureResult.partials.size()); for (ParcelCaptureResult parcelResult : totalCaptureResult.partials) { partials.add(new CaptureResult(parcelResult.cameraId, parcelResult.results, parcelResult.parent, parcelResult.sequenceId, parcelResult.frameNumber)); } TotalCaptureResult result = new TotalCaptureResult( totalCaptureResult.logicalCameraId, totalCaptureResult.results, totalCaptureResult.parent, totalCaptureResult.sequenceId, totalCaptureResult.frameNumber, partials, totalCaptureResult.sessionId, physicalResults); mCallback.onCaptureCompleted(mRequests.get(requestId), result); } else { Log.e(TAG,"Request id: " + requestId + " not found!"); } } } @Override public void onCaptureFailed(int requestId, CaptureFailure captureFailure) { if (mCallback != null) { if (mRequests.get(requestId) != null) { android.hardware.camera2.CaptureFailure failure = new android.hardware.camera2.CaptureFailure(captureFailure.request, captureFailure.reason, captureFailure.dropped, captureFailure.sequenceId, captureFailure.frameNumber, captureFailure.errorPhysicalCameraId); mCallback.onCaptureFailed(mRequests.get(requestId), failure); } else { Log.e(TAG,"Request id: " + requestId + " not found!"); } } } @Override public void onCaptureBufferLost(int requestId, long frameNumber, int outputStreamId) { if (mCallback != null) { if (mRequests.get(requestId) != null) { mCallback.onCaptureBufferLost(mRequests.get(requestId), frameNumber, outputStreamId); } else { Log.e(TAG,"Request id: " + requestId + " not found!"); } } } @Override public void onCaptureSequenceCompleted(int sequenceId, long frameNumber) { if (mCallback != null) { mCallback.onCaptureSequenceCompleted(sequenceId, frameNumber); } } @Override public void onCaptureSequenceAborted(int sequenceId) { if (mCallback != null) { mCallback.onCaptureSequenceAborted(sequenceId); } } } private class ImageProcessorImplStub extends IImageProcessorImpl.Stub { private final ImageProcessorImpl mImageProcessor; public ImageProcessorImplStub(ImageProcessorImpl imageProcessor) { mImageProcessor = imageProcessor; } @Override public void onNextImageAvailable(OutputConfigId outputConfigId, ParcelImage img, String physicalCameraId) { if (mImageProcessor != null) { mImageProcessor.onNextImageAvailable(outputConfigId.id, img.timestamp, new ImageReferenceImpl(img), physicalCameraId); } } } private class RequestProcessorStub implements RequestProcessorImpl { private final IRequestProcessorImpl mRequestProcessor; private final String mCameraId; public RequestProcessorStub(IRequestProcessorImpl requestProcessor, String cameraId) { mRequestProcessor = requestProcessor; mCameraId = cameraId; } @Override public void setImageProcessor(int outputConfigId, ImageProcessorImpl imageProcessor) { OutputConfigId configId = new OutputConfigId(); configId.id = outputConfigId; try { mRequestProcessor.setImageProcessor(configId, new ImageProcessorImplStub(imageProcessor)); } catch (RemoteException e) { Log.e(TAG, "Failed to set image processor due to remote exception!"); } } @Override public int submit(Request request, Callback callback) { ArrayList requests = new ArrayList<>(); requests.add(request); return submit(requests, callback); } @Override public int submit(List requests, Callback callback) { ArrayList captureRequests = new ArrayList<>(); int requestId = 0; for (Request request : requests) { captureRequests.add(initializeParcelable(request, requestId, mCameraId)); requestId++; } try { return mRequestProcessor.submitBurst(captureRequests, new RequestCallbackStub(requests, callback)); } catch (RemoteException e) { Log.e(TAG, "Failed to submit request due to remote exception!"); } return -1; } @Override public int setRepeating(Request request, Callback callback) { try { ArrayList requests = new ArrayList<>(); requests.add(request); return mRequestProcessor.setRepeating( initializeParcelable(request, 0, mCameraId), new RequestCallbackStub(requests, callback)); } catch (RemoteException e) { Log.e(TAG, "Failed to submit repeating request due to remote exception!"); } return -1; } @Override public void abortCaptures() { try { mRequestProcessor.abortCaptures(); } catch (RemoteException e) { Log.e(TAG, "Failed to abort requests due to remote exception!"); } } @Override public void stopRepeating() { try { mRequestProcessor.stopRepeating(); } catch (RemoteException e) { Log.e(TAG, "Failed to stop repeating request due to remote exception!"); } } } private class SessionProcessorImplStub extends ISessionProcessorImpl.Stub { private final SessionProcessorImpl mSessionProcessor; private String mCameraId = null; public SessionProcessorImplStub(SessionProcessorImpl sessionProcessor) { mSessionProcessor = sessionProcessor; } @Override public CameraSessionConfig initSession(String cameraId, OutputSurface previewSurface, OutputSurface burstSurface) { OutputSurfaceImplStub outputPreviewSurfaceImpl = new OutputSurfaceImplStub(previewSurface); OutputSurfaceImplStub outputBurstSurfaceImpl = new OutputSurfaceImplStub(burstSurface); Camera2SessionConfigImpl sessionConfig = mSessionProcessor.initSession(cameraId, mCharacteristicsHashMap, getApplicationContext(), outputPreviewSurfaceImpl, outputBurstSurfaceImpl, null /*imageAnalysisSurfaceConfig*/); List outputConfigs = sessionConfig.getOutputConfigs(); CameraSessionConfig ret = new CameraSessionConfig(); ret.outputConfigs = new ArrayList<>(); for (Camera2OutputConfigImpl output : outputConfigs) { CameraOutputConfig entry = new CameraOutputConfig(); entry.outputId = new OutputConfigId(); entry.outputId.id = output.getId(); entry.physicalCameraId = output.getPhysicalCameraId(); entry.surfaceGroupId = output.getSurfaceGroupId(); if (output instanceof SurfaceOutputConfigImpl) { SurfaceOutputConfigImpl surfaceConfig = (SurfaceOutputConfigImpl) output; entry.type = CameraOutputConfig.TYPE_SURFACE; entry.surface = surfaceConfig.getSurface(); } else if (output instanceof ImageReaderOutputConfigImpl) { ImageReaderOutputConfigImpl imageReaderOutputConfig = (ImageReaderOutputConfigImpl) output; entry.type = CameraOutputConfig.TYPE_IMAGEREADER; entry.size = new android.hardware.camera2.extension.Size(); entry.size.width = imageReaderOutputConfig.getSize().getWidth(); entry.size.height = imageReaderOutputConfig.getSize().getHeight(); entry.imageFormat = imageReaderOutputConfig.getImageFormat(); entry.capacity = imageReaderOutputConfig.getMaxImages(); } else if (output instanceof MultiResolutionImageReaderOutputConfigImpl) { MultiResolutionImageReaderOutputConfigImpl multiResReaderConfig = (MultiResolutionImageReaderOutputConfigImpl) output; entry.type = CameraOutputConfig.TYPE_MULTIRES_IMAGEREADER; entry.imageFormat = multiResReaderConfig.getImageFormat(); entry.capacity = multiResReaderConfig.getMaxImages(); } else { throw new IllegalStateException("Unknown output config type!"); } List sharedOutputs = output.getSurfaceSharingOutputConfigs(); if ((sharedOutputs != null) && (!sharedOutputs.isEmpty())) { entry.surfaceSharingOutputConfigs = new ArrayList<>(); for (Camera2OutputConfigImpl sharedOutput : sharedOutputs) { OutputConfigId outputId = new OutputConfigId(); outputId.id = sharedOutput.getId(); entry.surfaceSharingOutputConfigs.add(outputId); } } ret.outputConfigs.add(entry); } ret.sessionTemplateId = sessionConfig.getSessionTemplateId(); ret.sessionParameter = initializeParcelableMetadata( sessionConfig.getSessionParameters(), cameraId); mCameraId = cameraId; return ret; } @Override public void deInitSession() { mSessionProcessor.deInitSession(); } @Override public void onCaptureSessionStart(IRequestProcessorImpl requestProcessor) { mSessionProcessor.onCaptureSessionStart( new RequestProcessorStub(requestProcessor, mCameraId)); } @Override public void onCaptureSessionEnd() { mSessionProcessor.onCaptureSessionEnd(); } @Override public int startRepeating(ICaptureCallback callback) { return mSessionProcessor.startRepeating(new CaptureCallbackStub(callback)); } @Override public void stopRepeating() { mSessionProcessor.stopRepeating(); } @Override public int startCapture(ICaptureCallback callback, int jpegRotation, int jpegQuality) { HashMap, Object> paramMap = new HashMap<>(); paramMap.put(CaptureRequest.JPEG_ORIENTATION, jpegRotation); paramMap.put(CaptureRequest.JPEG_QUALITY, jpegQuality); mSessionProcessor.setParameters(paramMap); return mSessionProcessor.startCapture(new CaptureCallbackStub(callback)); } } private class OutputSurfaceImplStub implements OutputSurfaceImpl { private final Surface mSurface; private final Size mSize; private final int mImageFormat; public OutputSurfaceImplStub(OutputSurface outputSurface) { mSurface = outputSurface.surface; mSize = new Size(outputSurface.size.width, outputSurface.size.height); mImageFormat = outputSurface.imageFormat; } @Override public Surface getSurface() { return mSurface; } @Override public Size getSize() { return mSize; } @Override public int getImageFormat() { return mImageFormat; } } private class PreviewExtenderImplStub extends IPreviewExtenderImpl.Stub { private final PreviewExtenderImpl mPreviewExtender; private String mCameraId = null; public PreviewExtenderImplStub(PreviewExtenderImpl previewExtender) { mPreviewExtender = previewExtender; } @Override public void onInit(String cameraId, CameraMetadataNative cameraCharacteristics) { mCameraId = cameraId; CameraCharacteristics chars = new CameraCharacteristics(cameraCharacteristics); mCameraManager.registerDeviceStateListener(chars); mPreviewExtender.onInit(cameraId, chars, CameraExtensionsProxyService.this); } @Override public void onDeInit() { mPreviewExtender.onDeInit(); } @Override public CaptureStageImpl onPresetSession() { return initializeParcelable(mPreviewExtender.onPresetSession(), mCameraId); } @Override public CaptureStageImpl onEnableSession() { return initializeParcelable(mPreviewExtender.onEnableSession(), mCameraId); } @Override public CaptureStageImpl onDisableSession() { return initializeParcelable(mPreviewExtender.onDisableSession(), mCameraId); } @Override public void init(String cameraId, CameraMetadataNative chars) { CameraCharacteristics c = new CameraCharacteristics(chars); mCameraManager.registerDeviceStateListener(c); mPreviewExtender.init(cameraId, c); } @Override public boolean isExtensionAvailable(String cameraId, CameraMetadataNative chars) { CameraCharacteristics c = new CameraCharacteristics(chars); mCameraManager.registerDeviceStateListener(c); return mPreviewExtender.isExtensionAvailable(cameraId, c); } @Override public CaptureStageImpl getCaptureStage() { return initializeParcelable(mPreviewExtender.getCaptureStage(), mCameraId); } @Override public int getProcessorType() { ProcessorType processorType = mPreviewExtender.getProcessorType(); if (processorType == ProcessorType.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { return IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY; } else if (processorType == ProcessorType.PROCESSOR_TYPE_IMAGE_PROCESSOR) { return IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR; } else { return IPreviewExtenderImpl.PROCESSOR_TYPE_NONE; } } @Override public IPreviewImageProcessorImpl getPreviewImageProcessor() { PreviewImageProcessorImpl processor; try { processor = (PreviewImageProcessorImpl) mPreviewExtender.getProcessor(); } catch (ClassCastException e) { Log.e(TAG, "Failed casting preview processor!"); return null; } if (processor != null) { return new PreviewImageProcessorImplStub(processor); } return null; } @Override public IRequestUpdateProcessorImpl getRequestUpdateProcessor() { RequestUpdateProcessorImpl processor; try { processor = (RequestUpdateProcessorImpl) mPreviewExtender.getProcessor(); } catch (ClassCastException e) { Log.e(TAG, "Failed casting preview processor!"); return null; } if (processor != null) { return new RequestUpdateProcessorImplStub(processor, mCameraId); } return null; } @Override public List getSupportedResolutions() { if (INIT_API_SUPPORTED) { List> sizes = mPreviewExtender.getSupportedResolutions(); if ((sizes != null) && !sizes.isEmpty()) { return initializeParcelable(sizes); } } return null; } } private class ImageCaptureExtenderImplStub extends IImageCaptureExtenderImpl.Stub { private final ImageCaptureExtenderImpl mImageExtender; private String mCameraId = null; public ImageCaptureExtenderImplStub(ImageCaptureExtenderImpl imageExtender) { mImageExtender = imageExtender; } @Override public void onInit(String cameraId, CameraMetadataNative cameraCharacteristics) { CameraCharacteristics chars = new CameraCharacteristics(cameraCharacteristics); mCameraManager.registerDeviceStateListener(chars); mImageExtender.onInit(cameraId, chars, CameraExtensionsProxyService.this); mCameraId = cameraId; } @Override public void onDeInit() { mImageExtender.onDeInit(); } @Override public CaptureStageImpl onPresetSession() { return initializeParcelable(mImageExtender.onPresetSession(), mCameraId); } @Override public CaptureStageImpl onEnableSession() { return initializeParcelable(mImageExtender.onEnableSession(), mCameraId); } @Override public CaptureStageImpl onDisableSession() { return initializeParcelable(mImageExtender.onDisableSession(), mCameraId); } @Override public void init(String cameraId, CameraMetadataNative chars) { CameraCharacteristics c = new CameraCharacteristics(chars); mCameraManager.registerDeviceStateListener(c); mImageExtender.init(cameraId, c); } @Override public boolean isExtensionAvailable(String cameraId, CameraMetadataNative chars) { CameraCharacteristics c = new CameraCharacteristics(chars); mCameraManager.registerDeviceStateListener(c); return mImageExtender.isExtensionAvailable(cameraId, c); } @Override public ICaptureProcessorImpl getCaptureProcessor() { CaptureProcessorImpl captureProcessor = mImageExtender.getCaptureProcessor(); if (captureProcessor != null) { return new CaptureProcessorImplStub(captureProcessor); } return null; } @Override public List getCaptureStages() { List captureStages = mImageExtender.getCaptureStages(); if (captureStages != null) { ArrayList ret = new ArrayList<>(); for (androidx.camera.extensions.impl.CaptureStageImpl stage : captureStages) { ret.add(initializeParcelable(stage, mCameraId)); } return ret; } return null; } @Override public int getMaxCaptureStage() { return mImageExtender.getMaxCaptureStage(); } @Override public List getSupportedResolutions() { if (INIT_API_SUPPORTED) { List> sizes = mImageExtender.getSupportedResolutions(); if ((sizes != null) && !sizes.isEmpty()) { return initializeParcelable(sizes); } } return null; } @Override public LatencyRange getEstimatedCaptureLatencyRange( android.hardware.camera2.extension.Size outputSize) { if (EXTENSIONS_VERSION.startsWith(ADVANCED_VERSION_PREFIX)) { Size sz = new Size(outputSize.width, outputSize.height); Range latencyRange = mImageExtender.getEstimatedCaptureLatencyRange(sz); if (latencyRange != null) { LatencyRange ret = new LatencyRange(); ret.min = latencyRange.getLower(); ret.max = latencyRange.getUpper(); return ret; } } return null; } } private class CaptureProcessorImplStub extends ICaptureProcessorImpl.Stub { private final CaptureProcessorImpl mCaptureProcessor; public CaptureProcessorImplStub(CaptureProcessorImpl captureProcessor) { mCaptureProcessor = captureProcessor; } @Override public void onOutputSurface(Surface surface, int imageFormat) { mCaptureProcessor.onOutputSurface(surface, imageFormat); } @Override public void onResolutionUpdate(android.hardware.camera2.extension.Size size) { mCaptureProcessor.onResolutionUpdate(new android.util.Size(size.width, size.height)); } @Override public void onImageFormatUpdate(int imageFormat) { mCaptureProcessor.onImageFormatUpdate(imageFormat); } @Override public void process(List captureList) { HashMap> captureMap = new HashMap<>(); for (CaptureBundle captureBundle : captureList) { captureMap.put(captureBundle.stage, new Pair<> ( new ExtensionImage(captureBundle.captureImage), new TotalCaptureResult(captureBundle.captureResult, captureBundle.sequenceId))); } if (!captureMap.isEmpty()) { mCaptureProcessor.process(captureMap); } else { Log.e(TAG, "Process request with absent capture stages!"); } } } private class PreviewImageProcessorImplStub extends IPreviewImageProcessorImpl.Stub { private final PreviewImageProcessorImpl mProcessor; public PreviewImageProcessorImplStub(PreviewImageProcessorImpl processor) { mProcessor = processor; } @Override public void onOutputSurface(Surface surface, int imageFormat) { mProcessor.onOutputSurface(surface, imageFormat); } @Override public void onResolutionUpdate(android.hardware.camera2.extension.Size size) { mProcessor.onResolutionUpdate(new android.util.Size(size.width, size.height)); } @Override public void onImageFormatUpdate(int imageFormat) { mProcessor.onImageFormatUpdate(imageFormat); } @Override public void process(android.hardware.camera2.extension.ParcelImage image, CameraMetadataNative result, int sequenceId) { mProcessor.process(new ExtensionImage(image), new TotalCaptureResult(result, sequenceId)); } } private class RequestUpdateProcessorImplStub extends IRequestUpdateProcessorImpl.Stub { private final RequestUpdateProcessorImpl mProcessor; private final String mCameraId; public RequestUpdateProcessorImplStub(RequestUpdateProcessorImpl processor, String cameraId) { mProcessor = processor; mCameraId = cameraId; } @Override public void onOutputSurface(Surface surface, int imageFormat) { mProcessor.onOutputSurface(surface, imageFormat); } @Override public void onResolutionUpdate(android.hardware.camera2.extension.Size size) { mProcessor.onResolutionUpdate(new android.util.Size(size.width, size.height)); } @Override public void onImageFormatUpdate(int imageFormat) { mProcessor.onImageFormatUpdate(imageFormat); } @Override public CaptureStageImpl process(CameraMetadataNative result, int sequenceId) { return initializeParcelable( mProcessor.process(new TotalCaptureResult(result, sequenceId)), mCameraId); } } private class ImageReferenceImpl extends ExtensionImage implements androidx.camera.extensions.impl.advanced.ImageReferenceImpl { private final Object mImageLock = new Object(); private int mReferenceCount; private ImageReferenceImpl(ParcelImage parcelImage) { super(parcelImage); mReferenceCount = 1; } @Override public boolean increment() { synchronized (mImageLock) { if (mReferenceCount <= 0) { return false; } mReferenceCount++; } return true; } @Override public boolean decrement() { synchronized (mImageLock) { if (mReferenceCount <= 0) { return false; } mReferenceCount--; if (mReferenceCount <= 0) { close(); } } return true; } @Override public Image get() { return this; } } private class ExtensionImage extends android.media.Image { private final android.hardware.camera2.extension.ParcelImage mParcelImage; private GraphicBuffer mGraphicBuffer; private ImageReader.ImagePlane[] mPlanes; private ExtensionImage(android.hardware.camera2.extension.ParcelImage parcelImage) { mParcelImage = parcelImage; mIsImageValid = true; } @Override public int getFormat() { throwISEIfImageIsInvalid(); return mParcelImage.format; } @Override public int getWidth() { throwISEIfImageIsInvalid(); return mParcelImage.width; } @Override public HardwareBuffer getHardwareBuffer() { throwISEIfImageIsInvalid(); return mParcelImage.buffer; } @Override public int getHeight() { throwISEIfImageIsInvalid(); return mParcelImage.height; } @Override public long getTimestamp() { throwISEIfImageIsInvalid(); return mParcelImage.timestamp; } @Override public int getTransform() { throwISEIfImageIsInvalid(); return mParcelImage.transform; } @Override public int getScalingMode() { throwISEIfImageIsInvalid(); return mParcelImage.scalingMode; } @Override public Plane[] getPlanes() { throwISEIfImageIsInvalid(); if (mPlanes == null) { int fenceFd = mParcelImage.fence != null ? mParcelImage.fence.getFd() : -1; mGraphicBuffer = GraphicBuffer.createFromHardwareBuffer(mParcelImage.buffer); mPlanes = ImageReader.initializeImagePlanes(mParcelImage.planeCount, mGraphicBuffer, fenceFd, mParcelImage.format, mParcelImage.timestamp, mParcelImage.transform, mParcelImage.scalingMode, mParcelImage.crop); } // Shallow copy is fine. return mPlanes.clone(); } @Override protected final void finalize() throws Throwable { try { close(); } finally { super.finalize(); } } @Override public boolean isAttachable() { throwISEIfImageIsInvalid(); // Clients must always detach parcelable images return true; } @Override public Rect getCropRect() { throwISEIfImageIsInvalid(); return mParcelImage.crop; } @Override public void close() { mIsImageValid = false; if (mGraphicBuffer != null) { ImageReader.unlockGraphicBuffer(mGraphicBuffer); mGraphicBuffer.destroy(); mGraphicBuffer = null; } if (mPlanes != null) { mPlanes = null; } if (mParcelImage.buffer != null) { mParcelImage.buffer.close(); mParcelImage.buffer = null; } if (mParcelImage.fence != null) { try { mParcelImage.fence.close(); } catch (IOException e) { e.printStackTrace(); } mParcelImage.fence = null; } } } }