1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import android.annotation.NonNull; 20 import android.annotation.Nullable; 21 import android.annotation.RequiresPermission; 22 import android.content.Context; 23 import android.graphics.ImageFormat; 24 import android.graphics.SurfaceTexture; 25 import android.hardware.camera2.CameraAccessException; 26 import android.hardware.camera2.CameraCaptureSession; 27 import android.hardware.camera2.CameraCharacteristics; 28 import android.hardware.camera2.CameraDevice; 29 import android.hardware.camera2.CameraExtensionCharacteristics; 30 import android.hardware.camera2.CameraExtensionSession; 31 import android.hardware.camera2.CameraManager; 32 import android.hardware.camera2.CaptureFailure; 33 import android.hardware.camera2.CaptureRequest; 34 import android.hardware.camera2.CaptureResult; 35 import android.hardware.camera2.TotalCaptureResult; 36 import android.hardware.camera2.extension.CameraOutputConfig; 37 import android.hardware.camera2.extension.CameraSessionConfig; 38 import android.hardware.camera2.extension.CaptureStageImpl; 39 import android.hardware.camera2.extension.IAdvancedExtenderImpl; 40 import android.hardware.camera2.extension.ICaptureCallback; 41 import android.hardware.camera2.extension.IImageProcessorImpl; 42 import android.hardware.camera2.extension.IInitializeSessionCallback; 43 import android.hardware.camera2.extension.IRequestCallback; 44 import android.hardware.camera2.extension.IRequestProcessorImpl; 45 import android.hardware.camera2.extension.ISessionProcessorImpl; 46 import android.hardware.camera2.extension.OutputConfigId; 47 import android.hardware.camera2.extension.OutputSurface; 48 import android.hardware.camera2.extension.ParcelCaptureResult; 49 import android.hardware.camera2.extension.ParcelImage; 50 import android.hardware.camera2.extension.ParcelTotalCaptureResult; 51 import android.hardware.camera2.extension.Request; 52 import android.hardware.camera2.params.ExtensionSessionConfiguration; 53 import android.hardware.camera2.params.OutputConfiguration; 54 import android.hardware.camera2.params.SessionConfiguration; 55 import android.hardware.camera2.utils.SurfaceUtils; 56 import android.media.Image; 57 import android.media.ImageReader; 58 import android.os.Binder; 59 import android.os.Handler; 60 import android.os.HandlerThread; 61 import android.os.ParcelFileDescriptor; 62 import android.os.RemoteException; 63 import android.util.Log; 64 import android.util.Size; 65 import android.view.Surface; 66 67 import java.io.IOException; 68 import java.util.ArrayList; 69 import java.util.HashMap; 70 import java.util.List; 71 import java.util.Map; 72 import java.util.concurrent.Executor; 73 74 public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSession { 75 private static final String TAG = "CameraAdvancedExtensionSessionImpl"; 76 77 private final Executor mExecutor; 78 private final CameraDevice mCameraDevice; 79 private final long mExtensionClientId; 80 private final Handler mHandler; 81 private final HandlerThread mHandlerThread; 82 private final CameraExtensionSession.StateCallback mCallbacks; 83 private final IAdvancedExtenderImpl mAdvancedExtender; 84 // maps registered camera surfaces to extension output configs 85 private final HashMap<Surface, CameraOutputConfig> mCameraConfigMap = new HashMap<>(); 86 // maps camera extension output ids to camera registered image readers 87 private final HashMap<Integer, ImageReader> mReaderMap = new HashMap<>(); 88 private final RequestProcessor mRequestProcessor = new RequestProcessor(); 89 90 private Surface mClientRepeatingRequestSurface; 91 private Surface mClientCaptureSurface; 92 private CameraCaptureSession mCaptureSession = null; 93 private ISessionProcessorImpl mSessionProcessor = null; 94 private final InitializeSessionHandler mInitializeHandler; 95 96 private boolean mInitialized; 97 98 99 // Lock to synchronize cross-thread access to device public interface 100 final Object mInterfaceLock = new Object(); // access from this class and Session only! 101 102 /** 103 * @hide 104 */ 105 @RequiresPermission(android.Manifest.permission.CAMERA) createCameraAdvancedExtensionSession( @onNull CameraDevice cameraDevice, @NonNull Context ctx, @NonNull ExtensionSessionConfiguration config)106 public static CameraAdvancedExtensionSessionImpl createCameraAdvancedExtensionSession( 107 @NonNull CameraDevice cameraDevice, @NonNull Context ctx, 108 @NonNull ExtensionSessionConfiguration config) 109 throws CameraAccessException, RemoteException { 110 long clientId = CameraExtensionCharacteristics.registerClient(ctx); 111 if (clientId < 0) { 112 throw new UnsupportedOperationException("Unsupported extension!"); 113 } 114 115 String cameraId = cameraDevice.getId(); 116 CameraManager manager = ctx.getSystemService(CameraManager.class); 117 CameraCharacteristics chars = manager.getCameraCharacteristics(cameraId); 118 CameraExtensionCharacteristics extensionChars = new CameraExtensionCharacteristics(ctx, 119 cameraId, chars); 120 121 if (!CameraExtensionCharacteristics.isExtensionSupported(cameraDevice.getId(), 122 config.getExtension(), chars)) { 123 throw new UnsupportedOperationException("Unsupported extension type: " + 124 config.getExtension()); 125 } 126 127 if (config.getOutputConfigurations().isEmpty() || 128 config.getOutputConfigurations().size() > 2) { 129 throw new IllegalArgumentException("Unexpected amount of output surfaces, received: " + 130 config.getOutputConfigurations().size() + " expected <= 2"); 131 } 132 133 int suitableSurfaceCount = 0; 134 List<Size> supportedPreviewSizes = extensionChars.getExtensionSupportedSizes( 135 config.getExtension(), SurfaceTexture.class); 136 Surface repeatingRequestSurface = CameraExtensionUtils.getRepeatingRequestSurface( 137 config.getOutputConfigurations(), supportedPreviewSizes); 138 if (repeatingRequestSurface != null) { 139 suitableSurfaceCount++; 140 } 141 142 HashMap<Integer, List<Size>> supportedCaptureSizes = new HashMap<>(); 143 for (int format : CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS) { 144 List<Size> supportedSizes = extensionChars.getExtensionSupportedSizes( 145 config.getExtension(), format); 146 if (supportedSizes != null) { 147 supportedCaptureSizes.put(format, supportedSizes); 148 } 149 } 150 Surface burstCaptureSurface = CameraExtensionUtils.getBurstCaptureSurface( 151 config.getOutputConfigurations(), supportedCaptureSizes); 152 if (burstCaptureSurface != null) { 153 suitableSurfaceCount++; 154 } 155 156 if (suitableSurfaceCount != config.getOutputConfigurations().size()) { 157 throw new IllegalArgumentException("One or more unsupported output surfaces found!"); 158 } 159 160 IAdvancedExtenderImpl extender = CameraExtensionCharacteristics.initializeAdvancedExtension( 161 config.getExtension()); 162 extender.init(cameraId); 163 164 CameraAdvancedExtensionSessionImpl ret = new CameraAdvancedExtensionSessionImpl(clientId, 165 extender, cameraDevice, repeatingRequestSurface, burstCaptureSurface, 166 config.getStateCallback(), config.getExecutor()); 167 ret.initialize(); 168 169 return ret; 170 } 171 CameraAdvancedExtensionSessionImpl(long extensionClientId, @NonNull IAdvancedExtenderImpl extender, @NonNull CameraDevice cameraDevice, @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, @NonNull CameraExtensionSession.StateCallback callback, @NonNull Executor executor)172 private CameraAdvancedExtensionSessionImpl(long extensionClientId, 173 @NonNull IAdvancedExtenderImpl extender, @NonNull CameraDevice cameraDevice, 174 @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, 175 @NonNull CameraExtensionSession.StateCallback callback, @NonNull Executor executor) { 176 mExtensionClientId = extensionClientId; 177 mAdvancedExtender = extender; 178 mCameraDevice = cameraDevice; 179 mCallbacks = callback; 180 mExecutor = executor; 181 mClientRepeatingRequestSurface = repeatingRequestSurface; 182 mClientCaptureSurface = burstCaptureSurface; 183 mHandlerThread = new HandlerThread(TAG); 184 mHandlerThread.start(); 185 mHandler = new Handler(mHandlerThread.getLooper()); 186 mInitialized = false; 187 mInitializeHandler = new InitializeSessionHandler(); 188 } 189 190 /** 191 * @hide 192 */ initialize()193 public synchronized void initialize() throws CameraAccessException, RemoteException { 194 if (mInitialized) { 195 Log.d(TAG, "Session already initialized"); 196 return; 197 } 198 199 OutputSurface previewSurface = initializeParcelable(mClientRepeatingRequestSurface); 200 OutputSurface captureSurface = initializeParcelable(mClientCaptureSurface); 201 mSessionProcessor = mAdvancedExtender.getSessionProcessor(); 202 CameraSessionConfig sessionConfig = mSessionProcessor.initSession(mCameraDevice.getId(), 203 previewSurface, captureSurface); 204 List<CameraOutputConfig> outputConfigs = sessionConfig.outputConfigs; 205 // map camera output ids to output configurations 206 HashMap<Integer, OutputConfiguration> cameraOutputs = new HashMap<>(); 207 for (CameraOutputConfig output : outputConfigs) { 208 OutputConfiguration cameraOutput = null; 209 switch(output.type) { 210 case CameraOutputConfig.TYPE_SURFACE: 211 if (output.surface == null) { 212 Log.w(TAG, "Unsupported client output id: " + output.outputId.id + 213 ", skipping!"); 214 continue; 215 } 216 cameraOutput = new OutputConfiguration(output.surfaceGroupId, 217 output.surface); 218 break; 219 case CameraOutputConfig.TYPE_IMAGEREADER: 220 if ((output.imageFormat == ImageFormat.UNKNOWN) || (output.size.width <= 0) || 221 (output.size.height <= 0)) { 222 Log.w(TAG, "Unsupported client output id: " + output.outputId.id + 223 ", skipping!"); 224 continue; 225 } 226 ImageReader reader = ImageReader.newInstance(output.size.width, 227 output.size.height, output.imageFormat, output.capacity); 228 mReaderMap.put(output.outputId.id, reader); 229 cameraOutput = new OutputConfiguration(output.surfaceGroupId, 230 reader.getSurface()); 231 break; 232 case CameraOutputConfig.TYPE_MULTIRES_IMAGEREADER: 233 // Support for multi-resolution outputs to be added in future releases 234 default: 235 throw new IllegalArgumentException("Unsupported output config type: " + 236 output.type); 237 } 238 cameraOutput.setPhysicalCameraId(output.physicalCameraId); 239 cameraOutputs.put(output.outputId.id, cameraOutput); 240 } 241 242 ArrayList<OutputConfiguration> outputList = new ArrayList<>(); 243 for (CameraOutputConfig output : outputConfigs) { 244 if (!cameraOutputs.containsKey(output.outputId.id)) { 245 // Shared surface already removed by a previous iteration 246 continue; 247 } 248 OutputConfiguration outConfig = cameraOutputs.get(output.outputId.id); 249 if ((output.surfaceSharingOutputConfigs != null) && 250 !output.surfaceSharingOutputConfigs.isEmpty()) { 251 outConfig.enableSurfaceSharing(); 252 for (OutputConfigId outputId : output.surfaceSharingOutputConfigs) { 253 outConfig.addSurface(cameraOutputs.get(outputId.id).getSurface()); 254 cameraOutputs.remove(outputId.id); 255 } 256 } 257 outputList.add(outConfig); 258 mCameraConfigMap.put(outConfig.getSurface(), output); 259 } 260 261 SessionConfiguration sessionConfiguration = new SessionConfiguration( 262 SessionConfiguration.SESSION_REGULAR, outputList, 263 new CameraExtensionUtils.HandlerExecutor(mHandler), new SessionStateHandler()); 264 265 if ((sessionConfig.sessionParameter != null) && 266 (!sessionConfig.sessionParameter.isEmpty())) { 267 CaptureRequest.Builder requestBuilder = mCameraDevice.createCaptureRequest( 268 sessionConfig.sessionTemplateId); 269 CaptureRequest sessionRequest = requestBuilder.build(); 270 CameraMetadataNative.update(sessionRequest.getNativeMetadata(), 271 sessionConfig.sessionParameter); 272 sessionConfiguration.setSessionParameters(sessionRequest); 273 } 274 275 mCameraDevice.createCaptureSession(sessionConfiguration); 276 } 277 initializeParcelable(CaptureResult result)278 private static ParcelCaptureResult initializeParcelable(CaptureResult result) { 279 ParcelCaptureResult ret = new ParcelCaptureResult(); 280 ret.cameraId = result.getCameraId(); 281 ret.results = result.getNativeMetadata(); 282 ret.parent = result.getRequest(); 283 ret.sequenceId = result.getSequenceId(); 284 ret.frameNumber = result.getFrameNumber(); 285 286 return ret; 287 } 288 initializeParcelable(TotalCaptureResult totalResult)289 private static ParcelTotalCaptureResult initializeParcelable(TotalCaptureResult totalResult) { 290 ParcelTotalCaptureResult ret = new ParcelTotalCaptureResult(); 291 ret.logicalCameraId = totalResult.getCameraId(); 292 ret.results = totalResult.getNativeMetadata(); 293 ret.parent = totalResult.getRequest(); 294 ret.sequenceId = totalResult.getSequenceId(); 295 ret.frameNumber = totalResult.getFrameNumber(); 296 ret.sessionId = totalResult.getSessionId(); 297 ret.partials = new ArrayList<>(totalResult.getPartialResults().size()); 298 for (CaptureResult partial : totalResult.getPartialResults()) { 299 ret.partials.add(initializeParcelable(partial)); 300 } 301 Map<String, TotalCaptureResult> physicalResults = 302 totalResult.getPhysicalCameraTotalResults(); 303 ret.physicalResult = new ArrayList<>(physicalResults.size()); 304 for (TotalCaptureResult physicalResult : physicalResults.values()) { 305 ret.physicalResult.add(new PhysicalCaptureResultInfo(physicalResult.getCameraId(), 306 physicalResult.getNativeMetadata())); 307 } 308 309 return ret; 310 } 311 initializeParcelable(Surface s)312 private static OutputSurface initializeParcelable(Surface s) { 313 OutputSurface ret = new OutputSurface(); 314 if (s != null) { 315 ret.surface = s; 316 ret.size = new android.hardware.camera2.extension.Size(); 317 Size surfaceSize = SurfaceUtils.getSurfaceSize(s); 318 ret.size.width = surfaceSize.getWidth(); 319 ret.size.height = surfaceSize.getHeight(); 320 ret.imageFormat = SurfaceUtils.getSurfaceFormat(s); 321 } else { 322 ret.surface = null; 323 ret.size = new android.hardware.camera2.extension.Size(); 324 ret.size.width = -1; 325 ret.size.height = -1; 326 ret.imageFormat = ImageFormat.UNKNOWN; 327 } 328 329 return ret; 330 } 331 332 @Override getDevice()333 public @NonNull CameraDevice getDevice() { 334 synchronized (mInterfaceLock) { 335 return mCameraDevice; 336 } 337 } 338 339 @Override setRepeatingRequest(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)340 public int setRepeatingRequest(@NonNull CaptureRequest request, @NonNull Executor executor, 341 @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { 342 int seqId = -1; 343 synchronized (mInterfaceLock) { 344 if (!mInitialized) { 345 throw new IllegalStateException("Uninitialized component"); 346 } 347 348 if (mClientRepeatingRequestSurface == null) { 349 throw new IllegalArgumentException("No registered preview surface"); 350 } 351 352 if (!request.containsTarget(mClientRepeatingRequestSurface) || 353 (request.getTargets().size() != 1)) { 354 throw new IllegalArgumentException("Invalid repeating request output target!"); 355 } 356 357 try { 358 seqId = mSessionProcessor.startRepeating(new RequestCallbackHandler(request, 359 executor, listener)); 360 } catch (RemoteException e) { 361 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, 362 "Failed to enable repeating request, extension service failed to respond!"); 363 } 364 } 365 366 return seqId; 367 } 368 369 @Override capture(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)370 public int capture(@NonNull CaptureRequest request, 371 @NonNull Executor executor, 372 @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { 373 int seqId = -1; 374 synchronized (mInterfaceLock) { 375 if (!mInitialized) { 376 throw new IllegalStateException("Uninitialized component"); 377 } 378 379 if (mClientCaptureSurface == null) { 380 throw new IllegalArgumentException("No output surface registered for single" 381 + " requests!"); 382 } 383 384 if (!request.containsTarget(mClientCaptureSurface) || 385 (request.getTargets().size() != 1)) { 386 throw new IllegalArgumentException("Invalid single capture output target!"); 387 } 388 389 try { 390 // This will override the extension capture stage jpeg parameters with the user set 391 // jpeg quality and rotation. This will guarantee that client configured jpeg 392 // parameters always have highest priority. 393 Integer jpegRotation = request.get(CaptureRequest.JPEG_ORIENTATION); 394 if (jpegRotation == null) { 395 jpegRotation = CameraExtensionUtils.JPEG_DEFAULT_ROTATION; 396 } 397 Byte jpegQuality = request.get(CaptureRequest.JPEG_QUALITY); 398 if (jpegQuality == null) { 399 jpegQuality = CameraExtensionUtils.JPEG_DEFAULT_QUALITY; 400 } 401 402 seqId = mSessionProcessor.startCapture(new RequestCallbackHandler(request, 403 executor, listener), jpegRotation, jpegQuality); 404 } catch (RemoteException e) { 405 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, 406 "Failed to submit capture request, extension service failed to respond!"); 407 } 408 } 409 410 return seqId; 411 } 412 413 @Override stopRepeating()414 public void stopRepeating() throws CameraAccessException { 415 synchronized (mInterfaceLock) { 416 if (!mInitialized) { 417 throw new IllegalStateException("Uninitialized component"); 418 } 419 420 mCaptureSession.stopRepeating(); 421 422 try { 423 mSessionProcessor.stopRepeating(); 424 } catch (RemoteException e) { 425 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, 426 "Failed to notify about the end of repeating request, extension service" 427 + " failed to respond!"); 428 } 429 } 430 } 431 432 @Override close()433 public void close() throws CameraAccessException { 434 synchronized (mInterfaceLock) { 435 if (mInitialized) { 436 try { 437 mCaptureSession.stopRepeating(); 438 mSessionProcessor.stopRepeating(); 439 mSessionProcessor.onCaptureSessionEnd(); 440 } catch (RemoteException e) { 441 Log.e(TAG, "Failed to stop the repeating request or end the session," 442 + " , extension service does not respond!") ; 443 } 444 mCaptureSession.close(); 445 } 446 } 447 } 448 release(boolean skipCloseNotification)449 public void release(boolean skipCloseNotification) { 450 boolean notifyClose = false; 451 452 synchronized (mInterfaceLock) { 453 mHandlerThread.quitSafely(); 454 455 if (mSessionProcessor != null) { 456 try { 457 mSessionProcessor.deInitSession(); 458 } catch (RemoteException e) { 459 Log.e(TAG, "Failed to de-initialize session processor, extension service" 460 + " does not respond!") ; 461 } 462 mSessionProcessor = null; 463 } 464 465 if (mExtensionClientId >= 0) { 466 CameraExtensionCharacteristics.unregisterClient(mExtensionClientId); 467 if (mInitialized) { 468 notifyClose = true; 469 CameraExtensionCharacteristics.releaseSession(); 470 } 471 } 472 mInitialized = false; 473 474 for (ImageReader reader : mReaderMap.values()) { 475 reader.close(); 476 } 477 mReaderMap.clear(); 478 479 mClientRepeatingRequestSurface = null; 480 mClientCaptureSurface = null; 481 } 482 483 if (notifyClose && !skipCloseNotification) { 484 final long ident = Binder.clearCallingIdentity(); 485 try { 486 mExecutor.execute(() -> mCallbacks.onClosed( 487 CameraAdvancedExtensionSessionImpl.this)); 488 } finally { 489 Binder.restoreCallingIdentity(ident); 490 } 491 } 492 } 493 notifyConfigurationFailure()494 private void notifyConfigurationFailure() { 495 synchronized (mInterfaceLock) { 496 if (mInitialized) { 497 return; 498 } 499 } 500 501 release(true /*skipCloseNotification*/); 502 503 final long ident = Binder.clearCallingIdentity(); 504 try { 505 mExecutor.execute( 506 () -> mCallbacks.onConfigureFailed( 507 CameraAdvancedExtensionSessionImpl.this)); 508 } finally { 509 Binder.restoreCallingIdentity(ident); 510 } 511 } 512 513 private class SessionStateHandler extends 514 android.hardware.camera2.CameraCaptureSession.StateCallback { 515 @Override onClosed(@onNull CameraCaptureSession session)516 public void onClosed(@NonNull CameraCaptureSession session) { 517 release(false /*skipCloseNotification*/); 518 } 519 520 @Override onConfigureFailed(@onNull CameraCaptureSession session)521 public void onConfigureFailed(@NonNull CameraCaptureSession session) { 522 notifyConfigurationFailure(); 523 } 524 525 @Override onConfigured(@onNull CameraCaptureSession session)526 public void onConfigured(@NonNull CameraCaptureSession session) { 527 synchronized (mInterfaceLock) { 528 mCaptureSession = session; 529 try { 530 CameraExtensionCharacteristics.initializeSession(mInitializeHandler); 531 } catch (RemoteException e) { 532 Log.e(TAG, "Failed to initialize session! Extension service does" 533 + " not respond!"); 534 notifyConfigurationFailure(); 535 } 536 } 537 } 538 } 539 540 private class InitializeSessionHandler extends IInitializeSessionCallback.Stub { 541 @Override onSuccess()542 public void onSuccess() { 543 boolean status = true; 544 synchronized (mInterfaceLock) { 545 try { 546 mSessionProcessor.onCaptureSessionStart(mRequestProcessor); 547 mInitialized = true; 548 } catch (RemoteException e) { 549 Log.e(TAG, "Failed to start capture session," 550 + " extension service does not respond!"); 551 status = false; 552 mCaptureSession.close(); 553 } 554 } 555 556 if (status) { 557 final long ident = Binder.clearCallingIdentity(); 558 try { 559 mExecutor.execute( 560 () -> mCallbacks.onConfigured(CameraAdvancedExtensionSessionImpl.this)); 561 } finally { 562 Binder.restoreCallingIdentity(ident); 563 } 564 } else { 565 notifyConfigurationFailure(); 566 } 567 } 568 569 @Override onFailure()570 public void onFailure() { 571 mCaptureSession.close(); 572 Log.e(TAG, "Failed to initialize proxy service session!" 573 + " This can happen when trying to configure multiple " 574 + "concurrent extension sessions!"); 575 notifyConfigurationFailure(); 576 } 577 } 578 579 private final class RequestCallbackHandler extends ICaptureCallback.Stub { 580 private final CaptureRequest mClientRequest; 581 private final Executor mClientExecutor; 582 private final ExtensionCaptureCallback mClientCallbacks; 583 RequestCallbackHandler(@onNull CaptureRequest clientRequest, @NonNull Executor clientExecutor, @NonNull ExtensionCaptureCallback clientCallbacks)584 private RequestCallbackHandler(@NonNull CaptureRequest clientRequest, 585 @NonNull Executor clientExecutor, 586 @NonNull ExtensionCaptureCallback clientCallbacks) { 587 mClientRequest = clientRequest; 588 mClientExecutor = clientExecutor; 589 mClientCallbacks = clientCallbacks; 590 } 591 592 @Override onCaptureStarted(int captureSequenceId, long timestamp)593 public void onCaptureStarted(int captureSequenceId, long timestamp) { 594 final long ident = Binder.clearCallingIdentity(); 595 try { 596 mClientExecutor.execute( 597 () -> mClientCallbacks.onCaptureStarted( 598 CameraAdvancedExtensionSessionImpl.this, mClientRequest, 599 timestamp)); 600 } finally { 601 Binder.restoreCallingIdentity(ident); 602 } 603 } 604 605 @Override onCaptureProcessStarted(int captureSequenceId)606 public void onCaptureProcessStarted(int captureSequenceId) { 607 final long ident = Binder.clearCallingIdentity(); 608 try { 609 mClientExecutor.execute( 610 () -> mClientCallbacks.onCaptureProcessStarted( 611 CameraAdvancedExtensionSessionImpl.this, mClientRequest)); 612 } finally { 613 Binder.restoreCallingIdentity(ident); 614 } 615 } 616 617 @Override onCaptureFailed(int captureSequenceId)618 public void onCaptureFailed(int captureSequenceId) { 619 final long ident = Binder.clearCallingIdentity(); 620 try { 621 mClientExecutor.execute( 622 () -> mClientCallbacks.onCaptureFailed( 623 CameraAdvancedExtensionSessionImpl.this, mClientRequest)); 624 } finally { 625 Binder.restoreCallingIdentity(ident); 626 } 627 } 628 629 @Override onCaptureSequenceCompleted(int captureSequenceId)630 public void onCaptureSequenceCompleted(int captureSequenceId) { 631 final long ident = Binder.clearCallingIdentity(); 632 try { 633 mClientExecutor.execute( 634 () -> mClientCallbacks.onCaptureSequenceCompleted( 635 CameraAdvancedExtensionSessionImpl.this, captureSequenceId)); 636 } finally { 637 Binder.restoreCallingIdentity(ident); 638 } 639 } 640 641 @Override onCaptureSequenceAborted(int captureSequenceId)642 public void onCaptureSequenceAborted(int captureSequenceId) { 643 final long ident = Binder.clearCallingIdentity(); 644 try { 645 mClientExecutor.execute( 646 () -> mClientCallbacks.onCaptureSequenceAborted( 647 CameraAdvancedExtensionSessionImpl.this, captureSequenceId)); 648 } finally { 649 Binder.restoreCallingIdentity(ident); 650 } 651 } 652 } 653 654 private final class CaptureCallbackHandler extends CameraCaptureSession.CaptureCallback { 655 private final IRequestCallback mCallback; 656 CaptureCallbackHandler(IRequestCallback callback)657 public CaptureCallbackHandler(IRequestCallback callback) { 658 mCallback = callback; 659 } 660 661 @Override onCaptureBufferLost(CameraCaptureSession session, CaptureRequest request, Surface target, long frameNumber)662 public void onCaptureBufferLost(CameraCaptureSession session, CaptureRequest request, 663 Surface target, long frameNumber) { 664 try { 665 if (request.getTag() instanceof Integer) { 666 Integer requestId = (Integer) request.getTag(); 667 mCallback.onCaptureBufferLost(requestId, frameNumber, 668 mCameraConfigMap.get(target).outputId.id); 669 } else { 670 Log.e(TAG, "Invalid capture request tag!"); 671 } 672 } catch (RemoteException e) { 673 Log.e(TAG, "Failed to notify lost capture buffer, extension service doesn't" 674 + " respond!"); 675 } 676 } 677 678 @Override onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result)679 public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, 680 TotalCaptureResult result) { 681 try { 682 if (request.getTag() instanceof Integer) { 683 Integer requestId = (Integer) request.getTag(); 684 mCallback.onCaptureCompleted(requestId, initializeParcelable(result)); 685 } else { 686 Log.e(TAG, "Invalid capture request tag!"); 687 } 688 } catch (RemoteException e) { 689 Log.e(TAG, "Failed to notify capture result, extension service doesn't" 690 + " respond!"); 691 } 692 } 693 694 @Override onCaptureFailed(CameraCaptureSession session, CaptureRequest request, CaptureFailure failure)695 public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request, 696 CaptureFailure failure) { 697 try { 698 if (request.getTag() instanceof Integer) { 699 Integer requestId = (Integer) request.getTag(); 700 android.hardware.camera2.extension.CaptureFailure captureFailure = 701 new android.hardware.camera2.extension.CaptureFailure(); 702 captureFailure.request = request; 703 captureFailure.reason = failure.getReason(); 704 captureFailure.errorPhysicalCameraId = failure.getPhysicalCameraId(); 705 captureFailure.frameNumber = failure.getFrameNumber(); 706 captureFailure.sequenceId = failure.getSequenceId(); 707 captureFailure.dropped = !failure.wasImageCaptured(); 708 mCallback.onCaptureFailed(requestId, captureFailure); 709 } else { 710 Log.e(TAG, "Invalid capture request tag!"); 711 } 712 } catch (RemoteException e) { 713 Log.e(TAG, "Failed to notify capture failure, extension service doesn't" 714 + " respond!"); 715 } 716 } 717 718 @Override onCaptureProgressed(CameraCaptureSession session, CaptureRequest request, CaptureResult partialResult)719 public void onCaptureProgressed(CameraCaptureSession session, CaptureRequest request, 720 CaptureResult partialResult) { 721 try { 722 if (request.getTag() instanceof Integer) { 723 Integer requestId = (Integer) request.getTag(); 724 mCallback.onCaptureProgressed(requestId, initializeParcelable(partialResult)); 725 } else { 726 Log.e(TAG, "Invalid capture request tag!"); 727 } 728 } catch (RemoteException e) { 729 Log.e(TAG, "Failed to notify capture partial result, extension service doesn't" 730 + " respond!"); 731 } 732 } 733 734 @Override onCaptureSequenceAborted(CameraCaptureSession session, int sequenceId)735 public void onCaptureSequenceAborted(CameraCaptureSession session, int sequenceId) { 736 try { 737 mCallback.onCaptureSequenceAborted(sequenceId); 738 } catch (RemoteException e) { 739 Log.e(TAG, "Failed to notify aborted sequence, extension service doesn't" 740 + " respond!"); 741 } 742 } 743 744 @Override onCaptureSequenceCompleted(CameraCaptureSession session, int sequenceId, long frameNumber)745 public void onCaptureSequenceCompleted(CameraCaptureSession session, int sequenceId, 746 long frameNumber) { 747 try { 748 mCallback.onCaptureSequenceCompleted(sequenceId, frameNumber); 749 } catch (RemoteException e) { 750 Log.e(TAG, "Failed to notify sequence complete, extension service doesn't" 751 + " respond!"); 752 } 753 } 754 755 @Override onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber)756 public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, 757 long timestamp, long frameNumber) { 758 try { 759 if (request.getTag() instanceof Integer) { 760 Integer requestId = (Integer) request.getTag(); 761 mCallback.onCaptureStarted(requestId, frameNumber, timestamp); 762 } else { 763 Log.e(TAG, "Invalid capture request tag!"); 764 } 765 } catch (RemoteException e) { 766 Log.e(TAG, "Failed to notify capture started, extension service doesn't" 767 + " respond!"); 768 } 769 } 770 } 771 772 private static final class ImageReaderHandler implements ImageReader.OnImageAvailableListener { 773 private final OutputConfigId mOutputConfigId; 774 private final IImageProcessorImpl mIImageProcessor; 775 private final String mPhysicalCameraId; 776 ImageReaderHandler(int outputConfigId, IImageProcessorImpl iImageProcessor, String physicalCameraId)777 private ImageReaderHandler(int outputConfigId, 778 IImageProcessorImpl iImageProcessor, String physicalCameraId) { 779 mOutputConfigId = new OutputConfigId(); 780 mOutputConfigId.id = outputConfigId; 781 mIImageProcessor = iImageProcessor; 782 mPhysicalCameraId = physicalCameraId; 783 } 784 785 @Override onImageAvailable(ImageReader reader)786 public void onImageAvailable(ImageReader reader) { 787 if (mIImageProcessor == null) { 788 return; 789 } 790 791 Image img; 792 try { 793 img = reader.acquireNextImage(); 794 } catch (IllegalStateException e) { 795 Log.e(TAG, "Failed to acquire image, too many images pending!"); 796 return; 797 } 798 if (img == null) { 799 Log.e(TAG, "Invalid image!"); 800 return; 801 } 802 803 try { 804 reader.detachImage(img); 805 } catch(Exception e) { 806 Log.e(TAG, "Failed to detach image"); 807 img.close(); 808 return; 809 } 810 811 ParcelImage parcelImage = new ParcelImage(); 812 parcelImage.buffer = img.getHardwareBuffer(); 813 if (img.getFenceFd() >= 0) { 814 try { 815 parcelImage.fence = ParcelFileDescriptor.fromFd(img.getFenceFd()); 816 } catch (IOException e) { 817 Log.e(TAG,"Failed to parcel buffer fence!"); 818 } 819 } 820 parcelImage.width = img.getWidth(); 821 parcelImage.height = img.getHeight(); 822 parcelImage.format = img.getFormat(); 823 parcelImage.timestamp = img.getTimestamp(); 824 parcelImage.transform = img.getTransform(); 825 parcelImage.scalingMode = img.getScalingMode(); 826 parcelImage.planeCount = img.getPlaneCount(); 827 parcelImage.crop = img.getCropRect(); 828 829 try { 830 mIImageProcessor.onNextImageAvailable(mOutputConfigId, parcelImage, 831 mPhysicalCameraId); 832 } catch (RemoteException e) { 833 Log.e(TAG, "Failed to propagate image buffer on output surface id: " + 834 mOutputConfigId + " extension service does not respond!"); 835 } finally { 836 parcelImage.buffer.close(); 837 img.close(); 838 } 839 } 840 } 841 842 private final class RequestProcessor extends IRequestProcessorImpl.Stub { 843 @Override setImageProcessor(OutputConfigId outputConfigId, IImageProcessorImpl imageProcessor)844 public void setImageProcessor(OutputConfigId outputConfigId, 845 IImageProcessorImpl imageProcessor) { 846 synchronized (mInterfaceLock) { 847 if (mReaderMap.containsKey(outputConfigId.id)) { 848 ImageReader reader = mReaderMap.get(outputConfigId.id); 849 String physicalCameraId = null; 850 if (mCameraConfigMap.containsKey(reader.getSurface())) { 851 physicalCameraId = 852 mCameraConfigMap.get(reader.getSurface()).physicalCameraId; 853 reader.setOnImageAvailableListener(new ImageReaderHandler(outputConfigId.id, 854 imageProcessor, physicalCameraId), mHandler); 855 } else { 856 Log.e(TAG, "Camera output configuration for ImageReader with " + 857 " config Id " + outputConfigId.id + " not found!"); 858 } 859 } else { 860 Log.e(TAG, "ImageReader with output config id: " + outputConfigId.id + 861 " not found!"); 862 } 863 } 864 } 865 866 @Override submit(Request request, IRequestCallback callback)867 public int submit(Request request, IRequestCallback callback) { 868 ArrayList<Request> captureList = new ArrayList<>(); 869 captureList.add(request); 870 return submitBurst(captureList, callback); 871 } 872 873 @Override submitBurst(List<Request> requests, IRequestCallback callback)874 public int submitBurst(List<Request> requests, IRequestCallback callback) { 875 int seqId = -1; 876 try { 877 CaptureCallbackHandler captureCallback = new CaptureCallbackHandler(callback); 878 ArrayList<CaptureRequest> captureRequests = new ArrayList<>(); 879 for (Request request : requests) { 880 captureRequests.add(initializeCaptureRequest(mCameraDevice, request, 881 mCameraConfigMap)); 882 } 883 seqId = mCaptureSession.captureBurstRequests(captureRequests, 884 new CameraExtensionUtils.HandlerExecutor(mHandler), captureCallback); 885 } catch (CameraAccessException e) { 886 Log.e(TAG, "Failed to submit capture requests!"); 887 } catch (IllegalStateException e) { 888 Log.e(TAG, "Capture session closed!"); 889 } 890 891 return seqId; 892 } 893 894 @Override setRepeating(Request request, IRequestCallback callback)895 public int setRepeating(Request request, IRequestCallback callback) { 896 int seqId = -1; 897 try { 898 CaptureRequest repeatingRequest = initializeCaptureRequest(mCameraDevice, 899 request, mCameraConfigMap); 900 CaptureCallbackHandler captureCallback = new CaptureCallbackHandler(callback); 901 seqId = mCaptureSession.setSingleRepeatingRequest(repeatingRequest, 902 new CameraExtensionUtils.HandlerExecutor(mHandler), captureCallback); 903 } catch (CameraAccessException e) { 904 Log.e(TAG, "Failed to enable repeating request!"); 905 } catch (IllegalStateException e) { 906 Log.e(TAG, "Capture session closed!"); 907 } 908 909 return seqId; 910 } 911 912 @Override abortCaptures()913 public void abortCaptures() { 914 try { 915 mCaptureSession.abortCaptures(); 916 } catch (CameraAccessException e) { 917 Log.e(TAG, "Failed during capture abort!"); 918 } catch (IllegalStateException e) { 919 Log.e(TAG, "Capture session closed!"); 920 } 921 } 922 923 @Override stopRepeating()924 public void stopRepeating() { 925 try { 926 mCaptureSession.stopRepeating(); 927 } catch (CameraAccessException e) { 928 Log.e(TAG, "Failed during repeating capture stop!"); 929 } catch (IllegalStateException e) { 930 Log.e(TAG, "Capture session closed!"); 931 } 932 } 933 } 934 initializeCaptureRequest(CameraDevice cameraDevice, Request request, HashMap<Surface, CameraOutputConfig> surfaceIdMap)935 private static CaptureRequest initializeCaptureRequest(CameraDevice cameraDevice, 936 Request request, HashMap<Surface, CameraOutputConfig> surfaceIdMap) 937 throws CameraAccessException { 938 CaptureRequest.Builder builder = cameraDevice.createCaptureRequest(request.templateId); 939 for (OutputConfigId configId : request.targetOutputConfigIds) { 940 boolean found = false; 941 for (Map.Entry<Surface, CameraOutputConfig> entry : surfaceIdMap.entrySet()) { 942 if (entry.getValue().outputId.id == configId.id) { 943 builder.addTarget(entry.getKey()); 944 found = true; 945 break; 946 } 947 } 948 949 if (!found) { 950 Log.e(TAG, "Surface with output id: " + configId.id + 951 " not found among registered camera outputs!"); 952 } 953 } 954 955 builder.setTag(request.requestId); 956 CaptureRequest ret = builder.build(); 957 CameraMetadataNative.update(ret.getNativeMetadata(), request.parameters); 958 return ret; 959 } 960 } 961