1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import android.annotation.NonNull; 20 import android.annotation.Nullable; 21 import android.annotation.RequiresPermission; 22 import android.content.Context; 23 import android.graphics.ImageFormat; 24 import android.graphics.SurfaceTexture; 25 import android.hardware.SyncFence; 26 import android.hardware.camera2.CameraAccessException; 27 import android.hardware.camera2.CameraCaptureSession; 28 import android.hardware.camera2.CameraCharacteristics; 29 import android.hardware.camera2.CameraDevice; 30 import android.hardware.camera2.CameraExtensionCharacteristics; 31 import android.hardware.camera2.CameraExtensionSession; 32 import android.hardware.camera2.CaptureFailure; 33 import android.hardware.camera2.CaptureRequest; 34 import android.hardware.camera2.CaptureResult; 35 import android.hardware.camera2.TotalCaptureResult; 36 import android.hardware.camera2.extension.CameraOutputConfig; 37 import android.hardware.camera2.extension.CameraSessionConfig; 38 import android.hardware.camera2.extension.IAdvancedExtenderImpl; 39 import android.hardware.camera2.extension.ICaptureCallback; 40 import android.hardware.camera2.extension.IImageProcessorImpl; 41 import android.hardware.camera2.extension.IInitializeSessionCallback; 42 import android.hardware.camera2.extension.IRequestCallback; 43 import android.hardware.camera2.extension.IRequestProcessorImpl; 44 import android.hardware.camera2.extension.ISessionProcessorImpl; 45 import android.hardware.camera2.extension.LatencyPair; 46 import android.hardware.camera2.extension.OutputConfigId; 47 import android.hardware.camera2.extension.OutputSurface; 48 import android.hardware.camera2.extension.ParcelCaptureResult; 49 import android.hardware.camera2.extension.ParcelImage; 50 import android.hardware.camera2.extension.ParcelTotalCaptureResult; 51 import android.hardware.camera2.extension.Request; 52 import android.hardware.camera2.params.DynamicRangeProfiles; 53 import android.hardware.camera2.params.ExtensionSessionConfiguration; 54 import android.hardware.camera2.params.OutputConfiguration; 55 import android.hardware.camera2.params.SessionConfiguration; 56 import android.hardware.camera2.utils.ExtensionSessionStatsAggregator; 57 import android.hardware.camera2.utils.SurfaceUtils; 58 import android.media.Image; 59 import android.media.ImageReader; 60 import android.os.Binder; 61 import android.os.Handler; 62 import android.os.HandlerThread; 63 import android.os.IBinder; 64 import android.os.RemoteException; 65 import android.util.Log; 66 import android.util.Size; 67 import android.view.Surface; 68 69 import java.io.IOException; 70 import java.util.ArrayList; 71 import java.util.Arrays; 72 import java.util.HashMap; 73 import java.util.List; 74 import java.util.Map; 75 import java.util.concurrent.Executor; 76 77 public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSession { 78 private static final String TAG = "CameraAdvancedExtensionSessionImpl"; 79 80 private final Executor mExecutor; 81 private CameraDevice mCameraDevice; 82 private final Map<String, CameraMetadataNative> mCharacteristicsMap; 83 private final Handler mHandler; 84 private final HandlerThread mHandlerThread; 85 private final CameraExtensionSession.StateCallback mCallbacks; 86 private IAdvancedExtenderImpl mAdvancedExtender; 87 // maps registered camera surfaces to extension output configs 88 private final HashMap<Surface, CameraOutputConfig> mCameraConfigMap = new HashMap<>(); 89 // maps camera extension output ids to camera registered image readers 90 private final HashMap<Integer, ImageReader> mReaderMap = new HashMap<>(); 91 private RequestProcessor mRequestProcessor = new RequestProcessor(); 92 private final int mSessionId; 93 private IBinder mToken = null; 94 95 private Surface mClientRepeatingRequestSurface; 96 private Surface mClientCaptureSurface; 97 private Surface mClientPostviewSurface; 98 private CameraCaptureSession mCaptureSession = null; 99 private ISessionProcessorImpl mSessionProcessor = null; 100 private final InitializeSessionHandler mInitializeHandler; 101 private final ExtensionSessionStatsAggregator mStatsAggregator; 102 103 private boolean mInitialized; 104 private boolean mSessionClosed; 105 106 private final Context mContext; 107 108 // Lock to synchronize cross-thread access to device public interface 109 final Object mInterfaceLock; 110 111 /** 112 * @hide 113 */ 114 @RequiresPermission(android.Manifest.permission.CAMERA) createCameraAdvancedExtensionSession( @onNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice, @NonNull Map<String, CameraCharacteristics> characteristicsMap, @NonNull Context ctx, @NonNull ExtensionSessionConfiguration config, int sessionId, @NonNull IBinder token)115 public static CameraAdvancedExtensionSessionImpl createCameraAdvancedExtensionSession( 116 @NonNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice, 117 @NonNull Map<String, CameraCharacteristics> characteristicsMap, 118 @NonNull Context ctx, @NonNull ExtensionSessionConfiguration config, int sessionId, 119 @NonNull IBinder token) 120 throws CameraAccessException, RemoteException { 121 String cameraId = cameraDevice.getId(); 122 CameraExtensionCharacteristics extensionChars = new CameraExtensionCharacteristics(ctx, 123 cameraId, characteristicsMap); 124 125 Map<String, CameraMetadataNative> characteristicsMapNative = 126 CameraExtensionUtils.getCharacteristicsMapNative(characteristicsMap); 127 if (!CameraExtensionCharacteristics.isExtensionSupported(cameraDevice.getId(), 128 config.getExtension(), characteristicsMapNative)) { 129 throw new UnsupportedOperationException("Unsupported extension type: " + 130 config.getExtension()); 131 } 132 133 if (config.getOutputConfigurations().isEmpty() || 134 config.getOutputConfigurations().size() > 2) { 135 throw new IllegalArgumentException("Unexpected amount of output surfaces, received: " + 136 config.getOutputConfigurations().size() + " expected <= 2"); 137 } 138 139 for (OutputConfiguration c : config.getOutputConfigurations()) { 140 if (c.getDynamicRangeProfile() != DynamicRangeProfiles.STANDARD) { 141 throw new IllegalArgumentException("Unsupported dynamic range profile: " + 142 c.getDynamicRangeProfile()); 143 } 144 if (c.getStreamUseCase() != 145 CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT) { 146 throw new IllegalArgumentException("Unsupported stream use case: " + 147 c.getStreamUseCase()); 148 } 149 } 150 151 int suitableSurfaceCount = 0; 152 List<Size> supportedPreviewSizes = extensionChars.getExtensionSupportedSizes( 153 config.getExtension(), SurfaceTexture.class); 154 Surface repeatingRequestSurface = CameraExtensionUtils.getRepeatingRequestSurface( 155 config.getOutputConfigurations(), supportedPreviewSizes); 156 if (repeatingRequestSurface != null) { 157 suitableSurfaceCount++; 158 } 159 160 HashMap<Integer, List<Size>> supportedCaptureSizes = new HashMap<>(); 161 for (int format : CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS) { 162 List<Size> supportedSizes = extensionChars.getExtensionSupportedSizes( 163 config.getExtension(), format); 164 if (supportedSizes != null) { 165 supportedCaptureSizes.put(format, supportedSizes); 166 } 167 } 168 Surface burstCaptureSurface = CameraExtensionUtils.getBurstCaptureSurface( 169 config.getOutputConfigurations(), supportedCaptureSizes); 170 if (burstCaptureSurface != null) { 171 suitableSurfaceCount++; 172 } 173 174 if (suitableSurfaceCount != config.getOutputConfigurations().size()) { 175 throw new IllegalArgumentException("One or more unsupported output surfaces found!"); 176 } 177 178 Surface postviewSurface = null; 179 if (burstCaptureSurface != null && config.getPostviewOutputConfiguration() != null) { 180 CameraExtensionUtils.SurfaceInfo burstCaptureSurfaceInfo = 181 CameraExtensionUtils.querySurface(burstCaptureSurface); 182 Size burstCaptureSurfaceSize = 183 new Size(burstCaptureSurfaceInfo.mWidth, burstCaptureSurfaceInfo.mHeight); 184 HashMap<Integer, List<Size>> supportedPostviewSizes = new HashMap<>(); 185 for (int format : CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS) { 186 List<Size> supportedSizesPostview = extensionChars.getPostviewSupportedSizes( 187 config.getExtension(), burstCaptureSurfaceSize, format); 188 if (supportedSizesPostview != null) { 189 supportedPostviewSizes.put(format, supportedSizesPostview); 190 } 191 } 192 193 postviewSurface = CameraExtensionUtils.getPostviewSurface( 194 config.getPostviewOutputConfiguration(), supportedPostviewSizes, 195 burstCaptureSurfaceInfo.mFormat); 196 if (postviewSurface == null) { 197 throw new IllegalArgumentException("Unsupported output surface for postview!"); 198 } 199 } 200 201 IAdvancedExtenderImpl extender = CameraExtensionCharacteristics.initializeAdvancedExtension( 202 config.getExtension()); 203 extender.init(cameraId, characteristicsMapNative); 204 205 CameraAdvancedExtensionSessionImpl ret = new CameraAdvancedExtensionSessionImpl(ctx, 206 extender, cameraDevice, characteristicsMapNative, repeatingRequestSurface, 207 burstCaptureSurface, postviewSurface, config.getStateCallback(), 208 config.getExecutor(), sessionId, token); 209 210 ret.mStatsAggregator.setClientName(ctx.getOpPackageName()); 211 ret.mStatsAggregator.setExtensionType(config.getExtension()); 212 213 ret.initialize(); 214 215 return ret; 216 } 217 CameraAdvancedExtensionSessionImpl(Context ctx, @NonNull IAdvancedExtenderImpl extender, @NonNull CameraDeviceImpl cameraDevice, Map<String, CameraMetadataNative> characteristicsMap, @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, @Nullable Surface postviewSurface, @NonNull StateCallback callback, @NonNull Executor executor, int sessionId, @NonNull IBinder token)218 private CameraAdvancedExtensionSessionImpl(Context ctx, 219 @NonNull IAdvancedExtenderImpl extender, 220 @NonNull CameraDeviceImpl cameraDevice, 221 Map<String, CameraMetadataNative> characteristicsMap, 222 @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, 223 @Nullable Surface postviewSurface, 224 @NonNull StateCallback callback, @NonNull Executor executor, 225 int sessionId, 226 @NonNull IBinder token) { 227 mContext = ctx; 228 mAdvancedExtender = extender; 229 mCameraDevice = cameraDevice; 230 mCharacteristicsMap = characteristicsMap; 231 mCallbacks = callback; 232 mExecutor = executor; 233 mClientRepeatingRequestSurface = repeatingRequestSurface; 234 mClientCaptureSurface = burstCaptureSurface; 235 mClientPostviewSurface = postviewSurface; 236 mHandlerThread = new HandlerThread(TAG); 237 mHandlerThread.start(); 238 mHandler = new Handler(mHandlerThread.getLooper()); 239 mInitialized = false; 240 mSessionClosed = false; 241 mInitializeHandler = new InitializeSessionHandler(); 242 mSessionId = sessionId; 243 mToken = token; 244 mInterfaceLock = cameraDevice.mInterfaceLock; 245 246 mStatsAggregator = new ExtensionSessionStatsAggregator(mCameraDevice.getId(), 247 /*isAdvanced=*/true); 248 } 249 250 /** 251 * @hide 252 */ initialize()253 public synchronized void initialize() throws CameraAccessException, RemoteException { 254 if (mInitialized) { 255 Log.d(TAG, "Session already initialized"); 256 return; 257 } 258 259 OutputSurface previewSurface = initializeParcelable(mClientRepeatingRequestSurface); 260 OutputSurface captureSurface = initializeParcelable(mClientCaptureSurface); 261 OutputSurface postviewSurface = initializeParcelable(mClientPostviewSurface); 262 263 mSessionProcessor = mAdvancedExtender.getSessionProcessor(); 264 CameraSessionConfig sessionConfig = mSessionProcessor.initSession(mToken, 265 mCameraDevice.getId(), 266 mCharacteristicsMap, previewSurface, captureSurface, postviewSurface); 267 List<CameraOutputConfig> outputConfigs = sessionConfig.outputConfigs; 268 ArrayList<OutputConfiguration> outputList = new ArrayList<>(); 269 for (CameraOutputConfig output : outputConfigs) { 270 Surface outputSurface = initializeSurfrace(output); 271 if (outputSurface == null) { 272 continue; 273 } 274 OutputConfiguration cameraOutput = new OutputConfiguration(output.surfaceGroupId, 275 outputSurface); 276 277 if (output.isMultiResolutionOutput) { 278 cameraOutput.setMultiResolutionOutput(); 279 } 280 if ((output.sharedSurfaceConfigs != null) && !output.sharedSurfaceConfigs.isEmpty()) { 281 cameraOutput.enableSurfaceSharing(); 282 for (CameraOutputConfig sharedOutputConfig : output.sharedSurfaceConfigs) { 283 Surface sharedSurface = initializeSurfrace(sharedOutputConfig); 284 if (sharedSurface == null) { 285 continue; 286 } 287 cameraOutput.addSurface(sharedSurface); 288 mCameraConfigMap.put(sharedSurface, sharedOutputConfig); 289 } 290 } 291 292 // The extension processing logic needs to be able to match images to capture results via 293 // image and result timestamps. 294 cameraOutput.setTimestampBase(OutputConfiguration.TIMESTAMP_BASE_SENSOR); 295 cameraOutput.setReadoutTimestampEnabled(false); 296 cameraOutput.setPhysicalCameraId(output.physicalCameraId); 297 outputList.add(cameraOutput); 298 mCameraConfigMap.put(cameraOutput.getSurface(), output); 299 } 300 301 int sessionType = SessionConfiguration.SESSION_REGULAR; 302 if (sessionConfig.sessionType != -1 && 303 (sessionConfig.sessionType != SessionConfiguration.SESSION_HIGH_SPEED)) { 304 sessionType = sessionConfig.sessionType; 305 Log.v(TAG, "Using session type: " + sessionType); 306 } 307 308 SessionConfiguration sessionConfiguration = new SessionConfiguration(sessionType, 309 outputList, new CameraExtensionUtils.HandlerExecutor(mHandler), 310 new SessionStateHandler()); 311 312 if ((sessionConfig.sessionParameter != null) && 313 (!sessionConfig.sessionParameter.isEmpty())) { 314 CaptureRequest.Builder requestBuilder = mCameraDevice.createCaptureRequest( 315 sessionConfig.sessionTemplateId); 316 CaptureRequest sessionRequest = requestBuilder.build(); 317 CameraMetadataNative.update(sessionRequest.getNativeMetadata(), 318 sessionConfig.sessionParameter); 319 sessionConfiguration.setSessionParameters(sessionRequest); 320 } 321 322 mCameraDevice.createCaptureSession(sessionConfiguration); 323 } 324 initializeParcelable(CaptureResult result)325 private static ParcelCaptureResult initializeParcelable(CaptureResult result) { 326 ParcelCaptureResult ret = new ParcelCaptureResult(); 327 ret.cameraId = result.getCameraId(); 328 ret.results = result.getNativeMetadata(); 329 ret.parent = result.getRequest(); 330 ret.sequenceId = result.getSequenceId(); 331 ret.frameNumber = result.getFrameNumber(); 332 333 return ret; 334 } 335 initializeParcelable(TotalCaptureResult totalResult)336 private static ParcelTotalCaptureResult initializeParcelable(TotalCaptureResult totalResult) { 337 ParcelTotalCaptureResult ret = new ParcelTotalCaptureResult(); 338 ret.logicalCameraId = totalResult.getCameraId(); 339 ret.results = totalResult.getNativeMetadata(); 340 ret.parent = totalResult.getRequest(); 341 ret.sequenceId = totalResult.getSequenceId(); 342 ret.frameNumber = totalResult.getFrameNumber(); 343 ret.sessionId = totalResult.getSessionId(); 344 ret.partials = new ArrayList<>(totalResult.getPartialResults().size()); 345 for (CaptureResult partial : totalResult.getPartialResults()) { 346 ret.partials.add(initializeParcelable(partial)); 347 } 348 Map<String, TotalCaptureResult> physicalResults = 349 totalResult.getPhysicalCameraTotalResults(); 350 ret.physicalResult = new ArrayList<>(physicalResults.size()); 351 for (TotalCaptureResult physicalResult : physicalResults.values()) { 352 ret.physicalResult.add(new PhysicalCaptureResultInfo(physicalResult.getCameraId(), 353 physicalResult.getNativeMetadata())); 354 } 355 356 return ret; 357 } 358 initializeParcelable(Surface s)359 private static OutputSurface initializeParcelable(Surface s) { 360 OutputSurface ret = new OutputSurface(); 361 if (s != null) { 362 ret.surface = s; 363 ret.size = new android.hardware.camera2.extension.Size(); 364 Size surfaceSize = SurfaceUtils.getSurfaceSize(s); 365 ret.size.width = surfaceSize.getWidth(); 366 ret.size.height = surfaceSize.getHeight(); 367 ret.imageFormat = SurfaceUtils.getSurfaceFormat(s); 368 } else { 369 ret.surface = null; 370 ret.size = new android.hardware.camera2.extension.Size(); 371 ret.size.width = -1; 372 ret.size.height = -1; 373 ret.imageFormat = ImageFormat.UNKNOWN; 374 } 375 376 return ret; 377 } 378 379 @Override getDevice()380 public @NonNull CameraDevice getDevice() { 381 synchronized (mInterfaceLock) { 382 return mCameraDevice; 383 } 384 } 385 386 @Override getRealtimeStillCaptureLatency()387 public StillCaptureLatency getRealtimeStillCaptureLatency() throws CameraAccessException { 388 synchronized (mInterfaceLock) { 389 if (!mInitialized) { 390 throw new IllegalStateException("Uninitialized component"); 391 } 392 393 try { 394 LatencyPair latency = mSessionProcessor.getRealtimeCaptureLatency(); 395 if (latency != null) { 396 return new StillCaptureLatency(latency.first, latency.second); 397 } 398 399 return null; 400 } catch (RemoteException e) { 401 Log.e(TAG, "Failed to query realtime latency! Extension service does not " 402 + "respond"); 403 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); 404 } 405 } 406 } 407 408 @Override setRepeatingRequest(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)409 public int setRepeatingRequest(@NonNull CaptureRequest request, @NonNull Executor executor, 410 @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { 411 int seqId = -1; 412 synchronized (mInterfaceLock) { 413 if (!mInitialized) { 414 throw new IllegalStateException("Uninitialized component"); 415 } 416 417 if (mClientRepeatingRequestSurface == null) { 418 throw new IllegalArgumentException("No registered preview surface"); 419 } 420 421 if (!request.containsTarget(mClientRepeatingRequestSurface) || 422 (request.getTargets().size() != 1)) { 423 throw new IllegalArgumentException("Invalid repeating request output target!"); 424 } 425 426 try { 427 mSessionProcessor.setParameters(request); 428 429 seqId = mSessionProcessor.startRepeating(new RequestCallbackHandler(request, 430 executor, listener, mCameraDevice.getId())); 431 } catch (RemoteException e) { 432 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, 433 "Failed to enable repeating request, extension service failed to respond!"); 434 } 435 } 436 437 return seqId; 438 } 439 440 @Override capture(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)441 public int capture(@NonNull CaptureRequest request, 442 @NonNull Executor executor, 443 @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { 444 int seqId = -1; 445 synchronized (mInterfaceLock) { 446 if (!mInitialized) { 447 throw new IllegalStateException("Uninitialized component"); 448 } 449 450 validateCaptureRequestTargets(request); 451 452 if ((mClientCaptureSurface != null) && request.containsTarget(mClientCaptureSurface)) { 453 try { 454 boolean isPostviewRequested = request.containsTarget(mClientPostviewSurface); 455 mSessionProcessor.setParameters(request); 456 457 seqId = mSessionProcessor.startCapture(new RequestCallbackHandler(request, 458 executor, listener, mCameraDevice.getId()), isPostviewRequested); 459 } catch (RemoteException e) { 460 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, "Failed " + 461 " to submit capture request, extension service failed to respond!"); 462 } 463 } else if ((mClientRepeatingRequestSurface != null) && 464 request.containsTarget(mClientRepeatingRequestSurface)) { 465 try { 466 seqId = mSessionProcessor.startTrigger(request, new RequestCallbackHandler( 467 request, executor, listener, mCameraDevice.getId())); 468 } catch (RemoteException e) { 469 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, "Failed " + 470 " to submit trigger request, extension service failed to respond!"); 471 } 472 } else { 473 throw new IllegalArgumentException("Invalid single capture output target!"); 474 } 475 } 476 477 return seqId; 478 } 479 validateCaptureRequestTargets(@onNull CaptureRequest request)480 private void validateCaptureRequestTargets(@NonNull CaptureRequest request) { 481 if (request.getTargets().size() == 1) { 482 boolean containsCaptureTarget = 483 mClientCaptureSurface != null && request.containsTarget(mClientCaptureSurface); 484 boolean containsRepeatingTarget = 485 mClientRepeatingRequestSurface != null && 486 request.containsTarget(mClientRepeatingRequestSurface); 487 488 if (!containsCaptureTarget && !containsRepeatingTarget) { 489 throw new IllegalArgumentException("Target output combination requested is " + 490 "not supported!"); 491 } 492 } 493 494 if ((request.getTargets().size() == 2) && 495 (!request.getTargets().containsAll(Arrays.asList(mClientCaptureSurface, 496 mClientPostviewSurface)))) { 497 throw new IllegalArgumentException("Target output combination requested is " + 498 "not supported!"); 499 } 500 501 if (request.getTargets().size() > 2) { 502 throw new IllegalArgumentException("Target output combination requested is " + 503 "not supported!"); 504 } 505 } 506 507 @Override stopRepeating()508 public void stopRepeating() throws CameraAccessException { 509 synchronized (mInterfaceLock) { 510 if (!mInitialized) { 511 throw new IllegalStateException("Uninitialized component"); 512 } 513 514 mCaptureSession.stopRepeating(); 515 516 try { 517 mSessionProcessor.stopRepeating(); 518 } catch (RemoteException e) { 519 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, 520 "Failed to notify about the end of repeating request, extension service" 521 + " failed to respond!"); 522 } 523 } 524 } 525 526 @Override close()527 public void close() throws CameraAccessException { 528 synchronized (mInterfaceLock) { 529 if (mInitialized) { 530 try { 531 try { 532 mCaptureSession.stopRepeating(); 533 } catch (IllegalStateException e) { 534 // OK: already be closed, nothing else to do 535 } 536 mSessionProcessor.stopRepeating(); 537 mSessionProcessor.onCaptureSessionEnd(); 538 mSessionClosed = true; 539 } catch (RemoteException e) { 540 Log.e(TAG, "Failed to stop the repeating request or end the session," 541 + " , extension service does not respond!") ; 542 } 543 // Commit stats before closing the capture session 544 mStatsAggregator.commit(/*isFinal*/true); 545 mCaptureSession.close(); 546 } 547 } 548 } 549 550 /** 551 * Called by {@link CameraDeviceImpl} right before the capture session is closed, and before it 552 * calls {@link #release} 553 */ commitStats()554 public void commitStats() { 555 synchronized (mInterfaceLock) { 556 if (mInitialized) { 557 // Only commit stats if a capture session was initialized 558 mStatsAggregator.commit(/*isFinal*/true); 559 } 560 } 561 } 562 release(boolean skipCloseNotification)563 public void release(boolean skipCloseNotification) { 564 boolean notifyClose = false; 565 566 synchronized (mInterfaceLock) { 567 mHandlerThread.quitSafely(); 568 569 if (mSessionProcessor != null) { 570 try { 571 if (!mSessionClosed) { 572 mSessionProcessor.onCaptureSessionEnd(); 573 } 574 mSessionProcessor.deInitSession(mToken); 575 } catch (RemoteException e) { 576 Log.e(TAG, "Failed to de-initialize session processor, extension service" 577 + " does not respond!") ; 578 } 579 mSessionProcessor = null; 580 } 581 582 583 if (mToken != null) { 584 if (mInitialized || (mCaptureSession != null)) { 585 notifyClose = true; 586 CameraExtensionCharacteristics.releaseSession(); 587 } 588 CameraExtensionCharacteristics.unregisterClient(mContext, mToken); 589 } 590 mInitialized = false; 591 mToken = null; 592 593 for (ImageReader reader : mReaderMap.values()) { 594 reader.close(); 595 } 596 mReaderMap.clear(); 597 598 mClientRepeatingRequestSurface = null; 599 mClientCaptureSurface = null; 600 mCaptureSession = null; 601 mRequestProcessor = null; 602 mCameraDevice = null; 603 mAdvancedExtender = null; 604 } 605 606 if (notifyClose && !skipCloseNotification) { 607 final long ident = Binder.clearCallingIdentity(); 608 try { 609 mExecutor.execute(() -> mCallbacks.onClosed( 610 CameraAdvancedExtensionSessionImpl.this)); 611 } finally { 612 Binder.restoreCallingIdentity(ident); 613 } 614 } 615 } 616 notifyConfigurationFailure()617 private void notifyConfigurationFailure() { 618 synchronized (mInterfaceLock) { 619 if (mInitialized) { 620 return; 621 } 622 } 623 624 release(true /*skipCloseNotification*/); 625 626 final long ident = Binder.clearCallingIdentity(); 627 try { 628 mExecutor.execute( 629 () -> mCallbacks.onConfigureFailed( 630 CameraAdvancedExtensionSessionImpl.this)); 631 } finally { 632 Binder.restoreCallingIdentity(ident); 633 } 634 } 635 636 private class SessionStateHandler extends 637 android.hardware.camera2.CameraCaptureSession.StateCallback { 638 @Override onClosed(@onNull CameraCaptureSession session)639 public void onClosed(@NonNull CameraCaptureSession session) { 640 release(false /*skipCloseNotification*/); 641 } 642 643 @Override onConfigureFailed(@onNull CameraCaptureSession session)644 public void onConfigureFailed(@NonNull CameraCaptureSession session) { 645 notifyConfigurationFailure(); 646 } 647 648 @Override onConfigured(@onNull CameraCaptureSession session)649 public void onConfigured(@NonNull CameraCaptureSession session) { 650 synchronized (mInterfaceLock) { 651 mCaptureSession = session; 652 // Commit basic stats as soon as the capture session is created 653 mStatsAggregator.commit(/*isFinal*/false); 654 } 655 656 try { 657 CameraExtensionCharacteristics.initializeSession(mInitializeHandler); 658 } catch (RemoteException e) { 659 Log.e(TAG, "Failed to initialize session! Extension service does" 660 + " not respond!"); 661 notifyConfigurationFailure(); 662 } 663 } 664 } 665 666 private class InitializeSessionHandler extends IInitializeSessionCallback.Stub { 667 @Override onSuccess()668 public void onSuccess() { 669 mHandler.post(new Runnable() { 670 @Override 671 public void run() { 672 boolean status = true; 673 synchronized (mInterfaceLock) { 674 try { 675 if (mSessionProcessor != null) { 676 mInitialized = true; 677 mSessionProcessor.onCaptureSessionStart(mRequestProcessor); 678 } else { 679 Log.v(TAG, "Failed to start capture session, session " + 680 " released before extension start!"); 681 status = false; 682 } 683 } catch (RemoteException e) { 684 Log.e(TAG, "Failed to start capture session," 685 + " extension service does not respond!"); 686 status = false; 687 mInitialized = false; 688 } 689 } 690 691 if (status) { 692 final long ident = Binder.clearCallingIdentity(); 693 try { 694 mExecutor.execute(() -> mCallbacks.onConfigured( 695 CameraAdvancedExtensionSessionImpl.this)); 696 } finally { 697 Binder.restoreCallingIdentity(ident); 698 } 699 } else { 700 onFailure(); 701 } 702 } 703 }); 704 } 705 706 @Override onFailure()707 public void onFailure() { 708 mHandler.post(new Runnable() { 709 @Override 710 public void run() { 711 mCaptureSession.close(); 712 713 Log.e(TAG, "Failed to initialize proxy service session!" 714 + " This can happen when trying to configure multiple " 715 + "concurrent extension sessions!"); 716 notifyConfigurationFailure(); 717 } 718 }); 719 } 720 } 721 722 private final class RequestCallbackHandler extends ICaptureCallback.Stub { 723 private final CaptureRequest mClientRequest; 724 private final Executor mClientExecutor; 725 private final ExtensionCaptureCallback mClientCallbacks; 726 private final String mCameraId; 727 RequestCallbackHandler(@onNull CaptureRequest clientRequest, @NonNull Executor clientExecutor, @NonNull ExtensionCaptureCallback clientCallbacks, @NonNull String cameraId)728 private RequestCallbackHandler(@NonNull CaptureRequest clientRequest, 729 @NonNull Executor clientExecutor, 730 @NonNull ExtensionCaptureCallback clientCallbacks, 731 @NonNull String cameraId) { 732 mClientRequest = clientRequest; 733 mClientExecutor = clientExecutor; 734 mClientCallbacks = clientCallbacks; 735 mCameraId = cameraId; 736 } 737 738 @Override onCaptureStarted(int captureSequenceId, long timestamp)739 public void onCaptureStarted(int captureSequenceId, long timestamp) { 740 final long ident = Binder.clearCallingIdentity(); 741 try { 742 mClientExecutor.execute( 743 () -> mClientCallbacks.onCaptureStarted( 744 CameraAdvancedExtensionSessionImpl.this, mClientRequest, 745 timestamp)); 746 } finally { 747 Binder.restoreCallingIdentity(ident); 748 } 749 } 750 751 @Override onCaptureProcessStarted(int captureSequenceId)752 public void onCaptureProcessStarted(int captureSequenceId) { 753 final long ident = Binder.clearCallingIdentity(); 754 try { 755 mClientExecutor.execute( 756 () -> mClientCallbacks.onCaptureProcessStarted( 757 CameraAdvancedExtensionSessionImpl.this, mClientRequest)); 758 } finally { 759 Binder.restoreCallingIdentity(ident); 760 } 761 } 762 763 @Override onCaptureFailed(int captureSequenceId)764 public void onCaptureFailed(int captureSequenceId) { 765 final long ident = Binder.clearCallingIdentity(); 766 try { 767 mClientExecutor.execute( 768 () -> mClientCallbacks.onCaptureFailed( 769 CameraAdvancedExtensionSessionImpl.this, mClientRequest)); 770 } finally { 771 Binder.restoreCallingIdentity(ident); 772 } 773 } 774 775 @Override onCaptureSequenceCompleted(int captureSequenceId)776 public void onCaptureSequenceCompleted(int captureSequenceId) { 777 final long ident = Binder.clearCallingIdentity(); 778 try { 779 mClientExecutor.execute( 780 () -> mClientCallbacks.onCaptureSequenceCompleted( 781 CameraAdvancedExtensionSessionImpl.this, captureSequenceId)); 782 } finally { 783 Binder.restoreCallingIdentity(ident); 784 } 785 } 786 787 @Override onCaptureSequenceAborted(int captureSequenceId)788 public void onCaptureSequenceAborted(int captureSequenceId) { 789 final long ident = Binder.clearCallingIdentity(); 790 try { 791 mClientExecutor.execute( 792 () -> mClientCallbacks.onCaptureSequenceAborted( 793 CameraAdvancedExtensionSessionImpl.this, captureSequenceId)); 794 } finally { 795 Binder.restoreCallingIdentity(ident); 796 } 797 } 798 799 @Override onCaptureCompleted(long timestamp, int requestId, CameraMetadataNative result)800 public void onCaptureCompleted(long timestamp, int requestId, CameraMetadataNative result) { 801 if (result == null) { 802 Log.e(TAG,"Invalid capture result!"); 803 return; 804 } 805 806 result.set(CaptureResult.SENSOR_TIMESTAMP, timestamp); 807 TotalCaptureResult totalResult = new TotalCaptureResult(mCameraId, result, 808 mClientRequest, requestId, timestamp, new ArrayList<>(), mSessionId, 809 new PhysicalCaptureResultInfo[0]); 810 final long ident = Binder.clearCallingIdentity(); 811 try { 812 mExecutor.execute( 813 () -> mClientCallbacks.onCaptureResultAvailable( 814 CameraAdvancedExtensionSessionImpl.this, mClientRequest, 815 totalResult)); 816 } finally { 817 Binder.restoreCallingIdentity(ident); 818 } 819 } 820 821 @Override onCaptureProcessProgressed(int progress)822 public void onCaptureProcessProgressed(int progress) { 823 final long ident = Binder.clearCallingIdentity(); 824 try { 825 mExecutor.execute( 826 () -> mClientCallbacks.onCaptureProcessProgressed( 827 CameraAdvancedExtensionSessionImpl.this, mClientRequest, 828 progress)); 829 } finally { 830 Binder.restoreCallingIdentity(ident); 831 } 832 } 833 } 834 835 private final class CaptureCallbackHandler extends CameraCaptureSession.CaptureCallback { 836 private final IRequestCallback mCallback; 837 CaptureCallbackHandler(IRequestCallback callback)838 public CaptureCallbackHandler(IRequestCallback callback) { 839 mCallback = callback; 840 } 841 842 @Override onCaptureBufferLost(CameraCaptureSession session, CaptureRequest request, Surface target, long frameNumber)843 public void onCaptureBufferLost(CameraCaptureSession session, CaptureRequest request, 844 Surface target, long frameNumber) { 845 try { 846 if (request.getTag() instanceof Integer) { 847 Integer requestId = (Integer) request.getTag(); 848 mCallback.onCaptureBufferLost(requestId, frameNumber, 849 mCameraConfigMap.get(target).outputId.id); 850 } else { 851 Log.e(TAG, "Invalid capture request tag!"); 852 } 853 } catch (RemoteException e) { 854 Log.e(TAG, "Failed to notify lost capture buffer, extension service doesn't" 855 + " respond!"); 856 } 857 } 858 859 @Override onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result)860 public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, 861 TotalCaptureResult result) { 862 try { 863 if (request.getTag() instanceof Integer) { 864 Integer requestId = (Integer) request.getTag(); 865 mCallback.onCaptureCompleted(requestId, initializeParcelable(result)); 866 } else { 867 Log.e(TAG, "Invalid capture request tag!"); 868 } 869 } catch (RemoteException e) { 870 Log.e(TAG, "Failed to notify capture result, extension service doesn't" 871 + " respond!"); 872 } 873 } 874 875 @Override onCaptureFailed(CameraCaptureSession session, CaptureRequest request, CaptureFailure failure)876 public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request, 877 CaptureFailure failure) { 878 try { 879 if (request.getTag() instanceof Integer) { 880 Integer requestId = (Integer) request.getTag(); 881 android.hardware.camera2.extension.CaptureFailure captureFailure = 882 new android.hardware.camera2.extension.CaptureFailure(); 883 captureFailure.request = request; 884 captureFailure.reason = failure.getReason(); 885 captureFailure.errorPhysicalCameraId = failure.getPhysicalCameraId(); 886 captureFailure.frameNumber = failure.getFrameNumber(); 887 captureFailure.sequenceId = failure.getSequenceId(); 888 captureFailure.dropped = !failure.wasImageCaptured(); 889 mCallback.onCaptureFailed(requestId, captureFailure); 890 } else { 891 Log.e(TAG, "Invalid capture request tag!"); 892 } 893 } catch (RemoteException e) { 894 Log.e(TAG, "Failed to notify capture failure, extension service doesn't" 895 + " respond!"); 896 } 897 } 898 899 @Override onCaptureProgressed(CameraCaptureSession session, CaptureRequest request, CaptureResult partialResult)900 public void onCaptureProgressed(CameraCaptureSession session, CaptureRequest request, 901 CaptureResult partialResult) { 902 try { 903 if (request.getTag() instanceof Integer) { 904 Integer requestId = (Integer) request.getTag(); 905 mCallback.onCaptureProgressed(requestId, initializeParcelable(partialResult)); 906 } else { 907 Log.e(TAG, "Invalid capture request tag!"); 908 } 909 } catch (RemoteException e) { 910 Log.e(TAG, "Failed to notify capture partial result, extension service doesn't" 911 + " respond!"); 912 } 913 } 914 915 @Override onCaptureSequenceAborted(CameraCaptureSession session, int sequenceId)916 public void onCaptureSequenceAborted(CameraCaptureSession session, int sequenceId) { 917 try { 918 mCallback.onCaptureSequenceAborted(sequenceId); 919 } catch (RemoteException e) { 920 Log.e(TAG, "Failed to notify aborted sequence, extension service doesn't" 921 + " respond!"); 922 } 923 } 924 925 @Override onCaptureSequenceCompleted(CameraCaptureSession session, int sequenceId, long frameNumber)926 public void onCaptureSequenceCompleted(CameraCaptureSession session, int sequenceId, 927 long frameNumber) { 928 try { 929 mCallback.onCaptureSequenceCompleted(sequenceId, frameNumber); 930 } catch (RemoteException e) { 931 Log.e(TAG, "Failed to notify sequence complete, extension service doesn't" 932 + " respond!"); 933 } 934 } 935 936 @Override onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber)937 public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, 938 long timestamp, long frameNumber) { 939 try { 940 if (request.getTag() instanceof Integer) { 941 Integer requestId = (Integer) request.getTag(); 942 mCallback.onCaptureStarted(requestId, frameNumber, timestamp); 943 } else { 944 Log.e(TAG, "Invalid capture request tag!"); 945 } 946 } catch (RemoteException e) { 947 Log.e(TAG, "Failed to notify capture started, extension service doesn't" 948 + " respond!"); 949 } 950 } 951 } 952 953 private static final class ImageReaderHandler implements ImageReader.OnImageAvailableListener { 954 private final OutputConfigId mOutputConfigId; 955 private final IImageProcessorImpl mIImageProcessor; 956 private final String mPhysicalCameraId; 957 ImageReaderHandler(int outputConfigId, IImageProcessorImpl iImageProcessor, String physicalCameraId)958 private ImageReaderHandler(int outputConfigId, 959 IImageProcessorImpl iImageProcessor, String physicalCameraId) { 960 mOutputConfigId = new OutputConfigId(); 961 mOutputConfigId.id = outputConfigId; 962 mIImageProcessor = iImageProcessor; 963 mPhysicalCameraId = physicalCameraId; 964 } 965 966 @Override onImageAvailable(ImageReader reader)967 public void onImageAvailable(ImageReader reader) { 968 if (mIImageProcessor == null) { 969 return; 970 } 971 972 Image img; 973 try { 974 img = reader.acquireNextImage(); 975 } catch (IllegalStateException e) { 976 Log.e(TAG, "Failed to acquire image, too many images pending!"); 977 return; 978 } 979 if (img == null) { 980 Log.e(TAG, "Invalid image!"); 981 return; 982 } 983 984 try { 985 reader.detachImage(img); 986 } catch(Exception e) { 987 Log.e(TAG, "Failed to detach image"); 988 img.close(); 989 return; 990 } 991 992 ParcelImage parcelImage = new ParcelImage(); 993 parcelImage.buffer = img.getHardwareBuffer(); 994 try { 995 SyncFence fd = img.getFence(); 996 if (fd.isValid()) { 997 parcelImage.fence = fd.getFdDup(); 998 } 999 } catch (IOException e) { 1000 Log.e(TAG, "Failed to parcel buffer fence!"); 1001 } 1002 parcelImage.width = img.getWidth(); 1003 parcelImage.height = img.getHeight(); 1004 parcelImage.format = img.getFormat(); 1005 parcelImage.timestamp = img.getTimestamp(); 1006 parcelImage.transform = img.getTransform(); 1007 parcelImage.scalingMode = img.getScalingMode(); 1008 parcelImage.planeCount = img.getPlaneCount(); 1009 parcelImage.crop = img.getCropRect(); 1010 1011 try { 1012 mIImageProcessor.onNextImageAvailable(mOutputConfigId, parcelImage, 1013 mPhysicalCameraId); 1014 } catch (RemoteException e) { 1015 Log.e(TAG, "Failed to propagate image buffer on output surface id: " + 1016 mOutputConfigId + " extension service does not respond!"); 1017 } finally { 1018 parcelImage.buffer.close(); 1019 img.close(); 1020 } 1021 } 1022 } 1023 1024 private final class RequestProcessor extends IRequestProcessorImpl.Stub { 1025 @Override setImageProcessor(OutputConfigId outputConfigId, IImageProcessorImpl imageProcessor)1026 public void setImageProcessor(OutputConfigId outputConfigId, 1027 IImageProcessorImpl imageProcessor) { 1028 synchronized (mInterfaceLock) { 1029 if (mReaderMap.containsKey(outputConfigId.id)) { 1030 ImageReader reader = mReaderMap.get(outputConfigId.id); 1031 String physicalCameraId = null; 1032 if (mCameraConfigMap.containsKey(reader.getSurface())) { 1033 physicalCameraId = 1034 mCameraConfigMap.get(reader.getSurface()).physicalCameraId; 1035 reader.setOnImageAvailableListener(new ImageReaderHandler(outputConfigId.id, 1036 imageProcessor, physicalCameraId), mHandler); 1037 } else { 1038 Log.e(TAG, "Camera output configuration for ImageReader with " + 1039 " config Id " + outputConfigId.id + " not found!"); 1040 } 1041 } else { 1042 Log.e(TAG, "ImageReader with output config id: " + outputConfigId.id + 1043 " not found!"); 1044 } 1045 } 1046 } 1047 1048 @Override submit(Request request, IRequestCallback callback)1049 public int submit(Request request, IRequestCallback callback) { 1050 ArrayList<Request> captureList = new ArrayList<>(); 1051 captureList.add(request); 1052 return submitBurst(captureList, callback); 1053 } 1054 1055 @Override submitBurst(List<Request> requests, IRequestCallback callback)1056 public int submitBurst(List<Request> requests, IRequestCallback callback) { 1057 int seqId = -1; 1058 try { 1059 synchronized (mInterfaceLock) { 1060 if (!mInitialized) { 1061 return seqId; 1062 } 1063 1064 CaptureCallbackHandler captureCallback = new CaptureCallbackHandler(callback); 1065 ArrayList<CaptureRequest> captureRequests = new ArrayList<>(); 1066 for (Request request : requests) { 1067 captureRequests.add(initializeCaptureRequest(mCameraDevice, request, 1068 mCameraConfigMap)); 1069 } 1070 seqId = mCaptureSession.captureBurstRequests(captureRequests, 1071 new CameraExtensionUtils.HandlerExecutor(mHandler), captureCallback); 1072 } 1073 } catch (CameraAccessException e) { 1074 Log.e(TAG, "Failed to submit capture requests!"); 1075 } catch (IllegalStateException e) { 1076 Log.e(TAG, "Capture session closed!"); 1077 } 1078 1079 return seqId; 1080 } 1081 1082 @Override setRepeating(Request request, IRequestCallback callback)1083 public int setRepeating(Request request, IRequestCallback callback) { 1084 int seqId = -1; 1085 try { 1086 synchronized (mInterfaceLock) { 1087 if (!mInitialized) { 1088 return seqId; 1089 } 1090 1091 CaptureRequest repeatingRequest = initializeCaptureRequest(mCameraDevice, 1092 request, mCameraConfigMap); 1093 CaptureCallbackHandler captureCallback = new CaptureCallbackHandler(callback); 1094 seqId = mCaptureSession.setSingleRepeatingRequest(repeatingRequest, 1095 new CameraExtensionUtils.HandlerExecutor(mHandler), captureCallback); 1096 } 1097 } catch (CameraAccessException e) { 1098 Log.e(TAG, "Failed to enable repeating request!"); 1099 } catch (IllegalStateException e) { 1100 Log.e(TAG, "Capture session closed!"); 1101 } 1102 1103 return seqId; 1104 } 1105 1106 @Override abortCaptures()1107 public void abortCaptures() { 1108 try { 1109 synchronized (mInterfaceLock) { 1110 if (!mInitialized) { 1111 return; 1112 } 1113 1114 mCaptureSession.abortCaptures(); 1115 } 1116 } catch (CameraAccessException e) { 1117 Log.e(TAG, "Failed during capture abort!"); 1118 } catch (IllegalStateException e) { 1119 Log.e(TAG, "Capture session closed!"); 1120 } 1121 } 1122 1123 @Override stopRepeating()1124 public void stopRepeating() { 1125 try { 1126 synchronized (mInterfaceLock) { 1127 if (!mInitialized) { 1128 return; 1129 } 1130 1131 mCaptureSession.stopRepeating(); 1132 } 1133 } catch (CameraAccessException e) { 1134 Log.e(TAG, "Failed during repeating capture stop!"); 1135 } catch (IllegalStateException e) { 1136 Log.e(TAG, "Capture session closed!"); 1137 } 1138 } 1139 } 1140 initializeCaptureRequest(CameraDevice cameraDevice, Request request, HashMap<Surface, CameraOutputConfig> surfaceIdMap)1141 private static CaptureRequest initializeCaptureRequest(CameraDevice cameraDevice, 1142 Request request, HashMap<Surface, CameraOutputConfig> surfaceIdMap) 1143 throws CameraAccessException { 1144 CaptureRequest.Builder builder = cameraDevice.createCaptureRequest(request.templateId); 1145 for (OutputConfigId configId : request.targetOutputConfigIds) { 1146 boolean found = false; 1147 for (Map.Entry<Surface, CameraOutputConfig> entry : surfaceIdMap.entrySet()) { 1148 if (entry.getValue().outputId.id == configId.id) { 1149 builder.addTarget(entry.getKey()); 1150 found = true; 1151 break; 1152 } 1153 } 1154 1155 if (!found) { 1156 Log.e(TAG, "Surface with output id: " + configId.id + 1157 " not found among registered camera outputs!"); 1158 } 1159 } 1160 1161 builder.setTag(request.requestId); 1162 CaptureRequest ret = builder.build(); 1163 CameraMetadataNative.update(ret.getNativeMetadata(), request.parameters); 1164 return ret; 1165 } 1166 initializeSurfrace(CameraOutputConfig output)1167 private Surface initializeSurfrace(CameraOutputConfig output) { 1168 switch(output.type) { 1169 case CameraOutputConfig.TYPE_SURFACE: 1170 if (output.surface == null) { 1171 Log.w(TAG, "Unsupported client output id: " + output.outputId.id + 1172 ", skipping!"); 1173 return null; 1174 } 1175 return output.surface; 1176 case CameraOutputConfig.TYPE_IMAGEREADER: 1177 if ((output.imageFormat == ImageFormat.UNKNOWN) || (output.size.width <= 0) || 1178 (output.size.height <= 0)) { 1179 Log.w(TAG, "Unsupported client output id: " + output.outputId.id + 1180 ", skipping!"); 1181 return null; 1182 } 1183 ImageReader reader = ImageReader.newInstance(output.size.width, 1184 output.size.height, output.imageFormat, output.capacity); 1185 mReaderMap.put(output.outputId.id, reader); 1186 return reader.getSurface(); 1187 case CameraOutputConfig.TYPE_MULTIRES_IMAGEREADER: 1188 // Support for multi-resolution outputs to be added in future releases 1189 default: 1190 throw new IllegalArgumentException("Unsupported output config type: " + 1191 output.type); 1192 } 1193 } 1194 } 1195