1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_QUALITY; 20 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_ROTATION; 21 22 import android.annotation.NonNull; 23 import android.graphics.ImageFormat; 24 import android.hardware.camera2.CaptureResult; 25 import android.hardware.camera2.extension.CaptureBundle; 26 import android.hardware.camera2.extension.ICaptureProcessorImpl; 27 import android.media.Image; 28 import android.media.Image.Plane; 29 import android.media.ImageReader; 30 import android.media.ImageWriter; 31 import android.os.Handler; 32 import android.os.HandlerThread; 33 import android.os.IBinder; 34 import android.os.RemoteException; 35 import android.util.Log; 36 import android.view.Surface; 37 38 import java.nio.ByteBuffer; 39 import java.util.HashSet; 40 import java.util.Iterator; 41 import java.util.List; 42 import java.util.concurrent.ConcurrentLinkedQueue; 43 44 // Jpeg compress input YUV and queue back in the client target surface. 45 public class CameraExtensionJpegProcessor implements ICaptureProcessorImpl { 46 public final static String TAG = "CameraExtensionJpeg"; 47 private final static int JPEG_QUEUE_SIZE = 1; 48 49 private final Handler mHandler; 50 private final HandlerThread mHandlerThread; 51 private final ICaptureProcessorImpl mProcessor; 52 53 private ImageReader mYuvReader = null; 54 private android.hardware.camera2.extension.Size mResolution = null; 55 private int mFormat = -1; 56 private Surface mOutputSurface = null; 57 private ImageWriter mOutputWriter = null; 58 59 private static final class JpegParameters { 60 public HashSet<Long> mTimeStamps = new HashSet<>(); 61 public int mRotation = JPEG_DEFAULT_ROTATION; // CW multiple of 90 degrees 62 public int mQuality = JPEG_DEFAULT_QUALITY; // [0..100] 63 } 64 65 private ConcurrentLinkedQueue<JpegParameters> mJpegParameters = new ConcurrentLinkedQueue<>(); 66 CameraExtensionJpegProcessor(@onNull ICaptureProcessorImpl processor)67 public CameraExtensionJpegProcessor(@NonNull ICaptureProcessorImpl processor) { 68 mProcessor = processor; 69 mHandlerThread = new HandlerThread(TAG); 70 mHandlerThread.start(); 71 mHandler = new Handler(mHandlerThread.getLooper()); 72 } 73 close()74 public void close() { 75 mHandlerThread.quitSafely(); 76 77 if (mOutputWriter != null) { 78 mOutputWriter.close(); 79 mOutputWriter = null; 80 } 81 82 if (mYuvReader != null) { 83 mYuvReader.close(); 84 mYuvReader = null; 85 } 86 } 87 getJpegParameters(List<CaptureBundle> captureBundles)88 private static JpegParameters getJpegParameters(List<CaptureBundle> captureBundles) { 89 JpegParameters ret = new JpegParameters(); 90 if (!captureBundles.isEmpty()) { 91 // The quality and orientation settings must be equal for requests in a burst 92 93 Byte jpegQuality = captureBundles.get(0).captureResult.get(CaptureResult.JPEG_QUALITY); 94 if (jpegQuality != null) { 95 ret.mQuality = jpegQuality; 96 } else { 97 Log.w(TAG, "No jpeg quality set, using default: " + JPEG_DEFAULT_QUALITY); 98 } 99 100 Integer orientation = captureBundles.get(0).captureResult.get( 101 CaptureResult.JPEG_ORIENTATION); 102 if (orientation != null) { 103 // The jpeg encoder expects CCW rotation, convert from CW 104 ret.mRotation = (360 - (orientation % 360)) / 90; 105 } else { 106 Log.w(TAG, "No jpeg rotation set, using default: " + JPEG_DEFAULT_ROTATION); 107 } 108 109 for (CaptureBundle bundle : captureBundles) { 110 Long timeStamp = bundle.captureResult.get(CaptureResult.SENSOR_TIMESTAMP); 111 if (timeStamp != null) { 112 ret.mTimeStamps.add(timeStamp); 113 } else { 114 Log.e(TAG, "Capture bundle without valid sensor timestamp!"); 115 } 116 } 117 } 118 119 return ret; 120 } 121 122 /** 123 * Compresses a YCbCr image to jpeg, applying a crop and rotation. 124 * <p> 125 * The input is defined as a set of 3 planes of 8-bit samples, one plane for 126 * each channel of Y, Cb, Cr.<br> 127 * The Y plane is assumed to have the same width and height of the entire 128 * image.<br> 129 * The Cb and Cr planes are assumed to be downsampled by a factor of 2, to 130 * have dimensions (floor(width / 2), floor(height / 2)).<br> 131 * Each plane is specified by a direct java.nio.ByteBuffer, a pixel-stride, 132 * and a row-stride. So, the sample at coordinate (x, y) can be retrieved 133 * from byteBuffer[x * pixel_stride + y * row_stride]. 134 * <p> 135 * The pre-compression transformation is applied as follows: 136 * <ol> 137 * <li>The image is cropped to the rectangle from (cropLeft, cropTop) to 138 * (cropRight - 1, cropBottom - 1). So, a cropping-rectangle of (0, 0) - 139 * (width, height) is a no-op.</li> 140 * <li>The rotation is applied counter-clockwise relative to the coordinate 141 * space of the image, so a CCW rotation will appear CW when the image is 142 * rendered in scanline order. Only rotations which are multiples of 143 * 90-degrees are suppored, so the parameter 'rot90' specifies which 144 * multiple of 90 to rotate the image.</li> 145 * </ol> 146 * 147 * @param width the width of the image to compress 148 * @param height the height of the image to compress 149 * @param yBuf the buffer containing the Y component of the image 150 * @param yPStride the stride between adjacent pixels in the same row in 151 * yBuf 152 * @param yRStride the stride between adjacent rows in yBuf 153 * @param cbBuf the buffer containing the Cb component of the image 154 * @param cbPStride the stride between adjacent pixels in the same row in 155 * cbBuf 156 * @param cbRStride the stride between adjacent rows in cbBuf 157 * @param crBuf the buffer containing the Cr component of the image 158 * @param crPStride the stride between adjacent pixels in the same row in 159 * crBuf 160 * @param crRStride the stride between adjacent rows in crBuf 161 * @param outBuf a direct java.nio.ByteBuffer to hold the compressed jpeg. 162 * This must have enough capacity to store the result, or an 163 * error code will be returned. 164 * @param outBufCapacity the capacity of outBuf 165 * @param quality the jpeg-quality (1-100) to use 166 * @param cropLeft left-edge of the bounds of the image to crop to before 167 * rotation 168 * @param cropTop top-edge of the bounds of the image to crop to before 169 * rotation 170 * @param cropRight right-edge of the bounds of the image to crop to before 171 * rotation 172 * @param cropBottom bottom-edge of the bounds of the image to crop to 173 * before rotation 174 * @param rot90 the multiple of 90 to rotate the image CCW (after cropping) 175 */ compressJpegFromYUV420pNative( int width, int height, ByteBuffer yBuf, int yPStride, int yRStride, ByteBuffer cbBuf, int cbPStride, int cbRStride, ByteBuffer crBuf, int crPStride, int crRStride, ByteBuffer outBuf, int outBufCapacity, int quality, int cropLeft, int cropTop, int cropRight, int cropBottom, int rot90)176 private static native int compressJpegFromYUV420pNative( 177 int width, int height, 178 ByteBuffer yBuf, int yPStride, int yRStride, 179 ByteBuffer cbBuf, int cbPStride, int cbRStride, 180 ByteBuffer crBuf, int crPStride, int crRStride, 181 ByteBuffer outBuf, int outBufCapacity, 182 int quality, 183 int cropLeft, int cropTop, int cropRight, int cropBottom, 184 int rot90); 185 process(List<CaptureBundle> captureBundle)186 public void process(List<CaptureBundle> captureBundle) throws RemoteException { 187 JpegParameters jpegParams = getJpegParameters(captureBundle); 188 try { 189 mJpegParameters.add(jpegParams); 190 mProcessor.process(captureBundle); 191 } catch (Exception e) { 192 mJpegParameters.remove(jpegParams); 193 throw e; 194 } 195 } 196 onOutputSurface(Surface surface, int format)197 public void onOutputSurface(Surface surface, int format) throws RemoteException { 198 if (format != ImageFormat.JPEG) { 199 Log.e(TAG, "Unsupported output format: " + format); 200 return; 201 } 202 mOutputSurface = surface; 203 initializePipeline(); 204 } 205 206 @Override onResolutionUpdate(android.hardware.camera2.extension.Size size)207 public void onResolutionUpdate(android.hardware.camera2.extension.Size size) 208 throws RemoteException { 209 mResolution = size; 210 initializePipeline(); 211 } 212 onImageFormatUpdate(int format)213 public void onImageFormatUpdate(int format) throws RemoteException { 214 if (format != ImageFormat.YUV_420_888) { 215 Log.e(TAG, "Unsupported input format: " + format); 216 return; 217 } 218 mFormat = format; 219 initializePipeline(); 220 } 221 initializePipeline()222 private void initializePipeline() throws RemoteException { 223 if ((mFormat != -1) && (mOutputSurface != null) && (mResolution != null) && 224 (mYuvReader == null)) { 225 // Jpeg/blobs are expected to be configured with (w*h)x1 226 mOutputWriter = ImageWriter.newInstance(mOutputSurface, 1 /*maxImages*/, 227 ImageFormat.JPEG, mResolution.width * mResolution.height, 1); 228 mYuvReader = ImageReader.newInstance(mResolution.width, mResolution.height, mFormat, 229 JPEG_QUEUE_SIZE); 230 mYuvReader.setOnImageAvailableListener(new YuvCallback(), mHandler); 231 mProcessor.onOutputSurface(mYuvReader.getSurface(), mFormat); 232 mProcessor.onResolutionUpdate(mResolution); 233 mProcessor.onImageFormatUpdate(mFormat); 234 } 235 } 236 237 @Override asBinder()238 public IBinder asBinder() { 239 throw new UnsupportedOperationException("Binder IPC not supported!"); 240 } 241 242 private class YuvCallback implements ImageReader.OnImageAvailableListener { 243 @Override onImageAvailable(ImageReader reader)244 public void onImageAvailable(ImageReader reader) { 245 Image yuvImage = null; 246 Image jpegImage = null; 247 try { 248 yuvImage = mYuvReader.acquireNextImage(); 249 jpegImage = mOutputWriter.dequeueInputImage(); 250 } catch (IllegalStateException e) { 251 if (yuvImage != null) { 252 yuvImage.close(); 253 } 254 if (jpegImage != null) { 255 jpegImage.close(); 256 } 257 Log.e(TAG, "Failed to acquire processed yuv image or jpeg image!"); 258 return; 259 } 260 261 ByteBuffer jpegBuffer = jpegImage.getPlanes()[0].getBuffer(); 262 jpegBuffer.clear(); 263 // Jpeg/blobs are expected to be configured with (w*h)x1 264 int jpegCapacity = jpegImage.getWidth(); 265 266 Plane lumaPlane = yuvImage.getPlanes()[0]; 267 Plane crPlane = yuvImage.getPlanes()[1]; 268 Plane cbPlane = yuvImage.getPlanes()[2]; 269 270 Iterator<JpegParameters> jpegIter = mJpegParameters.iterator(); 271 JpegParameters jpegParams = null; 272 while(jpegIter.hasNext()) { 273 JpegParameters currentParams = jpegIter.next(); 274 if (currentParams.mTimeStamps.contains(yuvImage.getTimestamp())) { 275 jpegParams = currentParams; 276 jpegIter.remove(); 277 break; 278 } 279 } 280 if (jpegParams == null) { 281 if (mJpegParameters.isEmpty()) { 282 Log.w(TAG, "Empty jpeg settings queue! Using default jpeg orientation" 283 + " and quality!"); 284 jpegParams = new JpegParameters(); 285 jpegParams.mRotation = JPEG_DEFAULT_ROTATION; 286 jpegParams.mQuality = JPEG_DEFAULT_QUALITY; 287 } else { 288 Log.w(TAG, "No jpeg settings found with matching timestamp for current" 289 + " processed input!"); 290 Log.w(TAG, "Using values from the top of the queue!"); 291 jpegParams = mJpegParameters.poll(); 292 } 293 } 294 295 compressJpegFromYUV420pNative( 296 yuvImage.getWidth(), yuvImage.getHeight(), 297 lumaPlane.getBuffer(), lumaPlane.getPixelStride(), lumaPlane.getRowStride(), 298 crPlane.getBuffer(), crPlane.getPixelStride(), crPlane.getRowStride(), 299 cbPlane.getBuffer(), cbPlane.getPixelStride(), cbPlane.getRowStride(), 300 jpegBuffer, jpegCapacity, jpegParams.mQuality, 301 0, 0, yuvImage.getWidth(), yuvImage.getHeight(), 302 jpegParams.mRotation); 303 yuvImage.close(); 304 305 try { 306 mOutputWriter.queueInputImage(jpegImage); 307 } catch (IllegalStateException e) { 308 Log.e(TAG, "Failed to queue encoded result!"); 309 } finally { 310 jpegImage.close(); 311 } 312 } 313 } 314 } 315