1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.Manifest; 20 import android.annotation.IntDef; 21 import android.annotation.NonNull; 22 import android.annotation.Nullable; 23 import android.annotation.RequiresPermission; 24 import android.annotation.SystemApi; 25 import android.compat.annotation.UnsupportedAppUsage; 26 import android.graphics.ImageFormat; 27 import android.graphics.Rect; 28 import android.graphics.SurfaceTexture; 29 import android.hardware.HardwareBuffer; 30 import android.media.MediaCodecInfo.CodecCapabilities; 31 import android.os.Build; 32 import android.os.Bundle; 33 import android.os.Handler; 34 import android.os.IHwBinder; 35 import android.os.Looper; 36 import android.os.Message; 37 import android.os.PersistableBundle; 38 import android.view.Surface; 39 40 import java.io.IOException; 41 import java.lang.annotation.Retention; 42 import java.lang.annotation.RetentionPolicy; 43 import java.nio.ByteBuffer; 44 import java.nio.ByteOrder; 45 import java.nio.ReadOnlyBufferException; 46 import java.util.ArrayList; 47 import java.util.Arrays; 48 import java.util.BitSet; 49 import java.util.Collections; 50 import java.util.HashMap; 51 import java.util.HashSet; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Objects; 55 import java.util.Set; 56 import java.util.concurrent.BlockingQueue; 57 import java.util.concurrent.LinkedBlockingQueue; 58 import java.util.concurrent.locks.Lock; 59 import java.util.concurrent.locks.ReentrantLock; 60 61 /** 62 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 63 It is part of the Android low-level multimedia support infrastructure (normally used together 64 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 65 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 66 <p> 67 <center> 68 <img src="../../../images/media/mediacodec_buffers.svg" style="width: 540px; height: 205px" 69 alt="MediaCodec buffer flow diagram"> 70 </center> 71 <p> 72 In broad terms, a codec processes input data to generate output data. It processes data 73 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 74 (or receive) an empty input buffer, fill it up with data and send it to the codec for 75 processing. The codec uses up the data and transforms it into one of its empty output buffers. 76 Finally, you request (or receive) a filled output buffer, consume its contents and release it 77 back to the codec. 78 79 <h3 id=qualityFloor><a name="qualityFloor">Minimum Quality Floor for Video Encoding</h3> 80 <p> 81 Beginning with {@link android.os.Build.VERSION_CODES#S}, Android's Video MediaCodecs enforce a 82 minimum quality floor. The intent is to eliminate poor quality video encodings. This quality 83 floor is applied when the codec is in Variable Bitrate (VBR) mode; it is not applied when 84 the codec is in Constant Bitrate (CBR) mode. The quality floor enforcement is also restricted 85 to a particular size range; this size range is currently for video resolutions 86 larger than 320x240 up through 1920x1080. 87 88 <p> 89 When this quality floor is in effect, the codec and supporting framework code will work to 90 ensure that the generated video is of at least a "fair" or "good" quality. The metric 91 used to choose these targets is the VMAF (Video Multi-method Assessment Function) with a 92 target score of 70 for selected test sequences. 93 94 <p> 95 The typical effect is that 96 some videos will generate a higher bitrate than originally configured. This will be most 97 notable for videos which were configured with very low bitrates; the codec will use a bitrate 98 that is determined to be more likely to generate an "fair" or "good" quality video. Another 99 situation is where a video includes very complicated content (lots of motion and detail); 100 in such configurations, the codec will use extra bitrate as needed to avoid losing all of 101 the content's finer detail. 102 103 <p> 104 This quality floor will not impact content captured at high bitrates (a high bitrate should 105 already provide the codec with sufficient capacity to encode all of the detail). 106 The quality floor does not operate on CBR encodings. 107 The quality floor currently does not operate on resolutions of 320x240 or lower, nor on 108 videos with resolution above 1920x1080. 109 110 <h3>Data Types</h3> 111 <p> 112 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 113 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 114 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 115 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 116 You normally cannot access the raw video data when using a Surface, but you can use the 117 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 118 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 119 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 120 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 121 OutputImage(int)}. 122 123 <h4>Compressed Buffers</h4> 124 <p> 125 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 126 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 127 compressed video frame. For audio data this is normally a single access unit (an encoded audio 128 segment typically containing a few milliseconds of audio as dictated by the format type), but 129 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 130 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 131 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 132 133 <h4>Raw Audio Buffers</h4> 134 <p> 135 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 136 in channel order. Each PCM audio sample is either a 16 bit signed integer or a float, 137 in native byte order. 138 Raw audio buffers in the float PCM encoding are only possible 139 if the MediaFormat's {@linkplain MediaFormat#KEY_PCM_ENCODING} 140 is set to {@linkplain AudioFormat#ENCODING_PCM_FLOAT} during MediaCodec 141 {@link #configure configure(…)} 142 and confirmed by {@link #getOutputFormat} for decoders 143 or {@link #getInputFormat} for encoders. 144 A sample method to check for float PCM in the MediaFormat is as follows: 145 146 <pre class=prettyprint> 147 static boolean isPcmFloat(MediaFormat format) { 148 return format.getInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) 149 == AudioFormat.ENCODING_PCM_FLOAT; 150 }</pre> 151 152 In order to extract, in a short array, 153 one channel of a buffer containing 16 bit signed integer audio data, 154 the following code may be used: 155 156 <pre class=prettyprint> 157 // Assumes the buffer PCM encoding is 16 bit. 158 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 159 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 160 MediaFormat format = codec.getOutputFormat(bufferId); 161 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 162 int numChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 163 if (channelIx < 0 || channelIx >= numChannels) { 164 return null; 165 } 166 short[] res = new short[samples.remaining() / numChannels]; 167 for (int i = 0; i < res.length; ++i) { 168 res[i] = samples.get(i * numChannels + channelIx); 169 } 170 return res; 171 }</pre> 172 173 <h4>Raw Video Buffers</h4> 174 <p> 175 In ByteBuffer mode video buffers are laid out according to their {@linkplain 176 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 177 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 178 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 179 Video codecs may support three kinds of color formats: 180 <ul> 181 <li><strong>native raw video format:</strong> This is marked by {@link 182 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 183 <li><strong>flexible YUV buffers</strong> (such as {@link 184 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 185 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 186 OutputImage(int)}.</li> 187 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 188 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 189 For color formats that are equivalent to a flexible format, you can still use {@link 190 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 191 </ul> 192 <p> 193 All video codecs support flexible YUV 4:2:0 buffers since {@link 194 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 195 196 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 197 <p> 198 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 199 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 200 values to understand the layout of the raw output buffers. 201 <p class=note> 202 Note that on some devices the slice-height is advertised as 0. This could mean either that the 203 slice-height is the same as the frame height, or that the slice-height is the frame height 204 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 205 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 206 plane in planar formats is also not specified or defined, though usually it is half of the slice 207 height. 208 <p> 209 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 210 video frames; however, for most encondings the video (picture) only occupies a portion of the 211 video frame. This is represented by the 'crop rectangle'. 212 <p> 213 You need to use the following keys to get the crop rectangle of raw output images from the 214 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 215 entire video frame.The crop rectangle is understood in the context of the output frame 216 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 217 <table style="width: 0%"> 218 <thead> 219 <tr> 220 <th>Format Key</th> 221 <th>Type</th> 222 <th>Description</th> 223 </tr> 224 </thead> 225 <tbody> 226 <tr> 227 <td>{@link MediaFormat#KEY_CROP_LEFT}</td> 228 <td>Integer</td> 229 <td>The left-coordinate (x) of the crop rectangle</td> 230 </tr><tr> 231 <td>{@link MediaFormat#KEY_CROP_TOP}</td> 232 <td>Integer</td> 233 <td>The top-coordinate (y) of the crop rectangle</td> 234 </tr><tr> 235 <td>{@link MediaFormat#KEY_CROP_RIGHT}</td> 236 <td>Integer</td> 237 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 238 </tr><tr> 239 <td>{@link MediaFormat#KEY_CROP_BOTTOM}</td> 240 <td>Integer</td> 241 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 242 </tr><tr> 243 <td colspan=3> 244 The right and bottom coordinates can be understood as the coordinates of the right-most 245 valid column/bottom-most valid row of the cropped output image. 246 </td> 247 </tr> 248 </tbody> 249 </table> 250 <p> 251 The size of the video frame (before rotation) can be calculated as such: 252 <pre class=prettyprint> 253 MediaFormat format = decoder.getOutputFormat(…); 254 int width = format.getInteger(MediaFormat.KEY_WIDTH); 255 if (format.containsKey(MediaFormat.KEY_CROP_LEFT) 256 && format.containsKey(MediaFormat.KEY_CROP_RIGHT)) { 257 width = format.getInteger(MediaFormat.KEY_CROP_RIGHT) + 1 258 - format.getInteger(MediaFormat.KEY_CROP_LEFT); 259 } 260 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 261 if (format.containsKey(MediaFormat.KEY_CROP_TOP) 262 && format.containsKey(MediaFormat.KEY_CROP_BOTTOM)) { 263 height = format.getInteger(MediaFormat.KEY_CROP_BOTTOM) + 1 264 - format.getInteger(MediaFormat.KEY_CROP_TOP); 265 } 266 </pre> 267 <p class=note> 268 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 269 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 270 most devices it pointed to the top-left pixel of the entire frame. 271 272 <h3>States</h3> 273 <p> 274 During its life a codec conceptually exists in one of three states: Stopped, Executing or 275 Released. The Stopped collective state is actually the conglomeration of three states: 276 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 277 three sub-states: Flushed, Running and End-of-Stream. 278 <p> 279 <center> 280 <img src="../../../images/media/mediacodec_states.svg" style="width: 519px; height: 356px" 281 alt="MediaCodec state diagram"> 282 </center> 283 <p> 284 When you create a codec using one of the factory methods, the codec is in the Uninitialized 285 state. First, you need to configure it via {@link #configure configure(…)}, which brings 286 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 287 state you can process data through the buffer queue manipulation described above. 288 <p> 289 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 290 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 291 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 292 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 293 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 294 codec no longer accepts further input buffers, but still generates output buffers until the 295 end-of-stream is reached on the output. For decoders, you can move back to the Flushed sub-state 296 at any time while in the Executing state using {@link #flush}. 297 <p class=note> 298 <strong>Note:</strong> Going back to Flushed state is only supported for decoders, and may not 299 work for encoders (the behavior is undefined). 300 <p> 301 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 302 again. When you are done using a codec, you must release it by calling {@link #release}. 303 <p> 304 On rare occasions the codec may encounter an error and move to the Error state. This is 305 communicated using an invalid return value from a queuing operation, or sometimes via an 306 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 307 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 308 terminal Released state. 309 310 <h3>Creation</h3> 311 <p> 312 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 313 decoding a file or a stream, you can get the desired format from {@link 314 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 315 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 316 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 317 name of a codec that can handle that specific media format. Finally, create the codec using 318 {@link #createByCodecName}. 319 <p class=note> 320 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 321 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 322 MediaFormat#KEY_FRAME_RATE frame rate}. Use 323 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 324 to clear any existing frame rate setting in the format. 325 <p> 326 You can also create the preferred codec for a specific MIME type using {@link 327 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 328 This, however, cannot be used to inject features, and may create a codec that cannot handle the 329 specific desired media format. 330 331 <h4>Creating secure decoders</h4> 332 <p> 333 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 334 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 335 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 336 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 337 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 338 <p> 339 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 340 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 341 342 <h3>Initialization</h3> 343 <p> 344 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 345 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 346 specific media format. This is when you can specify the output {@link Surface} for video 347 producers – codecs that generate raw video data (e.g. video decoders). This is also when 348 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 349 some codecs can operate in multiple modes, you must specify whether you want it to work as a 350 decoder or an encoder. 351 <p> 352 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 353 output format in the Configured state. You can use this to verify the resulting configuration, 354 e.g. color formats, before starting the codec. 355 <p> 356 If you want to process raw input video buffers natively with a video consumer – a codec 357 that processes raw video input, such as a video encoder – create a destination Surface for 358 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 359 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 360 surface} by calling {@link #setInputSurface}. 361 362 <h4 id=EncoderProfiles><a name="EncoderProfiles"></a>Encoder Profiles</h4> 363 <p> 364 When using an encoder, it is recommended to set the desired codec {@link MediaFormat#KEY_PROFILE 365 profile} during {@link #configure configure()}. (This is only meaningful for 366 {@link MediaFormat#KEY_MIME media formats} for which profiles are defined.) 367 <p> 368 If a profile is not specified during {@code configure}, the encoder will choose a profile for the 369 session based on the available information. We will call this value the <i>default profile</i>. 370 The selection of the default profile is device specific and may not be deterministic 371 (could be ad hoc or even experimental). The encoder may choose a default profile that is not 372 suitable for the intended encoding session, which may result in the encoder ultimately rejecting 373 the session. 374 <p> 375 The encoder may reject the encoding session if the configured (or default if unspecified) profile 376 does not support the codec input (mainly the {@link MediaFormat#KEY_COLOR_FORMAT color format} for 377 video/image codecs, or the {@link MediaFormat#KEY_PCM_ENCODING sample encoding} and the {@link 378 MediaFormat#KEY_CHANNEL_COUNT number of channels} for audio codecs, but also possibly 379 {@link MediaFormat#KEY_WIDTH width}, {@link MediaFormat#KEY_HEIGHT height}, 380 {@link MediaFormat#KEY_FRAME_RATE frame rate}, {@link MediaFormat#KEY_BIT_RATE bitrate} or 381 {@link MediaFormat#KEY_SAMPLE_RATE sample rate}.) 382 Alternatively, the encoder may choose to (but is not required to) convert the input to support the 383 selected (or default) profile - or adjust the chosen profile based on the presumed or detected 384 input format - to ensure a successful encoding session. <b>Note</b>: Converting the input to match 385 an incompatible profile will in most cases result in decreased codec performance. 386 <p> 387 To ensure backward compatibility, the following guarantees are provided by Android: 388 <ul> 389 <li>The default video encoder profile always supports 8-bit YUV 4:2:0 color format ({@link 390 CodecCapabilities#COLOR_FormatYUV420Flexible COLOR_FormatYUV420Flexible} and equivalent 391 {@link CodecCapabilities#colorFormats supported formats}) for both Surface and ByteBuffer modes. 392 <li>The default video encoder profile always supports the default 8-bit RGBA color format in 393 Surface mode even if no such formats are enumerated in the {@link CodecCapabilities#colorFormats 394 supported formats}. 395 </ul> 396 <p class=note> 397 <b>Note</b>: the accepted profile can be queried through the {@link #getOutputFormat output 398 format} of the encoder after {@code configure} to allow applications to set up their 399 codec input to a format supported by the encoder profile. 400 <p> 401 <b>Implication:</b> 402 <ul> 403 <li>Applications that want to encode 4:2:2, 4:4:4, 10+ bit or HDR video input <b>MUST</b> configure 404 a suitable profile for encoders. 405 </ul> 406 407 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 408 <p> 409 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 410 to be prefixed by a number of buffers containing setup data, or codec specific data. When 411 processing such compressed formats, this data must be submitted to the codec after {@link 412 #start} and before any frame data. Such data must be marked using the flag {@link 413 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 414 <p> 415 Codec-specific data can also be included in the format passed to {@link #configure configure} in 416 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 417 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 418 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 419 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 420 specific data, you can choose to submit it using the specified number of buffers in the correct 421 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 422 codec-specific data and submit it as a single codec-config buffer. 423 <p> 424 Android uses the following codec-specific data buffers. These are also required to be set in 425 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 426 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 427 {@code "\x00\x00\x00\x01"}. 428 <p> 429 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 430 <table> 431 <thead> 432 <th>Format</th> 433 <th>CSD buffer #0</th> 434 <th>CSD buffer #1</th> 435 <th>CSD buffer #2</th> 436 </thead> 437 <tbody class=mid> 438 <tr> 439 <td>AAC</td> 440 <td>Decoder-specific information from ESDS<sup>*</sup></td> 441 <td class=NA>Not Used</td> 442 <td class=NA>Not Used</td> 443 </tr> 444 <tr> 445 <td>VORBIS</td> 446 <td>Identification header</td> 447 <td>Setup header</td> 448 <td class=NA>Not Used</td> 449 </tr> 450 <tr> 451 <td>OPUS</td> 452 <td>Identification header</td> 453 <td>Pre-skip in nanosecs<br> 454 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 455 This overrides the pre-skip value in the identification header.</td> 456 <td>Seek Pre-roll in nanosecs<br> 457 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 458 </tr> 459 <tr> 460 <td>FLAC</td> 461 <td>"fLaC", the FLAC stream marker in ASCII,<br> 462 followed by the STREAMINFO block (the mandatory metadata block),<br> 463 optionally followed by any number of other metadata blocks</td> 464 <td class=NA>Not Used</td> 465 <td class=NA>Not Used</td> 466 </tr> 467 <tr> 468 <td>MPEG-4</td> 469 <td>Decoder-specific information from ESDS<sup>*</sup></td> 470 <td class=NA>Not Used</td> 471 <td class=NA>Not Used</td> 472 </tr> 473 <tr> 474 <td>H.264 AVC</td> 475 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 476 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 477 <td class=NA>Not Used</td> 478 </tr> 479 <tr> 480 <td>H.265 HEVC</td> 481 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 482 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 483 PPS (Picture Parameter Sets<sup>*</sup>)</td> 484 <td class=NA>Not Used</td> 485 <td class=NA>Not Used</td> 486 </tr> 487 <tr> 488 <td>VP9</td> 489 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 490 (optional)</td> 491 <td class=NA>Not Used</td> 492 <td class=NA>Not Used</td> 493 </tr> 494 <tr> 495 <td>AV1</td> 496 <td>AV1 <a href="https://aomediacodec.github.io/av1-isobmff/#av1codecconfigurationbox-syntax"> 497 AV1CodecConfigurationRecord</a> Data (optional) 498 </td> 499 <td class=NA>Not Used</td> 500 <td class=NA>Not Used</td> 501 </tr> 502 </tbody> 503 </table> 504 505 <p class=note> 506 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 507 after start, before any output buffer or output format change has been returned, as the codec 508 specific data may be lost during the flush. You must resubmit the data using buffers marked with 509 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 510 <p> 511 Encoders (or codecs that generate compressed data) will create and return the codec specific data 512 before any valid output buffer in output buffers marked with the {@linkplain 513 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 514 meaningful timestamps. 515 516 <h3>Data Processing</h3> 517 <p> 518 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 519 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 520 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 521 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 522 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 523 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 524 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 525 <p> 526 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 527 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 528 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 529 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 530 <p> 531 The codec in turn will return a read-only output buffer via the {@link 532 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 533 response to a {@link #dequeueOutputBuffer dequeueOutputBuffer} call in synchronous mode. After the 534 output buffer has been processed, call one of the {@link #releaseOutputBuffer 535 releaseOutputBuffer} methods to return the buffer to the codec. 536 <p> 537 While you are not required to resubmit/release buffers immediately to the codec, holding onto 538 input and/or output buffers may stall the codec, and this behavior is device dependent. 539 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 540 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 541 hold onto to available buffers as little as possible. 542 <p> 543 Depending on the API version, you can process data in three ways: 544 <table> 545 <thead> 546 <tr> 547 <th>Processing Mode</th> 548 <th>API version <= 20<br>Jelly Bean/KitKat</th> 549 <th>API version >= 21<br>Lollipop and later</th> 550 </tr> 551 </thead> 552 <tbody> 553 <tr> 554 <td>Synchronous API using buffer arrays</td> 555 <td>Supported</td> 556 <td>Deprecated</td> 557 </tr> 558 <tr> 559 <td>Synchronous API using buffers</td> 560 <td class=NA>Not Available</td> 561 <td>Supported</td> 562 </tr> 563 <tr> 564 <td>Asynchronous API using buffers</td> 565 <td class=NA>Not Available</td> 566 <td>Supported</td> 567 </tr> 568 </tbody> 569 </table> 570 571 <h4>Asynchronous Processing using Buffers</h4> 572 <p> 573 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 574 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 575 mode changes the state transitions slightly, because you must call {@link #start} after {@link 576 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 577 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 578 sub-state and start passing available input buffers via the callback. 579 <p> 580 <center> 581 <img src="../../../images/media/mediacodec_async_states.svg" style="width: 516px; height: 353px" 582 alt="MediaCodec state diagram for asynchronous operation"> 583 </center> 584 <p> 585 MediaCodec is typically used like this in asynchronous mode: 586 <pre class=prettyprint> 587 MediaCodec codec = MediaCodec.createByCodecName(name); 588 MediaFormat mOutputFormat; // member variable 589 codec.setCallback(new MediaCodec.Callback() { 590 {@literal @Override} 591 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 592 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 593 // fill inputBuffer with valid data 594 … 595 codec.queueInputBuffer(inputBufferId, …); 596 } 597 598 {@literal @Override} 599 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 600 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 601 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 602 // bufferFormat is equivalent to mOutputFormat 603 // outputBuffer is ready to be processed or rendered. 604 … 605 codec.releaseOutputBuffer(outputBufferId, …); 606 } 607 608 {@literal @Override} 609 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 610 // Subsequent data will conform to new format. 611 // Can ignore if using getOutputFormat(outputBufferId) 612 mOutputFormat = format; // option B 613 } 614 615 {@literal @Override} 616 void onError(…) { 617 … 618 } 619 {@literal @Override} 620 void onCryptoError(…) { 621 … 622 } 623 }); 624 codec.configure(format, …); 625 mOutputFormat = codec.getOutputFormat(); // option B 626 codec.start(); 627 // wait for processing to complete 628 codec.stop(); 629 codec.release();</pre> 630 631 <h4>Synchronous Processing using Buffers</h4> 632 <p> 633 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 634 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 635 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 636 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 637 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 638 getInput}/{@link #getOutputBuffers OutputBuffers()}. 639 640 <p class=note> 641 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 642 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 643 #start} or after having dequeued an output buffer ID with the value of {@link 644 #INFO_OUTPUT_FORMAT_CHANGED}. 645 <p> 646 MediaCodec is typically used like this in synchronous mode: 647 <pre> 648 MediaCodec codec = MediaCodec.createByCodecName(name); 649 codec.configure(format, …); 650 MediaFormat outputFormat = codec.getOutputFormat(); // option B 651 codec.start(); 652 for (;;) { 653 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 654 if (inputBufferId >= 0) { 655 ByteBuffer inputBuffer = codec.getInputBuffer(…); 656 // fill inputBuffer with valid data 657 … 658 codec.queueInputBuffer(inputBufferId, …); 659 } 660 int outputBufferId = codec.dequeueOutputBuffer(…); 661 if (outputBufferId >= 0) { 662 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 663 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 664 // bufferFormat is identical to outputFormat 665 // outputBuffer is ready to be processed or rendered. 666 … 667 codec.releaseOutputBuffer(outputBufferId, …); 668 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 669 // Subsequent data will conform to new format. 670 // Can ignore if using getOutputFormat(outputBufferId) 671 outputFormat = codec.getOutputFormat(); // option B 672 } 673 } 674 codec.stop(); 675 codec.release();</pre> 676 677 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 678 <p> 679 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 680 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 681 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 682 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 683 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 684 between the size of the arrays and the number of input and output buffers used by the system, 685 although the array size provides an upper bound. 686 <pre> 687 MediaCodec codec = MediaCodec.createByCodecName(name); 688 codec.configure(format, …); 689 codec.start(); 690 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 691 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 692 for (;;) { 693 int inputBufferId = codec.dequeueInputBuffer(…); 694 if (inputBufferId >= 0) { 695 // fill inputBuffers[inputBufferId] with valid data 696 … 697 codec.queueInputBuffer(inputBufferId, …); 698 } 699 int outputBufferId = codec.dequeueOutputBuffer(…); 700 if (outputBufferId >= 0) { 701 // outputBuffers[outputBufferId] is ready to be processed or rendered. 702 … 703 codec.releaseOutputBuffer(outputBufferId, …); 704 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 705 outputBuffers = codec.getOutputBuffers(); 706 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 707 // Subsequent data will conform to new format. 708 MediaFormat format = codec.getOutputFormat(); 709 } 710 } 711 codec.stop(); 712 codec.release();</pre> 713 714 <h4>End-of-stream Handling</h4> 715 <p> 716 When you reach the end of the input data, you must signal it to the codec by specifying the 717 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 718 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 719 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 720 be ignored. 721 <p> 722 The codec will continue to return output buffers until it eventually signals the end of the 723 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 724 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 725 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 726 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 727 <p> 728 Do not submit additional input buffers after signaling the end of the input stream, unless the 729 codec has been flushed, or stopped and restarted. 730 731 <h4>Using an Output Surface</h4> 732 <p> 733 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 734 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 735 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 736 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 737 null}-s. 738 <p> 739 When using an output Surface, you can select whether or not to render each output buffer on the 740 surface. You have three choices: 741 <ul> 742 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 743 releaseOutputBuffer(bufferId, false)}.</li> 744 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 745 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 746 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 747 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 748 </ul> 749 <p> 750 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 751 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 752 It was not defined prior to that. 753 <p> 754 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 755 dynamically using {@link #setOutputSurface setOutputSurface}. 756 <p> 757 When rendering output to a Surface, the Surface may be configured to drop excessive frames (that 758 are not consumed by the Surface in a timely manner). Or it may be configured to not drop excessive 759 frames. In the latter mode if the Surface is not consuming output frames fast enough, it will 760 eventually block the decoder. Prior to {@link android.os.Build.VERSION_CODES#Q} the exact behavior 761 was undefined, with the exception that View surfaces (SurfaceView or TextureView) always dropped 762 excessive frames. Since {@link android.os.Build.VERSION_CODES#Q} the default behavior is to drop 763 excessive frames. Applications can opt out of this behavior for non-View surfaces (such as 764 ImageReader or SurfaceTexture) by targeting SDK {@link android.os.Build.VERSION_CODES#Q} and 765 setting the key {@link MediaFormat#KEY_ALLOW_FRAME_DROP} to {@code 0} 766 in their configure format. 767 768 <h4>Transformations When Rendering onto Surface</h4> 769 770 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 771 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 772 mode} will be automatically applied with one exception: 773 <p class=note> 774 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 775 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 776 and simple way to identify software decoders, or if they apply the rotation other than by trying 777 it out. 778 <p> 779 There are also some caveats. 780 <p class=note> 781 Note that the pixel aspect ratio is not considered when displaying the output onto the 782 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 783 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 784 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 785 square pixels (pixel aspect ratio or 1:1). 786 <p class=note> 787 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 788 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 789 by 90 or 270 degrees. 790 <p class=note> 791 When setting the video scaling mode, note that it must be reset after each time the output 792 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 793 do this after each time the output format changes. 794 795 <h4>Using an Input Surface</h4> 796 <p> 797 When using an input Surface, there are no accessible input buffers, as buffers are automatically 798 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 799 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 800 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 801 <p> 802 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 803 submitting data to the codec immediately after this call. 804 <p> 805 806 <h3>Seeking & Adaptive Playback Support</h3> 807 <p> 808 Video decoders (and in general codecs that consume compressed video data) behave differently 809 regarding seek and format change whether or not they support and are configured for adaptive 810 playback. You can check if a decoder supports {@linkplain 811 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 812 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 813 playback support for video decoders is only activated if you configure the codec to decode onto a 814 {@link Surface}. 815 816 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 817 <p> 818 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 819 stream boundary: the first frame must be a key frame. A <em>key frame</em> can be decoded 820 completely on its own (for most codecs this means an I-frame), and no frames that are to be 821 displayed after a key frame refer to frames before the key frame. 822 <p> 823 The following table summarizes suitable key frames for various video formats. 824 <table> 825 <thead> 826 <tr> 827 <th>Format</th> 828 <th>Suitable key frame</th> 829 </tr> 830 </thead> 831 <tbody class=mid> 832 <tr> 833 <td>VP9/VP8</td> 834 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 835 <i>(There is no specific name for such key frame.)</i></td> 836 </tr> 837 <tr> 838 <td>H.265 HEVC</td> 839 <td>IDR or CRA</td> 840 </tr> 841 <tr> 842 <td>H.264 AVC</td> 843 <td>IDR</td> 844 </tr> 845 <tr> 846 <td>MPEG-4<br>H.263<br>MPEG-2</td> 847 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 848 <i>(There is no specific name for such key frame.)</td> 849 </tr> 850 </tbody> 851 </table> 852 853 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 854 Surface)</h4> 855 <p> 856 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 857 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 858 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 859 before you call {@code flush}. It is important that the input data after a flush starts at a 860 suitable stream boundary/key frame. 861 <p class=note> 862 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 863 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 864 #configure configure(…)} - {@link #start} cycle is necessary. 865 866 <p class=note> 867 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 868 generally, before the first output buffer or output format change is received – you 869 will need to resubmit the codec-specific-data to the codec. See the <a 870 href="#CSD">codec-specific-data section</a> for more info. 871 872 <h4>For decoders that support and are configured for adaptive playback</h4> 873 <p> 874 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 875 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 876 discontinuity must start at a suitable stream boundary/key frame. 877 <p> 878 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 879 picture size or configuration mid-stream. To do this you must package the entire new 880 codec-specific configuration data together with the key frame into a single buffer (including 881 any start codes), and submit it as a <strong>regular</strong> input buffer. 882 <p> 883 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 884 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 885 onOutputFormatChanged} callback just after the picture-size change takes place and before any 886 frames with the new size have been returned. 887 <p class=note> 888 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 889 {@link #flush} shortly after you have changed the picture size. If you have not received 890 confirmation of the picture size change, you will need to repeat the request for the new picture 891 size. 892 893 <h3>Error handling</h3> 894 <p> 895 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 896 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 897 which you must catch or declare to pass up. MediaCodec methods throw {@code 898 IllegalStateException} when the method is called from a codec state that does not allow it; this 899 is typically due to incorrect application API usage. Methods involving secure buffers may throw 900 {@link CryptoException}, which has further error information obtainable from {@link 901 CryptoException#getErrorCode}. 902 <p> 903 Internal codec errors result in a {@link CodecException}, which may be due to media content 904 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 905 correctly using the API. The recommended action when receiving a {@code CodecException} 906 can be determined by calling {@link CodecException#isRecoverable} and {@link 907 CodecException#isTransient}: 908 <ul> 909 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 910 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 911 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 912 temporarily unavailable and the method may be retried at a later time.</li> 913 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 914 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 915 reset} or {@linkplain #release released}.</li> 916 </ul> 917 <p> 918 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 919 920 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 921 <p> 922 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 923 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 924 925 <style> 926 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 927 .api > tr > th { vertical-align: bottom; } 928 .api > tr > td { vertical-align: middle; } 929 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 930 .fn { text-align: left; } 931 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 932 .deg45 { 933 white-space: nowrap; background: none; border: none; vertical-align: bottom; 934 width: 30px; height: 83px; 935 } 936 .deg45 > div { 937 transform: skew(-45deg, 0deg) translate(1px, -67px); 938 transform-origin: bottom left 0; 939 width: 30px; height: 20px; 940 } 941 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 942 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 943 </style> 944 945 <table align="right" style="width: 0%"> 946 <thead> 947 <tr><th>Symbol</th><th>Meaning</th></tr> 948 </thead> 949 <tbody class=sml> 950 <tr><td>●</td><td>Supported</td></tr> 951 <tr><td>⁕</td><td>Semantics changed</td></tr> 952 <tr><td>○</td><td>Experimental support</td></tr> 953 <tr><td>[ ]</td><td>Deprecated</td></tr> 954 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 955 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 956 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 957 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 958 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 959 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 960 </tbody> 961 </table> 962 963 <table style="width: 100%;"> 964 <thead class=api> 965 <tr> 966 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 967 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 968 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 969 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 970 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 971 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 972 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 973 <th></th> 974 <th colspan="8">SDK Version</th> 975 </tr> 976 <tr> 977 <th colspan="7">State</th> 978 <th>Method</th> 979 <th>16</th> 980 <th>17</th> 981 <th>18</th> 982 <th>19</th> 983 <th>20</th> 984 <th>21</th> 985 <th>22</th> 986 <th>23</th> 987 </tr> 988 </thead> 989 <tbody class=api> 990 <tr> 991 <td></td> 992 <td></td> 993 <td></td> 994 <td></td> 995 <td></td> 996 <td></td> 997 <td></td> 998 <td class=fn>{@link #createByCodecName createByCodecName}</td> 999 <td>●</td> 1000 <td>●</td> 1001 <td>●</td> 1002 <td>●</td> 1003 <td>●</td> 1004 <td>●</td> 1005 <td>●</td> 1006 <td>●</td> 1007 </tr> 1008 <tr> 1009 <td></td> 1010 <td></td> 1011 <td></td> 1012 <td></td> 1013 <td></td> 1014 <td></td> 1015 <td></td> 1016 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 1017 <td>●</td> 1018 <td>●</td> 1019 <td>●</td> 1020 <td>●</td> 1021 <td>●</td> 1022 <td>●</td> 1023 <td>●</td> 1024 <td>●</td> 1025 </tr> 1026 <tr> 1027 <td></td> 1028 <td></td> 1029 <td></td> 1030 <td></td> 1031 <td></td> 1032 <td></td> 1033 <td></td> 1034 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 1035 <td>●</td> 1036 <td>●</td> 1037 <td>●</td> 1038 <td>●</td> 1039 <td>●</td> 1040 <td>●</td> 1041 <td>●</td> 1042 <td>●</td> 1043 </tr> 1044 <tr> 1045 <td></td> 1046 <td></td> 1047 <td></td> 1048 <td></td> 1049 <td></td> 1050 <td></td> 1051 <td></td> 1052 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 1053 <td></td> 1054 <td></td> 1055 <td></td> 1056 <td></td> 1057 <td></td> 1058 <td></td> 1059 <td></td> 1060 <td>●</td> 1061 </tr> 1062 <tr> 1063 <td>16+</td> 1064 <td>-</td> 1065 <td>-</td> 1066 <td>-</td> 1067 <td>-</td> 1068 <td>-</td> 1069 <td>-</td> 1070 <td class=fn>{@link #configure configure}</td> 1071 <td>●</td> 1072 <td>●</td> 1073 <td>●</td> 1074 <td>●</td> 1075 <td>●</td> 1076 <td>⁕</td> 1077 <td>●</td> 1078 <td>●</td> 1079 </tr> 1080 <tr> 1081 <td>-</td> 1082 <td>18+</td> 1083 <td>-</td> 1084 <td>-</td> 1085 <td>-</td> 1086 <td>-</td> 1087 <td>-</td> 1088 <td class=fn>{@link #createInputSurface createInputSurface}</td> 1089 <td></td> 1090 <td></td> 1091 <td>⎋</td> 1092 <td>⎋</td> 1093 <td>⎋</td> 1094 <td>⎋</td> 1095 <td>⎋</td> 1096 <td>⎋</td> 1097 </tr> 1098 <tr> 1099 <td>-</td> 1100 <td>-</td> 1101 <td>16+</td> 1102 <td>16+</td> 1103 <td>(16+)</td> 1104 <td>-</td> 1105 <td>-</td> 1106 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 1107 <td>●</td> 1108 <td>●</td> 1109 <td>▧</td> 1110 <td>▧</td> 1111 <td>▧</td> 1112 <td>⁕▧↩</td> 1113 <td>▧↩</td> 1114 <td>▧↩</td> 1115 </tr> 1116 <tr> 1117 <td>-</td> 1118 <td>-</td> 1119 <td>16+</td> 1120 <td>16+</td> 1121 <td>16+</td> 1122 <td>-</td> 1123 <td>-</td> 1124 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 1125 <td>●</td> 1126 <td>●</td> 1127 <td>●</td> 1128 <td>●</td> 1129 <td>●</td> 1130 <td>⁕↩</td> 1131 <td>↩</td> 1132 <td>↩</td> 1133 </tr> 1134 <tr> 1135 <td>-</td> 1136 <td>-</td> 1137 <td>16+</td> 1138 <td>16+</td> 1139 <td>16+</td> 1140 <td>-</td> 1141 <td>-</td> 1142 <td class=fn>{@link #flush flush}</td> 1143 <td>●</td> 1144 <td>●</td> 1145 <td>●</td> 1146 <td>●</td> 1147 <td>●</td> 1148 <td>●</td> 1149 <td>●</td> 1150 <td>●</td> 1151 </tr> 1152 <tr> 1153 <td>18+</td> 1154 <td>18+</td> 1155 <td>18+</td> 1156 <td>18+</td> 1157 <td>18+</td> 1158 <td>18+</td> 1159 <td>-</td> 1160 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1161 <td></td> 1162 <td></td> 1163 <td>●</td> 1164 <td>●</td> 1165 <td>●</td> 1166 <td>●</td> 1167 <td>●</td> 1168 <td>●</td> 1169 </tr> 1170 <tr> 1171 <td>-</td> 1172 <td>-</td> 1173 <td>(21+)</td> 1174 <td>21+</td> 1175 <td>(21+)</td> 1176 <td>-</td> 1177 <td>-</td> 1178 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1179 <td></td> 1180 <td></td> 1181 <td></td> 1182 <td></td> 1183 <td></td> 1184 <td>●</td> 1185 <td>●</td> 1186 <td>●</td> 1187 </tr> 1188 <tr> 1189 <td>-</td> 1190 <td>-</td> 1191 <td>16+</td> 1192 <td>(16+)</td> 1193 <td>(16+)</td> 1194 <td>-</td> 1195 <td>-</td> 1196 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1197 <td>●</td> 1198 <td>●</td> 1199 <td>●</td> 1200 <td>●</td> 1201 <td>●</td> 1202 <td>[⁕↩]</td> 1203 <td>[↩]</td> 1204 <td>[↩]</td> 1205 </tr> 1206 <tr> 1207 <td>-</td> 1208 <td>21+</td> 1209 <td>(21+)</td> 1210 <td>(21+)</td> 1211 <td>(21+)</td> 1212 <td>-</td> 1213 <td>-</td> 1214 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1215 <td></td> 1216 <td></td> 1217 <td></td> 1218 <td></td> 1219 <td></td> 1220 <td>●</td> 1221 <td>●</td> 1222 <td>●</td> 1223 </tr> 1224 <tr> 1225 <td>-</td> 1226 <td>-</td> 1227 <td>(21+)</td> 1228 <td>21+</td> 1229 <td>(21+)</td> 1230 <td>-</td> 1231 <td>-</td> 1232 <td class=fn>{@link #getInputImage getInputImage}</td> 1233 <td></td> 1234 <td></td> 1235 <td></td> 1236 <td></td> 1237 <td></td> 1238 <td>○</td> 1239 <td>●</td> 1240 <td>●</td> 1241 </tr> 1242 <tr> 1243 <td>18+</td> 1244 <td>18+</td> 1245 <td>18+</td> 1246 <td>18+</td> 1247 <td>18+</td> 1248 <td>18+</td> 1249 <td>-</td> 1250 <td class=fn>{@link #getName getName}</td> 1251 <td></td> 1252 <td></td> 1253 <td>●</td> 1254 <td>●</td> 1255 <td>●</td> 1256 <td>●</td> 1257 <td>●</td> 1258 <td>●</td> 1259 </tr> 1260 <tr> 1261 <td>-</td> 1262 <td>-</td> 1263 <td>(21+)</td> 1264 <td>21+</td> 1265 <td>21+</td> 1266 <td>-</td> 1267 <td>-</td> 1268 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1269 <td></td> 1270 <td></td> 1271 <td></td> 1272 <td></td> 1273 <td></td> 1274 <td>●</td> 1275 <td>●</td> 1276 <td>●</td> 1277 </tr> 1278 <tr> 1279 <td>-</td> 1280 <td>-</td> 1281 <td>16+</td> 1282 <td>16+</td> 1283 <td>16+</td> 1284 <td>-</td> 1285 <td>-</td> 1286 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1287 <td>●</td> 1288 <td>●</td> 1289 <td>●</td> 1290 <td>●</td> 1291 <td>●</td> 1292 <td>[⁕↩]</td> 1293 <td>[↩]</td> 1294 <td>[↩]</td> 1295 </tr> 1296 <tr> 1297 <td>-</td> 1298 <td>21+</td> 1299 <td>16+</td> 1300 <td>16+</td> 1301 <td>16+</td> 1302 <td>-</td> 1303 <td>-</td> 1304 <td class=fn>{@link #getOutputFormat()}</td> 1305 <td>●</td> 1306 <td>●</td> 1307 <td>●</td> 1308 <td>●</td> 1309 <td>●</td> 1310 <td>●</td> 1311 <td>●</td> 1312 <td>●</td> 1313 </tr> 1314 <tr> 1315 <td>-</td> 1316 <td>-</td> 1317 <td>(21+)</td> 1318 <td>21+</td> 1319 <td>21+</td> 1320 <td>-</td> 1321 <td>-</td> 1322 <td class=fn>{@link #getOutputFormat(int)}</td> 1323 <td></td> 1324 <td></td> 1325 <td></td> 1326 <td></td> 1327 <td></td> 1328 <td>●</td> 1329 <td>●</td> 1330 <td>●</td> 1331 </tr> 1332 <tr> 1333 <td>-</td> 1334 <td>-</td> 1335 <td>(21+)</td> 1336 <td>21+</td> 1337 <td>21+</td> 1338 <td>-</td> 1339 <td>-</td> 1340 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1341 <td></td> 1342 <td></td> 1343 <td></td> 1344 <td></td> 1345 <td></td> 1346 <td>○</td> 1347 <td>●</td> 1348 <td>●</td> 1349 </tr> 1350 <tr> 1351 <td>-</td> 1352 <td>-</td> 1353 <td>-</td> 1354 <td>16+</td> 1355 <td>(16+)</td> 1356 <td>-</td> 1357 <td>-</td> 1358 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1359 <td>●</td> 1360 <td>●</td> 1361 <td>●</td> 1362 <td>●</td> 1363 <td>●</td> 1364 <td>⁕</td> 1365 <td>●</td> 1366 <td>●</td> 1367 </tr> 1368 <tr> 1369 <td>-</td> 1370 <td>-</td> 1371 <td>-</td> 1372 <td>16+</td> 1373 <td>(16+)</td> 1374 <td>-</td> 1375 <td>-</td> 1376 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1377 <td>●</td> 1378 <td>●</td> 1379 <td>●</td> 1380 <td>●</td> 1381 <td>●</td> 1382 <td>⁕</td> 1383 <td>●</td> 1384 <td>●</td> 1385 </tr> 1386 <tr> 1387 <td>16+</td> 1388 <td>16+</td> 1389 <td>16+</td> 1390 <td>16+</td> 1391 <td>16+</td> 1392 <td>16+</td> 1393 <td>16+</td> 1394 <td class=fn>{@link #release release}</td> 1395 <td>●</td> 1396 <td>●</td> 1397 <td>●</td> 1398 <td>●</td> 1399 <td>●</td> 1400 <td>●</td> 1401 <td>●</td> 1402 <td>●</td> 1403 </tr> 1404 <tr> 1405 <td>-</td> 1406 <td>-</td> 1407 <td>-</td> 1408 <td>16+</td> 1409 <td>16+</td> 1410 <td>-</td> 1411 <td>-</td> 1412 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1413 <td>●</td> 1414 <td>●</td> 1415 <td>●</td> 1416 <td>●</td> 1417 <td>●</td> 1418 <td>⁕</td> 1419 <td>●</td> 1420 <td>⁕</td> 1421 </tr> 1422 <tr> 1423 <td>-</td> 1424 <td>-</td> 1425 <td>-</td> 1426 <td>21+</td> 1427 <td>21+</td> 1428 <td>-</td> 1429 <td>-</td> 1430 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1431 <td></td> 1432 <td></td> 1433 <td></td> 1434 <td></td> 1435 <td></td> 1436 <td>⎆</td> 1437 <td>⎆</td> 1438 <td>⎆</td> 1439 </tr> 1440 <tr> 1441 <td>21+</td> 1442 <td>21+</td> 1443 <td>21+</td> 1444 <td>21+</td> 1445 <td>21+</td> 1446 <td>21+</td> 1447 <td>-</td> 1448 <td class=fn>{@link #reset reset}</td> 1449 <td></td> 1450 <td></td> 1451 <td></td> 1452 <td></td> 1453 <td></td> 1454 <td>●</td> 1455 <td>●</td> 1456 <td>●</td> 1457 </tr> 1458 <tr> 1459 <td>21+</td> 1460 <td>-</td> 1461 <td>-</td> 1462 <td>-</td> 1463 <td>-</td> 1464 <td>-</td> 1465 <td>-</td> 1466 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1467 <td></td> 1468 <td></td> 1469 <td></td> 1470 <td></td> 1471 <td></td> 1472 <td>●</td> 1473 <td>●</td> 1474 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1475 </tr> 1476 <tr> 1477 <td>-</td> 1478 <td>23+</td> 1479 <td>-</td> 1480 <td>-</td> 1481 <td>-</td> 1482 <td>-</td> 1483 <td>-</td> 1484 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1485 <td></td> 1486 <td></td> 1487 <td></td> 1488 <td></td> 1489 <td></td> 1490 <td></td> 1491 <td></td> 1492 <td>⎋</td> 1493 </tr> 1494 <tr> 1495 <td>23+</td> 1496 <td>23+</td> 1497 <td>23+</td> 1498 <td>23+</td> 1499 <td>23+</td> 1500 <td>(23+)</td> 1501 <td>(23+)</td> 1502 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1503 <td></td> 1504 <td></td> 1505 <td></td> 1506 <td></td> 1507 <td></td> 1508 <td></td> 1509 <td></td> 1510 <td>○ ⎆</td> 1511 </tr> 1512 <tr> 1513 <td>-</td> 1514 <td>23+</td> 1515 <td>23+</td> 1516 <td>23+</td> 1517 <td>23+</td> 1518 <td>-</td> 1519 <td>-</td> 1520 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1521 <td></td> 1522 <td></td> 1523 <td></td> 1524 <td></td> 1525 <td></td> 1526 <td></td> 1527 <td></td> 1528 <td>⎆</td> 1529 </tr> 1530 <tr> 1531 <td>19+</td> 1532 <td>19+</td> 1533 <td>19+</td> 1534 <td>19+</td> 1535 <td>19+</td> 1536 <td>(19+)</td> 1537 <td>-</td> 1538 <td class=fn>{@link #setParameters setParameters}</td> 1539 <td></td> 1540 <td></td> 1541 <td></td> 1542 <td>●</td> 1543 <td>●</td> 1544 <td>●</td> 1545 <td>●</td> 1546 <td>●</td> 1547 </tr> 1548 <tr> 1549 <td>-</td> 1550 <td>(16+)</td> 1551 <td>(16+)</td> 1552 <td>16+</td> 1553 <td>(16+)</td> 1554 <td>(16+)</td> 1555 <td>-</td> 1556 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1557 <td>⎆</td> 1558 <td>⎆</td> 1559 <td>⎆</td> 1560 <td>⎆</td> 1561 <td>⎆</td> 1562 <td>⎆</td> 1563 <td>⎆</td> 1564 <td>⎆</td> 1565 </tr> 1566 <tr> 1567 <td>(29+)</td> 1568 <td>29+</td> 1569 <td>29+</td> 1570 <td>29+</td> 1571 <td>(29+)</td> 1572 <td>(29+)</td> 1573 <td>-</td> 1574 <td class=fn>{@link #setAudioPresentation setAudioPresentation}</td> 1575 <td></td> 1576 <td></td> 1577 <td></td> 1578 <td></td> 1579 <td></td> 1580 <td></td> 1581 <td></td> 1582 <td></td> 1583 </tr> 1584 <tr> 1585 <td>-</td> 1586 <td>-</td> 1587 <td>18+</td> 1588 <td>18+</td> 1589 <td>-</td> 1590 <td>-</td> 1591 <td>-</td> 1592 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1593 <td></td> 1594 <td></td> 1595 <td>⎋</td> 1596 <td>⎋</td> 1597 <td>⎋</td> 1598 <td>⎋</td> 1599 <td>⎋</td> 1600 <td>⎋</td> 1601 </tr> 1602 <tr> 1603 <td>-</td> 1604 <td>16+</td> 1605 <td>21+(⇄)</td> 1606 <td>-</td> 1607 <td>-</td> 1608 <td>-</td> 1609 <td>-</td> 1610 <td class=fn>{@link #start start}</td> 1611 <td>●</td> 1612 <td>●</td> 1613 <td>●</td> 1614 <td>●</td> 1615 <td>●</td> 1616 <td>⁕</td> 1617 <td>●</td> 1618 <td>●</td> 1619 </tr> 1620 <tr> 1621 <td>-</td> 1622 <td>-</td> 1623 <td>16+</td> 1624 <td>16+</td> 1625 <td>16+</td> 1626 <td>-</td> 1627 <td>-</td> 1628 <td class=fn>{@link #stop stop}</td> 1629 <td>●</td> 1630 <td>●</td> 1631 <td>●</td> 1632 <td>●</td> 1633 <td>●</td> 1634 <td>●</td> 1635 <td>●</td> 1636 <td>●</td> 1637 </tr> 1638 </tbody> 1639 </table> 1640 */ 1641 final public class MediaCodec { 1642 1643 /** 1644 * Per buffer metadata includes an offset and size specifying 1645 * the range of valid data in the associated codec (output) buffer. 1646 */ 1647 public final static class BufferInfo { 1648 /** 1649 * Update the buffer metadata information. 1650 * 1651 * @param newOffset the start-offset of the data in the buffer. 1652 * @param newSize the amount of data (in bytes) in the buffer. 1653 * @param newTimeUs the presentation timestamp in microseconds. 1654 * @param newFlags buffer flags associated with the buffer. This 1655 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1656 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1657 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1658 public void set( 1659 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1660 offset = newOffset; 1661 size = newSize; 1662 presentationTimeUs = newTimeUs; 1663 flags = newFlags; 1664 } 1665 1666 /** 1667 * The start-offset of the data in the buffer. 1668 */ 1669 public int offset; 1670 1671 /** 1672 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1673 * the buffer has no data in it and can be discarded. The only 1674 * use of a 0-size buffer is to carry the end-of-stream marker. 1675 */ 1676 public int size; 1677 1678 /** 1679 * The presentation timestamp in microseconds for the buffer. 1680 * This is derived from the presentation timestamp passed in 1681 * with the corresponding input buffer. This should be ignored for 1682 * a 0-sized buffer. 1683 */ 1684 public long presentationTimeUs; 1685 1686 /** 1687 * Buffer flags associated with the buffer. A combination of 1688 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1689 * 1690 * <p>Encoded buffers that are key frames are marked with 1691 * {@link #BUFFER_FLAG_KEY_FRAME}. 1692 * 1693 * <p>The last output buffer corresponding to the input buffer 1694 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1695 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1696 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1697 * marker. 1698 */ 1699 @BufferFlag 1700 public int flags; 1701 1702 /** @hide */ 1703 @NonNull dup()1704 public BufferInfo dup() { 1705 BufferInfo copy = new BufferInfo(); 1706 copy.set(offset, size, presentationTimeUs, flags); 1707 return copy; 1708 } 1709 }; 1710 1711 // The follow flag constants MUST stay in sync with their equivalents 1712 // in MediaCodec.h ! 1713 1714 /** 1715 * This indicates that the (encoded) buffer marked as such contains 1716 * the data for a key frame. 1717 * 1718 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1719 */ 1720 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1721 1722 /** 1723 * This indicates that the (encoded) buffer marked as such contains 1724 * the data for a key frame. 1725 */ 1726 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1727 1728 /** 1729 * This indicated that the buffer marked as such contains codec 1730 * initialization / codec specific data instead of media data. 1731 */ 1732 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1733 1734 /** 1735 * This signals the end of stream, i.e. no buffers will be available 1736 * after this, unless of course, {@link #flush} follows. 1737 */ 1738 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1739 1740 /** 1741 * This indicates that the buffer only contains part of a frame, 1742 * and the decoder should batch the data until a buffer without 1743 * this flag appears before decoding the frame. 1744 */ 1745 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1746 1747 /** 1748 * This indicates that the buffer contains non-media data for the 1749 * muxer to process. 1750 * 1751 * All muxer data should start with a FOURCC header that determines the type of data. 1752 * 1753 * For example, when it contains Exif data sent to a MediaMuxer track of 1754 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1755 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1756 * 1757 * @hide 1758 */ 1759 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1760 1761 /** 1762 * This indicates that the buffer is decoded and updates the internal state of the decoder, 1763 * but does not produce any output buffer. 1764 * 1765 * When a buffer has this flag set, 1766 * {@link OnFrameRenderedListener#onFrameRendered(MediaCodec, long, long)} and 1767 * {@link Callback#onOutputBufferAvailable(MediaCodec, int, BufferInfo)} will not be called for 1768 * that given buffer. 1769 * 1770 * For example, when seeking to a certain frame, that frame may need to reference previous 1771 * frames in order for it to produce output. The preceding frames can be marked with this flag 1772 * so that they are only decoded and their data is used when decoding the latter frame that 1773 * should be initially displayed post-seek. 1774 * Another example would be trick play, trick play is when a video is fast-forwarded and only a 1775 * subset of the frames is to be rendered on the screen. The frames not to be rendered can be 1776 * marked with this flag for the same reason as the above one. 1777 * Marking frames with this flag improves the overall performance of playing a video stream as 1778 * fewer frames need to be passed back to the app. 1779 * 1780 * In {@link CodecCapabilities#FEATURE_TunneledPlayback}, buffers marked with this flag 1781 * are not rendered on the output surface. 1782 * 1783 * A frame should not be marked with this flag and {@link #BUFFER_FLAG_END_OF_STREAM} 1784 * simultaneously, doing so will produce a {@link InvalidBufferFlagsException} 1785 */ 1786 public static final int BUFFER_FLAG_DECODE_ONLY = 32; 1787 1788 /** @hide */ 1789 @IntDef( 1790 flag = true, 1791 value = { 1792 BUFFER_FLAG_SYNC_FRAME, 1793 BUFFER_FLAG_KEY_FRAME, 1794 BUFFER_FLAG_CODEC_CONFIG, 1795 BUFFER_FLAG_END_OF_STREAM, 1796 BUFFER_FLAG_PARTIAL_FRAME, 1797 BUFFER_FLAG_MUXER_DATA, 1798 BUFFER_FLAG_DECODE_ONLY, 1799 }) 1800 @Retention(RetentionPolicy.SOURCE) 1801 public @interface BufferFlag {} 1802 1803 private EventHandler mEventHandler; 1804 private EventHandler mOnFirstTunnelFrameReadyHandler; 1805 private EventHandler mOnFrameRenderedHandler; 1806 private EventHandler mCallbackHandler; 1807 private Callback mCallback; 1808 private OnFirstTunnelFrameReadyListener mOnFirstTunnelFrameReadyListener; 1809 private OnFrameRenderedListener mOnFrameRenderedListener; 1810 private final Object mListenerLock = new Object(); 1811 private MediaCodecInfo mCodecInfo; 1812 private final Object mCodecInfoLock = new Object(); 1813 private MediaCrypto mCrypto; 1814 1815 private static final int EVENT_CALLBACK = 1; 1816 private static final int EVENT_SET_CALLBACK = 2; 1817 private static final int EVENT_FRAME_RENDERED = 3; 1818 private static final int EVENT_FIRST_TUNNEL_FRAME_READY = 4; 1819 1820 private static final int CB_INPUT_AVAILABLE = 1; 1821 private static final int CB_OUTPUT_AVAILABLE = 2; 1822 private static final int CB_ERROR = 3; 1823 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1824 private static final String EOS_AND_DECODE_ONLY_ERROR_MESSAGE = "An input buffer cannot have " 1825 + "both BUFFER_FLAG_END_OF_STREAM and BUFFER_FLAG_DECODE_ONLY flags"; 1826 private static final int CB_CRYPTO_ERROR = 6; 1827 1828 private class EventHandler extends Handler { 1829 private MediaCodec mCodec; 1830 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1831 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1832 super(looper); 1833 mCodec = codec; 1834 } 1835 1836 @Override handleMessage(@onNull Message msg)1837 public void handleMessage(@NonNull Message msg) { 1838 switch (msg.what) { 1839 case EVENT_CALLBACK: 1840 { 1841 handleCallback(msg); 1842 break; 1843 } 1844 case EVENT_SET_CALLBACK: 1845 { 1846 mCallback = (MediaCodec.Callback) msg.obj; 1847 break; 1848 } 1849 case EVENT_FRAME_RENDERED: 1850 Map<String, Object> map = (Map<String, Object>)msg.obj; 1851 for (int i = 0; ; ++i) { 1852 Object mediaTimeUs = map.get(i + "-media-time-us"); 1853 Object systemNano = map.get(i + "-system-nano"); 1854 OnFrameRenderedListener onFrameRenderedListener; 1855 synchronized (mListenerLock) { 1856 onFrameRenderedListener = mOnFrameRenderedListener; 1857 } 1858 if (mediaTimeUs == null || systemNano == null 1859 || onFrameRenderedListener == null) { 1860 break; 1861 } 1862 onFrameRenderedListener.onFrameRendered( 1863 mCodec, (long)mediaTimeUs, (long)systemNano); 1864 } 1865 break; 1866 case EVENT_FIRST_TUNNEL_FRAME_READY: 1867 OnFirstTunnelFrameReadyListener onFirstTunnelFrameReadyListener; 1868 synchronized (mListenerLock) { 1869 onFirstTunnelFrameReadyListener = mOnFirstTunnelFrameReadyListener; 1870 } 1871 if (onFirstTunnelFrameReadyListener == null) { 1872 break; 1873 } 1874 onFirstTunnelFrameReadyListener.onFirstTunnelFrameReady(mCodec); 1875 break; 1876 default: 1877 { 1878 break; 1879 } 1880 } 1881 } 1882 handleCallback(@onNull Message msg)1883 private void handleCallback(@NonNull Message msg) { 1884 if (mCallback == null) { 1885 return; 1886 } 1887 1888 switch (msg.arg1) { 1889 case CB_INPUT_AVAILABLE: 1890 { 1891 int index = msg.arg2; 1892 synchronized(mBufferLock) { 1893 switch (mBufferMode) { 1894 case BUFFER_MODE_LEGACY: 1895 validateInputByteBufferLocked(mCachedInputBuffers, index); 1896 break; 1897 case BUFFER_MODE_BLOCK: 1898 while (mQueueRequests.size() <= index) { 1899 mQueueRequests.add(null); 1900 } 1901 QueueRequest request = mQueueRequests.get(index); 1902 if (request == null) { 1903 request = new QueueRequest(mCodec, index); 1904 mQueueRequests.set(index, request); 1905 } 1906 request.setAccessible(true); 1907 break; 1908 default: 1909 throw new IllegalStateException( 1910 "Unrecognized buffer mode: " + mBufferMode); 1911 } 1912 } 1913 mCallback.onInputBufferAvailable(mCodec, index); 1914 break; 1915 } 1916 1917 case CB_OUTPUT_AVAILABLE: 1918 { 1919 int index = msg.arg2; 1920 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1921 synchronized(mBufferLock) { 1922 switch (mBufferMode) { 1923 case BUFFER_MODE_LEGACY: 1924 validateOutputByteBufferLocked(mCachedOutputBuffers, index, info); 1925 break; 1926 case BUFFER_MODE_BLOCK: 1927 while (mOutputFrames.size() <= index) { 1928 mOutputFrames.add(null); 1929 } 1930 OutputFrame frame = mOutputFrames.get(index); 1931 if (frame == null) { 1932 frame = new OutputFrame(index); 1933 mOutputFrames.set(index, frame); 1934 } 1935 frame.setBufferInfo(info); 1936 frame.setAccessible(true); 1937 break; 1938 default: 1939 throw new IllegalStateException( 1940 "Unrecognized buffer mode: " + mBufferMode); 1941 } 1942 } 1943 mCallback.onOutputBufferAvailable( 1944 mCodec, index, info); 1945 break; 1946 } 1947 1948 case CB_ERROR: 1949 { 1950 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1951 break; 1952 } 1953 1954 case CB_CRYPTO_ERROR: 1955 { 1956 mCallback.onCryptoError(mCodec, (MediaCodec.CryptoException) msg.obj); 1957 break; 1958 } 1959 1960 case CB_OUTPUT_FORMAT_CHANGE: 1961 { 1962 mCallback.onOutputFormatChanged(mCodec, 1963 new MediaFormat((Map<String, Object>) msg.obj)); 1964 break; 1965 } 1966 1967 default: 1968 { 1969 break; 1970 } 1971 } 1972 } 1973 } 1974 1975 private boolean mHasSurface = false; 1976 1977 /** 1978 * Instantiate the preferred decoder supporting input data of the given mime type. 1979 * 1980 * The following is a partial list of defined mime types and their semantics: 1981 * <ul> 1982 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1983 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1984 * <li>"video/avc" - H.264/AVC video 1985 * <li>"video/hevc" - H.265/HEVC video 1986 * <li>"video/mp4v-es" - MPEG4 video 1987 * <li>"video/3gpp" - H.263 video 1988 * <li>"audio/3gpp" - AMR narrowband audio 1989 * <li>"audio/amr-wb" - AMR wideband audio 1990 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1991 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1992 * <li>"audio/vorbis" - vorbis audio 1993 * <li>"audio/g711-alaw" - G.711 alaw audio 1994 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1995 * </ul> 1996 * 1997 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1998 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1999 * given format. 2000 * 2001 * @param type The mime type of the input data. 2002 * @throws IOException if the codec cannot be created. 2003 * @throws IllegalArgumentException if type is not a valid mime type. 2004 * @throws NullPointerException if type is null. 2005 */ 2006 @NonNull createDecoderByType(@onNull String type)2007 public static MediaCodec createDecoderByType(@NonNull String type) 2008 throws IOException { 2009 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 2010 } 2011 2012 /** 2013 * Instantiate the preferred encoder supporting output data of the given mime type. 2014 * 2015 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 2016 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 2017 * given format. 2018 * 2019 * @param type The desired mime type of the output data. 2020 * @throws IOException if the codec cannot be created. 2021 * @throws IllegalArgumentException if type is not a valid mime type. 2022 * @throws NullPointerException if type is null. 2023 */ 2024 @NonNull createEncoderByType(@onNull String type)2025 public static MediaCodec createEncoderByType(@NonNull String type) 2026 throws IOException { 2027 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 2028 } 2029 2030 /** 2031 * If you know the exact name of the component you want to instantiate 2032 * use this method to instantiate it. Use with caution. 2033 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 2034 * @param name The name of the codec to be instantiated. 2035 * @throws IOException if the codec cannot be created. 2036 * @throws IllegalArgumentException if name is not valid. 2037 * @throws NullPointerException if name is null. 2038 */ 2039 @NonNull createByCodecName(@onNull String name)2040 public static MediaCodec createByCodecName(@NonNull String name) 2041 throws IOException { 2042 return new MediaCodec(name, false /* nameIsType */, false /* encoder */); 2043 } 2044 2045 /** 2046 * This is the same as createByCodecName, but allows for instantiating a codec on behalf of a 2047 * client process. This is used for system apps or system services that create MediaCodecs on 2048 * behalf of other processes and will reclaim resources as necessary from processes with lower 2049 * priority than the client process, rather than processes with lower priority than the system 2050 * app or system service. Likely to be used with information obtained from 2051 * {@link android.media.MediaCodecList}. 2052 * @param name 2053 * @param clientPid 2054 * @param clientUid 2055 * @throws IOException if the codec cannot be created. 2056 * @throws IllegalArgumentException if name is not valid. 2057 * @throws NullPointerException if name is null. 2058 * @throws SecurityException if the MEDIA_RESOURCE_OVERRIDE_PID permission is not granted. 2059 * 2060 * @hide 2061 */ 2062 @NonNull 2063 @SystemApi 2064 @RequiresPermission(Manifest.permission.MEDIA_RESOURCE_OVERRIDE_PID) createByCodecNameForClient(@onNull String name, int clientPid, int clientUid)2065 public static MediaCodec createByCodecNameForClient(@NonNull String name, int clientPid, 2066 int clientUid) throws IOException { 2067 return new MediaCodec(name, false /* nameIsType */, false /* encoder */, clientPid, 2068 clientUid); 2069 } 2070 MediaCodec(@onNull String name, boolean nameIsType, boolean encoder)2071 private MediaCodec(@NonNull String name, boolean nameIsType, boolean encoder) { 2072 this(name, nameIsType, encoder, -1 /* pid */, -1 /* uid */); 2073 } 2074 MediaCodec(@onNull String name, boolean nameIsType, boolean encoder, int pid, int uid)2075 private MediaCodec(@NonNull String name, boolean nameIsType, boolean encoder, int pid, 2076 int uid) { 2077 Looper looper; 2078 if ((looper = Looper.myLooper()) != null) { 2079 mEventHandler = new EventHandler(this, looper); 2080 } else if ((looper = Looper.getMainLooper()) != null) { 2081 mEventHandler = new EventHandler(this, looper); 2082 } else { 2083 mEventHandler = null; 2084 } 2085 mCallbackHandler = mEventHandler; 2086 mOnFirstTunnelFrameReadyHandler = mEventHandler; 2087 mOnFrameRenderedHandler = mEventHandler; 2088 2089 mBufferLock = new Object(); 2090 2091 // save name used at creation 2092 mNameAtCreation = nameIsType ? null : name; 2093 2094 native_setup(name, nameIsType, encoder, pid, uid); 2095 } 2096 2097 private String mNameAtCreation; 2098 2099 @Override finalize()2100 protected void finalize() { 2101 native_finalize(); 2102 mCrypto = null; 2103 } 2104 2105 /** 2106 * Returns the codec to its initial (Uninitialized) state. 2107 * 2108 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 2109 * error has occured to reset the codec to its initial state after creation. 2110 * 2111 * @throws CodecException if an unrecoverable error has occured and the codec 2112 * could not be reset. 2113 * @throws IllegalStateException if in the Released state. 2114 */ reset()2115 public final void reset() { 2116 freeAllTrackedBuffers(); // free buffers first 2117 native_reset(); 2118 mCrypto = null; 2119 } 2120 native_reset()2121 private native final void native_reset(); 2122 2123 /** 2124 * Free up resources used by the codec instance. 2125 * 2126 * Make sure you call this when you're done to free up any opened 2127 * component instance instead of relying on the garbage collector 2128 * to do this for you at some point in the future. 2129 */ release()2130 public final void release() { 2131 freeAllTrackedBuffers(); // free buffers first 2132 native_release(); 2133 mCrypto = null; 2134 } 2135 native_release()2136 private native final void native_release(); 2137 2138 /** 2139 * If this codec is to be used as an encoder, pass this flag. 2140 */ 2141 public static final int CONFIGURE_FLAG_ENCODE = 1; 2142 2143 /** 2144 * If this codec is to be used with {@link LinearBlock} and/or {@link 2145 * HardwareBuffer}, pass this flag. 2146 * <p> 2147 * When this flag is set, the following APIs throw {@link IncompatibleWithBlockModelException}. 2148 * <ul> 2149 * <li>{@link #getInputBuffer} 2150 * <li>{@link #getInputImage} 2151 * <li>{@link #getInputBuffers} 2152 * <li>{@link #getOutputBuffer} 2153 * <li>{@link #getOutputImage} 2154 * <li>{@link #getOutputBuffers} 2155 * <li>{@link #queueInputBuffer} 2156 * <li>{@link #queueSecureInputBuffer} 2157 * <li>{@link #dequeueInputBuffer} 2158 * <li>{@link #dequeueOutputBuffer} 2159 * </ul> 2160 */ 2161 public static final int CONFIGURE_FLAG_USE_BLOCK_MODEL = 2; 2162 2163 /** 2164 * This flag should be used on a secure decoder only. MediaCodec configured with this 2165 * flag does decryption in a separate thread. The flag requires MediaCodec to operate 2166 * asynchronously and will throw CryptoException if any, in the onCryptoError() 2167 * callback. Applications should override the default implementation of 2168 * onCryptoError() and access the associated CryptoException. 2169 * 2170 * CryptoException thrown will contain {@link MediaCodec.CryptoInfo} 2171 * This can be accessed using getCryptoInfo() 2172 */ 2173 public static final int CONFIGURE_FLAG_USE_CRYPTO_ASYNC = 4; 2174 2175 /** @hide */ 2176 @IntDef( 2177 flag = true, 2178 value = { 2179 CONFIGURE_FLAG_ENCODE, 2180 CONFIGURE_FLAG_USE_BLOCK_MODEL, 2181 CONFIGURE_FLAG_USE_CRYPTO_ASYNC, 2182 }) 2183 @Retention(RetentionPolicy.SOURCE) 2184 public @interface ConfigureFlag {} 2185 2186 /** 2187 * Thrown when the codec is configured for block model and an incompatible API is called. 2188 */ 2189 public class IncompatibleWithBlockModelException extends RuntimeException { IncompatibleWithBlockModelException()2190 IncompatibleWithBlockModelException() { } 2191 IncompatibleWithBlockModelException(String message)2192 IncompatibleWithBlockModelException(String message) { 2193 super(message); 2194 } 2195 IncompatibleWithBlockModelException(String message, Throwable cause)2196 IncompatibleWithBlockModelException(String message, Throwable cause) { 2197 super(message, cause); 2198 } 2199 IncompatibleWithBlockModelException(Throwable cause)2200 IncompatibleWithBlockModelException(Throwable cause) { 2201 super(cause); 2202 } 2203 } 2204 2205 /** 2206 * Thrown when a buffer is marked with an invalid combination of flags 2207 * (e.g. both {@link #BUFFER_FLAG_END_OF_STREAM} and {@link #BUFFER_FLAG_DECODE_ONLY}) 2208 */ 2209 public class InvalidBufferFlagsException extends RuntimeException { InvalidBufferFlagsException(String message)2210 InvalidBufferFlagsException(String message) { 2211 super(message); 2212 } 2213 } 2214 2215 /** 2216 * Configures a component. 2217 * 2218 * @param format The format of the input data (decoder) or the desired 2219 * format of the output data (encoder). Passing {@code null} 2220 * as {@code format} is equivalent to passing an 2221 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2222 * @param surface Specify a surface on which to render the output of this 2223 * decoder. Pass {@code null} as {@code surface} if the 2224 * codec does not generate raw video output (e.g. not a video 2225 * decoder) and/or if you want to configure the codec for 2226 * {@link ByteBuffer} output. 2227 * @param crypto Specify a crypto object to facilitate secure decryption 2228 * of the media data. Pass {@code null} as {@code crypto} for 2229 * non-secure codecs. 2230 * Please note that {@link MediaCodec} does NOT take ownership 2231 * of the {@link MediaCrypto} object; it is the application's 2232 * responsibility to properly cleanup the {@link MediaCrypto} object 2233 * when not in use. 2234 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2235 * component as an encoder. 2236 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2237 * or the format is unacceptable (e.g. missing a mandatory key), 2238 * or the flags are not set properly 2239 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2240 * @throws IllegalStateException if not in the Uninitialized state. 2241 * @throws CryptoException upon DRM error. 2242 * @throws CodecException upon codec error. 2243 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)2244 public void configure( 2245 @Nullable MediaFormat format, 2246 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2247 @ConfigureFlag int flags) { 2248 configure(format, surface, crypto, null, flags); 2249 } 2250 2251 /** 2252 * Configure a component to be used with a descrambler. 2253 * @param format The format of the input data (decoder) or the desired 2254 * format of the output data (encoder). Passing {@code null} 2255 * as {@code format} is equivalent to passing an 2256 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2257 * @param surface Specify a surface on which to render the output of this 2258 * decoder. Pass {@code null} as {@code surface} if the 2259 * codec does not generate raw video output (e.g. not a video 2260 * decoder) and/or if you want to configure the codec for 2261 * {@link ByteBuffer} output. 2262 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2263 * component as an encoder. 2264 * @param descrambler Specify a descrambler object to facilitate secure 2265 * descrambling of the media data, or null for non-secure codecs. 2266 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2267 * or the format is unacceptable (e.g. missing a mandatory key), 2268 * or the flags are not set properly 2269 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2270 * @throws IllegalStateException if not in the Uninitialized state. 2271 * @throws CryptoException upon DRM error. 2272 * @throws CodecException upon codec error. 2273 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)2274 public void configure( 2275 @Nullable MediaFormat format, @Nullable Surface surface, 2276 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 2277 configure(format, surface, null, 2278 descrambler != null ? descrambler.getBinder() : null, flags); 2279 } 2280 2281 private static final int BUFFER_MODE_INVALID = -1; 2282 private static final int BUFFER_MODE_LEGACY = 0; 2283 private static final int BUFFER_MODE_BLOCK = 1; 2284 private int mBufferMode = BUFFER_MODE_INVALID; 2285 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2286 private void configure( 2287 @Nullable MediaFormat format, @Nullable Surface surface, 2288 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 2289 @ConfigureFlag int flags) { 2290 if (crypto != null && descramblerBinder != null) { 2291 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 2292 } 2293 2294 String[] keys = null; 2295 Object[] values = null; 2296 2297 if (format != null) { 2298 Map<String, Object> formatMap = format.getMap(); 2299 keys = new String[formatMap.size()]; 2300 values = new Object[formatMap.size()]; 2301 2302 int i = 0; 2303 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 2304 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 2305 int sessionId = 0; 2306 try { 2307 sessionId = (Integer)entry.getValue(); 2308 } 2309 catch (Exception e) { 2310 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 2311 } 2312 keys[i] = "audio-hw-sync"; 2313 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 2314 } else { 2315 keys[i] = entry.getKey(); 2316 values[i] = entry.getValue(); 2317 } 2318 ++i; 2319 } 2320 } 2321 2322 mHasSurface = surface != null; 2323 mCrypto = crypto; 2324 synchronized (mBufferLock) { 2325 if ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) != 0) { 2326 mBufferMode = BUFFER_MODE_BLOCK; 2327 } else { 2328 mBufferMode = BUFFER_MODE_LEGACY; 2329 } 2330 } 2331 2332 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 2333 } 2334 2335 /** 2336 * Dynamically sets the output surface of a codec. 2337 * <p> 2338 * This can only be used if the codec was configured with an output surface. The 2339 * new output surface should have a compatible usage type to the original output surface. 2340 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 2341 * to ImageReader (software readable) output. 2342 * @param surface the output surface to use. It must not be {@code null}. 2343 * @throws IllegalStateException if the codec does not support setting the output 2344 * surface in the current state. 2345 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 2346 */ setOutputSurface(@onNull Surface surface)2347 public void setOutputSurface(@NonNull Surface surface) { 2348 if (!mHasSurface) { 2349 throw new IllegalStateException("codec was not configured for an output surface"); 2350 } 2351 native_setSurface(surface); 2352 } 2353 native_setSurface(@onNull Surface surface)2354 private native void native_setSurface(@NonNull Surface surface); 2355 2356 /** 2357 * Create a persistent input surface that can be used with codecs that normally have an input 2358 * surface, such as video encoders. A persistent input can be reused by subsequent 2359 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 2360 * most one codec or recorder instance concurrently. 2361 * <p> 2362 * The application is responsible for calling release() on the Surface when done. 2363 * 2364 * @return an input surface that can be used with {@link #setInputSurface}. 2365 */ 2366 @NonNull createPersistentInputSurface()2367 public static Surface createPersistentInputSurface() { 2368 return native_createPersistentInputSurface(); 2369 } 2370 2371 static class PersistentSurface extends Surface { 2372 @SuppressWarnings("unused") PersistentSurface()2373 PersistentSurface() {} // used by native 2374 2375 @Override release()2376 public void release() { 2377 native_releasePersistentInputSurface(this); 2378 super.release(); 2379 } 2380 2381 private long mPersistentObject; 2382 }; 2383 2384 /** 2385 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2386 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2387 * lieu of {@link #createInputSurface}. 2388 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2389 * @throws IllegalStateException if not in the Configured state or does not require an input 2390 * surface. 2391 * @throws IllegalArgumentException if the surface was not created by 2392 * {@link #createPersistentInputSurface}. 2393 */ setInputSurface(@onNull Surface surface)2394 public void setInputSurface(@NonNull Surface surface) { 2395 if (!(surface instanceof PersistentSurface)) { 2396 throw new IllegalArgumentException("not a PersistentSurface"); 2397 } 2398 native_setInputSurface(surface); 2399 } 2400 2401 @NonNull native_createPersistentInputSurface()2402 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2403 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2404 private native final void native_setInputSurface(@NonNull Surface surface); 2405 native_setCallback(@ullable Callback cb)2406 private native final void native_setCallback(@Nullable Callback cb); 2407 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2408 private native final void native_configure( 2409 @Nullable String[] keys, @Nullable Object[] values, 2410 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2411 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2412 2413 /** 2414 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2415 * may only be called after {@link #configure} and before {@link #start}. 2416 * <p> 2417 * The application is responsible for calling release() on the Surface when 2418 * done. 2419 * <p> 2420 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2421 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2422 * unexpected results. 2423 * @throws IllegalStateException if not in the Configured state. 2424 */ 2425 @NonNull createInputSurface()2426 public native final Surface createInputSurface(); 2427 2428 /** 2429 * After successfully configuring the component, call {@code start}. 2430 * <p> 2431 * Call {@code start} also if the codec is configured in asynchronous mode, 2432 * and it has just been flushed, to resume requesting input buffers. 2433 * @throws IllegalStateException if not in the Configured state 2434 * or just after {@link #flush} for a codec that is configured 2435 * in asynchronous mode. 2436 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2437 * for start may be attributed to future method calls. 2438 */ start()2439 public final void start() { 2440 native_start(); 2441 } native_start()2442 private native final void native_start(); 2443 2444 /** 2445 * Finish the decode/encode session, note that the codec instance 2446 * remains active and ready to be {@link #start}ed again. 2447 * To ensure that it is available to other client call {@link #release} 2448 * and don't just rely on garbage collection to eventually do this for you. 2449 * @throws IllegalStateException if in the Released state. 2450 */ stop()2451 public final void stop() { 2452 native_stop(); 2453 freeAllTrackedBuffers(); 2454 2455 synchronized (mListenerLock) { 2456 if (mCallbackHandler != null) { 2457 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2458 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2459 } 2460 if (mOnFirstTunnelFrameReadyHandler != null) { 2461 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 2462 } 2463 if (mOnFrameRenderedHandler != null) { 2464 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2465 } 2466 } 2467 } 2468 native_stop()2469 private native final void native_stop(); 2470 2471 /** 2472 * Flush both input and output ports of the component. 2473 * <p> 2474 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2475 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2476 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2477 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2478 * invalid, and all buffers are owned by the codec. 2479 * <p> 2480 * If the codec is configured in asynchronous mode, call {@link #start} 2481 * after {@code flush} has returned to resume codec operations. The codec 2482 * will not request input buffers until this has happened. 2483 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2484 * callbacks that were not handled prior to calling {@code flush}. 2485 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2486 * should be discarded.</strong> 2487 * <p> 2488 * If the codec is configured in synchronous mode, codec will resume 2489 * automatically if it is configured with an input surface. Otherwise, it 2490 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2491 * 2492 * @throws IllegalStateException if not in the Executing state. 2493 * @throws MediaCodec.CodecException upon codec error. 2494 */ flush()2495 public final void flush() { 2496 synchronized(mBufferLock) { 2497 invalidateByteBuffersLocked(mCachedInputBuffers); 2498 invalidateByteBuffersLocked(mCachedOutputBuffers); 2499 mValidInputIndices.clear(); 2500 mValidOutputIndices.clear(); 2501 mDequeuedInputBuffers.clear(); 2502 mDequeuedOutputBuffers.clear(); 2503 } 2504 native_flush(); 2505 } 2506 native_flush()2507 private native final void native_flush(); 2508 2509 /** 2510 * Thrown when an internal codec error occurs. 2511 */ 2512 public final static class CodecException extends IllegalStateException { 2513 @UnsupportedAppUsage CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2514 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2515 super(detailMessage); 2516 mErrorCode = errorCode; 2517 mActionCode = actionCode; 2518 2519 // TODO get this from codec 2520 final String sign = errorCode < 0 ? "neg_" : ""; 2521 mDiagnosticInfo = 2522 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2523 } 2524 2525 /** 2526 * Returns true if the codec exception is a transient issue, 2527 * perhaps due to resource constraints, and that the method 2528 * (or encoding/decoding) may be retried at a later time. 2529 */ 2530 public boolean isTransient() { 2531 return mActionCode == ACTION_TRANSIENT; 2532 } 2533 2534 /** 2535 * Returns true if the codec cannot proceed further, 2536 * but can be recovered by stopping, configuring, 2537 * and starting again. 2538 */ 2539 public boolean isRecoverable() { 2540 return mActionCode == ACTION_RECOVERABLE; 2541 } 2542 2543 /** 2544 * Retrieve the error code associated with a CodecException 2545 */ 2546 public int getErrorCode() { 2547 return mErrorCode; 2548 } 2549 2550 /** 2551 * Retrieve a developer-readable diagnostic information string 2552 * associated with the exception. Do not show this to end-users, 2553 * since this string will not be localized or generally 2554 * comprehensible to end-users. 2555 */ 2556 public @NonNull String getDiagnosticInfo() { 2557 return mDiagnosticInfo; 2558 } 2559 2560 /** 2561 * This indicates required resource was not able to be allocated. 2562 */ 2563 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2564 2565 /** 2566 * This indicates the resource manager reclaimed the media resource used by the codec. 2567 * <p> 2568 * With this exception, the codec must be released, as it has moved to terminal state. 2569 */ 2570 public static final int ERROR_RECLAIMED = 1101; 2571 2572 /** @hide */ 2573 @IntDef({ 2574 ERROR_INSUFFICIENT_RESOURCE, 2575 ERROR_RECLAIMED, 2576 }) 2577 @Retention(RetentionPolicy.SOURCE) 2578 public @interface ReasonCode {} 2579 2580 /* Must be in sync with android_media_MediaCodec.cpp */ 2581 private final static int ACTION_TRANSIENT = 1; 2582 private final static int ACTION_RECOVERABLE = 2; 2583 2584 private final String mDiagnosticInfo; 2585 private final int mErrorCode; 2586 private final int mActionCode; 2587 } 2588 2589 /** 2590 * Thrown when a crypto error occurs while queueing a secure input buffer. 2591 */ 2592 public final static class CryptoException extends RuntimeException 2593 implements MediaDrmThrowable { 2594 public CryptoException(int errorCode, @Nullable String detailMessage) { 2595 this(detailMessage, errorCode, 0, 0, 0, null); 2596 } 2597 2598 /** 2599 * @hide 2600 */ 2601 public CryptoException(String message, int errorCode, int vendorError, int oemError, 2602 int errorContext, @Nullable CryptoInfo cryptoInfo) { 2603 super(message); 2604 mErrorCode = errorCode; 2605 mVendorError = vendorError; 2606 mOemError = oemError; 2607 mErrorContext = errorContext; 2608 mCryptoInfo = cryptoInfo; 2609 } 2610 2611 /** 2612 * This indicates that the requested key was not found when trying to 2613 * perform a decrypt operation. The operation can be retried after adding 2614 * the correct decryption key. 2615 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_NO_KEY}. 2616 */ 2617 public static final int ERROR_NO_KEY = MediaDrm.ErrorCodes.ERROR_NO_KEY; 2618 2619 /** 2620 * This indicates that the key used for decryption is no longer 2621 * valid due to license term expiration. The operation can be retried 2622 * after updating the expired keys. 2623 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_KEY_EXPIRED}. 2624 */ 2625 public static final int ERROR_KEY_EXPIRED = MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED; 2626 2627 /** 2628 * This indicates that a required crypto resource was not able to be 2629 * allocated while attempting the requested operation. The operation 2630 * can be retried if the app is able to release resources. 2631 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_RESOURCE_BUSY} 2632 */ 2633 public static final int ERROR_RESOURCE_BUSY = MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY; 2634 2635 /** 2636 * This indicates that the output protection levels supported by the 2637 * device are not sufficient to meet the requirements set by the 2638 * content owner in the license policy. 2639 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_OUTPUT_PROTECTION} 2640 */ 2641 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 2642 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION; 2643 2644 /** 2645 * This indicates that decryption was attempted on a session that is 2646 * not opened, which could be due to a failure to open the session, 2647 * closing the session prematurely, or the session being reclaimed 2648 * by the resource manager. 2649 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_SESSION_NOT_OPENED} 2650 */ 2651 public static final int ERROR_SESSION_NOT_OPENED = 2652 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED; 2653 2654 /** 2655 * This indicates that an operation was attempted that could not be 2656 * supported by the crypto system of the device in its current 2657 * configuration. It may occur when the license policy requires 2658 * device security features that aren't supported by the device, 2659 * or due to an internal error in the crypto system that prevents 2660 * the specified security policy from being met. 2661 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_UNSUPPORTED_OPERATION} 2662 */ 2663 public static final int ERROR_UNSUPPORTED_OPERATION = 2664 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION; 2665 2666 /** 2667 * This indicates that the security level of the device is not 2668 * sufficient to meet the requirements set by the content owner 2669 * in the license policy. 2670 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_SECURITY} 2671 */ 2672 public static final int ERROR_INSUFFICIENT_SECURITY = 2673 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY; 2674 2675 /** 2676 * This indicates that the video frame being decrypted exceeds 2677 * the size of the device's protected output buffers. When 2678 * encountering this error the app should try playing content 2679 * of a lower resolution. 2680 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_FRAME_TOO_LARGE} 2681 */ 2682 public static final int ERROR_FRAME_TOO_LARGE = MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE; 2683 2684 /** 2685 * This error indicates that session state has been 2686 * invalidated. It can occur on devices that are not capable 2687 * of retaining crypto session state across device 2688 * suspend/resume. The session must be closed and a new 2689 * session opened to resume operation. 2690 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_LOST_STATE} 2691 */ 2692 public static final int ERROR_LOST_STATE = MediaDrm.ErrorCodes.ERROR_LOST_STATE; 2693 2694 /** @hide */ 2695 @IntDef({ 2696 MediaDrm.ErrorCodes.ERROR_NO_KEY, 2697 MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED, 2698 MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY, 2699 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2700 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED, 2701 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION, 2702 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY, 2703 MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE, 2704 MediaDrm.ErrorCodes.ERROR_LOST_STATE, 2705 MediaDrm.ErrorCodes.ERROR_GENERIC_OEM, 2706 MediaDrm.ErrorCodes.ERROR_GENERIC_PLUGIN, 2707 MediaDrm.ErrorCodes.ERROR_LICENSE_PARSE, 2708 MediaDrm.ErrorCodes.ERROR_MEDIA_FRAMEWORK, 2709 MediaDrm.ErrorCodes.ERROR_ZERO_SUBSAMPLES 2710 }) 2711 @Retention(RetentionPolicy.SOURCE) 2712 public @interface CryptoErrorCode {} 2713 2714 /** 2715 * Returns error code associated with this {@link CryptoException}. 2716 * <p> 2717 * Please refer to {@link MediaDrm.ErrorCodes} for the general error 2718 * handling strategy and details about each possible return value. 2719 * 2720 * @return an error code defined in {@link MediaDrm.ErrorCodes}. 2721 */ 2722 @CryptoErrorCode 2723 public int getErrorCode() { 2724 return mErrorCode; 2725 } 2726 2727 /** 2728 * Returns CryptoInfo associated with this {@link CryptoException} 2729 * if any 2730 * 2731 * @return CryptoInfo object if any. {@link MediaCodec.CryptoException} 2732 */ 2733 public @Nullable CryptoInfo getCryptoInfo() { 2734 return mCryptoInfo; 2735 } 2736 2737 @Override 2738 public int getVendorError() { 2739 return mVendorError; 2740 } 2741 2742 @Override 2743 public int getOemError() { 2744 return mOemError; 2745 } 2746 2747 @Override 2748 public int getErrorContext() { 2749 return mErrorContext; 2750 } 2751 2752 private final int mErrorCode, mVendorError, mOemError, mErrorContext; 2753 private CryptoInfo mCryptoInfo; 2754 } 2755 2756 /** 2757 * After filling a range of the input buffer at the specified index 2758 * submit it to the component. Once an input buffer is queued to 2759 * the codec, it MUST NOT be used until it is later retrieved by 2760 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2761 * return value or a {@link Callback#onInputBufferAvailable} 2762 * callback. 2763 * <p> 2764 * Many decoders require the actual compressed data stream to be 2765 * preceded by "codec specific data", i.e. setup data used to initialize 2766 * the codec such as PPS/SPS in the case of AVC video or code tables 2767 * in the case of vorbis audio. 2768 * The class {@link android.media.MediaExtractor} provides codec 2769 * specific data as part of 2770 * the returned track format in entries named "csd-0", "csd-1" ... 2771 * <p> 2772 * These buffers can be submitted directly after {@link #start} or 2773 * {@link #flush} by specifying the flag {@link 2774 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2775 * codec with a {@link MediaFormat} containing these keys, they 2776 * will be automatically submitted by MediaCodec directly after 2777 * start. Therefore, the use of {@link 2778 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2779 * recommended only for advanced users. 2780 * <p> 2781 * To indicate that this is the final piece of input data (or rather that 2782 * no more input data follows unless the decoder is subsequently flushed) 2783 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2784 * <p class=note> 2785 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2786 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2787 * Surface output buffers, and the resulting frame timestamp was undefined. 2788 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2789 * Similarly, since frame timestamps can be used by the destination surface for rendering 2790 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2791 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2792 * SurfaceView specifics}).</strong> 2793 * 2794 * @param index The index of a client-owned input buffer previously returned 2795 * in a call to {@link #dequeueInputBuffer}. 2796 * @param offset The byte offset into the input buffer at which the data starts. 2797 * @param size The number of bytes of valid input data. 2798 * @param presentationTimeUs The presentation timestamp in microseconds for this 2799 * buffer. This is normally the media time at which this 2800 * buffer should be presented (rendered). When using an output 2801 * surface, this will be propagated as the {@link 2802 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2803 * conversion to nanoseconds). 2804 * @param flags A bitmask of flags 2805 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2806 * While not prohibited, most codecs do not use the 2807 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2808 * @throws IllegalStateException if not in the Executing state. 2809 * @throws MediaCodec.CodecException upon codec error. 2810 * @throws CryptoException if a crypto object has been specified in 2811 * {@link #configure} 2812 */ 2813 public final void queueInputBuffer( 2814 int index, 2815 int offset, int size, long presentationTimeUs, int flags) 2816 throws CryptoException { 2817 if ((flags & BUFFER_FLAG_DECODE_ONLY) != 0 2818 && (flags & BUFFER_FLAG_END_OF_STREAM) != 0) { 2819 throw new InvalidBufferFlagsException(EOS_AND_DECODE_ONLY_ERROR_MESSAGE); 2820 } 2821 synchronized(mBufferLock) { 2822 if (mBufferMode == BUFFER_MODE_BLOCK) { 2823 throw new IncompatibleWithBlockModelException("queueInputBuffer() " 2824 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2825 + "Please use getQueueRequest() to queue buffers"); 2826 } 2827 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 2828 mDequeuedInputBuffers.remove(index); 2829 } 2830 try { 2831 native_queueInputBuffer( 2832 index, offset, size, presentationTimeUs, flags); 2833 } catch (CryptoException | IllegalStateException e) { 2834 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 2835 throw e; 2836 } 2837 } 2838 2839 private native final void native_queueInputBuffer( 2840 int index, 2841 int offset, int size, long presentationTimeUs, int flags) 2842 throws CryptoException; 2843 2844 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2845 public static final int CRYPTO_MODE_AES_CTR = 1; 2846 public static final int CRYPTO_MODE_AES_CBC = 2; 2847 2848 /** 2849 * Metadata describing the structure of an encrypted input sample. 2850 * <p> 2851 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 2852 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 2853 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 2854 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 2855 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 2856 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 2857 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 2858 * <p> 2859 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 2860 * "Common encryption in ISO base media file format files". 2861 * <p> 2862 * <h3>ISO-CENC Schemes</h3> 2863 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 2864 * corresponding to each possible combination of an AES mode with the presence or absence of 2865 * patterned encryption. 2866 * 2867 * <table style="width: 0%"> 2868 * <thead> 2869 * <tr> 2870 * <th> </th> 2871 * <th>AES-CTR</th> 2872 * <th>AES-CBC</th> 2873 * </tr> 2874 * </thead> 2875 * <tbody> 2876 * <tr> 2877 * <th>Without Patterns</th> 2878 * <td>cenc</td> 2879 * <td>cbc1</td> 2880 * </tr><tr> 2881 * <th>With Patterns</th> 2882 * <td>cens</td> 2883 * <td>cbcs</td> 2884 * </tr> 2885 * </tbody> 2886 * </table> 2887 * 2888 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 2889 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 2890 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 2891 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 2892 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 2893 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 2894 * {@link #setPattern} is never called is all zeroes. 2895 * <p> 2896 * <h4>HLS SAMPLE-AES Audio</h4> 2897 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 2898 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 2899 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 2900 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 2901 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 2902 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 2903 * while still decrypting every block. 2904 */ 2905 public final static class CryptoInfo { 2906 /** 2907 * The number of subSamples that make up the buffer's contents. 2908 */ 2909 public int numSubSamples; 2910 /** 2911 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 2912 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 2913 */ 2914 public int[] numBytesOfClearData; 2915 /** 2916 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 2917 * as clear and {@link #numBytesOfClearData} must be specified. 2918 */ 2919 public int[] numBytesOfEncryptedData; 2920 /** 2921 * A 16-byte key id 2922 */ 2923 public byte[] key; 2924 /** 2925 * A 16-byte initialization vector 2926 */ 2927 public byte[] iv; 2928 /** 2929 * The type of encryption that has been applied, 2930 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2931 * and {@link #CRYPTO_MODE_AES_CBC} 2932 */ 2933 public int mode; 2934 2935 /** 2936 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 2937 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 2938 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 2939 */ 2940 public final static class Pattern { 2941 /** 2942 * Number of blocks to be encrypted in the pattern. If both this and 2943 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 2944 */ 2945 private int mEncryptBlocks; 2946 2947 /** 2948 * Number of blocks to be skipped (left clear) in the pattern. If both this and 2949 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 2950 */ 2951 private int mSkipBlocks; 2952 2953 /** 2954 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 2955 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 2956 */ 2957 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2958 set(blocksToEncrypt, blocksToSkip); 2959 } 2960 2961 /** 2962 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 2963 * parameters are zero, pattern encryption is inoperative. 2964 */ 2965 public void set(int blocksToEncrypt, int blocksToSkip) { 2966 mEncryptBlocks = blocksToEncrypt; 2967 mSkipBlocks = blocksToSkip; 2968 } 2969 2970 /** 2971 * Return the number of blocks to skip in a sample encryption pattern. 2972 */ 2973 public int getSkipBlocks() { 2974 return mSkipBlocks; 2975 } 2976 2977 /** 2978 * Return the number of blocks to encrypt in a sample encryption pattern. 2979 */ 2980 public int getEncryptBlocks() { 2981 return mEncryptBlocks; 2982 } 2983 }; 2984 2985 private static final Pattern ZERO_PATTERN = new Pattern(0, 0); 2986 2987 /** 2988 * The pattern applicable to the protected data in each subsample. 2989 */ 2990 private Pattern mPattern = ZERO_PATTERN; 2991 2992 /** 2993 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2994 * a {@link MediaCodec.CryptoInfo} instance. 2995 */ 2996 public void set( 2997 int newNumSubSamples, 2998 @NonNull int[] newNumBytesOfClearData, 2999 @NonNull int[] newNumBytesOfEncryptedData, 3000 @NonNull byte[] newKey, 3001 @NonNull byte[] newIV, 3002 int newMode) { 3003 numSubSamples = newNumSubSamples; 3004 numBytesOfClearData = newNumBytesOfClearData; 3005 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 3006 key = newKey; 3007 iv = newIV; 3008 mode = newMode; 3009 mPattern = ZERO_PATTERN; 3010 } 3011 3012 /** 3013 * Returns the {@link Pattern encryption pattern}. 3014 */ 3015 public @NonNull Pattern getPattern() { 3016 return new Pattern(mPattern.getEncryptBlocks(), mPattern.getSkipBlocks()); 3017 } 3018 3019 /** 3020 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 3021 * See {@link Pattern}. 3022 */ 3023 public void setPattern(Pattern newPattern) { 3024 if (newPattern == null) { 3025 newPattern = ZERO_PATTERN; 3026 } 3027 setPattern(newPattern.getEncryptBlocks(), newPattern.getSkipBlocks()); 3028 } 3029 3030 // Accessed from android_media_MediaExtractor.cpp. 3031 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 3032 mPattern = new Pattern(blocksToEncrypt, blocksToSkip); 3033 } 3034 3035 @Override 3036 public String toString() { 3037 StringBuilder builder = new StringBuilder(); 3038 builder.append(numSubSamples + " subsamples, key ["); 3039 String hexdigits = "0123456789abcdef"; 3040 for (int i = 0; i < key.length; i++) { 3041 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 3042 builder.append(hexdigits.charAt(key[i] & 0x0f)); 3043 } 3044 builder.append("], iv ["); 3045 for (int i = 0; i < iv.length; i++) { builder.append(iv[i] & 0xf0) >> 43046 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 3047 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 3048 } 3049 builder.append("], clear "); Arrays.toString(numBytesOfClearData)3050 builder.append(Arrays.toString(numBytesOfClearData)); 3051 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)3052 builder.append(Arrays.toString(numBytesOfEncryptedData)); 3053 builder.append(", pattern (encrypt: "); builder.append(mPattern.mEncryptBlocks)3054 builder.append(mPattern.mEncryptBlocks); 3055 builder.append(", skip: "); builder.append(mPattern.mSkipBlocks)3056 builder.append(mPattern.mSkipBlocks); 3057 builder.append(")"); 3058 return builder.toString(); 3059 } 3060 }; 3061 3062 /** 3063 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 3064 * potentially encrypted. 3065 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 3066 * 3067 * @param index The index of a client-owned input buffer previously returned 3068 * in a call to {@link #dequeueInputBuffer}. 3069 * @param offset The byte offset into the input buffer at which the data starts. 3070 * @param info Metadata required to facilitate decryption, the object can be 3071 * reused immediately after this call returns. 3072 * @param presentationTimeUs The presentation timestamp in microseconds for this 3073 * buffer. This is normally the media time at which this 3074 * buffer should be presented (rendered). 3075 * @param flags A bitmask of flags 3076 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3077 * While not prohibited, most codecs do not use the 3078 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3079 * @throws IllegalStateException if not in the Executing state. 3080 * @throws MediaCodec.CodecException upon codec error. 3081 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 3082 * An error code associated with the exception helps identify the 3083 * reason for the failure. 3084 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)3085 public final void queueSecureInputBuffer( 3086 int index, 3087 int offset, 3088 @NonNull CryptoInfo info, 3089 long presentationTimeUs, 3090 int flags) throws CryptoException { 3091 if ((flags & BUFFER_FLAG_DECODE_ONLY) != 0 3092 && (flags & BUFFER_FLAG_END_OF_STREAM) != 0) { 3093 throw new InvalidBufferFlagsException(EOS_AND_DECODE_ONLY_ERROR_MESSAGE); 3094 } 3095 synchronized(mBufferLock) { 3096 if (mBufferMode == BUFFER_MODE_BLOCK) { 3097 throw new IncompatibleWithBlockModelException("queueSecureInputBuffer() " 3098 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3099 + "Please use getQueueRequest() to queue buffers"); 3100 } 3101 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 3102 mDequeuedInputBuffers.remove(index); 3103 } 3104 try { 3105 native_queueSecureInputBuffer( 3106 index, offset, info, presentationTimeUs, flags); 3107 } catch (CryptoException | IllegalStateException e) { 3108 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 3109 throw e; 3110 } 3111 } 3112 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)3113 private native final void native_queueSecureInputBuffer( 3114 int index, 3115 int offset, 3116 @NonNull CryptoInfo info, 3117 long presentationTimeUs, 3118 int flags) throws CryptoException; 3119 3120 /** 3121 * Returns the index of an input buffer to be filled with valid data 3122 * or -1 if no such buffer is currently available. 3123 * This method will return immediately if timeoutUs == 0, wait indefinitely 3124 * for the availability of an input buffer if timeoutUs < 0 or wait up 3125 * to "timeoutUs" microseconds if timeoutUs > 0. 3126 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 3127 * @throws IllegalStateException if not in the Executing state, 3128 * or codec is configured in asynchronous mode. 3129 * @throws MediaCodec.CodecException upon codec error. 3130 */ dequeueInputBuffer(long timeoutUs)3131 public final int dequeueInputBuffer(long timeoutUs) { 3132 synchronized (mBufferLock) { 3133 if (mBufferMode == BUFFER_MODE_BLOCK) { 3134 throw new IncompatibleWithBlockModelException("dequeueInputBuffer() " 3135 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3136 + "Please use MediaCodec.Callback objectes to get input buffer slots."); 3137 } 3138 } 3139 int res = native_dequeueInputBuffer(timeoutUs); 3140 if (res >= 0) { 3141 synchronized(mBufferLock) { 3142 validateInputByteBufferLocked(mCachedInputBuffers, res); 3143 } 3144 } 3145 return res; 3146 } 3147 native_dequeueInputBuffer(long timeoutUs)3148 private native final int native_dequeueInputBuffer(long timeoutUs); 3149 3150 /** 3151 * Section of memory that represents a linear block. Applications may 3152 * acquire a block via {@link LinearBlock#obtain} and queue all or part 3153 * of the block as an input buffer to a codec, or get a block allocated by 3154 * codec as an output buffer from {@link OutputFrame}. 3155 * 3156 * {@see QueueRequest#setLinearBlock} 3157 * {@see QueueRequest#setEncryptedLinearBlock} 3158 * {@see OutputFrame#getLinearBlock} 3159 */ 3160 public static final class LinearBlock { 3161 // No public constructors. LinearBlock()3162 private LinearBlock() {} 3163 3164 /** 3165 * Returns true if the buffer is mappable. 3166 * @throws IllegalStateException if invalid 3167 */ isMappable()3168 public boolean isMappable() { 3169 synchronized (mLock) { 3170 if (!mValid) { 3171 throw new IllegalStateException("The linear block is invalid"); 3172 } 3173 return mMappable; 3174 } 3175 } 3176 3177 /** 3178 * Map the memory and return the mapped region. 3179 * <p> 3180 * The returned memory region becomes inaccessible after 3181 * {@link #recycle}, or the buffer is queued to the codecs and not 3182 * returned to the client yet. 3183 * 3184 * @return mapped memory region as {@link ByteBuffer} object 3185 * @throws IllegalStateException if not mappable or invalid 3186 */ map()3187 public @NonNull ByteBuffer map() { 3188 synchronized (mLock) { 3189 if (!mValid) { 3190 throw new IllegalStateException("The linear block is invalid"); 3191 } 3192 if (!mMappable) { 3193 throw new IllegalStateException("The linear block is not mappable"); 3194 } 3195 if (mMapped == null) { 3196 mMapped = native_map(); 3197 } 3198 return mMapped; 3199 } 3200 } 3201 native_map()3202 private native ByteBuffer native_map(); 3203 3204 /** 3205 * Mark this block as ready to be recycled by the framework once it is 3206 * no longer in use. All operations to this object after 3207 * this call will cause exceptions, as well as attempt to access the 3208 * previously mapped memory region. Caller should clear all references 3209 * to this object after this call. 3210 * <p> 3211 * To avoid excessive memory consumption, it is recommended that callers 3212 * recycle buffers as soon as they no longer need the buffers 3213 * 3214 * @throws IllegalStateException if invalid 3215 */ recycle()3216 public void recycle() { 3217 synchronized (mLock) { 3218 if (!mValid) { 3219 throw new IllegalStateException("The linear block is invalid"); 3220 } 3221 if (mMapped != null) { 3222 mMapped.setAccessible(false); 3223 mMapped = null; 3224 } 3225 native_recycle(); 3226 mValid = false; 3227 mNativeContext = 0; 3228 } 3229 3230 if (!mInternal) { 3231 sPool.offer(this); 3232 } 3233 } 3234 native_recycle()3235 private native void native_recycle(); 3236 native_obtain(int capacity, String[] codecNames)3237 private native void native_obtain(int capacity, String[] codecNames); 3238 3239 @Override finalize()3240 protected void finalize() { 3241 native_recycle(); 3242 } 3243 3244 /** 3245 * Returns true if it is possible to allocate a linear block that can be 3246 * passed to all listed codecs as input buffers without copying the 3247 * content. 3248 * <p> 3249 * Note that even if this function returns true, {@link #obtain} may 3250 * still throw due to invalid arguments or allocation failure. 3251 * 3252 * @param codecNames list of codecs that the client wants to use a 3253 * linear block without copying. Null entries are 3254 * ignored. 3255 */ isCodecCopyFreeCompatible(@onNull String[] codecNames)3256 public static boolean isCodecCopyFreeCompatible(@NonNull String[] codecNames) { 3257 return native_checkCompatible(codecNames); 3258 } 3259 native_checkCompatible(@onNull String[] codecNames)3260 private static native boolean native_checkCompatible(@NonNull String[] codecNames); 3261 3262 /** 3263 * Obtain a linear block object no smaller than {@code capacity}. 3264 * If {@link #isCodecCopyFreeCompatible} with the same 3265 * {@code codecNames} returned true, the returned 3266 * {@link LinearBlock} object can be queued to the listed codecs without 3267 * copying. The returned {@link LinearBlock} object is always 3268 * read/write mappable. 3269 * 3270 * @param capacity requested capacity of the linear block in bytes 3271 * @param codecNames list of codecs that the client wants to use this 3272 * linear block without copying. Null entries are 3273 * ignored. 3274 * @return a linear block object. 3275 * @throws IllegalArgumentException if the capacity is invalid or 3276 * codecNames contains invalid name 3277 * @throws IOException if an error occurred while allocating a buffer 3278 */ obtain( int capacity, @NonNull String[] codecNames)3279 public static @Nullable LinearBlock obtain( 3280 int capacity, @NonNull String[] codecNames) { 3281 LinearBlock buffer = sPool.poll(); 3282 if (buffer == null) { 3283 buffer = new LinearBlock(); 3284 } 3285 synchronized (buffer.mLock) { 3286 buffer.native_obtain(capacity, codecNames); 3287 } 3288 return buffer; 3289 } 3290 3291 // Called from native setInternalStateLocked(long context, boolean isMappable)3292 private void setInternalStateLocked(long context, boolean isMappable) { 3293 mNativeContext = context; 3294 mMappable = isMappable; 3295 mValid = (context != 0); 3296 mInternal = true; 3297 } 3298 3299 private static final BlockingQueue<LinearBlock> sPool = 3300 new LinkedBlockingQueue<>(); 3301 3302 private final Object mLock = new Object(); 3303 private boolean mValid = false; 3304 private boolean mMappable = false; 3305 private ByteBuffer mMapped = null; 3306 private long mNativeContext = 0; 3307 private boolean mInternal = false; 3308 } 3309 3310 /** 3311 * Map a {@link HardwareBuffer} object into {@link Image}, so that the content of the buffer is 3312 * accessible. Depending on the usage and pixel format of the hardware buffer, it may not be 3313 * mappable; this method returns null in that case. 3314 * 3315 * @param hardwareBuffer {@link HardwareBuffer} to map. 3316 * @return Mapped {@link Image} object, or null if the buffer is not mappable. 3317 */ mapHardwareBuffer(@onNull HardwareBuffer hardwareBuffer)3318 public static @Nullable Image mapHardwareBuffer(@NonNull HardwareBuffer hardwareBuffer) { 3319 return native_mapHardwareBuffer(hardwareBuffer); 3320 } 3321 native_mapHardwareBuffer( @onNull HardwareBuffer hardwareBuffer)3322 private static native @Nullable Image native_mapHardwareBuffer( 3323 @NonNull HardwareBuffer hardwareBuffer); 3324 native_closeMediaImage(long context)3325 private static native void native_closeMediaImage(long context); 3326 3327 /** 3328 * Builder-like class for queue requests. Use this class to prepare a 3329 * queue request and send it. 3330 */ 3331 public final class QueueRequest { 3332 // No public constructor QueueRequest(@onNull MediaCodec codec, int index)3333 private QueueRequest(@NonNull MediaCodec codec, int index) { 3334 mCodec = codec; 3335 mIndex = index; 3336 } 3337 3338 /** 3339 * Set a linear block to this queue request. Exactly one buffer must be 3340 * set for a queue request before calling {@link #queue}. It is possible 3341 * to use the same {@link LinearBlock} object for multiple queue 3342 * requests. The behavior is undefined if the range of the buffer 3343 * overlaps for multiple requests, or the application writes into the 3344 * region being processed by the codec. 3345 * 3346 * @param block The linear block object 3347 * @param offset The byte offset into the input buffer at which the data starts. 3348 * @param size The number of bytes of valid input data. 3349 * @return this object 3350 * @throws IllegalStateException if a buffer is already set 3351 */ setLinearBlock( @onNull LinearBlock block, int offset, int size)3352 public @NonNull QueueRequest setLinearBlock( 3353 @NonNull LinearBlock block, 3354 int offset, 3355 int size) { 3356 if (!isAccessible()) { 3357 throw new IllegalStateException("The request is stale"); 3358 } 3359 if (mLinearBlock != null || mHardwareBuffer != null) { 3360 throw new IllegalStateException("Cannot set block twice"); 3361 } 3362 mLinearBlock = block; 3363 mOffset = offset; 3364 mSize = size; 3365 mCryptoInfo = null; 3366 return this; 3367 } 3368 3369 /** 3370 * Set an encrypted linear block to this queue request. Exactly one buffer must be 3371 * set for a queue request before calling {@link #queue}. It is possible 3372 * to use the same {@link LinearBlock} object for multiple queue 3373 * requests. The behavior is undefined if the range of the buffer 3374 * overlaps for multiple requests, or the application writes into the 3375 * region being processed by the codec. 3376 * 3377 * @param block The linear block object 3378 * @param offset The byte offset into the input buffer at which the data starts. 3379 * @param size The number of bytes of valid input data. 3380 * @param cryptoInfo Metadata describing the structure of the encrypted input sample. 3381 * @return this object 3382 * @throws IllegalStateException if a buffer is already set 3383 */ setEncryptedLinearBlock( @onNull LinearBlock block, int offset, int size, @NonNull MediaCodec.CryptoInfo cryptoInfo)3384 public @NonNull QueueRequest setEncryptedLinearBlock( 3385 @NonNull LinearBlock block, 3386 int offset, 3387 int size, 3388 @NonNull MediaCodec.CryptoInfo cryptoInfo) { 3389 Objects.requireNonNull(cryptoInfo); 3390 if (!isAccessible()) { 3391 throw new IllegalStateException("The request is stale"); 3392 } 3393 if (mLinearBlock != null || mHardwareBuffer != null) { 3394 throw new IllegalStateException("Cannot set block twice"); 3395 } 3396 mLinearBlock = block; 3397 mOffset = offset; 3398 mSize = size; 3399 mCryptoInfo = cryptoInfo; 3400 return this; 3401 } 3402 3403 /** 3404 * Set a harware graphic buffer to this queue request. Exactly one buffer must 3405 * be set for a queue request before calling {@link #queue}. 3406 * <p> 3407 * Note: buffers should have format {@link HardwareBuffer#YCBCR_420_888}, 3408 * a single layer, and an appropriate usage ({@link HardwareBuffer#USAGE_CPU_READ_OFTEN} 3409 * for software codecs and {@link HardwareBuffer#USAGE_VIDEO_ENCODE} for hardware) 3410 * for codecs to recognize. Codecs may throw exception if the buffer is not recognizable. 3411 * 3412 * @param buffer The hardware graphic buffer object 3413 * @return this object 3414 * @throws IllegalStateException if a buffer is already set 3415 */ setHardwareBuffer( @onNull HardwareBuffer buffer)3416 public @NonNull QueueRequest setHardwareBuffer( 3417 @NonNull HardwareBuffer buffer) { 3418 if (!isAccessible()) { 3419 throw new IllegalStateException("The request is stale"); 3420 } 3421 if (mLinearBlock != null || mHardwareBuffer != null) { 3422 throw new IllegalStateException("Cannot set block twice"); 3423 } 3424 mHardwareBuffer = buffer; 3425 return this; 3426 } 3427 3428 /** 3429 * Set timestamp to this queue request. 3430 * 3431 * @param presentationTimeUs The presentation timestamp in microseconds for this 3432 * buffer. This is normally the media time at which this 3433 * buffer should be presented (rendered). When using an output 3434 * surface, this will be propagated as the {@link 3435 * SurfaceTexture#getTimestamp timestamp} for the frame (after 3436 * conversion to nanoseconds). 3437 * @return this object 3438 */ setPresentationTimeUs(long presentationTimeUs)3439 public @NonNull QueueRequest setPresentationTimeUs(long presentationTimeUs) { 3440 if (!isAccessible()) { 3441 throw new IllegalStateException("The request is stale"); 3442 } 3443 mPresentationTimeUs = presentationTimeUs; 3444 return this; 3445 } 3446 3447 /** 3448 * Set flags to this queue request. 3449 * 3450 * @param flags A bitmask of flags 3451 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3452 * While not prohibited, most codecs do not use the 3453 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3454 * @return this object 3455 */ setFlags(@ufferFlag int flags)3456 public @NonNull QueueRequest setFlags(@BufferFlag int flags) { 3457 if (!isAccessible()) { 3458 throw new IllegalStateException("The request is stale"); 3459 } 3460 mFlags = flags; 3461 return this; 3462 } 3463 3464 /** 3465 * Add an integer parameter. 3466 * See {@link MediaFormat} for an exhaustive list of supported keys with 3467 * values of type int, that can also be set with {@link MediaFormat#setInteger}. 3468 * 3469 * If there was {@link MediaCodec#setParameters} 3470 * call with the same key which is not processed by the codec yet, the 3471 * value set from this method will override the unprocessed value. 3472 * 3473 * @return this object 3474 */ setIntegerParameter( @onNull String key, int value)3475 public @NonNull QueueRequest setIntegerParameter( 3476 @NonNull String key, int value) { 3477 if (!isAccessible()) { 3478 throw new IllegalStateException("The request is stale"); 3479 } 3480 mTuningKeys.add(key); 3481 mTuningValues.add(Integer.valueOf(value)); 3482 return this; 3483 } 3484 3485 /** 3486 * Add a long parameter. 3487 * See {@link MediaFormat} for an exhaustive list of supported keys with 3488 * values of type long, that can also be set with {@link MediaFormat#setLong}. 3489 * 3490 * If there was {@link MediaCodec#setParameters} 3491 * call with the same key which is not processed by the codec yet, the 3492 * value set from this method will override the unprocessed value. 3493 * 3494 * @return this object 3495 */ setLongParameter( @onNull String key, long value)3496 public @NonNull QueueRequest setLongParameter( 3497 @NonNull String key, long value) { 3498 if (!isAccessible()) { 3499 throw new IllegalStateException("The request is stale"); 3500 } 3501 mTuningKeys.add(key); 3502 mTuningValues.add(Long.valueOf(value)); 3503 return this; 3504 } 3505 3506 /** 3507 * Add a float parameter. 3508 * See {@link MediaFormat} for an exhaustive list of supported keys with 3509 * values of type float, that can also be set with {@link MediaFormat#setFloat}. 3510 * 3511 * If there was {@link MediaCodec#setParameters} 3512 * call with the same key which is not processed by the codec yet, the 3513 * value set from this method will override the unprocessed value. 3514 * 3515 * @return this object 3516 */ setFloatParameter( @onNull String key, float value)3517 public @NonNull QueueRequest setFloatParameter( 3518 @NonNull String key, float value) { 3519 if (!isAccessible()) { 3520 throw new IllegalStateException("The request is stale"); 3521 } 3522 mTuningKeys.add(key); 3523 mTuningValues.add(Float.valueOf(value)); 3524 return this; 3525 } 3526 3527 /** 3528 * Add a {@link ByteBuffer} parameter. 3529 * See {@link MediaFormat} for an exhaustive list of supported keys with 3530 * values of byte buffer, that can also be set with {@link MediaFormat#setByteBuffer}. 3531 * 3532 * If there was {@link MediaCodec#setParameters} 3533 * call with the same key which is not processed by the codec yet, the 3534 * value set from this method will override the unprocessed value. 3535 * 3536 * @return this object 3537 */ setByteBufferParameter( @onNull String key, @NonNull ByteBuffer value)3538 public @NonNull QueueRequest setByteBufferParameter( 3539 @NonNull String key, @NonNull ByteBuffer value) { 3540 if (!isAccessible()) { 3541 throw new IllegalStateException("The request is stale"); 3542 } 3543 mTuningKeys.add(key); 3544 mTuningValues.add(value); 3545 return this; 3546 } 3547 3548 /** 3549 * Add a string parameter. 3550 * See {@link MediaFormat} for an exhaustive list of supported keys with 3551 * values of type string, that can also be set with {@link MediaFormat#setString}. 3552 * 3553 * If there was {@link MediaCodec#setParameters} 3554 * call with the same key which is not processed by the codec yet, the 3555 * value set from this method will override the unprocessed value. 3556 * 3557 * @return this object 3558 */ setStringParameter( @onNull String key, @NonNull String value)3559 public @NonNull QueueRequest setStringParameter( 3560 @NonNull String key, @NonNull String value) { 3561 if (!isAccessible()) { 3562 throw new IllegalStateException("The request is stale"); 3563 } 3564 mTuningKeys.add(key); 3565 mTuningValues.add(value); 3566 return this; 3567 } 3568 3569 /** 3570 * Finish building a queue request and queue the buffers with tunings. 3571 */ queue()3572 public void queue() { 3573 if (!isAccessible()) { 3574 throw new IllegalStateException("The request is stale"); 3575 } 3576 if (mLinearBlock == null && mHardwareBuffer == null) { 3577 throw new IllegalStateException("No block is set"); 3578 } 3579 setAccessible(false); 3580 if (mLinearBlock != null) { 3581 mCodec.native_queueLinearBlock( 3582 mIndex, mLinearBlock, mOffset, mSize, mCryptoInfo, 3583 mPresentationTimeUs, mFlags, 3584 mTuningKeys, mTuningValues); 3585 } else if (mHardwareBuffer != null) { 3586 mCodec.native_queueHardwareBuffer( 3587 mIndex, mHardwareBuffer, mPresentationTimeUs, mFlags, 3588 mTuningKeys, mTuningValues); 3589 } 3590 clear(); 3591 } 3592 clear()3593 @NonNull QueueRequest clear() { 3594 mLinearBlock = null; 3595 mOffset = 0; 3596 mSize = 0; 3597 mCryptoInfo = null; 3598 mHardwareBuffer = null; 3599 mPresentationTimeUs = 0; 3600 mFlags = 0; 3601 mTuningKeys.clear(); 3602 mTuningValues.clear(); 3603 return this; 3604 } 3605 isAccessible()3606 boolean isAccessible() { 3607 return mAccessible; 3608 } 3609 setAccessible(boolean accessible)3610 @NonNull QueueRequest setAccessible(boolean accessible) { 3611 mAccessible = accessible; 3612 return this; 3613 } 3614 3615 private final MediaCodec mCodec; 3616 private final int mIndex; 3617 private LinearBlock mLinearBlock = null; 3618 private int mOffset = 0; 3619 private int mSize = 0; 3620 private MediaCodec.CryptoInfo mCryptoInfo = null; 3621 private HardwareBuffer mHardwareBuffer = null; 3622 private long mPresentationTimeUs = 0; 3623 private @BufferFlag int mFlags = 0; 3624 private final ArrayList<String> mTuningKeys = new ArrayList<>(); 3625 private final ArrayList<Object> mTuningValues = new ArrayList<>(); 3626 3627 private boolean mAccessible = false; 3628 } 3629 native_queueLinearBlock( int index, @NonNull LinearBlock block, int offset, int size, @Nullable CryptoInfo cryptoInfo, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3630 private native void native_queueLinearBlock( 3631 int index, 3632 @NonNull LinearBlock block, 3633 int offset, 3634 int size, 3635 @Nullable CryptoInfo cryptoInfo, 3636 long presentationTimeUs, 3637 int flags, 3638 @NonNull ArrayList<String> keys, 3639 @NonNull ArrayList<Object> values); 3640 native_queueHardwareBuffer( int index, @NonNull HardwareBuffer buffer, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3641 private native void native_queueHardwareBuffer( 3642 int index, 3643 @NonNull HardwareBuffer buffer, 3644 long presentationTimeUs, 3645 int flags, 3646 @NonNull ArrayList<String> keys, 3647 @NonNull ArrayList<Object> values); 3648 3649 private final ArrayList<QueueRequest> mQueueRequests = new ArrayList<>(); 3650 3651 /** 3652 * Return a {@link QueueRequest} object for an input slot index. 3653 * 3654 * @param index input slot index from 3655 * {@link Callback#onInputBufferAvailable} 3656 * @return queue request object 3657 * @throws IllegalStateException if not using block model 3658 * @throws IllegalArgumentException if the input slot is not available or 3659 * the index is out of range 3660 */ getQueueRequest(int index)3661 public @NonNull QueueRequest getQueueRequest(int index) { 3662 synchronized (mBufferLock) { 3663 if (mBufferMode != BUFFER_MODE_BLOCK) { 3664 throw new IllegalStateException("The codec is not configured for block model"); 3665 } 3666 if (index < 0 || index >= mQueueRequests.size()) { 3667 throw new IndexOutOfBoundsException("Expected range of index: [0," 3668 + (mQueueRequests.size() - 1) + "]; actual: " + index); 3669 } 3670 QueueRequest request = mQueueRequests.get(index); 3671 if (request == null) { 3672 throw new IllegalArgumentException("Unavailable index: " + index); 3673 } 3674 if (!request.isAccessible()) { 3675 throw new IllegalArgumentException( 3676 "The request is stale at index " + index); 3677 } 3678 return request.clear(); 3679 } 3680 } 3681 3682 /** 3683 * If a non-negative timeout had been specified in the call 3684 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 3685 */ 3686 public static final int INFO_TRY_AGAIN_LATER = -1; 3687 3688 /** 3689 * The output format has changed, subsequent data will follow the new 3690 * format. {@link #getOutputFormat()} returns the new format. Note, that 3691 * you can also use the new {@link #getOutputFormat(int)} method to 3692 * get the format for a specific output buffer. This frees you from 3693 * having to track output format changes. 3694 */ 3695 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 3696 3697 /** 3698 * The output buffers have changed, the client must refer to the new 3699 * set of output buffers returned by {@link #getOutputBuffers} from 3700 * this point on. 3701 * 3702 * <p>Additionally, this event signals that the video scaling mode 3703 * may have been reset to the default.</p> 3704 * 3705 * @deprecated This return value can be ignored as {@link 3706 * #getOutputBuffers} has been deprecated. Client should 3707 * request a current buffer using on of the get-buffer or 3708 * get-image methods each time one has been dequeued. 3709 */ 3710 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 3711 3712 /** @hide */ 3713 @IntDef({ 3714 INFO_TRY_AGAIN_LATER, 3715 INFO_OUTPUT_FORMAT_CHANGED, 3716 INFO_OUTPUT_BUFFERS_CHANGED, 3717 }) 3718 @Retention(RetentionPolicy.SOURCE) 3719 public @interface OutputBufferInfo {} 3720 3721 /** 3722 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 3723 * Returns the index of an output buffer that has been successfully 3724 * decoded or one of the INFO_* constants. 3725 * @param info Will be filled with buffer meta data. 3726 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 3727 * @throws IllegalStateException if not in the Executing state, 3728 * or codec is configured in asynchronous mode. 3729 * @throws MediaCodec.CodecException upon codec error. 3730 */ 3731 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)3732 public final int dequeueOutputBuffer( 3733 @NonNull BufferInfo info, long timeoutUs) { 3734 synchronized (mBufferLock) { 3735 if (mBufferMode == BUFFER_MODE_BLOCK) { 3736 throw new IncompatibleWithBlockModelException("dequeueOutputBuffer() " 3737 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3738 + "Please use MediaCodec.Callback objects to get output buffer slots."); 3739 } 3740 } 3741 int res = native_dequeueOutputBuffer(info, timeoutUs); 3742 synchronized (mBufferLock) { 3743 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 3744 cacheBuffersLocked(false /* input */); 3745 } else if (res >= 0) { 3746 validateOutputByteBufferLocked(mCachedOutputBuffers, res, info); 3747 if (mHasSurface || mCachedOutputBuffers == null) { 3748 mDequeuedOutputInfos.put(res, info.dup()); 3749 } 3750 } 3751 } 3752 return res; 3753 } 3754 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)3755 private native final int native_dequeueOutputBuffer( 3756 @NonNull BufferInfo info, long timeoutUs); 3757 3758 /** 3759 * If you are done with a buffer, use this call to return the buffer to the codec 3760 * or to render it on the output surface. If you configured the codec with an 3761 * output surface, setting {@code render} to {@code true} will first send the buffer 3762 * to that output surface. The surface will release the buffer back to the codec once 3763 * it is no longer used/displayed. 3764 * 3765 * Once an output buffer is released to the codec, it MUST NOT 3766 * be used until it is later retrieved by {@link #getOutputBuffer} in response 3767 * to a {@link #dequeueOutputBuffer} return value or a 3768 * {@link Callback#onOutputBufferAvailable} callback. 3769 * 3770 * @param index The index of a client-owned output buffer previously returned 3771 * from a call to {@link #dequeueOutputBuffer}. 3772 * @param render If a valid surface was specified when configuring the codec, 3773 * passing true renders this output buffer to the surface. 3774 * @throws IllegalStateException if not in the Executing state. 3775 * @throws MediaCodec.CodecException upon codec error. 3776 */ releaseOutputBuffer(int index, boolean render)3777 public final void releaseOutputBuffer(int index, boolean render) { 3778 releaseOutputBufferInternal(index, render, false /* updatePTS */, 0 /* dummy */); 3779 } 3780 3781 /** 3782 * If you are done with a buffer, use this call to update its surface timestamp 3783 * and return it to the codec to render it on the output surface. If you 3784 * have not specified an output surface when configuring this video codec, 3785 * this call will simply return the buffer to the codec.<p> 3786 * 3787 * The timestamp may have special meaning depending on the destination surface. 3788 * 3789 * <table> 3790 * <tr><th>SurfaceView specifics</th></tr> 3791 * <tr><td> 3792 * If you render your buffer on a {@link android.view.SurfaceView}, 3793 * you can use the timestamp to render the buffer at a specific time (at the 3794 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 3795 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 3796 * Currently, this is set as within one (1) second. A few notes: 3797 * 3798 * <ul> 3799 * <li>the buffer will not be returned to the codec until the timestamp 3800 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 3801 * <li>buffers are processed sequentially, so you may block subsequent buffers to 3802 * be displayed on the {@link android.view.Surface}. This is important if you 3803 * want to react to user action, e.g. stop the video or seek. 3804 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 3805 * rendered at the same VSYNC, the last one will be shown, and the other ones 3806 * will be dropped. 3807 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 3808 * time, the {@link android.view.Surface} will ignore the timestamp, and 3809 * display the buffer at the earliest feasible time. In this mode it will not 3810 * drop frames. 3811 * <li>for best performance and quality, call this method when you are about 3812 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 3813 * about 33 msec. 3814 * </ul> 3815 * </td></tr> 3816 * </table> 3817 * 3818 * Once an output buffer is released to the codec, it MUST NOT 3819 * be used until it is later retrieved by {@link #getOutputBuffer} in response 3820 * to a {@link #dequeueOutputBuffer} return value or a 3821 * {@link Callback#onOutputBufferAvailable} callback. 3822 * 3823 * @param index The index of a client-owned output buffer previously returned 3824 * from a call to {@link #dequeueOutputBuffer}. 3825 * @param renderTimestampNs The timestamp to associate with this buffer when 3826 * it is sent to the Surface. 3827 * @throws IllegalStateException if not in the Executing state. 3828 * @throws MediaCodec.CodecException upon codec error. 3829 */ releaseOutputBuffer(int index, long renderTimestampNs)3830 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 3831 releaseOutputBufferInternal( 3832 index, true /* render */, true /* updatePTS */, renderTimestampNs); 3833 } 3834 releaseOutputBufferInternal( int index, boolean render, boolean updatePts, long renderTimestampNs)3835 private void releaseOutputBufferInternal( 3836 int index, boolean render, boolean updatePts, long renderTimestampNs) { 3837 BufferInfo info = null; 3838 synchronized(mBufferLock) { 3839 switch (mBufferMode) { 3840 case BUFFER_MODE_LEGACY: 3841 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 3842 mDequeuedOutputBuffers.remove(index); 3843 if (mHasSurface || mCachedOutputBuffers == null) { 3844 info = mDequeuedOutputInfos.remove(index); 3845 } 3846 break; 3847 case BUFFER_MODE_BLOCK: 3848 OutputFrame frame = mOutputFrames.get(index); 3849 frame.setAccessible(false); 3850 frame.clear(); 3851 break; 3852 default: 3853 throw new IllegalStateException( 3854 "Unrecognized buffer mode: " + mBufferMode); 3855 } 3856 } 3857 releaseOutputBuffer( 3858 index, render, updatePts, renderTimestampNs); 3859 } 3860 3861 @UnsupportedAppUsage releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)3862 private native final void releaseOutputBuffer( 3863 int index, boolean render, boolean updatePTS, long timeNs); 3864 3865 /** 3866 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 3867 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 3868 * encoders receiving input from a Surface created by {@link #createInputSurface}. 3869 * @throws IllegalStateException if not in the Executing state. 3870 * @throws MediaCodec.CodecException upon codec error. 3871 */ signalEndOfInputStream()3872 public native final void signalEndOfInputStream(); 3873 3874 /** 3875 * Call this after dequeueOutputBuffer signals a format change by returning 3876 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 3877 * You can also call this after {@link #configure} returns 3878 * successfully to get the output format initially configured 3879 * for the codec. Do this to determine what optional 3880 * configuration parameters were supported by the codec. 3881 * 3882 * @throws IllegalStateException if not in the Executing or 3883 * Configured state. 3884 * @throws MediaCodec.CodecException upon codec error. 3885 */ 3886 @NonNull getOutputFormat()3887 public final MediaFormat getOutputFormat() { 3888 return new MediaFormat(getFormatNative(false /* input */)); 3889 } 3890 3891 /** 3892 * Call this after {@link #configure} returns successfully to 3893 * get the input format accepted by the codec. Do this to 3894 * determine what optional configuration parameters were 3895 * supported by the codec. 3896 * 3897 * @throws IllegalStateException if not in the Executing or 3898 * Configured state. 3899 * @throws MediaCodec.CodecException upon codec error. 3900 */ 3901 @NonNull getInputFormat()3902 public final MediaFormat getInputFormat() { 3903 return new MediaFormat(getFormatNative(true /* input */)); 3904 } 3905 3906 /** 3907 * Returns the output format for a specific output buffer. 3908 * 3909 * @param index The index of a client-owned input buffer previously 3910 * returned from a call to {@link #dequeueInputBuffer}. 3911 * 3912 * @return the format for the output buffer, or null if the index 3913 * is not a dequeued output buffer. 3914 */ 3915 @NonNull getOutputFormat(int index)3916 public final MediaFormat getOutputFormat(int index) { 3917 return new MediaFormat(getOutputFormatNative(index)); 3918 } 3919 3920 @NonNull getFormatNative(boolean input)3921 private native final Map<String, Object> getFormatNative(boolean input); 3922 3923 @NonNull getOutputFormatNative(int index)3924 private native final Map<String, Object> getOutputFormatNative(int index); 3925 3926 // used to track dequeued buffers 3927 private static class BufferMap { 3928 // various returned representations of the codec buffer 3929 private static class CodecBuffer { 3930 private Image mImage; 3931 private ByteBuffer mByteBuffer; 3932 free()3933 public void free() { 3934 if (mByteBuffer != null) { 3935 // all of our ByteBuffers are direct 3936 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 3937 mByteBuffer = null; 3938 } 3939 if (mImage != null) { 3940 mImage.close(); 3941 mImage = null; 3942 } 3943 } 3944 setImage(@ullable Image image)3945 public void setImage(@Nullable Image image) { 3946 free(); 3947 mImage = image; 3948 } 3949 setByteBuffer(@ullable ByteBuffer buffer)3950 public void setByteBuffer(@Nullable ByteBuffer buffer) { 3951 free(); 3952 mByteBuffer = buffer; 3953 } 3954 } 3955 3956 private final Map<Integer, CodecBuffer> mMap = 3957 new HashMap<Integer, CodecBuffer>(); 3958 remove(int index)3959 public void remove(int index) { 3960 CodecBuffer buffer = mMap.get(index); 3961 if (buffer != null) { 3962 buffer.free(); 3963 mMap.remove(index); 3964 } 3965 } 3966 put(int index, @Nullable ByteBuffer newBuffer)3967 public void put(int index, @Nullable ByteBuffer newBuffer) { 3968 CodecBuffer buffer = mMap.get(index); 3969 if (buffer == null) { // likely 3970 buffer = new CodecBuffer(); 3971 mMap.put(index, buffer); 3972 } 3973 buffer.setByteBuffer(newBuffer); 3974 } 3975 put(int index, @Nullable Image newImage)3976 public void put(int index, @Nullable Image newImage) { 3977 CodecBuffer buffer = mMap.get(index); 3978 if (buffer == null) { // likely 3979 buffer = new CodecBuffer(); 3980 mMap.put(index, buffer); 3981 } 3982 buffer.setImage(newImage); 3983 } 3984 clear()3985 public void clear() { 3986 for (CodecBuffer buffer: mMap.values()) { 3987 buffer.free(); 3988 } 3989 mMap.clear(); 3990 } 3991 } 3992 3993 private ByteBuffer[] mCachedInputBuffers; 3994 private ByteBuffer[] mCachedOutputBuffers; 3995 private BitSet mValidInputIndices = new BitSet(); 3996 private BitSet mValidOutputIndices = new BitSet(); 3997 3998 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 3999 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 4000 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 4001 new HashMap<Integer, BufferInfo>(); 4002 final private Object mBufferLock; 4003 invalidateByteBufferLocked( @ullable ByteBuffer[] buffers, int index, boolean input)4004 private void invalidateByteBufferLocked( 4005 @Nullable ByteBuffer[] buffers, int index, boolean input) { 4006 if (buffers == null) { 4007 if (index >= 0) { 4008 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4009 indices.clear(index); 4010 } 4011 } else if (index >= 0 && index < buffers.length) { 4012 ByteBuffer buffer = buffers[index]; 4013 if (buffer != null) { 4014 buffer.setAccessible(false); 4015 } 4016 } 4017 } 4018 validateInputByteBufferLocked( @ullable ByteBuffer[] buffers, int index)4019 private void validateInputByteBufferLocked( 4020 @Nullable ByteBuffer[] buffers, int index) { 4021 if (buffers == null) { 4022 if (index >= 0) { 4023 mValidInputIndices.set(index); 4024 } 4025 } else if (index >= 0 && index < buffers.length) { 4026 ByteBuffer buffer = buffers[index]; 4027 if (buffer != null) { 4028 buffer.setAccessible(true); 4029 buffer.clear(); 4030 } 4031 } 4032 } 4033 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index, boolean input)4034 private void revalidateByteBuffer( 4035 @Nullable ByteBuffer[] buffers, int index, boolean input) { 4036 synchronized(mBufferLock) { 4037 if (buffers == null) { 4038 if (index >= 0) { 4039 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4040 indices.set(index); 4041 } 4042 } else if (index >= 0 && index < buffers.length) { 4043 ByteBuffer buffer = buffers[index]; 4044 if (buffer != null) { 4045 buffer.setAccessible(true); 4046 } 4047 } 4048 } 4049 } 4050 validateOutputByteBufferLocked( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)4051 private void validateOutputByteBufferLocked( 4052 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 4053 if (buffers == null) { 4054 if (index >= 0) { 4055 mValidOutputIndices.set(index); 4056 } 4057 } else if (index >= 0 && index < buffers.length) { 4058 ByteBuffer buffer = buffers[index]; 4059 if (buffer != null) { 4060 buffer.setAccessible(true); 4061 buffer.limit(info.offset + info.size).position(info.offset); 4062 } 4063 } 4064 } 4065 invalidateByteBuffersLocked(@ullable ByteBuffer[] buffers)4066 private void invalidateByteBuffersLocked(@Nullable ByteBuffer[] buffers) { 4067 if (buffers != null) { 4068 for (ByteBuffer buffer: buffers) { 4069 if (buffer != null) { 4070 buffer.setAccessible(false); 4071 } 4072 } 4073 } 4074 } 4075 freeByteBufferLocked(@ullable ByteBuffer buffer)4076 private void freeByteBufferLocked(@Nullable ByteBuffer buffer) { 4077 if (buffer != null /* && buffer.isDirect() */) { 4078 // all of our ByteBuffers are direct 4079 java.nio.NioUtils.freeDirectBuffer(buffer); 4080 } 4081 } 4082 freeByteBuffersLocked(@ullable ByteBuffer[] buffers)4083 private void freeByteBuffersLocked(@Nullable ByteBuffer[] buffers) { 4084 if (buffers != null) { 4085 for (ByteBuffer buffer: buffers) { 4086 freeByteBufferLocked(buffer); 4087 } 4088 } 4089 } 4090 freeAllTrackedBuffers()4091 private void freeAllTrackedBuffers() { 4092 synchronized(mBufferLock) { 4093 freeByteBuffersLocked(mCachedInputBuffers); 4094 freeByteBuffersLocked(mCachedOutputBuffers); 4095 mCachedInputBuffers = null; 4096 mCachedOutputBuffers = null; 4097 mValidInputIndices.clear(); 4098 mValidOutputIndices.clear(); 4099 mDequeuedInputBuffers.clear(); 4100 mDequeuedOutputBuffers.clear(); 4101 mQueueRequests.clear(); 4102 mOutputFrames.clear(); 4103 } 4104 } 4105 cacheBuffersLocked(boolean input)4106 private void cacheBuffersLocked(boolean input) { 4107 ByteBuffer[] buffers = null; 4108 try { 4109 buffers = getBuffers(input); 4110 invalidateByteBuffersLocked(buffers); 4111 } catch (IllegalStateException e) { 4112 // we don't get buffers in async mode 4113 } 4114 if (buffers != null) { 4115 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4116 for (int i = 0; i < buffers.length; ++i) { 4117 ByteBuffer buffer = buffers[i]; 4118 if (buffer == null || !indices.get(i)) { 4119 continue; 4120 } 4121 buffer.setAccessible(true); 4122 if (!input) { 4123 BufferInfo info = mDequeuedOutputInfos.get(i); 4124 if (info != null) { 4125 buffer.limit(info.offset + info.size).position(info.offset); 4126 } 4127 } 4128 } 4129 indices.clear(); 4130 } 4131 if (input) { 4132 mCachedInputBuffers = buffers; 4133 } else { 4134 mCachedOutputBuffers = buffers; 4135 } 4136 } 4137 4138 /** 4139 * Retrieve the set of input buffers. Call this after start() 4140 * returns. After calling this method, any ByteBuffers 4141 * previously returned by an earlier call to this method MUST no 4142 * longer be used. 4143 * 4144 * @deprecated Use the new {@link #getInputBuffer} method instead 4145 * each time an input buffer is dequeued. 4146 * 4147 * <b>Note:</b> As of API 21, dequeued input buffers are 4148 * automatically {@link java.nio.Buffer#clear cleared}. 4149 * 4150 * <em>Do not use this method if using an input surface.</em> 4151 * 4152 * @throws IllegalStateException if not in the Executing state, 4153 * or codec is configured in asynchronous mode. 4154 * @throws MediaCodec.CodecException upon codec error. 4155 */ 4156 @NonNull getInputBuffers()4157 public ByteBuffer[] getInputBuffers() { 4158 synchronized (mBufferLock) { 4159 if (mBufferMode == BUFFER_MODE_BLOCK) { 4160 throw new IncompatibleWithBlockModelException("getInputBuffers() " 4161 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4162 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4163 + "objects and attach to QueueRequest objects."); 4164 } 4165 if (mCachedInputBuffers == null) { 4166 cacheBuffersLocked(true /* input */); 4167 } 4168 if (mCachedInputBuffers == null) { 4169 throw new IllegalStateException(); 4170 } 4171 // FIXME: check codec status 4172 return mCachedInputBuffers; 4173 } 4174 } 4175 4176 /** 4177 * Retrieve the set of output buffers. Call this after start() 4178 * returns and whenever dequeueOutputBuffer signals an output 4179 * buffer change by returning {@link 4180 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 4181 * ByteBuffers previously returned by an earlier call to this 4182 * method MUST no longer be used. 4183 * 4184 * @deprecated Use the new {@link #getOutputBuffer} method instead 4185 * each time an output buffer is dequeued. This method is not 4186 * supported if codec is configured in asynchronous mode. 4187 * 4188 * <b>Note:</b> As of API 21, the position and limit of output 4189 * buffers that are dequeued will be set to the valid data 4190 * range. 4191 * 4192 * <em>Do not use this method if using an output surface.</em> 4193 * 4194 * @throws IllegalStateException if not in the Executing state, 4195 * or codec is configured in asynchronous mode. 4196 * @throws MediaCodec.CodecException upon codec error. 4197 */ 4198 @NonNull getOutputBuffers()4199 public ByteBuffer[] getOutputBuffers() { 4200 synchronized (mBufferLock) { 4201 if (mBufferMode == BUFFER_MODE_BLOCK) { 4202 throw new IncompatibleWithBlockModelException("getOutputBuffers() " 4203 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4204 + "Please use getOutputFrame to get output frames."); 4205 } 4206 if (mCachedOutputBuffers == null) { 4207 cacheBuffersLocked(false /* input */); 4208 } 4209 if (mCachedOutputBuffers == null) { 4210 throw new IllegalStateException(); 4211 } 4212 // FIXME: check codec status 4213 return mCachedOutputBuffers; 4214 } 4215 } 4216 4217 /** 4218 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 4219 * object for a dequeued input buffer index to contain the input data. 4220 * 4221 * After calling this method any ByteBuffer or Image object 4222 * previously returned for the same input index MUST no longer 4223 * be used. 4224 * 4225 * @param index The index of a client-owned input buffer previously 4226 * returned from a call to {@link #dequeueInputBuffer}, 4227 * or received via an onInputBufferAvailable callback. 4228 * 4229 * @return the input buffer, or null if the index is not a dequeued 4230 * input buffer, or if the codec is configured for surface input. 4231 * 4232 * @throws IllegalStateException if not in the Executing state. 4233 * @throws MediaCodec.CodecException upon codec error. 4234 */ 4235 @Nullable getInputBuffer(int index)4236 public ByteBuffer getInputBuffer(int index) { 4237 synchronized (mBufferLock) { 4238 if (mBufferMode == BUFFER_MODE_BLOCK) { 4239 throw new IncompatibleWithBlockModelException("getInputBuffer() " 4240 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4241 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4242 + "objects and attach to QueueRequest objects."); 4243 } 4244 } 4245 ByteBuffer newBuffer = getBuffer(true /* input */, index); 4246 synchronized (mBufferLock) { 4247 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 4248 mDequeuedInputBuffers.put(index, newBuffer); 4249 } 4250 return newBuffer; 4251 } 4252 4253 /** 4254 * Returns a writable Image object for a dequeued input buffer 4255 * index to contain the raw input video frame. 4256 * 4257 * After calling this method any ByteBuffer or Image object 4258 * previously returned for the same input index MUST no longer 4259 * be used. 4260 * 4261 * @param index The index of a client-owned input buffer previously 4262 * returned from a call to {@link #dequeueInputBuffer}, 4263 * or received via an onInputBufferAvailable callback. 4264 * 4265 * @return the input image, or null if the index is not a 4266 * dequeued input buffer, or not a ByteBuffer that contains a 4267 * raw image. 4268 * 4269 * @throws IllegalStateException if not in the Executing state. 4270 * @throws MediaCodec.CodecException upon codec error. 4271 */ 4272 @Nullable getInputImage(int index)4273 public Image getInputImage(int index) { 4274 synchronized (mBufferLock) { 4275 if (mBufferMode == BUFFER_MODE_BLOCK) { 4276 throw new IncompatibleWithBlockModelException("getInputImage() " 4277 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4278 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4279 + "objects and attach to QueueRequest objects."); 4280 } 4281 } 4282 Image newImage = getImage(true /* input */, index); 4283 synchronized (mBufferLock) { 4284 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 4285 mDequeuedInputBuffers.put(index, newImage); 4286 } 4287 return newImage; 4288 } 4289 4290 /** 4291 * Returns a read-only ByteBuffer for a dequeued output buffer 4292 * index. The position and limit of the returned buffer are set 4293 * to the valid output data. 4294 * 4295 * After calling this method, any ByteBuffer or Image object 4296 * previously returned for the same output index MUST no longer 4297 * be used. 4298 * 4299 * @param index The index of a client-owned output buffer previously 4300 * returned from a call to {@link #dequeueOutputBuffer}, 4301 * or received via an onOutputBufferAvailable callback. 4302 * 4303 * @return the output buffer, or null if the index is not a dequeued 4304 * output buffer, or the codec is configured with an output surface. 4305 * 4306 * @throws IllegalStateException if not in the Executing state. 4307 * @throws MediaCodec.CodecException upon codec error. 4308 */ 4309 @Nullable getOutputBuffer(int index)4310 public ByteBuffer getOutputBuffer(int index) { 4311 synchronized (mBufferLock) { 4312 if (mBufferMode == BUFFER_MODE_BLOCK) { 4313 throw new IncompatibleWithBlockModelException("getOutputBuffer() " 4314 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4315 + "Please use getOutputFrame() to get output frames."); 4316 } 4317 } 4318 ByteBuffer newBuffer = getBuffer(false /* input */, index); 4319 synchronized (mBufferLock) { 4320 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4321 mDequeuedOutputBuffers.put(index, newBuffer); 4322 } 4323 return newBuffer; 4324 } 4325 4326 /** 4327 * Returns a read-only Image object for a dequeued output buffer 4328 * index that contains the raw video frame. 4329 * 4330 * After calling this method, any ByteBuffer or Image object previously 4331 * returned for the same output index MUST no longer be used. 4332 * 4333 * @param index The index of a client-owned output buffer previously 4334 * returned from a call to {@link #dequeueOutputBuffer}, 4335 * or received via an onOutputBufferAvailable callback. 4336 * 4337 * @return the output image, or null if the index is not a 4338 * dequeued output buffer, not a raw video frame, or if the codec 4339 * was configured with an output surface. 4340 * 4341 * @throws IllegalStateException if not in the Executing state. 4342 * @throws MediaCodec.CodecException upon codec error. 4343 */ 4344 @Nullable getOutputImage(int index)4345 public Image getOutputImage(int index) { 4346 synchronized (mBufferLock) { 4347 if (mBufferMode == BUFFER_MODE_BLOCK) { 4348 throw new IncompatibleWithBlockModelException("getOutputImage() " 4349 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4350 + "Please use getOutputFrame() to get output frames."); 4351 } 4352 } 4353 Image newImage = getImage(false /* input */, index); 4354 synchronized (mBufferLock) { 4355 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4356 mDequeuedOutputBuffers.put(index, newImage); 4357 } 4358 return newImage; 4359 } 4360 4361 /** 4362 * A single output frame and its associated metadata. 4363 */ 4364 public static final class OutputFrame { 4365 // No public constructor OutputFrame(int index)4366 OutputFrame(int index) { 4367 mIndex = index; 4368 } 4369 4370 /** 4371 * Returns the output linear block, or null if this frame is empty. 4372 * 4373 * @throws IllegalStateException if this output frame is not linear. 4374 */ getLinearBlock()4375 public @Nullable LinearBlock getLinearBlock() { 4376 if (mHardwareBuffer != null) { 4377 throw new IllegalStateException("This output frame is not linear"); 4378 } 4379 return mLinearBlock; 4380 } 4381 4382 /** 4383 * Returns the output hardware graphic buffer, or null if this frame is empty. 4384 * 4385 * @throws IllegalStateException if this output frame is not graphic. 4386 */ getHardwareBuffer()4387 public @Nullable HardwareBuffer getHardwareBuffer() { 4388 if (mLinearBlock != null) { 4389 throw new IllegalStateException("This output frame is not graphic"); 4390 } 4391 return mHardwareBuffer; 4392 } 4393 4394 /** 4395 * Returns the presentation timestamp in microseconds. 4396 */ getPresentationTimeUs()4397 public long getPresentationTimeUs() { 4398 return mPresentationTimeUs; 4399 } 4400 4401 /** 4402 * Returns the buffer flags. 4403 */ getFlags()4404 public @BufferFlag int getFlags() { 4405 return mFlags; 4406 } 4407 4408 /** 4409 * Returns a read-only {@link MediaFormat} for this frame. The returned 4410 * object is valid only until the client calls {@link MediaCodec#releaseOutputBuffer}. 4411 */ getFormat()4412 public @NonNull MediaFormat getFormat() { 4413 return mFormat; 4414 } 4415 4416 /** 4417 * Returns an unmodifiable set of the names of entries that has changed from 4418 * the previous frame. The entries may have been removed/changed/added. 4419 * Client can find out what the change is by querying {@link MediaFormat} 4420 * object returned from {@link #getFormat}. 4421 */ getChangedKeys()4422 public @NonNull Set<String> getChangedKeys() { 4423 if (mKeySet.isEmpty() && !mChangedKeys.isEmpty()) { 4424 mKeySet.addAll(mChangedKeys); 4425 } 4426 return Collections.unmodifiableSet(mKeySet); 4427 } 4428 clear()4429 void clear() { 4430 mLinearBlock = null; 4431 mHardwareBuffer = null; 4432 mFormat = null; 4433 mChangedKeys.clear(); 4434 mKeySet.clear(); 4435 mLoaded = false; 4436 } 4437 isAccessible()4438 boolean isAccessible() { 4439 return mAccessible; 4440 } 4441 setAccessible(boolean accessible)4442 void setAccessible(boolean accessible) { 4443 mAccessible = accessible; 4444 } 4445 setBufferInfo(MediaCodec.BufferInfo info)4446 void setBufferInfo(MediaCodec.BufferInfo info) { 4447 mPresentationTimeUs = info.presentationTimeUs; 4448 mFlags = info.flags; 4449 } 4450 isLoaded()4451 boolean isLoaded() { 4452 return mLoaded; 4453 } 4454 setLoaded(boolean loaded)4455 void setLoaded(boolean loaded) { 4456 mLoaded = loaded; 4457 } 4458 4459 private final int mIndex; 4460 private LinearBlock mLinearBlock = null; 4461 private HardwareBuffer mHardwareBuffer = null; 4462 private long mPresentationTimeUs = 0; 4463 private @BufferFlag int mFlags = 0; 4464 private MediaFormat mFormat = null; 4465 private final ArrayList<String> mChangedKeys = new ArrayList<>(); 4466 private final Set<String> mKeySet = new HashSet<>(); 4467 private boolean mAccessible = false; 4468 private boolean mLoaded = false; 4469 } 4470 4471 private final ArrayList<OutputFrame> mOutputFrames = new ArrayList<>(); 4472 4473 /** 4474 * Returns an {@link OutputFrame} object. 4475 * 4476 * @param index output buffer index from 4477 * {@link Callback#onOutputBufferAvailable} 4478 * @return {@link OutputFrame} object describing the output buffer 4479 * @throws IllegalStateException if not using block model 4480 * @throws IllegalArgumentException if the output buffer is not available or 4481 * the index is out of range 4482 */ getOutputFrame(int index)4483 public @NonNull OutputFrame getOutputFrame(int index) { 4484 synchronized (mBufferLock) { 4485 if (mBufferMode != BUFFER_MODE_BLOCK) { 4486 throw new IllegalStateException("The codec is not configured for block model"); 4487 } 4488 if (index < 0 || index >= mOutputFrames.size()) { 4489 throw new IndexOutOfBoundsException("Expected range of index: [0," 4490 + (mQueueRequests.size() - 1) + "]; actual: " + index); 4491 } 4492 OutputFrame frame = mOutputFrames.get(index); 4493 if (frame == null) { 4494 throw new IllegalArgumentException("Unavailable index: " + index); 4495 } 4496 if (!frame.isAccessible()) { 4497 throw new IllegalArgumentException( 4498 "The output frame is stale at index " + index); 4499 } 4500 if (!frame.isLoaded()) { 4501 native_getOutputFrame(frame, index); 4502 frame.setLoaded(true); 4503 } 4504 return frame; 4505 } 4506 } 4507 native_getOutputFrame(OutputFrame frame, int index)4508 private native void native_getOutputFrame(OutputFrame frame, int index); 4509 4510 /** 4511 * The content is scaled to the surface dimensions 4512 */ 4513 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 4514 4515 /** 4516 * The content is scaled, maintaining its aspect ratio, the whole 4517 * surface area is used, content may be cropped. 4518 * <p class=note> 4519 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 4520 * configure the pixel aspect ratio for a {@link Surface}. 4521 * <p class=note> 4522 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 4523 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 4524 */ 4525 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 4526 4527 /** @hide */ 4528 @IntDef({ 4529 VIDEO_SCALING_MODE_SCALE_TO_FIT, 4530 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 4531 }) 4532 @Retention(RetentionPolicy.SOURCE) 4533 public @interface VideoScalingMode {} 4534 4535 /** 4536 * If a surface has been specified in a previous call to {@link #configure} 4537 * specifies the scaling mode to use. The default is "scale to fit". 4538 * <p class=note> 4539 * The scaling mode may be reset to the <strong>default</strong> each time an 4540 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 4541 * must call this method after every buffer change event (and before the first output buffer is 4542 * released for rendering) to ensure consistent scaling mode. 4543 * <p class=note> 4544 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 4545 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 4546 * 4547 * @throws IllegalArgumentException if mode is not recognized. 4548 * @throws IllegalStateException if in the Released state. 4549 */ setVideoScalingMode(@ideoScalingMode int mode)4550 public native final void setVideoScalingMode(@VideoScalingMode int mode); 4551 4552 /** 4553 * Sets the audio presentation. 4554 * @param presentation see {@link AudioPresentation}. In particular, id should be set. 4555 */ setAudioPresentation(@onNull AudioPresentation presentation)4556 public void setAudioPresentation(@NonNull AudioPresentation presentation) { 4557 if (presentation == null) { 4558 throw new NullPointerException("audio presentation is null"); 4559 } 4560 native_setAudioPresentation(presentation.getPresentationId(), presentation.getProgramId()); 4561 } 4562 native_setAudioPresentation(int presentationId, int programId)4563 private native void native_setAudioPresentation(int presentationId, int programId); 4564 4565 /** 4566 * Retrieve the codec name. 4567 * 4568 * If the codec was created by createDecoderByType or createEncoderByType, what component is 4569 * chosen is not known beforehand. This method returns the name of the codec that was 4570 * selected by the platform. 4571 * 4572 * <strong>Note:</strong> Implementations may provide multiple aliases (codec 4573 * names) for the same underlying codec, any of which can be used to instantiate the same 4574 * underlying codec in {@link MediaCodec#createByCodecName}. This method returns the 4575 * name used to create the codec in this case. 4576 * 4577 * @throws IllegalStateException if in the Released state. 4578 */ 4579 @NonNull getName()4580 public final String getName() { 4581 // get canonical name to handle exception 4582 String canonicalName = getCanonicalName(); 4583 return mNameAtCreation != null ? mNameAtCreation : canonicalName; 4584 } 4585 4586 /** 4587 * Retrieve the underlying codec name. 4588 * 4589 * This method is similar to {@link #getName}, except that it returns the underlying component 4590 * name even if an alias was used to create this MediaCodec object by name, 4591 * 4592 * @throws IllegalStateException if in the Released state. 4593 */ 4594 @NonNull getCanonicalName()4595 public native final String getCanonicalName(); 4596 4597 /** 4598 * Return Metrics data about the current codec instance. 4599 * 4600 * @return a {@link PersistableBundle} containing the set of attributes and values 4601 * available for the media being handled by this instance of MediaCodec 4602 * The attributes are descibed in {@link MetricsConstants}. 4603 * 4604 * Additional vendor-specific fields may also be present in 4605 * the return value. 4606 */ getMetrics()4607 public PersistableBundle getMetrics() { 4608 PersistableBundle bundle = native_getMetrics(); 4609 return bundle; 4610 } 4611 native_getMetrics()4612 private native PersistableBundle native_getMetrics(); 4613 4614 /** 4615 * Change a video encoder's target bitrate on the fly. The value is an 4616 * Integer object containing the new bitrate in bps. 4617 * 4618 * @see #setParameters(Bundle) 4619 */ 4620 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 4621 4622 /** 4623 * Temporarily suspend/resume encoding of input data. While suspended 4624 * input data is effectively discarded instead of being fed into the 4625 * encoder. This parameter really only makes sense to use with an encoder 4626 * in "surface-input" mode, as the client code has no control over the 4627 * input-side of the encoder in that case. 4628 * The value is an Integer object containing the value 1 to suspend 4629 * or the value 0 to resume. 4630 * 4631 * @see #setParameters(Bundle) 4632 */ 4633 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 4634 4635 /** 4636 * When {@link #PARAMETER_KEY_SUSPEND} is present, the client can also 4637 * optionally use this key to specify the timestamp (in micro-second) 4638 * at which the suspend/resume operation takes effect. 4639 * 4640 * Note that the specified timestamp must be greater than or equal to the 4641 * timestamp of any previously queued suspend/resume operations. 4642 * 4643 * The value is a long int, indicating the timestamp to suspend/resume. 4644 * 4645 * @see #setParameters(Bundle) 4646 */ 4647 public static final String PARAMETER_KEY_SUSPEND_TIME = "drop-start-time-us"; 4648 4649 /** 4650 * Specify an offset (in micro-second) to be added on top of the timestamps 4651 * onward. A typical use case is to apply an adjust to the timestamps after 4652 * a period of pause by the user. 4653 * 4654 * This parameter can only be used on an encoder in "surface-input" mode. 4655 * 4656 * The value is a long int, indicating the timestamp offset to be applied. 4657 * 4658 * @see #setParameters(Bundle) 4659 */ 4660 public static final String PARAMETER_KEY_OFFSET_TIME = "time-offset-us"; 4661 4662 /** 4663 * Request that the encoder produce a sync frame "soon". 4664 * Provide an Integer with the value 0. 4665 * 4666 * @see #setParameters(Bundle) 4667 */ 4668 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 4669 4670 /** 4671 * Set the HDR10+ metadata on the next queued input frame. 4672 * 4673 * Provide a byte array of data that's conforming to the 4674 * user_data_registered_itu_t_t35() syntax of SEI message for ST 2094-40. 4675 *<p> 4676 * For decoders: 4677 *<p> 4678 * When a decoder is configured for one of the HDR10+ profiles that uses 4679 * out-of-band metadata (such as {@link 4680 * MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus} or {@link 4681 * MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), this 4682 * parameter sets the HDR10+ metadata on the next input buffer queued 4683 * to the decoder. A decoder supporting these profiles must propagate 4684 * the metadata to the format of the output buffer corresponding to this 4685 * particular input buffer (under key {@link MediaFormat#KEY_HDR10_PLUS_INFO}). 4686 * The metadata should be applied to that output buffer and the buffers 4687 * following it (in display order), until the next output buffer (in 4688 * display order) upon which an HDR10+ metadata is set. 4689 *<p> 4690 * This parameter shouldn't be set if the decoder is not configured for 4691 * an HDR10+ profile that uses out-of-band metadata. In particular, 4692 * it shouldn't be set for HDR10+ profiles that uses in-band metadata 4693 * where the metadata is embedded in the input buffers, for example 4694 * {@link MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}. 4695 *<p> 4696 * For encoders: 4697 *<p> 4698 * When an encoder is configured for one of the HDR10+ profiles and the 4699 * operates in byte buffer input mode (instead of surface input mode), 4700 * this parameter sets the HDR10+ metadata on the next input buffer queued 4701 * to the encoder. For the HDR10+ profiles that uses out-of-band metadata 4702 * (such as {@link MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus}, 4703 * or {@link MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), 4704 * the metadata must be propagated to the format of the output buffer 4705 * corresponding to this particular input buffer (under key {@link 4706 * MediaFormat#KEY_HDR10_PLUS_INFO}). For the HDR10+ profiles that uses 4707 * in-band metadata (such as {@link 4708 * MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}), the 4709 * metadata info must be embedded in the corresponding output buffer itself. 4710 *<p> 4711 * This parameter shouldn't be set if the encoder is not configured for 4712 * an HDR10+ profile, or if it's operating in surface input mode. 4713 *<p> 4714 * 4715 * @see MediaFormat#KEY_HDR10_PLUS_INFO 4716 */ 4717 public static final String PARAMETER_KEY_HDR10_PLUS_INFO = MediaFormat.KEY_HDR10_PLUS_INFO; 4718 4719 /** 4720 * Enable/disable low latency decoding mode. 4721 * When enabled, the decoder doesn't hold input and output data more than 4722 * required by the codec standards. 4723 * The value is an Integer object containing the value 1 to enable 4724 * or the value 0 to disable. 4725 * 4726 * @see #setParameters(Bundle) 4727 * @see MediaFormat#KEY_LOW_LATENCY 4728 */ 4729 public static final String PARAMETER_KEY_LOW_LATENCY = 4730 MediaFormat.KEY_LOW_LATENCY; 4731 4732 /** 4733 * Control video peek of the first frame when a codec is configured for tunnel mode with 4734 * {@link MediaFormat#KEY_AUDIO_SESSION_ID} while the {@link AudioTrack} is paused. 4735 *<p> 4736 * When disabled (1) after a {@link #flush} or {@link #start}, (2) while the corresponding 4737 * {@link AudioTrack} is paused and (3) before any buffers are queued, the first frame is not to 4738 * be rendered until either this parameter is enabled or the corresponding {@link AudioTrack} 4739 * has begun playback. Once the frame is decoded and ready to be rendered, 4740 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called but the frame is 4741 * not rendered. The surface continues to show the previously-rendered content, or black if the 4742 * surface is new. A subsequent call to {@link AudioTrack#play} renders this frame and triggers 4743 * a callback to {@link OnFrameRenderedListener#onFrameRendered}, and video playback begins. 4744 *<p> 4745 * <b>Note</b>: To clear any previously rendered content and show black, configure the 4746 * MediaCodec with {@code KEY_PUSH_BLANK_BUFFERS_ON_STOP(1)}, and call {@link #stop} before 4747 * pushing new video frames to the codec. 4748 *<p> 4749 * When enabled (1) after a {@link #flush} or {@link #start} and (2) while the corresponding 4750 * {@link AudioTrack} is paused, the first frame is rendered as soon as it is decoded, or 4751 * immediately, if it has already been decoded. If not already decoded, when the frame is 4752 * decoded and ready to be rendered, 4753 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called. The frame is then 4754 * immediately rendered and {@link OnFrameRenderedListener#onFrameRendered} is subsequently 4755 * called. 4756 *<p> 4757 * The value is an Integer object containing the value 1 to enable or the value 0 to disable. 4758 *<p> 4759 * The default for this parameter is <b>enabled</b>. Once a frame has been rendered, changing 4760 * this parameter has no effect until a subsequent {@link #flush} or 4761 * {@link #stop}/{@link #start}. 4762 * 4763 * @see #setParameters(Bundle) 4764 */ 4765 public static final String PARAMETER_KEY_TUNNEL_PEEK = "tunnel-peek"; 4766 4767 /** 4768 * Communicate additional parameter changes to the component instance. 4769 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 4770 * 4771 * @param params The bundle of parameters to set. 4772 * @throws IllegalStateException if in the Released state. 4773 */ setParameters(@ullable Bundle params)4774 public final void setParameters(@Nullable Bundle params) { 4775 if (params == null) { 4776 return; 4777 } 4778 4779 String[] keys = new String[params.size()]; 4780 Object[] values = new Object[params.size()]; 4781 4782 int i = 0; 4783 for (final String key: params.keySet()) { 4784 if (key.equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 4785 int sessionId = 0; 4786 try { 4787 sessionId = (Integer)params.get(key); 4788 } catch (Exception e) { 4789 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 4790 } 4791 keys[i] = "audio-hw-sync"; 4792 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 4793 } else { 4794 keys[i] = key; 4795 Object value = params.get(key); 4796 4797 // Bundle's byte array is a byte[], JNI layer only takes ByteBuffer 4798 if (value instanceof byte[]) { 4799 values[i] = ByteBuffer.wrap((byte[])value); 4800 } else { 4801 values[i] = value; 4802 } 4803 } 4804 ++i; 4805 } 4806 4807 setParameters(keys, values); 4808 } 4809 4810 /** 4811 * Sets an asynchronous callback for actionable MediaCodec events. 4812 * 4813 * If the client intends to use the component in asynchronous mode, 4814 * a valid callback should be provided before {@link #configure} is called. 4815 * 4816 * When asynchronous callback is enabled, the client should not call 4817 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 4818 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 4819 * <p> 4820 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 4821 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 4822 * even if an input surface was created. 4823 * 4824 * @param cb The callback that will run. Use {@code null} to clear a previously 4825 * set callback (before {@link #configure configure} is called and run 4826 * in synchronous mode). 4827 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 4828 * callbacks are done on the default thread (the caller's thread or the 4829 * main thread.) 4830 */ setCallback(@ullable Callback cb, @Nullable Handler handler)4831 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 4832 if (cb != null) { 4833 synchronized (mListenerLock) { 4834 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 4835 // NOTE: there are no callbacks on the handler at this time, but check anyways 4836 // even if we were to extend this to be callable dynamically, it must 4837 // be called when codec is flushed, so no messages are pending. 4838 if (newHandler != mCallbackHandler) { 4839 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 4840 mCallbackHandler.removeMessages(EVENT_CALLBACK); 4841 mCallbackHandler = newHandler; 4842 } 4843 } 4844 } else if (mCallbackHandler != null) { 4845 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 4846 mCallbackHandler.removeMessages(EVENT_CALLBACK); 4847 } 4848 4849 if (mCallbackHandler != null) { 4850 // set java callback on main handler 4851 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 4852 mCallbackHandler.sendMessage(msg); 4853 4854 // set native handler here, don't post to handler because 4855 // it may cause the callback to be delayed and set in a wrong state. 4856 // Note that native codec may start sending events to the callback 4857 // handler after this returns. 4858 native_setCallback(cb); 4859 } 4860 } 4861 4862 /** 4863 * Sets an asynchronous callback for actionable MediaCodec events on the default 4864 * looper. 4865 * <p> 4866 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 4867 * @param cb The callback that will run. Use {@code null} to clear a previously 4868 * set callback (before {@link #configure configure} is called and run 4869 * in synchronous mode). 4870 * @see #setCallback(Callback, Handler) 4871 */ setCallback(@ullable Callback cb)4872 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 4873 setCallback(cb, null /* handler */); 4874 } 4875 4876 /** 4877 * Listener to be called when the first output frame has been decoded 4878 * and is ready to be rendered for a codec configured for tunnel mode with 4879 * {@code KEY_AUDIO_SESSION_ID}. 4880 * 4881 * @see MediaCodec#setOnFirstTunnelFrameReadyListener 4882 */ 4883 public interface OnFirstTunnelFrameReadyListener { 4884 4885 /** 4886 * Called when the first output frame has been decoded and is ready to be 4887 * rendered. 4888 */ onFirstTunnelFrameReady(@onNull MediaCodec codec)4889 void onFirstTunnelFrameReady(@NonNull MediaCodec codec); 4890 } 4891 4892 /** 4893 * Registers a callback to be invoked when the first output frame has been decoded 4894 * and is ready to be rendered on a codec configured for tunnel mode with {@code 4895 * KEY_AUDIO_SESSION_ID}. 4896 * 4897 * @param handler the callback will be run on the handler's thread. If {@code 4898 * null}, the callback will be run on the default thread, which is the looper from 4899 * which the codec was created, or a new thread if there was none. 4900 * 4901 * @param listener the callback that will be run. If {@code null}, clears any registered 4902 * listener. 4903 */ setOnFirstTunnelFrameReadyListener( @ullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener)4904 public void setOnFirstTunnelFrameReadyListener( 4905 @Nullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener) { 4906 synchronized (mListenerLock) { 4907 mOnFirstTunnelFrameReadyListener = listener; 4908 if (listener != null) { 4909 EventHandler newHandler = getEventHandlerOn( 4910 handler, 4911 mOnFirstTunnelFrameReadyHandler); 4912 if (newHandler != mOnFirstTunnelFrameReadyHandler) { 4913 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 4914 } 4915 mOnFirstTunnelFrameReadyHandler = newHandler; 4916 } else if (mOnFirstTunnelFrameReadyHandler != null) { 4917 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 4918 } 4919 native_enableOnFirstTunnelFrameReadyListener(listener != null); 4920 } 4921 } 4922 native_enableOnFirstTunnelFrameReadyListener(boolean enable)4923 private native void native_enableOnFirstTunnelFrameReadyListener(boolean enable); 4924 4925 /** 4926 * Listener to be called when an output frame has rendered on the output surface 4927 * 4928 * @see MediaCodec#setOnFrameRenderedListener 4929 */ 4930 public interface OnFrameRenderedListener { 4931 4932 /** 4933 * Called when an output frame has rendered on the output surface. 4934 * <p> 4935 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 4936 * render timing samples, and can be significantly delayed and batched. Starting with 4937 * Android {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE}, a callback will always 4938 * be received for each rendered frame providing the MediaCodec is still in the executing 4939 * state when the callback is dispatched. Prior to Android 4940 * {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE}, some frames may have been 4941 * rendered even if there was no callback generated. 4942 * 4943 * @param codec the MediaCodec instance 4944 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 4945 * This is usually the same as specified in {@link #queueInputBuffer}; however, 4946 * some codecs may alter the media time by applying some time-based transformation, 4947 * such as frame rate conversion. In that case, presentation time corresponds 4948 * to the actual output frame rendered. 4949 * @param nanoTime The system time when the frame was rendered. 4950 * 4951 * @see System#nanoTime 4952 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)4953 public void onFrameRendered( 4954 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 4955 } 4956 4957 /** 4958 * Registers a callback to be invoked when an output frame is rendered on the output surface. 4959 * <p> 4960 * This method can be called in any codec state, but will only have an effect in the 4961 * Executing state for codecs that render buffers to the output surface. 4962 * <p> 4963 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 4964 * render timing samples, and can be significantly delayed and batched. Some frames may have 4965 * been rendered even if there was no callback generated. 4966 * 4967 * @param listener the callback that will be run 4968 * @param handler the callback will be run on the handler's thread. If {@code null}, 4969 * the callback will be run on the default thread, which is the looper 4970 * from which the codec was created, or a new thread if there was none. 4971 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)4972 public void setOnFrameRenderedListener( 4973 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 4974 synchronized (mListenerLock) { 4975 mOnFrameRenderedListener = listener; 4976 if (listener != null) { 4977 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 4978 if (newHandler != mOnFrameRenderedHandler) { 4979 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 4980 } 4981 mOnFrameRenderedHandler = newHandler; 4982 } else if (mOnFrameRenderedHandler != null) { 4983 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 4984 } 4985 native_enableOnFrameRenderedListener(listener != null); 4986 } 4987 } 4988 native_enableOnFrameRenderedListener(boolean enable)4989 private native void native_enableOnFrameRenderedListener(boolean enable); 4990 4991 /** 4992 * Returns a list of vendor parameter names. 4993 * <p> 4994 * This method can be called in any codec state except for released state. 4995 * 4996 * @return a list containing supported vendor parameters; an empty 4997 * list if no vendor parameters are supported. The order of the 4998 * parameters is arbitrary. 4999 * @throws IllegalStateException if in the Released state. 5000 */ 5001 @NonNull getSupportedVendorParameters()5002 public List<String> getSupportedVendorParameters() { 5003 return native_getSupportedVendorParameters(); 5004 } 5005 5006 @NonNull native_getSupportedVendorParameters()5007 private native List<String> native_getSupportedVendorParameters(); 5008 5009 /** 5010 * Contains description of a parameter. 5011 */ 5012 public static class ParameterDescriptor { ParameterDescriptor()5013 private ParameterDescriptor() {} 5014 5015 /** 5016 * Returns the name of the parameter. 5017 */ 5018 @NonNull getName()5019 public String getName() { 5020 return mName; 5021 } 5022 5023 /** 5024 * Returns the type of the parameter. 5025 * {@link MediaFormat#TYPE_NULL} is never returned. 5026 */ 5027 @MediaFormat.Type getType()5028 public int getType() { 5029 return mType; 5030 } 5031 5032 @Override equals(Object o)5033 public boolean equals(Object o) { 5034 if (o == null) { 5035 return false; 5036 } 5037 if (!(o instanceof ParameterDescriptor)) { 5038 return false; 5039 } 5040 ParameterDescriptor other = (ParameterDescriptor) o; 5041 return this.mName.equals(other.mName) && this.mType == other.mType; 5042 } 5043 5044 @Override hashCode()5045 public int hashCode() { 5046 return Arrays.asList( 5047 (Object) mName, 5048 (Object) Integer.valueOf(mType)).hashCode(); 5049 } 5050 5051 private String mName; 5052 private @MediaFormat.Type int mType; 5053 } 5054 5055 /** 5056 * Describe a parameter with the name. 5057 * <p> 5058 * This method can be called in any codec state except for released state. 5059 * 5060 * @param name name of the parameter to describe, typically one from 5061 * {@link #getSupportedVendorParameters}. 5062 * @return {@link ParameterDescriptor} object that describes the parameter. 5063 * {@code null} if unrecognized / not able to describe. 5064 * @throws IllegalStateException if in the Released state. 5065 */ 5066 @Nullable getParameterDescriptor(@onNull String name)5067 public ParameterDescriptor getParameterDescriptor(@NonNull String name) { 5068 return native_getParameterDescriptor(name); 5069 } 5070 5071 @Nullable native_getParameterDescriptor(@onNull String name)5072 private native ParameterDescriptor native_getParameterDescriptor(@NonNull String name); 5073 5074 /** 5075 * Subscribe to vendor parameters, so that these parameters will be present in 5076 * {@link #getOutputFormat} and changes to these parameters generate 5077 * output format change event. 5078 * <p> 5079 * Unrecognized parameter names or standard (non-vendor) parameter names will be ignored. 5080 * {@link #reset} also resets the list of subscribed parameters. 5081 * If a parameter in {@code names} is already subscribed, it will remain subscribed. 5082 * <p> 5083 * This method can be called in any codec state except for released state. When called in 5084 * running state with newly subscribed parameters, it takes effect no later than the 5085 * processing of the subsequently queued buffer. For the new parameters, the codec will generate 5086 * output format change event. 5087 * <p> 5088 * Note that any vendor parameters set in a {@link #configure} or 5089 * {@link #setParameters} call are automatically subscribed. 5090 * <p> 5091 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 5092 * for output format change events. 5093 * 5094 * @param names names of the vendor parameters to subscribe. This may be an empty list, 5095 * and in that case this method will not change the list of subscribed parameters. 5096 * @throws IllegalStateException if in the Released state. 5097 */ subscribeToVendorParameters(@onNull List<String> names)5098 public void subscribeToVendorParameters(@NonNull List<String> names) { 5099 native_subscribeToVendorParameters(names); 5100 } 5101 native_subscribeToVendorParameters(@onNull List<String> names)5102 private native void native_subscribeToVendorParameters(@NonNull List<String> names); 5103 5104 /** 5105 * Unsubscribe from vendor parameters, so that these parameters will not be present in 5106 * {@link #getOutputFormat} and changes to these parameters no longer generate 5107 * output format change event. 5108 * <p> 5109 * Unrecognized parameter names, standard (non-vendor) parameter names will be ignored. 5110 * {@link #reset} also resets the list of subscribed parameters. 5111 * If a parameter in {@code names} is already unsubscribed, it will remain unsubscribed. 5112 * <p> 5113 * This method can be called in any codec state except for released state. When called in 5114 * running state with newly unsubscribed parameters, it takes effect no later than the 5115 * processing of the subsequently queued buffer. For the removed parameters, the codec will 5116 * generate output format change event. 5117 * <p> 5118 * Note that any vendor parameters set in a {@link #configure} or 5119 * {@link #setParameters} call are automatically subscribed, and with this method 5120 * they can be unsubscribed. 5121 * <p> 5122 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 5123 * for output format change events. 5124 * 5125 * @param names names of the vendor parameters to unsubscribe. This may be an empty list, 5126 * and in that case this method will not change the list of subscribed parameters. 5127 * @throws IllegalStateException if in the Released state. 5128 */ unsubscribeFromVendorParameters(@onNull List<String> names)5129 public void unsubscribeFromVendorParameters(@NonNull List<String> names) { 5130 native_unsubscribeFromVendorParameters(names); 5131 } 5132 native_unsubscribeFromVendorParameters(@onNull List<String> names)5133 private native void native_unsubscribeFromVendorParameters(@NonNull List<String> names); 5134 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)5135 private EventHandler getEventHandlerOn( 5136 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 5137 if (handler == null) { 5138 return mEventHandler; 5139 } else { 5140 Looper looper = handler.getLooper(); 5141 if (lastHandler.getLooper() == looper) { 5142 return lastHandler; 5143 } else { 5144 return new EventHandler(this, looper); 5145 } 5146 } 5147 } 5148 5149 /** 5150 * MediaCodec callback interface. Used to notify the user asynchronously 5151 * of various MediaCodec events. 5152 */ 5153 public static abstract class Callback { 5154 /** 5155 * Called when an input buffer becomes available. 5156 * 5157 * @param codec The MediaCodec object. 5158 * @param index The index of the available input buffer. 5159 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)5160 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 5161 5162 /** 5163 * Called when an output buffer becomes available. 5164 * 5165 * @param codec The MediaCodec object. 5166 * @param index The index of the available output buffer. 5167 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 5168 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)5169 public abstract void onOutputBufferAvailable( 5170 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 5171 5172 /** 5173 * Called when the MediaCodec encountered an error 5174 * 5175 * @param codec The MediaCodec object. 5176 * @param e The {@link MediaCodec.CodecException} object describing the error. 5177 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)5178 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 5179 5180 /** 5181 * Called only when MediaCodec encountered a crypto(decryption) error when using 5182 * a decoder configured with CONFIGURE_FLAG_USE_CRYPTO_ASYNC flag along with crypto 5183 * or descrambler object. 5184 * 5185 * @param codec The MediaCodec object 5186 * @param e The {@link MediaCodec.CryptoException} object with error details. 5187 */ onCryptoError(@onNull MediaCodec codec, @NonNull CryptoException e)5188 public void onCryptoError(@NonNull MediaCodec codec, @NonNull CryptoException e) { 5189 /* 5190 * A default implementation for backward compatibility. 5191 * Use of CONFIGURE_FLAG_USE_CRYPTO_ASYNC requires override of this callback 5192 * to receive CrytoInfo. Without an orverride an exception is thrown. 5193 */ 5194 throw new IllegalStateException( 5195 "Client must override onCryptoError when the codec is " + 5196 "configured with CONFIGURE_FLAG_USE_CRYPTO_ASYNC.", e); 5197 } 5198 5199 /** 5200 * Called when the output format has changed 5201 * 5202 * @param codec The MediaCodec object. 5203 * @param format The new output format. 5204 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)5205 public abstract void onOutputFormatChanged( 5206 @NonNull MediaCodec codec, @NonNull MediaFormat format); 5207 } 5208 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)5209 private void postEventFromNative( 5210 int what, int arg1, int arg2, @Nullable Object obj) { 5211 synchronized (mListenerLock) { 5212 EventHandler handler = mEventHandler; 5213 if (what == EVENT_CALLBACK) { 5214 handler = mCallbackHandler; 5215 } else if (what == EVENT_FIRST_TUNNEL_FRAME_READY) { 5216 handler = mOnFirstTunnelFrameReadyHandler; 5217 } else if (what == EVENT_FRAME_RENDERED) { 5218 handler = mOnFrameRenderedHandler; 5219 } 5220 if (handler != null) { 5221 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 5222 handler.sendMessage(msg); 5223 } 5224 } 5225 } 5226 5227 @UnsupportedAppUsage setParameters(@onNull String[] keys, @NonNull Object[] values)5228 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 5229 5230 /** 5231 * Get the codec info. If the codec was created by createDecoderByType 5232 * or createEncoderByType, what component is chosen is not known beforehand, 5233 * and thus the caller does not have the MediaCodecInfo. 5234 * @throws IllegalStateException if in the Released state. 5235 */ 5236 @NonNull getCodecInfo()5237 public MediaCodecInfo getCodecInfo() { 5238 // Get the codec name first. If the codec is already released, 5239 // IllegalStateException will be thrown here. 5240 String name = getName(); 5241 synchronized (mCodecInfoLock) { 5242 if (mCodecInfo == null) { 5243 // Get the codec info for this codec itself first. Only initialize 5244 // the full codec list if this somehow fails because it can be slow. 5245 mCodecInfo = getOwnCodecInfo(); 5246 if (mCodecInfo == null) { 5247 mCodecInfo = MediaCodecList.getInfoFor(name); 5248 } 5249 } 5250 return mCodecInfo; 5251 } 5252 } 5253 5254 @NonNull getOwnCodecInfo()5255 private native final MediaCodecInfo getOwnCodecInfo(); 5256 5257 @NonNull 5258 @UnsupportedAppUsage getBuffers(boolean input)5259 private native final ByteBuffer[] getBuffers(boolean input); 5260 5261 @Nullable getBuffer(boolean input, int index)5262 private native final ByteBuffer getBuffer(boolean input, int index); 5263 5264 @Nullable getImage(boolean input, int index)5265 private native final Image getImage(boolean input, int index); 5266 native_init()5267 private static native final void native_init(); 5268 native_setup( @onNull String name, boolean nameIsType, boolean encoder, int pid, int uid)5269 private native final void native_setup( 5270 @NonNull String name, boolean nameIsType, boolean encoder, int pid, int uid); 5271 native_finalize()5272 private native final void native_finalize(); 5273 5274 static { 5275 System.loadLibrary("media_jni"); native_init()5276 native_init(); 5277 } 5278 5279 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) 5280 private long mNativeContext = 0; 5281 private final Lock mNativeContextLock = new ReentrantLock(); 5282 lockAndGetContext()5283 private final long lockAndGetContext() { 5284 mNativeContextLock.lock(); 5285 return mNativeContext; 5286 } 5287 setAndUnlockContext(long context)5288 private final void setAndUnlockContext(long context) { 5289 mNativeContext = context; 5290 mNativeContextLock.unlock(); 5291 } 5292 5293 /** @hide */ 5294 public static class MediaImage extends Image { 5295 private final boolean mIsReadOnly; 5296 private final int mWidth; 5297 private final int mHeight; 5298 private final int mFormat; 5299 private long mTimestamp; 5300 private final Plane[] mPlanes; 5301 private final ByteBuffer mBuffer; 5302 private final ByteBuffer mInfo; 5303 private final int mXOffset; 5304 private final int mYOffset; 5305 private final long mBufferContext; 5306 5307 private final static int TYPE_YUV = 1; 5308 5309 private final int mTransform = 0; //Default no transform 5310 private final int mScalingMode = 0; //Default frozen scaling mode 5311 5312 @Override getFormat()5313 public int getFormat() { 5314 throwISEIfImageIsInvalid(); 5315 return mFormat; 5316 } 5317 5318 @Override getHeight()5319 public int getHeight() { 5320 throwISEIfImageIsInvalid(); 5321 return mHeight; 5322 } 5323 5324 @Override getWidth()5325 public int getWidth() { 5326 throwISEIfImageIsInvalid(); 5327 return mWidth; 5328 } 5329 5330 @Override getTransform()5331 public int getTransform() { 5332 throwISEIfImageIsInvalid(); 5333 return mTransform; 5334 } 5335 5336 @Override getScalingMode()5337 public int getScalingMode() { 5338 throwISEIfImageIsInvalid(); 5339 return mScalingMode; 5340 } 5341 5342 @Override getTimestamp()5343 public long getTimestamp() { 5344 throwISEIfImageIsInvalid(); 5345 return mTimestamp; 5346 } 5347 5348 @Override 5349 @NonNull getPlanes()5350 public Plane[] getPlanes() { 5351 throwISEIfImageIsInvalid(); 5352 return Arrays.copyOf(mPlanes, mPlanes.length); 5353 } 5354 5355 @Override close()5356 public void close() { 5357 if (mIsImageValid) { 5358 if (mBuffer != null) { 5359 java.nio.NioUtils.freeDirectBuffer(mBuffer); 5360 } 5361 if (mBufferContext != 0) { 5362 native_closeMediaImage(mBufferContext); 5363 } 5364 mIsImageValid = false; 5365 } 5366 } 5367 5368 /** 5369 * Set the crop rectangle associated with this frame. 5370 * <p> 5371 * The crop rectangle specifies the region of valid pixels in the image, 5372 * using coordinates in the largest-resolution plane. 5373 */ 5374 @Override setCropRect(@ullable Rect cropRect)5375 public void setCropRect(@Nullable Rect cropRect) { 5376 if (mIsReadOnly) { 5377 throw new ReadOnlyBufferException(); 5378 } 5379 super.setCropRect(cropRect); 5380 } 5381 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)5382 public MediaImage( 5383 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 5384 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 5385 mTimestamp = timestamp; 5386 mIsImageValid = true; 5387 mIsReadOnly = buffer.isReadOnly(); 5388 mBuffer = buffer.duplicate(); 5389 5390 // save offsets and info 5391 mXOffset = xOffset; 5392 mYOffset = yOffset; 5393 mInfo = info; 5394 5395 mBufferContext = 0; 5396 5397 int cbPlaneOffset = -1; 5398 int crPlaneOffset = -1; 5399 int planeOffsetInc = -1; 5400 int pixelStride = -1; 5401 5402 // read media-info. See MediaImage2 5403 if (info.remaining() == 104) { 5404 int type = info.getInt(); 5405 if (type != TYPE_YUV) { 5406 throw new UnsupportedOperationException("unsupported type: " + type); 5407 } 5408 int numPlanes = info.getInt(); 5409 if (numPlanes != 3) { 5410 throw new RuntimeException("unexpected number of planes: " + numPlanes); 5411 } 5412 mWidth = info.getInt(); 5413 mHeight = info.getInt(); 5414 if (mWidth < 1 || mHeight < 1) { 5415 throw new UnsupportedOperationException( 5416 "unsupported size: " + mWidth + "x" + mHeight); 5417 } 5418 int bitDepth = info.getInt(); 5419 if (bitDepth != 8 && bitDepth != 10) { 5420 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 5421 } 5422 int bitDepthAllocated = info.getInt(); 5423 if (bitDepthAllocated != 8 && bitDepthAllocated != 16) { 5424 throw new UnsupportedOperationException( 5425 "unsupported allocated bit depth: " + bitDepthAllocated); 5426 } 5427 if (bitDepth == 8 && bitDepthAllocated == 8) { 5428 mFormat = ImageFormat.YUV_420_888; 5429 planeOffsetInc = 1; 5430 pixelStride = 2; 5431 } else if (bitDepth == 10 && bitDepthAllocated == 16) { 5432 mFormat = ImageFormat.YCBCR_P010; 5433 planeOffsetInc = 2; 5434 pixelStride = 4; 5435 } else { 5436 throw new UnsupportedOperationException("couldn't infer ImageFormat" 5437 + " bitDepth: " + bitDepth + " bitDepthAllocated: " + bitDepthAllocated); 5438 } 5439 5440 mPlanes = new MediaPlane[numPlanes]; 5441 for (int ix = 0; ix < numPlanes; ix++) { 5442 int planeOffset = info.getInt(); 5443 int colInc = info.getInt(); 5444 int rowInc = info.getInt(); 5445 int horiz = info.getInt(); 5446 int vert = info.getInt(); 5447 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 5448 throw new UnsupportedOperationException("unexpected subsampling: " 5449 + horiz + "x" + vert + " on plane " + ix); 5450 } 5451 if (colInc < 1 || rowInc < 1) { 5452 throw new UnsupportedOperationException("unexpected strides: " 5453 + colInc + " pixel, " + rowInc + " row on plane " + ix); 5454 } 5455 buffer.clear(); 5456 buffer.position(mBuffer.position() + planeOffset 5457 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 5458 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 5459 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 5460 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 5461 if ((mFormat == ImageFormat.YUV_420_888 || mFormat == ImageFormat.YCBCR_P010) 5462 && ix == 1) { 5463 cbPlaneOffset = planeOffset; 5464 } else if ((mFormat == ImageFormat.YUV_420_888 5465 || mFormat == ImageFormat.YCBCR_P010) && ix == 2) { 5466 crPlaneOffset = planeOffset; 5467 } 5468 } 5469 } else { 5470 throw new UnsupportedOperationException( 5471 "unsupported info length: " + info.remaining()); 5472 } 5473 5474 // Validate chroma semiplanerness. 5475 if (mFormat == ImageFormat.YCBCR_P010) { 5476 if (crPlaneOffset != cbPlaneOffset + planeOffsetInc) { 5477 throw new UnsupportedOperationException("Invalid plane offsets" 5478 + " cbPlaneOffset: " + cbPlaneOffset + " crPlaneOffset: " + crPlaneOffset); 5479 } 5480 if (mPlanes[1].getPixelStride() != pixelStride 5481 || mPlanes[2].getPixelStride() != pixelStride) { 5482 throw new UnsupportedOperationException("Invalid pixelStride"); 5483 } 5484 } 5485 5486 if (cropRect == null) { 5487 cropRect = new Rect(0, 0, mWidth, mHeight); 5488 } 5489 cropRect.offset(-xOffset, -yOffset); 5490 super.setCropRect(cropRect); 5491 } 5492 MediaImage( @onNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, int width, int height, int format, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context)5493 public MediaImage( 5494 @NonNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, 5495 int width, int height, int format, boolean readOnly, 5496 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context) { 5497 if (buffers.length != rowStrides.length || buffers.length != pixelStrides.length) { 5498 throw new IllegalArgumentException( 5499 "buffers, rowStrides and pixelStrides should have the same length"); 5500 } 5501 mWidth = width; 5502 mHeight = height; 5503 mFormat = format; 5504 mTimestamp = timestamp; 5505 mIsImageValid = true; 5506 mIsReadOnly = readOnly; 5507 mBuffer = null; 5508 mInfo = null; 5509 mPlanes = new MediaPlane[buffers.length]; 5510 for (int i = 0; i < buffers.length; ++i) { 5511 mPlanes[i] = new MediaPlane(buffers[i], rowStrides[i], pixelStrides[i]); 5512 } 5513 5514 // save offsets and info 5515 mXOffset = xOffset; 5516 mYOffset = yOffset; 5517 5518 if (cropRect == null) { 5519 cropRect = new Rect(0, 0, mWidth, mHeight); 5520 } 5521 cropRect.offset(-xOffset, -yOffset); 5522 super.setCropRect(cropRect); 5523 5524 mBufferContext = context; 5525 } 5526 5527 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)5528 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 5529 mData = buffer; 5530 mRowInc = rowInc; 5531 mColInc = colInc; 5532 } 5533 5534 @Override getRowStride()5535 public int getRowStride() { 5536 throwISEIfImageIsInvalid(); 5537 return mRowInc; 5538 } 5539 5540 @Override getPixelStride()5541 public int getPixelStride() { 5542 throwISEIfImageIsInvalid(); 5543 return mColInc; 5544 } 5545 5546 @Override 5547 @NonNull getBuffer()5548 public ByteBuffer getBuffer() { 5549 throwISEIfImageIsInvalid(); 5550 return mData; 5551 } 5552 5553 private final int mRowInc; 5554 private final int mColInc; 5555 private final ByteBuffer mData; 5556 } 5557 } 5558 5559 public final static class MetricsConstants 5560 { MetricsConstants()5561 private MetricsConstants() {} 5562 5563 /** 5564 * Key to extract the codec being used 5565 * from the {@link MediaCodec#getMetrics} return value. 5566 * The value is a String. 5567 */ 5568 public static final String CODEC = "android.media.mediacodec.codec"; 5569 5570 /** 5571 * Key to extract the MIME type 5572 * from the {@link MediaCodec#getMetrics} return value. 5573 * The value is a String. 5574 */ 5575 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 5576 5577 /** 5578 * Key to extract what the codec mode 5579 * from the {@link MediaCodec#getMetrics} return value. 5580 * The value is a String. Values will be one of the constants 5581 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 5582 */ 5583 public static final String MODE = "android.media.mediacodec.mode"; 5584 5585 /** 5586 * The value returned for the key {@link #MODE} when the 5587 * codec is a audio codec. 5588 */ 5589 public static final String MODE_AUDIO = "audio"; 5590 5591 /** 5592 * The value returned for the key {@link #MODE} when the 5593 * codec is a video codec. 5594 */ 5595 public static final String MODE_VIDEO = "video"; 5596 5597 /** 5598 * Key to extract the flag indicating whether the codec is running 5599 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 5600 * The value is an integer. 5601 * A 0 indicates decoder; 1 indicates encoder. 5602 */ 5603 public static final String ENCODER = "android.media.mediacodec.encoder"; 5604 5605 /** 5606 * Key to extract the flag indicating whether the codec is running 5607 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 5608 * The value is an integer. 5609 */ 5610 public static final String SECURE = "android.media.mediacodec.secure"; 5611 5612 /** 5613 * Key to extract the width (in pixels) of the video track 5614 * from the {@link MediaCodec#getMetrics} return value. 5615 * The value is an integer. 5616 */ 5617 public static final String WIDTH = "android.media.mediacodec.width"; 5618 5619 /** 5620 * Key to extract the height (in pixels) of the video track 5621 * from the {@link MediaCodec#getMetrics} return value. 5622 * The value is an integer. 5623 */ 5624 public static final String HEIGHT = "android.media.mediacodec.height"; 5625 5626 /** 5627 * Key to extract the rotation (in degrees) to properly orient the video 5628 * from the {@link MediaCodec#getMetrics} return. 5629 * The value is a integer. 5630 */ 5631 public static final String ROTATION = "android.media.mediacodec.rotation"; 5632 5633 } 5634 } 5635
