1 /*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "CCodecBufferChannel"
19 #include <utils/Log.h>
20
21 #include <algorithm>
22 #include <atomic>
23 #include <list>
24 #include <numeric>
25
26 #include <C2AllocatorGralloc.h>
27 #include <C2PlatformSupport.h>
28 #include <C2BlockInternal.h>
29 #include <C2Config.h>
30 #include <C2Debug.h>
31
32 #include <android/hardware/cas/native/1.0/IDescrambler.h>
33 #include <android/hardware/drm/1.0/types.h>
34 #include <android-base/properties.h>
35 #include <android-base/stringprintf.h>
36 #include <binder/MemoryBase.h>
37 #include <binder/MemoryDealer.h>
38 #include <cutils/properties.h>
39 #include <gui/Surface.h>
40 #include <hidlmemory/FrameworkUtils.h>
41 #include <media/openmax/OMX_Core.h>
42 #include <media/stagefright/foundation/ABuffer.h>
43 #include <media/stagefright/foundation/ALookup.h>
44 #include <media/stagefright/foundation/AMessage.h>
45 #include <media/stagefright/foundation/AUtils.h>
46 #include <media/stagefright/foundation/hexdump.h>
47 #include <media/stagefright/MediaCodec.h>
48 #include <media/stagefright/MediaCodecConstants.h>
49 #include <media/stagefright/SkipCutBuffer.h>
50 #include <media/MediaCodecBuffer.h>
51 #include <mediadrm/ICrypto.h>
52 #include <system/window.h>
53
54 #include "CCodecBufferChannel.h"
55 #include "Codec2Buffer.h"
56
57 namespace android {
58
59 using android::base::StringPrintf;
60 using hardware::hidl_handle;
61 using hardware::hidl_string;
62 using hardware::hidl_vec;
63 using hardware::fromHeap;
64 using hardware::HidlMemory;
65
66 using namespace hardware::cas::V1_0;
67 using namespace hardware::cas::native::V1_0;
68
69 using CasStatus = hardware::cas::V1_0::Status;
70 using DrmBufferType = hardware::drm::V1_0::BufferType;
71
72 namespace {
73
74 constexpr size_t kSmoothnessFactor = 4;
75 constexpr size_t kRenderingDepth = 3;
76
77 // This is for keeping IGBP's buffer dropping logic in legacy mode other
78 // than making it non-blocking. Do not change this value.
79 const static size_t kDequeueTimeoutNs = 0;
80
81 } // namespace
82
QueueGuard(CCodecBufferChannel::QueueSync & sync)83 CCodecBufferChannel::QueueGuard::QueueGuard(
84 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
85 Mutex::Autolock l(mSync.mGuardLock);
86 // At this point it's guaranteed that mSync is not under state transition,
87 // as we are holding its mutex.
88
89 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
90 if (count->value == -1) {
91 mRunning = false;
92 } else {
93 ++count->value;
94 mRunning = true;
95 }
96 }
97
~QueueGuard()98 CCodecBufferChannel::QueueGuard::~QueueGuard() {
99 if (mRunning) {
100 // We are not holding mGuardLock at this point so that QueueSync::stop() can
101 // keep holding the lock until mCount reaches zero.
102 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
103 --count->value;
104 count->cond.broadcast();
105 }
106 }
107
start()108 void CCodecBufferChannel::QueueSync::start() {
109 Mutex::Autolock l(mGuardLock);
110 // If stopped, it goes to running state; otherwise no-op.
111 Mutexed<Counter>::Locked count(mCount);
112 if (count->value == -1) {
113 count->value = 0;
114 }
115 }
116
stop()117 void CCodecBufferChannel::QueueSync::stop() {
118 Mutex::Autolock l(mGuardLock);
119 Mutexed<Counter>::Locked count(mCount);
120 if (count->value == -1) {
121 // no-op
122 return;
123 }
124 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
125 // mCount can only decrement. In other words, threads that acquired the lock
126 // are allowed to finish execution but additional threads trying to acquire
127 // the lock at this point will block, and then get QueueGuard at STOPPED
128 // state.
129 while (count->value != 0) {
130 count.waitForCondition(count->cond);
131 }
132 count->value = -1;
133 }
134
135 // Input
136
Input()137 CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
138
139 // CCodecBufferChannel
140
CCodecBufferChannel(const std::shared_ptr<CCodecCallback> & callback)141 CCodecBufferChannel::CCodecBufferChannel(
142 const std::shared_ptr<CCodecCallback> &callback)
143 : mHeapSeqNum(-1),
144 mCCodecCallback(callback),
145 mFrameIndex(0u),
146 mFirstValidFrameIndex(0u),
147 mMetaMode(MODE_NONE),
148 mInputMetEos(false),
149 mSendEncryptedInfoBuffer(false) {
150 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
151 {
152 Mutexed<Input>::Locked input(mInput);
153 input->buffers.reset(new DummyInputBuffers(""));
154 input->extraBuffers.flush();
155 input->inputDelay = 0u;
156 input->pipelineDelay = 0u;
157 input->numSlots = kSmoothnessFactor;
158 input->numExtraSlots = 0u;
159 input->lastFlushIndex = 0u;
160 }
161 {
162 Mutexed<Output>::Locked output(mOutput);
163 output->outputDelay = 0u;
164 output->numSlots = kSmoothnessFactor;
165 }
166 {
167 Mutexed<BlockPools>::Locked pools(mBlockPools);
168 pools->outputPoolId = C2BlockPool::BASIC_LINEAR;
169 }
170 }
171
~CCodecBufferChannel()172 CCodecBufferChannel::~CCodecBufferChannel() {
173 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
174 mCrypto->unsetHeap(mHeapSeqNum);
175 }
176 }
177
setComponent(const std::shared_ptr<Codec2Client::Component> & component)178 void CCodecBufferChannel::setComponent(
179 const std::shared_ptr<Codec2Client::Component> &component) {
180 mComponent = component;
181 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
182 mName = mComponentName.c_str();
183 }
184
setInputSurface(const std::shared_ptr<InputSurfaceWrapper> & surface)185 status_t CCodecBufferChannel::setInputSurface(
186 const std::shared_ptr<InputSurfaceWrapper> &surface) {
187 ALOGV("[%s] setInputSurface", mName);
188 mInputSurface = surface;
189 return mInputSurface->connect(mComponent);
190 }
191
signalEndOfInputStream()192 status_t CCodecBufferChannel::signalEndOfInputStream() {
193 if (mInputSurface == nullptr) {
194 return INVALID_OPERATION;
195 }
196 return mInputSurface->signalEndOfInputStream();
197 }
198
queueInputBufferInternal(sp<MediaCodecBuffer> buffer,std::shared_ptr<C2LinearBlock> encryptedBlock,size_t blockSize)199 status_t CCodecBufferChannel::queueInputBufferInternal(
200 sp<MediaCodecBuffer> buffer,
201 std::shared_ptr<C2LinearBlock> encryptedBlock,
202 size_t blockSize) {
203 int64_t timeUs;
204 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
205
206 if (mInputMetEos) {
207 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
208 return OK;
209 }
210
211 int32_t flags = 0;
212 int32_t tmp = 0;
213 bool eos = false;
214 bool tunnelFirstFrame = false;
215 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
216 eos = true;
217 mInputMetEos = true;
218 ALOGV("[%s] input EOS", mName);
219 }
220 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
221 flags |= C2FrameData::FLAG_CODEC_CONFIG;
222 }
223 if (buffer->meta()->findInt32("tunnel-first-frame", &tmp) && tmp) {
224 tunnelFirstFrame = true;
225 }
226 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
227 std::list<std::unique_ptr<C2Work>> items;
228 std::unique_ptr<C2Work> work(new C2Work);
229 work->input.ordinal.timestamp = timeUs;
230 work->input.ordinal.frameIndex = mFrameIndex++;
231 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
232 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
233 // Keep client timestamp in customOrdinal
234 work->input.ordinal.customOrdinal = timeUs;
235 work->input.buffers.clear();
236
237 sp<Codec2Buffer> copy;
238 bool usesFrameReassembler = false;
239
240 if (buffer->size() > 0u) {
241 Mutexed<Input>::Locked input(mInput);
242 std::shared_ptr<C2Buffer> c2buffer;
243 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
244 return -ENOENT;
245 }
246 // TODO: we want to delay copying buffers.
247 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
248 copy = input->buffers->cloneAndReleaseBuffer(buffer);
249 if (copy != nullptr) {
250 (void)input->extraBuffers.assignSlot(copy);
251 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
252 return UNKNOWN_ERROR;
253 }
254 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
255 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
256 mName, released ? "" : "not ");
257 buffer.clear();
258 } else {
259 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
260 "buffer starvation on component.", mName);
261 }
262 }
263 if (input->frameReassembler) {
264 usesFrameReassembler = true;
265 input->frameReassembler.process(buffer, &items);
266 } else {
267 int32_t cvo = 0;
268 if (buffer->meta()->findInt32("cvo", &cvo)) {
269 int32_t rotation = cvo % 360;
270 // change rotation to counter-clock wise.
271 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
272
273 Mutexed<OutputSurface>::Locked output(mOutputSurface);
274 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
275 output->rotation[frameIndex] = rotation;
276 }
277 work->input.buffers.push_back(c2buffer);
278 if (encryptedBlock) {
279 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
280 kParamIndexEncryptedBuffer,
281 encryptedBlock->share(0, blockSize, C2Fence())));
282 }
283 }
284 } else if (eos) {
285 flags |= C2FrameData::FLAG_END_OF_STREAM;
286 }
287 if (usesFrameReassembler) {
288 if (!items.empty()) {
289 items.front()->input.configUpdate = std::move(mParamsToBeSet);
290 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
291 }
292 } else {
293 work->input.flags = (C2FrameData::flags_t)flags;
294 // TODO: fill info's
295
296 work->input.configUpdate = std::move(mParamsToBeSet);
297 if (tunnelFirstFrame) {
298 C2StreamTunnelHoldRender::input tunnelHoldRender{
299 0u /* stream */,
300 C2_TRUE /* value */
301 };
302 work->input.configUpdate.push_back(C2Param::Copy(tunnelHoldRender));
303 }
304 work->worklets.clear();
305 work->worklets.emplace_back(new C2Worklet);
306
307 items.push_back(std::move(work));
308
309 eos = eos && buffer->size() > 0u;
310 }
311 if (eos) {
312 work.reset(new C2Work);
313 work->input.ordinal.timestamp = timeUs;
314 work->input.ordinal.frameIndex = mFrameIndex++;
315 // WORKAROUND: keep client timestamp in customOrdinal
316 work->input.ordinal.customOrdinal = timeUs;
317 work->input.buffers.clear();
318 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
319 work->worklets.emplace_back(new C2Worklet);
320 items.push_back(std::move(work));
321 }
322 c2_status_t err = C2_OK;
323 if (!items.empty()) {
324 {
325 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
326 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
327 for (const std::unique_ptr<C2Work> &work : items) {
328 watcher->onWorkQueued(
329 work->input.ordinal.frameIndex.peeku(),
330 std::vector(work->input.buffers),
331 now);
332 }
333 }
334 err = mComponent->queue(&items);
335 }
336 if (err != C2_OK) {
337 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
338 for (const std::unique_ptr<C2Work> &work : items) {
339 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
340 }
341 } else {
342 Mutexed<Input>::Locked input(mInput);
343 bool released = false;
344 if (buffer) {
345 released = input->buffers->releaseBuffer(buffer, nullptr, true);
346 } else if (copy) {
347 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
348 }
349 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
350 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
351 }
352
353 feedInputBufferIfAvailableInternal();
354 return err;
355 }
356
setParameters(std::vector<std::unique_ptr<C2Param>> & params)357 status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> ¶ms) {
358 QueueGuard guard(mSync);
359 if (!guard.isRunning()) {
360 ALOGD("[%s] setParameters is only supported in the running state.", mName);
361 return -ENOSYS;
362 }
363 mParamsToBeSet.insert(mParamsToBeSet.end(),
364 std::make_move_iterator(params.begin()),
365 std::make_move_iterator(params.end()));
366 params.clear();
367 return OK;
368 }
369
attachBuffer(const std::shared_ptr<C2Buffer> & c2Buffer,const sp<MediaCodecBuffer> & buffer)370 status_t CCodecBufferChannel::attachBuffer(
371 const std::shared_ptr<C2Buffer> &c2Buffer,
372 const sp<MediaCodecBuffer> &buffer) {
373 if (!buffer->copy(c2Buffer)) {
374 return -ENOSYS;
375 }
376 return OK;
377 }
378
ensureDecryptDestination(size_t size)379 void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
380 if (!mDecryptDestination || mDecryptDestination->size() < size) {
381 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
382 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
383 mCrypto->unsetHeap(mHeapSeqNum);
384 }
385 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
386 if (mCrypto) {
387 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
388 }
389 }
390 }
391
getHeapSeqNum(const sp<HidlMemory> & memory)392 int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
393 CHECK(mCrypto);
394 auto it = mHeapSeqNumMap.find(memory);
395 int32_t heapSeqNum = -1;
396 if (it == mHeapSeqNumMap.end()) {
397 heapSeqNum = mCrypto->setHeap(memory);
398 mHeapSeqNumMap.emplace(memory, heapSeqNum);
399 } else {
400 heapSeqNum = it->second;
401 }
402 return heapSeqNum;
403 }
404
attachEncryptedBuffer(const sp<hardware::HidlMemory> & memory,bool secure,const uint8_t * key,const uint8_t * iv,CryptoPlugin::Mode mode,CryptoPlugin::Pattern pattern,size_t offset,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,const sp<MediaCodecBuffer> & buffer)405 status_t CCodecBufferChannel::attachEncryptedBuffer(
406 const sp<hardware::HidlMemory> &memory,
407 bool secure,
408 const uint8_t *key,
409 const uint8_t *iv,
410 CryptoPlugin::Mode mode,
411 CryptoPlugin::Pattern pattern,
412 size_t offset,
413 const CryptoPlugin::SubSample *subSamples,
414 size_t numSubSamples,
415 const sp<MediaCodecBuffer> &buffer) {
416 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
417 static const C2MemoryUsage kDefaultReadWriteUsage{
418 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
419
420 size_t size = 0;
421 for (size_t i = 0; i < numSubSamples; ++i) {
422 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
423 }
424 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
425 std::shared_ptr<C2LinearBlock> block;
426 c2_status_t err = pool->fetchLinearBlock(
427 size,
428 secure ? kSecureUsage : kDefaultReadWriteUsage,
429 &block);
430 if (err != C2_OK) {
431 return NO_MEMORY;
432 }
433 if (!secure) {
434 ensureDecryptDestination(size);
435 }
436 ssize_t result = -1;
437 ssize_t codecDataOffset = 0;
438 if (mCrypto) {
439 AString errorDetailMsg;
440 int32_t heapSeqNum = getHeapSeqNum(memory);
441 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
442 hardware::drm::V1_0::DestinationBuffer dst;
443 if (secure) {
444 dst.type = DrmBufferType::NATIVE_HANDLE;
445 dst.secureMemory = hardware::hidl_handle(block->handle());
446 } else {
447 dst.type = DrmBufferType::SHARED_MEMORY;
448 IMemoryToSharedBuffer(
449 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
450 }
451 result = mCrypto->decrypt(
452 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
453 dst, &errorDetailMsg);
454 if (result < 0) {
455 return result;
456 }
457 if (dst.type == DrmBufferType::SHARED_MEMORY) {
458 C2WriteView view = block->map().get();
459 if (view.error() != C2_OK) {
460 return false;
461 }
462 if (view.size() < result) {
463 return false;
464 }
465 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
466 }
467 } else {
468 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
469 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
470 hidl_vec<SubSample> hidlSubSamples;
471 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
472
473 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
474 hardware::cas::native::V1_0::DestinationBuffer dst;
475 if (secure) {
476 dst.type = BufferType::NATIVE_HANDLE;
477 dst.secureMemory = hardware::hidl_handle(block->handle());
478 } else {
479 dst.type = BufferType::SHARED_MEMORY;
480 dst.nonsecureMemory = src;
481 }
482
483 CasStatus status = CasStatus::OK;
484 hidl_string detailedError;
485 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
486
487 if (key != nullptr) {
488 sctrl = (ScramblingControl)key[0];
489 // Adjust for the PES offset
490 codecDataOffset = key[2] | (key[3] << 8);
491 }
492
493 auto returnVoid = mDescrambler->descramble(
494 sctrl,
495 hidlSubSamples,
496 src,
497 0,
498 dst,
499 0,
500 [&status, &result, &detailedError] (
501 CasStatus _status, uint32_t _bytesWritten,
502 const hidl_string& _detailedError) {
503 status = _status;
504 result = (ssize_t)_bytesWritten;
505 detailedError = _detailedError;
506 });
507
508 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
509 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
510 mName, returnVoid.description().c_str(), status, result);
511 return UNKNOWN_ERROR;
512 }
513
514 if (result < codecDataOffset) {
515 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
516 return BAD_VALUE;
517 }
518 }
519 if (!secure) {
520 C2WriteView view = block->map().get();
521 if (view.error() != C2_OK) {
522 return UNKNOWN_ERROR;
523 }
524 if (view.size() < result) {
525 return UNKNOWN_ERROR;
526 }
527 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
528 }
529 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
530 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
531 if (!buffer->copy(c2Buffer)) {
532 return -ENOSYS;
533 }
534 return OK;
535 }
536
queueInputBuffer(const sp<MediaCodecBuffer> & buffer)537 status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
538 QueueGuard guard(mSync);
539 if (!guard.isRunning()) {
540 ALOGD("[%s] No more buffers should be queued at current state.", mName);
541 return -ENOSYS;
542 }
543 return queueInputBufferInternal(buffer);
544 }
545
queueSecureInputBuffer(const sp<MediaCodecBuffer> & buffer,bool secure,const uint8_t * key,const uint8_t * iv,CryptoPlugin::Mode mode,CryptoPlugin::Pattern pattern,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,AString * errorDetailMsg)546 status_t CCodecBufferChannel::queueSecureInputBuffer(
547 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
548 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
549 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
550 AString *errorDetailMsg) {
551 QueueGuard guard(mSync);
552 if (!guard.isRunning()) {
553 ALOGD("[%s] No more buffers should be queued at current state.", mName);
554 return -ENOSYS;
555 }
556
557 if (!hasCryptoOrDescrambler()) {
558 return -ENOSYS;
559 }
560 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
561
562 std::shared_ptr<C2LinearBlock> block;
563 size_t allocSize = buffer->size();
564 size_t bufferSize = 0;
565 c2_status_t blockRes = C2_OK;
566 bool copied = false;
567 if (mSendEncryptedInfoBuffer) {
568 static const C2MemoryUsage kDefaultReadWriteUsage{
569 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
570 constexpr int kAllocGranule0 = 1024 * 64;
571 constexpr int kAllocGranule1 = 1024 * 1024;
572 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
573 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
574 if (allocSize <= kAllocGranule1) {
575 bufferSize = align(allocSize, kAllocGranule0);
576 } else {
577 bufferSize = align(allocSize, kAllocGranule1);
578 }
579 blockRes = pool->fetchLinearBlock(
580 bufferSize, kDefaultReadWriteUsage, &block);
581
582 if (blockRes == C2_OK) {
583 C2WriteView view = block->map().get();
584 if (view.error() == C2_OK && view.size() == bufferSize) {
585 copied = true;
586 // TODO: only copy clear sections
587 memcpy(view.data(), buffer->data(), allocSize);
588 }
589 }
590 }
591
592 if (!copied) {
593 block.reset();
594 }
595
596 ssize_t result = -1;
597 ssize_t codecDataOffset = 0;
598 if (numSubSamples == 1
599 && subSamples[0].mNumBytesOfClearData == 0
600 && subSamples[0].mNumBytesOfEncryptedData == 0) {
601 // We don't need to go through crypto or descrambler if the input is empty.
602 result = 0;
603 } else if (mCrypto != nullptr) {
604 hardware::drm::V1_0::DestinationBuffer destination;
605 if (secure) {
606 destination.type = DrmBufferType::NATIVE_HANDLE;
607 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
608 } else {
609 destination.type = DrmBufferType::SHARED_MEMORY;
610 IMemoryToSharedBuffer(
611 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
612 }
613 hardware::drm::V1_0::SharedBuffer source;
614 encryptedBuffer->fillSourceBuffer(&source);
615 result = mCrypto->decrypt(
616 key, iv, mode, pattern, source, buffer->offset(),
617 subSamples, numSubSamples, destination, errorDetailMsg);
618 if (result < 0) {
619 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
620 return result;
621 }
622 if (destination.type == DrmBufferType::SHARED_MEMORY) {
623 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
624 }
625 } else {
626 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
627 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
628 hidl_vec<SubSample> hidlSubSamples;
629 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
630
631 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
632 encryptedBuffer->fillSourceBuffer(&srcBuffer);
633
634 DestinationBuffer dstBuffer;
635 if (secure) {
636 dstBuffer.type = BufferType::NATIVE_HANDLE;
637 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
638 } else {
639 dstBuffer.type = BufferType::SHARED_MEMORY;
640 dstBuffer.nonsecureMemory = srcBuffer;
641 }
642
643 CasStatus status = CasStatus::OK;
644 hidl_string detailedError;
645 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
646
647 if (key != nullptr) {
648 sctrl = (ScramblingControl)key[0];
649 // Adjust for the PES offset
650 codecDataOffset = key[2] | (key[3] << 8);
651 }
652
653 auto returnVoid = mDescrambler->descramble(
654 sctrl,
655 hidlSubSamples,
656 srcBuffer,
657 0,
658 dstBuffer,
659 0,
660 [&status, &result, &detailedError] (
661 CasStatus _status, uint32_t _bytesWritten,
662 const hidl_string& _detailedError) {
663 status = _status;
664 result = (ssize_t)_bytesWritten;
665 detailedError = _detailedError;
666 });
667
668 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
669 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
670 mName, returnVoid.description().c_str(), status, result);
671 return UNKNOWN_ERROR;
672 }
673
674 if (result < codecDataOffset) {
675 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
676 return BAD_VALUE;
677 }
678
679 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
680
681 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
682 encryptedBuffer->copyDecryptedContentFromMemory(result);
683 }
684 }
685
686 buffer->setRange(codecDataOffset, result - codecDataOffset);
687
688 return queueInputBufferInternal(buffer, block, bufferSize);
689 }
690
feedInputBufferIfAvailable()691 void CCodecBufferChannel::feedInputBufferIfAvailable() {
692 QueueGuard guard(mSync);
693 if (!guard.isRunning()) {
694 ALOGV("[%s] We're not running --- no input buffer reported", mName);
695 return;
696 }
697 feedInputBufferIfAvailableInternal();
698 }
699
feedInputBufferIfAvailableInternal()700 void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
701 if (mInputMetEos) {
702 return;
703 }
704 {
705 Mutexed<Output>::Locked output(mOutput);
706 if (!output->buffers ||
707 output->buffers->hasPending() ||
708 output->buffers->numActiveSlots() >= output->numSlots) {
709 return;
710 }
711 }
712 size_t numActiveSlots = 0;
713 while (!mPipelineWatcher.lock()->pipelineFull()) {
714 sp<MediaCodecBuffer> inBuffer;
715 size_t index;
716 {
717 Mutexed<Input>::Locked input(mInput);
718 numActiveSlots = input->buffers->numActiveSlots();
719 if (numActiveSlots >= input->numSlots) {
720 break;
721 }
722 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
723 ALOGV("[%s] no new buffer available", mName);
724 break;
725 }
726 }
727 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
728 mCallback->onInputBufferAvailable(index, inBuffer);
729 }
730 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
731 }
732
renderOutputBuffer(const sp<MediaCodecBuffer> & buffer,int64_t timestampNs)733 status_t CCodecBufferChannel::renderOutputBuffer(
734 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
735 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
736 std::shared_ptr<C2Buffer> c2Buffer;
737 bool released = false;
738 {
739 Mutexed<Output>::Locked output(mOutput);
740 if (output->buffers) {
741 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
742 }
743 }
744 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
745 // set to true.
746 sendOutputBuffers();
747 // input buffer feeding may have been gated by pending output buffers
748 feedInputBufferIfAvailable();
749 if (!c2Buffer) {
750 if (released) {
751 std::call_once(mRenderWarningFlag, [this] {
752 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
753 "timestamp or render=true with non-video buffers. Apps should "
754 "call releaseOutputBuffer() with render=false for those.",
755 mName);
756 });
757 }
758 return INVALID_OPERATION;
759 }
760
761 #if 0
762 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
763 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
764 for (const std::shared_ptr<const C2Info> &info : infoParams) {
765 AString res;
766 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
767 if (ix) res.append(", ");
768 res.append(*((int32_t*)info.get() + (ix / 4)));
769 }
770 ALOGV(" [%s]", res.c_str());
771 }
772 #endif
773 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
774 std::static_pointer_cast<const C2StreamRotationInfo::output>(
775 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
776 bool flip = rotation && (rotation->flip & 1);
777 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
778
779 {
780 Mutexed<OutputSurface>::Locked output(mOutputSurface);
781 if (output->surface == nullptr) {
782 ALOGI("[%s] cannot render buffer without surface", mName);
783 return OK;
784 }
785 int64_t frameIndex;
786 buffer->meta()->findInt64("frameIndex", &frameIndex);
787 if (output->rotation.count(frameIndex) != 0) {
788 auto it = output->rotation.find(frameIndex);
789 quarters = (it->second / 90) & 3;
790 output->rotation.erase(it);
791 }
792 }
793
794 uint32_t transform = 0;
795 switch (quarters) {
796 case 0: // no rotation
797 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
798 break;
799 case 1: // 90 degrees counter-clockwise
800 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
801 : HAL_TRANSFORM_ROT_270;
802 break;
803 case 2: // 180 degrees
804 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
805 break;
806 case 3: // 90 degrees clockwise
807 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
808 : HAL_TRANSFORM_ROT_90;
809 break;
810 }
811
812 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
813 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
814 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
815 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
816 if (surfaceScaling) {
817 videoScalingMode = surfaceScaling->value;
818 }
819
820 // Use dataspace from format as it has the default aspects already applied
821 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
822 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
823
824 // HDR static info
825 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
826 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
827 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
828
829 // HDR10 plus info
830 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
831 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
832 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
833 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
834 hdr10PlusInfo.reset();
835 }
836
837 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
838 if (blocks.size() != 1u) {
839 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
840 return UNKNOWN_ERROR;
841 }
842 const C2ConstGraphicBlock &block = blocks.front();
843
844 // TODO: revisit this after C2Fence implementation.
845 android::IGraphicBufferProducer::QueueBufferInput qbi(
846 timestampNs,
847 false, // droppable
848 dataSpace,
849 Rect(blocks.front().crop().left,
850 blocks.front().crop().top,
851 blocks.front().crop().right(),
852 blocks.front().crop().bottom()),
853 videoScalingMode,
854 transform,
855 Fence::NO_FENCE, 0);
856 if (hdrStaticInfo || hdr10PlusInfo) {
857 HdrMetadata hdr;
858 if (hdrStaticInfo) {
859 // If mastering max and min luminance fields are 0, do not use them.
860 // It indicates the value may not be present in the stream.
861 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
862 hdrStaticInfo->mastering.minLuminance > 0.0f) {
863 struct android_smpte2086_metadata smpte2086_meta = {
864 .displayPrimaryRed = {
865 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
866 },
867 .displayPrimaryGreen = {
868 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
869 },
870 .displayPrimaryBlue = {
871 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
872 },
873 .whitePoint = {
874 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
875 },
876 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
877 .minLuminance = hdrStaticInfo->mastering.minLuminance,
878 };
879 hdr.validTypes |= HdrMetadata::SMPTE2086;
880 hdr.smpte2086 = smpte2086_meta;
881 }
882 // If the content light level fields are 0, do not use them, it
883 // indicates the value may not be present in the stream.
884 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
885 struct android_cta861_3_metadata cta861_meta = {
886 .maxContentLightLevel = hdrStaticInfo->maxCll,
887 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
888 };
889 hdr.validTypes |= HdrMetadata::CTA861_3;
890 hdr.cta8613 = cta861_meta;
891 }
892 }
893 if (hdr10PlusInfo) {
894 hdr.validTypes |= HdrMetadata::HDR10PLUS;
895 hdr.hdr10plus.assign(
896 hdr10PlusInfo->m.value,
897 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
898 }
899 qbi.setHdrMetadata(hdr);
900 }
901 // we don't have dirty regions
902 qbi.setSurfaceDamage(Region::INVALID_REGION);
903 android::IGraphicBufferProducer::QueueBufferOutput qbo;
904 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
905 if (result != OK) {
906 ALOGI("[%s] queueBuffer failed: %d", mName, result);
907 if (result == NO_INIT) {
908 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
909 }
910 return result;
911 }
912
913 if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
914 ALOGD("[%s] queue buffer successful", mName);
915 } else {
916 ALOGV("[%s] queue buffer successful", mName);
917 }
918
919 int64_t mediaTimeUs = 0;
920 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
921 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
922
923 return OK;
924 }
925
discardBuffer(const sp<MediaCodecBuffer> & buffer)926 status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
927 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
928 bool released = false;
929 {
930 Mutexed<Input>::Locked input(mInput);
931 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
932 released = true;
933 }
934 }
935 {
936 Mutexed<Output>::Locked output(mOutput);
937 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
938 released = true;
939 }
940 }
941 if (released) {
942 sendOutputBuffers();
943 feedInputBufferIfAvailable();
944 } else {
945 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
946 }
947 return OK;
948 }
949
getInputBufferArray(Vector<sp<MediaCodecBuffer>> * array)950 void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
951 array->clear();
952 Mutexed<Input>::Locked input(mInput);
953
954 if (!input->buffers->isArrayMode()) {
955 input->buffers = input->buffers->toArrayMode(input->numSlots);
956 }
957
958 input->buffers->getArray(array);
959 }
960
getOutputBufferArray(Vector<sp<MediaCodecBuffer>> * array)961 void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
962 array->clear();
963 Mutexed<Output>::Locked output(mOutput);
964
965 if (!output->buffers->isArrayMode()) {
966 output->buffers = output->buffers->toArrayMode(output->numSlots);
967 }
968
969 output->buffers->getArray(array);
970 }
971
start(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,bool buffersBoundToCodec)972 status_t CCodecBufferChannel::start(
973 const sp<AMessage> &inputFormat,
974 const sp<AMessage> &outputFormat,
975 bool buffersBoundToCodec) {
976 C2StreamBufferTypeSetting::input iStreamFormat(0u);
977 C2StreamBufferTypeSetting::output oStreamFormat(0u);
978 C2ComponentKindSetting kind;
979 C2PortReorderBufferDepthTuning::output reorderDepth;
980 C2PortReorderKeySetting::output reorderKey;
981 C2PortActualDelayTuning::input inputDelay(0);
982 C2PortActualDelayTuning::output outputDelay(0);
983 C2ActualPipelineDelayTuning pipelineDelay(0);
984 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
985
986 c2_status_t err = mComponent->query(
987 {
988 &iStreamFormat,
989 &oStreamFormat,
990 &kind,
991 &reorderDepth,
992 &reorderKey,
993 &inputDelay,
994 &pipelineDelay,
995 &outputDelay,
996 &secureMode,
997 },
998 {},
999 C2_DONT_BLOCK,
1000 nullptr);
1001 if (err == C2_BAD_INDEX) {
1002 if (!iStreamFormat || !oStreamFormat || !kind) {
1003 return UNKNOWN_ERROR;
1004 }
1005 } else if (err != C2_OK) {
1006 return UNKNOWN_ERROR;
1007 }
1008
1009 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
1010 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
1011 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
1012
1013 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
1014 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
1015
1016 // TODO: get this from input format
1017 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1018
1019 // secure mode is a static parameter (shall not change in the executing state)
1020 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1021
1022 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
1023 int poolMask = GetCodec2PoolMask();
1024 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
1025
1026 if (inputFormat != nullptr) {
1027 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
1028 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
1029 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1030 API_REFLECTION |
1031 API_VALUES |
1032 API_CURRENT_VALUES |
1033 API_DEPENDENCY |
1034 API_SAME_INPUT_BUFFER);
1035 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1036 C2StreamSampleRateInfo::input sampleRate(0u);
1037 C2StreamChannelCountInfo::input channelCount(0u);
1038 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
1039 std::shared_ptr<C2BlockPool> pool;
1040 {
1041 Mutexed<BlockPools>::Locked pools(mBlockPools);
1042
1043 // set default allocator ID.
1044 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
1045 : preferredLinearId;
1046
1047 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1048 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1049 std::vector<std::unique_ptr<C2Param>> params;
1050 C2ApiFeaturesSetting featuresSetting{apiFeatures};
1051 std::vector<C2Param *> stackParams({&featuresSetting});
1052 if (audioEncoder) {
1053 stackParams.push_back(&encoderFrameSize);
1054 stackParams.push_back(&sampleRate);
1055 stackParams.push_back(&channelCount);
1056 stackParams.push_back(&pcmEncoding);
1057 } else {
1058 encoderFrameSize.invalidate();
1059 sampleRate.invalidate();
1060 channelCount.invalidate();
1061 pcmEncoding.invalidate();
1062 }
1063 err = mComponent->query(stackParams,
1064 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1065 C2_DONT_BLOCK,
1066 ¶ms);
1067 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1068 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1069 mName, params.size(), asString(err), err);
1070 } else if (params.size() == 1) {
1071 C2PortAllocatorsTuning::input *inputAllocators =
1072 C2PortAllocatorsTuning::input::From(params[0].get());
1073 if (inputAllocators && inputAllocators->flexCount() > 0) {
1074 std::shared_ptr<C2Allocator> allocator;
1075 // verify allocator IDs and resolve default allocator
1076 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1077 if (allocator) {
1078 pools->inputAllocatorId = allocator->getId();
1079 } else {
1080 ALOGD("[%s] component requested invalid input allocator ID %u",
1081 mName, inputAllocators->m.values[0]);
1082 }
1083 }
1084 }
1085 if (featuresSetting) {
1086 apiFeatures = featuresSetting.value;
1087 }
1088
1089 // TODO: use C2Component wrapper to associate this pool with ourselves
1090 if ((poolMask >> pools->inputAllocatorId) & 1) {
1091 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1092 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1093 mName, pools->inputAllocatorId,
1094 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1095 asString(err), err);
1096 } else {
1097 err = C2_NOT_FOUND;
1098 }
1099 if (err != C2_OK) {
1100 C2BlockPool::local_id_t inputPoolId =
1101 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1102 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1103 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1104 mName, (unsigned long long)inputPoolId,
1105 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1106 asString(err), err);
1107 if (err != C2_OK) {
1108 return NO_MEMORY;
1109 }
1110 }
1111 pools->inputPool = pool;
1112 }
1113
1114 bool forceArrayMode = false;
1115 Mutexed<Input>::Locked input(mInput);
1116 input->inputDelay = inputDelayValue;
1117 input->pipelineDelay = pipelineDelayValue;
1118 input->numSlots = numInputSlots;
1119 input->extraBuffers.flush();
1120 input->numExtraSlots = 0u;
1121 input->lastFlushIndex = mFrameIndex.load(std::memory_order_relaxed);
1122 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1123 input->frameReassembler.init(
1124 pool,
1125 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1126 encoderFrameSize.value,
1127 sampleRate.value,
1128 channelCount.value,
1129 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1130 }
1131 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1132 // For encrypted content, framework decrypts source buffer (ashmem) into
1133 // C2Buffers. Thus non-conforming codecs can process these.
1134 if (!buffersBoundToCodec
1135 && !input->frameReassembler
1136 && (hasCryptoOrDescrambler() || conforming)) {
1137 input->buffers.reset(new SlotInputBuffers(mName));
1138 } else if (graphic) {
1139 if (mInputSurface) {
1140 input->buffers.reset(new DummyInputBuffers(mName));
1141 } else if (mMetaMode == MODE_ANW) {
1142 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
1143 // This is to ensure buffers do not get released prematurely.
1144 // TODO: handle this without going into array mode
1145 forceArrayMode = true;
1146 } else {
1147 input->buffers.reset(new GraphicInputBuffers(mName));
1148 }
1149 } else {
1150 if (hasCryptoOrDescrambler()) {
1151 int32_t capacity = kLinearBufferSize;
1152 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1153 if ((size_t)capacity > kMaxLinearBufferSize) {
1154 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1155 capacity = kMaxLinearBufferSize;
1156 }
1157 if (mDealer == nullptr) {
1158 mDealer = new MemoryDealer(
1159 align(capacity, MemoryDealer::getAllocationAlignment())
1160 * (numInputSlots + 1),
1161 "EncryptedLinearInputBuffers");
1162 mDecryptDestination = mDealer->allocate((size_t)capacity);
1163 }
1164 if (mCrypto != nullptr && mHeapSeqNum < 0) {
1165 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1166 mHeapSeqNum = mCrypto->setHeap(heap);
1167 } else {
1168 mHeapSeqNum = -1;
1169 }
1170 input->buffers.reset(new EncryptedLinearInputBuffers(
1171 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
1172 numInputSlots, mName));
1173 forceArrayMode = true;
1174 } else {
1175 input->buffers.reset(new LinearInputBuffers(mName));
1176 }
1177 }
1178 input->buffers->setFormat(inputFormat);
1179
1180 if (err == C2_OK) {
1181 input->buffers->setPool(pool);
1182 } else {
1183 // TODO: error
1184 }
1185
1186 if (forceArrayMode) {
1187 input->buffers = input->buffers->toArrayMode(numInputSlots);
1188 }
1189 }
1190
1191 if (outputFormat != nullptr) {
1192 sp<IGraphicBufferProducer> outputSurface;
1193 uint32_t outputGeneration;
1194 int maxDequeueCount = 0;
1195 {
1196 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1197 maxDequeueCount = output->maxDequeueBuffers = numOutputSlots +
1198 reorderDepth.value + kRenderingDepth;
1199 outputSurface = output->surface ?
1200 output->surface->getIGraphicBufferProducer() : nullptr;
1201 if (outputSurface) {
1202 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1203 }
1204 outputGeneration = output->generation;
1205 }
1206
1207 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
1208 C2BlockPool::local_id_t outputPoolId_;
1209 C2BlockPool::local_id_t prevOutputPoolId;
1210
1211 {
1212 Mutexed<BlockPools>::Locked pools(mBlockPools);
1213
1214 prevOutputPoolId = pools->outputPoolId;
1215
1216 // set default allocator ID.
1217 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
1218 : preferredLinearId;
1219
1220 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1221 // unsuccessful.
1222 std::vector<std::unique_ptr<C2Param>> params;
1223 err = mComponent->query({ },
1224 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1225 C2_DONT_BLOCK,
1226 ¶ms);
1227 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1228 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1229 mName, params.size(), asString(err), err);
1230 } else if (err == C2_OK && params.size() == 1) {
1231 C2PortAllocatorsTuning::output *outputAllocators =
1232 C2PortAllocatorsTuning::output::From(params[0].get());
1233 if (outputAllocators && outputAllocators->flexCount() > 0) {
1234 std::shared_ptr<C2Allocator> allocator;
1235 // verify allocator IDs and resolve default allocator
1236 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1237 if (allocator) {
1238 pools->outputAllocatorId = allocator->getId();
1239 } else {
1240 ALOGD("[%s] component requested invalid output allocator ID %u",
1241 mName, outputAllocators->m.values[0]);
1242 }
1243 }
1244 }
1245
1246 // use bufferqueue if outputting to a surface.
1247 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1248 // if unsuccessful.
1249 if (outputSurface) {
1250 params.clear();
1251 err = mComponent->query({ },
1252 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1253 C2_DONT_BLOCK,
1254 ¶ms);
1255 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1256 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1257 mName, params.size(), asString(err), err);
1258 } else if (err == C2_OK && params.size() == 1) {
1259 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1260 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1261 if (surfaceAllocator) {
1262 std::shared_ptr<C2Allocator> allocator;
1263 // verify allocator IDs and resolve default allocator
1264 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1265 if (allocator) {
1266 pools->outputAllocatorId = allocator->getId();
1267 } else {
1268 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1269 mName, surfaceAllocator->value);
1270 err = C2_BAD_VALUE;
1271 }
1272 }
1273 }
1274 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1275 && err != C2_OK
1276 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1277 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1278 }
1279 }
1280
1281 if ((poolMask >> pools->outputAllocatorId) & 1) {
1282 err = mComponent->createBlockPool(
1283 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1284 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1285 mName, pools->outputAllocatorId,
1286 (unsigned long long)pools->outputPoolId,
1287 asString(err));
1288 } else {
1289 err = C2_NOT_FOUND;
1290 }
1291 if (err != C2_OK) {
1292 // use basic pool instead
1293 pools->outputPoolId =
1294 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1295 }
1296
1297 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1298 // component.
1299 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1300 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1301
1302 std::vector<std::unique_ptr<C2SettingResult>> failures;
1303 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1304 ALOGD("[%s] Configured output block pool ids %llu => %s",
1305 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1306 outputPoolId_ = pools->outputPoolId;
1307 }
1308
1309 if (prevOutputPoolId != C2BlockPool::BASIC_LINEAR
1310 && prevOutputPoolId != C2BlockPool::BASIC_GRAPHIC) {
1311 c2_status_t err = mComponent->destroyBlockPool(prevOutputPoolId);
1312 if (err != C2_OK) {
1313 ALOGW("Failed to clean up previous block pool %llu - %s (%d)\n",
1314 (unsigned long long) prevOutputPoolId, asString(err), err);
1315 }
1316 }
1317
1318 Mutexed<Output>::Locked output(mOutput);
1319 output->outputDelay = outputDelayValue;
1320 output->numSlots = numOutputSlots;
1321 if (graphic) {
1322 if (outputSurface || !buffersBoundToCodec) {
1323 output->buffers.reset(new GraphicOutputBuffers(mName));
1324 } else {
1325 output->buffers.reset(new RawGraphicOutputBuffers(mName));
1326 }
1327 } else {
1328 output->buffers.reset(new LinearOutputBuffers(mName));
1329 }
1330 output->buffers->setFormat(outputFormat);
1331
1332 output->buffers->clearStash();
1333 if (reorderDepth) {
1334 output->buffers->setReorderDepth(reorderDepth.value);
1335 }
1336 if (reorderKey) {
1337 output->buffers->setReorderKey(reorderKey.value);
1338 }
1339
1340 // Try to set output surface to created block pool if given.
1341 if (outputSurface) {
1342 mComponent->setOutputSurface(
1343 outputPoolId_,
1344 outputSurface,
1345 outputGeneration,
1346 maxDequeueCount);
1347 } else {
1348 // configure CPU read consumer usage
1349 C2StreamUsageTuning::output outputUsage{0u, C2MemoryUsage::CPU_READ};
1350 std::vector<std::unique_ptr<C2SettingResult>> failures;
1351 err = mComponent->config({ &outputUsage }, C2_MAY_BLOCK, &failures);
1352 // do not print error message for now as most components may not yet
1353 // support this setting
1354 ALOGD_IF(err != C2_BAD_INDEX, "[%s] Configured output usage [%#llx]",
1355 mName, (long long)outputUsage.value);
1356 }
1357
1358 if (oStreamFormat.value == C2BufferData::LINEAR) {
1359 if (buffersBoundToCodec) {
1360 // WORKAROUND: if we're using early CSD workaround we convert to
1361 // array mode, to appease apps assuming the output
1362 // buffers to be of the same size.
1363 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1364 }
1365
1366 int32_t channelCount;
1367 int32_t sampleRate;
1368 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1369 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1370 int32_t delay = 0;
1371 int32_t padding = 0;;
1372 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1373 delay = 0;
1374 }
1375 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1376 padding = 0;
1377 }
1378 if (delay || padding) {
1379 // We need write access to the buffers, and we're already in
1380 // array mode.
1381 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
1382 }
1383 }
1384 }
1385 }
1386
1387 // Set up pipeline control. This has to be done after mInputBuffers and
1388 // mOutputBuffers are initialized to make sure that lingering callbacks
1389 // about buffers from the previous generation do not interfere with the
1390 // newly initialized pipeline capacity.
1391
1392 if (inputFormat || outputFormat) {
1393 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1394 watcher->inputDelay(inputDelayValue)
1395 .pipelineDelay(pipelineDelayValue)
1396 .outputDelay(outputDelayValue)
1397 .smoothnessFactor(kSmoothnessFactor);
1398 watcher->flush();
1399 }
1400
1401 mInputMetEos = false;
1402 mSync.start();
1403 return OK;
1404 }
1405
requestInitialInputBuffers()1406 status_t CCodecBufferChannel::requestInitialInputBuffers() {
1407 if (mInputSurface) {
1408 return OK;
1409 }
1410
1411 C2StreamBufferTypeSetting::output oStreamFormat(0u);
1412 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1413 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1414 if (err != C2_OK && err != C2_BAD_INDEX) {
1415 return UNKNOWN_ERROR;
1416 }
1417 size_t numInputSlots = mInput.lock()->numSlots;
1418
1419 struct ClientInputBuffer {
1420 size_t index;
1421 sp<MediaCodecBuffer> buffer;
1422 size_t capacity;
1423 };
1424 std::list<ClientInputBuffer> clientInputBuffers;
1425
1426 {
1427 Mutexed<Input>::Locked input(mInput);
1428 while (clientInputBuffers.size() < numInputSlots) {
1429 ClientInputBuffer clientInputBuffer;
1430 if (!input->buffers->requestNewBuffer(&clientInputBuffer.index,
1431 &clientInputBuffer.buffer)) {
1432 break;
1433 }
1434 clientInputBuffer.capacity = clientInputBuffer.buffer->capacity();
1435 clientInputBuffers.emplace_back(std::move(clientInputBuffer));
1436 }
1437 }
1438 if (clientInputBuffers.empty()) {
1439 ALOGW("[%s] start: cannot allocate memory at all", mName);
1440 return NO_MEMORY;
1441 } else if (clientInputBuffers.size() < numInputSlots) {
1442 ALOGD("[%s] start: cannot allocate memory for all slots, "
1443 "only %zu buffers allocated",
1444 mName, clientInputBuffers.size());
1445 } else {
1446 ALOGV("[%s] %zu initial input buffers available",
1447 mName, clientInputBuffers.size());
1448 }
1449 // Sort input buffers by their capacities in increasing order.
1450 clientInputBuffers.sort(
1451 [](const ClientInputBuffer& a, const ClientInputBuffer& b) {
1452 return a.capacity < b.capacity;
1453 });
1454
1455 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1456 mFlushedConfigs.lock()->swap(flushedConfigs);
1457 if (!flushedConfigs.empty()) {
1458 {
1459 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1460 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
1461 for (const std::unique_ptr<C2Work> &work : flushedConfigs) {
1462 watcher->onWorkQueued(
1463 work->input.ordinal.frameIndex.peeku(),
1464 std::vector(work->input.buffers),
1465 now);
1466 }
1467 }
1468 err = mComponent->queue(&flushedConfigs);
1469 if (err != C2_OK) {
1470 ALOGW("[%s] Error while queueing a flushed config", mName);
1471 return UNKNOWN_ERROR;
1472 }
1473 }
1474 if (oStreamFormat.value == C2BufferData::LINEAR &&
1475 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE)) {
1476 sp<MediaCodecBuffer> buffer = clientInputBuffers.front().buffer;
1477 // WORKAROUND: Some apps expect CSD available without queueing
1478 // any input. Queue an empty buffer to get the CSD.
1479 buffer->setRange(0, 0);
1480 buffer->meta()->clear();
1481 buffer->meta()->setInt64("timeUs", 0);
1482 if (queueInputBufferInternal(buffer) != OK) {
1483 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1484 mName);
1485 return UNKNOWN_ERROR;
1486 }
1487 clientInputBuffers.pop_front();
1488 }
1489
1490 for (const ClientInputBuffer& clientInputBuffer: clientInputBuffers) {
1491 mCallback->onInputBufferAvailable(
1492 clientInputBuffer.index,
1493 clientInputBuffer.buffer);
1494 }
1495
1496 return OK;
1497 }
1498
stop()1499 void CCodecBufferChannel::stop() {
1500 mSync.stop();
1501 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
1502 }
1503
reset()1504 void CCodecBufferChannel::reset() {
1505 stop();
1506 if (mInputSurface != nullptr) {
1507 mInputSurface.reset();
1508 }
1509 mPipelineWatcher.lock()->flush();
1510 {
1511 Mutexed<Input>::Locked input(mInput);
1512 input->buffers.reset(new DummyInputBuffers(""));
1513 input->extraBuffers.flush();
1514 }
1515 {
1516 Mutexed<Output>::Locked output(mOutput);
1517 output->buffers.reset();
1518 }
1519 }
1520
release()1521 void CCodecBufferChannel::release() {
1522 mComponent.reset();
1523 mInputAllocator.reset();
1524 mOutputSurface.lock()->surface.clear();
1525 {
1526 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1527 blockPools->inputPool.reset();
1528 blockPools->outputPoolIntf.reset();
1529 }
1530 setCrypto(nullptr);
1531 setDescrambler(nullptr);
1532 }
1533
flush(const std::list<std::unique_ptr<C2Work>> & flushedWork)1534 void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1535 ALOGV("[%s] flush", mName);
1536 std::list<std::unique_ptr<C2Work>> configs;
1537 mInput.lock()->lastFlushIndex = mFrameIndex.load(std::memory_order_relaxed);
1538 {
1539 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1540 for (const std::unique_ptr<C2Work> &work : flushedWork) {
1541 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
1542 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1543 watcher->onWorkDone(frameIndex);
1544 continue;
1545 }
1546 if (work->input.buffers.empty()
1547 || work->input.buffers.front() == nullptr
1548 || work->input.buffers.front()->data().linearBlocks().empty()) {
1549 ALOGD("[%s] no linear codec config data found", mName);
1550 watcher->onWorkDone(frameIndex);
1551 continue;
1552 }
1553 std::unique_ptr<C2Work> copy(new C2Work);
1554 copy->input.flags = C2FrameData::flags_t(
1555 work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1556 copy->input.ordinal = work->input.ordinal;
1557 copy->input.ordinal.frameIndex = mFrameIndex++;
1558 for (size_t i = 0; i < work->input.buffers.size(); ++i) {
1559 copy->input.buffers.push_back(watcher->onInputBufferReleased(frameIndex, i));
1560 }
1561 for (const std::unique_ptr<C2Param> ¶m : work->input.configUpdate) {
1562 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1563 }
1564 copy->input.infoBuffers.insert(
1565 copy->input.infoBuffers.begin(),
1566 work->input.infoBuffers.begin(),
1567 work->input.infoBuffers.end());
1568 copy->worklets.emplace_back(new C2Worklet);
1569 configs.push_back(std::move(copy));
1570 watcher->onWorkDone(frameIndex);
1571 ALOGV("[%s] stashed flushed codec config data", mName);
1572 }
1573 }
1574 mFlushedConfigs.lock()->swap(configs);
1575 {
1576 Mutexed<Input>::Locked input(mInput);
1577 input->buffers->flush();
1578 input->extraBuffers.flush();
1579 }
1580 {
1581 Mutexed<Output>::Locked output(mOutput);
1582 if (output->buffers) {
1583 output->buffers->flush(flushedWork);
1584 output->buffers->flushStash();
1585 }
1586 }
1587 }
1588
onWorkDone(std::unique_ptr<C2Work> work,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)1589 void CCodecBufferChannel::onWorkDone(
1590 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
1591 const C2StreamInitDataInfo::output *initData) {
1592 if (handleWork(std::move(work), outputFormat, initData)) {
1593 feedInputBufferIfAvailable();
1594 }
1595 }
1596
onInputBufferDone(uint64_t frameIndex,size_t arrayIndex)1597 void CCodecBufferChannel::onInputBufferDone(
1598 uint64_t frameIndex, size_t arrayIndex) {
1599 if (mInputSurface) {
1600 return;
1601 }
1602 std::shared_ptr<C2Buffer> buffer =
1603 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
1604 bool newInputSlotAvailable = false;
1605 {
1606 Mutexed<Input>::Locked input(mInput);
1607 if (input->lastFlushIndex >= frameIndex) {
1608 ALOGD("[%s] Ignoring stale input buffer done callback: "
1609 "last flush index = %lld, frameIndex = %lld",
1610 mName, input->lastFlushIndex.peekll(), (long long)frameIndex);
1611 } else {
1612 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1613 if (!newInputSlotAvailable) {
1614 (void)input->extraBuffers.expireComponentBuffer(buffer);
1615 }
1616 }
1617 }
1618 if (newInputSlotAvailable) {
1619 feedInputBufferIfAvailable();
1620 }
1621 }
1622
handleWork(std::unique_ptr<C2Work> work,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)1623 bool CCodecBufferChannel::handleWork(
1624 std::unique_ptr<C2Work> work,
1625 const sp<AMessage> &outputFormat,
1626 const C2StreamInitDataInfo::output *initData) {
1627 {
1628 Mutexed<Output>::Locked output(mOutput);
1629 if (!output->buffers) {
1630 return false;
1631 }
1632 }
1633
1634 // Whether the output buffer should be reported to the client or not.
1635 bool notifyClient = false;
1636
1637 if (work->result == C2_OK){
1638 notifyClient = true;
1639 } else if (work->result == C2_NOT_FOUND) {
1640 ALOGD("[%s] flushed work; ignored.", mName);
1641 } else {
1642 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1643 // the config update.
1644 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1645 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1646 return false;
1647 }
1648
1649 if ((work->input.ordinal.frameIndex -
1650 mFirstValidFrameIndex.load()).peek() < 0) {
1651 // Discard frames from previous generation.
1652 ALOGD("[%s] Discard frames from previous generation.", mName);
1653 notifyClient = false;
1654 }
1655
1656 if (mInputSurface == nullptr && (work->worklets.size() != 1u
1657 || !work->worklets.front()
1658 || !(work->worklets.front()->output.flags &
1659 C2FrameData::FLAG_INCOMPLETE))) {
1660 mPipelineWatcher.lock()->onWorkDone(
1661 work->input.ordinal.frameIndex.peeku());
1662 }
1663
1664 // NOTE: MediaCodec usage supposedly have only one worklet
1665 if (work->worklets.size() != 1u) {
1666 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1667 mName, work->worklets.size());
1668 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1669 return false;
1670 }
1671
1672 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1673
1674 std::shared_ptr<C2Buffer> buffer;
1675 // NOTE: MediaCodec usage supposedly have only one output stream.
1676 if (worklet->output.buffers.size() > 1u) {
1677 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1678 mName, worklet->output.buffers.size());
1679 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1680 return false;
1681 } else if (worklet->output.buffers.size() == 1u) {
1682 buffer = worklet->output.buffers[0];
1683 if (!buffer) {
1684 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1685 }
1686 }
1687
1688 std::optional<uint32_t> newInputDelay, newPipelineDelay, newOutputDelay, newReorderDepth;
1689 std::optional<C2Config::ordinal_key_t> newReorderKey;
1690 bool needMaxDequeueBufferCountUpdate = false;
1691 while (!worklet->output.configUpdate.empty()) {
1692 std::unique_ptr<C2Param> param;
1693 worklet->output.configUpdate.back().swap(param);
1694 worklet->output.configUpdate.pop_back();
1695 switch (param->coreIndex().coreIndex()) {
1696 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1697 C2PortReorderBufferDepthTuning::output reorderDepth;
1698 if (reorderDepth.updateFrom(*param)) {
1699 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1700 mName, reorderDepth.value);
1701 newReorderDepth = reorderDepth.value;
1702 needMaxDequeueBufferCountUpdate = true;
1703 } else {
1704 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1705 mName);
1706 }
1707 break;
1708 }
1709 case C2PortReorderKeySetting::CORE_INDEX: {
1710 C2PortReorderKeySetting::output reorderKey;
1711 if (reorderKey.updateFrom(*param)) {
1712 newReorderKey = reorderKey.value;
1713 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1714 mName, reorderKey.value);
1715 } else {
1716 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1717 }
1718 break;
1719 }
1720 case C2PortActualDelayTuning::CORE_INDEX: {
1721 if (param->isGlobal()) {
1722 C2ActualPipelineDelayTuning pipelineDelay;
1723 if (pipelineDelay.updateFrom(*param)) {
1724 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1725 mName, pipelineDelay.value);
1726 newPipelineDelay = pipelineDelay.value;
1727 (void)mPipelineWatcher.lock()->pipelineDelay(
1728 pipelineDelay.value);
1729 }
1730 }
1731 if (param->forInput()) {
1732 C2PortActualDelayTuning::input inputDelay;
1733 if (inputDelay.updateFrom(*param)) {
1734 ALOGV("[%s] onWorkDone: updating input delay %u",
1735 mName, inputDelay.value);
1736 newInputDelay = inputDelay.value;
1737 (void)mPipelineWatcher.lock()->inputDelay(
1738 inputDelay.value);
1739 }
1740 }
1741 if (param->forOutput()) {
1742 C2PortActualDelayTuning::output outputDelay;
1743 if (outputDelay.updateFrom(*param)) {
1744 ALOGV("[%s] onWorkDone: updating output delay %u",
1745 mName, outputDelay.value);
1746 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
1747 newOutputDelay = outputDelay.value;
1748 needMaxDequeueBufferCountUpdate = true;
1749
1750 }
1751 }
1752 break;
1753 }
1754 case C2PortTunnelSystemTime::CORE_INDEX: {
1755 C2PortTunnelSystemTime::output frameRenderTime;
1756 if (frameRenderTime.updateFrom(*param)) {
1757 ALOGV("[%s] onWorkDone: frame rendered (sys:%lld ns, media:%lld us)",
1758 mName, (long long)frameRenderTime.value,
1759 (long long)worklet->output.ordinal.timestamp.peekll());
1760 mCCodecCallback->onOutputFramesRendered(
1761 worklet->output.ordinal.timestamp.peek(), frameRenderTime.value);
1762 }
1763 break;
1764 }
1765 case C2StreamTunnelHoldRender::CORE_INDEX: {
1766 C2StreamTunnelHoldRender::output firstTunnelFrameHoldRender;
1767 if (!(worklet->output.flags & C2FrameData::FLAG_INCOMPLETE)) break;
1768 if (!firstTunnelFrameHoldRender.updateFrom(*param)) break;
1769 if (firstTunnelFrameHoldRender.value != C2_TRUE) break;
1770 ALOGV("[%s] onWorkDone: first tunnel frame ready", mName);
1771 mCCodecCallback->onFirstTunnelFrameReady();
1772 break;
1773 }
1774 default:
1775 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1776 mName, param->index());
1777 break;
1778 }
1779 }
1780 if (newInputDelay || newPipelineDelay) {
1781 Mutexed<Input>::Locked input(mInput);
1782 size_t newNumSlots =
1783 newInputDelay.value_or(input->inputDelay) +
1784 newPipelineDelay.value_or(input->pipelineDelay) +
1785 kSmoothnessFactor;
1786 if (input->buffers->isArrayMode()) {
1787 if (input->numSlots >= newNumSlots) {
1788 input->numExtraSlots = 0;
1789 } else {
1790 input->numExtraSlots = newNumSlots - input->numSlots;
1791 }
1792 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1793 mName, input->numExtraSlots);
1794 } else {
1795 input->numSlots = newNumSlots;
1796 }
1797 }
1798 size_t numOutputSlots = 0;
1799 uint32_t reorderDepth = 0;
1800 bool outputBuffersChanged = false;
1801 if (newReorderKey || newReorderDepth || needMaxDequeueBufferCountUpdate) {
1802 Mutexed<Output>::Locked output(mOutput);
1803 if (!output->buffers) {
1804 return false;
1805 }
1806 numOutputSlots = output->numSlots;
1807 if (newReorderKey) {
1808 output->buffers->setReorderKey(newReorderKey.value());
1809 }
1810 if (newReorderDepth) {
1811 output->buffers->setReorderDepth(newReorderDepth.value());
1812 }
1813 reorderDepth = output->buffers->getReorderDepth();
1814 if (newOutputDelay) {
1815 output->outputDelay = newOutputDelay.value();
1816 numOutputSlots = newOutputDelay.value() + kSmoothnessFactor;
1817 if (output->numSlots < numOutputSlots) {
1818 output->numSlots = numOutputSlots;
1819 if (output->buffers->isArrayMode()) {
1820 OutputBuffersArray *array =
1821 (OutputBuffersArray *)output->buffers.get();
1822 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1823 mName, numOutputSlots);
1824 array->grow(numOutputSlots);
1825 outputBuffersChanged = true;
1826 }
1827 }
1828 }
1829 numOutputSlots = output->numSlots;
1830 }
1831 if (outputBuffersChanged) {
1832 mCCodecCallback->onOutputBuffersChanged();
1833 }
1834 if (needMaxDequeueBufferCountUpdate) {
1835 int maxDequeueCount = 0;
1836 {
1837 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1838 maxDequeueCount = output->maxDequeueBuffers =
1839 numOutputSlots + reorderDepth + kRenderingDepth;
1840 if (output->surface) {
1841 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1842 }
1843 }
1844 if (maxDequeueCount > 0) {
1845 mComponent->setOutputSurfaceMaxDequeueCount(maxDequeueCount);
1846 }
1847 }
1848
1849 int32_t flags = 0;
1850 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1851 flags |= MediaCodec::BUFFER_FLAG_EOS;
1852 ALOGV("[%s] onWorkDone: output EOS", mName);
1853 }
1854
1855 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1856 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1857 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1858 // shall correspond to the client input timesamp (in customOrdinal). By using the
1859 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1860 // produces multiple output.
1861 c2_cntr64_t timestamp =
1862 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1863 - work->input.ordinal.timestamp;
1864 if (mInputSurface != nullptr) {
1865 // When using input surface we need to restore the original input timestamp.
1866 timestamp = work->input.ordinal.customOrdinal;
1867 }
1868 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1869 mName,
1870 work->input.ordinal.customOrdinal.peekll(),
1871 work->input.ordinal.timestamp.peekll(),
1872 worklet->output.ordinal.timestamp.peekll(),
1873 timestamp.peekll());
1874
1875 // csd cannot be re-ordered and will always arrive first.
1876 if (initData != nullptr) {
1877 Mutexed<Output>::Locked output(mOutput);
1878 if (output->buffers && outputFormat) {
1879 output->buffers->updateSkipCutBuffer(outputFormat);
1880 output->buffers->setFormat(outputFormat);
1881 }
1882 if (!notifyClient) {
1883 return false;
1884 }
1885 size_t index;
1886 sp<MediaCodecBuffer> outBuffer;
1887 if (output->buffers && output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
1888 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1889 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1890 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1891
1892 output.unlock();
1893 mCallback->onOutputBufferAvailable(index, outBuffer);
1894 } else {
1895 ALOGD("[%s] onWorkDone: unable to register csd", mName);
1896 output.unlock();
1897 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1898 return false;
1899 }
1900 }
1901
1902 if (notifyClient && !buffer && !flags) {
1903 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
1904 mName, work->input.ordinal.frameIndex.peekull());
1905 notifyClient = false;
1906 }
1907
1908 if (buffer) {
1909 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1910 // TODO: properly translate these to metadata
1911 switch (info->coreIndex().coreIndex()) {
1912 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
1913 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
1914 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1915 }
1916 break;
1917 default:
1918 break;
1919 }
1920 }
1921 }
1922
1923 {
1924 Mutexed<Output>::Locked output(mOutput);
1925 if (!output->buffers) {
1926 return false;
1927 }
1928 output->buffers->pushToStash(
1929 buffer,
1930 notifyClient,
1931 timestamp.peek(),
1932 flags,
1933 outputFormat,
1934 worklet->output.ordinal);
1935 }
1936 sendOutputBuffers();
1937 return true;
1938 }
1939
sendOutputBuffers()1940 void CCodecBufferChannel::sendOutputBuffers() {
1941 OutputBuffers::BufferAction action;
1942 size_t index;
1943 sp<MediaCodecBuffer> outBuffer;
1944 std::shared_ptr<C2Buffer> c2Buffer;
1945
1946 while (true) {
1947 Mutexed<Output>::Locked output(mOutput);
1948 if (!output->buffers) {
1949 return;
1950 }
1951 action = output->buffers->popFromStashAndRegister(
1952 &c2Buffer, &index, &outBuffer);
1953 switch (action) {
1954 case OutputBuffers::SKIP:
1955 return;
1956 case OutputBuffers::DISCARD:
1957 break;
1958 case OutputBuffers::NOTIFY_CLIENT:
1959 output.unlock();
1960 mCallback->onOutputBufferAvailable(index, outBuffer);
1961 break;
1962 case OutputBuffers::REALLOCATE:
1963 if (!output->buffers->isArrayMode()) {
1964 output->buffers =
1965 output->buffers->toArrayMode(output->numSlots);
1966 }
1967 static_cast<OutputBuffersArray*>(output->buffers.get())->
1968 realloc(c2Buffer);
1969 output.unlock();
1970 mCCodecCallback->onOutputBuffersChanged();
1971 break;
1972 case OutputBuffers::RETRY:
1973 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
1974 mName);
1975 return;
1976 default:
1977 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
1978 "corrupted BufferAction value (%d) "
1979 "returned from popFromStashAndRegister.",
1980 mName, int(action));
1981 return;
1982 }
1983 }
1984 }
1985
setSurface(const sp<Surface> & newSurface)1986 status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1987 static std::atomic_uint32_t surfaceGeneration{0};
1988 uint32_t generation = (getpid() << 10) |
1989 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1990 & ((1 << 10) - 1));
1991
1992 sp<IGraphicBufferProducer> producer;
1993 int maxDequeueCount = mOutputSurface.lock()->maxDequeueBuffers;
1994 if (newSurface) {
1995 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
1996 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
1997 newSurface->setMaxDequeuedBufferCount(maxDequeueCount);
1998 producer = newSurface->getIGraphicBufferProducer();
1999 producer->setGenerationNumber(generation);
2000 } else {
2001 ALOGE("[%s] setting output surface to null", mName);
2002 return INVALID_OPERATION;
2003 }
2004
2005 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
2006 C2BlockPool::local_id_t outputPoolId;
2007 {
2008 Mutexed<BlockPools>::Locked pools(mBlockPools);
2009 outputPoolId = pools->outputPoolId;
2010 outputPoolIntf = pools->outputPoolIntf;
2011 }
2012
2013 if (outputPoolIntf) {
2014 if (mComponent->setOutputSurface(
2015 outputPoolId,
2016 producer,
2017 generation,
2018 maxDequeueCount) != C2_OK) {
2019 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
2020 return INVALID_OPERATION;
2021 }
2022 }
2023
2024 {
2025 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2026 output->surface = newSurface;
2027 output->generation = generation;
2028 }
2029
2030 return OK;
2031 }
2032
elapsed()2033 PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
2034 // When client pushed EOS, we want all the work to be done quickly.
2035 // Otherwise, component may have stalled work due to input starvation up to
2036 // the sum of the delay in the pipeline.
2037 size_t n = 0;
2038 if (!mInputMetEos) {
2039 size_t outputDelay = mOutput.lock()->outputDelay;
2040 Mutexed<Input>::Locked input(mInput);
2041 n = input->inputDelay + input->pipelineDelay + outputDelay;
2042 }
2043 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
2044 }
2045
setMetaMode(MetaMode mode)2046 void CCodecBufferChannel::setMetaMode(MetaMode mode) {
2047 mMetaMode = mode;
2048 }
2049
setCrypto(const sp<ICrypto> & crypto)2050 void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
2051 if (mCrypto != nullptr) {
2052 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
2053 mCrypto->unsetHeap(entry.second);
2054 }
2055 mHeapSeqNumMap.clear();
2056 if (mHeapSeqNum >= 0) {
2057 mCrypto->unsetHeap(mHeapSeqNum);
2058 mHeapSeqNum = -1;
2059 }
2060 }
2061 mCrypto = crypto;
2062 }
2063
setDescrambler(const sp<IDescrambler> & descrambler)2064 void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
2065 mDescrambler = descrambler;
2066 }
2067
toStatusT(c2_status_t c2s,c2_operation_t c2op)2068 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2069 // C2_OK is always translated to OK.
2070 if (c2s == C2_OK) {
2071 return OK;
2072 }
2073
2074 // Operation-dependent translation
2075 // TODO: Add as necessary
2076 switch (c2op) {
2077 case C2_OPERATION_Component_start:
2078 switch (c2s) {
2079 case C2_NO_MEMORY:
2080 return NO_MEMORY;
2081 default:
2082 return UNKNOWN_ERROR;
2083 }
2084 default:
2085 break;
2086 }
2087
2088 // Backup operation-agnostic translation
2089 switch (c2s) {
2090 case C2_BAD_INDEX:
2091 return BAD_INDEX;
2092 case C2_BAD_VALUE:
2093 return BAD_VALUE;
2094 case C2_BLOCKING:
2095 return WOULD_BLOCK;
2096 case C2_DUPLICATE:
2097 return ALREADY_EXISTS;
2098 case C2_NO_INIT:
2099 return NO_INIT;
2100 case C2_NO_MEMORY:
2101 return NO_MEMORY;
2102 case C2_NOT_FOUND:
2103 return NAME_NOT_FOUND;
2104 case C2_TIMED_OUT:
2105 return TIMED_OUT;
2106 case C2_BAD_STATE:
2107 case C2_CANCELED:
2108 case C2_CANNOT_DO:
2109 case C2_CORRUPTED:
2110 case C2_OMITTED:
2111 case C2_REFUSED:
2112 return UNKNOWN_ERROR;
2113 default:
2114 return -static_cast<status_t>(c2s);
2115 }
2116 }
2117
2118 } // namespace android
2119