1 /*
2  * Copyright 2018, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifdef __LP64__
18 #define OMX_ANDROID_COMPILE_AS_32BIT_ON_64BIT_PLATFORMS
19 #endif
20 
21 //#define LOG_NDEBUG 0
22 #define LOG_TAG "C2OMXNode"
23 #include <log/log.h>
24 
25 #include <C2AllocatorGralloc.h>
26 #include <C2BlockInternal.h>
27 #include <C2Component.h>
28 #include <C2Config.h>
29 #include <C2PlatformSupport.h>
30 
31 #include <OMX_Component.h>
32 #include <OMX_Index.h>
33 #include <OMX_IndexExt.h>
34 
35 #include <android/fdsan.h>
36 #include <media/stagefright/foundation/ColorUtils.h>
37 #include <media/stagefright/omx/OMXUtils.h>
38 #include <media/stagefright/MediaErrors.h>
39 #include <ui/Fence.h>
40 #include <ui/GraphicBuffer.h>
41 #include <utils/Thread.h>
42 
43 #include "utils/Codec2Mapper.h"
44 #include "C2OMXNode.h"
45 
46 namespace android {
47 
48 namespace {
49 
50 constexpr OMX_U32 kPortIndexInput = 0;
51 
52 class Buffer2D : public C2Buffer {
53 public:
Buffer2D(C2ConstGraphicBlock block)54     explicit Buffer2D(C2ConstGraphicBlock block) : C2Buffer({ block }) {}
55 };
56 
57 }  // namespace
58 
59 class C2OMXNode::QueueThread : public Thread {
60 public:
QueueThread()61     QueueThread() : Thread(false) {}
62     ~QueueThread() override = default;
queue(const std::shared_ptr<Codec2Client::Component> & comp,int fenceFd,std::unique_ptr<C2Work> && work,android::base::unique_fd && fd0,android::base::unique_fd && fd1)63     void queue(
64             const std::shared_ptr<Codec2Client::Component> &comp,
65             int fenceFd,
66             std::unique_ptr<C2Work> &&work,
67             android::base::unique_fd &&fd0,
68             android::base::unique_fd &&fd1) {
69         Mutexed<Jobs>::Locked jobs(mJobs);
70         auto it = jobs->queues.try_emplace(comp, comp).first;
71         it->second.workList.emplace_back(
72                 std::move(work), fenceFd, std::move(fd0), std::move(fd1));
73         jobs->cond.broadcast();
74     }
75 
setDataspace(android_dataspace dataspace)76     void setDataspace(android_dataspace dataspace) {
77         Mutexed<Jobs>::Locked jobs(mJobs);
78         ColorUtils::convertDataSpaceToV0(dataspace);
79         jobs->configUpdate.emplace_back(new C2StreamDataSpaceInfo::input(0u, dataspace));
80         int32_t standard;
81         int32_t transfer;
82         int32_t range;
83         ColorUtils::getColorConfigFromDataSpace(dataspace, &range, &standard, &transfer);
84         std::unique_ptr<C2StreamColorAspectsInfo::input> colorAspects =
85             std::make_unique<C2StreamColorAspectsInfo::input>(0u);
86         if (C2Mapper::map(standard, &colorAspects->primaries, &colorAspects->matrix)
87                 && C2Mapper::map(transfer, &colorAspects->transfer)
88                 && C2Mapper::map(range, &colorAspects->range)) {
89             jobs->configUpdate.push_back(std::move(colorAspects));
90         }
91     }
92 
setPriority(int priority)93     void setPriority(int priority) {
94         androidSetThreadPriority(getTid(), priority);
95     }
96 
97 protected:
threadLoop()98     bool threadLoop() override {
99         constexpr nsecs_t kIntervalNs = nsecs_t(10) * 1000 * 1000;  // 10ms
100         constexpr nsecs_t kWaitNs = kIntervalNs * 2;
101         for (int i = 0; i < 2; ++i) {
102             Mutexed<Jobs>::Locked jobs(mJobs);
103             nsecs_t nowNs = systemTime();
104             bool queued = false;
105             for (auto it = jobs->queues.begin(); it != jobs->queues.end(); ) {
106                 Queue &queue = it->second;
107                 if (queue.workList.empty()
108                         || (queue.lastQueuedTimestampNs != 0 &&
109                             nowNs - queue.lastQueuedTimestampNs < kIntervalNs)) {
110                     ++it;
111                     continue;
112                 }
113                 std::shared_ptr<Codec2Client::Component> comp = queue.component.lock();
114                 if (!comp) {
115                     it = jobs->queues.erase(it);
116                     continue;
117                 }
118                 std::list<std::unique_ptr<C2Work>> items;
119                 std::vector<int> fenceFds;
120                 std::vector<android::base::unique_fd> uniqueFds;
121                 while (!queue.workList.empty()) {
122                     items.push_back(std::move(queue.workList.front().work));
123                     fenceFds.push_back(queue.workList.front().fenceFd);
124                     uniqueFds.push_back(std::move(queue.workList.front().fd0));
125                     uniqueFds.push_back(std::move(queue.workList.front().fd1));
126                     queue.workList.pop_front();
127                 }
128                 for (const std::unique_ptr<C2Param> &param : jobs->configUpdate) {
129                     items.front()->input.configUpdate.emplace_back(C2Param::Copy(*param));
130                 }
131 
132                 jobs.unlock();
133                 for (int fenceFd : fenceFds) {
134                     sp<Fence> fence(new Fence(fenceFd));
135                     fence->waitForever(LOG_TAG);
136                 }
137                 queue.lastQueuedTimestampNs = nowNs;
138                 comp->queue(&items);
139                 for (android::base::unique_fd &ufd : uniqueFds) {
140                     (void)ufd.release();
141                 }
142                 jobs.lock();
143 
144                 it = jobs->queues.upper_bound(comp);
145                 queued = true;
146             }
147             if (queued) {
148                 jobs->configUpdate.clear();
149                 return true;
150             }
151             if (i == 0) {
152                 jobs.waitForConditionRelative(jobs->cond, kWaitNs);
153             }
154         }
155         return true;
156     }
157 
158 private:
159     struct WorkFence {
WorkFenceandroid::C2OMXNode::QueueThread::WorkFence160         WorkFence(std::unique_ptr<C2Work> &&w, int fd) : work(std::move(w)), fenceFd(fd) {}
161 
WorkFenceandroid::C2OMXNode::QueueThread::WorkFence162         WorkFence(
163                 std::unique_ptr<C2Work> &&w,
164                 int fd,
165                 android::base::unique_fd &&uniqueFd0,
166                 android::base::unique_fd &&uniqueFd1)
167             : work(std::move(w)),
168               fenceFd(fd),
169               fd0(std::move(uniqueFd0)),
170               fd1(std::move(uniqueFd1)) {}
171 
172         std::unique_ptr<C2Work> work;
173         int fenceFd;
174         android::base::unique_fd fd0;
175         android::base::unique_fd fd1;
176     };
177     struct Queue {
Queueandroid::C2OMXNode::QueueThread::Queue178         Queue(const std::shared_ptr<Codec2Client::Component> &comp)
179             : component(comp), lastQueuedTimestampNs(0) {}
180         Queue(const Queue &) = delete;
181         Queue &operator =(const Queue &) = delete;
182 
183         std::weak_ptr<Codec2Client::Component> component;
184         std::list<WorkFence> workList;
185         nsecs_t lastQueuedTimestampNs;
186     };
187     struct Jobs {
188         std::map<std::weak_ptr<Codec2Client::Component>,
189                  Queue,
190                  std::owner_less<std::weak_ptr<Codec2Client::Component>>> queues;
191         std::vector<std::unique_ptr<C2Param>> configUpdate;
192         Condition cond;
193     };
194     Mutexed<Jobs> mJobs;
195 };
196 
C2OMXNode(const std::shared_ptr<Codec2Client::Component> & comp)197 C2OMXNode::C2OMXNode(const std::shared_ptr<Codec2Client::Component> &comp)
198     : mComp(comp), mFrameIndex(0), mWidth(0), mHeight(0), mUsage(0),
199       mAdjustTimestampGapUs(0), mFirstInputFrame(true),
200       mQueueThread(new QueueThread) {
201     android_fdsan_set_error_level(ANDROID_FDSAN_ERROR_LEVEL_WARN_ALWAYS);
202     mQueueThread->run("C2OMXNode", PRIORITY_AUDIO);
203 
204     Mutexed<android_dataspace>::Locked ds(mDataspace);
205     *ds = HAL_DATASPACE_UNKNOWN;
206 }
207 
freeNode()208 status_t C2OMXNode::freeNode() {
209     mComp.reset();
210     android_fdsan_set_error_level(ANDROID_FDSAN_ERROR_LEVEL_WARN_ONCE);
211     return mQueueThread->requestExitAndWait();
212 }
213 
sendCommand(OMX_COMMANDTYPE cmd,OMX_S32 param)214 status_t C2OMXNode::sendCommand(OMX_COMMANDTYPE cmd, OMX_S32 param) {
215     if (cmd == OMX_CommandStateSet && param == OMX_StateLoaded) {
216         // Reset first input frame so if C2OMXNode is recycled, the timestamp does not become
217         // negative. This is a workaround for HW codecs that do not handle timestamp rollover.
218         mFirstInputFrame = true;
219     }
220     return ERROR_UNSUPPORTED;
221 }
222 
getParameter(OMX_INDEXTYPE index,void * params,size_t size)223 status_t C2OMXNode::getParameter(OMX_INDEXTYPE index, void *params, size_t size) {
224     status_t err = ERROR_UNSUPPORTED;
225     switch ((uint32_t)index) {
226         case OMX_IndexParamConsumerUsageBits: {
227             OMX_U32 *usage = (OMX_U32 *)params;
228             *usage = mUsage;
229             err = OK;
230             break;
231         }
232         case OMX_IndexParamPortDefinition: {
233             if (size < sizeof(OMX_PARAM_PORTDEFINITIONTYPE)) {
234                 return BAD_VALUE;
235             }
236             OMX_PARAM_PORTDEFINITIONTYPE *pDef = (OMX_PARAM_PORTDEFINITIONTYPE *)params;
237             if (pDef->nPortIndex != kPortIndexInput) {
238                 break;
239             }
240 
241             pDef->nBufferCountActual = 16;
242 
243             // WORKAROUND: having more slots improve performance while consuming
244             // more memory. This is a temporary workaround to reduce memory for
245             // larger-than-4K scenario.
246             if (mWidth * mHeight > 4096 * 2340) {
247                 std::shared_ptr<Codec2Client::Component> comp = mComp.lock();
248                 C2PortActualDelayTuning::input inputDelay(0);
249                 C2ActualPipelineDelayTuning pipelineDelay(0);
250                 c2_status_t c2err = C2_NOT_FOUND;
251                 if (comp) {
252                     c2err = comp->query(
253                             {&inputDelay, &pipelineDelay}, {}, C2_DONT_BLOCK, nullptr);
254                 }
255                 if (c2err == C2_OK || c2err == C2_BAD_INDEX) {
256                     pDef->nBufferCountActual = 4;
257                     pDef->nBufferCountActual += (inputDelay ? inputDelay.value : 0u);
258                     pDef->nBufferCountActual += (pipelineDelay ? pipelineDelay.value : 0u);
259                 }
260             }
261 
262             pDef->eDomain = OMX_PortDomainVideo;
263             pDef->format.video.nFrameWidth = mWidth;
264             pDef->format.video.nFrameHeight = mHeight;
265             pDef->format.video.eColorFormat = OMX_COLOR_FormatAndroidOpaque;
266             err = OK;
267             break;
268         }
269         default:
270             break;
271     }
272     return err;
273 }
274 
setParameter(OMX_INDEXTYPE index,const void * params,size_t size)275 status_t C2OMXNode::setParameter(OMX_INDEXTYPE index, const void *params, size_t size) {
276     if (params == NULL) {
277         return BAD_VALUE;
278     }
279     switch ((uint32_t)index) {
280         case OMX_IndexParamMaxFrameDurationForBitrateControl:
281             // handle max/fixed frame duration control
282             if (size != sizeof(OMX_PARAM_U32TYPE)) {
283                 return BAD_VALUE;
284             }
285             // The incoming number is an int32_t contained in OMX_U32.
286             mAdjustTimestampGapUs = (int32_t)((OMX_PARAM_U32TYPE*)params)->nU32;
287             return OK;
288 
289         case OMX_IndexParamConsumerUsageBits:
290             if (size != sizeof(OMX_U32)) {
291                 return BAD_VALUE;
292             }
293             mUsage = *((OMX_U32 *)params);
294             return OK;
295     }
296     return ERROR_UNSUPPORTED;
297 }
298 
getConfig(OMX_INDEXTYPE index,void * config,size_t size)299 status_t C2OMXNode::getConfig(OMX_INDEXTYPE index, void *config, size_t size) {
300     (void)index;
301     (void)config;
302     (void)size;
303     return ERROR_UNSUPPORTED;
304 }
305 
setConfig(OMX_INDEXTYPE index,const void * config,size_t size)306 status_t C2OMXNode::setConfig(OMX_INDEXTYPE index, const void *config, size_t size) {
307     (void)index;
308     (void)config;
309     (void)size;
310     return ERROR_UNSUPPORTED;
311 }
312 
setPortMode(OMX_U32 portIndex,IOMX::PortMode mode)313 status_t C2OMXNode::setPortMode(OMX_U32 portIndex, IOMX::PortMode mode) {
314     (void)portIndex;
315     (void)mode;
316     return ERROR_UNSUPPORTED;
317 }
318 
prepareForAdaptivePlayback(OMX_U32 portIndex,OMX_BOOL enable,OMX_U32 maxFrameWidth,OMX_U32 maxFrameHeight)319 status_t C2OMXNode::prepareForAdaptivePlayback(
320         OMX_U32 portIndex, OMX_BOOL enable,
321         OMX_U32 maxFrameWidth, OMX_U32 maxFrameHeight) {
322     (void)portIndex;
323     (void)enable;
324     (void)maxFrameWidth;
325     (void)maxFrameHeight;
326     return ERROR_UNSUPPORTED;
327 }
328 
configureVideoTunnelMode(OMX_U32 portIndex,OMX_BOOL tunneled,OMX_U32 audioHwSync,native_handle_t ** sidebandHandle)329 status_t C2OMXNode::configureVideoTunnelMode(
330         OMX_U32 portIndex, OMX_BOOL tunneled,
331         OMX_U32 audioHwSync, native_handle_t **sidebandHandle) {
332     (void)portIndex;
333     (void)tunneled;
334     (void)audioHwSync;
335     *sidebandHandle = nullptr;
336     return ERROR_UNSUPPORTED;
337 }
338 
getGraphicBufferUsage(OMX_U32 portIndex,OMX_U32 * usage)339 status_t C2OMXNode::getGraphicBufferUsage(OMX_U32 portIndex, OMX_U32* usage) {
340     (void)portIndex;
341     *usage = 0;
342     return ERROR_UNSUPPORTED;
343 }
344 
setInputSurface(const sp<IOMXBufferSource> & bufferSource)345 status_t C2OMXNode::setInputSurface(const sp<IOMXBufferSource> &bufferSource) {
346     c2_status_t err = GetCodec2PlatformAllocatorStore()->fetchAllocator(
347             C2PlatformAllocatorStore::GRALLOC,
348             &mAllocator);
349     if (err != OK) {
350         return UNKNOWN_ERROR;
351     }
352     mBufferSource = bufferSource;
353     return OK;
354 }
355 
allocateSecureBuffer(OMX_U32 portIndex,size_t size,buffer_id * buffer,void ** bufferData,sp<NativeHandle> * nativeHandle)356 status_t C2OMXNode::allocateSecureBuffer(
357         OMX_U32 portIndex, size_t size, buffer_id *buffer,
358         void **bufferData, sp<NativeHandle> *nativeHandle) {
359     (void)portIndex;
360     (void)size;
361     (void)nativeHandle;
362     *buffer = 0;
363     *bufferData = nullptr;
364     return ERROR_UNSUPPORTED;
365 }
366 
useBuffer(OMX_U32 portIndex,const OMXBuffer & omxBuf,buffer_id * buffer)367 status_t C2OMXNode::useBuffer(
368         OMX_U32 portIndex, const OMXBuffer &omxBuf, buffer_id *buffer) {
369     (void)portIndex;
370     (void)omxBuf;
371     *buffer = 0;
372     return ERROR_UNSUPPORTED;
373 }
374 
freeBuffer(OMX_U32 portIndex,buffer_id buffer)375 status_t C2OMXNode::freeBuffer(OMX_U32 portIndex, buffer_id buffer) {
376     (void)portIndex;
377     (void)buffer;
378     return ERROR_UNSUPPORTED;
379 }
380 
fillBuffer(buffer_id buffer,const OMXBuffer & omxBuf,int fenceFd)381 status_t C2OMXNode::fillBuffer(
382         buffer_id buffer, const OMXBuffer &omxBuf, int fenceFd) {
383     (void)buffer;
384     (void)omxBuf;
385     (void)fenceFd;
386     return ERROR_UNSUPPORTED;
387 }
388 
emptyBuffer(buffer_id buffer,const OMXBuffer & omxBuf,OMX_U32 flags,OMX_TICKS timestamp,int fenceFd)389 status_t C2OMXNode::emptyBuffer(
390         buffer_id buffer, const OMXBuffer &omxBuf,
391         OMX_U32 flags, OMX_TICKS timestamp, int fenceFd) {
392     std::shared_ptr<Codec2Client::Component> comp = mComp.lock();
393     if (!comp) {
394         return NO_INIT;
395     }
396 
397     uint32_t c2Flags = (flags & OMX_BUFFERFLAG_EOS)
398             ? C2FrameData::FLAG_END_OF_STREAM : 0;
399     std::shared_ptr<C2GraphicBlock> block;
400 
401     android::base::unique_fd fd0, fd1;
402     C2Handle *handle = nullptr;
403     if (omxBuf.mBufferType == OMXBuffer::kBufferTypeANWBuffer
404             && omxBuf.mGraphicBuffer != nullptr) {
405         std::shared_ptr<C2GraphicAllocation> alloc;
406         handle = WrapNativeCodec2GrallocHandle(
407                 omxBuf.mGraphicBuffer->handle,
408                 omxBuf.mGraphicBuffer->width,
409                 omxBuf.mGraphicBuffer->height,
410                 omxBuf.mGraphicBuffer->format,
411                 omxBuf.mGraphicBuffer->usage,
412                 omxBuf.mGraphicBuffer->stride);
413         if (handle != nullptr) {
414             // unique_fd takes ownership of the fds, we'll get warning if these
415             // fds get closed by somebody else. Onwership will be released before
416             // we return, so that the fds get closed as usually when this function
417             // goes out of scope (when both items and block are gone).
418             native_handle_t *nativeHandle = reinterpret_cast<native_handle_t*>(handle);
419             fd0.reset(nativeHandle->numFds > 0 ? nativeHandle->data[0] : -1);
420             fd1.reset(nativeHandle->numFds > 1 ? nativeHandle->data[1] : -1);
421         }
422         c2_status_t err = mAllocator->priorGraphicAllocation(handle, &alloc);
423         if (err != OK) {
424             (void)fd0.release();
425             (void)fd1.release();
426             native_handle_close(handle);
427             native_handle_delete(handle);
428             return UNKNOWN_ERROR;
429         }
430         block = _C2BlockFactory::CreateGraphicBlock(alloc);
431     } else if (!(flags & OMX_BUFFERFLAG_EOS)) {
432         return BAD_VALUE;
433     }
434 
435     std::unique_ptr<C2Work> work(new C2Work);
436     work->input.flags = (C2FrameData::flags_t)c2Flags;
437     work->input.ordinal.timestamp = timestamp;
438 
439     // WORKAROUND: adjust timestamp based on gapUs
440     {
441         work->input.ordinal.customOrdinal = timestamp; // save input timestamp
442         if (mFirstInputFrame) {
443             // grab timestamps on first frame
444             mPrevInputTimestamp = timestamp;
445             mPrevCodecTimestamp = timestamp;
446             mFirstInputFrame = false;
447         } else if (mAdjustTimestampGapUs > 0) {
448             work->input.ordinal.timestamp =
449                 mPrevCodecTimestamp
450                         + c2_min((timestamp - mPrevInputTimestamp).peek(), mAdjustTimestampGapUs);
451         } else if (mAdjustTimestampGapUs < 0) {
452             work->input.ordinal.timestamp = mPrevCodecTimestamp - mAdjustTimestampGapUs;
453         }
454         mPrevInputTimestamp = work->input.ordinal.customOrdinal;
455         mPrevCodecTimestamp = work->input.ordinal.timestamp;
456         ALOGV("adjusting %lld to %lld (gap=%lld)",
457               work->input.ordinal.customOrdinal.peekll(),
458               work->input.ordinal.timestamp.peekll(),
459               (long long)mAdjustTimestampGapUs);
460     }
461 
462     work->input.ordinal.frameIndex = mFrameIndex++;
463     work->input.buffers.clear();
464     if (block) {
465         std::shared_ptr<C2Buffer> c2Buffer(
466                 new Buffer2D(block->share(
467                         C2Rect(block->width(), block->height()), ::C2Fence())));
468         work->input.buffers.push_back(c2Buffer);
469     }
470     work->worklets.clear();
471     work->worklets.emplace_back(new C2Worklet);
472     mBufferIdsInUse.lock()->emplace(work->input.ordinal.frameIndex.peeku(), buffer);
473     mQueueThread->queue(comp, fenceFd, std::move(work), std::move(fd0), std::move(fd1));
474 
475     return OK;
476 }
477 
getExtensionIndex(const char * parameterName,OMX_INDEXTYPE * index)478 status_t C2OMXNode::getExtensionIndex(
479         const char *parameterName, OMX_INDEXTYPE *index) {
480     (void)parameterName;
481     *index = OMX_IndexMax;
482     return ERROR_UNSUPPORTED;
483 }
484 
dispatchMessage(const omx_message & msg)485 status_t C2OMXNode::dispatchMessage(const omx_message& msg) {
486     if (msg.type != omx_message::EVENT) {
487         return ERROR_UNSUPPORTED;
488     }
489     if (msg.u.event_data.event != OMX_EventDataSpaceChanged) {
490         return ERROR_UNSUPPORTED;
491     }
492     android_dataspace dataSpace = (android_dataspace)msg.u.event_data.data1;
493     uint32_t pixelFormat = msg.u.event_data.data3;
494 
495     ALOGD("dataspace changed to %#x pixel format: %#x", dataSpace, pixelFormat);
496     mQueueThread->setDataspace(dataSpace);
497 
498     Mutexed<android_dataspace>::Locked ds(mDataspace);
499     *ds = dataSpace;
500     return OK;
501 }
502 
getSource()503 sp<IOMXBufferSource> C2OMXNode::getSource() {
504     return mBufferSource;
505 }
506 
setFrameSize(uint32_t width,uint32_t height)507 void C2OMXNode::setFrameSize(uint32_t width, uint32_t height) {
508     mWidth = width;
509     mHeight = height;
510 }
511 
onInputBufferDone(c2_cntr64_t index)512 void C2OMXNode::onInputBufferDone(c2_cntr64_t index) {
513     if (!mBufferSource) {
514         ALOGD("Buffer source not set (index=%llu)", index.peekull());
515         return;
516     }
517 
518     int32_t bufferId = 0;
519     {
520         decltype(mBufferIdsInUse)::Locked bufferIds(mBufferIdsInUse);
521         auto it = bufferIds->find(index.peeku());
522         if (it == bufferIds->end()) {
523             ALOGV("Untracked input index %llu (maybe already removed)", index.peekull());
524             return;
525         }
526         bufferId = it->second;
527         (void)bufferIds->erase(it);
528     }
529     (void)mBufferSource->onInputBufferEmptied(bufferId, -1);
530 }
531 
getDataspace()532 android_dataspace C2OMXNode::getDataspace() {
533     return *mDataspace.lock();
534 }
535 
setPriority(int priority)536 void C2OMXNode::setPriority(int priority) {
537     mQueueThread->setPriority(priority);
538 }
539 
540 }  // namespace android
541