1 /*
2  * Copyright (c) 2021-2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifdef VIDEO_SUPPORT
17 
18 #define HST_LOG_TAG "FfmpegVideoDecoderPlugin"
19 
20 #include "video_ffmpeg_decoder_plugin.h"
21 #include <cstring>
22 #include <map>
23 #include <set>
24 #include "plugin/common/plugin_caps_builder.h"
25 #include "plugin/common/surface_memory.h"
26 #include "plugins/ffmpeg_adapter/utils/ffmpeg_utils.h"
27 
28 namespace {
29 // register plugins
30 using namespace OHOS::Media::Plugin;
31 using namespace Ffmpeg;
32 void UpdatePluginDefinition(const AVCodec* codec, CodecPluginDef& definition);
33 
34 std::map<std::string, std::shared_ptr<const AVCodec>> codecMap;
35 
36 constexpr size_t BUFFER_QUEUE_SIZE = 8;
37 constexpr int32_t STRIDE_ALIGN = 16;
38 
39 std::set<AVCodecID> supportedCodec = {AV_CODEC_ID_H264};
40 
VideoFfmpegDecoderCreator(const std::string & name)41 std::shared_ptr<CodecPlugin> VideoFfmpegDecoderCreator(const std::string& name)
42 {
43     return std::make_shared<VideoFfmpegDecoderPlugin>(name);
44 }
45 
RegisterVideoDecoderPlugins(const std::shared_ptr<Register> & reg)46 Status RegisterVideoDecoderPlugins(const std::shared_ptr<Register>& reg)
47 {
48     const AVCodec* codec = nullptr;
49     void* iter = nullptr;
50     MEDIA_LOG_I("registering video decoders");
51     while ((codec = av_codec_iterate(&iter))) {
52         if (!av_codec_is_decoder(codec) || codec->type != AVMEDIA_TYPE_VIDEO) {
53             continue;
54         }
55         if (supportedCodec.find(codec->id) == supportedCodec.end()) {
56             MEDIA_LOG_DD("codec " PUBLIC_LOG_S "(" PUBLIC_LOG_S ") is not supported right now",
57                          codec->name, codec->long_name);
58             continue;
59         }
60         CodecPluginDef definition;
61         definition.name = "videodecoder_" + std::string(codec->name);
62         definition.pluginType = PluginType::VIDEO_DECODER;
63         definition.rank = 100; // 100
64         definition.creator = VideoFfmpegDecoderCreator;
65         UpdatePluginDefinition(codec, definition);
66         // do not delete the codec in the deleter
67         codecMap[definition.name] = std::shared_ptr<AVCodec>(const_cast<AVCodec*>(codec), [](void* ptr) {});
68         if (reg->AddPlugin(definition) != Status::OK) {
69             MEDIA_LOG_W("register plugin " PUBLIC_LOG_S "(" PUBLIC_LOG_S ") failed",
70                         codec->name, codec->long_name);
71         }
72     }
73     return Status::OK;
74 }
75 
UnRegisterVideoDecoderPlugins()76 void UnRegisterVideoDecoderPlugins()
77 {
78     codecMap.clear();
79 }
80 
UpdateInCaps(const AVCodec * codec,CodecPluginDef & definition)81 void UpdateInCaps(const AVCodec* codec, CodecPluginDef& definition)
82 {
83     CapabilityBuilder incapBuilder;
84     switch (codec->id) {
85         case AV_CODEC_ID_H264:
86             incapBuilder.SetMime(OHOS::Media::MEDIA_MIME_VIDEO_H264);
87             incapBuilder.SetVideoBitStreamFormatList({VideoBitStreamFormat::AVC1, VideoBitStreamFormat::ANNEXB});
88             break;
89         default:
90             incapBuilder.SetMime("video/unknown");
91             MEDIA_LOG_I("codec is not supported right now");
92             break;
93     }
94     definition.inCaps.push_back(incapBuilder.Build());
95 }
96 
UpdateOutCaps(const AVCodec * codec,CodecPluginDef & definition)97 void UpdateOutCaps(const AVCodec* codec, CodecPluginDef& definition)
98 {
99     CapabilityBuilder outcapBuilder;
100     outcapBuilder.SetMime(OHOS::Media::MEDIA_MIME_VIDEO_RAW);
101     if (codec->pix_fmts != nullptr) {
102         DiscreteCapability<VideoPixelFormat> values;
103         size_t index = 0;
104         for (index = 0; codec->pix_fmts[index] != 0; ++index) {
105             auto supportFormat = ConvertPixelFormatFromFFmpeg(codec->pix_fmts[index]);
106             if (supportFormat != VideoPixelFormat::UNKNOWN) {
107                 values.push_back(supportFormat);
108             }
109         }
110         if (index) {
111             outcapBuilder.SetVideoPixelFormatList(values);
112         }
113     }
114     definition.outCaps.push_back(outcapBuilder.Build());
115 }
116 
UpdatePluginDefinition(const AVCodec * codec,CodecPluginDef & definition)117 void UpdatePluginDefinition(const AVCodec* codec, CodecPluginDef& definition)
118 {
119     UpdateInCaps(codec, definition);
120     UpdateOutCaps(codec, definition);
121 }
122 } // namespace
123 
124 PLUGIN_DEFINITION(FFmpegVideoDecoders, LicenseType::LGPL, RegisterVideoDecoderPlugins, UnRegisterVideoDecoderPlugins);
125 
126 namespace OHOS {
127 namespace Media {
128 namespace Plugin {
VideoFfmpegDecoderPlugin(std::string name)129 VideoFfmpegDecoderPlugin::VideoFfmpegDecoderPlugin(std::string name)
130     : CodecPlugin(std::move(name)), outBufferQ_("vdecPluginQueue", BUFFER_QUEUE_SIZE)
131 {
132     for (int32_t i = 0; i < AV_NUM_DATA_POINTERS; i++) {
133         scaleData_[i] = nullptr;
134         scaleLineSize_[i] = 0;
135     }
136     isAllocScaleData_ = false;
137 }
138 
Init()139 Status VideoFfmpegDecoderPlugin::Init()
140 {
141     OSAL::ScopedLock l(avMutex_);
142     auto iter = codecMap.find(pluginName_);
143     if (iter == codecMap.end()) {
144         MEDIA_LOG_W("cannot find codec with name " PUBLIC_LOG_S, pluginName_.c_str());
145         return Status::ERROR_UNSUPPORTED_FORMAT;
146     }
147     avCodec_ = iter->second;
148     cachedFrame_ = std::shared_ptr<AVFrame>(av_frame_alloc(), [](AVFrame* fp) { av_frame_free(&fp); });
149     videoDecParams_[Tag::REQUIRED_OUT_BUFFER_CNT] = (uint32_t)BUFFER_QUEUE_SIZE;
150     if (!decodeTask_) {
151         decodeTask_ = std::make_shared<OHOS::Media::OSAL::Task>("videoFfmpegDecThread");
152         decodeTask_->RegisterHandler([this] { ReceiveFrameBuffer(); });
153     }
154     state_ = State::INITIALIZED;
155     MEDIA_LOG_I("Init success");
156     return Status::OK;
157 }
158 
Deinit()159 Status VideoFfmpegDecoderPlugin::Deinit()
160 {
161     OSAL::ScopedLock l(avMutex_);
162     avCodec_.reset();
163     cachedFrame_.reset();
164     ResetLocked();
165     if (decodeTask_) {
166         decodeTask_->Stop();
167         decodeTask_.reset();
168     }
169     if (scale_) {
170         scale_.reset();
171     }
172     state_ = State::DESTROYED;
173     return Status::OK;
174 }
175 
SetParameter(Tag tag,const ValueType & value)176 Status VideoFfmpegDecoderPlugin::SetParameter(Tag tag, const ValueType& value)
177 {
178     OSAL::ScopedLock l(avMutex_);
179     if (videoDecParams_.count(tag)) {
180         videoDecParams_[tag] = value;
181     } else {
182         videoDecParams_.insert(std::make_pair(tag, value));
183     }
184     return Status::OK;
185 }
186 
GetParameter(Tag tag,ValueType & value)187 Status VideoFfmpegDecoderPlugin::GetParameter(Tag tag, ValueType& value)
188 {
189     OSAL::ScopedLock l(avMutex_);
190     auto res = videoDecParams_.find(tag);
191     if (res != videoDecParams_.end()) {
192         value = res->second;
193         return Status::OK;
194     }
195     return Status::ERROR_INVALID_PARAMETER;
196 }
197 
198 template <typename T>
FindInParameterMapThenAssignLocked(Tag tag,T & assign)199 void VideoFfmpegDecoderPlugin::FindInParameterMapThenAssignLocked(Tag tag, T& assign)
200 {
201     auto iter = videoDecParams_.find(tag);
202     if (iter != videoDecParams_.end() && Any::IsSameTypeWith<T>(iter->second)) {
203         assign = Plugin::AnyCast<T>(iter->second);
204     } else {
205         MEDIA_LOG_W("parameter " PUBLIC_LOG_D32 " is not found or type mismatch", static_cast<int32_t>(tag));
206     }
207 }
208 
CreateCodecContext()209 Status VideoFfmpegDecoderPlugin::CreateCodecContext()
210 {
211     auto context = avcodec_alloc_context3(avCodec_.get());
212     if (context == nullptr) {
213         MEDIA_LOG_E("cannot allocate codec context");
214         return Status::ERROR_UNKNOWN;
215     }
216     avCodecContext_ = std::shared_ptr<AVCodecContext>(context, [](AVCodecContext* ptr) {
217         if (ptr != nullptr) {
218             if (ptr->extradata) {
219                 av_free(ptr->extradata);
220                 ptr->extradata = nullptr;
221             }
222             avcodec_free_context(&ptr);
223         }
224     });
225     MEDIA_LOG_I("CreateCodecContext success");
226     return Status::OK;
227 }
228 
InitCodecContext()229 void VideoFfmpegDecoderPlugin::InitCodecContext()
230 {
231     avCodecContext_->codec_type = AVMEDIA_TYPE_VIDEO;
232     FindInParameterMapThenAssignLocked<int64_t>(Tag::MEDIA_BITRATE, avCodecContext_->bit_rate);
233     FindInParameterMapThenAssignLocked<std::uint32_t>(Tag::VIDEO_WIDTH, width_);
234     FindInParameterMapThenAssignLocked<std::uint32_t>(Tag::VIDEO_HEIGHT, height_);
235     FindInParameterMapThenAssignLocked<Plugin::VideoPixelFormat>(Tag::VIDEO_PIXEL_FORMAT, pixelFormat_);
236     MEDIA_LOG_D("bitRate: " PUBLIC_LOG_D64 ", width: " PUBLIC_LOG_U32 ", height: " PUBLIC_LOG_U32
237                 ", pixelFormat: " PUBLIC_LOG_U32, avCodecContext_->bit_rate, width_, height_, pixelFormat_);
238     SetCodecExtraData();
239     // Reset coded_width/_height to prevent it being reused from last time when
240     // the codec is opened again, causing a mismatch and possible segfault/corruption.
241     avCodecContext_->coded_width = 0;
242     avCodecContext_->coded_height = 0;
243     avCodecContext_->workaround_bugs |= FF_BUG_AUTODETECT;
244     avCodecContext_->err_recognition = 1;
245 }
246 
DeinitCodecContext()247 void VideoFfmpegDecoderPlugin::DeinitCodecContext()
248 {
249     if (avCodecContext_ == nullptr) {
250         return;
251     }
252     if (avCodecContext_->extradata) {
253         av_free(avCodecContext_->extradata);
254         avCodecContext_->extradata = nullptr;
255     }
256     avCodecContext_->extradata_size = 0;
257     avCodecContext_->opaque = nullptr;
258     avCodecContext_->width = 0;
259     avCodecContext_->height = 0;
260     avCodecContext_->coded_width = 0;
261     avCodecContext_->coded_height = 0;
262     avCodecContext_->time_base.den = 0;
263     avCodecContext_->time_base.num = 0;
264     avCodecContext_->ticks_per_frame = 0;
265     avCodecContext_->sample_aspect_ratio.num = 0;
266     avCodecContext_->sample_aspect_ratio.den = 0;
267     avCodecContext_->get_buffer2 = nullptr;
268 }
269 
SetCodecExtraData()270 void VideoFfmpegDecoderPlugin::SetCodecExtraData()
271 {
272     auto iter = videoDecParams_.find(Tag::MEDIA_CODEC_CONFIG);
273     if (iter == videoDecParams_.end() || Any::IsSameTypeWith<std::vector<uint8_t>>(iter->second)) {
274         return;
275     }
276     auto codecConfig = Plugin::AnyCast<std::vector<uint8_t>>(iter->second);
277     int configSize = static_cast<int>(codecConfig.size());
278     if (configSize > 0) {
279         auto allocSize = AlignUp(configSize + AV_INPUT_BUFFER_PADDING_SIZE, STRIDE_ALIGN);
280         avCodecContext_->extradata = static_cast<uint8_t*>(av_mallocz(allocSize));
281         auto ret = memcpy_s(avCodecContext_->extradata, configSize, codecConfig.data(), configSize);
282         if (ret != EOK) {
283             MEDIA_LOG_W("memcpy into buffer failed with code " PUBLIC_LOG_D32, ret);
284         }
285         avCodecContext_->extradata_size = configSize;
286         MEDIA_LOG_I("SetCodecExtraData success");
287     }
288 }
289 
OpenCodecContext()290 Status VideoFfmpegDecoderPlugin::OpenCodecContext()
291 {
292     const AVCodec* vdec = avcodec_find_decoder(avCodecContext_->codec_id);
293     if (vdec == nullptr) {
294         MEDIA_LOG_E("Codec: " PUBLIC_LOG_D32 " is not found", static_cast<int32_t>(avCodecContext_->codec_id));
295         DeinitCodecContext();
296         return Status::ERROR_INVALID_PARAMETER;
297     }
298     auto res = avcodec_open2(avCodecContext_.get(), avCodec_.get(), nullptr);
299     if (res != 0) {
300         MEDIA_LOG_E("avcodec open error " PUBLIC_LOG_S " when start decoder ", AVStrError(res).c_str());
301         DeinitCodecContext();
302         return Status::ERROR_UNKNOWN;
303     }
304     MEDIA_LOG_I("OpenCodecContext success");
305     return Status::OK;
306 }
307 
CloseCodecContext()308 Status VideoFfmpegDecoderPlugin::CloseCodecContext()
309 {
310     Status ret = Status::OK;
311     if (avCodecContext_ != nullptr) {
312         auto res = avcodec_close(avCodecContext_.get());
313         if (res != 0) {
314             DeinitCodecContext();
315             MEDIA_LOG_E("avcodec close error " PUBLIC_LOG_S " when stop decoder", AVStrError(res).c_str());
316             ret = Status::ERROR_UNKNOWN;
317         }
318         avCodecContext_.reset();
319     }
320     return ret;
321 }
322 
Prepare()323 Status VideoFfmpegDecoderPlugin::Prepare()
324 {
325     {
326         OSAL::ScopedLock l(avMutex_);
327         if (state_ != State::INITIALIZED && state_ != State::PREPARED) {
328             return Status::ERROR_WRONG_STATE;
329         }
330         if (CreateCodecContext() != Status::OK) {
331             MEDIA_LOG_E("Create codec context fail");
332             return Status::ERROR_UNKNOWN;
333         }
334         InitCodecContext();
335 #ifdef DUMP_RAW_DATA
336         dumpFd_ = std::fopen("./vdec_out.yuv", "wb");
337 #endif
338         state_ = State::PREPARED;
339     }
340     avPacket_ = std::shared_ptr<AVPacket>(av_packet_alloc(), [](AVPacket* ptr) {
341         av_packet_free(&ptr);
342     });
343     outBufferQ_.SetActive(true);
344     MEDIA_LOG_I("Prepare success");
345     return Status::OK;
346 }
347 
ResetLocked()348 Status VideoFfmpegDecoderPlugin::ResetLocked()
349 {
350     videoDecParams_.clear();
351     avCodecContext_.reset();
352     outBufferQ_.Clear();
353     if (scaleData_[0] != nullptr) {
354         if (isAllocScaleData_) {
355             av_free(scaleData_[0]);
356             isAllocScaleData_ = false;
357         }
358         for (int32_t i = 0; i < AV_NUM_DATA_POINTERS; i++) {
359             scaleData_[i] = nullptr;
360             scaleLineSize_[i] = 0;
361         }
362     }
363 #ifdef DUMP_RAW_DATA
364     if (dumpFd_) {
365         std::fclose(dumpFd_);
366         dumpFd_ = nullptr;
367     }
368 #endif
369     state_ = State::INITIALIZED;
370     return Status::OK;
371 }
372 
Reset()373 Status VideoFfmpegDecoderPlugin::Reset()
374 {
375     OSAL::ScopedLock l(avMutex_);
376     return ResetLocked();
377 }
378 
Start()379 Status VideoFfmpegDecoderPlugin::Start()
380 {
381     {
382         OSAL::ScopedLock l(avMutex_);
383         if (state_ != State::PREPARED) {
384             return Status::ERROR_WRONG_STATE;
385         }
386         if (OpenCodecContext() != Status::OK) {
387             MEDIA_LOG_E("Open codec context fail");
388             return Status::ERROR_UNKNOWN;
389         }
390         state_ = State::RUNNING;
391     }
392     outBufferQ_.SetActive(true);
393     decodeTask_->Start();
394     MEDIA_LOG_I("Start success");
395     return Status::OK;
396 }
397 
Stop()398 Status VideoFfmpegDecoderPlugin::Stop()
399 {
400     Status ret = Status::OK;
401     {
402         OSAL::ScopedLock l(avMutex_);
403         ret = CloseCodecContext();
404 #ifdef DUMP_RAW_DATA
405         if (dumpFd_) {
406             std::fclose(dumpFd_);
407             dumpFd_ = nullptr;
408         }
409 #endif
410         state_ = State::INITIALIZED;
411     }
412     outBufferQ_.SetActive(false);
413     decodeTask_->Stop();
414     MEDIA_LOG_I("Stop success");
415     return ret;
416 }
417 
QueueOutputBuffer(const std::shared_ptr<Buffer> & outputBuffer,int32_t timeoutMs)418 Status VideoFfmpegDecoderPlugin::QueueOutputBuffer(const std::shared_ptr<Buffer>& outputBuffer, int32_t timeoutMs)
419 {
420     outBufferQ_.Push(outputBuffer);
421     MEDIA_LOG_DD("QueueOutputBuffer success");
422     return Status::OK;
423 }
424 
Flush()425 Status VideoFfmpegDecoderPlugin::Flush()
426 {
427     OSAL::ScopedLock l(avMutex_);
428     if (avCodecContext_ != nullptr) {
429         avcodec_flush_buffers(avCodecContext_.get());
430     }
431     return Status::OK;
432 }
433 
QueueInputBuffer(const std::shared_ptr<Buffer> & inputBuffer,int32_t timeoutMs)434 Status VideoFfmpegDecoderPlugin::QueueInputBuffer(const std::shared_ptr<Buffer>& inputBuffer, int32_t timeoutMs)
435 {
436     if (inputBuffer->IsEmpty() && !(inputBuffer->flag & BUFFER_FLAG_EOS)) {
437         MEDIA_LOG_E("decoder does not support fd buffer");
438         return Status::ERROR_INVALID_DATA;
439     }
440     Status ret = Status::OK;
441     {
442         OSAL::ScopedLock l(avMutex_);
443         ret = SendBufferLocked(inputBuffer);
444     }
445     NotifyInputBufferDone(inputBuffer);
446     MEDIA_LOG_DD("QueueInputBuffer ret: " PUBLIC_LOG_U32, ret);
447     return ret;
448 }
449 
SendBufferLocked(const std::shared_ptr<Buffer> & inputBuffer)450 Status VideoFfmpegDecoderPlugin::SendBufferLocked(const std::shared_ptr<Buffer>& inputBuffer)
451 {
452     if (state_ != State::RUNNING) {
453         MEDIA_LOG_W("SendBufferLocked in wrong state: " PUBLIC_LOG_D32, state_);
454         return Status::ERROR_WRONG_STATE;
455     }
456     if (inputBuffer && !(inputBuffer->flag & BUFFER_FLAG_EOS)) {
457         auto inputMemory = inputBuffer->GetMemory();
458         if (inputMemory == nullptr) {
459             MEDIA_LOG_E("SendBufferLocked inputBuffer GetMemory nullptr");
460             return Status::ERROR_UNKNOWN;
461         }
462         const uint8_t* ptr = inputMemory->GetReadOnlyData();
463         auto bufferLength = inputMemory->GetSize();
464         size_t bufferEnd = bufferLength;
465         // pad to data if needed
466         if ((bufferLength % AV_INPUT_BUFFER_PADDING_SIZE != 0) &&
467             (bufferLength - bufferEnd + bufferLength % AV_INPUT_BUFFER_PADDING_SIZE < AV_INPUT_BUFFER_PADDING_SIZE)) {
468             if (paddedBufferSize_ < bufferLength + AV_INPUT_BUFFER_PADDING_SIZE) {
469                 paddedBufferSize_ = bufferLength + AV_INPUT_BUFFER_PADDING_SIZE;
470                 paddedBuffer_.reserve(paddedBufferSize_);
471                 MEDIA_LOG_I("increase padded buffer size to " PUBLIC_LOG_ZU, paddedBufferSize_);
472             }
473             paddedBuffer_.assign(ptr, ptr + bufferLength);
474             paddedBuffer_.insert(paddedBuffer_.end(), AV_INPUT_BUFFER_PADDING_SIZE, 0);
475             ptr = paddedBuffer_.data();
476         }
477         avPacket_->data = const_cast<uint8_t*>(ptr);
478         avPacket_->size = static_cast<int32_t>(bufferLength);
479         avPacket_->pts = static_cast<int64_t>(inputBuffer->pts);
480     }
481     auto ret = avcodec_send_packet(avCodecContext_.get(), avPacket_.get());
482     av_packet_unref(avPacket_.get());
483     if (ret < 0) {
484         MEDIA_LOG_DD("send buffer error " PUBLIC_LOG_S, AVStrError(ret).c_str());
485         return Status::ERROR_NO_MEMORY;
486     }
487     return Status::OK;
488 }
489 
490 #ifdef DUMP_RAW_DATA
DumpVideoRawOutData()491 void VideoFfmpegDecoderPlugin::DumpVideoRawOutData()
492 {
493     if (dumpFd_ == nullptr) {
494         return;
495     }
496     if (pixelFormat_ == VideoPixelFormat::YUV420P) {
497         if (scaleData_[0] != nullptr && scaleLineSize_[0] != 0) {
498             std::fwrite(reinterpret_cast<const char*>(scaleData_[0]),
499                         scaleLineSize_[0] * height_, 1, dumpFd_);
500         }
501         if (scaleData_[1] != nullptr && scaleLineSize_[1] != 0) {
502             std::fwrite(reinterpret_cast<const char*>(scaleData_[1]),
503                         scaleLineSize_[1] * height_ / 2, 1, dumpFd_); // 2
504         }
505         if (scaleData_[2] != nullptr && scaleLineSize_[2] != 0) { // 2
506             std::fwrite(reinterpret_cast<const char*>(scaleData_[2]),
507                         scaleLineSize_[2] * height_ / 2, 1, dumpFd_); // 2
508         }
509     } else if (pixelFormat_ == VideoPixelFormat::NV21 || pixelFormat_ == VideoPixelFormat::NV12) {
510         if (scaleData_[0] != nullptr && scaleLineSize_[0] != 0) {
511             std::fwrite(reinterpret_cast<const char*>(scaleData_[0]),
512                         scaleLineSize_[0] * height_, 1, dumpFd_);
513         }
514         if (scaleData_[1] != nullptr && scaleLineSize_[1] != 0) {
515             std::fwrite(reinterpret_cast<const char*>(scaleData_[1]),
516                         scaleLineSize_[1] * height_ / 2, 1, dumpFd_); // 2
517         }
518     } else if (pixelFormat_ == VideoPixelFormat::RGBA || pixelFormat_ == VideoPixelFormat::ARGB ||
519                pixelFormat_ == VideoPixelFormat::ABGR || pixelFormat_ == VideoPixelFormat::BGRA) {
520         if (scaleData_[0] != nullptr && scaleLineSize_[0] != 0) {
521             std::fwrite(reinterpret_cast<const char*>(scaleData_[0]),
522                         scaleLineSize_[0] * height_, 1, dumpFd_);
523         }
524     }
525 }
526 #endif
527 
ScaleVideoFrame()528 Status VideoFfmpegDecoderPlugin::ScaleVideoFrame()
529 {
530     if (ConvertPixelFormatFromFFmpeg(static_cast<AVPixelFormat>(cachedFrame_->format)) == pixelFormat_ &&
531         static_cast<uint32_t>(cachedFrame_->width) == width_ &&
532         static_cast<uint32_t>(cachedFrame_->height) == height_) {
533         for (int32_t i = 0; cachedFrame_->linesize[i] > 0; i++) {
534             scaleData_[i] = cachedFrame_->data[i];
535             scaleLineSize_[i] = cachedFrame_->linesize[i];
536         }
537         return Status::OK;
538     }
539     if (!scale_) {
540         scale_ = std::make_shared<Ffmpeg::Scale>();
541         Ffmpeg::ScalePara scalePara {
542             static_cast<int32_t>(cachedFrame_->width),
543             static_cast<int32_t>(cachedFrame_->height),
544             static_cast<AVPixelFormat>(cachedFrame_->format),
545             static_cast<int32_t>(width_),
546             static_cast<int32_t>(height_),
547             Ffmpeg::ConvertPixelFormatToFFmpeg(pixelFormat_),
548             STRIDE_ALIGN
549         };
550         FALSE_RETURN_V_MSG(scale_->Init(scalePara, scaleData_, scaleLineSize_) == Status::OK,
551             Status::ERROR_UNKNOWN, "Scale init error");
552         isAllocScaleData_ = true;
553     }
554     auto res = scale_->Convert(cachedFrame_->data, cachedFrame_->linesize, scaleData_, scaleLineSize_);
555     FALSE_RETURN_V_MSG_E(res == Status::OK, Status::ERROR_UNKNOWN, "Scale convert fail.");
556     MEDIA_LOG_D("ScaleVideoFrame success");
557     return Status::OK;
558 }
559 
560 #ifndef OHOS_LITE
WriteYuvDataStride(const std::shared_ptr<Buffer> & frameBuffer,int32_t stride)561 Status VideoFfmpegDecoderPlugin::WriteYuvDataStride(const std::shared_ptr<Buffer>& frameBuffer, int32_t stride)
562 {
563     auto frameBufferMem = frameBuffer->GetMemory();
564     if (frameBufferMem == nullptr) {
565         MEDIA_LOG_E("WriteYuvDataStride frameBuffer GetMemory nullptr");
566         return Status::ERROR_UNKNOWN;
567     }
568     size_t srcPos = 0;
569     size_t dstPos = 0;
570     if (pixelFormat_ == VideoPixelFormat::YUV420P) {
571         auto writeSize = scaleLineSize_[0];
572         for (uint32_t colNum = 0; colNum < height_; colNum++) {
573             frameBufferMem->Write(scaleData_[0] + srcPos, writeSize, dstPos);
574             dstPos += stride;
575         }
576         srcPos = 0;
577         writeSize = scaleLineSize_[1];
578         for (uint32_t colNum = 0; colNum < height_; colNum++) {
579             frameBufferMem->Write(scaleData_[1] + srcPos, writeSize, dstPos);
580             dstPos += stride;
581         }
582         srcPos = 0;
583         writeSize = scaleLineSize_[2]; // 2
584         for (uint32_t colNum = 0; colNum < height_; colNum++) {
585             frameBufferMem->Write(scaleData_[2] + srcPos, writeSize, dstPos); // 2
586             dstPos += stride;
587         }
588     } else if ((pixelFormat_ == VideoPixelFormat::NV12) || (pixelFormat_ == VideoPixelFormat::NV21)) {
589         auto writeSize = scaleLineSize_[0];
590         for (uint32_t colNum = 0; colNum < height_; colNum++) {
591             frameBufferMem->Write(scaleData_[0] + srcPos, writeSize, dstPos);
592             dstPos += stride;
593         }
594         srcPos = 0;
595         writeSize = scaleLineSize_[1];
596         for (uint32_t colNum = 0; colNum < height_; colNum++) {
597             frameBufferMem->Write(scaleData_[1] + srcPos, writeSize, dstPos);
598             dstPos += stride;
599         }
600     } else {
601         return Status::ERROR_UNSUPPORTED_FORMAT;
602     }
603     MEDIA_LOG_D("WriteYuvDataStride success");
604     return Status::OK;
605 }
606 
WriteRgbDataStride(const std::shared_ptr<Buffer> & frameBuffer,int32_t stride)607 Status VideoFfmpegDecoderPlugin::WriteRgbDataStride(const std::shared_ptr<Buffer>& frameBuffer, int32_t stride)
608 {
609     auto frameBufferMem = frameBuffer->GetMemory();
610     if (frameBufferMem == nullptr) {
611         MEDIA_LOG_E("WriteRgbDataStride frameBuffer GetMemory nullptr");
612         return Status::ERROR_UNKNOWN;
613     }
614     if (pixelFormat_ == VideoPixelFormat::RGBA || pixelFormat_ == VideoPixelFormat::ARGB ||
615         pixelFormat_ == VideoPixelFormat::ABGR || pixelFormat_ == VideoPixelFormat::BGRA) {
616         size_t srcPos = 0;
617         size_t dstPos = 0;
618         auto writeSize = scaleLineSize_[0];
619         for (uint32_t colNum = 0; colNum < height_; colNum++) {
620             frameBufferMem->Write(scaleData_[0] + srcPos, writeSize, dstPos);
621             dstPos += static_cast<size_t>(stride);
622         }
623     } else {
624         return Status::ERROR_UNSUPPORTED_FORMAT;
625     }
626     MEDIA_LOG_D("WriteRgbDataStride success");
627     return Status::OK;
628 }
629 #endif
630 
WriteYuvData(const std::shared_ptr<Buffer> & frameBuffer)631 Status VideoFfmpegDecoderPlugin::WriteYuvData(const std::shared_ptr<Buffer>& frameBuffer)
632 {
633     auto frameBufferMem = frameBuffer->GetMemory();
634     if (frameBufferMem == nullptr) {
635         return Status::ERROR_NULL_POINTER;
636     }
637 #ifndef OHOS_LITE
638     if (frameBufferMem->GetMemoryType() == Plugin::MemoryType::SURFACE_BUFFER) {
639         std::shared_ptr<Plugin::SurfaceMemory> surfaceMemory =
640                 Plugin::ReinterpretPointerCast<Plugin::SurfaceMemory>(frameBufferMem);
641         auto stride = surfaceMemory->GetSurfaceBufferStride();
642         if (stride % width_) {
643             return WriteYuvDataStride(frameBuffer, stride);
644         }
645     }
646 #endif
647     size_t ySize = static_cast<size_t>(scaleLineSize_[0] * height_);
648     // AV_PIX_FMT_YUV420P: scaleLineSize_[0] = scaleLineSize_[1] * 2 = scaleLineSize_[2] * 2
649     // AV_PIX_FMT_NV12: scaleLineSize_[0] = scaleLineSize_[1]
650     size_t uvSize = static_cast<size_t>(scaleLineSize_[1] * height_ / 2); // 2
651     size_t frameSize = 0;
652     if (pixelFormat_ == VideoPixelFormat::YUV420P) {
653         frameSize = ySize + (uvSize * 2); // 2
654     } else if (pixelFormat_ == VideoPixelFormat::NV21 || pixelFormat_ == VideoPixelFormat::NV12) {
655         frameSize = ySize + uvSize;
656     }
657     FALSE_RETURN_V_MSG_E(frameBufferMem->GetCapacity() >= frameSize, Status::ERROR_NO_MEMORY,
658                          "output buffer size is not enough: real[" PUBLIC_LOG "zu], need[" PUBLIC_LOG "zu]",
659                          frameBufferMem->GetCapacity(), frameSize);
660     if (pixelFormat_ == VideoPixelFormat::YUV420P) {
661         frameBufferMem->Write(scaleData_[0], ySize);
662         frameBufferMem->Write(scaleData_[1], uvSize);
663         frameBufferMem->Write(scaleData_[2], uvSize); // 2
664     } else if ((pixelFormat_ == VideoPixelFormat::NV12) || (pixelFormat_ == VideoPixelFormat::NV21)) {
665         frameBufferMem->Write(scaleData_[0], ySize);
666         frameBufferMem->Write(scaleData_[1], uvSize);
667     } else {
668         return Status::ERROR_UNSUPPORTED_FORMAT;
669     }
670     MEDIA_LOG_DD("WriteYuvData success");
671     return Status::OK;
672 }
673 
WriteRgbData(const std::shared_ptr<Buffer> & frameBuffer)674 Status VideoFfmpegDecoderPlugin::WriteRgbData(const std::shared_ptr<Buffer>& frameBuffer)
675 {
676     auto frameBufferMem = frameBuffer->GetMemory();
677     if (frameBufferMem == nullptr) {
678         MEDIA_LOG_E("WriteRgbData frameBuffer GetMemory nullptr");
679         return Status::ERROR_UNKNOWN;
680     }
681 #ifndef OHOS_LITE
682     if (frameBufferMem->GetMemoryType() == Plugin::MemoryType::SURFACE_BUFFER) {
683         std::shared_ptr<Plugin::SurfaceMemory> surfaceMemory =
684                 Plugin::ReinterpretPointerCast<Plugin::SurfaceMemory>(frameBufferMem);
685         auto stride = surfaceMemory->GetSurfaceBufferStride();
686         if (stride % width_) {
687             return WriteRgbDataStride(frameBuffer, stride);
688         }
689     }
690 #endif
691     size_t frameSize = static_cast<size_t>(scaleLineSize_[0] * height_);
692     FALSE_RETURN_V_MSG_E(frameBufferMem->GetCapacity() >= frameSize, Status::ERROR_NO_MEMORY,
693                          "output buffer size is not enough: real[" PUBLIC_LOG "zu], need[" PUBLIC_LOG "zu]",
694                          frameBufferMem->GetCapacity(), frameSize);
695     if (pixelFormat_ == VideoPixelFormat::RGBA || pixelFormat_ == VideoPixelFormat::ARGB ||
696         pixelFormat_ == VideoPixelFormat::ABGR || pixelFormat_ == VideoPixelFormat::BGRA) {
697         frameBufferMem->Write(scaleData_[0], frameSize);
698     } else {
699         return Status::ERROR_UNSUPPORTED_FORMAT;
700     }
701     MEDIA_LOG_D("WriteRgbData success");
702     return Status::OK;
703 }
704 
FillFrameBuffer(const std::shared_ptr<Buffer> & frameBuffer)705 Status VideoFfmpegDecoderPlugin::FillFrameBuffer(const std::shared_ptr<Buffer>& frameBuffer)
706 {
707     MEDIA_LOG_DD("receive one frame: " PUBLIC_LOG_D32 ", picture type: " PUBLIC_LOG_D32 ", pixel format: "
708                  PUBLIC_LOG_D32 ", packet size: " PUBLIC_LOG_D32, cachedFrame_->key_frame,
709                  static_cast<int32_t>(cachedFrame_->pict_type), cachedFrame_->format, cachedFrame_->pkt_size);
710     FALSE_RETURN_V_MSG_E((cachedFrame_->flags & AV_FRAME_FLAG_CORRUPT) == 0, Status::ERROR_INVALID_DATA,
711                          "decoded frame is corrupt");
712     auto ret = ScaleVideoFrame();
713     FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "ScaleVideoFrame fail: " PUBLIC_LOG_D32, ret);
714     auto bufferMeta = frameBuffer->GetBufferMeta();
715     if (bufferMeta != nullptr && bufferMeta->GetType() == BufferMetaType::VIDEO) {
716         std::shared_ptr<VideoBufferMeta> videoMeta = ReinterpretPointerCast<VideoBufferMeta>(bufferMeta);
717         videoMeta->videoPixelFormat = pixelFormat_;
718         videoMeta->height = height_;
719         videoMeta->width = width_;
720         for (int i = 0; scaleLineSize_[i] > 0; ++i) {
721             videoMeta->stride.emplace_back(scaleLineSize_[i]);
722         }
723         videoMeta->planes = videoMeta->stride.size();
724     }
725 #ifdef DUMP_RAW_DATA
726     DumpVideoRawOutData();
727 #endif
728     auto newFormat = ConvertPixelFormatToFFmpeg(pixelFormat_);
729     if (IsYuvFormat(newFormat)) {
730         FALSE_RETURN_V_MSG_E(WriteYuvData(frameBuffer) == Status::OK, Status::ERROR_UNSUPPORTED_FORMAT,
731                              "Unsupported pixel format: " PUBLIC_LOG_U32, pixelFormat_);
732     } else if (IsRgbFormat(newFormat)) {
733         FALSE_RETURN_V_MSG_E(WriteRgbData(frameBuffer) == Status::OK, Status::ERROR_UNSUPPORTED_FORMAT,
734                              "Unsupported pixel format: " PUBLIC_LOG_U32, pixelFormat_);
735     } else {
736         MEDIA_LOG_E("Unsupported pixel format: " PUBLIC_LOG_U32, pixelFormat_);
737         return Status::ERROR_UNSUPPORTED_FORMAT;
738     }
739     frameBuffer->pts = static_cast<uint64_t>(cachedFrame_->pts);
740     MEDIA_LOG_DD("FillFrameBuffer success");
741     return Status::OK;
742 }
743 
ReceiveBufferLocked(const std::shared_ptr<Buffer> & frameBuffer)744 Status VideoFfmpegDecoderPlugin::ReceiveBufferLocked(const std::shared_ptr<Buffer>& frameBuffer)
745 {
746     if (state_ != State::RUNNING) {
747         MEDIA_LOG_W("ReceiveBufferLocked in wrong state: " PUBLIC_LOG_D32, state_);
748         return Status::ERROR_WRONG_STATE;
749     }
750     Status status;
751     auto ret = avcodec_receive_frame(avCodecContext_.get(), cachedFrame_.get());
752     if (ret >= 0) {
753         status = FillFrameBuffer(frameBuffer);
754     } else if (ret == AVERROR_EOF) {
755         MEDIA_LOG_I("eos received");
756         auto frameBufferMem = frameBuffer->GetMemory();
757         if (frameBufferMem == nullptr) {
758             MEDIA_LOG_E("ReceiveBufferLocked frameBuffer GetMemory nullptr");
759             return Status::ERROR_UNKNOWN;
760         }
761         frameBufferMem->Reset();
762         frameBuffer->flag |= BUFFER_FLAG_EOS;
763         avcodec_flush_buffers(avCodecContext_.get());
764         status = Status::END_OF_STREAM;
765     } else {
766         MEDIA_LOG_DD("video decoder receive error: " PUBLIC_LOG_S, AVStrError(ret).c_str());
767         status = Status::ERROR_TIMED_OUT;
768     }
769     av_frame_unref(cachedFrame_.get());
770     MEDIA_LOG_DD("ReceiveBufferLocked status: " PUBLIC_LOG_U32, status);
771     return status;
772 }
773 
ReceiveFrameBuffer()774 void VideoFfmpegDecoderPlugin::ReceiveFrameBuffer()
775 {
776     std::shared_ptr<Buffer> frameBuffer = outBufferQ_.Pop();
777     if (frameBuffer == nullptr || frameBuffer->IsEmpty()) {
778         MEDIA_LOG_W("cannot fetch valid buffer to output");
779         return;
780     }
781     auto frameMeta = frameBuffer->GetBufferMeta();
782     if (frameMeta == nullptr || frameMeta->GetType() != BufferMetaType::VIDEO) {
783         MEDIA_LOG_W("output buffer is not video buffer");
784         return;
785     }
786     Status status;
787     {
788         OSAL::ScopedLock l(avMutex_);
789         status = ReceiveBufferLocked(frameBuffer);
790     }
791     if (status == Status::OK || status == Status::END_OF_STREAM) {
792         NotifyOutputBufferDone(frameBuffer);
793     } else {
794         outBufferQ_.Push(frameBuffer);
795     }
796 }
797 
NotifyInputBufferDone(const std::shared_ptr<Buffer> & input)798 void VideoFfmpegDecoderPlugin::NotifyInputBufferDone(const std::shared_ptr<Buffer>& input)
799 {
800     if (dataCb_ != nullptr) {
801         dataCb_->OnInputBufferDone(input);
802     }
803 }
804 
NotifyOutputBufferDone(const std::shared_ptr<Buffer> & output)805 void VideoFfmpegDecoderPlugin::NotifyOutputBufferDone(const std::shared_ptr<Buffer>& output)
806 {
807     if (dataCb_ != nullptr) {
808         dataCb_->OnOutputBufferDone(output);
809     }
810 }
811 
GetAllocator()812 std::shared_ptr<Allocator> VideoFfmpegDecoderPlugin::GetAllocator()
813 {
814     return nullptr;
815 }
816 } // namespace Plugin
817 } // namespace Media
818 } // namespace OHOS
819 #endif
820