1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #define MEDIA_PIPELINE
16 
17 #include <malloc.h>
18 #include <map>
19 #include <unistd.h>
20 #include <vector>
21 #include "avcodec_video_decoder.h"
22 #include "avcodec_errors.h"
23 #include "avcodec_trace.h"
24 #include "common/log.h"
25 #include "media_description.h"
26 #include "surface_type.h"
27 #include "buffer/avbuffer_queue_consumer.h"
28 #include "meta/meta_key.h"
29 #include "meta/meta.h"
30 #include "video_decoder_adapter.h"
31 #include "media_core.h"
32 #include "avcodec_sysevent.h"
33 
34 namespace {
35 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "VideoDecoderAdapter" };
36 }
37 
38 namespace OHOS {
39 namespace Media {
40 using namespace MediaAVCodec;
41 const std::string VIDEO_INPUT_BUFFER_QUEUE_NAME = "VideoDecoderInputBufferQueue";
42 
VideoDecoderCallback(std::shared_ptr<VideoDecoderAdapter> videoDecoder)43 VideoDecoderCallback::VideoDecoderCallback(std::shared_ptr<VideoDecoderAdapter> videoDecoder)
44 {
45     MEDIA_LOG_I("VideoDecoderCallback instances create.");
46     videoDecoderAdapter_ = videoDecoder;
47 }
48 
~VideoDecoderCallback()49 VideoDecoderCallback::~VideoDecoderCallback()
50 {
51     MEDIA_LOG_I("~VideoDecoderCallback()");
52 }
53 
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)54 void VideoDecoderCallback::OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode)
55 {
56     if (auto videoDecoderAdapter = videoDecoderAdapter_.lock()) {
57         videoDecoderAdapter->OnError(errorType, errorCode);
58     } else {
59         MEDIA_LOG_I("invalid videoDecoderAdapter");
60     }
61 }
62 
OnOutputFormatChanged(const MediaAVCodec::Format & format)63 void VideoDecoderCallback::OnOutputFormatChanged(const MediaAVCodec::Format &format)
64 {
65     if (auto videoDecoderAdapter = videoDecoderAdapter_.lock()) {
66         videoDecoderAdapter->OnOutputFormatChanged(format);
67     } else {
68         MEDIA_LOG_I("invalid videoDecoderAdapter");
69     }
70 }
71 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)72 void VideoDecoderCallback::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
73 {
74     if (auto videoDecoderAdapter = videoDecoderAdapter_.lock()) {
75         videoDecoderAdapter->OnInputBufferAvailable(index, buffer);
76     } else {
77         MEDIA_LOG_I("invalid videoDecoderAdapter");
78     }
79 }
80 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)81 void VideoDecoderCallback::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
82 {
83     if (auto videoDecoderAdapter = videoDecoderAdapter_.lock()) {
84         videoDecoderAdapter->OnOutputBufferAvailable(index, buffer);
85     } else {
86         MEDIA_LOG_I("invalid videoDecoderAdapter");
87     }
88 }
89 
VideoDecoderAdapter()90 VideoDecoderAdapter::VideoDecoderAdapter()
91 {
92     MEDIA_LOG_I("VideoDecoderAdapter instances create.");
93 }
94 
~VideoDecoderAdapter()95 VideoDecoderAdapter::~VideoDecoderAdapter()
96 {
97     MEDIA_LOG_I("~VideoDecoderAdapter.");
98     FALSE_RETURN_MSG(mediaCodec_ != nullptr, "mediaCodec_ is nullptr");
99     mediaCodec_->Release();
100 }
101 
Init(MediaAVCodec::AVCodecType type,bool isMimeType,const std::string & name)102 Status VideoDecoderAdapter::Init(MediaAVCodec::AVCodecType type, bool isMimeType, const std::string &name)
103 {
104     MEDIA_LOG_I("mediaCodec_->Init.");
105 
106     Format format;
107     int32_t ret;
108     std::shared_ptr<Media::Meta> callerInfo = std::make_shared<Media::Meta>();
109     callerInfo->SetData(Media::Tag::AV_CODEC_FORWARD_CALLER_PID, appPid_);
110     callerInfo->SetData(Media::Tag::AV_CODEC_FORWARD_CALLER_UID, appUid_);
111     callerInfo->SetData(Media::Tag::AV_CODEC_FORWARD_CALLER_PROCESS_NAME, bundleName_);
112     format.SetMeta(callerInfo);
113     mediaCodecName_ = "";
114     if (isMimeType) {
115         ret = MediaAVCodec::VideoDecoderFactory::CreateByMime(name, format, mediaCodec_);
116         MEDIA_LOG_I("VideoDecoderAdapter::Init CreateByMime errorCode %{public}d", ret);
117     } else {
118         ret = MediaAVCodec::VideoDecoderFactory::CreateByName(name, format, mediaCodec_);
119         MEDIA_LOG_I("VideoDecoderAdapter::Init CreateByName errorCode %{public}d", ret);
120     }
121 
122     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, Status::ERROR_INVALID_STATE, "mediaCodec_ is nullptr");
123     mediaCodecName_ = name;
124     return Status::OK;
125 }
126 
Configure(const Format & format)127 Status VideoDecoderAdapter::Configure(const Format &format)
128 {
129     MEDIA_LOG_I("VideoDecoderAdapter->Configure.");
130     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, Status::ERROR_INVALID_STATE, "mediaCodec_ is nullptr");
131     int32_t ret = mediaCodec_->Configure(format);
132     isConfigured_ = ret == AVCodecServiceErrCode::AVCS_ERR_OK;
133     return isConfigured_ ? Status::OK : Status::ERROR_INVALID_DATA;
134 }
135 
SetParameter(const Format & format)136 int32_t VideoDecoderAdapter::SetParameter(const Format &format)
137 {
138     MEDIA_LOG_I("SetParameter enter.");
139     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, AVCodecServiceErrCode::AVCS_ERR_INVALID_VAL, "mediaCodec_ is nullptr");
140     return mediaCodec_->SetParameter(format);
141 }
142 
Start()143 Status VideoDecoderAdapter::Start()
144 {
145     MEDIA_LOG_I("Start enter.");
146     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, Status::ERROR_INVALID_STATE, "mediaCodec_ is nullptr");
147     FALSE_RETURN_V_MSG(isConfigured_, Status::ERROR_INVALID_STATE, "mediaCodec_ is not configured");
148     int32_t ret = mediaCodec_->Start();
149     if (ret != AVCodecServiceErrCode::AVCS_ERR_OK) {
150         std::string instanceId = std::to_string(instanceId_);
151         struct VideoCodecFaultInfo videoCodecFaultInfo;
152         videoCodecFaultInfo.appName = bundleName_;
153         videoCodecFaultInfo.instanceId = instanceId;
154         videoCodecFaultInfo.callerType = "player_framework";
155         videoCodecFaultInfo.videoCodec = mediaCodecName_;
156         videoCodecFaultInfo.errMsg = "mediaCodec_ start failed";
157         FaultVideoCodecEventWrite(videoCodecFaultInfo);
158     }
159     return ret == AVCodecServiceErrCode::AVCS_ERR_OK ? Status::OK : Status::ERROR_INVALID_STATE;
160 }
161 
Stop()162 Status VideoDecoderAdapter::Stop()
163 {
164     MEDIA_LOG_I("Stop enter.");
165     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, Status::ERROR_INVALID_STATE, "mediaCodec_ is nullptr");
166     FALSE_RETURN_V_MSG(isConfigured_, Status::ERROR_INVALID_STATE, "mediaCodec_ is not configured");
167     mediaCodec_->Stop();
168     return Status::OK;
169 }
170 
Flush()171 Status VideoDecoderAdapter::Flush()
172 {
173     MEDIA_LOG_I("Flush enter.");
174     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, Status::ERROR_INVALID_STATE, "mediaCodec_ is nullptr");
175     FALSE_RETURN_V_MSG(isConfigured_, Status::ERROR_INVALID_STATE, "mediaCodec_ is not configured");
176     int32_t ret = mediaCodec_->Flush();
177     std::unique_lock<std::mutex> lock(mutex_);
178     if (inputBufferQueueConsumer_ != nullptr) {
179         for (auto &buffer : bufferVector_) {
180             inputBufferQueueConsumer_->DetachBuffer(buffer);
181         }
182         bufferVector_.clear();
183         inputBufferQueueConsumer_->SetQueueSize(0);
184     }
185     return ret == AVCodecServiceErrCode::AVCS_ERR_OK ? Status::OK : Status::ERROR_INVALID_STATE;
186 }
187 
Reset()188 Status VideoDecoderAdapter::Reset()
189 {
190     MEDIA_LOG_I("Reset enter.");
191     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, Status::ERROR_INVALID_STATE, "mediaCodec_ is nullptr");
192     mediaCodec_->Reset();
193     isConfigured_ = false;
194     std::unique_lock<std::mutex> lock(mutex_);
195     if (inputBufferQueueConsumer_ != nullptr) {
196         for (auto &buffer : bufferVector_) {
197             inputBufferQueueConsumer_->DetachBuffer(buffer);
198         }
199         bufferVector_.clear();
200         inputBufferQueueConsumer_->SetQueueSize(0);
201     }
202     return Status::OK;
203 }
204 
Release()205 Status VideoDecoderAdapter::Release()
206 {
207     MEDIA_LOG_I("Release enter.");
208     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, Status::ERROR_INVALID_STATE, "mediaCodec_ is nullptr");
209     int32_t ret = mediaCodec_->Release();
210     currentTime_ = -1;
211     return ret == AVCodecServiceErrCode::AVCS_ERR_OK ? Status::OK : Status::ERROR_INVALID_STATE;
212 }
213 
ResetRenderTime()214 void VideoDecoderAdapter::ResetRenderTime()
215 {
216     currentTime_ = -1;
217 }
218 
SetCallback(const std::shared_ptr<MediaAVCodec::MediaCodecCallback> & callback)219 int32_t VideoDecoderAdapter::SetCallback(const std::shared_ptr<MediaAVCodec::MediaCodecCallback> &callback)
220 {
221     MEDIA_LOG_I("SetCallback enter.");
222     callback_ = callback;
223     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, AVCodecServiceErrCode::AVCS_ERR_INVALID_VAL, "mediaCodec_ is nullptr");
224     std::shared_ptr<MediaAVCodec::MediaCodecCallback> mediaCodecCallback
225         = std::make_shared<VideoDecoderCallback>(shared_from_this());
226     return mediaCodec_->SetCallback(mediaCodecCallback);
227 }
228 
PrepareInputBufferQueue()229 void VideoDecoderAdapter::PrepareInputBufferQueue()
230 {
231     if (inputBufferQueue_ != nullptr && inputBufferQueue_-> GetQueueSize() > 0) {
232         MEDIA_LOG_W("InputBufferQueue already create");
233         return;
234     }
235     inputBufferQueue_ = AVBufferQueue::Create(0,
236         MemoryType::UNKNOWN_MEMORY, VIDEO_INPUT_BUFFER_QUEUE_NAME, true);
237     inputBufferQueueProducer_ = inputBufferQueue_->GetProducer();
238     inputBufferQueueConsumer_ = inputBufferQueue_->GetConsumer();
239 }
240 
GetBufferQueueProducer()241 sptr<AVBufferQueueProducer> VideoDecoderAdapter::GetBufferQueueProducer()
242 {
243     return inputBufferQueueProducer_;
244 }
245 
GetBufferQueueConsumer()246 sptr<AVBufferQueueConsumer> VideoDecoderAdapter::GetBufferQueueConsumer()
247 {
248     return inputBufferQueueConsumer_;
249 }
250 
AquireAvailableInputBuffer()251 void VideoDecoderAdapter::AquireAvailableInputBuffer()
252 {
253     AVCodecTrace trace("VideoDecoderAdapter::AquireAvailableInputBuffer");
254     if (inputBufferQueueConsumer_ == nullptr) {
255         MEDIA_LOG_E("inputBufferQueueConsumer_ is null");
256         return;
257     }
258     std::unique_lock<std::mutex> lock(mutex_);
259     std::shared_ptr<AVBuffer> tmpBuffer;
260     if (inputBufferQueueConsumer_->AcquireBuffer(tmpBuffer) == Status::OK) {
261         FALSE_RETURN_MSG(tmpBuffer->meta_ != nullptr, "tmpBuffer is nullptr.");
262         uint32_t index;
263         FALSE_RETURN_MSG(tmpBuffer->meta_->GetData(Tag::REGULAR_TRACK_ID, index), "get index failed.");
264         if (tmpBuffer->flag_ & (uint32_t)(Plugins::AVBufferFlag::EOS)) {
265             MEDIA_LOG_I("AquireAvailableInputBuffer for eos, index: %{public}u,  bufferid: %{public}" PRIu64
266                 ", pts: %{public}" PRIu64", flag: %{public}u", index, tmpBuffer->GetUniqueId(),
267                 tmpBuffer->pts_, tmpBuffer->flag_);
268             tmpBuffer->memory_->SetSize(0); // to make sure hisi codec know this eos buffer
269         }
270         FALSE_RETURN_MSG(mediaCodec_ != nullptr, "mediaCodec_ is nullptr.");
271         int32_t ret = mediaCodec_->QueueInputBuffer(index);
272         if (ret != ERR_OK) {
273             MEDIA_LOG_E("QueueInputBuffer failed, index: %{public}u,  bufferid: %{public}" PRIu64
274                 ", pts: %{public}" PRIu64", flag: %{public}u, errCode: %{public}d", index, tmpBuffer->GetUniqueId(),
275                 tmpBuffer->pts_, tmpBuffer->flag_, ret);
276             if (ret == AVCS_ERR_DECRYPT_FAILED) {
277                 eventReceiver_->OnEvent({"video_decoder_adapter", EventType::EVENT_ERROR,
278                     MSERR_DRM_VERIFICATION_FAILED});
279             }
280         } else {
281             MEDIA_LOG_D("QueueInputBuffer success, index: %{public}u,  bufferid: %{public}" PRIu64
282                 ", pts: %{public}" PRIu64", flag: %{public}u", index, tmpBuffer->GetUniqueId(),
283                 tmpBuffer->pts_, tmpBuffer->flag_);
284         }
285     } else {
286         MEDIA_LOG_E("AcquireBuffer failed.");
287     }
288 }
289 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)290 void VideoDecoderAdapter::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
291 {
292     FALSE_RETURN_MSG(buffer != nullptr && buffer->meta_ != nullptr, "meta_ is nullptr.");
293     MEDIA_LOG_D("OnInputBufferAvailable enter. index: %{public}u, bufferid: %{public}" PRIu64", pts: %{public}" PRIu64
294         ", flag: %{public}u", index, buffer->GetUniqueId(), buffer->pts_, buffer->flag_);
295     buffer->meta_->SetData(Tag::REGULAR_TRACK_ID, index);
296     if (inputBufferQueueConsumer_ == nullptr) {
297         MEDIA_LOG_E("inputBufferQueueConsumer_ is null");
298         return;
299     }
300     std::unique_lock<std::mutex> lock(mutex_);
301     if (inputBufferQueueConsumer_->IsBufferInQueue(buffer)) {
302         if (inputBufferQueueConsumer_->ReleaseBuffer(buffer) != Status::OK) {
303             MEDIA_LOG_E("IsBufferInQueue ReleaseBuffer failed. index: %{public}u, bufferid: %{public}" PRIu64
304                 ", pts: %{public}" PRIu64", flag: %{public}u", index, buffer->GetUniqueId(),
305                 buffer->pts_, buffer->flag_);
306         } else {
307             MEDIA_LOG_D("IsBufferInQueue ReleaseBuffer success. index: %{public}u, bufferid: %{public}" PRIu64
308                 ", pts: %{public}" PRIu64", flag: %{public}u", index, buffer->GetUniqueId(),
309                 buffer->pts_, buffer->flag_);
310         }
311     } else {
312         uint32_t size = inputBufferQueueConsumer_->GetQueueSize() + 1;
313         MEDIA_LOG_I("AttachBuffer enter. index: %{public}u,  size: %{public}u , bufferid: %{public}" PRIu64,
314             index, size, buffer->GetUniqueId());
315         inputBufferQueueConsumer_->SetQueueSizeAndAttachBuffer(size, buffer, false);
316         bufferVector_.push_back(buffer);
317     }
318 }
319 
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)320 void VideoDecoderAdapter::OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode)
321 {
322     FALSE_RETURN_MSG(callback_ != nullptr, "OnError callback_ is nullptr");
323     callback_->OnError(errorType, errorCode);
324 }
325 
OnOutputFormatChanged(const MediaAVCodec::Format & format)326 void VideoDecoderAdapter::OnOutputFormatChanged(const MediaAVCodec::Format &format)
327 {
328     FALSE_RETURN_MSG(callback_ != nullptr, "OnOutputFormatChanged callback_ is nullptr");
329     callback_->OnOutputFormatChanged(format);
330 }
331 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)332 void VideoDecoderAdapter::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
333 {
334     AVCodecTrace trace("VideoDecoderAdapter::OnOutputBufferAvailable");
335     if (buffer != nullptr) {
336         MEDIA_LOG_D("OnOutputBufferAvailable start. index: %{public}u, bufferid: %{public}" PRIu64
337             ", pts: %{public}" PRIu64 ", flag: %{public}u", index, buffer->GetUniqueId(), buffer->pts_, buffer->flag_);
338     } else {
339         MEDIA_LOG_D("OnOutputBufferAvailable start. buffer is nullptr, index: %{public}u", index);
340     }
341     FALSE_RETURN_MSG(callback_ != nullptr, "callback_ is nullptr");
342     callback_->OnOutputBufferAvailable(index, buffer);
343 }
344 
GetOutputFormat(Format & format)345 int32_t VideoDecoderAdapter::GetOutputFormat(Format &format)
346 {
347     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, AVCodecServiceErrCode::AVCS_ERR_INVALID_VAL,
348         "GetOutputFormat mediaCodec_ is nullptr");
349     return mediaCodec_->GetOutputFormat(format);
350 }
351 
ReleaseOutputBuffer(uint32_t index,bool render)352 int32_t VideoDecoderAdapter::ReleaseOutputBuffer(uint32_t index, bool render)
353 {
354     AVCodecTrace trace("VideoDecoderAdapter::ReleaseOutputBuffer");
355     MEDIA_LOG_D("VideoDecoderAdapter::ReleaseOutputBuffer");
356     mediaCodec_->ReleaseOutputBuffer(index, render);
357     return 0;
358 }
359 
RenderOutputBufferAtTime(uint32_t index,int64_t renderTimestampNs)360 int32_t VideoDecoderAdapter::RenderOutputBufferAtTime(uint32_t index, int64_t renderTimestampNs)
361 {
362     AVCodecTrace trace("VideoDecoderAdapter::RenderOutputBufferAtTime");
363     MEDIA_LOG_D("VideoDecoderAdapter::RenderOutputBufferAtTime");
364     mediaCodec_->RenderOutputBufferAtTime(index, renderTimestampNs);
365     return 0;
366 }
367 
SetOutputSurface(sptr<Surface> videoSurface)368 int32_t VideoDecoderAdapter::SetOutputSurface(sptr<Surface> videoSurface)
369 {
370     MEDIA_LOG_I("VideoDecoderAdapter::SetOutputSurface");
371     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, AVCodecServiceErrCode::AVCS_ERR_INVALID_VAL, "mediaCodec_ is nullptr");
372     return mediaCodec_->SetOutputSurface(videoSurface);
373 }
374 
SetDecryptConfig(const sptr<DrmStandard::IMediaKeySessionService> & keySession,const bool svpFlag)375 int32_t VideoDecoderAdapter::SetDecryptConfig(const sptr<DrmStandard::IMediaKeySessionService> &keySession,
376     const bool svpFlag)
377 {
378 #ifdef SUPPORT_DRM
379     FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, AVCodecServiceErrCode::AVCS_ERR_INVALID_VAL, "mediaCodec_ is nullptr");
380     FALSE_RETURN_V_MSG(keySession != nullptr, AVCodecServiceErrCode::AVCS_ERR_INVALID_VAL, "mediaCodec_ is nullptr");
381     return mediaCodec_->SetDecryptConfig(keySession, svpFlag);
382 #else
383     return 0;
384 #endif
385 }
386 
SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> & receiver)387 void VideoDecoderAdapter::SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> &receiver)
388 {
389     eventReceiver_ = receiver;
390 }
391 
SetCallingInfo(int32_t appUid,int32_t appPid,const std::string & bundleName,uint64_t instanceId)392 void VideoDecoderAdapter::SetCallingInfo(int32_t appUid, int32_t appPid, const std::string& bundleName,
393     uint64_t instanceId)
394 {
395     appUid_ = appUid;
396     appPid_ = appPid;
397     bundleName_ = bundleName;
398     instanceId_ = instanceId;
399 }
400 
OnDumpInfo(int32_t fd)401 void VideoDecoderAdapter::OnDumpInfo(int32_t fd)
402 {
403     MEDIA_LOG_D("VideoDecoderAdapter::OnDumpInfo called.");
404     std::string dumpString;
405     dumpString += "VideoDecoderAdapter media codec name is:" + mediaCodecName_ + "\n";
406     if (inputBufferQueue_ != nullptr) {
407         dumpString += "VideoDecoderAdapter buffer size is:" + std::to_string(inputBufferQueue_->GetQueueSize()) + "\n";
408     }
409     if (fd < 0) {
410         MEDIA_LOG_E("VideoDecoderAdapter::OnDumpInfo fd is invalid.");
411         return;
412     }
413     int ret = write(fd, dumpString.c_str(), dumpString.size());
414     if (ret < 0) {
415         MEDIA_LOG_E("VideoDecoderAdapter::OnDumpInfo write failed.");
416         return;
417     }
418 }
419 } // namespace Media
420 } // namespace OHOS
421