1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "surface_decoder_adapter.h"
17 #include <ctime>
18 #include "avcodec_info.h"
19 #include "avcodec_common.h"
20 #include "codec_server.h"
21 #include "meta/format.h"
22 #include "media_description.h"
23 #include "avcodec_trace.h"
24 
25 namespace {
26 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_ONLY_PRERELEASE, LOG_DOMAIN_SYSTEM_PLAYER,
27                                                 "SurfaceDecoderAdapter" };
28 }
29 
30 namespace OHOS {
31 namespace Media {
32 
33 constexpr int64_t VARIABLE_INCREMENT_INTERVAL = 1;
34 
35 class SurfaceDecoderAdapterCallback : public MediaAVCodec::MediaCodecCallback {
36 public:
SurfaceDecoderAdapterCallback(std::shared_ptr<SurfaceDecoderAdapter> surfaceDecoderAdapter)37     explicit SurfaceDecoderAdapterCallback(std::shared_ptr<SurfaceDecoderAdapter> surfaceDecoderAdapter)
38         : surfaceDecoderAdapter_(std::move(surfaceDecoderAdapter))
39     {
40     }
41 
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)42     void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode) override
43     {
44         if (auto surfaceDecoderAdapter = surfaceDecoderAdapter_.lock()) {
45             surfaceDecoderAdapter->decoderAdapterCallback_->OnError(errorType, errorCode);
46         } else {
47             MEDIA_LOG_I("invalid surfaceEncoderAdapter");
48         }
49     }
50 
OnOutputFormatChanged(const MediaAVCodec::Format & format)51     void OnOutputFormatChanged(const MediaAVCodec::Format &format) override
52     {
53     }
54 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)55     void OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
56     {
57         if (auto surfaceDecoderAdapter = surfaceDecoderAdapter_.lock()) {
58             surfaceDecoderAdapter->OnInputBufferAvailable(index, buffer);
59         } else {
60             MEDIA_LOG_I("invalid surfaceDecoderAdapter");
61         }
62     }
63 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)64     void OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
65     {
66         if (auto surfaceDecoderAdapter = surfaceDecoderAdapter_.lock()) {
67             MEDIA_LOG_D("OnOutputBuffer flag " PUBLIC_LOG_D32, buffer->flag_);
68             surfaceDecoderAdapter->OnOutputBufferAvailable(index, buffer);
69             if (buffer->flag_ == 1) {
70                 int64_t lastBufferPts = surfaceDecoderAdapter->GetLastBufferPts();
71                 int64_t frameNum = surfaceDecoderAdapter->GetFrameNum();
72                 MEDIA_LOG_I("lastBuffer PTS: " PUBLIC_LOG_D64 " frameNum: " PUBLIC_LOG_D64,
73                     lastBufferPts, frameNum);
74                 surfaceDecoderAdapter->decoderAdapterCallback_->OnBufferEos(lastBufferPts, frameNum);
75             }
76         } else {
77             MEDIA_LOG_I("invalid surfaceDecoderAdapter");
78         }
79     }
80 
81 private:
82     std::weak_ptr<SurfaceDecoderAdapter> surfaceDecoderAdapter_;
83 };
84 
85 class AVBufferAvailableListener : public OHOS::Media::IConsumerListener {
86 public:
AVBufferAvailableListener(std::shared_ptr<SurfaceDecoderAdapter> surfaceDecoderAdapter)87     explicit AVBufferAvailableListener(std::shared_ptr<SurfaceDecoderAdapter> surfaceDecoderAdapter)
88         : surfaceDecoderAdapter_(std::move(surfaceDecoderAdapter))
89     {
90     }
91 
OnBufferAvailable()92     void OnBufferAvailable() override
93     {
94         if (auto surfaceDecoderAdapter = surfaceDecoderAdapter_.lock()) {
95             surfaceDecoderAdapter->AcquireAvailableInputBuffer();
96         } else {
97             MEDIA_LOG_I("invalid surfaceDecoderAdapter");
98         }
99     }
100 private:
101     std::weak_ptr<SurfaceDecoderAdapter> surfaceDecoderAdapter_;
102 };
103 
SurfaceDecoderAdapter()104 SurfaceDecoderAdapter::SurfaceDecoderAdapter()
105 {
106     MEDIA_LOG_I("encoder adapter create");
107 }
108 
~SurfaceDecoderAdapter()109 SurfaceDecoderAdapter::~SurfaceDecoderAdapter()
110 {
111     MEDIA_LOG_I("encoder adapter destroy");
112     if (codecServer_) {
113         codecServer_->Release();
114     }
115     codecServer_ = nullptr;
116 }
117 
Init(const std::string & mime)118 Status SurfaceDecoderAdapter::Init(const std::string &mime)
119 {
120     MEDIA_LOG_I("Init mime: " PUBLIC_LOG_S, mime.c_str());
121     codecServer_ = MediaAVCodec::VideoDecoderFactory::CreateByMime(mime);
122     if (!codecServer_) {
123         MEDIA_LOG_I("Create codecServer failed");
124         return Status::ERROR_UNKNOWN;
125     }
126     if (!releaseBufferTask_) {
127         releaseBufferTask_ = std::make_shared<Task>("SurfaceDecoder");
128         releaseBufferTask_->RegisterJob([this] {
129             ReleaseBuffer();
130             return 0;
131         });
132     }
133     return Status::OK;
134 }
135 
Configure(const Format & format)136 Status SurfaceDecoderAdapter::Configure(const Format &format)
137 {
138     MEDIA_LOG_I("Configure");
139     if (!codecServer_) {
140         return Status::ERROR_UNKNOWN;
141     }
142     int32_t ret = codecServer_->Configure(format);
143     return ret == 0 ? Status::OK : Status::ERROR_UNKNOWN;
144 }
145 
GetFrameNum()146 int64_t SurfaceDecoderAdapter::GetFrameNum()
147 {
148     return frameNum_.load();
149 }
150 
GetLastBufferPts()151 int64_t SurfaceDecoderAdapter::GetLastBufferPts()
152 {
153     return lastBufferPts_.load();
154 }
155 
GetInputBufferQueue()156 sptr<OHOS::Media::AVBufferQueueProducer> SurfaceDecoderAdapter::GetInputBufferQueue()
157 {
158     MEDIA_LOG_I("GetInputBufferQueue");
159     if (inputBufferQueue_ != nullptr && inputBufferQueue_-> GetQueueSize() > 0) {
160         MEDIA_LOG_I("InputBufferQueue already create");
161         return inputBufferQueueProducer_;
162     }
163     inputBufferQueue_ = AVBufferQueue::Create(0,
164         MemoryType::UNKNOWN_MEMORY, "inputBufferQueue", true);
165     inputBufferQueueProducer_ = inputBufferQueue_->GetProducer();
166     inputBufferQueueConsumer_ = inputBufferQueue_->GetConsumer();
167     sptr<IConsumerListener> listener = new AVBufferAvailableListener(shared_from_this());
168     inputBufferQueueConsumer_->SetBufferAvailableListener(listener);
169     return inputBufferQueueProducer_;
170 }
171 
SetDecoderAdapterCallback(const std::shared_ptr<DecoderAdapterCallback> & decoderAdapterCallback)172 Status SurfaceDecoderAdapter::SetDecoderAdapterCallback(
173     const std::shared_ptr<DecoderAdapterCallback> &decoderAdapterCallback)
174 {
175     MEDIA_LOG_I("SetDecoderAdapterCallback");
176     std::shared_ptr<MediaAVCodec::MediaCodecCallback> surfaceDecoderAdapterCallback =
177         std::make_shared<SurfaceDecoderAdapterCallback>(shared_from_this());
178     decoderAdapterCallback_ = decoderAdapterCallback;
179     if (!codecServer_) {
180         return Status::ERROR_UNKNOWN;
181     }
182     int32_t ret = codecServer_->SetCallback(surfaceDecoderAdapterCallback);
183     if (ret == 0) {
184         return Status::OK;
185     } else {
186         return Status::ERROR_UNKNOWN;
187     }
188 }
189 
SetOutputSurface(sptr<Surface> surface)190 Status SurfaceDecoderAdapter::SetOutputSurface(sptr<Surface> surface)
191 {
192     MEDIA_LOG_I("SetOutputSurface");
193     if (!codecServer_) {
194         return Status::ERROR_UNKNOWN;
195     }
196     int32_t ret = codecServer_->SetOutputSurface(surface);
197     if (ret == 0) {
198         MEDIA_LOG_I("SetOutputSurface success");
199         return Status::OK;
200     } else {
201         MEDIA_LOG_I("SetOutputSurface fail");
202         return Status::ERROR_UNKNOWN;
203     }
204 }
205 
Start()206 Status SurfaceDecoderAdapter::Start()
207 {
208     MEDIA_LOG_I("Start");
209     if (!codecServer_) {
210         return Status::ERROR_UNKNOWN;
211     }
212     int32_t ret;
213     isThreadExit_ = false;
214     if (releaseBufferTask_) {
215         releaseBufferTask_->Start();
216     }
217     ret = codecServer_->Prepare();
218     if (ret == 0) {
219         MEDIA_LOG_I("Prepare success");
220     } else {
221         MEDIA_LOG_I("Prepare fail");
222         return Status::ERROR_UNKNOWN;
223     }
224     ret = codecServer_->Start();
225     if (ret == 0) {
226         MEDIA_LOG_I("Start success");
227         return Status::OK;
228     } else {
229         MEDIA_LOG_I("Start fail");
230         return Status::ERROR_UNKNOWN;
231     }
232 }
233 
Stop()234 Status SurfaceDecoderAdapter::Stop()
235 {
236     MEDIA_LOG_I("Stop");
237     if (releaseBufferTask_) {
238         isThreadExit_ = true;
239         releaseBufferCondition_.notify_all();
240         releaseBufferTask_->Stop();
241         MEDIA_LOG_I("releaseBufferTask_ Stop");
242     }
243     if (!codecServer_) {
244         return Status::OK;
245     }
246     int32_t ret = codecServer_->Stop();
247     MEDIA_LOG_I("codecServer_ Stop");
248     if (ret == 0) {
249         return Status::OK;
250     } else {
251         return Status::ERROR_UNKNOWN;
252     }
253 }
254 
Pause()255 Status SurfaceDecoderAdapter::Pause()
256 {
257     MEDIA_LOG_I("Pause");
258     return Status::OK;
259 }
260 
Resume()261 Status SurfaceDecoderAdapter::Resume()
262 {
263     MEDIA_LOG_I("Resume");
264     return Status::OK;
265 }
266 
Flush()267 Status SurfaceDecoderAdapter::Flush()
268 {
269     MEDIA_LOG_I("Flush");
270     if (!codecServer_) {
271         return Status::ERROR_UNKNOWN;
272     }
273     int32_t ret = codecServer_->Flush();
274     if (ret == 0) {
275         return Status::OK;
276     } else {
277         return Status::ERROR_UNKNOWN;
278     }
279 }
280 
Release()281 Status SurfaceDecoderAdapter::Release()
282 {
283     MEDIA_LOG_I("Release");
284     if (!codecServer_) {
285         return Status::OK;
286     }
287     int32_t ret = codecServer_->Release();
288     if (ret == 0) {
289         return Status::OK;
290     } else {
291         return Status::ERROR_UNKNOWN;
292     }
293 }
294 
SetParameter(const Format & format)295 Status SurfaceDecoderAdapter::SetParameter(const Format &format)
296 {
297     MEDIA_LOG_I("SetParameter");
298     if (!codecServer_) {
299         return Status::ERROR_UNKNOWN;
300     }
301     int32_t ret = codecServer_->SetParameter(format);
302     if (ret == 0) {
303         return Status::OK;
304     } else {
305         return Status::ERROR_UNKNOWN;
306     }
307 }
308 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)309 void SurfaceDecoderAdapter::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
310 {
311     FALSE_RETURN_MSG(buffer != nullptr && buffer->meta_ != nullptr, "meta_ is nullptr.");
312     MEDIA_LOG_D("OnInputBufferAvailable enter. index: %{public}u, bufferid: %{public}" PRIu64", pts: %{public}" PRIu64
313         ", flag: %{public}u", index, buffer->GetUniqueId(), buffer->pts_, buffer->flag_);
314     buffer->meta_->SetData(Tag::REGULAR_TRACK_ID, index);
315     if (inputBufferQueueConsumer_ == nullptr) {
316         MEDIA_LOG_E("inputBufferQueueConsumer_ is null");
317         return;
318     }
319     if (inputBufferQueueConsumer_->IsBufferInQueue(buffer)) {
320         if (inputBufferQueueConsumer_->ReleaseBuffer(buffer) != Status::OK) {
321             MEDIA_LOG_E("IsBufferInQueue ReleaseBuffer failed. index: %{public}u, bufferid: %{public}" PRIu64
322                 ", pts: %{public}" PRIu64", flag: %{public}u", index, buffer->GetUniqueId(),
323                 buffer->pts_, buffer->flag_);
324         } else {
325             MEDIA_LOG_D("IsBufferInQueue ReleaseBuffer success. index: %{public}u, bufferid: %{public}" PRIu64
326                 ", pts: %{public}" PRIu64", flag: %{public}u", index, buffer->GetUniqueId(),
327                 buffer->pts_, buffer->flag_);
328         }
329     } else {
330         uint32_t size = inputBufferQueueConsumer_->GetQueueSize() + 1;
331         MEDIA_LOG_I("AttachBuffer enter. index: %{public}u,  size: %{public}u , bufferid: %{public}" PRIu64,
332             index, size, buffer->GetUniqueId());
333         inputBufferQueueConsumer_->SetQueueSize(size);
334         inputBufferQueueConsumer_->AttachBuffer(buffer, false);
335     }
336 }
337 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)338 void SurfaceDecoderAdapter::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
339 {
340     {
341         MediaAVCodec::AVCodecTrace trace("SurfaceDecoderAdapter::OnOutputBufferAvailable");
342         std::lock_guard<std::mutex> lock(releaseBufferMutex_);
343         if (buffer->flag_ == 1) {
344             dropIndexs_.push_back(index);
345         } else if (buffer->pts_ > lastBufferPts_.load()) {
346             lastBufferPts_ = buffer->pts_;
347             frameNum_.fetch_add(VARIABLE_INCREMENT_INTERVAL, std::memory_order_relaxed);
348             indexs_.push_back(index);
349         } else {
350             MEDIA_LOG_D("OnOutputBufferAvailable drop index: %{public}u" PRIu32, index);
351             dropIndexs_.push_back(index);
352         }
353     }
354     releaseBufferCondition_.notify_all();
355     MEDIA_LOG_D("OnOutputBufferAvailable end");
356 }
357 
AcquireAvailableInputBuffer()358 void SurfaceDecoderAdapter::AcquireAvailableInputBuffer()
359 {
360     std::shared_ptr<AVBuffer> filledInputBuffer;
361     Status ret = inputBufferQueueConsumer_->AcquireBuffer(filledInputBuffer);
362     if (ret != Status::OK) {
363         MEDIA_LOG_E("AcquireBuffer fail");
364         return;
365     }
366     FALSE_RETURN_MSG(filledInputBuffer->meta_ != nullptr, "filledInputBuffer meta is nullptr.");
367     uint32_t index;
368     FALSE_RETURN_MSG(filledInputBuffer->meta_->GetData(Tag::REGULAR_TRACK_ID, index), "get index failed.");
369     FALSE_RETURN_MSG(codecServer_ != nullptr, "codecServer_ is nullptr.");
370     if (codecServer_->QueueInputBuffer(index) != ERR_OK) {
371         MEDIA_LOG_E("QueueInputBuffer failed, index: %{public}u,  bufferid: %{public}" PRIu64
372             ", pts: %{public}" PRIu64", flag: %{public}u", index, filledInputBuffer->GetUniqueId(),
373             filledInputBuffer->pts_, filledInputBuffer->flag_);
374     } else {
375         MEDIA_LOG_D("QueueInputBuffer success, index: %{public}u,  bufferid: %{public}" PRIu64
376             ", pts: %{public}" PRIu64", flag: %{public}u", index, filledInputBuffer->GetUniqueId(),
377             filledInputBuffer->pts_, filledInputBuffer->flag_);
378     }
379 }
380 
ReleaseBuffer()381 void SurfaceDecoderAdapter::ReleaseBuffer()
382 {
383     MEDIA_LOG_I("ReleaseBuffer");
384     while (!isThreadExit_) {
385         std::vector<uint32_t> indexs;
386         std::vector<uint32_t> dropIndexs;
387         {
388             std::unique_lock<std::mutex> lock(releaseBufferMutex_);
389             releaseBufferCondition_.wait(lock, [this] {
390                 return isThreadExit_ || !indexs_.empty();
391             });
392             indexs = indexs_;
393             indexs_.clear();
394             dropIndexs = dropIndexs_;
395             dropIndexs_.clear();
396         }
397         for (auto &index : indexs) {
398             codecServer_->ReleaseOutputBuffer(index, true);
399         }
400         for (auto &dropIndex : dropIndexs) {
401             codecServer_->ReleaseOutputBuffer(dropIndex, false);
402         }
403     }
404     MEDIA_LOG_I("ReleaseBuffer end");
405 }
406 } // namespace MEDIA
407 } // namespace OHOS
408