1 /*
2  * Copyright (c) 2023-2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef MEDIA_PIPELINE_VIDEO_SINK_FILTER_H
17 #define MEDIA_PIPELINE_VIDEO_SINK_FILTER_H
18 
19 #include <atomic>
20 #include "plugin/plugin_time.h"
21 #include "avcodec_common.h"
22 #include "surface/surface.h"
23 #include "osal/task/condition_variable.h"
24 #include "osal/task/mutex.h"
25 #include "osal/task/task.h"
26 #include "video_sink.h"
27 #include "sink/media_synchronous_sink.h"
28 #include "common/status.h"
29 #include "meta/meta.h"
30 #include "meta/format.h"
31 #include "filter/filter.h"
32 #include "media_sync_manager.h"
33 #include "foundation/multimedia/drm_framework/services/drm_service/ipc/i_keysession_service.h"
34 #include "common/media_core.h"
35 #include "common/seek_callback.h"
36 
37 namespace OHOS {
38 namespace Media {
39 class VideoDecoderAdapter;
40 namespace Pipeline {
41 class DecoderSurfaceFilter : public Filter, public std::enable_shared_from_this<DecoderSurfaceFilter> {
42 public:
43     explicit DecoderSurfaceFilter(const std::string& name, FilterType type);
44     ~DecoderSurfaceFilter() override;
45 
46     void Init(const std::shared_ptr<EventReceiver> &receiver,
47         const std::shared_ptr<FilterCallback> &callback) override;
48     Status Configure(const std::shared_ptr<Meta> &parameter);
49     Status DoInitAfterLink() override;
50     Status DoPrepare() override;
51     Status DoStart() override;
52     Status DoPause() override;
53     Status DoPauseDragging() override;
54     Status DoResume() override;
55     Status DoResumeDragging() override;
56     Status DoStop() override;
57     Status DoFlush() override;
58     Status DoRelease() override;
59     Status DoPreroll() override;
60     Status DoWaitPrerollDone(bool render) override;
61     Status DoSetPlayRange(int64_t start, int64_t end) override;
62     Status DoProcessInputBuffer(int recvArg, bool dropFrame) override;
63     Status DoProcessOutputBuffer(int recvArg, bool dropFrame, bool byIdx, uint32_t idx, int64_t renderTime) override;
64 
65     void SetParameter(const std::shared_ptr<Meta>& parameter) override;
66     void GetParameter(std::shared_ptr<Meta>& parameter) override;
67     void SetInterruptState(bool isInterruptNeeded);
68 
69     Status LinkNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override;
70     Status UpdateNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override;
71     Status UnLinkNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override;
72     void OnLinkedResult(const sptr<AVBufferQueueProducer> &outputBufferQueue, std::shared_ptr<Meta> &meta);
73     void OnUpdatedResult(std::shared_ptr<Meta> &meta);
74 
75     void OnUnlinkedResult(std::shared_ptr<Meta> &meta);
76     FilterType GetFilterType();
77     void DrainOutputBuffer(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer);
78     Status SetVideoSurface(sptr<Surface> videoSurface);
79 
80     Status SetDecryptConfig(const sptr<DrmStandard::IMediaKeySessionService> &keySessionProxy,
81         bool svp);
82 
83     void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode);
84 
85     sptr<AVBufferQueueProducer> GetInputBufferQueue();
86     void SetSyncCenter(std::shared_ptr<MediaSyncManager> syncCenter);
87 
88     void SetSeekTime(int64_t seekTimeUs);
89     void ResetSeekInfo();
90     Status HandleInputBuffer();
91     void OnDumpInfo(int32_t fd);
92 
93     void SetBitrateStart();
94     void OnOutputFormatChanged(const MediaAVCodec::Format &format);
95 
96     void SetCallingInfo(int32_t appUid, int32_t appPid, const std::string& bundleName, uint64_t instanceId);
97 
98     Status GetLagInfo(int32_t& lagTimes, int32_t& maxLagDuration, int32_t& avgLagDuration);
99 
100     Status StartSeekContinous();
101     Status StopSeekContinous();
102     void RegisterVideoFrameReadyCallback(std::shared_ptr<VideoFrameReadyCallback> &callback);
103     void DeregisterVideoFrameReadyCallback();
104     int32_t GetDecRateUpperLimit();
105     void ConsumeVideoFrame(uint32_t index, bool isRender, int64_t renderTimeNs = 0L);
106 
107 protected:
108     Status OnLinked(StreamType inType, const std::shared_ptr<Meta> &meta,
109         const std::shared_ptr<FilterLinkCallback> &callback) override;
110     Status OnUpdated(StreamType inType, const std::shared_ptr<Meta> &meta,
111         const std::shared_ptr<FilterLinkCallback> &callback) override;
112     Status OnUnLinked(StreamType inType, const std::shared_ptr<FilterLinkCallback>& callback) override;
113 
114 private:
115     void RenderLoop();
116     std::string GetCodecName(std::string mimeType);
117     int64_t CalculateNextRender(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer);
118     void ParseDecodeRateLimit();
119     void RenderNextOutput(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer);
120     Status ReleaseOutputBuffer(int index, bool render, const std::shared_ptr<AVBuffer> &outBuffer, int64_t renderTime);
121     bool AcquireNextRenderBuffer(bool byIdx, uint32_t &index, std::shared_ptr<AVBuffer> &outBuffer);
122     bool DrainSeekContinuous(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer);
123     bool DrainPreroll(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer);
124     bool DrainSeekClosest(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer);
125 
126     std::string name_;
127     FilterType filterType_;
128     std::shared_ptr<EventReceiver> eventReceiver_;
129     std::shared_ptr<FilterCallback> filterCallback_;
130     std::shared_ptr<FilterLinkCallback> onLinkedResultCallback_;
131     std::shared_ptr<VideoDecoderAdapter> videoDecoder_;
132     std::shared_ptr<VideoSink> videoSink_;
133     std::string codecMimeType_;
134     std::shared_ptr<Meta> configureParameter_;
135     std::shared_ptr<Meta> meta_;
136 
137     std::shared_ptr<Filter> nextFilter_;
138     Format configFormat_;
139 
140     std::atomic<uint64_t> renderFrameCnt_{0};
141     std::atomic<uint64_t> discardFrameCnt_{0};
142 
143     bool refreshTotalPauseTime_{false};
144     int64_t latestBufferTime_{HST_TIME_NONE};
145     int64_t latestPausedTime_{HST_TIME_NONE};
146     int64_t totalPausedTime_{0};
147     int64_t stopTime_{0};
148     sptr<Surface> videoSurface_;
149     bool isDrmProtected_ = false;
150     sptr<DrmStandard::IMediaKeySessionService> keySessionServiceProxy_;
151     bool svpFlag_ = false;
152     std::atomic<bool> isPaused_{false};
153     std::atomic<bool> isSeek_{false};
154     int64_t seekTimeUs_{0};
155     std::list<std::pair<int, std::shared_ptr<AVBuffer>>> outputBuffers_;
156     std::mutex mutex_;
157     std::unique_ptr<std::thread> readThread_ = nullptr;
158     std::atomic<bool> isThreadExit_ = true;
159     std::condition_variable condBufferAvailable_;
160 
161     std::atomic<int32_t> bitrateChange_{0};
162     int32_t surfaceWidth_{0};
163     int32_t surfaceHeight_{0};
164     std::atomic<bool> isRenderStarted_{false};
165     std::atomic<bool> isInterruptNeeded_{false};
166     Mutex formatChangeMutex_{};
167     int32_t rateUpperLimit_{0};
168 
169     std::mutex prerollMutex_ {};
170     std::atomic<bool> inPreroll_ {false};
171     std::condition_variable prerollDoneCond_ {};
172     std::atomic<bool> prerollDone_ {true};
173     std::atomic<bool> eosNext_ {false};
174     bool isFirstFrameAfterResume_ {true};
175 
176     int32_t appUid_ = -1;
177     int32_t appPid_ = -1;
178     std::string bundleName_;
179     uint64_t instanceId_ = 0;
180     int64_t playRangeStartTime_ = -1;
181     int64_t playRangeEndTime_ = -1;
182 
183     std::shared_ptr<VideoFrameReadyCallback> videoFrameReadyCallback_;
184     bool isInSeekContinous_{false};
185     std::unordered_map<uint32_t, std::shared_ptr<AVBuffer>> outputBufferMap_;
186     std::mutex draggingMutex_ {};
187 };
188 } // namespace Pipeline
189 } // namespace Media
190 } // namespace OHOS
191 #endif // MEDIA_PIPELINE_VIDEO_SINK_FILTER_H
192