1 /*
2  * Copyright (C) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "video_sample.h"
17 
18 #include <chrono>
19 #include <unistd.h>
20 
21 #include "native_avformat.h"
22 #include "sync_fence.h"
23 
24 #include "video_processing_callback_impl.h"
25 #include "video_processing_callback_native.h"
26 #include "video_processing_impl.h"
27 
28 using namespace OHOS;
29 using namespace OHOS::Media;
30 using namespace std;
31 using OHOS::Surface;
32 using std::mutex;
33 using OHOS::SurfaceBuffer;
34 using OHOS::IBufferConsumerListener;
35 
36 namespace {
37 constexpr std::chrono::seconds STOP_TIMEOUT(10);
38 const int INPUT_FRAME_NUMBER = 10;
39 const int QUEUE_BUFFER_SIZE = 5;
40 }
41 
42 class TestConsumerListener : public IBufferConsumerListener {
43 public:
TestConsumerListener(VideoSample * sample)44     explicit TestConsumerListener(VideoSample *sample) : sample_(sample) {};
~TestConsumerListener()45     ~TestConsumerListener() {}
OnBufferAvailable()46     void OnBufferAvailable() override
47     {
48         sample_->OnBufferAvailable();
49     }
50 
51 private:
52     VideoSample *sample_ = nullptr;
53 };
54 
OnError(OH_VideoProcessing * videoProcessor,VideoProcessing_ErrorCode error,void * userData)55 static void OnError(OH_VideoProcessing* videoProcessor, VideoProcessing_ErrorCode error, void* userData)
56 {
57     VideoSample* sample = reinterpret_cast<VideoSample*>(userData);
58     sample->UpdateErrorCount();
59 }
60 
OnState(OH_VideoProcessing * videoProcessor,VideoProcessing_State state,void * userData)61 static void OnState(OH_VideoProcessing* videoProcessor, VideoProcessing_State state, void* userData)
62 {
63     VideoSample* sample = reinterpret_cast<VideoSample*>(userData);
64     if (state == VIDEO_PROCESSING_STATE_STOPPED) {
65         sample->NotifyCv();
66     }
67 }
68 
OnNewOutputBufferCall(OH_VideoProcessing * videoProcessor,uint32_t index,void * userData)69 static void OnNewOutputBufferCall(OH_VideoProcessing* videoProcessor, uint32_t index, void* userData)
70 {
71     VideoSample* sample = reinterpret_cast<VideoSample*>(userData);
72     VideoProcessing_ErrorCode ret = OH_VideoProcessing_RenderOutputBuffer(videoProcessor, index);
73     if (ret != VIDEO_PROCESSING_SUCCESS) {
74         sample->UpdateErrorCount();
75     }
76 }
77 
OnNewOutputBufferCallImpl(OH_VideoProcessing * videoProcessor,uint32_t index,void * userData)78 static void OnNewOutputBufferCallImpl(OH_VideoProcessing* videoProcessor, uint32_t index, void* userData)
79 {
80     VideoSample* sample = reinterpret_cast<VideoSample*>(userData);
81     VideoProcessing_ErrorCode ret = videoProcessor->GetObj()->RenderOutputBuffer(index);
82     if (ret != VIDEO_PROCESSING_SUCCESS) {
83         sample->UpdateErrorCount();
84     }
85 }
86 
VideoSample()87 VideoSample::VideoSample()
88 {
89 }
90 
~VideoSample()91 VideoSample::~VideoSample()
92 {
93     if (callback_ || callbackImpl_) {
94         if (!isImpl_) {
95             OH_VideoProcessingCallback_Destroy(callback_);
96             callback_ = nullptr;
97         } else {
98             VideoProcessing_Callback::Destroy(callbackImpl_);
99             callbackImpl_ = nullptr;
100         }
101     }
102     if (rect_) {
103         delete rect_;
104         rect_ = nullptr;
105     }
106     if (!isImpl_) {
107         OH_VideoProcessing_Destroy(videoProcessor_);
108     } else {
109         OH_VideoProcessing::Destroy(videoProcessorImpl_);
110     }
111     if (cs_) {
112         for (; !inputBufferAvilQue_.empty(); inputBufferAvilQue_.pop()) {
113             cs_->ReleaseBuffer(inputBufferAvilQue_.front(), -1);
114         }
115         cs_->UnregisterConsumerListener();
116     }
117     cs_ = nullptr;
118     OH_VideoProcessing_DeinitializeEnvironment();
119 }
120 
InitVideoSample(VideoProcessParam param)121 int32_t VideoSample::InitVideoSample(VideoProcessParam param)
122 {
123     OH_VideoProcessing_InitializeEnvironment();
124     param_ = param;
125     OH_VideoProcessing_Create(&videoProcessor_, VIDEO_PROCESSING_TYPE_DETAIL_ENHANCER);
126     cs_ = Surface::CreateSurfaceAsConsumer();
127     sptr<IBufferConsumerListener> listener = new TestConsumerListener(this);
128     cs_->RegisterConsumerListener(listener);
129     auto p = cs_->GetProducer();
130     sptr<Surface> ps = Surface::CreateSurfaceAsProducer(p);
131     outWindow_ = CreateNativeWindowFromSurface(&ps);
132     cs_->SetQueueSize(QUEUE_BUFFER_SIZE);
133     (void)OH_NativeWindow_NativeWindowHandleOpt(outWindow_, SET_BUFFER_GEOMETRY, param_.outWidth, param_.outHeight);
134     (void)OH_NativeWindow_NativeWindowHandleOpt(outWindow_, SET_USAGE,
135         NATIVEBUFFER_USAGE_CPU_READ | NATIVEBUFFER_USAGE_CPU_WRITE |
136         NATIVEBUFFER_USAGE_MEM_DMA | NATIVEBUFFER_USAGE_HW_RENDER);
137     (void)OH_NativeWindow_NativeWindowHandleOpt(outWindow_, SET_FORMAT, param_.outFmt);
138     OH_VideoProcessing_SetSurface(videoProcessor_, outWindow_);
139     OH_VideoProcessing_GetSurface(videoProcessor_, &inWindow_);
140     SetInputWindowParam();
141     OH_VideoProcessingCallback_Create(&callback_);
142     OH_VideoProcessingCallback_BindOnError(callback_, OnError);
143     OH_VideoProcessingCallback_BindOnState(callback_, OnState);
144     OH_VideoProcessingCallback_BindOnNewOutputBuffer(callback_, OnNewOutputBufferCall);
145     OH_VideoProcessing_RegisterCallback(videoProcessor_, callback_, this);
146     OH_AVFormat* parameter = OH_AVFormat_Create();
147     OH_AVFormat_SetIntValue(parameter, VIDEO_DETAIL_ENHANCER_PARAMETER_KEY_QUALITY_LEVEL, qualityLevel_);
148     OH_VideoProcessing_SetParameter(videoProcessor_, parameter);
149     return VIDEO_PROCESSING_SUCCESS;
150 }
151 
InitVideoSampleImpl(VideoProcessParam param)152 int32_t VideoSample::InitVideoSampleImpl(VideoProcessParam param)
153 {
154     param_ = param;
155     OH_VideoProcessing::Create(&videoProcessorImpl_, VIDEO_PROCESSING_TYPE_DETAIL_ENHANCER);
156     cs_ = Surface::CreateSurfaceAsConsumer();
157     sptr<IBufferConsumerListener> listener = new TestConsumerListener(this);
158     cs_->RegisterConsumerListener(listener);
159     auto p = cs_->GetProducer();
160     sptr<Surface> ps = Surface::CreateSurfaceAsProducer(p);
161     outWindow_ = CreateNativeWindowFromSurface(&ps);
162     cs_->SetQueueSize(QUEUE_BUFFER_SIZE);
163     (void)OH_NativeWindow_NativeWindowHandleOpt(outWindow_, SET_BUFFER_GEOMETRY, param_.outWidth, param_.outHeight);
164     (void)OH_NativeWindow_NativeWindowHandleOpt(outWindow_, SET_USAGE,
165         NATIVEBUFFER_USAGE_CPU_READ | NATIVEBUFFER_USAGE_CPU_WRITE |
166         NATIVEBUFFER_USAGE_MEM_DMA | NATIVEBUFFER_USAGE_HW_RENDER);
167     (void)OH_NativeWindow_NativeWindowHandleOpt(outWindow_, SET_FORMAT, param_.outFmt);
168     videoProcessorImpl_->GetObj()->SetSurface(outWindow_);
169     videoProcessorImpl_->GetObj()->GetSurface(&inWindow_);
170     SetInputWindowParam();
171     VideoProcessing_Callback::Create(&callbackImpl_);
172     callbackImpl_->GetObj()->BindOnError(OnError);
173     callbackImpl_->GetObj()->BindOnState(OnState);
174     callbackImpl_->GetObj()->BindOnNewOutputBuffer(OnNewOutputBufferCallImpl);
175     videoProcessorImpl_->GetObj()->RegisterCallback(callbackImpl_, this);
176     OH_AVFormat* parameter = OH_AVFormat_Create();
177     OH_AVFormat_SetIntValue(parameter, VIDEO_DETAIL_ENHANCER_PARAMETER_KEY_QUALITY_LEVEL, qualityLevel_);
178     videoProcessorImpl_->GetObj()->SetParameter(parameter);
179     return VIDEO_PROCESSING_SUCCESS;
180 }
181 
SetInputWindowParam()182 void VideoSample::SetInputWindowParam()
183 {
184     (void)OH_NativeWindow_NativeWindowHandleOpt(inWindow_, SET_BUFFER_GEOMETRY, param_.inWidth, param_.inHeight);
185     (void)OH_NativeWindow_NativeWindowHandleOpt(inWindow_, SET_USAGE,
186         NATIVEBUFFER_USAGE_CPU_READ | NATIVEBUFFER_USAGE_CPU_WRITE |
187         NATIVEBUFFER_USAGE_MEM_DMA | NATIVEBUFFER_USAGE_HW_RENDER);
188     (void)OH_NativeWindow_NativeWindowHandleOpt(inWindow_, SET_FORMAT, param_.inFmt);
189 
190     rect_ = new Region::Rect();
191     rect_->x = 0;
192     rect_->y = 0;
193     rect_->w = param_.inWidth;
194     rect_->h = param_.inHeight;
195     region_.rects = rect_;
196 }
197 
InputFunc()198 int32_t VideoSample::InputFunc()
199 {
200     for (int i = 0; i < INPUT_FRAME_NUMBER; i++) {
201         int fenceFd = -1;
202         OHNativeWindowBuffer *ohNativeWindowBuffer;
203         OH_NativeWindow_NativeWindowRequestBuffer(inWindow_, &ohNativeWindowBuffer, &fenceFd);
204         if (fenceFd > 0) {
205             close(fenceFd);
206         }
207         OH_NativeBuffer *nativeBuffer = nullptr;
208         OH_NativeBuffer_FromNativeWindowBuffer(ohNativeWindowBuffer, &nativeBuffer);
209         OH_NativeWindow_NativeWindowFlushBuffer(inWindow_, ohNativeWindowBuffer, -1, region_);
210     }
211     return 0;
212 }
213 
StartProcess()214 int32_t VideoSample::StartProcess()
215 {
216     OH_VideoProcessing_Start(videoProcessor_);
217     inputLoop_ = make_unique<thread>(&VideoSample::InputFunc, this);
218     return VIDEO_PROCESSING_SUCCESS;
219 }
220 
StartProcessImpl()221 int32_t VideoSample::StartProcessImpl()
222 {
223     videoProcessorImpl_->GetObj()->Start();
224     inputLoop_ = make_unique<thread>(&VideoSample::InputFunc, this);
225     return VIDEO_PROCESSING_SUCCESS;
226 }
227 
WaitAndStopSample()228 int32_t VideoSample::WaitAndStopSample()
229 {
230     inputLoop_->join();
231     int32_t ret = OH_VideoProcessing_Stop(videoProcessor_);
232     unique_lock<mutex> lock(mutex_);
233     if (cv_.wait_for(lock, STOP_TIMEOUT) == std::cv_status::timeout) {
234         std::cout << "waiting stop state timeout" << std::endl;
235     }
236     return ret;
237 }
238 
WaitAndStopSampleImpl()239 int32_t VideoSample::WaitAndStopSampleImpl()
240 {
241     inputLoop_->join();
242     int32_t ret = videoProcessorImpl_->GetObj()->Stop();
243     unique_lock<mutex> lock(mutex_);
244     if (cv_.wait_for(lock, STOP_TIMEOUT) == std::cv_status::timeout) {
245         std::cout << "waiting stop state timeout" << std::endl;
246     }
247     return ret;
248 }
249 
SetSurfaceOnRunningImpl()250 int32_t VideoSample::SetSurfaceOnRunningImpl()
251 {
252     OH_VideoProcessing* videoProcessing2 = nullptr;
253     OH_VideoProcessing::Create(&videoProcessing2, VIDEO_PROCESSING_TYPE_DETAIL_ENHANCER);
254     OHNativeWindow* window2 = nullptr;
255     videoProcessing2->GetObj()->GetSurface(&window2);
256     int32_t ret = videoProcessorImpl_->GetObj()->SetSurface(window2);
257     return ret;
258 }
259 
OnBufferAvailable()260 void VideoSample::OnBufferAvailable()
261 {
262     unique_lock<mutex> lock(mutexListener_);
263     sptr<SurfaceBuffer> buffer;
264     Rect damage = {};
265     int32_t fence = -1;
266     int64_t timestamp = 0;
267     cs_->AcquireBuffer(buffer, fence, timestamp, damage);
268     inputBufferAvilQue_.push(buffer);
269     lock.unlock();
270 }
271 
UpdateErrorCount()272 void VideoSample::UpdateErrorCount()
273 {
274     errCount_++;
275 }
276 
SetQualityLevel(VideoDetailEnhancer_QualityLevel level)277 void VideoSample::SetQualityLevel(VideoDetailEnhancer_QualityLevel level)
278 {
279     qualityLevel_ = level;
280 }
281 
SetImplLoader(bool isImpl)282 void VideoSample::SetImplLoader(bool isImpl)
283 {
284     isImpl_ = isImpl;
285 }
286 
NotifyCv()287 void VideoSample::NotifyCv()
288 {
289     cv_.notify_all();
290 }