1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef POST_PROCESSING_H
17 #define POST_PROCESSING_H
18 
19 #include <array>
20 #include <memory>
21 #include <string>
22 #include <tuple>
23 #include <type_traits>
24 #include "surface.h"
25 #include "avcodec_errors.h"
26 #include "avcodec_trace.h"
27 #include "codecbase.h"
28 #include "meta/format.h"
29 #include "media_description.h"
30 #include "controller.h"
31 #include "dynamic_controller.h"
32 #include "state_machine.h"
33 #include "utils.h"
34 #include "post_processing_callback.h"
35 
36 namespace OHOS {
37 namespace MediaAVCodec {
38 namespace PostProcessing {
39 
40 template<typename T, typename = IsDerivedController<T>>
41 class PostProcessing {
42 public:
Create(const std::shared_ptr<CodecBase> codec,const Format & format,int32_t & ret)43     static std::unique_ptr<PostProcessing<T>> Create(const std::shared_ptr<CodecBase> codec,
44         const Format& format, int32_t& ret)
45     {
46         AVCODEC_SYNC_TRACE;
47         auto p = std::make_unique<PostProcessing<T>>(codec);
48         if (!p) {
49             AVCODEC_LOGE("Create post processing failed");
50             ret = AVCS_ERR_NO_MEMORY;
51             return nullptr;
52         }
53         ret = p->Init(format);
54         if (ret != AVCS_ERR_OK) {
55             return nullptr;
56         }
57         return p;
58     }
59 
PostProcessing(std::shared_ptr<CodecBase> codec)60     explicit PostProcessing(std::shared_ptr<CodecBase> codec) : codec_(codec) {}
61 
~PostProcessing()62     ~PostProcessing()
63     {
64         callbackUserData_ = nullptr;
65     }
66 
SetCallback(const Callback & callback,void * userData)67     int32_t SetCallback(const Callback& callback, void* userData)
68     {
69         AVCODEC_SYNC_TRACE;
70         callback_ = callback;
71         callbackUserData_ = userData;
72         return AVCS_ERR_OK;
73     }
74 
SetOutputSurface(sptr<Surface> surface)75     int32_t SetOutputSurface(sptr<Surface> surface)
76     {
77         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null");
78         AVCODEC_SYNC_TRACE;
79         switch (state_.Get()) {
80             case State::CONFIGURED:
81                 {
82                     config_.outputSurface = surface;
83                     return AVCS_ERR_OK;
84                 }
85             case State::PREPARED:
86                 [[fallthrough]];
87             case State::RUNNING:
88                 [[fallthrough]];
89             case State::FLUSHED:
90                 [[fallthrough]];
91             case State::STOPPED:
92                 {
93                     int32_t ret = controller_->SetOutputSurface(surface);
94                     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Set output surface failed");
95                     config_.outputSurface = surface;
96                     return ret;
97                 }
98             default:
99                 {
100                     AVCODEC_LOGE("Invalid post processing state: %{public}s", state_.Name());
101                     return AVCS_ERR_UNKNOWN;
102                 }
103         }
104     }
105 
Prepare()106     int32_t Prepare()
107     {
108         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null");
109         CHECK_AND_RETURN_RET_LOG(state_.Get() == State::CONFIGURED, AVCS_ERR_INVALID_STATE,
110             "Invalid post processing state: %{public}s", state_.Name());
111         CHECK_AND_RETURN_RET_LOG(config_.outputSurface != nullptr, AVCS_ERR_INVALID_OPERATION,
112             "Output surface is not set");
113 
114         AVCODEC_SYNC_TRACE;
115 
116         int32_t ret = controller_->Create();
117         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed");
118 
119         SetOutputSurfaceTransform();
120         ret = controller_->SetOutputSurface(config_.outputSurface);
121         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed");
122 
123         ret = SetDecoderInputSurface();
124         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed");
125 
126         ret = controller_->SetCallback(static_cast<void*>(&callback_), callbackUserData_);
127         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed");
128 
129         ret = ConfigureController();
130         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed");
131 
132         ret = controller_->Prepare();
133         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed");
134 
135         state_.Set(State::PREPARED);
136         return AVCS_ERR_OK;
137     }
138 
Start()139     int32_t Start()
140     {
141         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null");
142         CHECK_AND_RETURN_RET_LOG(state_.Get() == State::PREPARED || state_.Get() == State::STOPPED ||
143                                  state_.Get() == State::FLUSHED,
144                                  AVCS_ERR_INVALID_OPERATION, "Post processing is not prepared");
145         AVCODEC_SYNC_TRACE;
146         int32_t ret = controller_->Start();
147         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Start failed");
148         state_.Set(State::RUNNING);
149         return AVCS_ERR_OK;
150     }
151 
Stop()152     int32_t Stop()
153     {
154         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null");
155         CHECK_AND_RETURN_RET_LOG(state_.Get() == State::RUNNING || state_.Get() == State::FLUSHED,
156                                  AVCS_ERR_INVALID_STATE, "Invalid post processing state: %{public}s", state_.Name());
157         AVCODEC_SYNC_TRACE;
158         int32_t ret = controller_->Stop();
159         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Start failed");
160         state_.Set(State::STOPPED);
161         return AVCS_ERR_OK;
162     }
163 
Flush()164     int32_t Flush()
165     {
166         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null");
167         CHECK_AND_RETURN_RET_LOG(state_.Get() == State::RUNNING, AVCS_ERR_INVALID_STATE,
168             "Invalid post processing state: %{public}s", state_.Name());
169         AVCODEC_SYNC_TRACE;
170         int32_t ret = controller_->Flush();
171         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Flush failed");
172         state_.Set(State::FLUSHED);
173         return AVCS_ERR_OK;
174     }
175 
Reset()176     int32_t Reset()
177     {
178         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null");
179         AVCODEC_SYNC_TRACE;
180         int32_t ret = controller_->Reset();
181         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Reset failed");
182         codec_.reset();
183         state_.Set(State::DISABLED);
184         return AVCS_ERR_OK;
185     }
186 
Release()187     int32_t Release()
188     {
189         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null");
190         AVCODEC_SYNC_TRACE;
191         config_.inputSurface = nullptr;
192         config_.outputSurface = nullptr;
193         controller_->Release();
194         controller_->Destroy();
195         controller_->UnloadInterfaces();
196         codec_.reset();
197         state_.Set(State::DISABLED);
198         return AVCS_ERR_OK;
199     }
200 
ReleaseOutputBuffer(uint32_t index,bool render)201     int32_t ReleaseOutputBuffer(uint32_t index, bool render)
202     {
203         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null");
204         CHECK_AND_RETURN_RET_LOG(state_.Get() == State::RUNNING, AVCS_ERR_INVALID_STATE,
205             "Invalid post processing state: %{public}s", state_.Name());
206         AVCODEC_SYNC_TRACE;
207         int32_t ret = controller_->ReleaseOutputBuffer(index, render);
208         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "ReleaseOutputBuffer failed");
209         return AVCS_ERR_OK;
210     }
211 
GetOutputFormat(Format & format)212     void GetOutputFormat(Format& format)
213     {
214         CHECK_AND_RETURN_LOG(controller_, "Post processing controller is null");
215         AVCODEC_SYNC_TRACE;
216         int32_t ret = controller_->GetOutputFormat(format);
217         CHECK_AND_RETURN_LOG(ret == AVCS_ERR_OK, "GetOutputFormat failed");
218         return;
219     }
220 private:
Init(const Format & format)221     int32_t Init(const Format& format)
222     {
223         controller_ = std::make_unique<T>();
224         CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_NO_MEMORY, "Create post processing controller failed");
225         CHECK_AND_RETURN_RET_LOG(controller_->LoadInterfaces(), AVCS_ERR_VIDEO_UNSUPPORT_COLOR_SPACE_CONVERSION,
226                                  "Initialize interfaces failed.");
227 
228         CreateConfiguration(format);
229 
230         CapabilityInfo output{
231             .colorSpaceType = config_.outputColorSpaceType,
232             .metadataType = config_.outputMetadataType,
233             .pixelFormat = config_.outputPixelFormat
234         };
235 
236         constexpr int32_t hdrVividVideoColorSpaceTypeList[]{
237             0x440504, // BT2020 HLG Limit
238             0x440404 // BT2020 PQ Limit
239         };
240         constexpr int32_t hdrVividVideoMetadataType{3}; // HDR Vivid Video
241         constexpr int32_t hdrVividVideoPixelFormatList[]{35, 36};
242 
243         CapabilityInfo input;
244         input.metadataType = hdrVividVideoMetadataType;
245         bool supported{false};
246         for (auto colorSpaceType : hdrVividVideoColorSpaceTypeList) {
247             for (auto pixelFormat : hdrVividVideoPixelFormatList) {
248                 input.colorSpaceType = colorSpaceType,
249                 input.pixelFormat = pixelFormat,
250                 supported |= controller_->IsColorSpaceConversionSupported(input, output);
251             }
252         }
253         CHECK_AND_RETURN_RET_LOG(supported, AVCS_ERR_UNSUPPORT, "No capability found");
254 
255         state_.Set(State::CONFIGURED);
256         return AVCS_ERR_OK;
257     }
258 
CreateConfiguration(const Format & format)259     void CreateConfiguration(const Format& format)
260     {
261         format_ = format;
262         constexpr int32_t colorSpaceTypeBt709Limited{0x410101}; // OH_COLORSPACE_BT709_LIMIT
263         constexpr int32_t pixelFormatNV12{24}; // NATIVEBUFFER_PIXEL_FMT_YCBCR_420_SP
264         constexpr int32_t pixelFormatNV21{25}; // NATIVEBUFFER_PIXEL_FMT_YCRCB_420_SP
265 
266         // the field is checked before
267         int32_t width;
268         (void)format.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width);
269         int32_t height;
270         (void)format.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height);
271 
272         int32_t pixelFormat;
273         if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, pixelFormat)) {
274             pixelFormat = static_cast<int32_t>(VideoPixelFormat::NV12);
275             (void)format_.PutIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, pixelFormat);
276         }
277         int32_t rotation;
278         if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_ROTATION_ANGLE, rotation)) {
279             rotation = 0; // rotation 0
280         }
281         int32_t scalingMode;
282         if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_SCALE_TYPE, scalingMode)) {
283             scalingMode = 0; // No scaling
284         }
285         switch (pixelFormat) {
286             case static_cast<int32_t>(VideoPixelFormat::NV12):
287                 pixelFormat = pixelFormatNV12;
288                 break;
289             case static_cast<int32_t>(VideoPixelFormat::NV21):
290                 pixelFormat = pixelFormatNV21;
291                 break;
292             default:
293                 AVCODEC_LOGE("Unsupported pixel format %{public}d", pixelFormat);
294         }
295 
296         config_.width = width;
297         config_.height = height;
298         config_.outputColorSpaceType = colorSpaceTypeBt709Limited;
299         config_.outputMetadataType = 0; // see OH_COLORSPACE_NONE
300         config_.outputPixelFormat = pixelFormat;
301         config_.rotation = rotation;
302         config_.scalingMode = scalingMode;
303     }
304 
SetDecoderInputSurface()305     int32_t SetDecoderInputSurface()
306     {
307         CHECK_AND_RETURN_RET_LOG(codec_ != nullptr, AVCS_ERR_UNKNOWN, "Decoder is not found");
308         sptr<Surface> surface = nullptr;
309         int32_t ret = controller_->CreateInputSurface(surface);
310         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK && surface != nullptr, ret, "Create input surface failed");
311         GSError gsRet = surface->SetSurfaceSourceType(OHSurfaceSource::OH_SURFACE_SOURCE_VIDEO);
312         EXPECT_AND_LOGW(gsRet != GSERROR_OK, "Set surface source type failed, %{public}s", GSErrorStr(gsRet).c_str());
313         ret = codec_->SetOutputSurface(surface);
314         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Set output surface of decoder failed");
315         config_.inputSurface = surface;
316         return ret;
317     }
318 
SetOutputSurfaceTransform()319     void SetOutputSurfaceTransform()
320     {
321         CHECK_AND_RETURN_LOG(config_.outputSurface != nullptr, "Output surface is null");
322 
323         GSError err{GSERROR_OK};
324         switch (config_.rotation) {
325             case VIDEO_ROTATION_0:
326                 err = config_.outputSurface->SetTransform(GRAPHIC_ROTATE_NONE);
327                 break;
328             case VIDEO_ROTATION_90:
329                 err = config_.outputSurface->SetTransform(GRAPHIC_ROTATE_270);
330                 break;
331             case VIDEO_ROTATION_180:
332                 err = config_.outputSurface->SetTransform(GRAPHIC_ROTATE_180);
333                 break;
334             case VIDEO_ROTATION_270:
335                 err = config_.outputSurface->SetTransform(GRAPHIC_ROTATE_90);
336                 break;
337             default:
338                 break;
339         }
340         if (err != GSERROR_OK) {
341             AVCODEC_LOGE("Set transform failed");
342         }
343 
344         switch (config_.scalingMode) {
345             case SCALING_MODE_SCALE_TO_WINDOW:
346                 [[fallthrough]];
347             case SCALING_MODE_SCALE_CROP:
348                 err = config_.outputSurface->SetScalingMode(static_cast<ScalingMode>(config_.scalingMode));
349                 if (err != GSERROR_OK) {
350                     AVCODEC_LOGE("Set transform failed");
351                 }
352                 break;
353             default:
354                 break;
355         }
356     }
357 
ConfigureController()358     int32_t ConfigureController()
359     {
360         constexpr std::string_view keyPrimaries{"colorspace_primaries"};
361         constexpr std::string_view keyTransFunc{"colorspace_trans_func"};
362         constexpr std::string_view keyMatrix{"colorspace_matrix"};
363         constexpr std::string_view keyRange{"colorspace_range"};
364         constexpr std::string_view keyMetadataType{"hdr_metadata_type"};
365         constexpr std::string_view keyRenderIntent{"render_intent"};
366         constexpr std::string_view keyPixelFormat{"pixel_format"};
367         constexpr int32_t primaries{1};
368         constexpr int32_t transFunc{1};
369         constexpr int32_t matrix{1};
370         constexpr int32_t range{2};
371         constexpr int32_t metadataType{0};
372         constexpr int32_t renderIntent{2};
373         Format format(format_);
374         format.PutIntValue(keyPrimaries, primaries);
375         format.PutIntValue(keyTransFunc, transFunc);
376         format.PutIntValue(keyMatrix, matrix);
377         format.PutIntValue(keyRange, range);
378         format.PutIntValue(keyMetadataType, metadataType);
379         format.PutIntValue(keyRenderIntent, renderIntent);
380         format.PutIntValue(keyPixelFormat, config_.outputPixelFormat);
381         return controller_->Configure(format);
382     }
383 
384     struct Configuration {
385         int32_t width{0};
386         int32_t height{0};
387         int32_t inputColorSpaceType{0};
388         int32_t inputMetadataType{0};
389         int32_t inputPixelFormat{0};
390         sptr<Surface> inputSurface{nullptr};
391         int32_t outputColorSpaceType{0};
392         int32_t outputMetadataType{0};
393         int32_t outputPixelFormat{0};
394         sptr<Surface> outputSurface{nullptr};
395         int32_t rotation{0};
396         int32_t scalingMode{0};
397     };
398 
399     StateMachine state_;
400     Configuration config_;
401     Format format_;
402     std::unique_ptr<Controller<T>> controller_{nullptr};
403     std::shared_ptr<CodecBase> codec_{nullptr};
404     static constexpr HiviewDFX::HiLogLabel LABEL{LogLabel("PostProcessing")};
405     Callback callback_;
406     void* callbackUserData_{nullptr};
407 };
408 
409 using DynamicPostProcessing = PostProcessing<DynamicController>;
410 
411 } // namespace PostProcessing
412 } // namespace OHOS
413 } // namespace MediaAVCodec
414 
415 #endif // POST_PROCESSING_H