1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "image_processing_native.h"
17 
18 #include <cinttypes>
19 
20 #include "common/native_mfmagic.h"
21 #include "pixelmap_native_impl.h"
22 
23 #include "vpe_log.h"
24 
25 using namespace OHOS;
26 using namespace OHOS::Media;
27 using namespace OHOS::Media::VideoProcessingEngine;
28 
29 namespace {
30 const std::unordered_map<AlgoErrorCode, ImageProcessing_ErrorCode> ERROR_MAP = {
31     { ALGO_SUCCESS,                         IMAGE_PROCESSING_SUCCESS },
32     { ALGO_ERROR_INVALID_PARAMETER,         IMAGE_PROCESSING_ERROR_INVALID_PARAMETER },
33     { ALGO_ERROR_UNKNOWN,                   IMAGE_PROCESSING_ERROR_UNKNOWN },
34     { ALGO_ERROR_INITIALIZE_FAILED,         IMAGE_PROCESSING_ERROR_INITIALIZE_FAILED },
35     { ALGO_ERROR_CREATE_FAILED,             IMAGE_PROCESSING_ERROR_CREATE_FAILED },
36     { ALGO_ERROR_PROCESS_FAILED,            IMAGE_PROCESSING_ERROR_PROCESS_FAILED },
37     { ALGO_ERROR_UNSUPPORTED_PROCESSING,    IMAGE_PROCESSING_ERROR_UNSUPPORTED_PROCESSING },
38     { ALGO_ERROR_OPERATION_NOT_PERMITTED,   IMAGE_PROCESSING_ERROR_OPERATION_NOT_PERMITTED },
39     { ALGO_ERROR_NO_MEMORY,                 IMAGE_PROCESSING_ERROR_NO_MEMORY },
40     { ALGO_ERROR_INVALID_INSTANCE,          IMAGE_PROCESSING_ERROR_INVALID_INSTANCE },
41     { ALGO_ERROR_INVALID_VALUE,             IMAGE_PROCESSING_ERROR_INVALID_VALUE },
42 };
43 
44 const std::unordered_map<OHOS::Media::PixelFormat, GraphicPixelFormat> FORMAT_MAP = {
45     { OHOS::Media::PixelFormat::RGBA_8888, GraphicPixelFormat::GRAPHIC_PIXEL_FMT_RGBA_8888 },
46     { OHOS::Media::PixelFormat::BGRA_8888, GraphicPixelFormat::GRAPHIC_PIXEL_FMT_BGRA_8888 },
47 };
48 
GetBytesPerPixel(OHOS::Media::PixelFormat format)49 float GetBytesPerPixel(OHOS::Media::PixelFormat format)
50 {
51     float bytes;
52     switch (format) {
53         case OHOS::Media::PixelFormat::RGBA_8888:
54         case OHOS::Media::PixelFormat::BGRA_8888:
55             bytes = 4.0f; // 4.0 size bytes
56             break;
57         default:
58             bytes = 3.0f; // 3.0 size bytes
59             break;
60     }
61     return bytes;
62 }
63 }
64 
Initialize()65 ImageProcessing_ErrorCode ImageProcessingNative::Initialize()
66 {
67     std::lock_guard<std::mutex> lock(lock_);
68     if (isInitialized_) {
69         VPE_LOGE("Already initialize!");
70         return IMAGE_PROCESSING_ERROR_OPERATION_NOT_PERMITTED;
71     }
72     auto errorCode = InitializeInner();
73     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
74         return errorCode;
75     }
76     isInitialized_ = true;
77     return IMAGE_PROCESSING_SUCCESS;
78 }
79 
Deinitialize()80 ImageProcessing_ErrorCode ImageProcessingNative::Deinitialize()
81 {
82     std::lock_guard<std::mutex> lock(lock_);
83     if (!isInitialized_) {
84         VPE_LOGE("Already deinitialize!");
85         return IMAGE_PROCESSING_ERROR_OPERATION_NOT_PERMITTED;
86     }
87     auto errorCode = DeinitializeInner();
88     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
89         return errorCode;
90     }
91     isInitialized_ = false;
92     return IMAGE_PROCESSING_SUCCESS;
93 }
94 
SetParameter(const OH_AVFormat * parameter)95 ImageProcessing_ErrorCode ImageProcessingNative::SetParameter(const OH_AVFormat* parameter)
96 {
97     if (parameter == nullptr) {
98         VPE_LOGE("parameter is null!");
99         return IMAGE_PROCESSING_ERROR_INVALID_PARAMETER;
100     }
101     return SetParameter(parameter->format_);
102 }
103 
GetParameter(OH_AVFormat * parameter)104 ImageProcessing_ErrorCode ImageProcessingNative::GetParameter(OH_AVFormat* parameter)
105 {
106     if (parameter == nullptr) {
107         VPE_LOGE("parameter is null!");
108         return IMAGE_PROCESSING_ERROR_INVALID_PARAMETER;
109     }
110     return GetParameter(parameter->format_);
111 }
112 
ConvertColorSpace(OH_PixelmapNative * sourceImage,OH_PixelmapNative * destinationImage)113 ImageProcessing_ErrorCode ImageProcessingNative::ConvertColorSpace(OH_PixelmapNative* sourceImage,
114     OH_PixelmapNative* destinationImage)
115 {
116     if (sourceImage == nullptr || sourceImage->GetInnerPixelmap() == nullptr ||
117         destinationImage == nullptr || destinationImage->GetInnerPixelmap() == nullptr) {
118         VPE_LOGE("sourceImage or destinationImage is null or empty!");
119         return IMAGE_PROCESSING_ERROR_INVALID_PARAMETER;
120     }
121     sptr<SurfaceBuffer> srcBuffer = nullptr;
122     auto errorCode = GetSurfaceBufferFromPixelMap(sourceImage->GetInnerPixelmap(), srcBuffer);
123     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
124         return errorCode;
125     }
126     sptr<SurfaceBuffer> dstBuffer = nullptr;
127     std::shared_ptr<PixelMap> dstImage = destinationImage->GetInnerPixelmap();
128     errorCode = GetSurfaceBufferFromPixelMapNoCopy(dstImage, dstBuffer);
129     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
130         return errorCode;
131     }
132     errorCode = ConvertColorSpace(srcBuffer, dstBuffer);
133     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
134         return errorCode;
135     }
136     return SetSurfaceBufferToPixelMap(dstBuffer, dstImage);
137 }
138 
Compose(OH_PixelmapNative * sourceImage,OH_PixelmapNative * sourceGainmap,OH_PixelmapNative * destinationImage)139 ImageProcessing_ErrorCode ImageProcessingNative::Compose(OH_PixelmapNative* sourceImage,
140     OH_PixelmapNative* sourceGainmap, OH_PixelmapNative* destinationImage)
141 {
142     if (sourceImage == nullptr || sourceImage->GetInnerPixelmap() == nullptr ||
143         sourceGainmap == nullptr || sourceGainmap->GetInnerPixelmap() == nullptr ||
144         destinationImage == nullptr || destinationImage->GetInnerPixelmap() == nullptr) {
145             VPE_LOGE("sourceImage, sourceGainmap or destinationImage is null or empty!");
146             return IMAGE_PROCESSING_ERROR_INVALID_PARAMETER;
147     }
148     sptr<SurfaceBuffer> srcBuffer = nullptr;
149     auto errorCode = GetSurfaceBufferFromPixelMap(sourceImage->GetInnerPixelmap(), srcBuffer);
150     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
151         return errorCode;
152     }
153     sptr<SurfaceBuffer> srcGainmap = nullptr;
154     errorCode = GetSurfaceBufferFromPixelMap(sourceGainmap->GetInnerPixelmap(), srcGainmap);
155     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
156         return errorCode;
157     }
158     sptr<SurfaceBuffer> dstBuffer = nullptr;
159     std::shared_ptr<PixelMap> dstImage = destinationImage->GetInnerPixelmap();
160     errorCode = GetSurfaceBufferFromPixelMapNoCopy(dstImage, dstBuffer);
161     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
162         return errorCode;
163     }
164     errorCode = Compose(srcBuffer, srcGainmap, dstBuffer);
165     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
166         return errorCode;
167     }
168     return SetSurfaceBufferToPixelMap(dstBuffer, dstImage);
169 }
170 
Decompose(OH_PixelmapNative * sourceImage,OH_PixelmapNative * destinationImage,OH_PixelmapNative * destinationGainmap)171 ImageProcessing_ErrorCode ImageProcessingNative::Decompose(OH_PixelmapNative* sourceImage,
172     OH_PixelmapNative* destinationImage, OH_PixelmapNative* destinationGainmap)
173 {
174     if (sourceImage == nullptr || sourceImage->GetInnerPixelmap() == nullptr ||
175         destinationImage == nullptr || destinationImage->GetInnerPixelmap() == nullptr ||
176         destinationGainmap == nullptr || destinationGainmap->GetInnerPixelmap() == nullptr) {
177             VPE_LOGE("sourceImage, destinationImage or destinationGainmap is null or empty!");
178             return IMAGE_PROCESSING_ERROR_INVALID_PARAMETER;
179     }
180     sptr<SurfaceBuffer> srcBuffer = nullptr;
181     auto errorCode = GetSurfaceBufferFromPixelMap(sourceImage->GetInnerPixelmap(), srcBuffer);
182     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
183         return errorCode;
184     }
185     sptr<SurfaceBuffer> dstBuffer = nullptr;
186     std::shared_ptr<PixelMap> dstImage = destinationImage->GetInnerPixelmap();
187     errorCode = GetSurfaceBufferFromPixelMapNoCopy(dstImage, dstBuffer);
188     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
189         return errorCode;
190     }
191     sptr<SurfaceBuffer> dstGainmap = nullptr;
192     std::shared_ptr<PixelMap> dstImageGainmap = destinationGainmap->GetInnerPixelmap();
193     errorCode = GetSurfaceBufferFromPixelMapNoCopy(dstImageGainmap, dstGainmap);
194     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
195         return errorCode;
196     }
197     errorCode = Decompose(srcBuffer, dstBuffer, dstGainmap);
198     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
199         return errorCode;
200     }
201     errorCode = SetSurfaceBufferToPixelMap(dstBuffer, dstImage);
202     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
203         return errorCode;
204     }
205     return SetSurfaceBufferToPixelMap(dstGainmap, dstImageGainmap);
206 }
207 
GenerateMetadata(OH_PixelmapNative * sourceImage)208 ImageProcessing_ErrorCode ImageProcessingNative::GenerateMetadata(OH_PixelmapNative* sourceImage)
209 {
210     if (sourceImage == nullptr || sourceImage->GetInnerPixelmap() == nullptr) {
211             VPE_LOGE("sourceImage is null or empty!");
212             return IMAGE_PROCESSING_ERROR_INVALID_PARAMETER;
213     }
214     sptr<SurfaceBuffer> srcBuffer = nullptr;
215     auto errorCode = GetSurfaceBufferFromPixelMap(sourceImage->GetInnerPixelmap(), srcBuffer);
216     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
217         return errorCode;
218     }
219     return GenerateMetadata(srcBuffer); // Need to copy metadata to PixelMap
220 }
221 
EnhanceDetail(OH_PixelmapNative * sourceImage,OH_PixelmapNative * destinationImage)222 ImageProcessing_ErrorCode ImageProcessingNative::EnhanceDetail(OH_PixelmapNative* sourceImage,
223     OH_PixelmapNative* destinationImage)
224 {
225     if (sourceImage == nullptr || sourceImage->GetInnerPixelmap() == nullptr ||
226         destinationImage == nullptr || destinationImage->GetInnerPixelmap() == nullptr) {
227         VPE_LOGE("sourceImage or destinationImage is null or empty!");
228         return IMAGE_PROCESSING_ERROR_INVALID_PARAMETER;
229     }
230     sptr<SurfaceBuffer> srcBuffer = nullptr;
231     auto errorCode = GetSurfaceBufferFromPixelMap(sourceImage->GetInnerPixelmap(), srcBuffer);
232     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
233         return errorCode;
234     }
235     sptr<SurfaceBuffer> dstBuffer = nullptr;
236     std::shared_ptr<PixelMap> dstImage = destinationImage->GetInnerPixelmap();
237     errorCode = GetSurfaceBufferFromPixelMapNoCopy(dstImage, dstBuffer);
238     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
239         return errorCode;
240     }
241     errorCode = EnhanceDetail(srcBuffer, dstBuffer);
242     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
243         return errorCode;
244     }
245     return SetSurfaceBufferToPixelMap(dstBuffer, dstImage);
246 }
247 
AlgoErrorToNdk(AlgoErrorCode errorCode)248 ImageProcessing_ErrorCode ImageProcessingNative::AlgoErrorToNdk(AlgoErrorCode errorCode)
249 {
250     auto it = ERROR_MAP.find(errorCode);
251     if (it == ERROR_MAP.end()) {
252         VPE_LOGE("Invalid error code:%{public}d", errorCode);
253         return IMAGE_PROCESSING_ERROR_UNKNOWN;
254     }
255     return it->second;
256 }
257 
GetSurfaceBufferFromPixelMap(const std::shared_ptr<PixelMap> & pixelMap,sptr<SurfaceBuffer> & surfaceBuffer)258 ImageProcessing_ErrorCode ImageProcessingNative::GetSurfaceBufferFromPixelMap(const std::shared_ptr<PixelMap>& pixelMap,
259     sptr<SurfaceBuffer>& surfaceBuffer)
260 {
261     if (pixelMap->GetAllocatorType() == AllocatorType::DMA_ALLOC) {
262         surfaceBuffer = reinterpret_cast<SurfaceBuffer*>(pixelMap->GetFd());
263         return IMAGE_PROCESSING_SUCCESS;
264     }
265     auto errorCode = CreateSurfaceBufferFromPixelMap(pixelMap, surfaceBuffer);
266     if (errorCode != IMAGE_PROCESSING_SUCCESS) {
267         return errorCode;
268     }
269     return CopyPixelMapToSurfaceBuffer(pixelMap, surfaceBuffer);
270 }
271 
GetSurfaceBufferFromPixelMapNoCopy(const std::shared_ptr<PixelMap> & pixelMap,sptr<SurfaceBuffer> & surfaceBuffer)272 ImageProcessing_ErrorCode ImageProcessingNative::GetSurfaceBufferFromPixelMapNoCopy(
273     const std::shared_ptr<PixelMap>& pixelMap, sptr<SurfaceBuffer>& surfaceBuffer)
274 {
275     if (pixelMap->GetAllocatorType() == AllocatorType::DMA_ALLOC) {
276         surfaceBuffer = reinterpret_cast<SurfaceBuffer*>(pixelMap->GetFd());
277         return IMAGE_PROCESSING_SUCCESS;
278     }
279     return CreateSurfaceBufferFromPixelMap(pixelMap, surfaceBuffer);
280 }
281 
SetSurfaceBufferToPixelMap(const sptr<SurfaceBuffer> & surfaceBuffer,std::shared_ptr<PixelMap> & pixelMap)282 ImageProcessing_ErrorCode ImageProcessingNative::SetSurfaceBufferToPixelMap(const sptr<SurfaceBuffer>& surfaceBuffer,
283     std::shared_ptr<PixelMap>& pixelMap)
284 {
285     if (pixelMap->GetAllocatorType() == AllocatorType::DMA_ALLOC) {
286         return IMAGE_PROCESSING_SUCCESS;
287     }
288     return CopySurfaceBufferToPixelMap(surfaceBuffer, pixelMap);
289 }
290 
InitializeInner()291 ImageProcessing_ErrorCode ImageProcessingNative::InitializeInner()
292 {
293     return IMAGE_PROCESSING_SUCCESS;
294 }
295 
DeinitializeInner()296 ImageProcessing_ErrorCode ImageProcessingNative::DeinitializeInner()
297 {
298     return IMAGE_PROCESSING_SUCCESS;
299 }
300 
301 ImageProcessing_ErrorCode ImageProcessingNative::SetParameter([[maybe_unused]] const OHOS::Media::Format& parameter)
302 {
303     return IMAGE_PROCESSING_SUCCESS;
304 }
305 
306 ImageProcessing_ErrorCode ImageProcessingNative::GetParameter([[maybe_unused]] OHOS::Media::Format& parameter)
307 {
308     return IMAGE_PROCESSING_SUCCESS;
309 }
310 
311 ImageProcessing_ErrorCode ImageProcessingNative::ConvertColorSpace([[maybe_unused]] sptr<SurfaceBuffer>& sourceImage,
312     [[maybe_unused]] sptr<SurfaceBuffer>& destinationImage)
313 {
314     return IMAGE_PROCESSING_ERROR_UNSUPPORTED_PROCESSING;
315 }
316 
317 ImageProcessing_ErrorCode ImageProcessingNative::Compose([[maybe_unused]] sptr<SurfaceBuffer>& sourceImage,
318     [[maybe_unused]] sptr<SurfaceBuffer>& sourceGainmap, [[maybe_unused]] sptr<SurfaceBuffer>& destinationImage)
319 {
320     return IMAGE_PROCESSING_ERROR_UNSUPPORTED_PROCESSING;
321 }
322 
323 ImageProcessing_ErrorCode ImageProcessingNative::Decompose([[maybe_unused]] sptr<SurfaceBuffer>& sourceImage,
324     [[maybe_unused]] sptr<SurfaceBuffer>& destinationImage, [[maybe_unused]] sptr<SurfaceBuffer>& destinationGainmap)
325 {
326     return IMAGE_PROCESSING_ERROR_UNSUPPORTED_PROCESSING;
327 }
328 
329 ImageProcessing_ErrorCode ImageProcessingNative::GenerateMetadata([[maybe_unused]] sptr<SurfaceBuffer>& sourceImage)
330 {
331     return IMAGE_PROCESSING_ERROR_UNSUPPORTED_PROCESSING;
332 }
333 
334 ImageProcessing_ErrorCode ImageProcessingNative::EnhanceDetail([[maybe_unused]] sptr<SurfaceBuffer>& sourceImage,
335     [[maybe_unused]] sptr<SurfaceBuffer>& destinationImage)
336 {
337     return IMAGE_PROCESSING_ERROR_UNSUPPORTED_PROCESSING;
338 }
339 
CreateSurfaceBufferFromPixelMap(const std::shared_ptr<PixelMap> & pixelMap,sptr<SurfaceBuffer> & surfaceBuffer)340 ImageProcessing_ErrorCode ImageProcessingNative::CreateSurfaceBufferFromPixelMap(
341     const std::shared_ptr<PixelMap>& pixelMap, sptr<SurfaceBuffer>& surfaceBuffer)
342 {
343     surfaceBuffer = SurfaceBuffer::Create();
344     if (surfaceBuffer == nullptr) {
345         VPE_LOGE("Failed to create SurfaceBuffer!");
346         return IMAGE_PROCESSING_ERROR_CREATE_FAILED;
347     }
348     return ConvertPixelMapToSurfaceBuffer(pixelMap, surfaceBuffer);
349 }
350 
ConvertPixelMapToSurfaceBuffer(const std::shared_ptr<PixelMap> & pixelMap,sptr<SurfaceBuffer> & surfaceBuffer)351 ImageProcessing_ErrorCode ImageProcessingNative::ConvertPixelMapToSurfaceBuffer(
352     const std::shared_ptr<PixelMap>& pixelMap, sptr<SurfaceBuffer>& surfaceBuffer)
353 {
354     auto it = FORMAT_MAP.find(pixelMap->GetPixelFormat());
355     if (it == FORMAT_MAP.end()) {
356         VPE_LOGE("Unsupported format: %{public}d", pixelMap->GetPixelFormat());
357         return IMAGE_PROCESSING_ERROR_UNSUPPORTED_PROCESSING;
358     }
359     BufferRequestConfig requestCfg = {};
360     requestCfg.width = pixelMap->GetWidth();
361     requestCfg.height = pixelMap->GetHeight();
362     requestCfg.usage = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA;
363     requestCfg.strideAlignment = requestCfg.width;
364     requestCfg.format = it->second;
365     requestCfg.timeout = 0;
366     requestCfg.colorGamut = GraphicColorGamut::GRAPHIC_COLOR_GAMUT_SRGB;
367     requestCfg.transform = GraphicTransformType::GRAPHIC_ROTATE_NONE;
368     if (surfaceBuffer->Alloc(requestCfg) != GSERROR_OK) {
369         VPE_LOGE("Failed to allocate buffer for image!");
370         return IMAGE_PROCESSING_ERROR_NO_MEMORY;
371     }
372     return IMAGE_PROCESSING_SUCCESS;
373 }
374 
CopyPixelMapToSurfaceBuffer(const std::shared_ptr<PixelMap> & pixelMap,sptr<SurfaceBuffer> & surfaceBuffer)375 ImageProcessing_ErrorCode ImageProcessingNative::CopyPixelMapToSurfaceBuffer(
376     const std::shared_ptr<PixelMap>& pixelMap, sptr<SurfaceBuffer>& surfaceBuffer)
377 {
378     int32_t rowSize = pixelMap->GetWidth() * GetBytesPerPixel(pixelMap->GetPixelFormat());
379     for (int i = 0; i < pixelMap->GetHeight(); i++) {
380         if (memcpy_s(static_cast<uint8_t*>(surfaceBuffer->GetVirAddr()) + i * surfaceBuffer->GetStride(),
381             rowSize, pixelMap->GetPixels() + i * pixelMap->GetRowStride(), rowSize) != EOK) {
382             VPE_LOGE("Failed to copy image buffer!");
383             return IMAGE_PROCESSING_ERROR_PROCESS_FAILED;
384             }
385     }
386     return IMAGE_PROCESSING_SUCCESS;
387 }
388 
CopySurfaceBufferToPixelMap(const sptr<SurfaceBuffer> & surfaceBuffer,std::shared_ptr<PixelMap> & pixelMap)389 ImageProcessing_ErrorCode ImageProcessingNative::CopySurfaceBufferToPixelMap(const sptr<SurfaceBuffer>& surfaceBuffer,
390     std::shared_ptr<PixelMap>& pixelMap)
391 {
392     int rowSize = std::min(surfaceBuffer->GetStride(), pixelMap->GetRowStride());
393     for (int i = 0; i < surfaceBuffer->GetHeight(); ++i) {
394         if (memcpy_s(static_cast<uint8_t*>(pixelMap->GetWritablePixels()) + i * pixelMap->GetRowStride(), rowSize,
395             static_cast<uint8_t*>(surfaceBuffer->GetVirAddr()) + i * surfaceBuffer->GetStride(), rowSize) != EOK) {
396             VPE_LOGE("Failed to copy image buffer!");
397             return IMAGE_PROCESSING_ERROR_PROCESS_FAILED;
398         }
399     }
400     return IMAGE_PROCESSING_SUCCESS;
401 }
402