1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_NDEBUG 0
18 #define LOG_TAG "DepthProcessorTest"
19 
20 #include <array>
21 #include <random>
22 
23 #include <gtest/gtest.h>
24 
25 #include "../common/DepthPhotoProcessor.h"
26 #include "../utils/ExifUtils.h"
27 #include "NV12Compressor.h"
28 
29 using namespace android;
30 using namespace android::camera3;
31 
32 static const size_t kTestBufferWidth = 640;
33 static const size_t kTestBufferHeight = 480;
34 static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeight)) * 3) / 2);
35 static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);
36 static const size_t kSeed = 1234;
37 
generateColorJpegBuffer(int jpegQuality,ExifOrientation orientationValue,bool includeExif,bool switchDimensions,std::vector<uint8_t> * colorJpegBuffer)38 void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,
39         bool switchDimensions, std::vector<uint8_t> *colorJpegBuffer /*out*/) {
40     ASSERT_NE(colorJpegBuffer, nullptr);
41 
42     std::array<uint8_t, kTestBufferNV12Size> colorSourceBuffer;
43     std::default_random_engine gen(kSeed);
44     std::uniform_int_distribution<int> uniDist(0, UINT8_MAX - 1);
45     for (size_t i = 0; i < colorSourceBuffer.size(); i++) {
46         colorSourceBuffer[i] = uniDist(gen);
47     }
48 
49     size_t width = kTestBufferWidth;
50     size_t height = kTestBufferHeight;
51     if (switchDimensions) {
52         width = kTestBufferHeight;
53         height = kTestBufferWidth;
54     }
55 
56     NV12Compressor jpegCompressor;
57     if (includeExif) {
58         ASSERT_TRUE(jpegCompressor.compressWithExifOrientation(
59                 reinterpret_cast<const unsigned char*> (colorSourceBuffer.data()), width, height,
60                 jpegQuality, orientationValue));
61     } else {
62         ASSERT_TRUE(jpegCompressor.compress(
63                 reinterpret_cast<const unsigned char*> (colorSourceBuffer.data()), width, height,
64                 jpegQuality));
65     }
66 
67     *colorJpegBuffer = std::move(jpegCompressor.getCompressedData());
68     ASSERT_FALSE(colorJpegBuffer->empty());
69 }
70 
generateDepth16Buffer(std::array<uint16_t,kTestBufferDepthSize> * depth16Buffer)71 void generateDepth16Buffer(std::array<uint16_t, kTestBufferDepthSize> *depth16Buffer /*out*/) {
72     ASSERT_NE(depth16Buffer, nullptr);
73     std::default_random_engine gen(kSeed+1);
74     std::uniform_int_distribution<int> uniDist(0, UINT16_MAX - 1);
75     for (size_t i = 0; i < depth16Buffer->size(); i++) {
76         (*depth16Buffer)[i] = uniDist(gen);
77     }
78 }
79 
TEST(DepthProcessorTest,BadInput)80 TEST(DepthProcessorTest, BadInput) {
81     int jpegQuality = 95;
82 
83     DepthPhotoInputFrame inputFrame;
84     // Worst case both depth and confidence maps have the same size as the main color image.
85     inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
86 
87     std::vector<uint8_t> colorJpegBuffer;
88     generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
89             /*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer);
90 
91     std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;
92     generateDepth16Buffer(&depth16Buffer);
93 
94     std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
95     size_t actualDepthPhotoSize = 0;
96 
97     inputFrame.mMainJpegWidth = kTestBufferWidth;
98     inputFrame.mMainJpegHeight = kTestBufferHeight;
99     inputFrame.mJpegQuality = jpegQuality;
100     ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
101                 &actualDepthPhotoSize), 0);
102 
103     inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
104     inputFrame.mMainJpegSize = colorJpegBuffer.size();
105     ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
106                 &actualDepthPhotoSize), 0);
107 
108     inputFrame.mDepthMapBuffer = depth16Buffer.data();
109     inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
110     inputFrame.mDepthMapHeight = kTestBufferHeight;
111     ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), nullptr,
112                 &actualDepthPhotoSize), 0);
113 
114     ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
115                 nullptr), 0);
116 }
117 
TEST(DepthProcessorTest,BasicDepthPhotoValidation)118 TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
119     int jpegQuality = 95;
120 
121     std::vector<uint8_t> colorJpegBuffer;
122     generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
123             /*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer);
124 
125     std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;
126     generateDepth16Buffer(&depth16Buffer);
127 
128     DepthPhotoInputFrame inputFrame;
129     inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
130     inputFrame.mMainJpegSize = colorJpegBuffer.size();
131     // Worst case both depth and confidence maps have the same size as the main color image.
132     inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
133     inputFrame.mMainJpegWidth = kTestBufferWidth;
134     inputFrame.mMainJpegHeight = kTestBufferHeight;
135     inputFrame.mJpegQuality = jpegQuality;
136     inputFrame.mDepthMapBuffer = depth16Buffer.data();
137     inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
138     inputFrame.mDepthMapHeight = kTestBufferHeight;
139 
140     std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
141     size_t actualDepthPhotoSize = 0;
142     ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
143                 &actualDepthPhotoSize), 0);
144     ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));
145 
146     // The final depth photo must consist of three jpeg images:
147     //  - the main color image
148     //  - the depth map image
149     //  - the confidence map image
150     size_t mainJpegSize = 0;
151     ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize,
152                 &mainJpegSize), OK);
153     ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize));
154     size_t depthMapSize = 0;
155     ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
156                 actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
157     ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
158 }
159 
TEST(DepthProcessorTest,TestDepthPhotoExifOrientation)160 TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
161     int jpegQuality = 95;
162 
163     ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,
164             ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,
165             ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };
166     for (auto exifOrientation : exifOrientations) {
167         std::vector<uint8_t> colorJpegBuffer;
168         generateColorJpegBuffer(jpegQuality, exifOrientation, /*includeExif*/ true,
169                 /*switchDimensions*/ false, &colorJpegBuffer);
170         if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) {
171             auto jpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
172             ASSERT_EQ(NV12Compressor::getExifOrientation(colorJpegBuffer.data(),
173                     colorJpegBuffer.size(), &jpegExifOrientation), OK);
174             ASSERT_EQ(exifOrientation, jpegExifOrientation);
175         }
176 
177         std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;
178         generateDepth16Buffer(&depth16Buffer);
179 
180         DepthPhotoInputFrame inputFrame;
181         inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
182         inputFrame.mMainJpegSize = colorJpegBuffer.size();
183         // Worst case both depth and confidence maps have the same size as the main color image.
184         inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
185         inputFrame.mMainJpegWidth = kTestBufferWidth;
186         inputFrame.mMainJpegHeight = kTestBufferHeight;
187         inputFrame.mJpegQuality = jpegQuality;
188         inputFrame.mDepthMapBuffer = depth16Buffer.data();
189         inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
190         inputFrame.mDepthMapHeight = kTestBufferHeight;
191 
192         std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
193         size_t actualDepthPhotoSize = 0;
194         ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
195                     depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
196         ASSERT_TRUE((actualDepthPhotoSize > 0) &&
197                 (depthPhotoBuffer.size() >= actualDepthPhotoSize));
198 
199         size_t mainJpegSize = 0;
200         ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize,
201                 &mainJpegSize), OK);
202         ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize));
203         size_t depthMapSize = 0;
204         ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
205                 actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
206         ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
207         size_t confidenceMapSize = actualDepthPhotoSize - (mainJpegSize + depthMapSize);
208 
209         //Depth and confidence images must have the same EXIF orientation as the source
210         auto depthJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
211         ASSERT_EQ(NV12Compressor::getExifOrientation(depthPhotoBuffer.data() + mainJpegSize,
212                 depthMapSize, &depthJpegExifOrientation), OK);
213         if (exifOrientation == ORIENTATION_UNDEFINED) {
214             // In case of undefined or missing EXIF orientation, always expect 0 degrees in the
215             // depth map.
216             ASSERT_EQ(depthJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES);
217         } else {
218             ASSERT_EQ(depthJpegExifOrientation, exifOrientation);
219         }
220 
221         auto confidenceJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
222         ASSERT_EQ(NV12Compressor::getExifOrientation(
223                 depthPhotoBuffer.data() + mainJpegSize + depthMapSize,
224                 confidenceMapSize, &confidenceJpegExifOrientation), OK);
225         if (exifOrientation == ORIENTATION_UNDEFINED) {
226             // In case of undefined or missing EXIF orientation, always expect 0 degrees in the
227             // confidence map.
228             ASSERT_EQ(confidenceJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES);
229         } else {
230             ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
231         }
232     }
233 }
234 
TEST(DepthProcessorTest,TestDephtPhotoPhysicalRotation)235 TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
236     int jpegQuality = 95;
237 
238     // In case of physical rotation, the EXIF orientation must always be 0.
239     auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES;
240     DepthPhotoOrientation depthOrientations[] = {
241             DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES,
242             DepthPhotoOrientation::DEPTH_ORIENTATION_90_DEGREES,
243             DepthPhotoOrientation::DEPTH_ORIENTATION_180_DEGREES,
244             DepthPhotoOrientation::DEPTH_ORIENTATION_270_DEGREES };
245     for (auto depthOrientation : depthOrientations) {
246         std::vector<uint8_t> colorJpegBuffer;
247         bool switchDimensions = false;
248         size_t expectedWidth = kTestBufferWidth;
249         size_t expectedHeight = kTestBufferHeight;
250         if ((depthOrientation == DepthPhotoOrientation::DEPTH_ORIENTATION_90_DEGREES) ||
251                 (depthOrientation == DepthPhotoOrientation::DEPTH_ORIENTATION_270_DEGREES)) {
252             switchDimensions = true;
253             expectedWidth = kTestBufferHeight;
254             expectedHeight = kTestBufferWidth;
255         }
256         generateColorJpegBuffer(jpegQuality, exifOrientation, /*includeExif*/ true,
257                 switchDimensions, &colorJpegBuffer);
258         auto jpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
259         ASSERT_EQ(NV12Compressor::getExifOrientation(colorJpegBuffer.data(), colorJpegBuffer.size(),
260                 &jpegExifOrientation), OK);
261         ASSERT_EQ(exifOrientation, jpegExifOrientation);
262 
263         std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;
264         generateDepth16Buffer(&depth16Buffer);
265 
266         DepthPhotoInputFrame inputFrame;
267         inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
268         inputFrame.mMainJpegSize = colorJpegBuffer.size();
269         // Worst case both depth and confidence maps have the same size as the main color image.
270         inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
271         inputFrame.mMainJpegWidth = kTestBufferWidth;
272         inputFrame.mMainJpegHeight = kTestBufferHeight;
273         inputFrame.mJpegQuality = jpegQuality;
274         inputFrame.mDepthMapBuffer = depth16Buffer.data();
275         inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
276         inputFrame.mDepthMapHeight = kTestBufferHeight;
277         inputFrame.mOrientation = depthOrientation;
278 
279         std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
280         size_t actualDepthPhotoSize = 0;
281         ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
282                     depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
283         ASSERT_TRUE((actualDepthPhotoSize > 0) &&
284                 (depthPhotoBuffer.size() >= actualDepthPhotoSize));
285 
286         size_t mainJpegSize = 0;
287         ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize,
288                 &mainJpegSize), OK);
289         ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize));
290         size_t depthMapSize = 0;
291         ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
292                 actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
293         ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
294         size_t confidenceMapSize = actualDepthPhotoSize - (mainJpegSize + depthMapSize);
295 
296         //Depth and confidence images must have the same EXIF orientation as the source
297         auto depthJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
298         ASSERT_EQ(NV12Compressor::getExifOrientation(depthPhotoBuffer.data() + mainJpegSize,
299                 depthMapSize, &depthJpegExifOrientation), OK);
300         ASSERT_EQ(depthJpegExifOrientation, exifOrientation);
301         size_t depthMapWidth, depthMapHeight;
302         ASSERT_EQ(NV12Compressor::getJpegImageDimensions(depthPhotoBuffer.data() + mainJpegSize,
303                 depthMapSize, &depthMapWidth, &depthMapHeight), OK);
304         ASSERT_EQ(depthMapWidth, expectedWidth);
305         ASSERT_EQ(depthMapHeight, expectedHeight);
306 
307         auto confidenceJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
308         ASSERT_EQ(NV12Compressor::getExifOrientation(
309                 depthPhotoBuffer.data() + mainJpegSize + depthMapSize, confidenceMapSize,
310                 &confidenceJpegExifOrientation), OK);
311         ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
312         size_t confidenceMapWidth, confidenceMapHeight;
313         ASSERT_EQ(NV12Compressor::getJpegImageDimensions(
314                 depthPhotoBuffer.data() + mainJpegSize + depthMapSize, confidenceMapSize,
315                 &confidenceMapWidth, &confidenceMapHeight), OK);
316         ASSERT_EQ(confidenceMapWidth, expectedWidth);
317         ASSERT_EQ(confidenceMapHeight, expectedHeight);
318     }
319 }
320