1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /* Header-only library for various helpers of test harness
18  * See packages/modules/NeuralNetworks/runtime/test/TestGenerated.cpp for how this is used.
19  */
20 #ifndef ANDROID_FRAMEWORKS_ML_NN_TOOLS_TEST_GENERATOR_TEST_HARNESS_LEGACY_TEST_HARNESS_H
21 #define ANDROID_FRAMEWORKS_ML_NN_TOOLS_TEST_GENERATOR_TEST_HARNESS_LEGACY_TEST_HARNESS_H
22 
23 #include <gmock/gmock.h>
24 #include <gtest/gtest.h>
25 
26 #include <cmath>
27 #include <functional>
28 #include <map>
29 #include <string>
30 #include <tuple>
31 #include <utility>
32 #include <vector>
33 
34 namespace test_helper {
35 
36 constexpr const size_t gMaximumNumberOfErrorMessages = 10;
37 
38 // TODO: Figure out the build dependency to make including "CpuOperationUtils.h" work.
convertFloat16ToFloat32(const _Float16 * input,std::vector<float> * output)39 inline void convertFloat16ToFloat32(const _Float16* input, std::vector<float>* output) {
40     for (size_t i = 0; i < output->size(); ++i) {
41         (*output)[i] = static_cast<float>(input[i]);
42     }
43 }
44 
45 // This class is a workaround for two issues our code relies on:
46 // 1. sizeof(bool) is implementation defined.
47 // 2. vector<bool> does not allow direct pointer access via the data() method.
48 class bool8 {
49    public:
bool8()50     bool8() : mValue() {}
bool8(bool value)51     /* implicit */ bool8(bool value) : mValue(value) {}
52     inline operator bool() const { return mValue != 0; }
53 
54    private:
55     uint8_t mValue;
56 };
57 
58 static_assert(sizeof(bool8) == 1, "size of bool8 must be 8 bits");
59 
60 typedef std::map<int, std::vector<uint32_t>> OperandDimensions;
61 typedef std::map<int, std::vector<float>> Float32Operands;
62 typedef std::map<int, std::vector<int32_t>> Int32Operands;
63 typedef std::map<int, std::vector<uint8_t>> Quant8AsymmOperands;
64 typedef std::map<int, std::vector<int16_t>> Quant16SymmOperands;
65 typedef std::map<int, std::vector<_Float16>> Float16Operands;
66 typedef std::map<int, std::vector<bool8>> Bool8Operands;
67 typedef std::map<int, std::vector<int8_t>> Quant8ChannelOperands;
68 typedef std::map<int, std::vector<uint16_t>> Quant16AsymmOperands;
69 typedef std::map<int, std::vector<int8_t>> Quant8SymmOperands;
70 struct MixedTyped {
71     static constexpr size_t kNumTypes = 9;
72     OperandDimensions operandDimensions;
73     Float32Operands float32Operands;
74     Int32Operands int32Operands;
75     Quant8AsymmOperands quant8AsymmOperands;
76     Quant16SymmOperands quant16SymmOperands;
77     Float16Operands float16Operands;
78     Bool8Operands bool8Operands;
79     Quant8ChannelOperands quant8ChannelOperands;
80     Quant16AsymmOperands quant16AsymmOperands;
81     Quant8SymmOperands quant8SymmOperands;
82 };
83 typedef std::pair<MixedTyped, MixedTyped> MixedTypedExampleType;
84 
85 // Mixed-typed examples
86 typedef struct {
87     MixedTypedExampleType operands;
88     // Specifies the RANDOM_MULTINOMIAL distribution tolerance.
89     // If set to greater than zero, the input is compared as log-probabilities
90     // to the output and must be within this tolerance to pass.
91     float expectedMultinomialDistributionTolerance = 0.0;
92 } MixedTypedExample;
93 
94 // Go through all index-value pairs of a given input type
95 template <typename T>
for_each(const std::map<int,std::vector<T>> & idx_and_data,std::function<void (int,const std::vector<T> &)> execute)96 inline void for_each(const std::map<int, std::vector<T>>& idx_and_data,
97                      std::function<void(int, const std::vector<T>&)> execute) {
98     for (auto& i : idx_and_data) {
99         execute(i.first, i.second);
100     }
101 }
102 
103 // non-const variant of for_each
104 template <typename T>
for_each(std::map<int,std::vector<T>> & idx_and_data,std::function<void (int,std::vector<T> &)> execute)105 inline void for_each(std::map<int, std::vector<T>>& idx_and_data,
106                      std::function<void(int, std::vector<T>&)> execute) {
107     for (auto& i : idx_and_data) {
108         execute(i.first, i.second);
109     }
110 }
111 
112 // Go through all index-value pairs of a given input type
113 template <typename T>
for_each(const std::map<int,std::vector<T>> & golden,std::map<int,std::vector<T>> & test,std::function<void (int,const std::vector<T> &,std::vector<T> &)> execute)114 inline void for_each(const std::map<int, std::vector<T>>& golden,
115                      std::map<int, std::vector<T>>& test,
116                      std::function<void(int, const std::vector<T>&, std::vector<T>&)> execute) {
117     for_each<T>(golden, [&test, &execute](int index, const std::vector<T>& g) {
118         auto& t = test[index];
119         execute(index, g, t);
120     });
121 }
122 
123 // Go through all index-value pairs of a given input type
124 template <typename T>
for_each(const std::map<int,std::vector<T>> & golden,const std::map<int,std::vector<T>> & test,std::function<void (int,const std::vector<T> &,const std::vector<T> &)> execute)125 inline void for_each(
126         const std::map<int, std::vector<T>>& golden, const std::map<int, std::vector<T>>& test,
127         std::function<void(int, const std::vector<T>&, const std::vector<T>&)> execute) {
128     for_each<T>(golden, [&test, &execute](int index, const std::vector<T>& g) {
129         auto t = test.find(index);
130         ASSERT_NE(t, test.end());
131         execute(index, g, t->second);
132     });
133 }
134 
135 // internal helper for for_all
136 template <typename T>
for_all_internal(std::map<int,std::vector<T>> & idx_and_data,std::function<void (int,void *,size_t)> execute_this)137 inline void for_all_internal(std::map<int, std::vector<T>>& idx_and_data,
138                              std::function<void(int, void*, size_t)> execute_this) {
139     for_each<T>(idx_and_data, [&execute_this](int idx, std::vector<T>& m) {
140         execute_this(idx, static_cast<void*>(m.data()), m.size() * sizeof(T));
141     });
142 }
143 
144 // Go through all index-value pairs of all input types
145 // expects a functor that takes (int index, void *raw data, size_t sz)
for_all(MixedTyped & idx_and_data,std::function<void (int,void *,size_t)> execute_this)146 inline void for_all(MixedTyped& idx_and_data,
147                     std::function<void(int, void*, size_t)> execute_this) {
148     for_all_internal(idx_and_data.float32Operands, execute_this);
149     for_all_internal(idx_and_data.int32Operands, execute_this);
150     for_all_internal(idx_and_data.quant8AsymmOperands, execute_this);
151     for_all_internal(idx_and_data.quant16SymmOperands, execute_this);
152     for_all_internal(idx_and_data.float16Operands, execute_this);
153     for_all_internal(idx_and_data.bool8Operands, execute_this);
154     for_all_internal(idx_and_data.quant8ChannelOperands, execute_this);
155     for_all_internal(idx_and_data.quant16AsymmOperands, execute_this);
156     for_all_internal(idx_and_data.quant8SymmOperands, execute_this);
157     static_assert(9 == MixedTyped::kNumTypes,
158                   "Number of types in MixedTyped changed, but for_all function wasn't updated");
159 }
160 
161 // Const variant of internal helper for for_all
162 template <typename T>
for_all_internal(const std::map<int,std::vector<T>> & idx_and_data,std::function<void (int,const void *,size_t)> execute_this)163 inline void for_all_internal(const std::map<int, std::vector<T>>& idx_and_data,
164                              std::function<void(int, const void*, size_t)> execute_this) {
165     for_each<T>(idx_and_data, [&execute_this](int idx, const std::vector<T>& m) {
166         execute_this(idx, static_cast<const void*>(m.data()), m.size() * sizeof(T));
167     });
168 }
169 
170 // Go through all index-value pairs (const variant)
171 // expects a functor that takes (int index, const void *raw data, size_t sz)
for_all(const MixedTyped & idx_and_data,std::function<void (int,const void *,size_t)> execute_this)172 inline void for_all(const MixedTyped& idx_and_data,
173                     std::function<void(int, const void*, size_t)> execute_this) {
174     for_all_internal(idx_and_data.float32Operands, execute_this);
175     for_all_internal(idx_and_data.int32Operands, execute_this);
176     for_all_internal(idx_and_data.quant8AsymmOperands, execute_this);
177     for_all_internal(idx_and_data.quant16SymmOperands, execute_this);
178     for_all_internal(idx_and_data.float16Operands, execute_this);
179     for_all_internal(idx_and_data.bool8Operands, execute_this);
180     for_all_internal(idx_and_data.quant8ChannelOperands, execute_this);
181     for_all_internal(idx_and_data.quant16AsymmOperands, execute_this);
182     for_all_internal(idx_and_data.quant8SymmOperands, execute_this);
183     static_assert(
184             9 == MixedTyped::kNumTypes,
185             "Number of types in MixedTyped changed, but const for_all function wasn't updated");
186 }
187 
188 // Helper template - resize test output per golden
189 template <typename T>
resize_accordingly_(const std::map<int,std::vector<T>> & golden,std::map<int,std::vector<T>> & test)190 inline void resize_accordingly_(const std::map<int, std::vector<T>>& golden,
191                                 std::map<int, std::vector<T>>& test) {
192     for_each<T>(golden, test,
193                 [](int, const std::vector<T>& g, std::vector<T>& t) { t.resize(g.size()); });
194 }
195 
196 template <>
197 inline void resize_accordingly_<uint32_t>(const OperandDimensions& golden,
198                                           OperandDimensions& test) {
199     for_each<uint32_t>(
200             golden, test,
201             [](int, const std::vector<uint32_t>& g, std::vector<uint32_t>& t) { t = g; });
202 }
203 
resize_accordingly(const MixedTyped & golden,MixedTyped & test)204 inline void resize_accordingly(const MixedTyped& golden, MixedTyped& test) {
205     resize_accordingly_(golden.operandDimensions, test.operandDimensions);
206     resize_accordingly_(golden.float32Operands, test.float32Operands);
207     resize_accordingly_(golden.int32Operands, test.int32Operands);
208     resize_accordingly_(golden.quant8AsymmOperands, test.quant8AsymmOperands);
209     resize_accordingly_(golden.quant16SymmOperands, test.quant16SymmOperands);
210     resize_accordingly_(golden.float16Operands, test.float16Operands);
211     resize_accordingly_(golden.bool8Operands, test.bool8Operands);
212     resize_accordingly_(golden.quant8ChannelOperands, test.quant8ChannelOperands);
213     resize_accordingly_(golden.quant16AsymmOperands, test.quant16AsymmOperands);
214     resize_accordingly_(golden.quant8SymmOperands, test.quant8SymmOperands);
215     static_assert(9 == MixedTyped::kNumTypes,
216                   "Number of types in MixedTyped changed, but resize_accordingly function wasn't "
217                   "updated");
218 }
219 
220 template <typename T>
filter_internal(const std::map<int,std::vector<T>> & golden,std::map<int,std::vector<T>> * filtered,std::function<bool (int)> is_ignored)221 void filter_internal(const std::map<int, std::vector<T>>& golden,
222                      std::map<int, std::vector<T>>* filtered, std::function<bool(int)> is_ignored) {
223     for_each<T>(golden, [filtered, &is_ignored](int index, const std::vector<T>& m) {
224         auto& g = *filtered;
225         if (!is_ignored(index)) g[index] = m;
226     });
227 }
228 
filter(const MixedTyped & golden,std::function<bool (int)> is_ignored)229 inline MixedTyped filter(const MixedTyped& golden, std::function<bool(int)> is_ignored) {
230     MixedTyped filtered;
231     filter_internal(golden.operandDimensions, &filtered.operandDimensions, is_ignored);
232     filter_internal(golden.float32Operands, &filtered.float32Operands, is_ignored);
233     filter_internal(golden.int32Operands, &filtered.int32Operands, is_ignored);
234     filter_internal(golden.quant8AsymmOperands, &filtered.quant8AsymmOperands, is_ignored);
235     filter_internal(golden.quant16SymmOperands, &filtered.quant16SymmOperands, is_ignored);
236     filter_internal(golden.float16Operands, &filtered.float16Operands, is_ignored);
237     filter_internal(golden.bool8Operands, &filtered.bool8Operands, is_ignored);
238     filter_internal(golden.quant8ChannelOperands, &filtered.quant8ChannelOperands, is_ignored);
239     filter_internal(golden.quant16AsymmOperands, &filtered.quant16AsymmOperands, is_ignored);
240     filter_internal(golden.quant8SymmOperands, &filtered.quant8SymmOperands, is_ignored);
241     static_assert(9 == MixedTyped::kNumTypes,
242                   "Number of types in MixedTyped changed, but compare function wasn't updated");
243     return filtered;
244 }
245 
246 // Compare results
247 template <typename T>
compare_(const std::map<int,std::vector<T>> & golden,const std::map<int,std::vector<T>> & test,std::function<void (T,T)> cmp)248 void compare_(const std::map<int, std::vector<T>>& golden,
249               const std::map<int, std::vector<T>>& test, std::function<void(T, T)> cmp) {
250     for_each<T>(golden, test, [&cmp](int index, const std::vector<T>& g, const std::vector<T>& t) {
251         for (unsigned int i = 0; i < g.size(); i++) {
252             SCOPED_TRACE(testing::Message()
253                          << "When comparing output " << index << " element " << i);
254             cmp(g[i], t[i]);
255         }
256     });
257 }
258 
259 // TODO: Allow passing accuracy criteria from spec.
260 // Currently we only need relaxed accuracy criteria on mobilenet tests, so we return the quant8
261 // tolerance simply based on the current test name.
getQuant8AllowedError()262 inline int getQuant8AllowedError() {
263     const ::testing::TestInfo* const testInfo =
264             ::testing::UnitTest::GetInstance()->current_test_info();
265     const std::string testCaseName = testInfo->test_case_name();
266     const std::string testName = testInfo->name();
267     // We relax the quant8 precision for all tests with mobilenet:
268     // - CTS/VTS GeneratedTest and DynamicOutputShapeTest with mobilenet
269     // - VTS CompilationCachingTest and CompilationCachingSecurityTest except for TOCTOU tests
270     if (testName.find("mobilenet") != std::string::npos ||
271         (testCaseName.find("CompilationCaching") != std::string::npos &&
272          testName.find("TOCTOU") == std::string::npos)) {
273         return 2;
274     } else {
275         return 1;
276     }
277 }
278 
279 inline void compare(const MixedTyped& golden, const MixedTyped& test, float fpAtol = 1e-5f,
280                     float fpRtol = 1e-5f) {
281     int quant8AllowedError = getQuant8AllowedError();
282     for_each<uint32_t>(
283             golden.operandDimensions, test.operandDimensions,
284             [](int index, const std::vector<uint32_t>& g, const std::vector<uint32_t>& t) {
285                 SCOPED_TRACE(testing::Message()
286                              << "When comparing dimensions for output " << index);
287                 EXPECT_EQ(g, t);
288             });
289     size_t totalNumberOfErrors = 0;
290     compare_<float>(golden.float32Operands, test.float32Operands,
291                     [&totalNumberOfErrors, fpAtol, fpRtol](float expected, float actual) {
292                         // Compute the range based on both absolute tolerance and relative tolerance
293                         float fpRange = fpAtol + fpRtol * std::abs(expected);
294                         if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
295                             EXPECT_NEAR(expected, actual, fpRange);
296                         }
297                         if (std::abs(expected - actual) > fpRange) {
298                             totalNumberOfErrors++;
299                         }
300                     });
301     compare_<int32_t>(golden.int32Operands, test.int32Operands,
302                       [&totalNumberOfErrors](int32_t expected, int32_t actual) {
303                           if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
304                               EXPECT_EQ(expected, actual);
305                           }
306                           if (expected != actual) {
307                               totalNumberOfErrors++;
308                           }
309                       });
310     compare_<uint8_t>(golden.quant8AsymmOperands, test.quant8AsymmOperands,
311                       [&totalNumberOfErrors, quant8AllowedError](uint8_t expected, uint8_t actual) {
312                           if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
313                               EXPECT_NEAR(expected, actual, quant8AllowedError);
314                           }
315                           if (std::abs(expected - actual) > quant8AllowedError) {
316                               totalNumberOfErrors++;
317                           }
318                       });
319     compare_<int16_t>(golden.quant16SymmOperands, test.quant16SymmOperands,
320                       [&totalNumberOfErrors](int16_t expected, int16_t actual) {
321                           if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
322                               EXPECT_NEAR(expected, actual, 1);
323                           }
324                           if (std::abs(expected - actual) > 1) {
325                               totalNumberOfErrors++;
326                           }
327                       });
328     compare_<_Float16>(golden.float16Operands, test.float16Operands,
329                        [&totalNumberOfErrors, fpAtol, fpRtol](_Float16 expected, _Float16 actual) {
330                            // Compute the range based on both absolute tolerance and relative
331                            // tolerance
332                            float fpRange = fpAtol + fpRtol * std::abs(static_cast<float>(expected));
333                            if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
334                                EXPECT_NEAR(expected, actual, fpRange);
335                            }
336                            if (std::abs(static_cast<float>(expected - actual)) > fpRange) {
337                                totalNumberOfErrors++;
338                            }
339                        });
340     compare_<bool8>(golden.bool8Operands, test.bool8Operands,
341                     [&totalNumberOfErrors](bool expected, bool actual) {
342                         if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
343                             EXPECT_EQ(expected, actual);
344                         }
345                         if (expected != actual) {
346                             totalNumberOfErrors++;
347                         }
348                     });
349     compare_<int8_t>(golden.quant8ChannelOperands, test.quant8ChannelOperands,
350                      [&totalNumberOfErrors, &quant8AllowedError](int8_t expected, int8_t actual) {
351                          if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
352                              EXPECT_NEAR(expected, actual, quant8AllowedError);
353                          }
354                          if (std::abs(static_cast<int>(expected) - static_cast<int>(actual)) >
355                              quant8AllowedError) {
356                              totalNumberOfErrors++;
357                          }
358                      });
359     compare_<uint16_t>(golden.quant16AsymmOperands, test.quant16AsymmOperands,
360                        [&totalNumberOfErrors](int16_t expected, int16_t actual) {
361                            if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
362                                EXPECT_NEAR(expected, actual, 1);
363                            }
364                            if (std::abs(expected - actual) > 1) {
365                                totalNumberOfErrors++;
366                            }
367                        });
368     compare_<int8_t>(golden.quant8SymmOperands, test.quant8SymmOperands,
369                      [&totalNumberOfErrors, quant8AllowedError](int8_t expected, int8_t actual) {
370                          if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
371                              EXPECT_NEAR(expected, actual, quant8AllowedError);
372                          }
373                          if (std::abs(static_cast<int>(expected) - static_cast<int>(actual)) >
374                              quant8AllowedError) {
375                              totalNumberOfErrors++;
376                          }
377                      });
378 
379     static_assert(9 == MixedTyped::kNumTypes,
380                   "Number of types in MixedTyped changed, but compare function wasn't updated");
381     EXPECT_EQ(size_t{0}, totalNumberOfErrors);
382 }
383 
384 // Calculates the expected probability from the unnormalized log-probability of
385 // each class in the input and compares it to the actual ocurrence of that class
386 // in the output.
expectMultinomialDistributionWithinTolerance(const MixedTyped & test,const MixedTypedExample & example)387 inline void expectMultinomialDistributionWithinTolerance(const MixedTyped& test,
388                                                          const MixedTypedExample& example) {
389     // TODO: These should be parameters but aren't currently preserved in the example.
390     const int kBatchSize = 1;
391     const int kNumClasses = 1024;
392     const int kNumSamples = 128;
393 
394     std::vector<int32_t> output = test.int32Operands.at(0);
395     std::vector<int> class_counts;
396     class_counts.resize(kNumClasses);
397     for (int index : output) {
398         class_counts[index]++;
399     }
400     std::vector<float> input;
401     Float32Operands float32Operands = example.operands.first.float32Operands;
402     if (!float32Operands.empty()) {
403         input = example.operands.first.float32Operands.at(0);
404     } else {
405         std::vector<_Float16> inputFloat16 = example.operands.first.float16Operands.at(0);
406         input.resize(inputFloat16.size());
407         convertFloat16ToFloat32(inputFloat16.data(), &input);
408     }
409     for (int b = 0; b < kBatchSize; ++b) {
410         float probability_sum = 0;
411         const int batch_index = kBatchSize * b;
412         for (int i = 0; i < kNumClasses; ++i) {
413             probability_sum += expf(input[batch_index + i]);
414         }
415         for (int i = 0; i < kNumClasses; ++i) {
416             float probability =
417                     static_cast<float>(class_counts[i]) / static_cast<float>(kNumSamples);
418             float probability_expected = expf(input[batch_index + i]) / probability_sum;
419             EXPECT_THAT(probability,
420                         ::testing::FloatNear(probability_expected,
421                                              example.expectedMultinomialDistributionTolerance));
422         }
423     }
424 }
425 
426 };  // namespace test_helper
427 
428 #endif  // ANDROID_FRAMEWORKS_ML_NN_TOOLS_TEST_GENERATOR_TEST_HARNESS_LEGACY_TEST_HARNESS_H
429