1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "GeneratedTestHarness.h"
18 
19 #include <android-base/logging.h>
20 #include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
21 #include <android/hardware/neuralnetworks/1.0/types.h>
22 #include <android/hardware/neuralnetworks/1.1/IDevice.h>
23 #include <android/hidl/allocator/1.0/IAllocator.h>
24 #include <android/hidl/memory/1.0/IMemory.h>
25 #include <hidlmemory/mapping.h>
26 
27 #include <gtest/gtest.h>
28 #include <iostream>
29 
30 #include "1.0/Callbacks.h"
31 #include "1.0/Utils.h"
32 #include "MemoryUtils.h"
33 #include "TestHarness.h"
34 #include "VtsHalNeuralnetworks.h"
35 
36 namespace android::hardware::neuralnetworks::V1_1::vts::functional {
37 
38 using namespace test_helper;
39 using hidl::memory::V1_0::IMemory;
40 using V1_0::DataLocation;
41 using V1_0::ErrorStatus;
42 using V1_0::IPreparedModel;
43 using V1_0::Operand;
44 using V1_0::OperandLifeTime;
45 using V1_0::OperandType;
46 using V1_0::Request;
47 using V1_0::implementation::ExecutionCallback;
48 using V1_0::implementation::PreparedModelCallback;
49 
createModel(const TestModel & testModel)50 Model createModel(const TestModel& testModel) {
51     // Model operands.
52     CHECK_EQ(testModel.referenced.size(), 0u);  // Not supported in 1.1.
53     hidl_vec<Operand> operands(testModel.main.operands.size());
54     size_t constCopySize = 0, constRefSize = 0;
55     for (uint32_t i = 0; i < testModel.main.operands.size(); i++) {
56         const auto& op = testModel.main.operands[i];
57 
58         DataLocation loc = {};
59         if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
60             loc = {.poolIndex = 0,
61                    .offset = static_cast<uint32_t>(constCopySize),
62                    .length = static_cast<uint32_t>(op.data.size())};
63             constCopySize += op.data.alignedSize();
64         } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
65             loc = {.poolIndex = 0,
66                    .offset = static_cast<uint32_t>(constRefSize),
67                    .length = static_cast<uint32_t>(op.data.size())};
68             constRefSize += op.data.alignedSize();
69         }
70 
71         operands[i] = {.type = static_cast<OperandType>(op.type),
72                        .dimensions = op.dimensions,
73                        .numberOfConsumers = op.numberOfConsumers,
74                        .scale = op.scale,
75                        .zeroPoint = op.zeroPoint,
76                        .lifetime = static_cast<OperandLifeTime>(op.lifetime),
77                        .location = loc};
78     }
79 
80     // Model operations.
81     hidl_vec<Operation> operations(testModel.main.operations.size());
82     std::transform(testModel.main.operations.begin(), testModel.main.operations.end(),
83                    operations.begin(), [](const TestOperation& op) -> Operation {
84                        return {.type = static_cast<OperationType>(op.type),
85                                .inputs = op.inputs,
86                                .outputs = op.outputs};
87                    });
88 
89     // Constant copies.
90     hidl_vec<uint8_t> operandValues(constCopySize);
91     for (uint32_t i = 0; i < testModel.main.operands.size(); i++) {
92         const auto& op = testModel.main.operands[i];
93         if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
94             const uint8_t* begin = op.data.get<uint8_t>();
95             const uint8_t* end = begin + op.data.size();
96             std::copy(begin, end, operandValues.data() + operands[i].location.offset);
97         }
98     }
99 
100     // Shared memory.
101     hidl_vec<hidl_memory> pools;
102     if (constRefSize > 0) {
103         hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
104         CHECK_NE(pools[0].size(), 0u);
105 
106         // load data
107         sp<IMemory> mappedMemory = mapMemory(pools[0]);
108         CHECK(mappedMemory.get() != nullptr);
109         uint8_t* mappedPtr =
110                 reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
111         CHECK(mappedPtr != nullptr);
112 
113         for (uint32_t i = 0; i < testModel.main.operands.size(); i++) {
114             const auto& op = testModel.main.operands[i];
115             if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
116                 const uint8_t* begin = op.data.get<uint8_t>();
117                 const uint8_t* end = begin + op.data.size();
118                 std::copy(begin, end, mappedPtr + operands[i].location.offset);
119             }
120         }
121     }
122 
123     return {.operands = std::move(operands),
124             .operations = std::move(operations),
125             .inputIndexes = testModel.main.inputIndexes,
126             .outputIndexes = testModel.main.outputIndexes,
127             .operandValues = std::move(operandValues),
128             .pools = std::move(pools),
129             .relaxComputationFloat32toFloat16 = testModel.isRelaxed};
130 }
131 
132 // Top level driver for models and examples generated by test_generator.py
133 // Test driver for those generated from ml/nn/runtime/test/spec
Execute(const sp<IDevice> & device,const TestModel & testModel)134 void Execute(const sp<IDevice>& device, const TestModel& testModel) {
135     const Model model = createModel(testModel);
136 
137     ExecutionContext context;
138     const Request request = context.createRequest(testModel);
139 
140     // Create IPreparedModel.
141     sp<IPreparedModel> preparedModel;
142     createPreparedModel(device, model, &preparedModel);
143     if (preparedModel == nullptr) return;
144 
145     // Launch execution.
146     sp<ExecutionCallback> executionCallback = new ExecutionCallback();
147     Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(request, executionCallback);
148     ASSERT_TRUE(executionLaunchStatus.isOk());
149     EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
150 
151     // Retrieve execution status.
152     executionCallback->wait();
153     ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());
154 
155     // Retrieve execution results.
156     const std::vector<TestBuffer> outputs = context.getOutputBuffers(request);
157 
158     // We want "close-enough" results.
159     checkResults(testModel, outputs);
160 }
161 
SetUp()162 void GeneratedTestBase::SetUp() {
163     testing::TestWithParam<GeneratedTestParam>::SetUp();
164     ASSERT_NE(kDevice, nullptr);
165     const bool deviceIsResponsive = kDevice->ping().isOk();
166     ASSERT_TRUE(deviceIsResponsive);
167 }
168 
getNamedModels(const FilterFn & filter)169 std::vector<NamedModel> getNamedModels(const FilterFn& filter) {
170     return TestModelManager::get().getTestModels(filter);
171 }
172 
getNamedModels(const FilterNameFn & filter)173 std::vector<NamedModel> getNamedModels(const FilterNameFn& filter) {
174     return TestModelManager::get().getTestModels(filter);
175 }
176 
printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam> & info)177 std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info) {
178     const auto& [namedDevice, namedModel] = info.param;
179     return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel));
180 }
181 
182 // Tag for the generated tests
183 class GeneratedTest : public GeneratedTestBase {};
184 
TEST_P(GeneratedTest,Test)185 TEST_P(GeneratedTest, Test) {
186     Execute(kDevice, kTestModel);
187 }
188 
189 INSTANTIATE_GENERATED_TEST(GeneratedTest,
__anonaf337bef0202(const TestModel& testModel) 190                            [](const TestModel& testModel) { return !testModel.expectFailure; });
191 
192 }  // namespace android::hardware::neuralnetworks::V1_1::vts::functional
193