1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Operations"
18 
19 #include <algorithm>
20 #include <functional>
21 #include <vector>
22 
23 #include "OperationResolver.h"
24 #include "Tracing.h"
25 #include "nnapi/Validation.h"
26 
27 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
28 #include <tensorflow/lite/kernels/internal/reference/reference_ops.h>
29 
30 #include "CpuOperationUtils.h"
31 #endif  // NN_INCLUDE_CPU_IMPLEMENTATION
32 
33 namespace android {
34 namespace nn {
35 
36 namespace resize_image {
37 
38 constexpr uint32_t kNumInputs = 4;
39 constexpr uint32_t kInputTensor = 0;
40 // The following two scalars represent output shape if INT32, scale if floating point.
41 constexpr uint32_t kOutputWidthParamScalar = 1;
42 constexpr uint32_t kOutputHeightParamScalar = 2;
43 constexpr uint32_t kLayoutScalar = 3;
44 constexpr uint32_t kNumOptionalInputs = 2;
45 constexpr uint32_t kAlignCornersScalar = 4;
46 constexpr uint32_t kHalfPixelCentersScalar = 5;
47 
48 constexpr uint32_t kNumOutputs = 1;
49 constexpr uint32_t kOutputTensor = 0;
50 
51 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
52 namespace {
53 
scaleHalfPixel(const int x,const float scale)54 inline float scaleHalfPixel(const int x, const float scale) {
55     return (static_cast<float>(x) + 0.5f) * scale;
56 }
57 
scaleLegacy(const int x,const float scale)58 inline float scaleLegacy(const int x, const float scale) {
59     return static_cast<float>(x) * scale;
60 }
61 
calculateResizeScale(int32_t inSize,int32_t outSize,bool alignCorners)62 inline float calculateResizeScale(int32_t inSize, int32_t outSize, bool alignCorners) {
63     return (alignCorners && outSize > 1) ? (inSize - 1) / static_cast<float>(outSize - 1)
64                                          : inSize / static_cast<float>(outSize);
65 }
66 
67 template <typename T>
resizeNearestNeighbor(const T * inputData,const Shape & inputShape,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)68 bool resizeNearestNeighbor(const T* inputData, const Shape& inputShape, bool alignCorners,
69                            bool halfPixelCenters, T* outputData, const Shape& outputShape) {
70     const int batchSize = getSizeOfDimension(inputShape, 0);
71     const int inHeight = getSizeOfDimension(inputShape, 1);
72     const int inWidth = getSizeOfDimension(inputShape, 2);
73     const int channels = getSizeOfDimension(inputShape, 3);
74     const int outHeight = getSizeOfDimension(outputShape, 1);
75     const int outWidth = getSizeOfDimension(outputShape, 2);
76 
77     const float heightScale = calculateResizeScale(inHeight, outHeight, alignCorners);
78     const float widthScale = calculateResizeScale(inWidth, outWidth, alignCorners);
79 
80     const std::function<float(const int, const float)> scaler =
81             halfPixelCenters ? scaleHalfPixel : scaleLegacy;
82 
83     for (int b = 0; b < batchSize; ++b) {
84         for (int y = 0; y < outHeight; ++y) {
85             int inY = std::min((alignCorners) ? static_cast<int>(roundf(scaler(y, heightScale)))
86                                               : static_cast<int>(floorf(scaler(y, heightScale))),
87                                inHeight - 1);
88             if (halfPixelCenters) {
89                 inY = std::max(static_cast<int>(0), inY);
90             }
91             for (int x = 0; x < outWidth; ++x) {
92                 int inX = std::min((alignCorners) ? static_cast<int>(roundf(scaler(x, widthScale)))
93                                                   : static_cast<int>(floorf(scaler(x, widthScale))),
94                                    inWidth - 1);
95                 if (halfPixelCenters) {
96                     inX = std::max(static_cast<int>(0), inX);
97                 }
98                 std::copy_n(inputData + b * inHeight * inWidth * channels +
99                                     inY * inWidth * channels + inX * channels,
100                             channels,
101                             outputData + b * outHeight * outWidth * channels +
102                                     y * outWidth * channels + x * channels);
103             }
104         }
105     }
106 
107     return true;
108 }
109 
110 template <typename T>
resizeImageOpNhwc(OperationType opType,const T * inputData,const Shape & inputShape,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)111 bool resizeImageOpNhwc(OperationType opType, const T* inputData, const Shape& inputShape,
112                        bool alignCorners, bool halfPixelCenters, T* outputData,
113                        const Shape& outputShape) {
114     NNTRACE_TRANS("resizeImageOpNhwc");
115     int32_t height = static_cast<int32_t>(getSizeOfDimension(outputShape, 1));
116     int32_t width = static_cast<int32_t>(getSizeOfDimension(outputShape, 2));
117     // We have to fake a tensor here, to satisfy tflite implementation.
118     int32_t outDimData[2] = {height, width};
119     Shape outDimShape;
120     outDimShape.dimensions = {2};
121 
122     if (opType == OperationType::RESIZE_BILINEAR) {
123         NNTRACE_COMP_SWITCH("optimized_ops::ResizeBilinear");
124         tflite::reference_ops::ResizeBilinear(
125                 {.align_corners = alignCorners, .half_pixel_centers = halfPixelCenters},
126                 convertShapeToTflshape(inputShape), inputData, convertShapeToTflshape(outDimShape),
127                 outDimData, convertShapeToTflshape(outputShape), outputData);
128     } else if (opType == OperationType::RESIZE_NEAREST_NEIGHBOR) {
129         // Align corners = true is not supported.
130         NNTRACE_COMP_SWITCH("ResizeNearestNeighbor");
131         resizeNearestNeighbor(inputData, inputShape, alignCorners, halfPixelCenters, outputData,
132                               outputShape);
133     }
134     return true;
135 }
136 
137 template <>
138 bool resizeImageOpNhwc<_Float16>(OperationType opType, const _Float16* inputData,
139                                  const Shape& inputShape, bool alignCorners, bool halfPixelCenters,
140                                  _Float16* outputData, const Shape& outputShape) {
141     NNTRACE_TRANS("resizeImageOpNhwcFloat16");
142     std::vector<float> inputData_float32(getNumberOfElements(inputShape));
143     convertFloat16ToFloat32(inputData, &inputData_float32);
144     std::vector<float> outputData_float32(getNumberOfElements(outputShape));
145     NN_RET_CHECK(resizeImageOpNhwc(opType, inputData_float32.data(), inputShape, alignCorners,
146                                    halfPixelCenters, outputData_float32.data(), outputShape));
147     convertFloat32ToFloat16(outputData_float32, outputData);
148     return true;
149 }
150 
151 template <typename T>
resizeImageOp(OperationType opType,const T * inputData,const Shape & inputShape,bool useNchw,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)152 bool resizeImageOp(OperationType opType, const T* inputData, const Shape& inputShape, bool useNchw,
153                    bool alignCorners, bool halfPixelCenters, T* outputData,
154                    const Shape& outputShape) {
155     InputWithLayout<T> input(useNchw);
156     OutputWithLayout<T> output(useNchw);
157     NN_RET_CHECK(input.initialize(inputData, inputShape));
158     NN_RET_CHECK(output.initialize(outputData, outputShape));
159     NN_RET_CHECK(resizeImageOpNhwc(opType, input.getNhwcBuffer(), input.getNhwcShape(),
160                                    alignCorners, halfPixelCenters, output.getNhwcBuffer(),
161                                    output.getNhwcShape()));
162     NN_RET_CHECK(output.commit());
163     return true;
164 }
165 
getOptionalScalar(const IOperationExecutionContext * context,uint32_t scalarIndex)166 inline bool getOptionalScalar(const IOperationExecutionContext* context, uint32_t scalarIndex) {
167     bool scalarValue = false;
168     if (context->getNumInputs() > scalarIndex) {
169         scalarValue = context->getInputValue<bool>(scalarIndex);
170     }
171     return scalarValue;
172 }
173 
174 }  // namespace
175 #endif  // NN_INCLUDE_CPU_IMPLEMENTATION
176 
validate(OperationType opType,const IOperationValidationContext * context)177 Result<Version> validate(OperationType opType, const IOperationValidationContext* context) {
178     const auto numInputs = context->getNumInputs();
179     if (opType == OperationType::RESIZE_BILINEAR) {
180         NN_RET_CHECK(numInputs >= kNumInputs - 1 && numInputs <= kNumInputs + kNumOptionalInputs);
181     } else if (opType == OperationType::RESIZE_NEAREST_NEIGHBOR) {
182         NN_RET_CHECK(numInputs >= kNumInputs && numInputs <= kNumInputs + kNumOptionalInputs);
183     } else {
184         NN_RET_CHECK_FAIL() << "Unsupported operation " << opType;
185     }
186     NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs);
187     auto inputType = context->getInputType(kInputTensor);
188     auto scalarType = context->getInputType(kOutputHeightParamScalar);
189     std::vector<OperandType> inExpectedTypes = {inputType, scalarType, scalarType};
190     auto minSupportedVersion = Version::ANDROID_OC_MR1;
191     NN_RET_CHECK(inputType == OperandType::TENSOR_FLOAT16 ||
192                  inputType == OperandType::TENSOR_FLOAT32 ||
193                  inputType == OperandType::TENSOR_QUANT8_ASYMM ||
194                  inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED)
195             << "Unsupported tensor type for operation " << opType;
196     if (inputType == OperandType::TENSOR_FLOAT16 || inputType == OperandType::TENSOR_QUANT8_ASYMM) {
197         minSupportedVersion = combineVersions(minSupportedVersion, Version::ANDROID_Q);
198     }
199     if (inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
200         minSupportedVersion = combineVersions(minSupportedVersion, Version::ANDROID_R);
201     }
202     if (scalarType != OperandType::INT32) {
203         minSupportedVersion = combineVersions(minSupportedVersion, Version::ANDROID_Q);
204         if (inputType == OperandType::TENSOR_FLOAT32) {
205             NN_RET_CHECK(scalarType == OperandType::FLOAT32);
206         } else if (inputType == OperandType::TENSOR_FLOAT16) {
207             NN_RET_CHECK(scalarType == OperandType::FLOAT16);
208         } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM ||
209                    inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
210             NN_RET_CHECK(scalarType == OperandType::FLOAT32);
211         }
212     }
213     if (numInputs < kNumInputs) {
214         minSupportedVersion = combineVersions(minSupportedVersion, Version::ANDROID_OC_MR1);
215     } else if (numInputs == kNumInputs) {
216         inExpectedTypes.push_back(OperandType::BOOL);
217         minSupportedVersion = combineVersions(minSupportedVersion, Version::ANDROID_Q);
218     } else {
219         while (inExpectedTypes.size() < numInputs) {
220             inExpectedTypes.push_back(OperandType::BOOL);
221         }
222         minSupportedVersion = combineVersions(minSupportedVersion, Version::ANDROID_R);
223     }
224     NN_RET_CHECK(validateInputTypes(context, inExpectedTypes));
225     NN_RET_CHECK(validateOutputTypes(context, {inputType}));
226     return minSupportedVersion;
227 }
228 
229 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
prepare(OperationType opType,IOperationExecutionContext * context)230 bool prepare(OperationType opType, IOperationExecutionContext* context) {
231     Shape input = context->getInputShape(kInputTensor);
232     NN_RET_CHECK_EQ(getNumberOfDimensions(input), 4);
233     const auto numInputs = context->getNumInputs();
234     const bool useNchw = getOptionalScalar(context, kLayoutScalar);
235     const bool alignCorners = getOptionalScalar(context, kAlignCornersScalar);
236     const bool halfPixelCenters = getOptionalScalar(context, kHalfPixelCentersScalar);
237 
238     NN_RET_CHECK(!halfPixelCenters || (halfPixelCenters && !alignCorners));
239 
240     // Only batches can be zero.
241     uint32_t batches = getSizeOfDimension(input, 0);
242     uint32_t inHeight = getSizeOfDimension(input, useNchw ? 2 : 1);
243     uint32_t inWidth = getSizeOfDimension(input, useNchw ? 3 : 2);
244     uint32_t channels = getSizeOfDimension(input, useNchw ? 1 : 3);
245     NN_RET_CHECK_GT(inHeight, 0);
246     NN_RET_CHECK_GT(inWidth, 0);
247     NN_RET_CHECK_GT(channels, 0);
248 
249     int32_t height, width;
250     auto scalarType = context->getInputType(kOutputHeightParamScalar);
251     if (scalarType == OperandType::INT32) {
252         height = context->getInputValue<int32_t>(kOutputHeightParamScalar);
253         width = context->getInputValue<int32_t>(kOutputWidthParamScalar);
254     } else if (scalarType == OperandType::FLOAT32) {
255         height = std::floor(static_cast<float>(inHeight) *
256                             context->getInputValue<float>(kOutputHeightParamScalar));
257         width = std::floor(static_cast<float>(inWidth) *
258                            context->getInputValue<float>(kOutputWidthParamScalar));
259     } else if (scalarType == OperandType::FLOAT16) {
260         height = std::floor(
261                 static_cast<float>(inHeight) *
262                 static_cast<float>(context->getInputValue<_Float16>(kOutputHeightParamScalar)));
263         width = std::floor(
264                 static_cast<float>(inWidth) *
265                 static_cast<float>(context->getInputValue<_Float16>(kOutputWidthParamScalar)));
266     } else {
267         NN_RET_CHECK_FAIL() << "Unsupported scalar type for operation " << opType;
268     }
269     NN_RET_CHECK_GT(height, 0);
270     NN_RET_CHECK_GT(width, 0);
271 
272     Shape output = input;
273     if (useNchw) {
274         output.dimensions = {batches, channels, (uint32_t)height, (uint32_t)width};
275     } else {
276         output.dimensions = {batches, (uint32_t)height, (uint32_t)width, channels};
277     }
278     return context->setOutputShape(kOutputTensor, output);
279 }
280 
execute(OperationType opType,IOperationExecutionContext * context)281 bool execute(OperationType opType, IOperationExecutionContext* context) {
282     // Bypass execution in the case of zero-sized input.
283     if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true;
284 
285     const bool useNchw = getOptionalScalar(context, kLayoutScalar);
286     const bool alignCorners = getOptionalScalar(context, kAlignCornersScalar);
287     const bool halfPixelCenters = getOptionalScalar(context, kHalfPixelCentersScalar);
288 
289     switch (context->getInputType(kInputTensor)) {
290         case OperandType::TENSOR_FLOAT16:
291             return resizeImageOp(opType, context->getInputBuffer<_Float16>(kInputTensor),
292                                  context->getInputShape(kInputTensor), useNchw, alignCorners,
293                                  halfPixelCenters,
294                                  context->getOutputBuffer<_Float16>(kOutputTensor),
295                                  context->getOutputShape(kOutputTensor));
296         case OperandType::TENSOR_FLOAT32:
297             return resizeImageOp(opType, context->getInputBuffer<float>(kInputTensor),
298                                  context->getInputShape(kInputTensor), useNchw, alignCorners,
299                                  halfPixelCenters, context->getOutputBuffer<float>(kOutputTensor),
300                                  context->getOutputShape(kOutputTensor));
301         case OperandType::TENSOR_QUANT8_ASYMM:
302             return resizeImageOp(opType, context->getInputBuffer<uint8_t>(kInputTensor),
303                                  context->getInputShape(kInputTensor), useNchw, alignCorners,
304                                  halfPixelCenters, context->getOutputBuffer<uint8_t>(kOutputTensor),
305                                  context->getOutputShape(kOutputTensor));
306         case OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
307             return resizeImageOp(opType, context->getInputBuffer<int8_t>(kInputTensor),
308                                  context->getInputShape(kInputTensor), useNchw, alignCorners,
309                                  halfPixelCenters, context->getOutputBuffer<int8_t>(kOutputTensor),
310                                  context->getOutputShape(kOutputTensor));
311 
312         default:
313             NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << opType;
314     }
315 }
316 #endif  // NN_INCLUDE_CPU_IMPLEMENTATION
317 
318 }  // namespace resize_image
319 
320 using std::placeholders::_1;
321 
322 NN_REGISTER_OPERATION(RESIZE_BILINEAR, "RESIZE_BILINEAR",
323                       std::bind(resize_image::validate, OperationType::RESIZE_BILINEAR, _1),
324                       std::bind(resize_image::prepare, OperationType::RESIZE_BILINEAR, _1),
325                       std::bind(resize_image::execute, OperationType::RESIZE_BILINEAR, _1),
326                       .allowZeroSizedInput = true);
327 
328 NN_REGISTER_OPERATION(RESIZE_NEAREST_NEIGHBOR, "RESIZE_NEAREST_NEIGHBOR",
329                       std::bind(resize_image::validate, OperationType::RESIZE_NEAREST_NEIGHBOR, _1),
330                       std::bind(resize_image::prepare, OperationType::RESIZE_NEAREST_NEIGHBOR, _1),
331                       std::bind(resize_image::execute, OperationType::RESIZE_NEAREST_NEIGHBOR, _1),
332                       .allowZeroSizedInput = true);
333 
334 }  // namespace nn
335 }  // namespace android
336