1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Operations"
18 
19 #include <algorithm>
20 #include <vector>
21 
22 #include "OperationResolver.h"
23 #include "Operations.h"
24 #include "Tracing.h"
25 
26 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
27 #include <tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h>
28 #include <tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h>
29 
30 #include "CpuOperationUtils.h"
31 #endif  // NN_INCLUDE_CPU_IMPLEMENTATION
32 
33 namespace android {
34 namespace nn {
35 namespace depthwise_conv_2d {
36 constexpr char kOperationName[] = "DEPTHWISE_CONV_2D";
37 
38 constexpr uint32_t kNumInputsArray[] = {8, 9, 11, 12, 14};
39 constexpr uint32_t kInputTensor = 0;
40 constexpr uint32_t kFilterTensor = 1;
41 constexpr uint32_t kBiasTensor = 2;
42 
43 constexpr uint32_t kNumOutputs = 1;
44 constexpr uint32_t kOutputTensor = 0;
45 
46 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
47 namespace {
48 
49 struct DepthwiseConv2dParam {
50     int32_t padding_left, padding_right;
51     int32_t padding_top, padding_bottom;
52     int32_t stride_width, stride_height;
53     int32_t dilation_width_factor = 1, dilation_height_factor = 1;
54     int32_t depth_multiplier;
55     int32_t activation;
56     bool useNchw = false;
57 
initializeandroid::nn::depthwise_conv_2d::__anond5a607290110::DepthwiseConv2dParam58     bool initialize(const IOperationExecutionContext* context) {
59         uint32_t inCount = context->getNumInputs();
60         int32_t padding_implicit = 0;
61         bool useImplicitPadding = false;
62         if ((inCount >= 9 && context->getInputType(8) == OperandType::BOOL) || inCount == 8) {
63             padding_implicit = context->getInputValue<int32_t>(3);
64             stride_width = context->getInputValue<int32_t>(4);
65             stride_height = context->getInputValue<int32_t>(5);
66             depth_multiplier = context->getInputValue<int32_t>(6);
67             activation = context->getInputValue<int32_t>(7);
68             if (inCount >= 9) {
69                 useNchw = context->getInputValue<bool>(8);
70             }
71             if (inCount == 11) {
72                 dilation_width_factor = context->getInputValue<int32_t>(9);
73                 dilation_height_factor = context->getInputValue<int32_t>(10);
74             }
75             useImplicitPadding = true;
76         } else if (inCount >= 11 && context->getInputType(8) == OperandType::INT32) {
77             padding_left = context->getInputValue<int32_t>(3);
78             padding_right = context->getInputValue<int32_t>(4);
79             padding_top = context->getInputValue<int32_t>(5);
80             padding_bottom = context->getInputValue<int32_t>(6);
81             stride_width = context->getInputValue<int32_t>(7);
82             stride_height = context->getInputValue<int32_t>(8);
83             depth_multiplier = context->getInputValue<int32_t>(9);
84             activation = context->getInputValue<int32_t>(10);
85             if (inCount >= 12) {
86                 useNchw = context->getInputValue<bool>(11);
87             }
88             if (inCount == 14) {
89                 dilation_width_factor = context->getInputValue<int32_t>(12);
90                 dilation_height_factor = context->getInputValue<int32_t>(13);
91             }
92         } else {
93             NN_RET_CHECK_FAIL() << "Unsupported input spec for operation " << kOperationName;
94         }
95         if (useImplicitPadding) {
96             Shape inputShape = context->getInputShape(kInputTensor);
97             Shape filterShape = context->getInputShape(kFilterTensor);
98             int32_t input_width = getSizeOfDimension(inputShape, useNchw ? 3 : 2);
99             int32_t input_height = getSizeOfDimension(inputShape, useNchw ? 2 : 1);
100             int32_t filter_width = getSizeOfDimension(filterShape, 2);
101             int32_t filter_height = getSizeOfDimension(filterShape, 1);
102             calculateExplicitPadding(input_width, stride_width, dilation_width_factor, filter_width,
103                                      padding_implicit, &padding_left, &padding_right);
104             calculateExplicitPadding(input_height, stride_height, dilation_height_factor,
105                                      filter_height, padding_implicit, &padding_top,
106                                      &padding_bottom);
107         }
108         NN_RET_CHECK_GE(padding_left, 0);
109         NN_RET_CHECK_GE(padding_right, 0);
110         NN_RET_CHECK_GE(padding_top, 0);
111         NN_RET_CHECK_GE(padding_bottom, 0);
112         NN_RET_CHECK_GT(stride_width, 0);
113         NN_RET_CHECK_GT(stride_height, 0);
114         NN_RET_CHECK_GT(dilation_width_factor, 0);
115         NN_RET_CHECK_GT(dilation_height_factor, 0);
116         NN_RET_CHECK_GT(depth_multiplier, 0);
117         NN_RET_CHECK_GE(activation, 0);
118         return true;
119     }
120 };
121 
122 #define ANDROID_NN_DEPTHWISE_CONV_PARAMETERS                    \
123     uint32_t height = getSizeOfDimension(inputShape, 1);        \
124     uint32_t width = getSizeOfDimension(inputShape, 2);         \
125     uint32_t filterHeight = getSizeOfDimension(filterShape, 1); \
126     uint32_t filterWidth = getSizeOfDimension(filterShape, 2);  \
127     uint32_t outHeight = getSizeOfDimension(outputShape, 1);    \
128     uint32_t outWidth = getSizeOfDimension(outputShape, 2);     \
129                                                                 \
130     uint32_t paddingHeight = (uint32_t)paddingTop;              \
131     uint32_t paddingWidth = (uint32_t)paddingLeft;
132 
depthwiseConvNhwc(const float * inputData,const Shape & inputShape,const float * filterData,const Shape & filterShape,const float * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t depthMultiplier,int32_t activation,float * outputData,const Shape & outputShape)133 bool depthwiseConvNhwc(const float* inputData, const Shape& inputShape, const float* filterData,
134                        const Shape& filterShape, const float* biasData, const Shape& biasShape,
135                        int32_t paddingLeft, int32_t paddingRight, int32_t paddingTop,
136                        int32_t paddingBottom, int32_t strideWidth, int32_t strideHeight,
137                        int32_t dilationWidthFactor, int32_t dilationHeightFactor,
138                        int32_t depthMultiplier, int32_t activation, float* outputData,
139                        const Shape& outputShape) {
140     NNTRACE_TRANS("depthwiseConvFloat32");
141 
142     ANDROID_NN_DEPTHWISE_CONV_PARAMETERS
143 
144     float output_activation_min, output_activation_max;
145     CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max);
146 
147     tflite::DepthwiseParams params{
148             .padding_values = {static_cast<int16>(paddingWidth), static_cast<int16>(paddingHeight),
149                                0 /*width_offset*/, 0 /*height_offset*/},
150             .stride_width = static_cast<int16>(strideWidth),
151             .stride_height = static_cast<int16>(strideHeight),
152             .dilation_width_factor = static_cast<int16>(dilationWidthFactor),
153             .dilation_height_factor = static_cast<int16>(dilationHeightFactor),
154             .depth_multiplier = static_cast<int16>(depthMultiplier),
155             .float_activation_min = output_activation_min,
156             .float_activation_max = output_activation_max,
157     };
158     NNTRACE_COMP_SWITCH("optimized_ops::DepthwiseConv");
159     tflite::reference_ops::DepthwiseConv(params, convertShapeToTflshape(inputShape), inputData,
160                                          convertShapeToTflshape(filterShape), filterData,
161                                          convertShapeToTflshape(biasShape), biasData,
162                                          convertShapeToTflshape(outputShape), outputData);
163 
164     return true;
165 }
166 
depthwiseConvNhwc(const _Float16 * inputData,const Shape & inputShape,const _Float16 * filterData,const Shape & filterShape,const _Float16 * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t depthMultiplier,int32_t activation,_Float16 * outputData,const Shape & outputShape)167 bool depthwiseConvNhwc(const _Float16* inputData, const Shape& inputShape,
168                        const _Float16* filterData, const Shape& filterShape,
169                        const _Float16* biasData, const Shape& biasShape, int32_t paddingLeft,
170                        int32_t paddingRight, int32_t paddingTop, int32_t paddingBottom,
171                        int32_t strideWidth, int32_t strideHeight, int32_t dilationWidthFactor,
172                        int32_t dilationHeightFactor, int32_t depthMultiplier, int32_t activation,
173                        _Float16* outputData, const Shape& outputShape) {
174     NNTRACE_TRANS("depthwiseConvFloat16");
175     std::vector<float> inputDataFloat32(getNumberOfElements(inputShape));
176     convertFloat16ToFloat32(inputData, &inputDataFloat32);
177     std::vector<float> filterDataFloat32(getNumberOfElements(filterShape));
178     convertFloat16ToFloat32(filterData, &filterDataFloat32);
179     std::vector<float> biasDataFloat32(getNumberOfElements(biasShape));
180     convertFloat16ToFloat32(biasData, &biasDataFloat32);
181 
182     std::vector<float> outputDataFloat32(getNumberOfElements(outputShape));
183     depthwiseConvNhwc(inputDataFloat32.data(), inputShape, filterDataFloat32.data(), filterShape,
184                       biasDataFloat32.data(), biasShape, paddingLeft, paddingRight, paddingTop,
185                       paddingBottom, strideWidth, strideHeight, dilationWidthFactor,
186                       dilationHeightFactor, depthMultiplier, activation, outputDataFloat32.data(),
187                       outputShape);
188 
189     convertFloat32ToFloat16(outputDataFloat32, outputData);
190     return true;
191 }
192 
depthwiseConvNhwc(const uint8_t * inputData,const Shape & inputShape,const uint8_t * filterData,const Shape & filterShape,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t depthMultiplier,int32_t activation,uint8_t * outputData,const Shape & outputShape)193 bool depthwiseConvNhwc(const uint8_t* inputData, const Shape& inputShape, const uint8_t* filterData,
194                        const Shape& filterShape, const int32_t* biasData, const Shape& biasShape,
195                        int32_t paddingLeft, int32_t paddingRight, int32_t paddingTop,
196                        int32_t paddingBottom, int32_t strideWidth, int32_t strideHeight,
197                        int32_t dilationWidthFactor, int32_t dilationHeightFactor,
198                        int32_t depthMultiplier, int32_t activation, uint8_t* outputData,
199                        const Shape& outputShape) {
200     NNTRACE_TRANS("depthwiseConvQuant8");
201 
202     ANDROID_NN_DEPTHWISE_CONV_PARAMETERS
203 
204     double real_multiplier = 0.0;
205     int32_t output_multiplier = 0;
206     int32_t output_shift = 0;
207     int32_t output_activation_min = 0;
208     int32_t output_activation_max = 0;
209 
210     NN_RET_CHECK(GetQuantizedConvolutionMultipler(inputShape, filterShape, biasShape, outputShape,
211                                                   &real_multiplier));
212     int exponent;
213     NN_RET_CHECK(QuantizeMultiplier(real_multiplier, &output_multiplier, &exponent));
214     output_shift = -exponent;
215     CalculateActivationRangeUint8(activation, outputShape, &output_activation_min,
216                                   &output_activation_max);
217 
218     tflite::DepthwiseParams params{
219             .padding_values = {static_cast<int16>(paddingWidth), static_cast<int16>(paddingHeight),
220                                0 /*width_offset*/, 0 /*height_offset*/},
221             .stride_width = static_cast<int16>(strideWidth),
222             .stride_height = static_cast<int16>(strideHeight),
223             .dilation_width_factor = static_cast<int16>(dilationWidthFactor),
224             .dilation_height_factor = static_cast<int16>(dilationHeightFactor),
225             .depth_multiplier = static_cast<int16>(depthMultiplier),
226             .input_offset = -inputShape.offset,
227             .weights_offset = -filterShape.offset,
228             .output_offset = outputShape.offset,
229             .output_multiplier = output_multiplier,
230             .output_shift = -output_shift,
231             .quantized_activation_min = output_activation_min,
232             .quantized_activation_max = output_activation_max,
233     };
234     NNTRACE_COMP_SWITCH("optimized_ops::DepthwiseConv");
235     tflite::reference_ops::DepthwiseConv(params, convertShapeToTflshape(inputShape), inputData,
236                                          convertShapeToTflshape(filterShape), filterData,
237                                          convertShapeToTflshape(biasShape), biasData,
238                                          convertShapeToTflshape(outputShape), outputData);
239     return true;
240 }
241 
242 // Passing input, filter and output shapes by value, so that we can change the
243 // offsets without modifying the actual shapes.
depthwiseConvNhwc(const int8_t * inputData,Shape inputShape,const int8_t * filterData,Shape filterShape,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t depthMultiplier,int32_t activation,int8_t * outputData,Shape outputShape)244 bool depthwiseConvNhwc(const int8_t* inputData, Shape inputShape, const int8_t* filterData,
245                        Shape filterShape, const int32_t* biasData, const Shape& biasShape,
246                        int32_t paddingLeft, int32_t paddingRight, int32_t paddingTop,
247                        int32_t paddingBottom, int32_t strideWidth, int32_t strideHeight,
248                        int32_t dilationWidthFactor, int32_t dilationHeightFactor,
249                        int32_t depthMultiplier, int32_t activation, int8_t* outputData,
250                        Shape outputShape) {
251     NNTRACE_TRANS("depthwiseConvQuant8");
252 
253     std::vector<uint8_t> unsignedInput(getNumberOfElements(inputShape));
254     convertInt8ToUInt8(inputData, &unsignedInput);
255     inputShape.offset += 128;
256 
257     std::vector<uint8_t> unsignedFilter(getNumberOfElements(filterShape));
258     convertInt8ToUInt8(filterData, &unsignedFilter);
259     filterShape.offset += 128;
260 
261     std::vector<uint8_t> unsignedOutput(getNumberOfElements(outputShape));
262     outputShape.offset += 128;
263 
264     NN_RET_CHECK(depthwiseConvNhwc(unsignedInput.data(), inputShape, unsignedFilter.data(),
265                                    filterShape, biasData, biasShape, paddingLeft, paddingRight,
266                                    paddingTop, paddingBottom, strideWidth, strideHeight,
267                                    dilationWidthFactor, dilationHeightFactor, depthMultiplier,
268                                    activation, unsignedOutput.data(), outputShape));
269 
270     convertUInt8ToInt8(unsignedOutput, outputData);
271 
272     return true;
273 }
274 
275 template <typename T>
depthwiseConvQuant8PerChannelNhwc(const T * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t depthMultiplier,int32_t activation,T * outputData,const Shape & outputShape)276 bool depthwiseConvQuant8PerChannelNhwc(
277         const T* inputData, const Shape& inputShape, const int8_t* filterData,
278         const Shape& filterShape, const float* filterScales, const int32_t* biasData,
279         const Shape& biasShape, int32_t paddingLeft, int32_t paddingRight, int32_t paddingTop,
280         int32_t paddingBottom, int32_t strideWidth, int32_t strideHeight,
281         int32_t dilationWidthFactor, int32_t dilationHeightFactor,
282 
283         int32_t depthMultiplier, int32_t activation, T* outputData, const Shape& outputShape) {
284     NNTRACE_TRANS("depthwiseConvQuant8");
285 
286     uint32_t paddingHeight = (uint32_t)paddingTop;
287     uint32_t paddingWidth = (uint32_t)paddingLeft;
288 
289     uint32_t numBatches = getSizeOfDimension(inputShape, 0);
290     uint32_t inputHeight = getSizeOfDimension(inputShape, 1);
291     uint32_t inputWidth = getSizeOfDimension(inputShape, 2);
292     uint32_t inputDepth = getSizeOfDimension(inputShape, 3);
293     uint32_t filterHeight = getSizeOfDimension(filterShape, 1);
294     uint32_t filterWidth = getSizeOfDimension(filterShape, 2);
295     uint32_t filterDepth = getSizeOfDimension(filterShape, 3);
296     uint32_t outputHeight = getSizeOfDimension(outputShape, 1);
297     uint32_t outputWidth = getSizeOfDimension(outputShape, 2);
298     uint32_t outputDepth = getSizeOfDimension(outputShape, 3);
299 
300     int32_t inputOffset = -inputShape.offset;
301     int32_t outputOffset = outputShape.offset;
302 
303     auto realMultiplier = std::vector<double>(outputDepth, .0f);
304     auto outputMultiplier = std::vector<int32_t>(outputDepth, 0);
305     auto outputShift = std::vector<int32_t>(outputDepth, .0f);
306 
307     for (int i = 0; i < outputDepth; ++i) {
308         Shape filterChannelShape = filterShape;
309         filterChannelShape.scale = filterScales[i];
310         Shape biasChannelShape = biasShape;
311         biasChannelShape.scale = filterScales[i] * inputShape.scale;
312         NN_RET_CHECK(GetQuantizedConvolutionMultipler(
313                 inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i]));
314         int exponent;
315         NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &exponent));
316         outputShift[i] = -exponent;
317     }
318 
319     int32_t output_activation_min = 0, output_activation_max = 0;
320     CalculateActivationRange<T>(activation, outputShape, &output_activation_min,
321                                 &output_activation_max);
322 
323     const T* inputBase = inputData;
324     T* outPtr = outputData;
325     for (uint32_t b = 0; b < numBatches; b++) {
326         for (uint32_t h = 0; h < outputHeight; h++) {
327             for (uint32_t w = 0; w < outputWidth; w++) {
328                 for (uint32_t ic = 0; ic < inputDepth; ic++) {
329                     for (uint32_t m = 0; m < depthMultiplier; m++) {
330                         int32_t wInputOrigin = static_cast<int32_t>(w) * strideWidth - paddingLeft;
331                         int32_t hInputOrigin = static_cast<int32_t>(h) * strideHeight - paddingTop;
332                         const int oc = m + ic * depthMultiplier;
333 
334                         int32_t sum = 0.0f;
335                         for (uint32_t i = 0; i < filterHeight; i++) {
336                             for (uint32_t j = 0; j < filterWidth; j++) {
337                                 int32_t hInput = hInputOrigin +
338                                                  dilationHeightFactor * static_cast<int32_t>(i);
339                                 int32_t wInput = wInputOrigin +
340                                                  dilationWidthFactor * static_cast<int32_t>(j);
341 
342                                 if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) &&
343                                     wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) {
344                                     uint32_t filterIndex =
345                                             i * filterWidth * filterDepth + j * filterDepth + oc;
346                                     uint32_t inputIndex = hInput * inputWidth * inputDepth +
347                                                           wInput * inputDepth + ic;
348                                     sum += (static_cast<int32_t>(filterData[filterIndex])) *
349                                            (static_cast<int32_t>(inputBase[inputIndex]) +
350                                             inputOffset);
351                                 }
352                             }
353                         }
354 
355                         sum += biasData[oc];
356                         sum = tflite::MultiplyByQuantizedMultiplier(sum, outputMultiplier[oc],
357                                                                     -outputShift[oc]);
358                         sum += outputOffset;
359                         sum = std::max(std::min(sum, output_activation_max), output_activation_min);
360                         outPtr[m] = static_cast<T>(sum);
361                     }
362                     outPtr += depthMultiplier;
363                 }
364             }
365         }
366         inputBase += inputHeight * inputWidth * inputDepth;
367     }
368 
369     return true;
370 }
371 
372 template <typename T_Input, typename T_Filter, typename T_Bias>
depthwiseConv(const T_Input * inputData,const Shape & inputShape,const T_Filter * filterData,const Shape & filterShape,const T_Bias * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t depthMultiplier,int32_t activation,bool useNchw,T_Input * outputData,const Shape & outputShape)373 bool depthwiseConv(const T_Input* inputData, const Shape& inputShape, const T_Filter* filterData,
374                    const Shape& filterShape, const T_Bias* biasData, const Shape& biasShape,
375                    int32_t paddingLeft, int32_t paddingRight, int32_t paddingTop,
376                    int32_t paddingBottom, int32_t strideWidth, int32_t strideHeight,
377                    int32_t dilationWidthFactor, int32_t dilationHeightFactor,
378                    int32_t depthMultiplier, int32_t activation, bool useNchw, T_Input* outputData,
379                    const Shape& outputShape) {
380     InputWithLayout<T_Input> input(useNchw);
381     OutputWithLayout<T_Input> output(useNchw);
382     NN_RET_CHECK(input.initialize(inputData, inputShape));
383     NN_RET_CHECK(output.initialize(outputData, outputShape));
384     NN_RET_CHECK(depthwiseConvNhwc(input.getNhwcBuffer(), input.getNhwcShape(), filterData,
385                                    filterShape, biasData, biasShape, paddingLeft, paddingRight,
386                                    paddingTop, paddingBottom, strideWidth, strideHeight,
387                                    dilationWidthFactor, dilationHeightFactor, depthMultiplier,
388                                    activation, output.getNhwcBuffer(), output.getNhwcShape()));
389     NN_RET_CHECK(output.commit());
390     return true;
391 }
392 
393 template <typename T>
depthwiseConvQuant8PerChannel(const T * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t depthMultiplier,int32_t activation,bool useNchw,T * outputData,const Shape & outputShape)394 bool depthwiseConvQuant8PerChannel(const T* inputData, const Shape& inputShape,
395                                    const int8_t* filterData, const Shape& filterShape,
396                                    const float* filterScales, const int32_t* biasData,
397                                    const Shape& biasShape, int32_t paddingLeft,
398                                    int32_t paddingRight, int32_t paddingTop, int32_t paddingBottom,
399                                    int32_t strideWidth, int32_t strideHeight,
400                                    int32_t dilationWidthFactor, int32_t dilationHeightFactor,
401                                    int32_t depthMultiplier, int32_t activation, bool useNchw,
402                                    T* outputData, const Shape& outputShape) {
403     InputWithLayout<T> input(useNchw);
404     OutputWithLayout<T> output(useNchw);
405     NN_RET_CHECK(input.initialize(inputData, inputShape));
406     NN_RET_CHECK(output.initialize(outputData, outputShape));
407     NN_RET_CHECK(depthwiseConvQuant8PerChannelNhwc(
408             input.getNhwcBuffer(), input.getNhwcShape(), filterData, filterShape, filterScales,
409             biasData, biasShape, paddingLeft, paddingRight, paddingTop, paddingBottom, strideWidth,
410             strideHeight, dilationWidthFactor, dilationHeightFactor, depthMultiplier, activation,
411             output.getNhwcBuffer(), output.getNhwcShape()));
412     NN_RET_CHECK(output.commit());
413     return true;
414 }
415 
416 #undef ANDROID_NN_DEPTHWISE_CONV_PARAMETERS
417 
418 }  // namespace
419 #endif  // NN_INCLUDE_CPU_IMPLEMENTATION
420 
validate(const IOperationValidationContext * context)421 Result<Version> validate(const IOperationValidationContext* context) {
422     const uint32_t numInputs = context->getNumInputs();
423     NN_RET_CHECK(
424             std::binary_search(std::begin(kNumInputsArray), std::end(kNumInputsArray), numInputs));
425     NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs);
426     auto inputType = context->getInputType(kInputTensor);
427     auto filterType = context->getInputType(kFilterTensor);
428     std::vector<OperandType> inExpectedTypes;
429     if (inputType == OperandType::TENSOR_FLOAT32) {
430         inExpectedTypes = {
431                 OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32,
432                 OperandType::TENSOR_FLOAT32, OperandType::INT32,
433                 OperandType::INT32,          OperandType::INT32,
434                 OperandType::INT32,          OperandType::INT32,
435         };
436     } else if (inputType == OperandType::TENSOR_FLOAT16) {
437         inExpectedTypes = {
438                 OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16,
439                 OperandType::TENSOR_FLOAT16, OperandType::INT32,
440                 OperandType::INT32,          OperandType::INT32,
441                 OperandType::INT32,          OperandType::INT32,
442         };
443     } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM ||
444                inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
445         NN_RET_CHECK(filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL ||
446                      filterType == inputType)
447                 << "Unsupported filter tensor type for operation " << kOperationName;
448         if (filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
449             NN_RET_CHECK_EQ(std::get<Operand::SymmPerChannelQuantParams>(
450                                     context->getInputExtraParams(kFilterTensor))
451                                     .channelDim,
452                             3)
453                     << "Unsupported filter tensor channel dimension for operation "
454                     << kOperationName;
455         }
456         inExpectedTypes = {
457                 inputType,          filterType,         OperandType::TENSOR_INT32,
458                 OperandType::INT32, OperandType::INT32, OperandType::INT32,
459                 OperandType::INT32, OperandType::INT32,
460         };
461     } else {
462         NN_RET_CHECK_FAIL() << "Unsupported input tensor type for operation " << kOperationName;
463     }
464 
465     // NeuralNetworks.h specifies that ANEURALNETWORKS_DEPTHWISE_CONV_2D's output must
466     // meet "outputScale > inputScale * filterScale" for the operand type
467     // ANEURALNETWORKS_TENSOR_QUANT8_ASYMM before API level 29. For other
468     // operand types (e.g., ANEURALNETWORKS_TENSOR_FLOAT32), this constraint
469     // does not apply, so by default the constraint is met.
470     bool meetsQuantizedScaleConstraintBeforeV1_2 = true;
471     if (inputType == OperandType::TENSOR_QUANT8_ASYMM) {
472         const float inputScale = context->getInputShape(kInputTensor).scale;
473         const float filterScale = context->getInputShape(kFilterTensor).scale;
474         const float outputScale = context->getInputShape(kOutputTensor).scale;
475         meetsQuantizedScaleConstraintBeforeV1_2 = (outputScale > inputScale * filterScale);
476     }
477 
478     bool withExplicitPadding = false;
479     bool withLayout = false;
480     bool withDilation = false;
481     if (numInputs >= 9) {
482         if (context->getInputType(8) == OperandType::INT32 && numInputs >= 11) {
483             std::vector<OperandType> explicitScalarTypes(3, OperandType::INT32);
484             inExpectedTypes.insert(inExpectedTypes.end(), explicitScalarTypes.begin(),
485                                    explicitScalarTypes.end());
486             withExplicitPadding = true;
487         }
488         int inputOffset = withExplicitPadding ? 3 : 0;
489         if (numInputs >= 9 + inputOffset) {
490             inExpectedTypes.push_back(OperandType::BOOL);
491             withLayout = true;
492         }
493         NN_RET_CHECK_NE(numInputs, 10 + inputOffset)
494                 << "Provided only one dilation factor value, two values are required for operation "
495                 << kOperationName;
496         if (numInputs == 11 + inputOffset) {
497             inExpectedTypes.push_back(OperandType::INT32);
498             inExpectedTypes.push_back(OperandType::INT32);
499             withDilation = true;
500         }
501     }
502 
503     auto minSupportedVersion = Version::ANDROID_OC_MR1;
504     if (inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
505         minSupportedVersion = Version::ANDROID_R;
506     } else if (inputType == OperandType::TENSOR_FLOAT16 ||
507                filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL || withLayout ||
508                withDilation || !meetsQuantizedScaleConstraintBeforeV1_2) {
509         minSupportedVersion = Version::ANDROID_Q;
510     } else {
511         minSupportedVersion = Version::ANDROID_OC_MR1;
512     }
513     NN_RET_CHECK(validateInputTypes(context, inExpectedTypes));
514     NN_RET_CHECK(validateOutputTypes(context, {inputType}));
515     return minSupportedVersion;
516 }
517 
518 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
prepare(IOperationExecutionContext * context)519 bool prepare(IOperationExecutionContext* context) {
520     Shape input = context->getInputShape(kInputTensor);
521     Shape filter = context->getInputShape(kFilterTensor);
522     Shape bias = context->getInputShape(kBiasTensor);
523 
524     if (filter.type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
525         NN_RET_CHECK(input.type == OperandType::TENSOR_QUANT8_ASYMM ||
526                      input.type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED);
527     } else {
528         NN_RET_CHECK(input.type == filter.type);
529     }
530     if (input.type == OperandType::TENSOR_QUANT8_ASYMM ||
531         input.type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
532         NN_RET_CHECK(bias.type == OperandType::TENSOR_INT32);
533     } else {
534         NN_RET_CHECK(input.type == bias.type);
535     }
536     NN_RET_CHECK_EQ(getNumberOfDimensions(input), 4);
537     NN_RET_CHECK_EQ(getNumberOfDimensions(filter), 4);
538     NN_RET_CHECK_EQ(getNumberOfDimensions(bias), 1);
539     NN_RET_CHECK_EQ(getSizeOfDimension(filter, 0), 1);
540     NN_RET_CHECK_EQ(getSizeOfDimension(filter, 3), getSizeOfDimension(bias, 0));
541 
542     DepthwiseConv2dParam param;
543     NN_RET_CHECK(param.initialize(context));
544 
545     uint32_t batches = getSizeOfDimension(input, 0);
546     uint32_t height = getSizeOfDimension(input, param.useNchw ? 2 : 1);
547     uint32_t width = getSizeOfDimension(input, param.useNchw ? 3 : 2);
548     uint32_t channels_in = getSizeOfDimension(input, param.useNchw ? 1 : 3);
549     uint32_t channels_out = getSizeOfDimension(filter, 3);
550     uint32_t filterHeight = getSizeOfDimension(filter, 1);
551     uint32_t filterWidth = getSizeOfDimension(filter, 2);
552 
553     NN_OPS_CHECK(param.depth_multiplier * channels_in == channels_out);
554     int32_t effectiveFilterWidth = (filterWidth - 1) * param.dilation_width_factor + 1;
555     int32_t effectiveFilterHeight = (filterHeight - 1) * param.dilation_height_factor + 1;
556     NN_RET_CHECK_GT(effectiveFilterWidth, param.padding_left);
557     NN_RET_CHECK_GT(effectiveFilterWidth, param.padding_right);
558     NN_RET_CHECK_GT(effectiveFilterHeight, param.padding_top);
559     NN_RET_CHECK_GT(effectiveFilterHeight, param.padding_bottom);
560 
561     uint32_t outHeight =
562             computeOutSize(height, filterHeight, param.stride_height, param.dilation_height_factor,
563                            param.padding_top, param.padding_bottom);
564     uint32_t outWidth =
565             computeOutSize(width, filterWidth, param.stride_width, param.dilation_width_factor,
566                            param.padding_left, param.padding_right);
567 
568     Shape output = context->getOutputShape(kOutputTensor);
569     output.type = input.type;
570     if (param.useNchw) {
571         output.dimensions = {batches, channels_out, outHeight, outWidth};
572     } else {
573         output.dimensions = {batches, outHeight, outWidth, channels_out};
574     }
575     return context->setOutputShape(kOutputTensor, output);
576 }
577 
execute(IOperationExecutionContext * context)578 bool execute(IOperationExecutionContext* context) {
579     // Bypass execution in the case of zero-sized input.
580     if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true;
581     DepthwiseConv2dParam param;
582     NN_RET_CHECK(param.initialize(context));
583     switch (context->getInputType(kInputTensor)) {
584         case OperandType::TENSOR_FLOAT32:
585             return depthwiseConv(context->getInputBuffer<float>(kInputTensor),
586                                  context->getInputShape(kInputTensor),
587                                  context->getInputBuffer<float>(kFilterTensor),
588                                  context->getInputShape(kFilterTensor),
589                                  context->getInputBuffer<float>(kBiasTensor),
590                                  context->getInputShape(kBiasTensor), param.padding_left,
591                                  param.padding_right, param.padding_top, param.padding_bottom,
592                                  param.stride_width, param.stride_height,
593                                  param.dilation_width_factor, param.dilation_height_factor,
594                                  param.depth_multiplier, param.activation, param.useNchw,
595                                  context->getOutputBuffer<float>(kOutputTensor),
596                                  context->getOutputShape(kOutputTensor));
597         case OperandType::TENSOR_FLOAT16:
598             return depthwiseConv(context->getInputBuffer<_Float16>(kInputTensor),
599                                  context->getInputShape(kInputTensor),
600                                  context->getInputBuffer<_Float16>(kFilterTensor),
601                                  context->getInputShape(kFilterTensor),
602                                  context->getInputBuffer<_Float16>(kBiasTensor),
603                                  context->getInputShape(kBiasTensor), param.padding_left,
604                                  param.padding_right, param.padding_top, param.padding_bottom,
605                                  param.stride_width, param.stride_height,
606                                  param.dilation_width_factor, param.dilation_height_factor,
607                                  param.depth_multiplier, param.activation, param.useNchw,
608                                  context->getOutputBuffer<_Float16>(kOutputTensor),
609                                  context->getOutputShape(kOutputTensor));
610         case OperandType::TENSOR_QUANT8_ASYMM:
611             if (context->getInputType(kFilterTensor) ==
612                 OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
613                 return depthwiseConvQuant8PerChannel(
614                         context->getInputBuffer<uint8_t>(kInputTensor),
615                         context->getInputShape(kInputTensor),
616                         context->getInputBuffer<int8_t>(kFilterTensor),
617                         context->getInputShape(kFilterTensor),
618                         std::get<Operand::SymmPerChannelQuantParams>(
619                                 context->getInputExtraParams(kFilterTensor))
620                                 .scales.data(),
621                         context->getInputBuffer<int32_t>(kBiasTensor),
622                         context->getInputShape(kBiasTensor), param.padding_left,
623                         param.padding_right, param.padding_top, param.padding_bottom,
624                         param.stride_width, param.stride_height, param.dilation_width_factor,
625                         param.dilation_height_factor, param.depth_multiplier, param.activation,
626                         param.useNchw, context->getOutputBuffer<uint8_t>(kOutputTensor),
627                         context->getOutputShape(kOutputTensor));
628             } else if (context->getInputType(kFilterTensor) == OperandType::TENSOR_QUANT8_ASYMM) {
629                 return depthwiseConv(context->getInputBuffer<uint8_t>(kInputTensor),
630                                      context->getInputShape(kInputTensor),
631                                      context->getInputBuffer<uint8_t>(kFilterTensor),
632                                      context->getInputShape(kFilterTensor),
633                                      context->getInputBuffer<int32_t>(kBiasTensor),
634                                      context->getInputShape(kBiasTensor), param.padding_left,
635                                      param.padding_right, param.padding_top, param.padding_bottom,
636                                      param.stride_width, param.stride_height,
637                                      param.dilation_width_factor, param.dilation_height_factor,
638                                      param.depth_multiplier, param.activation, param.useNchw,
639                                      context->getOutputBuffer<uint8_t>(kOutputTensor),
640                                      context->getOutputShape(kOutputTensor));
641             } else {
642                 NN_RET_CHECK_FAIL() << "Unsupported filter type for operation " << kOperationName;
643             }
644         case OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
645             if (context->getInputType(kFilterTensor) ==
646                 OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
647                 return depthwiseConvQuant8PerChannel(
648                         context->getInputBuffer<int8_t>(kInputTensor),
649                         context->getInputShape(kInputTensor),
650                         context->getInputBuffer<int8_t>(kFilterTensor),
651                         context->getInputShape(kFilterTensor),
652                         std::get<Operand::SymmPerChannelQuantParams>(
653                                 context->getInputExtraParams(kFilterTensor))
654                                 .scales.data(),
655                         context->getInputBuffer<int32_t>(kBiasTensor),
656                         context->getInputShape(kBiasTensor), param.padding_left,
657                         param.padding_right, param.padding_top, param.padding_bottom,
658                         param.stride_width, param.stride_height, param.dilation_width_factor,
659                         param.dilation_height_factor, param.depth_multiplier, param.activation,
660                         param.useNchw, context->getOutputBuffer<int8_t>(kOutputTensor),
661                         context->getOutputShape(kOutputTensor));
662             } else if (context->getInputType(kFilterTensor) ==
663                        OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
664                 return depthwiseConv(context->getInputBuffer<int8_t>(kInputTensor),
665                                      context->getInputShape(kInputTensor),
666                                      context->getInputBuffer<int8_t>(kFilterTensor),
667                                      context->getInputShape(kFilterTensor),
668                                      context->getInputBuffer<int32_t>(kBiasTensor),
669                                      context->getInputShape(kBiasTensor), param.padding_left,
670                                      param.padding_right, param.padding_top, param.padding_bottom,
671                                      param.stride_width, param.stride_height,
672                                      param.dilation_width_factor, param.dilation_height_factor,
673                                      param.depth_multiplier, param.activation, param.useNchw,
674                                      context->getOutputBuffer<int8_t>(kOutputTensor),
675                                      context->getOutputShape(kOutputTensor));
676             } else {
677                 NN_RET_CHECK_FAIL() << "Unsupported filter type for operation " << kOperationName;
678             }
679         default:
680             NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName;
681     }
682 }
683 #endif  // NN_INCLUDE_CPU_IMPLEMENTATION
684 
685 }  // namespace depthwise_conv_2d
686 
687 NN_REGISTER_OPERATION(DEPTHWISE_CONV_2D, depthwise_conv_2d::kOperationName,
688                       depthwise_conv_2d::validate, depthwise_conv_2d::prepare,
689                       depthwise_conv_2d::execute, .allowZeroSizedInput = true);
690 
691 }  // namespace nn
692 }  // namespace android
693