1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Operations"
18 
19 #include "SVDF.h"
20 
21 #include <algorithm>
22 #include <vector>
23 
24 #include "CpuExecutor.h"
25 #include "CpuOperationUtils.h"
26 #include "Tracing.h"
27 
28 namespace android {
29 namespace nn {
30 
SVDF(const Operation & operation,RunTimeOperandInfo * operands)31 SVDF::SVDF(const Operation& operation, RunTimeOperandInfo* operands) {
32     NNTRACE_TRANS("SVDF::SVDF");
33     input_ = GetInput(operation, operands, kInputTensor);
34     weights_feature_ = GetInput(operation, operands, kWeightsFeatureTensor);
35     weights_time_ = GetInput(operation, operands, kWeightsTimeTensor);
36     bias_ = GetInput(operation, operands, kBiasTensor);
37     state_in_ = GetInput(operation, operands, kStateInTensor);
38 
39     const auto& rankOperand = *GetInput(operation, operands, kRankParam);
40     params_.rank_ = getScalarDataWithDefault<int>(rankOperand, 0);
41     const auto& activationOperand = *GetInput(operation, operands, kActivationParam);
42     params_.activation_ = static_cast<TfLiteFusedActivation>(getScalarDataWithDefault<int>(
43             activationOperand, TfLiteFusedActivation::kTfLiteActNone));
44 
45     state_out_ = GetOutput(operation, operands, kStateOutTensor);
46     output_ = GetOutput(operation, operands, kOutputTensor);
47 }
48 
Prepare(const Operation & operation,RunTimeOperandInfo * operands,Shape * stateShape,Shape * outputShape)49 bool SVDF::Prepare(const Operation& operation, RunTimeOperandInfo* operands, Shape* stateShape,
50                    Shape* outputShape) {
51     NNTRACE_TRANS("SVDF::Prepare");
52     // Check we have all the inputs and outputs we need.
53     const int num_inputs = NumInputsWithValues(operation, operands);
54 
55     NN_CHECK(num_inputs == 6 || num_inputs == 7);
56     constexpr int requiredInputs[] = {
57             kInputTensor, kWeightsFeatureTensor, kWeightsTimeTensor, kStateInTensor,
58             kRankParam,   kActivationParam,
59     };
60     for (const int requiredInput : requiredInputs) {
61         NN_RET_CHECK(!IsNullInput(GetInput(operation, operands, requiredInput)))
62                 << "required input " << requiredInput << " is omitted";
63     }
64     NN_CHECK_EQ(NumOutputs(operation), 2);
65 
66     // Check that the scalar operands' buffers are large enough.
67     const auto& rankOperand = *GetInput(operation, operands, kRankParam);
68     NN_RET_CHECK(rankOperand.length >= sizeof(int));
69     const auto& activationOperand = *GetInput(operation, operands, kActivationParam);
70     NN_RET_CHECK(activationOperand.length >= sizeof(int));
71 
72     const RunTimeOperandInfo* input = GetInput(operation, operands, SVDF::kInputTensor);
73     const RunTimeOperandInfo* weights_feature =
74             GetInput(operation, operands, SVDF::kWeightsFeatureTensor);
75     const RunTimeOperandInfo* weights_time =
76             GetInput(operation, operands, SVDF::kWeightsTimeTensor);
77 
78     // Check all the parameters of tensor match within themselves and match the
79     // input configuration.
80     const int rank = getScalarData<int>(*GetInput(operation, operands, kRankParam));
81     const uint32_t batch_size = SizeOfDimension(input, 0);
82     const uint32_t num_filters = SizeOfDimension(weights_feature, 0);
83     NN_CHECK_EQ(num_filters % rank, 0);
84     const uint32_t num_units = num_filters / rank;
85     const uint32_t memory_size = SizeOfDimension(weights_time, 1);
86     NN_CHECK_EQ(SizeOfDimension(input, 1), SizeOfDimension(weights_feature, 1));
87     NN_CHECK_EQ(SizeOfDimension(weights_time, 0), num_filters);
88 
89     const RunTimeOperandInfo* bias = GetInput(operation, operands, kBiasTensor);
90     if (!IsNullInput(bias)) {
91         NN_CHECK_EQ(SizeOfDimension(bias, 0), num_units);
92     }
93 
94     // Resize state.
95     const Shape& inputShape = input->shape();
96     stateShape->type = inputShape.type;
97     stateShape->dimensions = {batch_size, memory_size * num_filters};
98     stateShape->offset = inputShape.offset;
99     stateShape->scale = inputShape.scale;
100 
101     // Resize output.
102     outputShape->type = inputShape.type;
103     outputShape->dimensions = {batch_size, num_units};
104     outputShape->offset = inputShape.offset;
105     outputShape->scale = inputShape.scale;
106 
107     return true;
108 }
109 
Eval()110 bool SVDF::Eval() {
111     NNTRACE_TRANS("SVDF::Eval");
112     switch (input_->type) {
113         case OperandType::TENSOR_FLOAT16: {
114             std::vector<float> inputDataFloat32(getNumberOfElements(input_->shape()));
115             convertFloat16ToFloat32(reinterpret_cast<_Float16*>(input_->buffer), &inputDataFloat32);
116             std::vector<float> inputStateDataFloat32(getNumberOfElements(state_in_->shape()));
117             convertFloat16ToFloat32(reinterpret_cast<_Float16*>(state_in_->buffer),
118                                     &inputStateDataFloat32);
119             std::vector<float> biasDataFloat32(getNumberOfElements(bias_->shape()));
120             if (!IsNullInput(bias_)) {
121                 convertFloat16ToFloat32(reinterpret_cast<_Float16*>(bias_->buffer),
122                                         &biasDataFloat32);
123             }
124             std::vector<float> weightsFeatureDataFloat32(
125                     getNumberOfElements(weights_feature_->shape()));
126             convertFloat16ToFloat32(reinterpret_cast<_Float16*>(weights_feature_->buffer),
127                                     &weightsFeatureDataFloat32);
128             std::vector<float> weightsTimeDataFloat32(getNumberOfElements(weights_time_->shape()));
129             convertFloat16ToFloat32(reinterpret_cast<_Float16*>(weights_time_->buffer),
130                                     &weightsTimeDataFloat32);
131             std::vector<float> outputDataFloat32(getNumberOfElements(output_->shape()));
132             std::vector<float> outputStateDataFloat32(getNumberOfElements(state_out_->shape()));
133 
134             EvalFloat32(inputDataFloat32.data(), inputStateDataFloat32.data(),
135                         biasDataFloat32.data(), weightsFeatureDataFloat32.data(),
136                         weightsTimeDataFloat32.data(), outputDataFloat32.data(),
137                         outputStateDataFloat32.data());
138             convertFloat32ToFloat16(outputDataFloat32,
139                                     reinterpret_cast<_Float16*>(output_->buffer));
140             convertFloat32ToFloat16(outputStateDataFloat32,
141                                     reinterpret_cast<_Float16*>(state_out_->buffer));
142             break;
143         }
144         case OperandType::TENSOR_FLOAT32: {
145             EvalFloat32(reinterpret_cast<float*>(input_->buffer),
146                         reinterpret_cast<float*>(state_in_->buffer),
147                         reinterpret_cast<float*>(bias_->buffer),
148                         reinterpret_cast<float*>(weights_feature_->buffer),
149                         reinterpret_cast<float*>(weights_time_->buffer),
150                         reinterpret_cast<float*>(output_->buffer),
151                         reinterpret_cast<float*>(state_out_->buffer));
152             break;
153         }
154         default: {
155             LOG(ERROR) << "Unsupported data type: " << static_cast<int>(input_->type);
156             return false;
157         }
158     }
159     return true;
160 }
161 
EvalFloat32(const float * inputData,const float * inputStateData,const float * biasData,const float * weightsFeatureData,const float * weightsTimeData,float * outputData,float * outputStateData)162 void SVDF::EvalFloat32(const float* inputData, const float* inputStateData, const float* biasData,
163                        const float* weightsFeatureData, const float* weightsTimeData,
164                        float* outputData, float* outputStateData) {
165     NNTRACE_COMP("SVDF::EvalFloat32");
166 
167     const int rank = params_.rank_;
168     const int batch_size = SizeOfDimension(input_, 0);
169     const int input_size = SizeOfDimension(input_, 1);
170     const int num_filters = SizeOfDimension(weights_feature_, 0);
171     const int num_units = num_filters / rank;
172     const int memory_size = SizeOfDimension(weights_time_, 1);
173 
174     memcpy(outputStateData, inputStateData, sizeof(float) * batch_size * memory_size * num_filters);
175     // Compute conv1d(inputs, weights_feature).
176     for (int b = 0; b < batch_size; b++) {
177         float* state_ptr_batch = outputStateData + b * memory_size * num_filters;
178         for (int c = 0; c < num_filters; c++) {
179             float* state_ptr = state_ptr_batch + c * memory_size;
180             state_ptr[memory_size - 1] = 0.0;
181         }
182     }
183 
184     // Clear scratch (the matmul is accumulative).
185     float scratch[batch_size * num_filters];
186     std::fill_n(scratch, batch_size * num_filters, 0.0f);
187     tflite::tensor_utils::MatrixBatchVectorMultiplyAccumulate(
188             weightsFeatureData, num_filters, input_size, inputData, batch_size, scratch);
189 
190     // Copy the latest activation from scratch into activation_state:
191     // The last, i.e. (memory_size-1)th entry for each batch, and filter.
192     for (int i = 0; i < batch_size * num_filters; ++i) {
193         outputStateData[i * memory_size + memory_size - 1] = scratch[i];
194     }
195 
196     // Begin ApplyTimeWeightsBiasAndActivation
197     // Compute matmul(state, weights_time).
198     for (int b = 0; b < batch_size; b++) {
199         float* state_out_ptr_batch = outputStateData + b * memory_size * num_filters;
200         float* scratch_ptr_batch = scratch + b * num_filters;
201         tflite::tensor_utils::BatchVectorBatchVectorDotProduct(
202                 weightsTimeData, state_out_ptr_batch, memory_size, num_filters, scratch_ptr_batch);
203     }
204 
205     // Reduction sum
206     tflite::tensor_utils::ReductionSumVector(scratch, outputData, batch_size * num_units, rank);
207 
208     // Add bias if provided.
209     if (!IsNullInput(bias_)) {
210         tflite::tensor_utils::VectorBatchVectorAdd(biasData, num_units, batch_size, outputData);
211     }
212 
213     // Apply activation.
214     tflite::tensor_utils::ApplyActivationToVector(outputData, batch_size * num_units,
215                                                   params_.activation_, outputData);
216     // Finished ApplyTimeWeightsBiasAndActivation
217 
218     // Right shift the state.
219     for (int b = 0; b < batch_size; b++) {
220         float* state_out_ptr_batch = outputStateData + b * memory_size * num_filters;
221         for (int f = 0; f < num_filters; f++) {
222             std::copy(state_out_ptr_batch + 1, state_out_ptr_batch + memory_size,
223                       state_out_ptr_batch);
224             state_out_ptr_batch[memory_size - 1] = 0.0;
225             state_out_ptr_batch += memory_size;
226         }
227     }
228 }
229 
230 }  // namespace nn
231 }  // namespace android
232