1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "EvsStateControl.h"
17
18 #include "FormatConvert.h"
19 #include "RenderDirectView.h"
20 #include "RenderPixelCopy.h"
21 #include "RenderTopView.h"
22
23 #include <android-base/logging.h>
24 #include <android/binder_manager.h>
25 #include <utils/SystemClock.h>
26
27 #include <inttypes.h>
28 #include <stdio.h>
29 #include <string.h>
30
31 using ::android::hardware::automotive::evs::V1_0::EvsResult;
32 using EvsDisplayState = ::android::hardware::automotive::evs::V1_0::DisplayState;
33 using BufferDesc_1_0 = ::android::hardware::automotive::evs::V1_0::BufferDesc;
34 using BufferDesc_1_1 = ::android::hardware::automotive::evs::V1_1::BufferDesc;
35
isSfReady()36 static bool isSfReady() {
37 return ::ndk::SpAIBinder(::AServiceManager_getService("SurfaceFlinger")).get() != nullptr;
38 }
39
40 // TODO: Seems like it'd be nice if the Vehicle HAL provided such helpers (but how & where?)
getPropType(VehicleProperty prop)41 inline constexpr VehiclePropertyType getPropType(VehicleProperty prop) {
42 return static_cast<VehiclePropertyType>(
43 static_cast<int32_t>(prop)
44 & static_cast<int32_t>(VehiclePropertyType::MASK));
45 }
46
EvsStateControl(android::sp<IVehicle> pVnet,android::sp<IEvsEnumerator> pEvs,android::sp<IEvsDisplay> pDisplay,const ConfigManager & config)47 EvsStateControl::EvsStateControl(android::sp<IVehicle> pVnet, android::sp<IEvsEnumerator> pEvs,
48 android::sp<IEvsDisplay> pDisplay, const ConfigManager& config) :
49 mVehicle(pVnet),
50 mEvs(pEvs),
51 mDisplay(pDisplay),
52 mConfig(config),
53 mCurrentState(OFF),
54 mEvsStats(EvsStats::build()) {
55 // Initialize the property value containers we'll be updating (they'll be zeroed by default)
56 static_assert(getPropType(VehicleProperty::GEAR_SELECTION) == VehiclePropertyType::INT32,
57 "Unexpected type for GEAR_SELECTION property");
58 static_assert(getPropType(VehicleProperty::TURN_SIGNAL_STATE) == VehiclePropertyType::INT32,
59 "Unexpected type for TURN_SIGNAL_STATE property");
60
61 mGearValue.prop = static_cast<int32_t>(VehicleProperty::GEAR_SELECTION);
62 mTurnSignalValue.prop = static_cast<int32_t>(VehicleProperty::TURN_SIGNAL_STATE);
63
64 // This way we only ever deal with cameras which exist in the system
65 // Build our set of cameras for the states we support
66 LOG(DEBUG) << "Requesting camera list";
67 mEvs->getCameraList_1_1(
68 [this, &config](hidl_vec<CameraDesc> cameraList) {
69 LOG(INFO) << "Camera list callback received " << cameraList.size() << "cameras.";
70 for (auto&& cam: cameraList) {
71 LOG(DEBUG) << "Found camera " << cam.v1.cameraId;
72 bool cameraConfigFound = false;
73
74 // Check our configuration for information about this camera
75 // Note that a camera can have a compound function string
76 // such that a camera can be "right/reverse" and be used for both.
77 // If more than one camera is listed for a given function, we'll
78 // list all of them and let the UX/rendering logic use one, some
79 // or all of them as appropriate.
80 for (auto&& info: config.getCameras()) {
81 if (cam.v1.cameraId == info.cameraId) {
82 // We found a match!
83 if (info.function.find("reverse") != std::string::npos) {
84 mCameraList[State::REVERSE].emplace_back(info);
85 mCameraDescList[State::REVERSE].emplace_back(cam);
86 }
87 if (info.function.find("right") != std::string::npos) {
88 mCameraList[State::RIGHT].emplace_back(info);
89 mCameraDescList[State::RIGHT].emplace_back(cam);
90 }
91 if (info.function.find("left") != std::string::npos) {
92 mCameraList[State::LEFT].emplace_back(info);
93 mCameraDescList[State::LEFT].emplace_back(cam);
94 }
95 if (info.function.find("park") != std::string::npos) {
96 mCameraList[State::PARKING].emplace_back(info);
97 mCameraDescList[State::PARKING].emplace_back(cam);
98 }
99 cameraConfigFound = true;
100 break;
101 }
102 }
103 if (!cameraConfigFound) {
104 LOG(WARNING) << "No config information for hardware camera "
105 << cam.v1.cameraId;
106 }
107 }
108 }
109 );
110
111 LOG(DEBUG) << "State controller ready";
112 }
113
startUpdateLoop()114 bool EvsStateControl::startUpdateLoop() {
115 // Create the thread and report success if it gets started
116 mRenderThread = std::thread([this](){ updateLoop(); });
117 return mRenderThread.joinable();
118 }
119
120
terminateUpdateLoop()121 void EvsStateControl::terminateUpdateLoop() {
122 if (mRenderThread.get_id() == std::this_thread::get_id()) {
123 // We should not join by ourselves
124 mRenderThread.detach();
125 } else if (mRenderThread.joinable()) {
126 // Join a rendering thread
127 mRenderThread.join();
128 }
129 }
130
131
postCommand(const Command & cmd,bool clear)132 void EvsStateControl::postCommand(const Command& cmd, bool clear) {
133 // Push the command onto the queue watched by updateLoop
134 mLock.lock();
135 if (clear) {
136 std::queue<Command> emptyQueue;
137 std::swap(emptyQueue, mCommandQueue);
138 }
139
140 mCommandQueue.push(cmd);
141 mLock.unlock();
142
143 // Send a signal to wake updateLoop in case it is asleep
144 mWakeSignal.notify_all();
145 }
146
147
updateLoop()148 void EvsStateControl::updateLoop() {
149 LOG(DEBUG) << "Starting EvsStateControl update loop";
150
151 bool run = true;
152 while (run) {
153 // Process incoming commands
154 sp<IEvsDisplay> displayHandle;
155 {
156 std::lock_guard <std::mutex> lock(mLock);
157 while (!mCommandQueue.empty()) {
158 const Command& cmd = mCommandQueue.front();
159 switch (cmd.operation) {
160 case Op::EXIT:
161 run = false;
162 break;
163 case Op::CHECK_VEHICLE_STATE:
164 // Just running selectStateForCurrentConditions below will take care of this
165 break;
166 case Op::TOUCH_EVENT:
167 // Implement this given the x/y location of the touch event
168 break;
169 }
170 mCommandQueue.pop();
171 }
172
173 displayHandle = mDisplay.promote();
174 }
175
176 if (!displayHandle) {
177 LOG(ERROR) << "We've lost the display";
178 break;
179 }
180
181 // Review vehicle state and choose an appropriate renderer
182 if (!selectStateForCurrentConditions()) {
183 LOG(ERROR) << "selectStateForCurrentConditions failed so we're going to die";
184 break;
185 }
186
187 // If we have an active renderer, give it a chance to draw
188 if (mCurrentRenderer) {
189 // Get the output buffer we'll use to display the imagery
190 BufferDesc_1_0 tgtBuffer = {};
191 displayHandle->getTargetBuffer([&tgtBuffer](const BufferDesc_1_0& buff) {
192 tgtBuffer = buff;
193 }
194 );
195
196 if (tgtBuffer.memHandle == nullptr) {
197 LOG(ERROR) << "Didn't get requested output buffer -- skipping this frame.";
198 run = false;
199 } else {
200 // Generate our output image
201 if (!mCurrentRenderer->drawFrame(convertBufferDesc(tgtBuffer))) {
202 // If drawing failed, we want to exit quickly so an app restart can happen
203 run = false;
204 }
205
206 // Send the finished image back for display
207 displayHandle->returnTargetBufferForDisplay(tgtBuffer);
208
209 if (!mFirstFrameIsDisplayed) {
210 mFirstFrameIsDisplayed = true;
211 // returnTargetBufferForDisplay() is finished, the frame should be displayed
212 mEvsStats.finishComputingFirstFrameLatency(android::uptimeMillis());
213 }
214 }
215 } else if (run) {
216 // No active renderer, so sleep until somebody wakes us with another command
217 // or exit if we received EXIT command
218 std::unique_lock<std::mutex> lock(mLock);
219 mWakeSignal.wait(lock);
220 }
221 }
222
223 LOG(WARNING) << "EvsStateControl update loop ending";
224
225 if (mCurrentRenderer) {
226 // Deactive the renderer
227 mCurrentRenderer->deactivate();
228 }
229
230 // If `ICarTelemetry` service was not ready before, we need to try sending data again.
231 mEvsStats.sendCollectedDataBlocking();
232
233 printf("Shutting down app due to state control loop ending\n");
234 LOG(ERROR) << "Shutting down app due to state control loop ending";
235 }
236
237
selectStateForCurrentConditions()238 bool EvsStateControl::selectStateForCurrentConditions() {
239 static int32_t sMockGear = mConfig.getMockGearSignal();
240 static int32_t sMockSignal = int32_t(VehicleTurnSignal::NONE);
241
242 if (mVehicle != nullptr) {
243 // Query the car state
244 if (invokeGet(&mGearValue) != StatusCode::OK) {
245 LOG(ERROR) << "GEAR_SELECTION not available from vehicle. Exiting.";
246 return false;
247 }
248 if ((mTurnSignalValue.prop == 0) || (invokeGet(&mTurnSignalValue) != StatusCode::OK)) {
249 // Silently treat missing turn signal state as no turn signal active
250 mTurnSignalValue.value.int32Values.setToExternal(&sMockSignal, 1);
251 mTurnSignalValue.prop = 0;
252 }
253 } else {
254 // While testing without a vehicle, behave as if we're in reverse for the first 20 seconds
255 static const int kShowTime = 20; // seconds
256
257 // See if it's time to turn off the default reverse camera
258 static std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now();
259 std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now();
260 if (std::chrono::duration_cast<std::chrono::seconds>(now - start).count() > kShowTime) {
261 // Switch to drive (which should turn off the reverse camera)
262 sMockGear = int32_t(VehicleGear::GEAR_DRIVE);
263 }
264
265 // Build the placeholder vehicle state values (treating single values as 1 element vectors)
266 mGearValue.value.int32Values.setToExternal(&sMockGear, 1);
267 mTurnSignalValue.value.int32Values.setToExternal(&sMockSignal, 1);
268 }
269
270 // Choose our desired EVS state based on the current car state
271 // TODO: Update this logic, and consider user input when choosing if a view should be presented
272 State desiredState = OFF;
273 if (mGearValue.value.int32Values[0] == int32_t(VehicleGear::GEAR_REVERSE)) {
274 desiredState = REVERSE;
275 } else if (mTurnSignalValue.value.int32Values[0] == int32_t(VehicleTurnSignal::RIGHT)) {
276 desiredState = RIGHT;
277 } else if (mTurnSignalValue.value.int32Values[0] == int32_t(VehicleTurnSignal::LEFT)) {
278 desiredState = LEFT;
279 } else if (mGearValue.value.int32Values[0] == int32_t(VehicleGear::GEAR_PARK)) {
280 desiredState = PARKING;
281 }
282
283 // Apply the desire state
284 return configureEvsPipeline(desiredState);
285 }
286
287
invokeGet(VehiclePropValue * pRequestedPropValue)288 StatusCode EvsStateControl::invokeGet(VehiclePropValue *pRequestedPropValue) {
289 StatusCode status = StatusCode::TRY_AGAIN;
290
291 // Call the Vehicle HAL, which will block until the callback is complete
292 mVehicle->get(*pRequestedPropValue,
293 [pRequestedPropValue, &status]
294 (StatusCode s, const VehiclePropValue& v) {
295 status = s;
296 if (s == StatusCode::OK) {
297 *pRequestedPropValue = v;
298 }
299 }
300 );
301
302 return status;
303 }
304
305
configureEvsPipeline(State desiredState)306 bool EvsStateControl::configureEvsPipeline(State desiredState) {
307 static bool isGlReady = false;
308
309 if (mCurrentState == desiredState) {
310 // Nothing to do here...
311 return true;
312 }
313
314 // Used by CarStats to accurately compute stats, it needs to be close to the beginning.
315 auto desiredStateTimeMillis = android::uptimeMillis();
316
317 LOG(DEBUG) << "Switching to state " << desiredState;
318 LOG(DEBUG) << " Current state " << mCurrentState
319 << " has " << mCameraList[mCurrentState].size() << " cameras";
320 LOG(DEBUG) << " Desired state " << desiredState
321 << " has " << mCameraList[desiredState].size() << " cameras";
322
323 if (!isGlReady && !isSfReady()) {
324 // Graphics is not ready yet; using CPU renderer.
325 if (mCameraList[desiredState].size() >= 1) {
326 mDesiredRenderer = std::make_unique<RenderPixelCopy>(mEvs,
327 mCameraList[desiredState][0]);
328 if (!mDesiredRenderer) {
329 LOG(ERROR) << "Failed to construct Pixel Copy renderer. Skipping state change.";
330 return false;
331 }
332 } else {
333 LOG(DEBUG) << "Unsupported, desiredState " << desiredState
334 << " has " << mCameraList[desiredState].size() << " cameras.";
335 }
336 } else {
337 // Assumes that SurfaceFlinger is available always after being launched.
338
339 // Do we need a new direct view renderer?
340 if (mCameraList[desiredState].size() == 1) {
341 // We have a camera assigned to this state for direct view.
342 mDesiredRenderer = std::make_unique<RenderDirectView>(mEvs,
343 mCameraDescList[desiredState][0],
344 mConfig);
345 if (!mDesiredRenderer) {
346 LOG(ERROR) << "Failed to construct direct renderer. Skipping state change.";
347 return false;
348 }
349 } else if (mCameraList[desiredState].size() > 1 || desiredState == PARKING) {
350 //TODO(b/140668179): RenderTopView needs to be updated to use new
351 // ConfigManager.
352 mDesiredRenderer = std::make_unique<RenderTopView>(mEvs,
353 mCameraList[desiredState],
354 mConfig);
355 if (!mDesiredRenderer) {
356 LOG(ERROR) << "Failed to construct top view renderer. Skipping state change.";
357 return false;
358 }
359 } else {
360 LOG(DEBUG) << "Unsupported, desiredState " << desiredState
361 << " has " << mCameraList[desiredState].size() << " cameras.";
362 }
363
364 // GL renderer is now ready.
365 isGlReady = true;
366 }
367
368 // Since we're changing states, shut down the current renderer
369 if (mCurrentRenderer != nullptr) {
370 mCurrentRenderer->deactivate();
371 mCurrentRenderer = nullptr; // It's a smart pointer, so destructs on assignment to null
372 }
373
374 // Now set the display state based on whether we have a video feed to show
375 sp<IEvsDisplay> displayHandle = mDisplay.promote();
376 if (!displayHandle) {
377 return false;
378 }
379
380 if (mDesiredRenderer == nullptr) {
381 LOG(DEBUG) << "Turning off the display";
382 displayHandle->setDisplayState(EvsDisplayState::NOT_VISIBLE);
383 } else {
384 mCurrentRenderer = std::move(mDesiredRenderer);
385
386 // Start the camera stream
387 LOG(DEBUG) << "EvsStartCameraStreamTiming start time: "
388 << android::elapsedRealtime() << " ms.";
389 if (!mCurrentRenderer->activate()) {
390 LOG(ERROR) << "New renderer failed to activate";
391 return false;
392 }
393
394 // Activate the display
395 LOG(DEBUG) << "EvsActivateDisplayTiming start time: "
396 << android::elapsedRealtime() << " ms.";
397 Return<EvsResult> result = displayHandle->setDisplayState(
398 EvsDisplayState::VISIBLE_ON_NEXT_FRAME);
399 if (result != EvsResult::OK) {
400 LOG(ERROR) << "setDisplayState returned an error "
401 << result.description();
402 return false;
403 }
404 }
405
406 // Record our current state
407 LOG(INFO) << "Activated state " << desiredState;
408 mCurrentState = desiredState;
409
410 mFirstFrameIsDisplayed = false; // Got a new renderer, mark first frame is not displayed.
411
412 if (mCurrentRenderer != nullptr && desiredState == State::REVERSE) {
413 // Start computing the latency when the evs state changes.
414 mEvsStats.startComputingFirstFrameLatency(desiredStateTimeMillis);
415 }
416
417 return true;
418 }
419