1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include "headset_wakeup_engine_impl.h"
16 #include "audio_system_manager.h"
17 #include "adapter_callback_service.h"
18 #include "intell_voice_log.h"
19 #include "history_info_mgr.h"
20 #include "intell_voice_util.h"
21 #include "intell_voice_service_manager.h"
22 #include "headset_host_manager.h"
23 
24 #define LOG_TAG "HeadsetWakeupEngineImpl"
25 
26 using namespace OHOS::HDI::IntelligentVoice::Engine::V1_0;
27 using namespace OHOS::IntellVoiceUtils;
28 using namespace OHOS::AudioStandard;
29 using namespace std;
30 
31 namespace OHOS {
32 namespace IntellVoiceEngine {
33 static constexpr int64_t RECOGNIZING_TIMEOUT_US = 10 * 1000 * 1000; //10s
34 static constexpr int64_t RECOGNIZE_COMPLETE_TIMEOUT_US = 1 * 1000; //1ms
35 static constexpr int64_t READ_CAPTURER_TIMEOUT_US = 10 * 1000 * 1000; //10s
36 
HeadsetWakeupEngineImpl()37 HeadsetWakeupEngineImpl::HeadsetWakeupEngineImpl() : ModuleStates(State(IDLE), "HeadsetWakeupEngineImpl")
38 {
39 }
40 
~HeadsetWakeupEngineImpl()41 HeadsetWakeupEngineImpl::~HeadsetWakeupEngineImpl()
42 {
43 }
44 
Init()45 bool HeadsetWakeupEngineImpl::Init()
46 {
47     if (!InitStates()) {
48         INTELL_VOICE_LOG_ERROR("init state failed");
49         return false;
50     }
51 
52     adapterListener_ = std::make_shared<WakeupAdapterListener>(
53         std::bind(&HeadsetWakeupEngineImpl::OnWakeupEvent, this, std::placeholders::_1));
54     if (adapterListener_ == nullptr) {
55         INTELL_VOICE_LOG_ERROR("adapterListener_ is nullptr");
56         return false;
57     }
58 
59     return true;
60 }
61 
InitStates()62 bool HeadsetWakeupEngineImpl::InitStates()
63 {
64     for (int i = IDLE; i <= READ_CAPTURER; i++) {
65         ForState(State(i))
66             .ACT(SET_LISTENER, HandleSetListener)
67             .ACT(SET_PARAM, HandleSetParam);
68     }
69 
70     ForState(IDLE)
71         .ACT(INIT, HandleInit)
72         .ACT(RESET_ADAPTER, HandleResetAdapter);
73 
74     ForState(INITIALIZING)
75         .ACT(INIT_DONE, HandleInitDone);
76 
77     ForState(INITIALIZED)
78         .ACT(START_RECOGNIZE, HandleStart);
79 
80     ForState(RECOGNIZING)
81         .WaitUntil(RECOGNIZING_TIMEOUT, std::bind(&HeadsetWakeupEngineImpl::HandleRecognizingTimeout,
82             this, std::placeholders::_1, std::placeholders::_2), RECOGNIZING_TIMEOUT_US)
83         .ACT(STOP_RECOGNIZE, HandleStop)
84         .ACT(RECOGNIZE_COMPLETE, HandleRecognizeComplete);
85 
86     ForState(RECOGNIZED)
87         .WaitUntil(RECOGNIZE_COMPLETE_TIMEOUT, std::bind(&HeadsetWakeupEngineImpl::HandleStopCapturer,
88             this, std::placeholders::_1, std::placeholders::_2), RECOGNIZE_COMPLETE_TIMEOUT_US)
89         .ACT(START_CAPTURER, HandleStartCapturer);
90 
91     ForState(READ_CAPTURER)
92         .WaitUntil(READ_CAPTURER_TIMEOUT, std::bind(&HeadsetWakeupEngineImpl::HandleStopCapturer,
93             this, std::placeholders::_1, std::placeholders::_2), READ_CAPTURER_TIMEOUT_US)
94         .ACT(READ, HandleRead)
95         .ACT(STOP_CAPTURER, HandleStopCapturer);
96 
97     FromState(INITIALIZING, READ_CAPTURER)
98         .ACT(RELEASE_ADAPTER, HandleRelease)
99         .ACT(RELEASE, HandleRelease);
100 
101     return IsStatesInitSucc();
102 }
103 
Handle(const StateMsg & msg)104 int32_t HeadsetWakeupEngineImpl::Handle(const StateMsg &msg)
105 {
106     if (!IsStatesInitSucc()) {
107         INTELL_VOICE_LOG_ERROR("failed to init state");
108         return -1;
109     }
110 
111     return ModuleStates::HandleMsg(msg);
112 }
113 
SetCallbackInner()114 bool HeadsetWakeupEngineImpl::SetCallbackInner()
115 {
116     INTELL_VOICE_LOG_INFO("enter");
117     if (adapter_ == nullptr) {
118         INTELL_VOICE_LOG_ERROR("adapter is nullptr");
119         return false;
120     }
121 
122     if (adapterListener_ == nullptr) {
123         INTELL_VOICE_LOG_ERROR("adapterListener_ is nullptr");
124         return false;
125     }
126 
127     callback_ = sptr<IIntellVoiceEngineCallback>(new (std::nothrow) AdapterCallbackService(adapterListener_));
128     if (callback_ == nullptr) {
129         INTELL_VOICE_LOG_ERROR("callback_ is nullptr");
130         return false;
131     }
132 
133     adapter_->SetCallback(callback_);
134     return true;
135 }
136 
AttachInner(const IntellVoiceEngineInfo & info)137 int32_t HeadsetWakeupEngineImpl::AttachInner(const IntellVoiceEngineInfo &info)
138 {
139     INTELL_VOICE_LOG_INFO("enter");
140     if (adapter_ == nullptr) {
141         INTELL_VOICE_LOG_ERROR("adapter is nullptr");
142         return -1;
143     }
144 
145     IntellVoiceEngineAdapterInfo adapterInfo = {
146     };
147 
148     return adapter_->Attach(adapterInfo);
149 }
150 
OnWakeupEvent(const OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent & event)151 void HeadsetWakeupEngineImpl::OnWakeupEvent(
152     const OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent &event)
153 {
154     INTELL_VOICE_LOG_INFO("enter, msgId:%{public}d, result:%{public}d", event.msgId, event.result);
155     if (event.msgId == INTELL_VOICE_ENGINE_MSG_INIT_DONE) {
156         std::thread(&HeadsetWakeupEngineImpl::OnInitDone, this, event.result).detach();
157     } else if (
158         event.msgId == static_cast<OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineMessageType>(
159             OHOS::HDI::IntelligentVoice::Engine::V1_2::INTELL_VOICE_ENGINE_MSG_HEADSET_RECOGNIZE_COMPLETE)) {
160         std::thread(&HeadsetWakeupEngineImpl::OnWakeupRecognition, this, event.result, event.info).detach();
161     } else {
162     }
163 }
164 
OnInitDone(int32_t result)165 void HeadsetWakeupEngineImpl::OnInitDone(int32_t result)
166 {
167     INTELL_VOICE_LOG_INFO("enter, result:%{public}d", result);
168     StateMsg msg(INIT_DONE, &result, sizeof(int32_t));
169     Handle(msg);
170 }
171 
OnWakeupRecognition(int32_t result,const std::string & info)172 void HeadsetWakeupEngineImpl::OnWakeupRecognition(int32_t result, const std::string &info)
173 {
174     INTELL_VOICE_LOG_INFO("enter, result:%{public}d", result);
175     OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent event;
176     event.msgId = static_cast<OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineMessageType>(
177         HDI::IntelligentVoice::Engine::V1_2::INTELL_VOICE_ENGINE_MSG_HEADSET_RECOGNIZE_COMPLETE);
178     event.result = static_cast<OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineErrors>(result);
179     event.info = info;
180     StateMsg msg(RECOGNIZE_COMPLETE, reinterpret_cast<void *>(&event),
181         sizeof(OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent));
182     Handle(msg);
183 }
184 
HandleInit(const StateMsg &,State & nextState)185 int32_t HeadsetWakeupEngineImpl::HandleInit(const StateMsg & /* msg */, State &nextState)
186 {
187     INTELL_VOICE_LOG_INFO("enter");
188     if (!EngineUtil::CreateAdapterInner(HeadsetHostManager::GetInstance(), WAKEUP_ADAPTER_TYPE)) {
189         INTELL_VOICE_LOG_ERROR("failed to create adapter");
190         return -1;
191     }
192 
193     if (!SetCallbackInner()) {
194         INTELL_VOICE_LOG_ERROR("failed to set callback");
195         return -1;
196     }
197 
198     EngineUtil::SetLanguage();
199     EngineUtil::SetArea();
200     adapter_->SetParameter("model_path=/data/local/tmp/encoder.om");
201     IntellVoiceEngineInfo info = {
202     };
203 
204     if (AttachInner(info) != 0) {
205         INTELL_VOICE_LOG_ERROR("failed to attach");
206         return -1;
207     }
208 
209     nextState = State(INITIALIZING);
210     return 0;
211 }
212 
HandleInitDone(const StateMsg & msg,State & nextState)213 int32_t HeadsetWakeupEngineImpl::HandleInitDone(const StateMsg &msg, State &nextState)
214 {
215     INTELL_VOICE_LOG_INFO("enter");
216     int32_t *result = reinterpret_cast<int32_t *>(msg.inMsg);
217     if ((result == nullptr) || (*result != 0)) {
218         INTELL_VOICE_LOG_ERROR("init done failed");
219         return -1;
220     }
221 
222     nextState = State(INITIALIZED);
223     return 0;
224 }
225 
HandleSetListener(const StateMsg & msg,State &)226 int32_t HeadsetWakeupEngineImpl::HandleSetListener(const StateMsg &msg, State & /* nextState */)
227 {
228     SetListenerMsg *listenerMsg = reinterpret_cast<SetListenerMsg *>(msg.inMsg);
229     if (listenerMsg == nullptr || adapterListener_ == nullptr) {
230         INTELL_VOICE_LOG_ERROR("listenerMsg or adapter listener is nullptr");
231         return -1;
232     }
233 
234     adapterListener_->SetCallback(listenerMsg->callback);
235     return 0;
236 }
237 
HandleStart(const StateMsg &,State & nextState)238 int32_t HeadsetWakeupEngineImpl::HandleStart(const StateMsg & /* msg */, State &nextState)
239 {
240     INTELL_VOICE_LOG_INFO("enter");
241     if (adapter_ == nullptr) {
242         INTELL_VOICE_LOG_ERROR("adapter is nullptr");
243         return -1;
244     }
245 
246     StartInfo info = {
247         .isLast = true,
248     };
249     if (adapter_->Start(info)) {
250         INTELL_VOICE_LOG_ERROR("start adapter failed");
251         return -1;
252     }
253 
254     if (!StartAudioSource()) {
255         INTELL_VOICE_LOG_ERROR("start audio source failed");
256         adapter_->Stop();
257         return -1;
258     }
259 
260     INTELL_VOICE_LOG_INFO("exit");
261     nextState = State(RECOGNIZING);
262     return 0;
263 }
264 
ReadThread()265 void HeadsetWakeupEngineImpl::ReadThread()
266 {
267     bool isEnd = false;
268     while (isReading_.load()) {
269         std::vector<uint8_t> audioStream;
270         bool hasAwakeWord = true;
271         int ret = ReadHeadsetStream(audioStream, hasAwakeWord);
272         if (hasAwakeWord && !isEnd) {
273             adapter_->WriteAudio(audioStream);
274         }
275         if (!hasAwakeWord && !isEnd) {
276             isEnd = true;
277             adapter_->SetParameter("end_of_pcm=true");
278         }
279         if (ret == -1) {
280             INTELL_VOICE_LOG_INFO("finish reading");
281             break;
282         }
283         WakeupSourceProcess::Write({ audioStream });
284     }
285 }
286 
StartAudioSource()287 bool HeadsetWakeupEngineImpl::StartAudioSource()
288 {
289     INTELL_VOICE_LOG_INFO("enter");
290     isReading_.store(true);
291 
292     WakeupSourceProcess::Init(1);
293 
294     std::thread t1(std::bind(&HeadsetWakeupEngineImpl::ReadThread, this));
295     readThread_ = std::move(t1);
296     return true;
297 }
298 
StopAudioSource()299 void HeadsetWakeupEngineImpl::StopAudioSource()
300 {
301     if (!isReading_.load()) {
302         INTELL_VOICE_LOG_INFO("already stop");
303         return;
304     }
305     isReading_.store(false);
306     readThread_.join();
307     StopReadingStream();
308     WakeupSourceProcess::Release();
309 }
310 
HandleStop(const StateMsg &,State & nextState)311 int32_t HeadsetWakeupEngineImpl::HandleStop(const StateMsg & /* msg */, State &nextState)
312 {
313     StopAudioSource();
314     EngineUtil::Stop();
315     nextState = State(INITIALIZED);
316     return 0;
317 }
318 
HandleRecognizeComplete(const StateMsg & msg,State & nextState)319 int32_t HeadsetWakeupEngineImpl::HandleRecognizeComplete(const StateMsg &msg, State &nextState)
320 {
321     EngineUtil::Stop();
322     OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent *event =
323         reinterpret_cast<OHOS::HDI::IntelligentVoice::Engine::V1_0::IntellVoiceEngineCallBackEvent *>(msg.inMsg);
324     if (event == nullptr) {
325         INTELL_VOICE_LOG_ERROR("event is nullptr");
326         return -1;
327     }
328 
329     adapterListener_->Notify(*event);
330 
331     if (event->result != 0) {
332         INTELL_VOICE_LOG_INFO("wakeup failed");
333         NotifyVerifyResult(false);
334         StopAudioSource();
335         nextState = State(INITIALIZED);
336     } else {
337         NotifyVerifyResult(true);
338         nextState = State(RECOGNIZED);
339     }
340     return 0;
341 }
342 
HandleStartCapturer(const StateMsg & msg,State & nextState)343 int32_t HeadsetWakeupEngineImpl::HandleStartCapturer(const StateMsg &msg, State &nextState)
344 {
345     INTELL_VOICE_LOG_INFO("enter");
346     int32_t *msgBody = reinterpret_cast<int32_t *>(msg.inMsg);
347     if (msgBody == nullptr) {
348         INTELL_VOICE_LOG_ERROR("msgBody is nullptr");
349         return -1;
350     }
351     channels_ = *msgBody;
352     nextState = State(READ_CAPTURER);
353     return 0;
354 }
355 
HandleRead(const StateMsg & msg,State &)356 int32_t HeadsetWakeupEngineImpl::HandleRead(const StateMsg &msg, State & /* nextState */)
357 {
358     CapturerData *capturerData = reinterpret_cast<CapturerData *>(msg.outMsg);
359     auto ret = WakeupSourceProcess::Read(capturerData->data, channels_);
360     if (ret != 0) {
361         INTELL_VOICE_LOG_ERROR("read capturer data failed");
362         return ret;
363     }
364 
365     ResetTimerDelay();
366     return 0;
367 }
368 
HandleStopCapturer(const StateMsg &,State & nextState)369 int32_t HeadsetWakeupEngineImpl::HandleStopCapturer(const StateMsg & /* msg */, State &nextState)
370 {
371     INTELL_VOICE_LOG_INFO("enter");
372     StopAudioSource();
373     nextState = State(INITIALIZED);
374     return 0;
375 }
376 
HandleRecognizingTimeout(const StateMsg &,State & nextState)377 int32_t HeadsetWakeupEngineImpl::HandleRecognizingTimeout(const StateMsg & /* msg */, State &nextState)
378 {
379     INTELL_VOICE_LOG_INFO("enter");
380     NotifyVerifyResult(false);
381     StopAudioSource();
382     EngineUtil::Stop();
383     nextState = State(INITIALIZED);
384     return 0;
385 }
386 
HandleResetAdapter(const StateMsg &,State & nextState)387 int32_t HeadsetWakeupEngineImpl::HandleResetAdapter(const StateMsg & /* msg */, State &nextState)
388 {
389     INTELL_VOICE_LOG_INFO("enter");
390     if (!EngineUtil::CreateAdapterInner(HeadsetHostManager::GetInstance(), WAKEUP_ADAPTER_TYPE)) {
391         INTELL_VOICE_LOG_ERROR("failed to create adapter");
392         return -1;
393     }
394 
395     adapter_->SetCallback(callback_);
396     EngineUtil::SetLanguage();
397     EngineUtil::SetArea();
398 
399     IntellVoiceEngineAdapterInfo adapterInfo = {
400     };
401 
402     if (adapter_->Attach(adapterInfo) != 0) {
403         INTELL_VOICE_LOG_ERROR("failed to attach");
404         EngineUtil::ReleaseAdapterInner(HeadsetHostManager::GetInstance());
405         return -1;
406     }
407 
408     nextState = State(INITIALIZING);
409     return 0;
410 }
411 
HandleRelease(const StateMsg &,State & nextState)412 int32_t HeadsetWakeupEngineImpl::HandleRelease(const StateMsg & /* msg */, State &nextState)
413 {
414     StopAudioSource();
415     if (adapter_ != nullptr) {
416         adapter_->Detach();
417         ReleaseAdapterInner(HeadsetHostManager::GetInstance());
418     }
419     nextState = State(IDLE);
420     return 0;
421 }
422 
HandleSetParam(const StateMsg & msg,State &)423 int32_t HeadsetWakeupEngineImpl::HandleSetParam(const StateMsg &msg, State & /* nextState */)
424 {
425     StringParam *param = reinterpret_cast<StringParam *>(msg.inMsg);
426     if (param == nullptr) {
427         INTELL_VOICE_LOG_INFO("param is nullptr");
428         return -1;
429     }
430 
431     return EngineUtil::SetParameter(param->strParam);
432 }
433 }
434 }