1 /*
2  * Copyright (c) 2020-2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <unistd.h>
17 #include <sys/time.h>
18 #include "player_sync.h"
19 #include "player_sink_type.h"
20 #include "media_log.h"
21 
22 namespace OHOS {
23 namespace Media {
24 const int32_t SYNC_STOP_NEGATIVE_MS = 6000;
25 const int32_t SYNC_STOP_PLUS_MS = 3000;
26 const int32_t SYNC_START_NEGATIVE_MS = 100;
27 const int32_t SYNC_START_PLUS_MS = 50;
28 const int32_t MAX_VID_LOST_FRAMENUM = 10;
29 const int32_t FIRST_VID_RENDER_WAIT_TIME_WHEN_NO_AUDIO = 50000;
30 const int32_t FIRST_AUDIO_RENDER_WAIT_VID_TIME = 10000000; /* audio wait first video frame time */
31 const int32_t SS2US = 1000000;
32 const int32_t US2MS = 1000;
33 const int64_t INT64_MAX_VALUE = 0x7fffffffffffffff;
34 
GetCurTimeUs()35 static int64_t GetCurTimeUs()
36 {
37     struct timeval ts = {0, 0};
38     gettimeofday(&ts, NULL);
39     return ((static_cast<int64_t>(ts.tv_sec)) * SS2US) + (static_cast<int64_t>(ts.tv_usec));
40 }
41 
CheckAVDiff(int64_t latenessMs,uint32_t & continueLost)42 SyncRet PlayerSync::CheckAVDiff(int64_t latenessMs, uint32_t &continueLost)
43 {
44     SyncRet syncRet = SYNC_RET_PLAY;
45 
46     if (continueLost >= MAX_VID_LOST_FRAMENUM) {
47         continueLost = 0;
48         return syncRet;
49     }
50     // sync lateness out range exception; pts jump err, just skip this frame
51     if (latenessMs > (int64_t)syncAttr_.syncStopRegion.vidNegativeTime
52         || latenessMs < (-(int64_t)syncAttr_.syncStopRegion.vidPlusTime)) {
53         syncRet = SYNC_RET_DROP;
54         continueLost++;
55         return syncRet;
56     }
57     if (latenessMs > (int64_t)syncAttr_.syncStartRegion.vidNegativeTime) {
58         syncRet = SYNC_RET_DROP;
59         continueLost++;
60     } else if (latenessMs < (-(int64_t)syncAttr_.syncStartRegion.vidPlusTime)) {
61         syncRet = SYNC_RET_REPEAT;
62     } else {
63         syncRet = SYNC_RET_PLAY;
64         continueLost = 0;
65     }
66     return syncRet;
67 }
68 
TPlayProcess(int64_t timestampUs,SyncRet & result)69 int32_t PlayerSync::TPlayProcess(int64_t timestampUs, SyncRet &result)
70 {
71     int64_t nowTsUs;
72     int64_t latenessUs = 0;
73     int64_t realTimeElapseUs = GetCurTimeUs() - frstVidFrameTime_;
74     int64_t trickTimeElapseUs = static_cast<int64_t>(realTimeElapseUs * speed_);
75 
76     if (tplayDirect_ == TPLAY_DIRECT_FORWARD) {
77         nowTsUs = firstVidFrameTs_ + trickTimeElapseUs;
78         latenessUs = (nowTsUs - timestampUs);
79     } else {  // backward
80         nowTsUs = firstVidFrameTs_ - trickTimeElapseUs;
81         if (nowTsUs < 0) {
82             nowTsUs = 0;
83         }
84         latenessUs = (timestampUs - nowTsUs);
85     }
86     int64_t latenessMs = (speed_ != 0) ? (latenessUs / (static_cast<int>(speed_ * MS_SCALE))) : 0;
87     result = CheckAVDiff(latenessMs, continousVidLost_);
88     return HI_SUCCESS;
89 }
90 
Reset(SyncChn syncChn)91 int32_t PlayerSync::Reset(SyncChn syncChn)
92 {
93     diffAvRenderTime_ = 0;
94 
95     if (syncChn == SYNC_CHN_VID) {
96         pthread_mutex_lock(&vidSyncLock_);
97         isFristVidFrame_ = true;
98         vidTimeSourceDelta_ = 0;
99         lastVideoTsUs_ = AV_INVALID_PTS;
100         continousVidLost_ = 0;
101         firstVidFrameTs_ = AV_INVALID_PTS;
102         frstVidFrameTime_ = 0;
103         pthread_mutex_unlock(&vidSyncLock_);
104     } else if (syncChn == SYNC_CHN_AUD) {
105         pthread_mutex_lock(&audSyncLock_);
106         isFristAudFrame_ = true;
107         audTimeSourceDelta_ = 0;
108         lastAudioTsUs_ = AV_INVALID_PTS;
109         lastAudioRealTsUs_ = AV_INVALID_PTS;
110         firstAudFrameTs_ = AV_INVALID_PTS;
111         firstAudFrameTime_ = 0;
112         pthread_mutex_unlock(&audSyncLock_);
113     } else {
114         MEDIA_ERR_LOG("invalid sync chn: %d", syncChn);
115         return HI_FAILURE;
116     }
117     return HI_SUCCESS;
118 }
119 
GetStatus(PlaySyncStatus & info)120 int32_t PlayerSync::GetStatus(PlaySyncStatus &info)
121 {
122     info.firstAudPts = (firstVidFrameTs_ == AV_INVALID_PTS) ? AV_INVALID_PTS : (firstVidFrameTs_ / MS_SCALE);
123     info.lastAudPts = (lastAudioTsUs_ == AV_INVALID_PTS) ? AV_INVALID_PTS : (lastAudioTsUs_ / MS_SCALE);
124     info.firstVidPts = (firstVidFrameTs_ == AV_INVALID_PTS) ? AV_INVALID_PTS : (firstVidFrameTs_ / MS_SCALE);
125     info.lastVidPts = (lastVideoTsUs_ == AV_INVALID_PTS) ? AV_INVALID_PTS : (lastVideoTsUs_ / MS_SCALE);
126     info.diffTime = diffAvRenderTime_;
127     info.localTime = AV_INVALID_PTS;
128     return HI_SUCCESS;
129 }
130 
GetAttr(PlayerSyncAttr & syncAttr)131 int32_t PlayerSync::GetAttr(PlayerSyncAttr &syncAttr)
132 {
133     syncAttr = syncAttr_;
134     return HI_SUCCESS;
135 }
136 
SetAttr(PlayerSyncAttr & syncAttr)137 int32_t PlayerSync::SetAttr(PlayerSyncAttr &syncAttr)
138 {
139     syncAttr_ = syncAttr;
140     return HI_SUCCESS;
141 }
142 
PlayerSync()143 PlayerSync::PlayerSync()
144 {
145     isVidEnable_ = false;
146     isAudEnable_ = false;
147     isFristVidFrame_ = false;
148     vidTimeSourceDelta_ = 0;
149     lastVideoTsUs_ = AV_INVALID_PTS;
150     firstVidFrameTs_ = 0;
151     frstVidFrameTime_ = 0;
152     continousVidLost_ = 0;
153     isFristAudFrame_ = false;
154     audTimeSourceDelta_ = 0;
155     lastAudioTsUs_ = AV_INVALID_PTS;
156     lastAudioRealTsUs_ = 0;
157     firstAudFrameTs_ = AV_INVALID_PTS;
158     firstAudFrameTime_ = 0;
159     isInTrickPlayMode_ = false;
160     speed_ = 1.0;
161     tplayDirect_ = TPLAY_DIRECT_BUTT;
162     syncAttr_.isQuickOutput = true;
163     syncAttr_.refType = PLAYER_SYNC_REF_AUD;
164     syncAttr_.syncStartRegion.vidNegativeTime = SYNC_START_NEGATIVE_MS;
165     syncAttr_.syncStartRegion.vidPlusTime = SYNC_START_PLUS_MS;
166     syncAttr_.syncStopRegion.vidNegativeTime = SYNC_STOP_NEGATIVE_MS;
167     syncAttr_.syncStopRegion.vidPlusTime = SYNC_STOP_PLUS_MS;
168     diffAvRenderTime_ = 0;
169     isInited_ = false;
170 }
171 
~PlayerSync()172 PlayerSync::~PlayerSync()
173 {
174     (void)Deinit();
175 }
176 
Deinit(void)177 int32_t PlayerSync::Deinit(void)
178 {
179     int32_t ret = HI_SUCCESS;
180     if (isInited_ == true) {
181         ret = pthread_mutex_destroy(&vidSyncLock_);
182         if (ret != 0) {
183             MEDIA_ERR_LOG("pthread_mutex_destroy vidSyncLock_ failed");
184         }
185         ret = pthread_mutex_destroy(&audSyncLock_);
186         if (ret != 0) {
187             MEDIA_ERR_LOG("pthread_mutex_destroy audSyncLock_ failed");
188         }
189         isInited_ = false;
190     }
191     return ret;
192 }
193 
Init(void)194 int32_t PlayerSync::Init(void)
195 {
196     if (isInited_) {
197         return HI_SUCCESS;
198     }
199     int32_t ret = pthread_mutex_init(&vidSyncLock_, nullptr);
200     if (ret != 0) {
201         MEDIA_ERR_LOG("pthread_mutex_init vidSyncLock_ failed");
202         return ret;
203     }
204     ret = pthread_mutex_init(&audSyncLock_, nullptr);
205     if (ret != 0) {
206         MEDIA_ERR_LOG("pthread_mutex_init vidSyncLock_ failed");
207         (void)pthread_mutex_destroy(&vidSyncLock_);
208         return ret;
209     }
210     (void)Reset(SYNC_CHN_VID);
211     (void)Reset(SYNC_CHN_AUD);
212     isInited_ = true;
213     return HI_SUCCESS;
214 }
215 
Start(SyncChn syncChn)216 int32_t PlayerSync::Start(SyncChn syncChn)
217 {
218     if (isInited_ == false) {
219         MEDIA_ERR_LOG("sync start before inited");
220         return HI_FAILURE;
221     }
222     if (syncChn == SYNC_CHN_VID) {
223         isVidEnable_ = true;
224     } else if (syncChn == SYNC_CHN_AUD) {
225         isAudEnable_ = true;
226     } else {
227         MEDIA_ERR_LOG("invalid sync chn: %d", syncChn);
228         return HI_FAILURE;
229     }
230     return HI_SUCCESS;
231 }
232 
SetSpeed(float speed,TplayDirect tplayDirect)233 int32_t PlayerSync::SetSpeed(float speed, TplayDirect  tplayDirect)
234 {
235     if (isInited_ == false) {
236         MEDIA_ERR_LOG("sync TPlay before inited");
237         return HI_FAILURE;
238     }
239     pthread_mutex_lock(&vidSyncLock_);
240     speed_ = speed;
241     isInTrickPlayMode_ = (speed_ != 1.0) ? true : false;
242     tplayDirect_ = tplayDirect;
243     pthread_mutex_unlock(&vidSyncLock_);
244     return HI_SUCCESS;
245 }
246 
247 /* only used to resume from tplay to normal */
Resume()248 int32_t PlayerSync::Resume()
249 {
250     if (isInited_ == false) {
251         MEDIA_ERR_LOG("sync Resume before inited");
252         return HI_FAILURE;
253     }
254     pthread_mutex_lock(&vidSyncLock_);
255     isInTrickPlayMode_ = false;
256     speed_ = 1.0;
257     pthread_mutex_unlock(&vidSyncLock_);
258     return HI_SUCCESS;
259 }
260 
Stop(SyncChn syncChn)261 int32_t PlayerSync::Stop(SyncChn syncChn)
262 {
263     if (isInited_ == false) {
264         MEDIA_ERR_LOG("sync Stop before inited");
265         return HI_FAILURE;
266     }
267     if (syncChn == SYNC_CHN_VID) {
268         isVidEnable_ = false;
269         Reset(syncChn);
270     } else if (syncChn == SYNC_CHN_AUD) {
271         isAudEnable_ = false;
272         Reset(syncChn);
273     } else {
274         MEDIA_ERR_LOG("invalid sync chn: %d", syncChn);
275         return HI_FAILURE;
276     }
277     return HI_SUCCESS;
278 }
279 
UpdateCurTimeWithAudio(int64_t & nowUs)280 void PlayerSync::UpdateCurTimeWithAudio(int64_t& nowUs)
281 {
282     if (syncAttr_.refType == PLAYER_SYNC_REF_AUD) {
283         // ref with audio pts
284         if (lastAudioTsUs_ != AV_INVALID_PTS) {
285             /* compensate for audio frame pts gap maybe too long, eg: 64ms */
286             nowUs = lastAudioTsUs_ + (GetCurTimeUs() - lastAudioRealTsUs_);
287         }
288     }
289 }
290 
OnVideoFirstFrame(int64_t ptsUs)291 SyncRet PlayerSync::OnVideoFirstFrame(int64_t ptsUs)
292 {
293     // caculate pts and clock delta of first frame
294     int64_t firstTimeUs = GetCurTimeUs();
295     vidTimeSourceDelta_ = firstTimeUs - ptsUs;
296     lastVideoTsUs_ = ptsUs;
297     frstVidFrameTime_ = firstTimeUs;
298     firstVidFrameTs_ = ptsUs;
299     isFristVidFrame_ = false;
300     return SYNC_RET_PLAY;
301 }
302 
ProcVidFrame(int64_t ptsMs,SyncRet & result)303 int32_t PlayerSync::ProcVidFrame(int64_t ptsMs, SyncRet &result)
304 {
305     int32_t ret = HI_SUCCESS;
306     int64_t ptsUs;
307     int64_t nowUs;
308 
309     if (!isInited_) {
310         MEDIA_ERR_LOG("sync ProcVidFrame before inited");
311         return HI_FAILURE;
312     }
313     pthread_mutex_lock(&vidSyncLock_);
314     if (!isVidEnable_) {
315         MEDIA_ERR_LOG("sync module have not enabled");
316         result = SYNC_RET_DROP;
317         ret = HI_FAILURE;
318         goto UNLOCK;
319     }
320     if (syncAttr_.refType == PLAYER_SYNC_REF_NONE) {
321         result = SYNC_RET_PLAY;
322         goto UNLOCK;
323     }
324     if (ptsMs > INT64_MAX_VALUE / US2MS) {
325         goto UNLOCK;
326     }
327     ptsUs = ptsMs * US2MS;
328     // first video frame quickoutput
329     if (isFristVidFrame_) {
330         if (!isAudEnable_) {
331             if (frstVidFrameTime_ == 0) {
332                 frstVidFrameTime_ = GetCurTimeUs();
333             }
334             if (GetCurTimeUs() - frstVidFrameTime_ < FIRST_VID_RENDER_WAIT_TIME_WHEN_NO_AUDIO) {
335                 result = SYNC_RET_REPEAT;
336                 goto UNLOCK;
337             }
338         }
339         result = OnVideoFirstFrame(ptsUs);
340         goto UNLOCK;
341     }
342     if (isInTrickPlayMode_) {
343         (void)TPlayProcess(ptsUs, result);
344         lastVideoTsUs_ = ptsUs;
345         goto UNLOCK;
346     }
347     nowUs = GetCurTimeUs() - vidTimeSourceDelta_;
348     UpdateCurTimeWithAudio(nowUs);
349     diffAvRenderTime_ = (nowUs - ptsUs) / MS_SCALE;
350     if (syncAttr_.refType == PLAYER_SYNC_REF_VID) {
351         if (diffAvRenderTime_ < (-(int64_t)(syncAttr_.syncStartRegion.vidPlusTime))) {
352             result = SYNC_RET_REPEAT;
353         } else { /* do not care the fps and decode speed, just play */
354             result = SYNC_RET_PLAY;
355         }
356     } else {
357         result = CheckAVDiff(diffAvRenderTime_, continousVidLost_);
358     }
359     lastVideoTsUs_ = ptsUs;
360     /* fall through */
361 UNLOCK:
362     pthread_mutex_unlock(&vidSyncLock_);
363     return ret;
364 }
365 
ProcAudFrame(int64_t ptsMs,SyncRet & result)366 int32_t PlayerSync::ProcAudFrame(int64_t ptsMs, SyncRet &result)
367 {
368     if (!isInited_) {
369         MEDIA_ERR_LOG("sync ProcAudFrame before inited");
370         return HI_FAILURE;
371     }
372     pthread_mutex_lock(&audSyncLock_);
373     result = SYNC_RET_DROP;
374     if (!isAudEnable_) {
375         MEDIA_ERR_LOG("sync module have not enabled");
376         pthread_mutex_unlock(&audSyncLock_);
377         return HI_FAILURE;
378     }
379     if (isInTrickPlayMode_) {
380         pthread_mutex_unlock(&audSyncLock_);
381         return HI_SUCCESS;
382     }
383     if (ptsMs > INT64_MAX_VALUE / US2MS) {
384         pthread_mutex_unlock(&audSyncLock_);
385         return HI_SUCCESS;
386     }
387     /* video stream exist, and wait for first video frame. video out first */
388     if (isFristVidFrame_ && isVidEnable_) {
389         if (firstAudFrameTime_ == 0) {
390             firstAudFrameTime_ = GetCurTimeUs();
391         }
392         if (GetCurTimeUs() - firstAudFrameTime_ <= FIRST_AUDIO_RENDER_WAIT_VID_TIME) {
393             result = SYNC_RET_REPEAT;
394             pthread_mutex_unlock(&audSyncLock_);
395             return HI_SUCCESS;
396         }
397     }
398     int64_t timeUs = ptsMs * US2MS;
399     int64_t nowUs;
400     if (isFristAudFrame_) {
401         if (lastVideoTsUs_ != AV_INVALID_PTS && (lastVideoTsUs_ < timeUs)) {
402             result = SYNC_RET_REPEAT;
403             pthread_mutex_unlock(&audSyncLock_);
404             return HI_SUCCESS;
405         }
406         // caculate pts and clock delta of first frame
407         int64_t firstTimeUs = GetCurTimeUs();
408         audTimeSourceDelta_ = firstTimeUs - timeUs;
409         isFristAudFrame_ = false;
410         firstAudFrameTs_ = timeUs;
411     }
412     if (syncAttr_.refType == PLAYER_SYNC_REF_AUD) {
413         // calc current shouldbe timestamp ref in sys clock
414         nowUs = GetCurTimeUs() - audTimeSourceDelta_;
415         int64_t latenessMs = (nowUs - timeUs) / MS_SCALE;
416         /* if have mechanism to fast or shorter audio sound play, handle here, now we just ignore */
417         if (latenessMs > (int64_t)syncAttr_.syncStopRegion.vidNegativeTime ||
418             latenessMs < (-(int64_t)syncAttr_.syncStopRegion.vidPlusTime)) {
419         }
420         lastAudioTsUs_ = timeUs;
421         lastAudioRealTsUs_ = GetCurTimeUs();
422     } else if (syncAttr_.refType == PLAYER_SYNC_REF_VID) {
423         MEDIA_ERR_LOG("current do not support ref video, if have audio");
424         result = SYNC_RET_DROP;
425         pthread_mutex_unlock(&audSyncLock_);
426         return HI_FAILURE;
427     }
428     result = SYNC_RET_PLAY;
429     pthread_mutex_unlock(&audSyncLock_);
430     return HI_SUCCESS;
431 }
432 };
433 };
434