/* * Copyright (c) 2023-2023 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "video_sink.h" #include #include "common/log.h" #include "media_sync_manager.h" #include "osal/task/jobutils.h" #include "syspara/parameters.h" namespace { constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "HiStreamer" }; constexpr int64_t LAG_LIMIT_TIME = 100; } namespace OHOS { namespace Media { namespace Pipeline { int64_t GetVideoLatencyFixDelay() { constexpr uint64_t defaultValue = 0; static uint64_t fixDelay = OHOS::system::GetUintParameter("debug.media_service.video_sync_fix_delay", defaultValue); MEDIA_LOG_I_SHORT("video_sync_fix_delay, pid:%{public}d, fixdelay: " PUBLIC_LOG_U64, getprocpid(), fixDelay); return (int64_t)fixDelay; } /// Video Key Frame Flag constexpr int BUFFER_FLAG_KEY_FRAME = 0x00000002; constexpr int64_t WAIT_TIME_US_THRESHOLD = 1500000; // max sleep time 1.5s constexpr int64_t SINK_TIME_US_THRESHOLD = 100000; // max sink time 100ms constexpr int64_t PER_SINK_TIME_THRESHOLD = 33000; // max per sink time 33ms // Video Sync Start Frame constexpr int VIDEO_SINK_START_FRAME = 4; constexpr int64_t WAIT_TIME_US_THRESHOLD_WARNING = 40000; // warning threshold 40ms VideoSink::VideoSink() { refreshTime_ = 0; syncerPriority_ = IMediaSynchronizer::VIDEO_SINK; fixDelay_ = GetVideoLatencyFixDelay(); MEDIA_LOG_I("VideoSink ctor called..."); } VideoSink::~VideoSink() { MEDIA_LOG_I("VideoSink dtor called..."); this->eventReceiver_ = nullptr; } void VideoSink::UpdateTimeAnchorActually(const std::shared_ptr& buffer) { auto syncCenter = syncCenter_.lock(); FALSE_RETURN(syncCenter != nullptr && buffer != nullptr); int64_t nowCt = syncCenter->GetClockTimeNow(); uint64_t latency = 0; (void)GetLatency(latency); syncCenter->UpdateTimeAnchor(nowCt, latency, buffer->pts_ - firstPts_, buffer->pts_, buffer->duration_, this); } int64_t VideoSink::DoSyncWrite(const std::shared_ptr& buffer) { FALSE_RETURN_V(buffer != nullptr, false); int64_t waitTime = 0; bool render = true; auto syncCenter = syncCenter_.lock(); if ((buffer->flag_ & BUFFER_FLAG_EOS) == 0) { int64_t nowCt = syncCenter ? syncCenter->GetClockTimeNow() : 0; if (isFirstFrame_) { FALSE_RETURN_V(syncCenter != nullptr, false); isFirstFrame_ = false; firstFrameNowct_ = nowCt; firstFramePts_ = buffer->pts_; } else { waitTime = CheckBufferLatenessMayWait(buffer); } if (syncCenter) { uint64_t latency = 0; (void)GetLatency(latency); render = syncCenter->UpdateTimeAnchor(nowCt + waitTime, latency, buffer->pts_ - firstPts_, buffer->pts_, buffer->duration_, this); } lagDetector_.CalcLag(buffer); lastTimeStamp_ = buffer->pts_ - firstPts_; } else { MEDIA_LOG_I("Videosink receives EOS."); if (syncCenter) { syncCenter->ReportEos(this); } return -1; } if ((render && waitTime >= 0) || lastFrameDropped_) { lastFrameDropped_ = false; renderFrameCnt_++; return waitTime > 0 ? waitTime : 0; } lastFrameDropped_ = true; discardFrameCnt_++; return -1; } void VideoSink::ResetSyncInfo() { ResetPrerollReported(); isFirstFrame_ = true; lastTimeStamp_ = HST_TIME_NONE; lastBufferTime_ = HST_TIME_NONE; seekFlag_ = false; lastPts_ = HST_TIME_NONE; lastClockTime_ = HST_TIME_NONE; lagDetector_.Reset(); } Status VideoSink::GetLatency(uint64_t& nanoSec) { nanoSec = 10; // 10 ns return Status::OK; } void VideoSink::SetSeekFlag() { seekFlag_ = true; // seek后首次不走平滑 } void VideoSink::SetLastPts(int64_t lastPts) { lastPts_ = lastPts; auto syncCenter = syncCenter_.lock(); if (syncCenter != nullptr) { lastClockTime_ = syncCenter->GetClockTimeNow(); } } int64_t VideoSink::CheckBufferLatenessMayWait(const std::shared_ptr& buffer) { FALSE_RETURN_V(buffer != nullptr, true); bool tooLate = false; auto syncCenter = syncCenter_.lock(); FALSE_RETURN_V(syncCenter != nullptr, true); auto pts = buffer->pts_ - firstPts_; auto ct4Buffer = syncCenter->GetClockTime(pts); MEDIA_LOG_D("VideoSink cur pts: " PUBLIC_LOG_D64 " us, ct4Buffer: " PUBLIC_LOG_D64 " us, buf_pts: " PUBLIC_LOG_D64 " us, fixDelay: " PUBLIC_LOG_D64 " us", pts, ct4Buffer, buffer->pts_, fixDelay_); FALSE_RETURN_V(ct4Buffer != Plugins::HST_TIME_NONE, 0); int64_t waitTimeUs = 0; if (lastBufferTime_ != HST_TIME_NONE && seekFlag_ == false) { int64_t thisBufferTime = lastBufferTime_ + pts - lastTimeStamp_; int64_t deltaTime = ct4Buffer - thisBufferTime; deltaTimeAccu_ = (deltaTimeAccu_ * 9 + deltaTime) / 10; // 9 10 for smoothing if (std::abs(deltaTimeAccu_) < 5 * HST_USECOND) { // 5ms ct4Buffer = thisBufferTime; } MEDIA_LOG_D("lastBfTime:" PUBLIC_LOG_D64" us, lastTS:" PUBLIC_LOG_D64, lastBufferTime_, lastTimeStamp_); } else { seekFlag_ = (seekFlag_ == true) ? false : seekFlag_; } auto nowCt = syncCenter->GetClockTimeNow(); uint64_t latency = 0; GetLatency(latency); float speed = GetSpeed(syncCenter->GetPlaybackRate()); auto diff = nowCt + (int64_t) latency - ct4Buffer + fixDelay_; // anhor diff auto diff2 = (nowCt - lastClockTime_) - static_cast((buffer->pts_ - lastPts_) / speed); // video diff auto diff3 = diff2 - PER_SINK_TIME_THRESHOLD; // video diff with PER_SINK_TIME_THRESHOLD if (discardFrameCnt_ + renderFrameCnt_ < VIDEO_SINK_START_FRAME) { diff = (nowCt - firstFrameNowct_) - (buffer->pts_ - firstFramePts_); MEDIA_LOG_I("VideoSink first few times diff is " PUBLIC_LOG_D64 " us", diff); } else if (diff < 0 && diff2 < SINK_TIME_US_THRESHOLD && diff < diff3) { // per frame render time reduced by 33ms diff = diff3; } MEDIA_LOG_D("VS ct4Bf:" PUBLIC_LOG_D64 "diff:" PUBLIC_LOG_D64 "nowCt:" PUBLIC_LOG_D64, ct4Buffer, diff, nowCt); if (diff < 0) { // buffer is early, diff < 0 or 0 < diff < 40ms(25Hz) render it waitTimeUs = 0 - diff; MEDIA_LOG_I_FALSE_D((waitTimeUs >= WAIT_TIME_US_THRESHOLD_WARNING), "buffer is too early waitTimeUs: " PUBLIC_LOG_D64, waitTimeUs); if (waitTimeUs > WAIT_TIME_US_THRESHOLD) { waitTimeUs = WAIT_TIME_US_THRESHOLD; } } else if (diff > 0 && Plugins::HstTime2Ms(diff * HST_USECOND) > 40) { // > 40ms, buffer is late tooLate = true; MEDIA_LOG_D("buffer is too late"); } lastBufferTime_ = ct4Buffer; bool dropFlag = tooLate && ((buffer->flag_ & BUFFER_FLAG_KEY_FRAME) == 0); // buffer is too late, drop it return dropFlag ? -1 : waitTimeUs; } void VideoSink::SetSyncCenter(std::shared_ptr syncCenter) { MEDIA_LOG_I("VideoSink::SetSyncCenter"); syncCenter_ = syncCenter; MediaSynchronousSink::Init(); } void VideoSink::SetEventReceiver(const std::shared_ptr &receiver) { this->eventReceiver_ = receiver; } void VideoSink::SetFirstPts(int64_t pts) { auto syncCenter = syncCenter_.lock(); if (firstPts_ == HST_TIME_NONE) { if (syncCenter && syncCenter->GetMediaStartPts() != HST_TIME_NONE) { firstPts_ = syncCenter->GetMediaStartPts(); } else { firstPts_ = pts; } MEDIA_LOG_I("video DoSyncWrite set firstPts = " PUBLIC_LOG_D64, firstPts_); } } Status VideoSink::SetParameter(const std::shared_ptr& meta) { UpdateMediaTimeRange(meta); return Status::OK; } float VideoSink::GetSpeed(float speed) { if (std::fabs(speed - 0) < 1e-9) { return 1.0f; } return speed; } Status VideoSink::GetLagInfo(int32_t& lagTimes, int32_t& maxLagDuration, int32_t& avgLagDuration) { lagDetector_.GetLagInfo(lagTimes, maxLagDuration, avgLagDuration); return Status::OK; } bool VideoSink::VideoLagDetector::CalcLag(std::shared_ptr buffer) { FALSE_RETURN_V(!(buffer->flag_ & (uint32_t)(Plugins::AVBufferFlag::EOS)), false); auto systemTimeMsNow = Plugins::GetCurrentMillisecond(); auto systemTimeMsDiff = systemTimeMsNow - lastSystemTimeMs_; auto bufferTimeUsNow = Plugins::Us2Ms(buffer->pts_); auto bufferTimeMsDiff = bufferTimeUsNow - lastBufferTimeMs_; auto lagTimeMs = systemTimeMsDiff - bufferTimeMsDiff; bool isVideoLag = lastSystemTimeMs_ > 0 && lagTimeMs >= LAG_LIMIT_TIME; MEDIA_LOG_I_FALSE_D(isVideoLag, "prePts " PUBLIC_LOG_D64 " curPts " PUBLIC_LOG_D64 " ptsDiff " PUBLIC_LOG_D64 " tDiff " PUBLIC_LOG_D64, lastBufferTimeMs_, bufferTimeUsNow, bufferTimeMsDiff, systemTimeMsDiff); lastSystemTimeMs_ = systemTimeMsNow; lastBufferTimeMs_ = bufferTimeUsNow; if (isVideoLag) { ResolveLagEvent(lagTimeMs); } return isVideoLag; } void VideoSink::VideoLagDetector::SetEventReceiver(const std::shared_ptr eventReceiver) { eventReceiver_ = eventReceiver; } void VideoSink::VideoLagDetector::ResolveLagEvent(const int64_t &lagTimeMs) { lagTimes_++; maxLagDuration_ = std::max(maxLagDuration_, lagTimeMs); totalLagDuration_ += lagTimeMs; FALSE_RETURN(eventReceiver_ != nullptr); eventReceiver_->OnDfxEvent({"VideoSink", DfxEventType::DFX_INFO_PLAYER_VIDEO_LAG, lagTimeMs}); } void VideoSink::VideoLagDetector::GetLagInfo(int32_t& lagTimes, int32_t& maxLagDuration, int32_t& avgLagDuration) { lagTimes = lagTimes_; maxLagDuration = static_cast(maxLagDuration_); if (lagTimes_ != 0) { avgLagDuration = static_cast(totalLagDuration_ / lagTimes_); } else { avgLagDuration = 0; } } void VideoSink::VideoLagDetector::Reset() { lagTimes_ = 0; maxLagDuration_ = 0; lastSystemTimeMs_ = 0; lastBufferTimeMs_ = 0; totalLagDuration_ = 0; } } // namespace Pipeline } // namespace MEDIA } // namespace OHOS