From d62248161973f8d12530f37fd4955bb417234935 Mon Sep 17 00:00:00 2001 From: xiongguangjie Date: Sun, 26 Jun 2022 00:54:31 +0800 Subject: [PATCH] fix multi aac frame in one frame result flv.js play audio error --- src/Common/MultiMediaSourceMuxer.cpp | 54 +------------------- src/Extension/AAC.cpp | 7 ++- src/Extension/Frame.h | 74 ++++++++++++++++++++++++++++ srt/SrtTransportImp.cpp | 16 +++++- srt/SrtTransportImp.hpp | 2 + 5 files changed, 97 insertions(+), 56 deletions(-) diff --git a/src/Common/MultiMediaSourceMuxer.cpp b/src/Common/MultiMediaSourceMuxer.cpp index cdfdde39..53515dba 100644 --- a/src/Common/MultiMediaSourceMuxer.cpp +++ b/src/Common/MultiMediaSourceMuxer.cpp @@ -354,64 +354,12 @@ void MultiMediaSourceMuxer::resetTracks() { } } -//该类实现frame级别的时间戳覆盖 -class FrameModifyStamp : public Frame{ -public: - typedef std::shared_ptr Ptr; - FrameModifyStamp(const Frame::Ptr &frame, Stamp &stamp){ - _frame = frame; - //覆盖时间戳 - stamp.revise(frame->dts(), frame->pts(), _dts, _pts, true); - } - ~FrameModifyStamp() override {} - - uint32_t dts() const override{ - return (uint32_t)_dts; - } - - uint32_t pts() const override{ - return (uint32_t)_pts; - } - - size_t prefixSize() const override { - return _frame->prefixSize(); - } - - bool keyFrame() const override { - return _frame->keyFrame(); - } - - bool configFrame() const override { - return _frame->configFrame(); - } - - bool cacheAble() const override { - return _frame->cacheAble(); - } - - char *data() const override { - return _frame->data(); - } - - size_t size() const override { - return _frame->size(); - } - - CodecId getCodecId() const override { - return _frame->getCodecId(); - } -private: - int64_t _dts; - int64_t _pts; - Frame::Ptr _frame; -}; - bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) { GET_CONFIG(bool, modify_stamp, General::kModifyStamp); auto frame = frame_in; if (modify_stamp) { //开启了时间戳覆盖 - frame = std::make_shared(frame, _stamp[frame->getTrackType()]); + frame = std::make_shared(frame, _stamp[frame->getTrackType()],true); } bool ret = false; diff --git a/src/Extension/AAC.cpp b/src/Extension/AAC.cpp index 33f28dce..1d923271 100644 --- a/src/Extension/AAC.cpp +++ b/src/Extension/AAC.cpp @@ -274,6 +274,9 @@ bool AACTrack::inputFrame(const Frame::Ptr &frame) { bool ret = false; //有adts头，尝试分帧 + int64_t dts = frame->dts(); + int64_t pts = frame->pts(); + auto ptr = frame->data(); auto end = frame->data() + frame->size(); while (ptr < end) { @@ -284,7 +287,7 @@ bool AACTrack::inputFrame(const Frame::Ptr &frame) { if (frame_len == frame->size()) { return inputFrame_l(frame); } - auto sub_frame = std::make_shared >(frame, (char *) ptr, frame_len, ADTS_HEADER_LEN); + auto sub_frame = std::make_shared >(frame, (char *) ptr, frame_len, ADTS_HEADER_LEN,dts,pts); ptr += frame_len; if (ptr > end) { WarnL << "invalid aac length in adts header: " << frame_len @@ -295,6 +298,8 @@ bool AACTrack::inputFrame(const Frame::Ptr &frame) { if (inputFrame_l(sub_frame)) { ret = true; } + dts += 1024*1000/getAudioSampleRate(); + pts += 1024*1000/getAudioSampleRate(); } return ret; } diff --git a/src/Extension/Frame.h b/src/Extension/Frame.h index 3fb26a11..497eb0c5 100644 --- a/src/Extension/Frame.h +++ b/src/Extension/Frame.h @@ -15,6 +15,7 @@ #include #include "Util/RingBuffer.h" #include "Network/Socket.h" +#include "Common/Stamp.h" namespace mediakit{ @@ -262,6 +263,27 @@ private: Frame::Ptr _parent_frame; }; +/** + * 一个Frame类中可以有多个帧(AAC)，时间戳会变化 + * ZLMediaKit会先把这种复合帧split成单个帧然后再处理 + * 一个复合帧可以通过无内存拷贝的方式切割成多个子Frame + * 提供该类的目的是切割复合帧时防止内存拷贝，提高性能 + */ +template +class FrameTSInternal : public Parent{ +public: + typedef std::shared_ptr Ptr; + FrameTSInternal(const Frame::Ptr &parent_frame, char *ptr, size_t size, size_t prefix_size,uint32_t dts,uint32_t pts) + : Parent(ptr, size, dts, pts, prefix_size) { + _parent_frame = parent_frame; + } + bool cacheAble() const override { + return _parent_frame->cacheAble(); + } +private: + Frame::Ptr _parent_frame; +}; + /** * 写帧接口的抽象接口类 */ @@ -498,6 +520,58 @@ private: FrameImp::Ptr _buffer; }; +//该类实现frame级别的时间戳覆盖 +class FrameStamp : public Frame{ +public: + typedef std::shared_ptr Ptr; + FrameStamp(const Frame::Ptr &frame, Stamp &stamp,bool modify_stamp){ + _frame = frame; + //覆盖时间戳 + stamp.revise(frame->dts(), frame->pts(), _dts, _pts, modify_stamp); + } + ~FrameStamp() override {} + + uint32_t dts() const override{ + return (uint32_t)_dts; + } + + uint32_t pts() const override{ + return (uint32_t)_pts; + } + + size_t prefixSize() const override { + return _frame->prefixSize(); + } + + bool keyFrame() const override { + return _frame->keyFrame(); + } + + bool configFrame() const override { + return _frame->configFrame(); + } + + bool cacheAble() const override { + return _frame->cacheAble(); + } + + char *data() const override { + return _frame->data(); + } + + size_t size() const override { + return _frame->size(); + } + + CodecId getCodecId() const override { + return _frame->getCodecId(); + } +private: + int64_t _dts; + int64_t _pts; + Frame::Ptr _frame; +}; + /** * 该对象可以把Buffer对象转换成可缓存的Frame对象 */ diff --git a/srt/SrtTransportImp.cpp b/srt/SrtTransportImp.cpp index abd7843b..ca5c0082 100644 --- a/srt/SrtTransportImp.cpp +++ b/srt/SrtTransportImp.cpp @@ -287,7 +287,10 @@ std::string SrtTransportImp::getIdentifier() const { bool SrtTransportImp::inputFrame(const Frame::Ptr &frame) { if (_muxer) { - return _muxer->inputFrame(frame); + //TraceL<<"before type "<getCodecName()<<" dts "<dts()<<" pts "<pts(); + auto frame_tmp = std::make_shared(frame, _type_to_stamp[frame->getTrackType()],false); + //TraceL<<"after type "<getCodecName()<<" dts "<dts()<<" pts "<pts(); + return _muxer->inputFrame(frame_tmp); } if (_cached_func.size() > 200) { WarnL << "cached frame of track(" << frame->getCodecName() << ") is too much, now dropped"; @@ -295,11 +298,17 @@ bool SrtTransportImp::inputFrame(const Frame::Ptr &frame) { } auto frame_cached = Frame::getCacheAbleFrame(frame); lock_guard lck(_func_mtx); - _cached_func.emplace_back([this, frame_cached]() { _muxer->inputFrame(frame_cached); }); + _cached_func.emplace_back([this, frame_cached]() { + //TraceL<<"before type "<getCodecName()<<" dts "<dts()<<" pts "<pts(); + auto frame_tmp = std::make_shared(frame_cached, _type_to_stamp[frame_cached->getTrackType()],false); + //TraceL<<"after type "<getCodecName()<<" dts "<dts()<<" pts "<pts(); + _muxer->inputFrame(frame_tmp); + }); return true; } bool SrtTransportImp::addTrack(const Track::Ptr &track) { + _type_to_stamp.emplace(track->getTrackType(),Stamp()); if (_muxer) { return _muxer->addTrack(track); } @@ -316,6 +325,9 @@ void SrtTransportImp::addTrackCompleted() { lock_guard lck(_func_mtx); _cached_func.emplace_back([this]() { _muxer->addTrackCompleted(); }); } + if(_type_to_stamp.size() >1){ + _type_to_stamp[TrackType::TrackAudio].syncTo(_type_to_stamp[TrackType::TrackVideo]); + } } void SrtTransportImp::doCachedFunc() { diff --git a/srt/SrtTransportImp.hpp b/srt/SrtTransportImp.hpp index 3a8f53a1..3ff24901 100644 --- a/srt/SrtTransportImp.hpp +++ b/srt/SrtTransportImp.hpp @@ -89,6 +89,8 @@ private: DecoderImp::Ptr _decoder; std::recursive_mutex _func_mtx; std::deque> _cached_func; + + std::unordered_map _type_to_stamp; }; } // namespace SRT