fix multi aac frame in one frame result flv.js play audio error

src/Common/MultiMediaSourceMuxer.cpp

@@ -354,64 +354,12 @@ void MultiMediaSourceMuxer::resetTracks() {
     }
 }
 
-//该类实现frame级别的时间戳覆盖
-class FrameModifyStamp : public Frame{
-public:
-    typedef std::shared_ptr<FrameModifyStamp> Ptr;
-    FrameModifyStamp(const Frame::Ptr &frame, Stamp &stamp){
-        _frame = frame;
-        //覆盖时间戳
-        stamp.revise(frame->dts(), frame->pts(), _dts, _pts, true);
-    }
-    ~FrameModifyStamp() override {}
-
-    uint32_t dts() const override{
-        return (uint32_t)_dts;
-    }
-
-    uint32_t pts() const override{
-        return (uint32_t)_pts;
-    }
-
-    size_t prefixSize() const override {
-        return _frame->prefixSize();
-    }
-
-    bool keyFrame() const override {
-        return _frame->keyFrame();
-    }
-
-    bool configFrame() const override {
-        return _frame->configFrame();
-    }
-
-    bool cacheAble() const override {
-        return _frame->cacheAble();
-    }
-
-    char *data() const override {
-        return _frame->data();
-    }
-
-    size_t size() const override {
-        return _frame->size();
-    }
-
-    CodecId getCodecId() const override {
-        return _frame->getCodecId();
-    }
-private:
-    int64_t _dts;
-    int64_t _pts;
-    Frame::Ptr _frame;
-};
-
 bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) {
     GET_CONFIG(bool, modify_stamp, General::kModifyStamp);
     auto frame = frame_in;
     if (modify_stamp) {
         //开启了时间戳覆盖
-        frame = std::make_shared<FrameModifyStamp>(frame, _stamp[frame->getTrackType()]);
+        frame = std::make_shared<FrameStamp>(frame, _stamp[frame->getTrackType()],true);
     }
 
     bool ret = false;

src/Extension/AAC.cpp

@@ -274,6 +274,9 @@ bool AACTrack::inputFrame(const Frame::Ptr &frame) {
 
     bool ret = false;
     //有adts头，尝试分帧
+    int64_t dts = frame->dts();
+    int64_t pts = frame->pts();
+
     auto ptr = frame->data();
     auto end = frame->data() + frame->size();
     while (ptr < end) {
@@ -284,7 +287,7 @@ bool AACTrack::inputFrame(const Frame::Ptr &frame) {
         if (frame_len == frame->size()) {
             return inputFrame_l(frame);
         }
-        auto sub_frame = std::make_shared<FrameInternal<FrameFromPtr> >(frame, (char *) ptr, frame_len, ADTS_HEADER_LEN);
+        auto sub_frame = std::make_shared<FrameTSInternal<FrameFromPtr> >(frame, (char *) ptr, frame_len, ADTS_HEADER_LEN,dts,pts);
         ptr += frame_len;
         if (ptr > end) {
             WarnL << "invalid aac length in adts header: " << frame_len
@@ -295,6 +298,8 @@ bool AACTrack::inputFrame(const Frame::Ptr &frame) {
         if (inputFrame_l(sub_frame)) {
             ret = true;
         }
+        dts += 1024*1000/getAudioSampleRate();
+        pts += 1024*1000/getAudioSampleRate();
     }
     return ret;
 }

src/Extension/Frame.h

@@ -15,6 +15,7 @@
 #include <functional>
 #include "Util/RingBuffer.h"
 #include "Network/Socket.h"
+#include "Common/Stamp.h"
 
 namespace mediakit{
 
@@ -262,6 +263,27 @@ private:
     Frame::Ptr _parent_frame;
 };
 
+/**
+ * 一个Frame类中可以有多个帧(AAC)，时间戳会变化
+ * ZLMediaKit会先把这种复合帧split成单个帧然后再处理
+ * 一个复合帧可以通过无内存拷贝的方式切割成多个子Frame
+ * 提供该类的目的是切割复合帧时防止内存拷贝，提高性能
+ */
+template<typename Parent>
+class FrameTSInternal : public Parent{
+public:
+    typedef std::shared_ptr<FrameTSInternal> Ptr;
+    FrameTSInternal(const Frame::Ptr &parent_frame, char *ptr, size_t size, size_t prefix_size,uint32_t dts,uint32_t pts)
+            : Parent(ptr, size, dts, pts, prefix_size) {
+        _parent_frame = parent_frame;
+    }
+    bool cacheAble() const override {
+        return _parent_frame->cacheAble();
+    }
+private:
+    Frame::Ptr _parent_frame;
+};
+
 /**
  * 写帧接口的抽象接口类
  */
@@ -498,6 +520,58 @@ private:
     FrameImp::Ptr _buffer;
 };
 
+//该类实现frame级别的时间戳覆盖
+class FrameStamp : public Frame{
+public:
+    typedef std::shared_ptr<FrameStamp> Ptr;
+    FrameStamp(const Frame::Ptr &frame, Stamp &stamp,bool modify_stamp){
+        _frame = frame;
+        //覆盖时间戳
+        stamp.revise(frame->dts(), frame->pts(), _dts, _pts, modify_stamp);
+    }
+    ~FrameStamp() override {}
+
+    uint32_t dts() const override{
+        return (uint32_t)_dts;
+    }
+
+    uint32_t pts() const override{
+        return (uint32_t)_pts;
+    }
+
+    size_t prefixSize() const override {
+        return _frame->prefixSize();
+    }
+
+    bool keyFrame() const override {
+        return _frame->keyFrame();
+    }
+
+    bool configFrame() const override {
+        return _frame->configFrame();
+    }
+
+    bool cacheAble() const override {
+        return _frame->cacheAble();
+    }
+
+    char *data() const override {
+        return _frame->data();
+    }
+
+    size_t size() const override {
+        return _frame->size();
+    }
+
+    CodecId getCodecId() const override {
+        return _frame->getCodecId();
+    }
+private:
+    int64_t _dts;
+    int64_t _pts;
+    Frame::Ptr _frame;
+};
+
 /**
  * 该对象可以把Buffer对象转换成可缓存的Frame对象
  */

srt/SrtTransportImp.cpp

@@ -287,7 +287,10 @@ std::string SrtTransportImp::getIdentifier() const {
 
 bool SrtTransportImp::inputFrame(const Frame::Ptr &frame) {
     if (_muxer) {
-        return _muxer->inputFrame(frame);
+        //TraceL<<"before type "<<frame->getCodecName()<<" dts "<<frame->dts()<<" pts "<<frame->pts();
+        auto frame_tmp = std::make_shared<FrameStamp>(frame, _type_to_stamp[frame->getTrackType()],false);
+        //TraceL<<"after type "<<frame_tmp->getCodecName()<<" dts "<<frame_tmp->dts()<<" pts "<<frame_tmp->pts();
+        return _muxer->inputFrame(frame_tmp);
     }
     if (_cached_func.size() > 200) {
         WarnL << "cached frame of track(" << frame->getCodecName() << ") is too much, now dropped";
@@ -295,11 +298,17 @@ bool SrtTransportImp::inputFrame(const Frame::Ptr &frame) {
     }
     auto frame_cached = Frame::getCacheAbleFrame(frame);
     lock_guard<recursive_mutex> lck(_func_mtx);
-    _cached_func.emplace_back([this, frame_cached]() { _muxer->inputFrame(frame_cached); });
+    _cached_func.emplace_back([this, frame_cached]() { 
+        //TraceL<<"before type "<<frame_cached->getCodecName()<<" dts "<<frame_cached->dts()<<" pts "<<frame_cached->pts();
+        auto frame_tmp = std::make_shared<FrameStamp>(frame_cached, _type_to_stamp[frame_cached->getTrackType()],false);
+        //TraceL<<"after type "<<frame_tmp->getCodecName()<<" dts "<<frame_tmp->dts()<<" pts "<<frame_tmp->pts();
+        _muxer->inputFrame(frame_tmp);
+    });
     return true;
 }
 
 bool SrtTransportImp::addTrack(const Track::Ptr &track) {
+    _type_to_stamp.emplace(track->getTrackType(),Stamp());
     if (_muxer) {
         return _muxer->addTrack(track);
     }
@@ -316,6 +325,9 @@ void SrtTransportImp::addTrackCompleted() {
         lock_guard<recursive_mutex> lck(_func_mtx);
         _cached_func.emplace_back([this]() { _muxer->addTrackCompleted(); });
     }
+    if(_type_to_stamp.size() >1){
+        _type_to_stamp[TrackType::TrackAudio].syncTo(_type_to_stamp[TrackType::TrackVideo]);
+    }
 }
 
 void SrtTransportImp::doCachedFunc() {

srt/SrtTransportImp.hpp

@@ -89,6 +89,8 @@ private:
     DecoderImp::Ptr _decoder;
     std::recursive_mutex _func_mtx;
     std::deque<std::function<void()>> _cached_func;
+
+    std::unordered_map<int, Stamp> _type_to_stamp;
 };
 
 } // namespace SRT