Send config frames once before sending DirectProxy RTP packets (#3611)

ext-codec/H264.cpp

@@ -117,7 +117,7 @@ size_t prefixSize(const char *ptr, size_t len) {
 H264Track::H264Track(const string &sps, const string &pps, int sps_prefix_len, int pps_prefix_len) {
     _sps = sps.substr(sps_prefix_len);
     _pps = pps.substr(pps_prefix_len);
-    update();
+    H264Track::update();
 }
 
 CodecId H264Track::getCodecId() const {
@@ -235,6 +235,10 @@ void H264Track::setExtraData(const uint8_t *data, size_t bytes) {
 }
 
 bool H264Track::update() {
+    _config_frames = std::vector<Frame::Ptr>{
+        createConfigFrame<H264Frame>(_sps),
+        createConfigFrame<H264Frame>(_pps)
+    };
     return getAVCInfo(_sps, _width, _height, _fps);
 }
 
@@ -284,23 +288,11 @@ bool H264Track::inputFrame_l(const Frame::Ptr &frame) {
 
 void H264Track::insertConfigFrame(const Frame::Ptr &frame) {
     if (!_sps.empty()) {
-        auto spsFrame = FrameImp::create<H264Frame>();
+        VideoTrack::inputFrame(createConfigFrame<H264Frame>(_sps, frame->dts(), frame->getIndex()));
-        spsFrame->_prefix_size = 4;
-        spsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
-        spsFrame->_buffer.append(_sps);
-        spsFrame->_dts = frame->dts();
-        spsFrame->setIndex(frame->getIndex());
-        VideoTrack::inputFrame(spsFrame);
     }
 
     if (!_pps.empty()) {
-        auto ppsFrame = FrameImp::create<H264Frame>();
+        VideoTrack::inputFrame(createConfigFrame<H264Frame>(_pps, frame->dts(), frame->getIndex()));
-        ppsFrame->_prefix_size = 4;
-        ppsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
-        ppsFrame->_buffer.append(_pps);
-        ppsFrame->_dts = frame->dts();
-        ppsFrame->setIndex(frame->getIndex());
-        VideoTrack::inputFrame(ppsFrame);
     }
 }
 

ext-codec/H264.h

@@ -115,6 +115,7 @@ public:
     toolkit::Buffer::Ptr getExtraData() const override;
     void setExtraData(const uint8_t *data, size_t size) override;
     bool update() override;
+    std::vector<Frame::Ptr> getConfigFrames() const override { return _config_frames; }
 
 private:
     Sdp::Ptr getSdp(uint8_t payload_type) const override;
@@ -129,7 +130,20 @@ private:
     float _fps = 0;
     std::string _sps;
     std::string _pps;
+    std::vector<Frame::Ptr> _config_frames;
 };
 
+template <typename FrameType>
+Frame::Ptr createConfigFrame(const std::string &data, uint64_t dts = 0, int index = 0) {
+    auto frame = FrameType::create();
+    frame->_prefix_size = 4;
+    frame->_buffer.assign("\x00\x00\x00\x01", 4);
+    frame->_buffer.append(data);
+    frame->_dts = dts;
+    frame->setIndex(index);
+    return frame;
+}
+
 }//namespace mediakit
+
 #endif //ZLMEDIAKIT_H264_H

ext-codec/H265.cpp

@@ -66,7 +66,7 @@ H265Track::H265Track(const string &vps,const string &sps, const string &pps,int
     _vps = vps.substr(vps_prefix_len);
     _sps = sps.substr(sps_prefix_len);
     _pps = pps.substr(pps_prefix_len);
-    update();
+    H265Track::update();
 }
 
 CodecId H265Track::getCodecId() const {
@@ -182,6 +182,11 @@ void H265Track::setExtraData(const uint8_t *data, size_t bytes) {
 }
 
 bool H265Track::update() {
+    _config_frames = std::vector<Frame::Ptr>{
+        createConfigFrame<H265Frame>(_vps),
+        createConfigFrame<H265Frame>(_sps),
+        createConfigFrame<H265Frame>(_pps)
+    };
     return getHEVCInfo(_vps, _sps, _width, _height, _fps);
 }
 
@@ -194,32 +199,13 @@ void H265Track::insertConfigFrame(const Frame::Ptr &frame) {
         return;
     }
     if (!_vps.empty()) {
-        auto vpsFrame = FrameImp::create<H265Frame>();
+        VideoTrack::inputFrame(createConfigFrame<H265Frame>(_vps));
-        vpsFrame->_prefix_size = 4;
-        vpsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
-        vpsFrame->_buffer.append(_vps);
-        vpsFrame->_dts = frame->dts();
-        vpsFrame->setIndex(frame->getIndex());
-        VideoTrack::inputFrame(vpsFrame);
     }
     if (!_sps.empty()) {
-        auto spsFrame = FrameImp::create<H265Frame>();
+        VideoTrack::inputFrame(createConfigFrame<H265Frame>(_sps));
-        spsFrame->_prefix_size = 4;
-        spsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
-        spsFrame->_buffer.append(_sps);
-        spsFrame->_dts = frame->dts();
-        spsFrame->setIndex(frame->getIndex());
-        VideoTrack::inputFrame(spsFrame);
     }
-
     if (!_pps.empty()) {
-        auto ppsFrame = FrameImp::create<H265Frame>();
+        VideoTrack::inputFrame(createConfigFrame<H265Frame>(_pps));
-        ppsFrame->_prefix_size = 4;
-        ppsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
-        ppsFrame->_buffer.append(_pps);
-        ppsFrame->_dts = frame->dts();
-        ppsFrame->setIndex(frame->getIndex());
-        VideoTrack::inputFrame(ppsFrame);
     }
 }
 

ext-codec/H265.h

@@ -142,6 +142,7 @@ public:
     toolkit::Buffer::Ptr getExtraData() const override;
     void setExtraData(const uint8_t *data, size_t size) override;
     bool update() override;
+    std::vector<Frame::Ptr> getConfigFrames() const override { return _config_frames; }
 
 private:
     Sdp::Ptr getSdp(uint8_t payload_type) const override;
@@ -157,6 +158,7 @@ private:
     std::string _vps;
     std::string _sps;
     std::string _pps;
+    std::vector<Frame::Ptr> _config_frames;
 };
 
 }//namespace mediakit

src/Extension/Track.h

@@ -109,6 +109,11 @@ public:
      * 返回视频fps
      */
     virtual float getVideoFps() const { return 0; }
+
+    /**
+     * 返回相关 sps/pps 等
+     */
+    virtual std::vector<Frame::Ptr> getConfigFrames() const { return std::vector<Frame::Ptr>{}; }
 };
 
 class VideoTrackImp : public VideoTrack {

webrtc/WebRtcPlayer.cpp

@@ -9,7 +9,10 @@
  */
 
 #include "WebRtcPlayer.h"
+
 #include "Common/config.h"
+#include "Extension/Factory.h"
+#include "Util/base64.h"
 
 using namespace std;
 
@@ -32,6 +35,9 @@ WebRtcPlayer::WebRtcPlayer(const EventPoller::Ptr &poller,
     _media_info = info;
     _play_src = src;
     CHECK(src);
+
+    GET_CONFIG(bool, direct_proxy, Rtsp::kDirectProxy);
+    _send_config_frames_once = direct_proxy;
 }
 
 void WebRtcPlayer::onStartWebRTC() {
@@ -56,6 +62,13 @@ void WebRtcPlayer::onStartWebRTC() {
             if (!strong_self) {
                 return;
             }
+
+            if (strong_self->_send_config_frames_once && !pkt->empty()) {
+                const auto &first_rtp = pkt->front();
+                strong_self->sendConfigFrames(first_rtp->getSeq(), first_rtp->sample_rate, first_rtp->getStamp(), first_rtp->ntp_stamp);
+                strong_self->_send_config_frames_once = false;
+            }
+
             size_t i = 0;
             pkt->for_each([&](const RtpPacket::Ptr &rtp) {
                 //TraceL<<"send track type:"<<rtp->type<<" ts:"<<rtp->getStamp()<<" ntp:"<<rtp->ntp_stamp<<" size:"<<rtp->getPayloadSize()<<" i:"<<i;
@@ -111,4 +124,37 @@ void WebRtcPlayer::onRtcConfigure(RtcConfigure &configure) const {
     configure.setPlayRtspInfo(playSrc->getSdp());
 }
 
+void WebRtcPlayer::sendConfigFrames(uint32_t before_seq, uint32_t sample_rate, uint32_t timestamp, uint64_t ntp_timestamp) {
+    auto play_src = _play_src.lock();
+    if (!play_src) {
+        return;
+    }
+    auto video_track = std::dynamic_pointer_cast<mediakit::VideoTrack>(play_src->getTrack(mediakit::TrackVideo));
+    if (!video_track) {
+        return;
+    }
+    auto frames = video_track->getConfigFrames();
+    if (frames.empty()) {
+        return;
+    }
+    auto encoder = mediakit::Factory::getRtpEncoderByCodecId(video_track->getCodecId(), 0);
+    if (!encoder) {
+        return;
+    }
+
+    GET_CONFIG(uint32_t, video_mtu, Rtp::kVideoMtuSize);
+    encoder->setRtpInfo(0, video_mtu, sample_rate, 0, 0, 0);
+
+    auto seq = before_seq - frames.size();
+    for (const auto &frame : video_track->getConfigFrames()) {
+        auto rtp = encoder->getRtpInfo().makeRtp(
+            TrackVideo, frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize(), false, 0);
+        auto header = rtp->getHeader();
+        header->seq = htons(seq++);
+        header->stamp = htonl(timestamp);
+        rtp->ntp_stamp = ntp_timestamp;
+        onSendRtp(rtp, false);
+    }
+}
+
 }// namespace mediakit

webrtc/WebRtcPlayer.h

@@ -31,11 +31,17 @@ protected:
 private:
     WebRtcPlayer(const EventPoller::Ptr &poller, const RtspMediaSource::Ptr &src, const MediaInfo &info);
 
+    void sendConfigFrames(uint32_t before_seq, uint32_t sample_rate, uint32_t timestamp, uint64_t ntp_timestamp);
+
 private:
     //媒体相关元数据
     MediaInfo _media_info;
     //播放的rtsp源
     std::weak_ptr<RtspMediaSource> _play_src;
+
+    // rtp 直接转发情况下通常会缺少 sps/pps, 在转发 rtp 前, 先发送一次相关帧信息, 部分情况下是可以播放的
+    bool _send_config_frames_once { false };
+
     //播放rtsp源的reader对象
     RtspMediaSource::RingType::RingReader::Ptr _reader;
 };