diff --git a/ext-codec/H264.cpp b/ext-codec/H264.cpp index 59471775..953c11fe 100644 --- a/ext-codec/H264.cpp +++ b/ext-codec/H264.cpp @@ -117,7 +117,7 @@ size_t prefixSize(const char *ptr, size_t len) { H264Track::H264Track(const string &sps, const string &pps, int sps_prefix_len, int pps_prefix_len) { _sps = sps.substr(sps_prefix_len); _pps = pps.substr(pps_prefix_len); - update(); + H264Track::update(); } CodecId H264Track::getCodecId() const { @@ -235,6 +235,10 @@ void H264Track::setExtraData(const uint8_t *data, size_t bytes) { } bool H264Track::update() { + _config_frames = std::vector{ + createConfigFrame(_sps), + createConfigFrame(_pps) + }; return getAVCInfo(_sps, _width, _height, _fps); } @@ -284,23 +288,11 @@ bool H264Track::inputFrame_l(const Frame::Ptr &frame) { void H264Track::insertConfigFrame(const Frame::Ptr &frame) { if (!_sps.empty()) { - auto spsFrame = FrameImp::create(); - spsFrame->_prefix_size = 4; - spsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - spsFrame->_buffer.append(_sps); - spsFrame->_dts = frame->dts(); - spsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(spsFrame); + VideoTrack::inputFrame(createConfigFrame(_sps, frame->dts(), frame->getIndex())); } if (!_pps.empty()) { - auto ppsFrame = FrameImp::create(); - ppsFrame->_prefix_size = 4; - ppsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - ppsFrame->_buffer.append(_pps); - ppsFrame->_dts = frame->dts(); - ppsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(ppsFrame); + VideoTrack::inputFrame(createConfigFrame(_pps, frame->dts(), frame->getIndex())); } } diff --git a/ext-codec/H264.h b/ext-codec/H264.h index cf054da0..9940854c 100644 --- a/ext-codec/H264.h +++ b/ext-codec/H264.h @@ -115,6 +115,7 @@ public: toolkit::Buffer::Ptr getExtraData() const override; void setExtraData(const uint8_t *data, size_t size) override; bool update() override; + std::vector getConfigFrames() const override { return _config_frames; } private: Sdp::Ptr getSdp(uint8_t payload_type) const override; @@ -129,7 +130,20 @@ private: float _fps = 0; std::string _sps; std::string _pps; + std::vector _config_frames; }; +template +Frame::Ptr createConfigFrame(const std::string &data, uint64_t dts = 0, int index = 0) { + auto frame = FrameType::create(); + frame->_prefix_size = 4; + frame->_buffer.assign("\x00\x00\x00\x01", 4); + frame->_buffer.append(data); + frame->_dts = dts; + frame->setIndex(index); + return frame; +} + }//namespace mediakit + #endif //ZLMEDIAKIT_H264_H diff --git a/ext-codec/H265.cpp b/ext-codec/H265.cpp index 3a999255..cc0831f6 100644 --- a/ext-codec/H265.cpp +++ b/ext-codec/H265.cpp @@ -66,7 +66,7 @@ H265Track::H265Track(const string &vps,const string &sps, const string &pps,int _vps = vps.substr(vps_prefix_len); _sps = sps.substr(sps_prefix_len); _pps = pps.substr(pps_prefix_len); - update(); + H265Track::update(); } CodecId H265Track::getCodecId() const { @@ -182,6 +182,11 @@ void H265Track::setExtraData(const uint8_t *data, size_t bytes) { } bool H265Track::update() { + _config_frames = std::vector{ + createConfigFrame(_vps), + createConfigFrame(_sps), + createConfigFrame(_pps) + }; return getHEVCInfo(_vps, _sps, _width, _height, _fps); } @@ -194,32 +199,13 @@ void H265Track::insertConfigFrame(const Frame::Ptr &frame) { return; } if (!_vps.empty()) { - auto vpsFrame = FrameImp::create(); - vpsFrame->_prefix_size = 4; - vpsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - vpsFrame->_buffer.append(_vps); - vpsFrame->_dts = frame->dts(); - vpsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(vpsFrame); + VideoTrack::inputFrame(createConfigFrame(_vps)); } if (!_sps.empty()) { - auto spsFrame = FrameImp::create(); - spsFrame->_prefix_size = 4; - spsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - spsFrame->_buffer.append(_sps); - spsFrame->_dts = frame->dts(); - spsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(spsFrame); + VideoTrack::inputFrame(createConfigFrame(_sps)); } - if (!_pps.empty()) { - auto ppsFrame = FrameImp::create(); - ppsFrame->_prefix_size = 4; - ppsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - ppsFrame->_buffer.append(_pps); - ppsFrame->_dts = frame->dts(); - ppsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(ppsFrame); + VideoTrack::inputFrame(createConfigFrame(_pps)); } } diff --git a/ext-codec/H265.h b/ext-codec/H265.h index 50775727..8132f444 100644 --- a/ext-codec/H265.h +++ b/ext-codec/H265.h @@ -142,6 +142,7 @@ public: toolkit::Buffer::Ptr getExtraData() const override; void setExtraData(const uint8_t *data, size_t size) override; bool update() override; + std::vector getConfigFrames() const override { return _config_frames; } private: Sdp::Ptr getSdp(uint8_t payload_type) const override; @@ -157,6 +158,7 @@ private: std::string _vps; std::string _sps; std::string _pps; + std::vector _config_frames; }; }//namespace mediakit diff --git a/src/Extension/Track.h b/src/Extension/Track.h index ba101282..006bd87a 100644 --- a/src/Extension/Track.h +++ b/src/Extension/Track.h @@ -109,6 +109,11 @@ public: * 返回视频fps */ virtual float getVideoFps() const { return 0; } + + /** + * 返回相关 sps/pps 等 + */ + virtual std::vector getConfigFrames() const { return std::vector{}; } }; class VideoTrackImp : public VideoTrack { diff --git a/webrtc/WebRtcPlayer.cpp b/webrtc/WebRtcPlayer.cpp index cfafae2b..b410a154 100644 --- a/webrtc/WebRtcPlayer.cpp +++ b/webrtc/WebRtcPlayer.cpp @@ -9,7 +9,10 @@ */ #include "WebRtcPlayer.h" + #include "Common/config.h" +#include "Extension/Factory.h" +#include "Util/base64.h" using namespace std; @@ -32,6 +35,9 @@ WebRtcPlayer::WebRtcPlayer(const EventPoller::Ptr &poller, _media_info = info; _play_src = src; CHECK(src); + + GET_CONFIG(bool, direct_proxy, Rtsp::kDirectProxy); + _send_config_frames_once = direct_proxy; } void WebRtcPlayer::onStartWebRTC() { @@ -56,6 +62,13 @@ void WebRtcPlayer::onStartWebRTC() { if (!strong_self) { return; } + + if (strong_self->_send_config_frames_once && !pkt->empty()) { + const auto &first_rtp = pkt->front(); + strong_self->sendConfigFrames(first_rtp->getSeq(), first_rtp->sample_rate, first_rtp->getStamp(), first_rtp->ntp_stamp); + strong_self->_send_config_frames_once = false; + } + size_t i = 0; pkt->for_each([&](const RtpPacket::Ptr &rtp) { //TraceL<<"send track type:"<type<<" ts:"<getStamp()<<" ntp:"<ntp_stamp<<" size:"<getPayloadSize()<<" i:"<getSdp()); } +void WebRtcPlayer::sendConfigFrames(uint32_t before_seq, uint32_t sample_rate, uint32_t timestamp, uint64_t ntp_timestamp) { + auto play_src = _play_src.lock(); + if (!play_src) { + return; + } + auto video_track = std::dynamic_pointer_cast(play_src->getTrack(mediakit::TrackVideo)); + if (!video_track) { + return; + } + auto frames = video_track->getConfigFrames(); + if (frames.empty()) { + return; + } + auto encoder = mediakit::Factory::getRtpEncoderByCodecId(video_track->getCodecId(), 0); + if (!encoder) { + return; + } + + GET_CONFIG(uint32_t, video_mtu, Rtp::kVideoMtuSize); + encoder->setRtpInfo(0, video_mtu, sample_rate, 0, 0, 0); + + auto seq = before_seq - frames.size(); + for (const auto &frame : video_track->getConfigFrames()) { + auto rtp = encoder->getRtpInfo().makeRtp( + TrackVideo, frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize(), false, 0); + auto header = rtp->getHeader(); + header->seq = htons(seq++); + header->stamp = htonl(timestamp); + rtp->ntp_stamp = ntp_timestamp; + onSendRtp(rtp, false); + } +} + }// namespace mediakit \ No newline at end of file diff --git a/webrtc/WebRtcPlayer.h b/webrtc/WebRtcPlayer.h index ccacd410..96e9e830 100644 --- a/webrtc/WebRtcPlayer.h +++ b/webrtc/WebRtcPlayer.h @@ -31,11 +31,17 @@ protected: private: WebRtcPlayer(const EventPoller::Ptr &poller, const RtspMediaSource::Ptr &src, const MediaInfo &info); + void sendConfigFrames(uint32_t before_seq, uint32_t sample_rate, uint32_t timestamp, uint64_t ntp_timestamp); + private: //媒体相关元数据 MediaInfo _media_info; //播放的rtsp源 std::weak_ptr _play_src; + + // rtp 直接转发情况下通常会缺少 sps/pps, 在转发 rtp 前, 先发送一次相关帧信息, 部分情况下是可以播放的 + bool _send_config_frames_once { false }; + //播放rtsp源的reader对象 RtspMediaSource::RingType::RingReader::Ptr _reader; };