From a91f4d29f2827c915a5135e231a2eef2c4d257be Mon Sep 17 00:00:00 2001 From: ziyue <1213642868@qq.com> Date: Thu, 24 Jun 2021 11:39:55 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84simulcast=E6=8E=A5=E6=94=B6?= =?UTF-8?q?=E5=8F=8A=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- webrtc/WebRtcTransport.cpp | 66 +++++++++++++------- webrtc/WebRtcTransport.h | 3 +- webrtc/offer-simulcast.sdp | 123 +++++-------------------------------- www/webrtc/index.html | 2 +- 4 files changed, 62 insertions(+), 132 deletions(-) diff --git a/webrtc/WebRtcTransport.cpp b/webrtc/WebRtcTransport.cpp index bdcf48fc..22e85d4f 100644 --- a/webrtc/WebRtcTransport.cpp +++ b/webrtc/WebRtcTransport.cpp @@ -761,35 +761,38 @@ void WebRtcTransportImp::onRtp_l(const char *buf, size_t len, bool rtx) { return; } #endif + auto &ref = info->receiver[ssrc]; if (!rtx) { //统计rtp接受情况，便于生成nack rtcp包 info->nack_ctx[ssrc].received(seq); //时间戳转换成毫秒 auto stamp_ms = ntohl(rtp->stamp) * uint64_t(1000) / info->plan_rtp->sample_rate; + //统计rtp收到的情况，好做rr汇报 - auto &ref = info->rtcp_context_recv[ssrc]; - if (!ref) { - ref = std::make_shared(info->plan_rtp->sample_rate, true); + auto &cxt_ref = info->rtcp_context_recv[ssrc]; + if (!cxt_ref) { + cxt_ref = std::make_shared(info->plan_rtp->sample_rate, true); } - ref->onRtp(seq, stamp_ms, len); + cxt_ref->onRtp(seq, stamp_ms, len); + //修改ext id至统一 - changeRtpExtId(*info, rtp, true, false); - } + string rid; + changeRtpExtId(*info, rtp, true, false, &rid); + if (!ref) { + ref = std::make_shared([info, this, rid](RtpPacket::Ptr rtp) mutable { + onSortedRtp(*info, rid, std::move(rtp)); + }); + info->nack_ctx[ssrc].setOnNack([info, this, ssrc](const FCI_NACK &nack) mutable { + onSendNack(*info, nack, ssrc); + }); + //recv simulcast ssrc --> RtpPayloadInfo + _rtp_info_ssrc[ssrc] = std::make_pair(false, info); + InfoL << "receive rtp of ssrc:" << ssrc; + } + } //解析并排序rtp - auto &ref = info->receiver[ssrc]; - if (!ref) { - ref = std::make_shared([info, this](RtpPacket::Ptr rtp) mutable { - onSortedRtp(*info, std::move(rtp)); - }); - info->nack_ctx[ssrc].setOnNack([info, this, ssrc](const FCI_NACK &nack) mutable { - onSendNack(*info, nack, ssrc); - }); - //recv simulcast ssrc --> RtpPayloadInfo - _rtp_info_ssrc[ssrc] = std::make_pair(false, info); - InfoL << "receive rtp of ssrc:" << ssrc; - } - + assert(ref); ref->inputRtp(info->media->type, info->plan_rtp->sample_rate, (uint8_t *) buf, len); return; } @@ -809,9 +812,9 @@ void WebRtcTransportImp::onRtp_l(const char *buf, size_t len, bool rtx) { auto origin_seq = payload[0] << 8 | payload[1]; rtp->seq = htons(origin_seq); if (info->offer_ssrc_rtp) { - //非simulcast + //非simulcast或音频 rtp->ssrc = htonl(info->offer_ssrc_rtp); - TraceL << "received rtx rtp,ssrc: " << ssrc << ", seq:" << origin_seq; + TraceL << "received rtx rtp,ssrc: " << ssrc << ", seq:" << origin_seq << ", pt:" << (int)rtp->pt; } else { //todo simulcast下，辅码流通过rtx传输？ //simulcast情况下，根据rtx的ssrc查找rtp的ssrc @@ -834,7 +837,7 @@ void WebRtcTransportImp::onSendNack(RtpPayloadInfo &info, const FCI_NACK &nack, /////////////////////////////////////////////////////////////////// -void WebRtcTransportImp::onSortedRtp(RtpPayloadInfo &info, RtpPacket::Ptr rtp) { +void WebRtcTransportImp::onSortedRtp(RtpPayloadInfo &info, const string &rid, RtpPacket::Ptr rtp) { if (info.media->type == TrackVideo && _pli_ticker.elapsedTime() > 2000) { //定期发送pli请求关键帧，方便非rtc等协议 _pli_ticker.resetTime(); @@ -848,7 +851,24 @@ void WebRtcTransportImp::onSortedRtp(RtpPayloadInfo &info, RtpPacket::Ptr rtp) { } if (_push_src) { - _push_src->onWrite(std::move(rtp), false); + if (rtp->type == TrackAudio) { + //音频 + for (auto &pr : _push_src_simulcast) { + pr.second->onWrite(rtp, false); + } + } else { + //视频 + auto &src = _push_src_simulcast[rid]; + if (!src) { + auto stream_id = rid.empty() ? _push_src->getId() : _push_src->getId() + "_" + rid; + auto src_imp = std::make_shared(_push_src->getVhost(), _push_src->getApp(), stream_id); + src_imp->setSdp(_push_src->getSdp()); + src_imp->setProtocolTranslation(_push_src->isRecording(Recorder::type_hls),_push_src->isRecording(Recorder::type_mp4)); + src_imp->setListener(shared_from_this()); + src = src_imp; + } + src->onWrite(std::move(rtp), false); + } } } diff --git a/webrtc/WebRtcTransport.h b/webrtc/WebRtcTransport.h index a309e6bd..c62f549e 100644 --- a/webrtc/WebRtcTransport.h +++ b/webrtc/WebRtcTransport.h @@ -358,7 +358,7 @@ private: unordered_map > receiver; }; - void onSortedRtp(RtpPayloadInfo &info, RtpPacket::Ptr rtp); + void onSortedRtp(RtpPayloadInfo &info, const string &rid, RtpPacket::Ptr rtp); void onSendNack(RtpPayloadInfo &info, const FCI_NACK &nack, uint32_t ssrc); void changeRtpExtId(RtpPayloadInfo &info, const RtpHeader *header, bool is_recv, bool is_rtx = false, string *rid_ptr = nullptr) const; @@ -380,6 +380,7 @@ private: Socket::Ptr _socket; //推流的rtsp源 RtspMediaSource::Ptr _push_src; + unordered_map _push_src_simulcast; //播放的rtsp源 RtspMediaSource::Ptr _play_src; //播放rtsp源的reader对象 diff --git a/webrtc/offer-simulcast.sdp b/webrtc/offer-simulcast.sdp index d6f1ff84..1bba3912 100644 --- a/webrtc/offer-simulcast.sdp +++ b/webrtc/offer-simulcast.sdp @@ -1,6 +1,5 @@ -# chrome的sdp v=0 -o=- 403371946498103831 2 IN IP4 127.0.0.1 +o=- 1520777637155103417 2 IN IP4 127.0.0.1 s=- t=0 0 a=group:BUNDLE 0 1 @@ -9,10 +8,10 @@ a=msid-semantic: WMS m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126 c=IN IP4 0.0.0.0 a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:pW4Z -a=ice-pwd:S38S++HW3eTcPTyytsNI1XVp +a=ice-ufrag:ma0v +a=ice-pwd:H8+DMsvKNQE+Qz1uS7cZby1+ a=ice-options:trickle -a=fingerprint:sha-256 04:32:7B:56:7D:F7:D4:EC:65:7C:04:6C:F8:0B:03:F0:35:A9:1A:C3:43:3E:18:95:67:E6:0D:D1:EE:C9:16:8C +a=fingerprint:sha-256 F1:30:4D:FE:6B:9A:BE:B4:31:65:30:C1:67:87:2F:74:23:7A:06:31:B0:49:DE:44:53:69:27:30:86:1F:E0:6C a=setup:actpass a=mid:0 a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level @@ -22,7 +21,7 @@ a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id a=sendrecv -a=msid:- a3c6a137-1291-45cd-b985-07a9bd365452 +a=msid:- 7bf9dab3-79e9-4969-b3d2-44beae8b4286 a=rtcp-mux a=rtpmap:111 opus/48000/2 a=rtcp-fb:111 transport-cc @@ -39,17 +38,17 @@ a=rtpmap:110 telephone-event/48000 a=rtpmap:112 telephone-event/32000 a=rtpmap:113 telephone-event/16000 a=rtpmap:126 telephone-event/8000 -a=ssrc:3626257331 cname:JSFJMbaE9Pu5tevN -a=ssrc:3626257331 msid:- a3c6a137-1291-45cd-b985-07a9bd365452 -a=ssrc:3626257331 mslabel:- -a=ssrc:3626257331 label:a3c6a137-1291-45cd-b985-07a9bd365452 +a=ssrc:3454457472 cname:kbZgD5tgXGqTwvD1 +a=ssrc:3454457472 msid:- 7bf9dab3-79e9-4969-b3d2-44beae8b4286 +a=ssrc:3454457472 mslabel:- +a=ssrc:3454457472 label:7bf9dab3-79e9-4969-b3d2-44beae8b4286 m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 102 121 127 120 125 107 108 109 35 36 124 119 123 118 114 115 116 c=IN IP4 0.0.0.0 a=rtcp:9 IN IP4 0.0.0.0 -a=ice-ufrag:pW4Z -a=ice-pwd:S38S++HW3eTcPTyytsNI1XVp +a=ice-ufrag:ma0v +a=ice-pwd:H8+DMsvKNQE+Qz1uS7cZby1+ a=ice-options:trickle -a=fingerprint:sha-256 04:32:7B:56:7D:F7:D4:EC:65:7C:04:6C:F8:0B:03:F0:35:A9:1A:C3:43:3E:18:95:67:E6:0D:D1:EE:C9:16:8C +a=fingerprint:sha-256 F1:30:4D:FE:6B:9A:BE:B4:31:65:30:C1:67:87:2F:74:23:7A:06:31:B0:49:DE:44:53:69:27:30:86:1F:E0:6C a=setup:actpass a=mid:1 a=extmap:14 urn:ietf:params:rtp-hdrext:toffset @@ -64,7 +63,7 @@ a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id a=sendrecv -a=msid:- 261e5384-9cf6-479d-9d59-aaf924d1a2ea +a=msid:- fa2b9d11-98f5-4ea9-bf0a-6098069c6940 a=rtcp-mux a=rtcp-rsize a=rtpmap:96 VP8/90000 @@ -143,7 +142,7 @@ a=rtcp-fb:124 transport-cc a=rtcp-fb:124 ccm fir a=rtcp-fb:124 nack a=rtcp-fb:124 nack pli -a=fmtp:124 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f +a=fmtp:124 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0032 a=rtpmap:119 rtx/90000 a=fmtp:119 apt=124 a=rtpmap:123 H264/90000 @@ -152,7 +151,7 @@ a=rtcp-fb:123 transport-cc a=rtcp-fb:123 ccm fir a=rtcp-fb:123 nack a=rtcp-fb:123 nack pli -a=fmtp:123 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f +a=fmtp:123 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032 a=rtpmap:118 rtx/90000 a=fmtp:118 apt=123 a=rtpmap:114 red/90000 @@ -162,94 +161,4 @@ a=rtpmap:116 ulpfec/90000 a=rid:q send a=rid:h send a=rid:f send -a=simulcast:send q;h;f - -#firefox的sdp -v=0 -o=mozilla...THIS_IS_SDPARTA-88.0.1 3954544078885279475 0 IN IP4 0.0.0.0 -s=- -t=0 0 -a=fingerprint:sha-256 9B:4F:D1:D2:A5:ED:08:BC:E8:D7:DD:D8:59:2C:E6:3D:19:F9:4C:67:9C:D9:9B:7B:C9:47:7A:3A:1F:05:C8:96 -a=group:BUNDLE 0 1 -a=ice-options:trickle -a=msid-semantic:WMS * -m=audio 9 UDP/TLS/RTP/SAVPF 109 9 0 8 101 -c=IN IP4 0.0.0.0 -a=sendrecv -a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level -a=extmap:2/recvonly urn:ietf:params:rtp-hdrext:csrc-audio-level -a=extmap:3 urn:ietf:params:rtp-hdrext:sdes:mid -a=fmtp:109 maxplaybackrate=48000;stereo=1;useinbandfec=1 -a=fmtp:101 0-15 -a=ice-pwd:92a9ced6d734f7ff2a45cde8b29572a9 -a=ice-ufrag:b986b945 -a=mid:0 -a=msid:- {ea61729a-c244-4c79-aeb7-b57765fefa26} -a=rtcp-mux -a=rtpmap:109 opus/48000/2 -a=rtpmap:9 G722/8000/1 -a=rtpmap:0 PCMU/8000 -a=rtpmap:8 PCMA/8000 -a=rtpmap:101 telephone-event/8000/1 -a=setup:actpass -a=ssrc:3000327501 cname:{12e7c547-559e-46e2-94db-f1ad474c95dc} -m=video 9 UDP/TLS/RTP/SAVPF 120 124 121 125 126 127 97 98 -c=IN IP4 0.0.0.0 -a=sendrecv -a=extmap:3 urn:ietf:params:rtp-hdrext:sdes:mid -a=extmap:4 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time -a=extmap:5 urn:ietf:params:rtp-hdrext:toffset -a=extmap:6/recvonly http://www.webrtc.org/experiments/rtp-hdrext/playout-delay -a=extmap:7 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 -a=extmap:8/sendonly urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id -a=extmap:9/sendonly urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id -a=fmtp:126 profile-level-id=42e01f;level-asymmetry-allowed=1;packetization-mode=1 -a=fmtp:97 profile-level-id=42e01f;level-asymmetry-allowed=1 -a=fmtp:120 max-fs=12288;max-fr=60 -a=fmtp:124 apt=120 -a=fmtp:121 max-fs=12288;max-fr=60 -a=fmtp:125 apt=121 -a=fmtp:127 apt=126 -a=fmtp:98 apt=97 -a=ice-pwd:92a9ced6d734f7ff2a45cde8b29572a9 -a=ice-ufrag:b986b945 -a=mid:1 -a=msid:- {3bfe1b80-20eb-4b42-b8b7-fac45fb281bf} -a=rid:q send -a=rid:h send -a=rid:f send -a=rtcp-fb:120 nack -a=rtcp-fb:120 nack pli -a=rtcp-fb:120 ccm fir -a=rtcp-fb:120 goog-remb -a=rtcp-fb:120 transport-cc -a=rtcp-fb:121 nack -a=rtcp-fb:121 nack pli -a=rtcp-fb:121 ccm fir -a=rtcp-fb:121 goog-remb -a=rtcp-fb:121 transport-cc -a=rtcp-fb:126 nack -a=rtcp-fb:126 nack pli -a=rtcp-fb:126 ccm fir -a=rtcp-fb:126 goog-remb -a=rtcp-fb:126 transport-cc -a=rtcp-fb:97 nack -a=rtcp-fb:97 nack pli -a=rtcp-fb:97 ccm fir -a=rtcp-fb:97 goog-remb -a=rtcp-fb:97 transport-cc -a=rtcp-mux -a=rtcp-rsize -a=rtpmap:120 VP8/90000 -a=rtpmap:124 rtx/90000 -a=rtpmap:121 VP9/90000 -a=rtpmap:125 rtx/90000 -a=rtpmap:126 H264/90000 -a=rtpmap:127 rtx/90000 -a=rtpmap:97 H264/90000 -a=rtpmap:98 rtx/90000 -a=setup:actpass -a=simulcast:send q;h;f -a=ssrc:2581133096 cname:{12e7c547-559e-46e2-94db-f1ad474c95dc} -a=ssrc:773854125 cname:{12e7c547-559e-46e2-94db-f1ad474c95dc} -a=ssrc:4100728001 cname:{12e7c547-559e-46e2-94db-f1ad474c95dc} +a=simulcast:send q;h;f \ No newline at end of file diff --git a/www/webrtc/index.html b/www/webrtc/index.html index 9007d34b..ee93d79a 100644 --- a/www/webrtc/index.html +++ b/www/webrtc/index.html @@ -108,7 +108,7 @@ element: document.getElementById('video'),// video 标签 debug: true,// 是否打印日志 zlmsdpUrl:document.getElementById('streamUrl').value,//流地址 - simulcast:false,//document.getElementById('simulcast').checked, + simulcast:document.getElementById('simulcast').checked, useCamera:document.getElementById('useCamera').checked, audioEnable:document.getElementById('audioEnable').checked, videoEnable:document.getElementById('videoEnable').checked,