From a28aeb214833b83126bd0016c60ab111df875e76 Mon Sep 17 00:00:00 2001 From: xiongguangjie Date: Sat, 19 Jun 2021 01:27:13 +0800 Subject: [PATCH 1/5] for frame merge must has vlc(video codec layer) data and flush rtmp a frame must has vcl --- src/Extension/Frame.cpp | 35 ++++++++++++++++++++++++++++++++++- src/Extension/Frame.h | 1 + src/Extension/H264Rtmp.cpp | 16 +++++++++++++--- src/Extension/H264Rtmp.h | 1 + 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/Extension/Frame.cpp b/src/Extension/Frame.cpp index 62753f59..62965553 100644 --- a/src/Extension/Frame.cpp +++ b/src/Extension/Frame.cpp @@ -267,11 +267,44 @@ bool FrameMerger::shouldDrop(const Frame::Ptr &frame) const{ } return false; } +bool FrameMerger::frameCacheHasVCL(List &frameCached) const{ + bool hasVCL = false; + bool isH264OrH265 = false; + frameCached.for_each([&hasVCL,&isH264OrH265](const Frame::Ptr &frame){ + switch (frame->getCodecId()) { + case CodecH264:{ + auto type = H264_TYPE(frame->data()[frame->prefixSize()]); + if(type >=H264Frame::NAL_B_P && type <= H264Frame::NAL_IDR){ + //有编码数据 + hasVCL=true; + } + isH264OrH265 = true; + break; + } + case CodecH265: { + //如果是新的一帧，前面的缓存需要输出 + auto type = H265_TYPE(frame->data()[frame->prefixSize()]); + if(type>=H265Frame::NAL_TRAIL_R &&type<= H265Frame::NAL_RSV_IRAP_VCL23){ + //有编码数据 + hasVCL=true; + } + isH264OrH265 = true; + break; + } + default: break; + } + }); + if(isH264OrH265){ + return hasVCL; + } + return true; + +} void FrameMerger::inputFrame(const Frame::Ptr &frame, const onOutput &cb) { if(shouldDrop(frame)){ return; } - if (willFlush(frame)) { + if (willFlush(frame) && frameCacheHasVCL(_frameCached)) { Frame::Ptr back = _frameCached.back(); Buffer::Ptr merged_frame = back; bool have_idr = back->keyFrame(); diff --git a/src/Extension/Frame.h b/src/Extension/Frame.h index 95a5ec6d..973b6359 100644 --- a/src/Extension/Frame.h +++ b/src/Extension/Frame.h @@ -476,6 +476,7 @@ private: bool willFlush(const Frame::Ptr &frame) const; void doMerge(BufferLikeString &buffer, const Frame::Ptr &frame) const; bool shouldDrop(const Frame::Ptr &frame) const; + bool frameCacheHasVCL(List &frameCached) const; private: int _type; diff --git a/src/Extension/H264Rtmp.cpp b/src/Extension/H264Rtmp.cpp index b3426fe8..6765b476 100644 --- a/src/Extension/H264Rtmp.cpp +++ b/src/Extension/H264Rtmp.cpp @@ -159,7 +159,7 @@ void H264RtmpEncoder::inputFrame(const Frame::Ptr &frame) { auto pcData = frame->data() + frame->prefixSize(); auto iLen = frame->size() - frame->prefixSize(); auto type = H264_TYPE(((uint8_t*)pcData)[0]); - if(type == H264Frame::NAL_SEI){ + if(type == H264Frame::NAL_SEI || type == H264Frame::NAL_AUD){ return; } @@ -182,10 +182,20 @@ void H264RtmpEncoder::inputFrame(const Frame::Ptr &frame) { break; } } - - if(_lastPacket && (_lastPacket->time_stamp != frame->dts() || ((pcData[1]&0x80) != 0 && type>=H264Frame::NAL_B_P && type<=H264Frame::NAL_IDR))) { + if((frame->configFrame() || frame->keyFrame()) && _lastPacket){ + // key frame or sps pps flush frame RtmpCodec::inputRtmp(_lastPacket); _lastPacket = nullptr; + _lastPacketHasVCL = false; + } + + if(_lastPacket && (_lastPacket->time_stamp != frame->dts() || ((pcData[1]&0x80) != 0 && type>=H264Frame::NAL_B_P && type<=H264Frame::NAL_IDR && _lastPacketHasVCL))) { + RtmpCodec::inputRtmp(_lastPacket); + _lastPacket = nullptr; + _lastPacketHasVCL = false; + } + if(type>=H264Frame::NAL_B_P && type<=H264Frame::NAL_IDR){ + _lastPacketHasVCL = true; } if(!_lastPacket) { diff --git a/src/Extension/H264Rtmp.h b/src/Extension/H264Rtmp.h index a7c243ca..c45da3a6 100644 --- a/src/Extension/H264Rtmp.h +++ b/src/Extension/H264Rtmp.h @@ -80,6 +80,7 @@ private: private: H264Track::Ptr _track; bool _gotSpsPps = false; + bool _lastPacketHasVCL = false; RtmpPacket::Ptr _lastPacket; }; From d215502ff548ba8b9028be97ec445bed9d3449d4 Mon Sep 17 00:00:00 2001 From: xiongguangjie Date: Sat, 19 Jun 2021 01:54:57 +0800 Subject: [PATCH 2/5] rtmp packet must has vlc when flush --- src/Extension/H264Rtmp.cpp | 4 ++-- src/Extension/H265Rtmp.cpp | 14 +++++++++++++- src/Extension/H265Rtmp.h | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Extension/H264Rtmp.cpp b/src/Extension/H264Rtmp.cpp index 6765b476..f3c7734c 100644 --- a/src/Extension/H264Rtmp.cpp +++ b/src/Extension/H264Rtmp.cpp @@ -182,8 +182,8 @@ void H264RtmpEncoder::inputFrame(const Frame::Ptr &frame) { break; } } - if((frame->configFrame() || frame->keyFrame()) && _lastPacket){ - // key frame or sps pps flush frame + if(frame->configFrame() && _lastPacket &&_lastPacketHasVCL){ + //sps pps flush frame RtmpCodec::inputRtmp(_lastPacket); _lastPacket = nullptr; _lastPacketHasVCL = false; diff --git a/src/Extension/H265Rtmp.cpp b/src/Extension/H265Rtmp.cpp index 84dddad6..cc46905b 100644 --- a/src/Extension/H265Rtmp.cpp +++ b/src/Extension/H265Rtmp.cpp @@ -169,9 +169,21 @@ void H265RtmpEncoder::inputFrame(const Frame::Ptr &frame) { return;// 防止sei aud 作为一帧 } - if (_lastPacket && (_lastPacket->time_stamp != frame->dts() || (type >=H264Frame::NAL_B_P && type<=H264Frame::NAL_IDR && (pcData[2]>>7 &0x01) !=0))) { + if(frame->configFrame() && _lastPacket &&_lastPacketHasVCL){ + // sps pps flush frame RtmpCodec::inputRtmp(_lastPacket); _lastPacket = nullptr; + _lastPacketHasVCL = false; + } + + if (_lastPacket && (_lastPacket->time_stamp != frame->dts() || (_lastPacketHasVCL &&type>=H265Frame::NAL_TRAIL_R &&type<= H265Frame::NAL_RSV_IRAP_VCL23 && (pcData[2]>>7 &0x01) !=0))) { + RtmpCodec::inputRtmp(_lastPacket); + _lastPacket = nullptr; + _lastPacketHasVCL = false; + } + + if(type>=H265Frame::NAL_TRAIL_R &&type<= H265Frame::NAL_RSV_IRAP_VCL23){ + _lastPacketHasVCL = true; } if(!_lastPacket) { diff --git a/src/Extension/H265Rtmp.h b/src/Extension/H265Rtmp.h index e4f97a08..6ba0f8a1 100644 --- a/src/Extension/H265Rtmp.h +++ b/src/Extension/H265Rtmp.h @@ -84,6 +84,7 @@ private: string _pps; H265Track::Ptr _track; RtmpPacket::Ptr _lastPacket; + bool _lastPacketHasVCL = false; }; }//namespace mediakit From 9b111fac6244f10be9ed637f43266d5cc9e26d1e Mon Sep 17 00:00:00 2001 From: xiongguangjie Date: Sat, 19 Jun 2021 02:01:28 +0800 Subject: [PATCH 3/5] add to do for h265 rtmp --- src/Extension/H265Rtmp.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Extension/H265Rtmp.cpp b/src/Extension/H265Rtmp.cpp index cc46905b..e5f2a8ec 100644 --- a/src/Extension/H265Rtmp.cpp +++ b/src/Extension/H265Rtmp.cpp @@ -191,7 +191,9 @@ void H265RtmpEncoder::inputFrame(const Frame::Ptr &frame) { int8_t flags = FLV_CODEC_H265; bool is_config = false; flags |= (((frame->configFrame() || frame->keyFrame()) ? FLV_KEY_FRAME : FLV_INTER_FRAME) << 4); - + // to do + // 必须是IDR帧才能是关键帧，否则有可能开始帧会花屏 SPS PPS VPS 打头的是一般I帧，但不一定是IDR帧 + // RtmpCodec::inputRtmp 时需要判断 是否是IDR帧,做出相应的修改 _lastPacket = RtmpPacket::create(); _lastPacket->buffer.push_back(flags); _lastPacket->buffer.push_back(!is_config); From e7e7906ecd1bbce36d51c55fc6bc57deccc68731 Mon Sep 17 00:00:00 2001 From: xgj Date: Fri, 25 Jun 2021 10:08:53 +0800 Subject: [PATCH 4/5] for firefox ingore ssrc is 0 --- webrtc/WebRtcTransport.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/webrtc/WebRtcTransport.cpp b/webrtc/WebRtcTransport.cpp index 22e85d4f..c5bc5bd1 100644 --- a/webrtc/WebRtcTransport.cpp +++ b/webrtc/WebRtcTransport.cpp @@ -792,6 +792,10 @@ void WebRtcTransportImp::onRtp_l(const char *buf, size_t len, bool rtx) { } } //解析并排序rtp + if(!ref){ + InfoL << "ignore no rtp receiver of ssrc:" << ssrc<<" is rtx:"<inputRtp(info->media->type, info->plan_rtp->sample_rate, (uint8_t *) buf, len); return; From c0f362af33a8f43f3403d55d668c97e46db7da8a Mon Sep 17 00:00:00 2001 From: xiongguangjie Date: Fri, 25 Jun 2021 22:03:52 +0800 Subject: [PATCH 5/5] for linux compile --start-group and --end-group --- tests/CMakeLists.txt | 8 +++++++- webrtc/WebRtcTransport.cpp | 7 +------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 64ec28a1..99326fbe 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,7 +62,13 @@ foreach(TEST_SRC ${TEST_SRC_LIST}) if(WIN32) set_target_properties(${TEST_EXE_NAME} PROPERTIES COMPILE_FLAGS ${VS_FALGS} ) endif(WIN32) - target_link_libraries(${TEST_EXE_NAME} ${LINK_LIB_LIST}) + if (CMAKE_SYSTEM_NAME MATCHES "Linux") + target_link_libraries(${TEST_EXE_NAME} -Wl,--start-group ${LINK_LIB_LIST} -Wl,--end-group) + else () + target_link_libraries(${TEST_EXE_NAME} ${LINK_LIB_LIST}) + endif () + + #target_link_libraries(${TEST_EXE_NAME} ${LINK_LIB_LIST}) endforeach() if(MSVC AND SDL2_FOUND AND AVCODEC_FOUND AND AVUTIL_FOUND) diff --git a/webrtc/WebRtcTransport.cpp b/webrtc/WebRtcTransport.cpp index 6ad1b896..2eb7cb1f 100644 --- a/webrtc/WebRtcTransport.cpp +++ b/webrtc/WebRtcTransport.cpp @@ -745,12 +745,7 @@ void WebRtcTransportImp::onRtp(const char *buf, size_t len) { } #endif //解析并排序rtp - if(!ref){ - InfoL << "ignore no rtp receiver of ssrc:" << ssrc<<" is rtx:"<inputRtp(info->media->type, info->plan_rtp->sample_rate, (uint8_t *) buf, len); + ref->inputRtp(track->media->type, track->plan_rtp->sample_rate, (uint8_t *) buf, len, false); return; }