From 375d36b4c8ac8ea7a4c92fc02fc2023354e18ede Mon Sep 17 00:00:00 2001 From: ziyue <1213642868@qq.com> Date: Thu, 29 Jul 2021 16:22:43 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A1=AE=E4=BF=9Drtp=E4=B8=A2=E5=8C=85?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E4=B8=A2=E5=BC=83gop=E5=90=8E=E7=BB=AD?= =?UTF-8?q?=E6=89=80=E6=9C=89=E5=B8=A7=EF=BC=8C=E9=98=B2=E6=AD=A2=E4=B8=A2?= =?UTF-8?q?=E5=8C=85=E8=8A=B1=E5=B1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Extension/H264Rtp.cpp | 194 +++++++++++++++++++------------------- src/Extension/H264Rtp.h | 9 +- src/Extension/H265Rtp.cpp | 94 +++++++++++------- src/Extension/H265Rtp.h | 11 ++- 4 files changed, 171 insertions(+), 137 deletions(-) diff --git a/src/Extension/H264Rtp.cpp b/src/Extension/H264Rtp.cpp index 0db7585a..73435357 100644 --- a/src/Extension/H264Rtp.cpp +++ b/src/Extension/H264Rtp.cpp @@ -46,7 +46,14 @@ H264Frame::Ptr H264RtpDecoder::obtainFrame() { } bool H264RtpDecoder::inputRtp(const RtpPacket::Ptr &rtp, bool key_pos) { - return decodeRtp(rtp); + auto seq = rtp->getSeq(); + auto ret = decodeRtp(rtp); + if (!_gop_dropped && seq != (uint16_t) (_last_seq + 1) && _last_seq) { + _gop_dropped = true; + WarnL << "start drop h264 gop, last seq:" << _last_seq << ", rtp:\r\n" << rtp->dumpString(); + } + _last_seq = seq; + return ret; } /* @@ -66,118 +73,115 @@ Table 1. Summary of NAL unit types and their payload structures 30-31 undefined - */ +bool H264RtpDecoder::singleFrame(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp){ + _frame->_buffer.assign("\x00\x00\x00\x01", 4); + _frame->_buffer.append((char *) ptr, size); + _frame->_pts = stamp; + auto key = _frame->keyFrame(); + outputFrame(rtp, _frame); + return key; +} + +bool H264RtpDecoder::unpackStapA(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp) { + //STAP-A 单一时间的组合包 + auto have_key_frame = false; + auto end = ptr + size; + while (ptr + 2 < end) { + uint16_t len = (ptr[0] << 8) | ptr[1]; + if (!len || ptr + len > end) { + WarnL << "invalid rtp data size:" << len << ",rtp:\r\n" << rtp->dumpString(); + _gop_dropped = true; + break; + } + ptr += 2; + if (singleFrame(rtp, ptr, len, stamp)) { + have_key_frame = true; + } + ptr += len; + } + return have_key_frame; +} + +bool H264RtpDecoder::mergeFu(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp, uint16_t seq){ + auto nal_suffix = *ptr & (~0x1F); + FuFlags *fu = (FuFlags *) (ptr + 1); + if (fu->start_bit) { + //该帧的第一个rtp包 + _frame->_buffer.assign("\x00\x00\x00\x01", 4); + _frame->_buffer.push_back(nal_suffix | fu->nal_type); + _frame->_pts = stamp; + _fu_dropped = false; + } + + if (_fu_dropped) { + //该帧不完整 + return false; + } + + if (!fu->start_bit && seq != (uint16_t) (_last_seq + 1)) { + //中间的或末尾的rtp包，其seq必须连续，否则说明rtp丢包，那么该帧不完整，必须得丢弃 + _fu_dropped = true; + _frame->_buffer.clear(); + return false; + } + + //后面追加数据 + _frame->_buffer.append((char *) ptr + 2, size - 2); + + if (!fu->end_bit) { + //非末尾包 + return fu->start_bit ? _frame->keyFrame() : false; + } + + //确保下一次fu必须收到第一个包 + _fu_dropped = true; + //该帧最后一个rtp包,输出frame + outputFrame(rtp, _frame); + return false; +} + bool H264RtpDecoder::decodeRtp(const RtpPacket::Ptr &rtp) { auto frame = rtp->getPayload(); auto length = rtp->getPayloadSize(); auto stamp = rtp->getStampMS(); auto seq = rtp->getSeq(); - auto nal_type = *frame & 0x1F; - auto nal_suffix = *frame & (~0x1F); + int nal = H264_TYPE(frame[0]); - if (nal_type >= 0 && nal_type < 24) { - //a full frame - _frame->_buffer.assign("\x00\x00\x00\x01", 4); - _frame->_buffer.append((char *) frame, length); - _frame->_pts = stamp; - auto key = _frame->keyFrame(); - onGetH264(_frame); - return (key); //i frame - } + switch (nal) { + case 24: + // 24 STAP-A Single-time aggregation packet 5.7.1 + return unpackStapA(rtp, frame + 1, length - 1, stamp); - switch (nal_type) { - case 24: { - // 24 STAP-A 单一时间的组合包 - bool haveIDR = false; - auto ptr = frame + 1; - while (true) { - size_t off = ptr - frame; - if (off >= length) { - break; - } - //获取当前nalu的大小 - uint16_t len = *ptr++; - len <<= 8; - len |= *ptr++; - if (off + len > length) { - break; - } - if (len > 0) { - //有有效数据 - _frame->_buffer.assign("\x00\x00\x00\x01", 4); - _frame->_buffer.append((char *) ptr, len); - _frame->_pts = stamp; - if ((ptr[0] & 0x1F) == H264Frame::NAL_IDR) { - haveIDR = true; - } - onGetH264(_frame); - } - ptr += len; - } - return haveIDR; - } - - case 28: { - //FU-A - FuFlags *fu = (FuFlags *) (frame + 1); - if (fu->start_bit) { - //该帧的第一个rtp包 FU-A start - //预留空间，防止频繁扩容拷贝 - _frame->_buffer.reserve(_max_frame_size); - _frame->_buffer.assign("\x00\x00\x00\x01", 4); - _frame->_buffer.push_back(nal_suffix | fu->nal_type); - _frame->_buffer.append((char *) frame + 2, length - 2); - _frame->_pts = stamp; - //该函数return时，保存下当前sequence,以便下次对比seq是否连续 - _last_seq = seq; - return _frame->keyFrame(); - } - - if (seq != (uint16_t) (_last_seq + 1)) { - //中间的或末尾的rtp包，其seq必须连续(如果回环了则判定为连续)，否则说明rtp丢包，那么该帧不完整，必须得丢弃 - _frame->_buffer.clear(); - WarnL << "rtp丢包: " << seq << " != " << _last_seq << " + 1,该帧被废弃"; - return false; - } - - if (!fu->end_bit) { - //该帧的中间rtp包 FU-A mid - _frame->_buffer.append((char *) frame + 2, length - 2); - //该函数return时，保存下当前sequence,以便下次对比seq是否连续 - _last_seq = seq; - return false; - } - - //该帧最后一个rtp包 FU-A end - _frame->_buffer.append((char *) frame + 2, length - 2); - _frame->_pts = stamp; - //计算最大的帧 - auto frame_size = _frame->size(); - if (frame_size > _max_frame_size) { - _max_frame_size = frame_size; - } - onGetH264(_frame); - return false; - } + case 28: + // 28 FU-A Fragmentation unit + return mergeFu(rtp, frame, length, stamp, seq); default: { - // 29 FU-B 单NAL单元B模式 - // 25 STAP-B 单一时间的组合包 - // 26 MTAP16 多个时间的组合包 - // 27 MTAP24 多个时间的组合包 - WarnL << "不支持的rtp类型:" << (int) nal_type << " " << seq; + if (nal < 24) { + //Single NAL Unit Packets + return singleFrame(rtp, frame, length, stamp); + } + _gop_dropped = true; + WarnL << "不支持该类型的264 RTP包, nal type:" << nal << ", rtp:\r\n" << rtp->dumpString(); return false; } } } -void H264RtpDecoder::onGetH264(const H264Frame::Ptr &frame) { +void H264RtpDecoder::outputFrame(const RtpPacket::Ptr &rtp, const H264Frame::Ptr &frame) { //rtsp没有dts，那么根据pts排序算法生成dts - _dts_generator.getDts(frame->_pts,frame->_dts); - RtpCodec::inputFrame(frame); + _dts_generator.getDts(frame->_pts, frame->_dts); + + if (frame->keyFrame() && _gop_dropped) { + _gop_dropped = false; + InfoL << "new gop received, rtp:\r\n" << rtp->dumpString(); + } + if (!_gop_dropped) { + RtpCodec::inputFrame(frame); + } _frame = obtainFrame(); } - //////////////////////////////////////////////////////////////////////// H264RtpEncoder::H264RtpEncoder(uint32_t ssrc, uint32_t mtu, uint32_t sample_rate, uint8_t pt, uint8_t interleaved) diff --git a/src/Extension/H264Rtp.h b/src/Extension/H264Rtp.h index 430718e9..edce6769 100644 --- a/src/Extension/H264Rtp.h +++ b/src/Extension/H264Rtp.h @@ -43,13 +43,18 @@ public: } private: + bool singleFrame(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp); + bool unpackStapA(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp); + bool mergeFu(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp, uint16_t seq); + bool decodeRtp(const RtpPacket::Ptr &rtp); - void onGetH264(const H264Frame::Ptr &frame); H264Frame::Ptr obtainFrame(); + void outputFrame(const RtpPacket::Ptr &rtp, const H264Frame::Ptr &frame); private: + bool _gop_dropped = false; + bool _fu_dropped = true; uint16_t _last_seq = 0; - size_t _max_frame_size = 0; H264Frame::Ptr _frame; DtsGenerator _dts_generator; }; diff --git a/src/Extension/H265Rtp.cpp b/src/Extension/H265Rtp.cpp index 9e780522..f783c709 100644 --- a/src/Extension/H265Rtp.cpp +++ b/src/Extension/H265Rtp.cpp @@ -12,6 +12,7 @@ namespace mediakit{ +//https://datatracker.ietf.org/doc/rfc7798/ //H265 nalu 头两个字节的定义 /* 0 1 @@ -41,7 +42,7 @@ H265Frame::Ptr H265RtpDecoder::obtainFrame() { #define CHECK_SIZE(total, size, ret) \ if (total < size) { \ - WarnL << "数据不够:" << total << " " << size; return ret; \ + WarnL << "invalid rtp data size:" << total << " < " << size << ",rtp:\r\n" << rtp->dumpString(); _gop_dropped = true; return ret; \ } // 4.4.2. Aggregation Packets (APs) (p25) @@ -68,9 +69,8 @@ H265Frame::Ptr H265RtpDecoder::obtainFrame() { | : ...OPTIONAL RTP padding | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ */ -bool H265RtpDecoder::unpackAp(const uint8_t *ptr, ssize_t size, uint32_t stamp){ +bool H265RtpDecoder::unpackAp(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp){ bool have_key_frame = false; - //忽略PayloadHdr CHECK_SIZE(size, 2, have_key_frame); ptr += 2; @@ -88,7 +88,7 @@ bool H265RtpDecoder::unpackAp(const uint8_t *ptr, ssize_t size, uint32_t stamp){ size -= 2; ptr += 2; CHECK_SIZE(size, nalu_size, have_key_frame) - if (singleFrame(ptr, nalu_size, stamp)) { + if (singleFrame(rtp, ptr, nalu_size, stamp)) { have_key_frame = true; } size -= nalu_size; @@ -119,7 +119,7 @@ bool H265RtpDecoder::unpackAp(const uint8_t *ptr, ssize_t size, uint32_t stamp){ +---------------+ */ -bool H265RtpDecoder::mergeFu(const uint8_t *ptr, ssize_t size, uint16_t seq, uint32_t stamp){ +bool H265RtpDecoder::mergeFu(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp, uint16_t seq){ CHECK_SIZE(size, 4, false); auto s_bit = ptr[2] >> 7; auto e_bit = (ptr[2] >> 6) & 0x01; @@ -127,15 +127,21 @@ bool H265RtpDecoder::mergeFu(const uint8_t *ptr, ssize_t size, uint16_t seq, uin if (s_bit) { //该帧的第一个rtp包 _frame->_buffer.assign("\x00\x00\x00\x01", 4); - //恢复nalu头两个字节 _frame->_buffer.push_back((type << 1) | (ptr[0] & 0x81)); _frame->_buffer.push_back(ptr[1]); + _frame->_pts = stamp; + _fu_dropped = false; } - if (!s_bit && seq != (uint16_t) (_last_seq + 1) && seq != 0) { + if (_fu_dropped) { + //该帧不完整 + return false; + } + + if (!s_bit && seq != (uint16_t) (_last_seq + 1)) { //中间的或末尾的rtp包，其seq必须连续，否则说明rtp丢包，那么该帧不完整，必须得丢弃 + _fu_dropped = true; _frame->_buffer.clear(); - WarnL << "rtp丢包: " << seq << " != " << _last_seq << " + 1,该帧被废弃"; return false; } @@ -152,64 +158,80 @@ bool H265RtpDecoder::mergeFu(const uint8_t *ptr, ssize_t size, uint16_t seq, uin CHECK_SIZE(size, 1, false); + //后面追加数据 + _frame->_buffer.append((char *) ptr, size); + if (!e_bit) { //非末尾包 - _last_seq = seq; - _frame->_buffer.append((char *) ptr, size); return s_bit ? _frame->keyFrame() : false; } + //确保下一次fu必须收到第一个包 + _fu_dropped = true; //该帧最后一个rtp包 - _frame->_pts = stamp; - _frame->_buffer.append((char *) ptr, size); - outputFrame(_frame); + outputFrame(rtp, _frame); return false; } -bool H265RtpDecoder::inputRtp(const RtpPacket::Ptr &rtp, bool ) { +bool H265RtpDecoder::inputRtp(const RtpPacket::Ptr &rtp, bool) { + auto seq = rtp->getSeq(); + auto ret = decodeRtp(rtp); + if (!_gop_dropped && seq != (uint16_t) (_last_seq + 1) && _last_seq) { + _gop_dropped = true; + WarnL << "start drop h265 gop, last seq:" << _last_seq << ", rtp:\r\n" << rtp->dumpString(); + } + _last_seq = seq; + return ret; +} + +bool H265RtpDecoder::decodeRtp(const RtpPacket::Ptr &rtp) { auto frame = rtp->getPayload(); auto length = rtp->getPayloadSize(); auto stamp = rtp->getStampMS(); auto seq = rtp->getSeq(); int nal = H265_TYPE(frame[0]); - if (nal > 50){ - // packet discard, Unsupported (HEVC) NAL type - WarnL << "不支持该类型的265 RTP包" << nal; - return false; - } switch (nal) { - case 50: - //4.4.4. PACI Packets (p32) - WarnL << "不支持该类型的265 RTP包" << nal; - return false; case 48: // aggregated packet (AP) - with two or more NAL units - return unpackAp(frame, length, stamp); - case 49: + return unpackAp(rtp, frame, length, stamp); + + case 49: // fragmentation unit (FU) - return mergeFu(frame, length, seq, stamp); - default: - // 4.4.1. Single NAL Unit Packets (p24) - return singleFrame(frame, length, stamp); + return mergeFu(rtp, frame, length, stamp, seq); + + default: { + if (nal < 48) { + // Single NAL Unit Packets (p24) + return singleFrame(rtp, frame, length, stamp); + } + _gop_dropped = true; + WarnL << "不支持该类型的265 RTP包, nal type" << nal << ", rtp:\r\n" << rtp->dumpString(); + return false; + } } } -bool H265RtpDecoder::singleFrame(const uint8_t *ptr, ssize_t size, uint32_t stamp){ - //a full frame +bool H265RtpDecoder::singleFrame(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp){ _frame->_buffer.assign("\x00\x00\x00\x01", 4); _frame->_buffer.append((char *) ptr, size); _frame->_pts = stamp; auto key = _frame->keyFrame(); - outputFrame(_frame); + outputFrame(rtp, _frame); return key; } -void H265RtpDecoder::outputFrame(const H265Frame::Ptr &frame) { +void H265RtpDecoder::outputFrame(const RtpPacket::Ptr &rtp, const H265Frame::Ptr &frame) { //rtsp没有dts，那么根据pts排序算法生成dts - _dts_generator.getDts(frame->_pts,frame->_dts); - //输出frame - RtpCodec::inputFrame(frame); + _dts_generator.getDts(frame->_pts, frame->_dts); + + if (frame->keyFrame() && _gop_dropped) { + _gop_dropped = false; + InfoL << "new gop received, rtp:\r\n" << rtp->dumpString(); + } + if (!_gop_dropped) { + RtpCodec::inputFrame(frame); + } _frame = obtainFrame(); } diff --git a/src/Extension/H265Rtp.h b/src/Extension/H265Rtp.h index 1af07623..5bbffe8c 100644 --- a/src/Extension/H265Rtp.h +++ b/src/Extension/H265Rtp.h @@ -44,15 +44,18 @@ public: } private: - bool unpackAp(const uint8_t *ptr, ssize_t size, uint32_t stamp); - bool mergeFu(const uint8_t *ptr, ssize_t size, uint16_t seq, uint32_t stamp); - bool singleFrame(const uint8_t *ptr, ssize_t size, uint32_t stamp); + bool unpackAp(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp); + bool mergeFu(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp, uint16_t seq); + bool singleFrame(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint32_t stamp); + bool decodeRtp(const RtpPacket::Ptr &rtp); H265Frame::Ptr obtainFrame(); - void outputFrame(const H265Frame::Ptr &frame); + void outputFrame(const RtpPacket::Ptr &rtp, const H265Frame::Ptr &frame); private: bool _using_donl_field = false; + bool _gop_dropped = false; + bool _fu_dropped = true; uint16_t _last_seq = 0; H265Frame::Ptr _frame; DtsGenerator _dts_generator;