From f4b2fd9c050507d69d4296e896cea68e55c0a817 Mon Sep 17 00:00:00 2001 From: cqm Date: Tue, 29 Nov 2022 20:38:59 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=20audio=5Ftranscode=20?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=9A=20-=20=E7=8B=AC=E7=AB=8B=E5=87=BA?= =?UTF-8?q?=20RtcMediaSource=EF=BC=8C=E5=B9=B6=E5=8F=AA=E5=AF=B9rtc?= =?UTF-8?q?=E5=BC=80=E6=94=BE=20-=20=E5=A2=9E=E5=8A=A0Rtc=20g711=E8=BD=AC?= =?UTF-8?q?=E7=A0=81=E5=BC=80=E5=85=B3=20-=20=E4=BF=AE=E6=94=B9=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/config.ini | 10 +- src/Common/MultiMediaSourceMuxer.cpp | 127 +---------------- src/Common/MultiMediaSourceMuxer.h | 9 -- webrtc/RtcMediaSource.cpp | 199 +++++++++++++++++++++++++++ webrtc/RtcMediaSource.h | 60 ++++++++ webrtc/WebRtcPusher.cpp | 2 +- webrtc/WebRtcTransport.cpp | 8 +- webrtc/WebRtcTransport.h | 1 + 8 files changed, 281 insertions(+), 135 deletions(-) create mode 100644 webrtc/RtcMediaSource.cpp create mode 100644 webrtc/RtcMediaSource.h diff --git a/conf/config.ini b/conf/config.ini index c1c85b79..d49d586e 100644 --- a/conf/config.ini +++ b/conf/config.ini @@ -50,9 +50,9 @@ auto_close=0 #此参数不应大于播放器超时时间;单位毫秒 continue_push_ms=15000 # 是否启用音频转码 -# 转码主要基于包过滤机制，代码实现详见 MultiMediaSourceMuxer 的 onTrackReady 和 onTrackFrame 方法，当前已实现 -# 1. AAC转Opus, 然后传入rtc中 -# 2. 非AAC音频转aac，然后传入rtmp中 +# 主要实现进出RTC音频流的自动转码，代码实现详见 RtcMediaSource.h/cpp，当前实现 +# 1. 将RTC收到的Opus音频流，转成AAC流，并传到MultiMediaSourceMuxer中的其他流中 +# 2. 将从MultiMediaSourceMuxer中收到的AAC音频流，转成OPUS，并送到RTC流中 # 音频转码底层使用FFMPEG来实现，需要打开FFMPEG, 即编译时必须指定 -DENABLE_FFMPEG=1, 当前已知支持FFMPEG 4.x 5.x 和 6.0， # 在ubuntu中可通过以下指令来安装: apt-get install libavcodec-dev libavutil-dev libswscale-dev libresample-dev # 此外音频转码正常都是用于webrtc的，一般也会开启WEBRTC, 即-DENABLE_WEBRTC=1, 此前必须自己装好libsrtp库, 安装过程详见wiki @@ -354,6 +354,10 @@ preferredCodecA=PCMU,PCMA,opus,mpeg4-generic #rtc支持的视频codec类型,在前面的优先级更高 #以下范例为所有支持的视频codec preferredCodecV=H264,H265,AV1,VP9,VP8 +# 是否开启RTC协议的G711转码，开启后 +# 能将传给rtc的g711音频转成opus +# 将由rtc流入g711音频转成aac，并转给其他协议流 +transcodeG711=0 [srt] #srt播放推流、播放超时时间,单位秒 diff --git a/src/Common/MultiMediaSourceMuxer.cpp b/src/Common/MultiMediaSourceMuxer.cpp index 11b51236..448bed88 100644 --- a/src/Common/MultiMediaSourceMuxer.cpp +++ b/src/Common/MultiMediaSourceMuxer.cpp @@ -11,9 +11,6 @@ #include #include "Common/config.h" #include "MultiMediaSourceMuxer.h" -#include "Extension/AAC.h" -#include "Extension/Opus.h" -#include "Extension/G711.h" #include "Rtp/RtpSender.h" #include "Record/HlsRecorder.h" #include "Record/HlsMediaSource.h" @@ -21,9 +18,8 @@ #include "Rtmp/RtmpMediaSourceMuxer.h" #include "TS/TSMediaSourceMuxer.h" #include "FMP4/FMP4MediaSourceMuxer.h" - -#ifdef ENABLE_FFMPEG -#include "Codec/Transcode.h" +#if defined(ENABLE_WEBRTC) +#include "webrtc/RtcMediaSource.h" #endif using namespace std; using namespace toolkit; @@ -122,7 +118,7 @@ MultiMediaSourceMuxer::MultiMediaSourceMuxer(const MediaTuple& tuple, float dur_ } if (option.enable_rtc) { #if defined(ENABLE_WEBRTC) - _rtc = std::make_shared(_tuple, option, std::make_shared(dur_sec), RTC_SCHEMA); + _rtc = std::make_shared(_tuple, option, std::make_shared(dur_sec)); #endif } if (option.enable_hls) { @@ -137,14 +133,6 @@ MultiMediaSourceMuxer::MultiMediaSourceMuxer(const MediaTuple& tuple, float dur_ if (option.enable_ts) { _ts = dynamic_pointer_cast(Recorder::createRecorder(Recorder::type_ts, _tuple, option)); } - if (option.audio_transcode) { -#if defined(ENABLE_FFMPEG) - InfoL << "enable audio_transcode"; -#else - InfoL << "without ffmpeg disable audio_transcode"; - _option.audio_transcode = false; -#endif - } if (option.enable_fmp4) { _fmp4 = dynamic_pointer_cast(Recorder::createRecorder(Recorder::type_fmp4, _tuple, option)); @@ -407,79 +395,9 @@ std::shared_ptr MultiMediaSourceMuxer::getMuxer(MediaSour bool MultiMediaSourceMuxer::onTrackReady(const Track::Ptr &track) { bool ret = false; - auto rtmp = _rtmp; - auto rtc = _rtc; -#if defined(ENABLE_FFMPEG) - if (_option.audio_transcode) { - if (track->getCodecId() == CodecAAC) { - if (rtmp) { - rtmp->addTrack(track); - rtmp = nullptr; - } - _audio_dec = nullptr; - _audio_enc = nullptr; - _opus_mute_maker = nullptr; - if (rtc) { - Track::Ptr newTrack(new OpusTrack()); - GET_CONFIG(int, bitrate, General::kOpusBitrate); - newTrack->setBitRate(bitrate); - rtc->addTrack(newTrack); - rtc = nullptr; - if (!hasMuteAudio()) { - // aac to opus - _audio_dec.reset(new FFmpegDecoder(track)); - _audio_enc.reset(new FFmpegEncoder(newTrack)); - _audio_dec->setOnDecode([this](const FFmpegFrame::Ptr & frame) { - _audio_enc->inputFrame(frame, false); - }); - _audio_enc->setOnEncode([this](const Frame::Ptr& frame) { - // fill data to _rtc - if (_rtc && _rtc->isEnabled()) - _rtc->inputFrame(frame); - }); - } - else { - _opus_mute_maker = std::make_shared(CodecOpus); - _opus_mute_maker->addDelegate([this](const Frame::Ptr &frame) { - if (_rtc && _rtc->isEnabled()) - _rtc->inputFrame(frame); - return true; - }); - } - } - } - else if (track->getTrackType() == TrackAudio) { - if (rtc) { - rtc->addTrack(track); - rtc = nullptr; - } - _audio_dec = nullptr; - _audio_enc = nullptr; - _opus_mute_maker = nullptr; - if (rtmp) { - Track::Ptr newTrack(new AACTrack(44100, std::dynamic_pointer_cast(track)->getAudioChannel())); - GET_CONFIG(int, bitrate, General::kAacBitrate); - newTrack->setBitRate(bitrate); - rtmp->addTrack(newTrack); - rtmp = nullptr; - - _audio_dec.reset(new FFmpegDecoder(track)); - _audio_enc.reset(new FFmpegEncoder(newTrack)); - _audio_dec->setOnDecode([this](const FFmpegFrame::Ptr & frame) { - _audio_enc->inputFrame(frame, false); - }); - _audio_enc->setOnEncode([this](const Frame::Ptr& frame) { - // fill aac frame to rtmp - if (_rtmp && _rtmp->isEnabled()) - _rtmp->inputFrame(frame); - }); - } - } - } -#endif - if (rtc && rtc->addTrack(track)) + if (_rtc && _rtc->addTrack(track)) ret = true; - if (rtmp && rtmp->addTrack(track)) + if (_rtmp && _rtmp->addTrack(track)) ret = true; if (_rtsp && _rtsp->addTrack(track)) ret = true; @@ -579,11 +497,6 @@ void MultiMediaSourceMuxer::resetTracks() { if (_rtc) { _rtc->resetTracks(); } -#if defined(ENABLE_FFMPEG) - _audio_dec = nullptr; - _audio_dec = nullptr; - _opus_mute_maker = nullptr; -#endif if (_fmp4) { _fmp4->resetTracks(); } @@ -606,36 +519,10 @@ bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) { } bool ret = false; - RtspMediaSourceMuxer::Ptr rtc; - RtmpMediaSourceMuxer::Ptr rtmp; - if (_rtc && _rtc->isEnabled()) - rtc = _rtc; - if (_rtmp && _rtmp->isEnabled()) - rtmp = _rtmp; -#if defined(ENABLE_FFMPEG) - if (_option.audio_transcode) { - if (frame->getCodecId() == CodecAAC) { - if (rtc) { - if (_audio_dec && rtc->readerCount()) - _audio_dec->inputFrame(frame, true, false, false); - rtc = nullptr; - } - } - else if (frame->getTrackType() == TrackAudio) { - if (rtmp) { - if (_audio_dec && (rtmp->readerCount() || !rtmp->isRegisted())) - _audio_dec->inputFrame(frame, true, false, false); - rtmp = nullptr; - } - } else if (_opus_mute_maker && rtc) { - _opus_mute_maker->inputFrame(frame); - } - } -#endif - if (rtc && rtc->inputFrame(frame)) + if (_rtc && _rtc->inputFrame(frame)) ret = true; - if (rtmp && rtmp->inputFrame(frame)) + if (_rtmp && _rtmp->inputFrame(frame)) ret = true; if (_rtsp && _rtsp->inputFrame(frame)) diff --git a/src/Common/MultiMediaSourceMuxer.h b/src/Common/MultiMediaSourceMuxer.h index 5042ef65..755bc33a 100644 --- a/src/Common/MultiMediaSourceMuxer.h +++ b/src/Common/MultiMediaSourceMuxer.h @@ -24,10 +24,6 @@ class RtmpMediaSourceMuxer; class TSMediaSourceMuxer; class FMP4MediaSourceMuxer; class RtpSender; -#ifdef ENABLE_FFMPEG -class FFmpegDecoder; -class FFmpegEncoder; -#endif class MultiMediaSourceMuxer : public MediaSourceEventInterceptor, public MediaSink, public std::enable_shared_from_this{ public: @@ -178,11 +174,6 @@ private: std::shared_ptr _rtsp; std::shared_ptr _ts; std::shared_ptr _rtc; -#if defined(ENABLE_FFMPEG) - MuteAudioMaker::Ptr _opus_mute_maker; - std::shared_ptr _audio_dec; - std::shared_ptr _audio_enc; -#endif MediaSinkInterface::Ptr _mp4; std::shared_ptr _hls; std::shared_ptr _hls_fmp4; diff --git a/webrtc/RtcMediaSource.cpp b/webrtc/RtcMediaSource.cpp new file mode 100644 index 00000000..6b050daa --- /dev/null +++ b/webrtc/RtcMediaSource.cpp @@ -0,0 +1,199 @@ +#include "RtcMediaSource.h" +#include "Common/config.h" +#include "Codec/Transcode.h" +#include "Extension/AAC.h" +#include "Extension/Opus.h" +#include "Extension/G711.h" +// for RTC configure +#include "WebRtcTransport.h" +namespace mediakit { + +bool needTransToOpus(CodecId codec) { + GET_CONFIG(int, transG711, Rtc::kTranscodeG711); + switch (codec) + { + case CodecG711U: + case CodecG711A: + return transG711; + case CodecAAC: + return true; + default: + return false; + } +} + +bool needTransToAac(CodecId codec) { + GET_CONFIG(int, transG711, Rtc::kTranscodeG711); + switch (codec) + { + case CodecG711U: + case CodecG711A: + return transG711; + case CodecOpus: + return true; + default: + return false; + } +} + +RtcMediaSourceMuxer::RtcMediaSourceMuxer(const MediaTuple& tuple, const ProtocolOption &option, const TitleSdp::Ptr &title) + : RtspMediaSourceMuxer(tuple, option, title, RTC_SCHEMA) +{ + if (_option.audio_transcode) { +#ifndef ENABLE_FFMPEG + WarnL << "without ffmpeg, skip transcode setting"; + _option.audio_transcode = false; +#endif + } +} + +RtspMediaSource::Ptr RtcMediaSourceImp::clone(const std::string &stream) { + auto tuple = _tuple; + tuple.stream = stream; + auto src_imp = std::make_shared(tuple); + src_imp->setSdp(getSdp()); + src_imp->setProtocolOption(getProtocolOption()); + return src_imp; +} + +bool RtcMediaSourceMuxer::inputFrame(const Frame::Ptr &frame) +{ + if (_clear_cache && _on_demand) { + _clear_cache = false; + _media_src->clearCache(); + } + if (_enabled || !_on_demand) { +#if defined(ENABLE_FFMPEG) + if (_option.audio_transcode && needTransToOpus(frame->getCodecId())) { + if (!_audio_dec) { // addTrack可能没调, 这边根据情况再调一次 + Track::Ptr track; + switch (frame->getCodecId()) + { + case CodecAAC: + if (frame->prefixSize()) { + std::string cfg = makeAacConfig((uint8_t *)(frame->data()), frame->prefixSize()); + track = std::make_shared(cfg); + } + else { + track = std::make_shared(44100, 2); + } + break; + case CodecG711A: + case CodecG711U: + track.reset(new G711Track(frame->getCodecId())); + break; + default: + break; + } + if (track) + addTrack(track); + if (!_audio_dec) return false; + } + if (readerCount()) { + _audio_dec->inputFrame(frame, true, false); + if (!_count) + InfoL << "start transcode " << frame->getCodecName() << "," << frame->pts() << "->Opus"; + _count++; + } + else if (_count) { + InfoL << "stop transcode with " << _count << " items"; + _count = 0; + } + return true; + } +#endif + return RtspMuxer::inputFrame(frame); + } + return false; +} + +#if defined(ENABLE_FFMPEG) + +bool RtcMediaSourceMuxer::addTrack(const Track::Ptr & track) +{ + Track::Ptr newTrack = track; + if (_option.audio_transcode && needTransToOpus(track->getCodecId())) { + newTrack = std::make_shared(); + GET_CONFIG(int, bitrate, General::kOpusBitrate); + newTrack->setBitRate(bitrate); + _audio_dec.reset(new FFmpegDecoder(track)); + _audio_enc.reset(new FFmpegEncoder(newTrack)); + // aac to opus + _audio_dec->setOnDecode([this](const FFmpegFrame::Ptr & frame) { + _audio_enc->inputFrame(frame, false); + }); + _audio_enc->setOnEncode([this](const Frame::Ptr& frame) { + RtspMuxer::inputFrame(frame); + }); + } + return RtspMuxer::addTrack(newTrack); +} + + +void RtcMediaSourceMuxer::resetTracks() +{ + RtspMuxer::resetTracks(); + _audio_dec = nullptr; + _audio_enc = nullptr; + if (_count) { + InfoL << "stop transcode with " << _count << " items"; + _count = 0; + } +} + +bool RtcMediaSourceImp::addTrack(const Track::Ptr &track) +{ + if (_muxer) { + Track::Ptr newTrack = track; + if (_option.audio_transcode && needTransToAac(track->getCodecId())) { + newTrack.reset(new AACTrack(44100, 2)); + GET_CONFIG(int, bitrate, General::kAacBitrate); + newTrack->setBitRate(bitrate); + _audio_dec.reset(new FFmpegDecoder(track)); + _audio_enc.reset(new FFmpegEncoder(newTrack)); + // hook data to newTack + track->addDelegate([this](const Frame::Ptr &frame) -> bool { + if (_all_track_ready && 0 == _muxer->totalReaderCount()) { + if (_count) { + InfoL << "stop transcode with " << _count << " items"; + _count = 0; + } + return true; + } + if (_audio_dec) { + if (!_count) + InfoL << "start transcode " << frame->getCodecName() << "," << frame->pts() << "->AAC"; + _count++; + _audio_dec->inputFrame(frame, true, false); + } + return true; + }); + _audio_dec->setOnDecode([this](const FFmpegFrame::Ptr & frame) { + _audio_enc->inputFrame(frame, false); + }); + _audio_enc->setOnEncode([newTrack](const Frame::Ptr& frame) { + newTrack->inputFrame(frame); + }); + } + + if (_muxer->addTrack(newTrack)) { + newTrack->addDelegate(_muxer); + return true; + } + } + return false; +} + +void RtcMediaSourceImp::resetTracks() +{ + RtspMediaSourceImp::resetTracks(); + _audio_dec = nullptr; + _audio_enc = nullptr; + if (_count) { + InfoL << "stop transcode with " << _count << " items"; + _count = 0; + } +} + +#endif +} diff --git a/webrtc/RtcMediaSource.h b/webrtc/RtcMediaSource.h new file mode 100644 index 00000000..73109d10 --- /dev/null +++ b/webrtc/RtcMediaSource.h @@ -0,0 +1,60 @@ +#ifndef ZLMEDIAKIT_RTCMEDIASOURCE_H +#define ZLMEDIAKIT_RTCMEDIASOURCE_H + +#include "Rtsp/RtspMediaSourceMuxer.h" +#include "Rtsp/RtspMediaSourceImp.h" +namespace mediakit { +class FFmpegDecoder; +class FFmpegEncoder; +bool needTransToOpus(CodecId codec); +bool needTransToAac(CodecId codec); + +class RtcMediaSourceImp : public RtspMediaSourceImp { +public: + typedef std::shared_ptr Ptr; + + RtcMediaSourceImp(const MediaTuple& tuple, int ringSize = RTP_GOP_SIZE) + : RtspMediaSourceImp(tuple, RTC_SCHEMA, ringSize) { + } + + RtspMediaSource::Ptr clone(const std::string &stream); +#if defined(ENABLE_FFMPEG) + ~RtcMediaSourceImp() override { resetTracks(); } + /** + * _demuxer触发的添加Track事件 + */ + bool addTrack(const Track::Ptr &track) override; + void resetTracks() override; +private: + int _count = 0; + std::shared_ptr _audio_dec; + std::shared_ptr _audio_enc; +#endif +}; + +class RtcMediaSourceMuxer : public RtspMediaSourceMuxer { +public: + typedef std::shared_ptr Ptr; + + RtcMediaSourceMuxer( const MediaTuple& tuple, + const ProtocolOption &option, + const TitleSdp::Ptr &title = nullptr); + + + bool inputFrame(const Frame::Ptr &frame) override; + +#if defined(ENABLE_FFMPEG) + ~RtcMediaSourceMuxer() override{resetTracks();} + + bool addTrack(const Track::Ptr & track) override; + void resetTracks() override; + +private: + int _count = 0; + std::shared_ptr _audio_dec; + std::shared_ptr _audio_enc; +#endif +}; + +} +#endif \ No newline at end of file diff --git a/webrtc/WebRtcPusher.cpp b/webrtc/WebRtcPusher.cpp index 0a62c0f3..3253bb26 100644 --- a/webrtc/WebRtcPusher.cpp +++ b/webrtc/WebRtcPusher.cpp @@ -10,7 +10,7 @@ #include "WebRtcPusher.h" #include "Common/config.h" -#include "Rtsp/RtspMediaSourceImp.h" +#include "RtcMediaSource.h" using namespace std; diff --git a/webrtc/WebRtcTransport.cpp b/webrtc/WebRtcTransport.cpp index 2f8f627a..4d540ede 100644 --- a/webrtc/WebRtcTransport.cpp +++ b/webrtc/WebRtcTransport.cpp @@ -18,7 +18,7 @@ #include "Rtsp/Rtsp.h" #include "Rtsp/RtpReceiver.h" #include "WebRtcTransport.h" - +#include "RtcMediaSource.h" #include "WebRtcEchoTest.h" #include "WebRtcPlayer.h" #include "WebRtcPusher.h" @@ -45,6 +45,9 @@ const string kTimeOutSec = RTC_FIELD "timeoutSec"; const string kExternIP = RTC_FIELD "externIP"; // 设置remb比特率，非0时关闭twcc并开启remb。该设置在rtc推流时有效，可以控制推流画质 const string kRembBitRate = RTC_FIELD "rembBitRate"; +// 是否转码G711音频，做到: 出rtc将g711转成aac，入rtc将g711转成opus +const string kTranscodeG711 = RTC_FIELD "transcodeG711"; + // webrtc单端口udp服务器 const string kPort = RTC_FIELD "port"; @@ -56,6 +59,7 @@ static onceToken token([]() { mINI::Instance()[kRembBitRate] = 0; mINI::Instance()[kPort] = 0; mINI::Instance()[kTcpPort] = 0; + mINI::Instance()[kTranscodeG711] = 0; }); } // namespace RTC @@ -1206,7 +1210,7 @@ void push_plugin(Session &sender, const WebRtcArgs &args, const WebRtcPluginMana } if (!push_src) { - push_src = std::make_shared(info, schema); + push_src = std::make_shared(info); push_src_ownership = push_src->getOwnership(); push_src->setProtocolOption(option); } diff --git a/webrtc/WebRtcTransport.h b/webrtc/WebRtcTransport.h index 2534ca3b..96891dfe 100644 --- a/webrtc/WebRtcTransport.h +++ b/webrtc/WebRtcTransport.h @@ -33,6 +33,7 @@ namespace Rtc { extern const std::string kPort; extern const std::string kTcpPort; extern const std::string kTimeOutSec; +extern const std::string kTranscodeG711; }//namespace RTC class WebRtcInterface {