diff --git a/player/test_player.cpp b/player/test_player.cpp index 7fa68bf2..40d3fcbb 100644 --- a/player/test_player.cpp +++ b/player/test_player.cpp @@ -84,7 +84,7 @@ int main(int argc, char *argv[]) { }); }); auto delegate = std::make_shared([decoder](const Frame::Ptr &frame) { - return decoder->inputFrame(frame, false); + return decoder->inputFrame(frame, false, true); }); videoTrack->addDelegate(delegate); } @@ -106,7 +106,7 @@ int main(int argc, char *argv[]) { audio_player->playPCM((const char *) (pcm->get()->data[0]), MIN(len, frame->get()->linesize[0])); }); auto audio_delegate = std::make_shared( [decoder](const Frame::Ptr &frame) { - return decoder->inputFrame(frame, false); + return decoder->inputFrame(frame, false, true); }); audioTrack->addDelegate(audio_delegate); } diff --git a/src/Codec/H264Encoder.cpp b/src/Codec/H264Encoder.cpp index b8bc3e9c..6fe0e3f7 100644 --- a/src/Codec/H264Encoder.cpp +++ b/src/Codec/H264Encoder.cpp @@ -9,17 +9,14 @@ */ #ifdef ENABLE_X264 + #include "H264Encoder.h" - #include "Util/TimeTicker.h" - using namespace toolkit; namespace mediakit { -H264Encoder::H264Encoder() { - -} +H264Encoder::H264Encoder() {} H264Encoder::~H264Encoder() { //* 清除图像区域 @@ -39,7 +36,6 @@ H264Encoder::~H264Encoder() { } } - /*typedef struct x264_param_t { CPU 标志位 @@ -212,7 +208,7 @@ Value的值就是fps。 void (*param_free)( void* ); } x264_param_t;*/ -bool H264Encoder::init(int iWidth, int iHeight, int iFps) { +bool H264Encoder::init(int iWidth, int iHeight, int iFps, int iBitRate) { if (_pX264Handle) { return true; } @@ -222,7 +218,7 @@ bool H264Encoder::init(int iWidth, int iHeight, int iFps) { x264_param_default_preset(pX264Param, "ultrafast", "zerolatency"); //* cpuFlags - pX264Param->i_threads = X264_SYNC_LOOKAHEAD_AUTO; //* 取空缓冲区继续使用不死锁的保证. + pX264Param->i_threads = X264_SYNC_LOOKAHEAD_AUTO; //* 取空缓冲区继续使用不死锁的保证. //* video Properties pX264Param->i_width = iWidth; //* 宽度. pX264Param->i_height = iHeight; //* 高度 @@ -230,21 +226,21 @@ bool H264Encoder::init(int iWidth, int iHeight, int iFps) { pX264Param->i_keyint_max = iFps * 3; //ffmpeg:gop_size 关键帧最大间隔 pX264Param->i_keyint_min = iFps * 1; //ffmpeg:keyint_min 关键帧最小间隔 //* Rate control Parameters - pX264Param->rc.i_bitrate = 5000; //* 码率(比特率,单位Kbps) - pX264Param->rc.i_qp_step = 1; //最大的在帧与帧之间进行切变的量化因子的变化量。ffmpeg:max_qdiff - pX264Param->rc.i_qp_min = 10; //ffmpeg:qmin;最小的量化因子。取值范围1-51。建议在10-30之间。 - pX264Param->rc.i_qp_max = 41; //ffmpeg:qmax;最大的量化因子。取值范围1-51。建议在10-30之间。 + pX264Param->rc.i_bitrate = iBitRate / 1000; //* 码率(比特率,单位Kbps) + pX264Param->rc.i_qp_step = 1; //最大的在帧与帧之间进行切变的量化因子的变化量。ffmpeg:max_qdiff + pX264Param->rc.i_qp_min = 10; //ffmpeg:qmin;最小的量化因子。取值范围1-51。建议在10-30之间。 + pX264Param->rc.i_qp_max = 41; //ffmpeg:qmax;最大的量化因子。取值范围1-51。建议在10-30之间。 pX264Param->rc.f_qcompress = 0.6;//ffmpeg:qcompress 量化器压缩比率0-1.越小则比特率越区域固定，但是越高越使量化器参数越固定 - pX264Param->analyse.i_me_range = 16; //ffmpeg:me_range 运动侦测的半径 - pX264Param->i_frame_reference = 3; //ffmpeg:refsB和P帧向前预测参考的帧数。取值范围1-16。 - //该值不影响解码的速度，但是越大解码 - //所需的内存越大。这个值在一般情况下 - //越大效果越好，但是超过6以后效果就 - //不明显了。 + pX264Param->analyse.i_me_range = 16; //ffmpeg:me_range 运动侦测的半径 + pX264Param->i_frame_reference = 3; //ffmpeg:refsB和P帧向前预测参考的帧数。取值范围1-16。 + //该值不影响解码的速度，但是越大解码 + //所需的内存越大。这个值在一般情况下 + //越大效果越好，但是超过6以后效果就 + //不明显了。 - pX264Param->analyse.i_trellis = 1; //ffmpeg:trellis + pX264Param->analyse.i_trellis = 1; //ffmpeg:trellis //pX264Param->analyse.i_me_method=X264_ME_DIA;//ffmpeg:me_method ME_ZERO 运动侦测的方式 - pX264Param->rc.f_qblur = 0.5; //ffmpeg:qblur + pX264Param->rc.f_qblur = 0.5; //ffmpeg:qblur //* bitstream parameters /*open-GOP @@ -268,7 +264,7 @@ bool H264Encoder::init(int iWidth, int iHeight, int iFps) { 由于B帧压缩性能好于P帧，因此open-GOP在编码性能上稍微优于close-GOP， 但为了兼容性和少一些麻烦，还是把opne-GOP关闭的好。*/ pX264Param->b_open_gop = 0; - pX264Param->i_bframe = 0; //最大B帧数. + pX264Param->i_bframe = 0; //最大B帧数. pX264Param->i_bframe_pyramid = 0; pX264Param->i_bframe_adaptive = X264_B_ADAPT_TRELLIS; //* Log @@ -304,20 +300,19 @@ bool H264Encoder::init(int iWidth, int iHeight, int iFps) { return true; } -int H264Encoder::inputData(char* apcYuv[3], int aiYuvLen[3], int64_t i64Pts, H264Frame** ppFrame) { +int H264Encoder::inputData(char *yuv[3], int linesize[3], int64_t cts, H264Frame **out_frame) { //TimeTicker1(5); - _pPicIn->img.i_stride[0] = aiYuvLen[0]; - _pPicIn->img.i_stride[1] = aiYuvLen[1]; - _pPicIn->img.i_stride[2] = aiYuvLen[2]; - _pPicIn->img.plane[0] = (uint8_t *) apcYuv[0]; - _pPicIn->img.plane[1] = (uint8_t *) apcYuv[1]; - _pPicIn->img.plane[2] = (uint8_t *) apcYuv[2]; - _pPicIn->i_pts = i64Pts; + _pPicIn->img.i_stride[0] = linesize[0]; + _pPicIn->img.i_stride[1] = linesize[1]; + _pPicIn->img.i_stride[2] = linesize[2]; + _pPicIn->img.plane[0] = (uint8_t *) yuv[0]; + _pPicIn->img.plane[1] = (uint8_t *) yuv[1]; + _pPicIn->img.plane[2] = (uint8_t *) yuv[2]; + _pPicIn->i_pts = cts; int iNal; - x264_nal_t* pNals; + x264_nal_t *pNals; - int iResult = x264_encoder_encode(_pX264Handle, &pNals, &iNal, _pPicIn, - _pPicOut); + int iResult = x264_encoder_encode(_pX264Handle, &pNals, &iNal, _pPicIn, _pPicOut); if (iResult <= 0) { return 0; } @@ -327,7 +322,7 @@ int H264Encoder::inputData(char* apcYuv[3], int aiYuvLen[3], int64_t i64Pts, H26 _aFrames[i].iLength = pNal.i_payload; _aFrames[i].pucData = pNal.p_payload; } - *ppFrame = _aFrames; + *out_frame = _aFrames; return iNal; } diff --git a/src/Codec/H264Encoder.h b/src/Codec/H264Encoder.h index 1229dd89..a63eb6ad 100644 --- a/src/Codec/H264Encoder.h +++ b/src/Codec/H264Encoder.h @@ -7,13 +7,10 @@ * LICENSE file in the root of the source tree. All contributing project authors * may be found in the AUTHORS file in the root of the source tree. */ - - #ifndef CODEC_H264ENCODER_H_ #define CODEC_H264ENCODER_H_ #include - #ifdef __cplusplus extern "C" { #endif //__cplusplus @@ -32,14 +29,16 @@ public: uint8_t *pucData; } H264Frame; - H264Encoder(void); - virtual ~H264Encoder(void); - bool init(int iWidth, int iHeight, int iFps); - int inputData(char *apcYuv[3], int aiYuvLen[3], int64_t i64Pts, H264Frame **ppFrame); + H264Encoder(); + ~H264Encoder(); + + bool init(int iWidth, int iHeight, int iFps, int iBitRate); + int inputData(char *yuv[3], int linesize[3], int64_t cts, H264Frame **out_frame); + private: - x264_t* _pX264Handle = nullptr; - x264_picture_t* _pPicIn = nullptr; - x264_picture_t* _pPicOut = nullptr; + x264_t *_pX264Handle = nullptr; + x264_picture_t *_pPicIn = nullptr; + x264_picture_t *_pPicOut = nullptr; H264Frame _aFrames[10]; }; diff --git a/src/Codec/Transcode.cpp b/src/Codec/Transcode.cpp index 26a0fb95..c9a4cb5f 100644 --- a/src/Codec/Transcode.cpp +++ b/src/Codec/Transcode.cpp @@ -9,13 +9,20 @@ */ #if defined(ENABLE_FFMPEG) - +#if !defined(_WIN32) +#include +#endif +#include "Util/File.h" +#include "Util/uv_errno.h" #include "Transcode.h" +#include "Extension/AAC.h" + #define MAX_DELAY_SECOND 3 using namespace std; using namespace toolkit; -using namespace mediakit; + +namespace mediakit { static string ffmpeg_err(int errnum) { char errbuf[AV_ERROR_MAX_STRING_SIZE]; @@ -23,7 +30,7 @@ static string ffmpeg_err(int errnum) { return errbuf; } -std::shared_ptr alloc_av_packet(){ +std::shared_ptr alloc_av_packet() { auto pkt = std::shared_ptr(av_packet_alloc(), [](AVPacket *pkt) { av_packet_free(&pkt); }); @@ -33,366 +40,146 @@ std::shared_ptr alloc_av_packet(){ } ////////////////////////////////////////////////////////////////////////////////////////// - -template -const AVCodec *getCodec(ARGS ...names); - -template -const AVCodec *getCodec(const char *name) { - auto codec = decoder ? avcodec_find_decoder_by_name(name) : avcodec_find_encoder_by_name(name); - if (codec) { - InfoL << (decoder ? "got decoder:" : "got encoder:") << name; +static void on_ffmpeg_log(void *ctx, int level, const char *fmt, va_list args) { + GET_CONFIG(bool, enable_ffmpeg_log, General::kEnableFFmpegLog); + if (!enable_ffmpeg_log) { + return; } - return codec; + LogLevel lev; + switch (level) { + case AV_LOG_FATAL: lev = LError; break; + case AV_LOG_ERROR: lev = LError; break; + case AV_LOG_WARNING: lev = LWarn; break; + case AV_LOG_INFO: lev = LInfo; break; + case AV_LOG_VERBOSE: lev = LDebug; break; + case AV_LOG_DEBUG: lev = LDebug; break; + case AV_LOG_TRACE: lev = LTrace; break; + default: lev = LTrace; break; + } + LoggerWrapper::printLogV(::toolkit::getLogger(), lev, __FILE__, ctx ? av_default_item_name(ctx) : "NULL", level, fmt, args); } -template -const AVCodec *getCodec(enum AVCodecID id) { - auto codec = decoder ? avcodec_find_decoder(id) : avcodec_find_encoder(id); - if (codec) { - InfoL << (decoder ? "got decoder:" : "got encoder:") << avcodec_get_name(id); - } - return codec; +static bool setupFFmpeg_l() { + av_log_set_level(AV_LOG_TRACE); + av_log_set_flags(AV_LOG_PRINT_LEVEL); + av_log_set_callback(on_ffmpeg_log); + avcodec_register_all(); + return true; } -template -const AVCodec *getCodec(First first, ARGS ...names) { - auto codec = getCodec(names...); - if (codec) { - return codec; +static void setupFFmpeg() { + static auto flag = setupFFmpeg_l(); +} + +static bool checkIfSupportedNvidia_l() { +#if !defined(_WIN32) + GET_CONFIG(bool, check_nvidia_dev, General::kCheckNvidiaDev); + if (!check_nvidia_dev) { + return false; } - return getCodec(first); + auto so = dlopen("libnvcuvid.so.1", RTLD_LAZY); + if (!so) { + WarnL << "libnvcuvid.so.1加载失败:" << get_uv_errmsg(); + return false; + } + dlclose(so); + + bool find_driver = false; + File::scanDir("/dev", [&](const string &path, bool is_dir) { + if (!is_dir && start_with(path, "/dev/nvidia")) { + //找到nvidia的驱动 + find_driver = true; + return false; + } + return true; + }, false); + + if (!find_driver) { + WarnL << "英伟达硬件编解码器驱动文件 /dev/nvidia* 不存在"; + } + return find_driver; +#else + return false; +#endif +} + +static bool checkIfSupportedNvidia() { + static auto ret = checkIfSupportedNvidia_l(); + return ret; } ////////////////////////////////////////////////////////////////////////////////////////// -FFmpegFrame::FFmpegFrame(std::shared_ptr frame) { - if (frame) { - _frame = std::move(frame); - } else { - _frame.reset(av_frame_alloc(), [](AVFrame *ptr) { - av_frame_free(&ptr); - }); - } -} - -FFmpegFrame::~FFmpegFrame() { - if (_data) { - delete[] _data; - _data = nullptr; - } -} - -AVFrame *FFmpegFrame::get() const { - return _frame.get(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////// - -FFmpegSwr::FFmpegSwr(AVSampleFormat output, int channel, int channel_layout, int samplerate) { - _target_format = output; - _target_channels = channel; - _target_channel_layout = channel_layout; - _target_samplerate = samplerate; -} - -FFmpegSwr::~FFmpegSwr() { - if (_ctx) { - swr_free(&_ctx); - } -} - -FFmpegFrame::Ptr FFmpegSwr::inputFrame(const FFmpegFrame::Ptr &frame) { - if (frame->get()->format == _target_format && - frame->get()->channels == _target_channels && - frame->get()->channel_layout == (uint64_t)_target_channel_layout && - frame->get()->sample_rate == _target_samplerate) { - //不转格式 - return frame; - } - if (!_ctx) { - _ctx = swr_alloc_set_opts(nullptr, _target_channel_layout, _target_format, _target_samplerate, - frame->get()->channel_layout, (AVSampleFormat) frame->get()->format, - frame->get()->sample_rate, 0, nullptr); - InfoL << "swr_alloc_set_opts:" << av_get_sample_fmt_name((enum AVSampleFormat) frame->get()->format) << " -> " - << av_get_sample_fmt_name(_target_format); - } - if (_ctx) { - auto out = std::make_shared(); - out->get()->format = _target_format; - out->get()->channel_layout = _target_channel_layout; - out->get()->channels = _target_channels; - out->get()->sample_rate = _target_samplerate; - out->get()->pkt_dts = frame->get()->pkt_dts; - out->get()->pts = frame->get()->pts; - - int ret = 0; - if(0 != (ret = swr_convert_frame(_ctx, out->get(), frame->get()))){ - WarnL << "swr_convert_frame failed:" << ffmpeg_err(ret); - return nullptr; - } - return out; - } - - return nullptr; -} - - -/////////////////////////////////////////////////////////////////////////// - -FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track) { -#if (LIBAVCODEC_VERSION_MAJOR < 58) - avcodec_register_all(); -#endif - const AVCodec *codec = nullptr; - const AVCodec *codec_default = nullptr; - switch (track->getCodecId()) { - case CodecH264: - codec_default = getCodec(AV_CODEC_ID_H264); - codec = getCodec("libopenh264", AV_CODEC_ID_H264, "h264_videotoolbox", "h264_cuvid"); - break; - case CodecH265: - codec_default = getCodec(AV_CODEC_ID_HEVC); - codec = getCodec(AV_CODEC_ID_HEVC, "hevc_videotoolbox", "hevc_cuvid"); - break; - case CodecAAC: - codec = getCodec(AV_CODEC_ID_AAC); - break; - case CodecG711A: - codec = getCodec(AV_CODEC_ID_PCM_ALAW); - break; - case CodecG711U: - codec = getCodec(AV_CODEC_ID_PCM_MULAW); - break; - case CodecOpus: - codec = getCodec(AV_CODEC_ID_OPUS); - break; - default: break; - } - - if (!codec) { - throw std::runtime_error("未找到解码器"); - } - - while (true) { - _context.reset(avcodec_alloc_context3(codec), [](AVCodecContext *ctx) { - avcodec_close(ctx); - avcodec_free_context(&ctx); - }); - - if (!_context) { - throw std::runtime_error("创建解码器失败"); - } - - //保存AVFrame的引用 -#ifdef FF_API_OLD_ENCDEC - _context->refcounted_frames = 1; -#endif - _context->flags |= AV_CODEC_FLAG_LOW_DELAY; - _context->flags2 |= AV_CODEC_FLAG2_FAST; - - switch (track->getCodecId()) { - case CodecG711A: - case CodecG711U: { - AudioTrack::Ptr audio = static_pointer_cast(track); - _context->channels = audio->getAudioChannel(); - _context->sample_rate = audio->getAudioSampleRate(); - _context->channel_layout = av_get_default_channel_layout(_context->channels); - break; - } - default: - break; - } - AVDictionary *dict = nullptr; - av_dict_set(&dict, "threads", "auto", 0); - av_dict_set(&dict, "zerolatency", "1", 0); - av_dict_set(&dict, "strict", "-2", 0); - - if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) { - /* we do not send complete frames */ - _context->flags |= AV_CODEC_FLAG_TRUNCATED; - } else { - // 此时业务层应该需要合帧 - _do_merger = true; - } - - int ret = avcodec_open2(_context.get(), codec, &dict); - av_dict_free(&dict); - if (ret >= 0) { - //成功 - InfoL << "打开解码器成功:" << codec->name; - break; - } - - if (codec_default && codec_default != codec) { - //硬件编解码器打开失败，尝试软件的 - WarnL << "打开解码器" << codec->name << "失败，原因是:" << ffmpeg_err(ret) << ", 再尝试打开解码器" << codec_default->name; - codec = codec_default; - continue; - } - throw std::runtime_error(StrPrinter << "打开解码器" << codec->name << "失败:" << ffmpeg_err(ret)); - } - - if (track->getTrackType() == TrackVideo) { - startThread("decoder thread"); - } -} - -FFmpegDecoder::~FFmpegDecoder() { - stopThread(); -} - -void FFmpegDecoder::flush() { - while (true) { - auto out_frame = std::make_shared(); - auto ret = avcodec_receive_frame(_context.get(), out_frame->get()); - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { - break; - } - if (ret < 0) { - WarnL << "avcodec_receive_frame failed:" << ffmpeg_err(ret); - break; - } - onDecode(out_frame); - } -} - -const AVCodecContext *FFmpegDecoder::getContext() const { - return _context.get(); -} - -bool FFmpegDecoder::inputFrame_l(const Frame::Ptr &frame) { - if (_do_merger) { - return _merger.inputFrame(frame, [&](uint32_t dts, uint32_t pts, const Buffer::Ptr &buffer, bool have_idr) { - decodeFrame(buffer->data(), buffer->size(), dts, pts); - }); - } - return decodeFrame(frame->data(), frame->size(), frame->dts(), frame->pts()); -} - -bool FFmpegDecoder::inputFrame(const Frame::Ptr &frame, bool may_async) { - if (!may_async || !TaskManager::isEnabled()) { - return inputFrame_l(frame); - } - auto frame_cache = Frame::getCacheAbleFrame(frame); - addDecodeTask(frame->keyFrame(), [this, frame_cache]() { - inputFrame_l(frame_cache); - //此处模拟解码太慢导致的主动丢帧 - //usleep(100 * 1000); - }); - return true; -} - -bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint32_t dts, uint32_t pts) { - TimeTicker2(30, TraceL); - - auto pkt = alloc_av_packet(); - pkt->data = (uint8_t *) data; - pkt->size = size; - pkt->dts = dts; - pkt->pts = pts; - - auto ret = avcodec_send_packet(_context.get(), pkt.get()); - if (ret < 0) { - if (ret != AVERROR_INVALIDDATA) { - WarnL << "avcodec_send_packet failed:" << ffmpeg_err(ret); - } - return false; - } - - while (true) { - auto out_frame = std::make_shared(); - ret = avcodec_receive_frame(_context.get(), out_frame->get()); - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { - break; - } - if (ret < 0) { - WarnL << "avcodec_receive_frame failed:" << ffmpeg_err(ret); - break; - } - if (pts - out_frame->get()->pts > MAX_DELAY_SECOND * 1000 && _ticker.createdTime() > 10 * 1000) { - //后面的帧才忽略,防止Track无法ready - WarnL << "解码时，忽略" << MAX_DELAY_SECOND << "秒前的数据:" << pts << " " << out_frame->get()->pts; - continue; - } - onDecode(out_frame); - } - return true; -} - -void FFmpegDecoder::setOnDecode(FFmpegDecoder::onDec cb) { - _cb = std::move(cb); -} - -void FFmpegDecoder::onDecode(const FFmpegFrame::Ptr &frame) { - if (_cb) { - _cb(frame); - } -} - -//////////////////////////////////////////////////////////////////////// - -void TaskManager::pushExit(){ - { - lock_guard lck(_task_mtx); - _exit = true; - _task.clear(); - _task.emplace_back([](){ - throw ThreadExitException(); - }); - } - _sem.post(10); -} - -void TaskManager::addEncodeTask(function task) { +bool TaskManager::addEncodeTask(function task) { { lock_guard lck(_task_mtx); _task.emplace_back(std::move(task)); - if (_task.size() > 30) { + if (_task.size() > _max_task) { WarnL << "encoder thread task is too more, now drop frame!"; _task.pop_front(); } } _sem.post(); + return true; } -void TaskManager::addDecodeTask(bool key_frame, function task) { +bool TaskManager::addDecodeTask(bool key_frame, function task) { { lock_guard lck(_task_mtx); if (_decode_drop_start) { if (!key_frame) { TraceL << "decode thread drop frame"; - return; + return false; } _decode_drop_start = false; InfoL << "decode thread stop drop frame"; } _task.emplace_back(std::move(task)); - if (_task.size() > 30) { + if (_task.size() > _max_task) { _decode_drop_start = true; WarnL << "decode thread start drop frame"; } } _sem.post(); + return true; +} + +void TaskManager::setMaxTaskSize(size_t size) { + CHECK(size >= 3 && size <= 1000, "async task size limited to 3 ~ 1000, now size is:", size); + _max_task = size; } void TaskManager::startThread(const string &name) { _thread.reset(new thread([this, name]() { onThreadRun(name); - }), [this](thread *ptr) { - pushExit(); + }), [](thread *ptr) { ptr->join(); delete ptr; }); } -void TaskManager::stopThread() { +void TaskManager::stopThread(bool drop_task) { + TimeTicker(); + if (!_thread) { + return; + } + { + lock_guard lck(_task_mtx); + if (drop_task) { + _exit = true; + _task.clear(); + } + _task.emplace_back([]() { + throw ThreadExitException(); + }); + } + _sem.post(10); _thread = nullptr; } TaskManager::~TaskManager() { - stopThread(); + stopThread(true); } bool TaskManager::isEnabled() const { @@ -423,9 +210,432 @@ void TaskManager::onThreadRun(const string &name) { } catch (std::exception &ex) { WarnL << ex.what(); continue; + } catch (...) { + WarnL << "catch one unknown exception"; + throw; } } InfoL << name << " exited!"; } -#endif//ENABLE_FFMPEG \ No newline at end of file +////////////////////////////////////////////////////////////////////////////////////////// + +FFmpegFrame::FFmpegFrame(std::shared_ptr frame) { + if (frame) { + _frame = std::move(frame); + } else { + _frame.reset(av_frame_alloc(), [](AVFrame *ptr) { + av_frame_free(&ptr); + }); + } +} + +FFmpegFrame::~FFmpegFrame() { + if (_data) { + delete[] _data; + _data = nullptr; + } +} + +AVFrame *FFmpegFrame::get() const { + return _frame.get(); +} + +void FFmpegFrame::fillPicture(AVPixelFormat target_format, int target_width, int target_height) { + assert(_data == nullptr); + _data = new char[avpicture_get_size(target_format, target_width, target_height)]; + avpicture_fill((AVPicture *) _frame.get(), (uint8_t *) _data, target_format, target_width, target_height); +} + +/////////////////////////////////////////////////////////////////////////// + +template +AVCodec *getCodec(ARGS ...names); + +template +AVCodec *getCodec(const char *name) { + auto codec = decoder ? avcodec_find_decoder_by_name(name) : avcodec_find_encoder_by_name(name); + if (codec) { + InfoL << (decoder ? "got decoder:" : "got encoder:") << name; + } else { + TraceL << (decoder ? "decoder:" : "encoder:") << name << " not found"; + } + return codec; +} + +template +AVCodec *getCodec(enum AVCodecID id) { + auto codec = decoder ? avcodec_find_decoder(id) : avcodec_find_encoder(id); + if (codec) { + InfoL << (decoder ? "got decoder:" : "got encoder:") << avcodec_get_name(id); + } else { + TraceL << (decoder ? "decoder:" : "encoder:") << avcodec_get_name(id) << " not found"; + } + return codec; +} + +template +AVCodec *getCodec(First first, ARGS ...names) { + auto codec = getCodec(names...); + if (codec) { + return codec; + } + return getCodec(first); +} + +FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num) { + setupFFmpeg(); + AVCodec *codec = nullptr; + AVCodec *codec_default = nullptr; + switch (track->getCodecId()) { + case CodecH264: + codec_default = getCodec(AV_CODEC_ID_H264); + if (checkIfSupportedNvidia()) { + codec = getCodec("libopenh264", AV_CODEC_ID_H264, "h264_qsv", "h264_videotoolbox", "h264_cuvid", "h264_nvmpi"); + } else { + codec = getCodec("libopenh264", AV_CODEC_ID_H264, "h264_qsv", "h264_videotoolbox", "h264_nvmpi"); + } + break; + case CodecH265: + codec_default = getCodec(AV_CODEC_ID_HEVC); + if (checkIfSupportedNvidia()) { + codec = getCodec(AV_CODEC_ID_HEVC, "hevc_qsv", "hevc_videotoolbox", "hevc_cuvid", "hevc_nvmpi"); + } else { + codec = getCodec(AV_CODEC_ID_HEVC, "hevc_qsv", "hevc_videotoolbox", "hevc_nvmpi"); + } + break; + case CodecAAC: + codec = getCodec(AV_CODEC_ID_AAC); + break; + case CodecG711A: + codec = getCodec(AV_CODEC_ID_PCM_ALAW); + break; + case CodecG711U: + codec = getCodec(AV_CODEC_ID_PCM_MULAW); + break; + case CodecOpus: + codec = getCodec(AV_CODEC_ID_OPUS); + break; + case CodecVP8: + codec = getCodec(AV_CODEC_ID_VP8); + break; + case CodecVP9: + codec = getCodec(AV_CODEC_ID_VP9); + break; + default: + break; + } + + if (!codec) { + throw std::runtime_error("未找到解码器"); + } + + while (true) { + _context.reset(avcodec_alloc_context3(codec), [](AVCodecContext *ctx) { + avcodec_free_context(&ctx); + }); + + if (!_context) { + throw std::runtime_error("创建解码器失败"); + } + + //保存AVFrame的引用 + _context->refcounted_frames = 1; + _context->flags |= AV_CODEC_FLAG_LOW_DELAY; + _context->flags2 |= AV_CODEC_FLAG2_FAST; + if (track->getTrackType() == TrackVideo) { + _context->width = static_pointer_cast(track)->getVideoWidth(); + _context->height = static_pointer_cast(track)->getVideoHeight(); + } + + switch (track->getCodecId()) { + case CodecG711A: + case CodecG711U: { + AudioTrack::Ptr audio = static_pointer_cast(track); + _context->channels = audio->getAudioChannel(); + _context->sample_rate = audio->getAudioSampleRate(); + _context->channel_layout = av_get_default_channel_layout(_context->channels); + break; + } + default: + break; + } + AVDictionary *dict = nullptr; + if (thread_num <= 0) { + av_dict_set(&dict, "threads", "auto", 0); + } else { + av_dict_set(&dict, "threads", to_string(MIN(thread_num, thread::hardware_concurrency())).data(), 0); + } + av_dict_set(&dict, "zerolatency", "1", 0); + av_dict_set(&dict, "strict", "-2", 0); + + if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) { + /* we do not send complete frames */ + _context->flags |= AV_CODEC_FLAG_TRUNCATED; + } else { + // 此时业务层应该需要合帧 + _do_merger = true; + } + + int ret = avcodec_open2(_context.get(), codec, &dict); + av_dict_free(&dict); + if (ret >= 0) { + //成功 + InfoL << "打开解码器成功:" << codec->name; + break; + } + + if (codec_default && codec_default != codec) { + //硬件编解码器打开失败，尝试软件的 + WarnL << "打开解码器" << codec->name << "失败，原因是:" << ffmpeg_err(ret) << ", 再尝试打开解码器" << codec_default->name; + codec = codec_default; + continue; + } + throw std::runtime_error(StrPrinter << "打开解码器" << codec->name << "失败:" << ffmpeg_err(ret)); + } +} + +FFmpegDecoder::~FFmpegDecoder() { + stopThread(true); + if (_do_merger) { + _merger.inputFrame(nullptr, [&](uint32_t dts, uint32_t pts, const Buffer::Ptr &buffer, bool have_idr) { + decodeFrame(buffer->data(), buffer->size(), dts, pts, false); + }); + } + flush(); +} + +void FFmpegDecoder::flush() { + while (true) { + auto out_frame = std::make_shared(); + auto ret = avcodec_receive_frame(_context.get(), out_frame->get()); + if (ret == AVERROR(EAGAIN)) { + avcodec_send_packet(_context.get(), nullptr); + continue; + } + if (ret == AVERROR_EOF) { + break; + } + if (ret < 0) { + WarnL << "avcodec_receive_frame failed:" << ffmpeg_err(ret); + break; + } + onDecode(out_frame); + } +} + +const AVCodecContext *FFmpegDecoder::getContext() const { + return _context.get(); +} + +bool FFmpegDecoder::inputFrame_l(const Frame::Ptr &frame, bool live, bool enable_merge) { + if (_do_merger && enable_merge) { + return _merger.inputFrame(frame, [&](uint32_t dts, uint32_t pts, const Buffer::Ptr &buffer, bool have_idr) { + decodeFrame(buffer->data(), buffer->size(), dts, pts, live); + }); + } + + return decodeFrame(frame->data(), frame->size(), frame->dts(), frame->pts(), live); +} + +bool FFmpegDecoder::inputFrame(const Frame::Ptr &frame, bool live, bool async, bool enable_merge) { + if (async && !TaskManager::isEnabled() && getContext()->codec_type == AVMEDIA_TYPE_VIDEO) { + //开启异步编码，且为视频，尝试启动异步解码线程 + startThread("decoder thread"); + } + + if (!async || !TaskManager::isEnabled()) { + return inputFrame_l(frame, live, enable_merge); + } + + auto frame_cache = Frame::getCacheAbleFrame(frame); + return addDecodeTask(frame->keyFrame(), [this, live, frame_cache, enable_merge]() { + inputFrame_l(frame_cache, live, enable_merge); + //此处模拟解码太慢导致的主动丢帧 + //usleep(100 * 1000); + }); +} + +bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint32_t dts, uint32_t pts, bool live) { + TimeTicker2(30, TraceL); + + auto pkt = alloc_av_packet(); + pkt->data = (uint8_t *) data; + pkt->size = size; + pkt->dts = dts; + pkt->pts = pts; + + auto ret = avcodec_send_packet(_context.get(), pkt.get()); + if (ret < 0) { + if (ret != AVERROR_INVALIDDATA) { + WarnL << "avcodec_send_packet failed:" << ffmpeg_err(ret); + } + return false; + } + + while (true) { + auto out_frame = std::make_shared(); + ret = avcodec_receive_frame(_context.get(), out_frame->get()); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } + if (ret < 0) { + WarnL << "avcodec_receive_frame failed:" << ffmpeg_err(ret); + break; + } + if (live && pts - out_frame->get()->pts > MAX_DELAY_SECOND * 1000 && _ticker.createdTime() > 10 * 1000) { + //后面的帧才忽略,防止Track无法ready + WarnL << "解码时，忽略" << MAX_DELAY_SECOND << "秒前的数据:" << pts << " " << out_frame->get()->pts; + continue; + } + onDecode(out_frame); + } + return true; +} + +void FFmpegDecoder::setOnDecode(FFmpegDecoder::onDec cb) { + _cb = std::move(cb); +} + +void FFmpegDecoder::onDecode(const FFmpegFrame::Ptr &frame) { + if (_cb) { + _cb(frame); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////// + +FFmpegSwr::FFmpegSwr(AVSampleFormat output, int channel, int channel_layout, int samplerate) { + _target_format = output; + _target_channels = channel; + _target_channel_layout = channel_layout; + _target_samplerate = samplerate; +} + +FFmpegSwr::~FFmpegSwr() { + if (_ctx) { + swr_free(&_ctx); + } +} + +FFmpegFrame::Ptr FFmpegSwr::inputFrame(const FFmpegFrame::Ptr &frame) { + if (frame->get()->format == _target_format && + frame->get()->channels == _target_channels && + frame->get()->channel_layout == _target_channel_layout && + frame->get()->sample_rate == _target_samplerate) { + //不转格式 + return frame; + } + if (!_ctx) { + _ctx = swr_alloc_set_opts(nullptr, _target_channel_layout, _target_format, _target_samplerate, + frame->get()->channel_layout, (AVSampleFormat) frame->get()->format, + frame->get()->sample_rate, 0, nullptr); + InfoL << "swr_alloc_set_opts:" << av_get_sample_fmt_name((enum AVSampleFormat) frame->get()->format) << " -> " + << av_get_sample_fmt_name(_target_format); + } + if (_ctx) { + auto out = std::make_shared(); + out->get()->format = _target_format; + out->get()->channel_layout = _target_channel_layout; + out->get()->channels = _target_channels; + out->get()->sample_rate = _target_samplerate; + out->get()->pkt_dts = frame->get()->pkt_dts; + out->get()->pts = frame->get()->pts; + + int ret = 0; + if (0 != (ret = swr_convert_frame(_ctx, out->get(), frame->get()))) { + WarnL << "swr_convert_frame failed:" << ffmpeg_err(ret); + return nullptr; + } + return out; + } + + return nullptr; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////// + +FFmpegSws::FFmpegSws(AVPixelFormat output, int width, int height) { + _target_format = output; + _target_width = width; + _target_height = height; +} + +FFmpegSws::~FFmpegSws() { + if (_ctx) { + sws_freeContext(_ctx); + _ctx = nullptr; + } +} + +int FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame, uint8_t *data) { + TimeTicker2(30, TraceL); + if (!_target_width) { + _target_width = frame->get()->width; + } + if (!_target_height) { + _target_height = frame->get()->height; + } + AVFrame dst; + memset(&dst, 0, sizeof(dst)); + avpicture_fill((AVPicture *) &dst, data, _target_format, _target_width, _target_height); + if (!_ctx) { + _ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format, + _target_width, _target_height, _target_format, SWS_FAST_BILINEAR, NULL, NULL, NULL); + InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> " + << av_get_pix_fmt_name(_target_format); + } + assert(_ctx); + int ret = 0; + if (0 >= (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height, dst.data, + dst.linesize))) { + WarnL << "sws_scale failed:" << ffmpeg_err(ret); + } + return ret; +} + +FFmpegFrame::Ptr FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame) { + TimeTicker2(30, TraceL); + + if (!_target_width) { + _target_width = frame->get()->width; + } + if (!_target_height) { + _target_height = frame->get()->height; + } + if (frame->get()->format == _target_format && frame->get()->width == _target_width + && frame->get()->height == _target_height) { + //不转格式 + return frame; + } + if (!_ctx) { + _ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format, + _target_width, _target_height, _target_format, + SWS_FAST_BILINEAR, NULL, NULL, NULL); + InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> " + << av_get_pix_fmt_name(_target_format); + } + if (_ctx) { + auto out = std::make_shared(); + if (!out->get()->data[0]) { + out->fillPicture(_target_format, _target_width, _target_height); + } + int ret = 0; + if (0 == (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height, + out->get()->data, out->get()->linesize))) { + WarnL << "sws_scale failed:" << ffmpeg_err(ret); + return nullptr; + } + + out->get()->format = _target_format; + out->get()->width = _target_width; + out->get()->height = _target_height; + out->get()->pkt_dts = frame->get()->pkt_dts; + out->get()->pts = frame->get()->pts; + return out; + } + return nullptr; +} + +} //namespace mediakit +#endif//ENABLE_FFMPEG diff --git a/src/Codec/Transcode.h b/src/Codec/Transcode.h index 9a81e01e..ac10dea6 100644 --- a/src/Codec/Transcode.h +++ b/src/Codec/Transcode.h @@ -8,23 +8,29 @@ * may be found in the AUTHORS file in the root of the source tree. */ -#ifndef FFMpegDecoder_H_ -#define FFMpegDecoder_H_ +#ifndef ZLMEDIAKIT_TRANSCODE_H +#define ZLMEDIAKIT_TRANSCODE_H + +#if defined(ENABLE_FFMPEG) #include "Util/TimeTicker.h" #include "Common/MediaSink.h" -#if defined(ENABLE_FFMPEG) - #ifdef __cplusplus extern "C" { #endif +#include "libswscale/swscale.h" +#include "libavutil/avutil.h" +#include "libavutil/pixdesc.h" #include "libavcodec/avcodec.h" #include "libswresample/swresample.h" +#include "libavutil/audio_fifo.h" #ifdef __cplusplus } #endif +namespace mediakit { + class FFmpegFrame { public: using Ptr = std::shared_ptr; @@ -33,6 +39,7 @@ public: ~FFmpegFrame(); AVFrame *get() const; + void fillPicture(AVPixelFormat target_format, int target_width, int target_height); private: char *_data = nullptr; @@ -45,7 +52,6 @@ public: FFmpegSwr(AVSampleFormat output, int channel, int channel_layout, int samplerate); ~FFmpegSwr(); - FFmpegFrame::Ptr inputFrame(const FFmpegFrame::Ptr &frame); private: @@ -59,19 +65,19 @@ private: class TaskManager { public: TaskManager() = default; - ~TaskManager(); + virtual ~TaskManager(); + + void setMaxTaskSize(size_t size); + void stopThread(bool drop_task); protected: void startThread(const std::string &name); - void stopThread(); - - void addEncodeTask(std::function task); - void addDecodeTask(bool key_frame, std::function task); + bool addEncodeTask(std::function task); + bool addDecodeTask(bool key_frame, std::function task); bool isEnabled() const; private: void onThreadRun(const std::string &name); - void pushExit(); private: class ThreadExitException : public std::runtime_error { @@ -83,39 +89,55 @@ private: private: bool _decode_drop_start = false; bool _exit = false; + size_t _max_task = 30; std::mutex _task_mtx; toolkit::semaphore _sem; toolkit::List > _task; std::shared_ptr _thread; }; -class FFmpegDecoder : private TaskManager { +class FFmpegDecoder : public TaskManager { public: using Ptr = std::shared_ptr; using onDec = std::function; - FFmpegDecoder(const mediakit::Track::Ptr &track); - ~FFmpegDecoder(); + FFmpegDecoder(const Track::Ptr &track, int thread_num = 2); + ~FFmpegDecoder() override; - bool inputFrame(const mediakit::Frame::Ptr &frame, bool may_async = true); + bool inputFrame(const Frame::Ptr &frame, bool live, bool async, bool enable_merge = true); void setOnDecode(onDec cb); void flush(); const AVCodecContext *getContext() const; private: void onDecode(const FFmpegFrame::Ptr &frame); - bool inputFrame_l(const mediakit::Frame::Ptr &frame); - bool decodeFrame(const char *data, size_t size, uint32_t dts, uint32_t pts); + bool inputFrame_l(const Frame::Ptr &frame, bool live, bool enable_merge); + bool decodeFrame(const char *data, size_t size, uint32_t dts, uint32_t pts, bool live); private: bool _do_merger = false; toolkit::Ticker _ticker; onDec _cb; std::shared_ptr _context; - mediakit::FrameMerger _merger { mediakit::FrameMerger::h264_prefix }; + FrameMerger _merger{FrameMerger::h264_prefix}; }; +class FFmpegSws { +public: + using Ptr = std::shared_ptr; + + FFmpegSws(AVPixelFormat output, int width, int height); + ~FFmpegSws(); + FFmpegFrame::Ptr inputFrame(const FFmpegFrame::Ptr &frame); + int inputFrame(const FFmpegFrame::Ptr &frame, uint8_t *data); + +private: + int _target_width; + int _target_height; + SwsContext *_ctx = nullptr; + AVPixelFormat _target_format; +}; + +}//namespace mediakit #endif// ENABLE_FFMPEG -#endif /* FFMpegDecoder_H_ */ - - +#endif //ZLMEDIAKIT_TRANSCODE_H diff --git a/src/Common/Device.cpp b/src/Common/Device.cpp index c0bd682f..c6be0721 100644 --- a/src/Common/Device.cpp +++ b/src/Common/Device.cpp @@ -28,22 +28,22 @@ using namespace std; namespace mediakit { -bool DevChannel::inputYUV(char* apcYuv[3], int aiYuvLen[3], uint32_t uiStamp) { +bool DevChannel::inputYUV(char *yuv[3], int linesize[3], uint32_t cts) { #ifdef ENABLE_X264 //TimeTicker1(50); if (!_pH264Enc) { _pH264Enc.reset(new H264Encoder()); - if (!_pH264Enc->init(_video->iWidth, _video->iHeight, _video->iFrameRate)) { + if (!_pH264Enc->init(_video->iWidth, _video->iHeight, _video->iFrameRate, _video->iBitRate)) { _pH264Enc.reset(); WarnL << "H264Encoder init failed!"; } } if (_pH264Enc) { - H264Encoder::H264Frame *pOut; - int iFrames = _pH264Enc->inputData(apcYuv, aiYuvLen, uiStamp, &pOut); + H264Encoder::H264Frame *out_frames; + int frames = _pH264Enc->inputData(yuv, linesize, cts, &out_frames); bool ret = false; - for (int i = 0; i < iFrames; i++) { - ret = inputH264((char *) pOut[i].pucData, pOut[i].iLength, uiStamp) ? true : ret; + for (int i = 0; i < frames; i++) { + ret = inputH264((char *) out_frames[i].pucData, out_frames[i].iLength, cts) ? true : ret; } return ret; } diff --git a/src/Common/Device.h b/src/Common/Device.h index 6eb9c81d..f1ab266f 100644 --- a/src/Common/Device.h +++ b/src/Common/Device.h @@ -29,6 +29,7 @@ public: int iWidth; int iHeight; float iFrameRate; + int iBitRate = 2 * 1024 * 1024; }; class AudioInfo { @@ -104,19 +105,19 @@ public: /** * 输入yuv420p视频帧，内部会完成编码并调用inputH264方法 - * @param apcYuv - * @param aiYuvLen - * @param uiStamp + * @param yuv yuv420p数据指针 + * @param linesize yuv420p数据linesize + * @param cts 采集时间戳，单位毫秒 */ - bool inputYUV(char *apcYuv[3], int aiYuvLen[3], uint32_t uiStamp); + bool inputYUV(char *yuv[3], int linesize[3], uint32_t cts); /** * 输入pcm数据，内部会完成编码并调用inputAAC方法 - * @param pcData - * @param iDataLen - * @param uiStamp + * @param data pcm数据指针，int16整形 + * @param len pcm数据长度 + * @param cts 采集时间戳，单位毫秒 */ - bool inputPCM(char *pcData, int iDataLen, uint32_t uiStamp); + bool inputPCM(char *data, int len, uint32_t cts); private: MediaOriginType getOriginType(MediaSource &sender) const override; diff --git a/src/Common/config.cpp b/src/Common/config.cpp index c1613af0..50e111d9 100644 --- a/src/Common/config.cpp +++ b/src/Common/config.cpp @@ -42,7 +42,7 @@ bool loadIniConfig(const char *ini_path){ namespace Broadcast { const string kBroadcastMediaChanged = "kBroadcastMediaChanged"; const string kBroadcastRecordMP4 = "kBroadcastRecordMP4"; -const string kBroadcastRecordTs = "kBroadcastRecoredTs"; +const string kBroadcastRecordTs = "kBroadcastRecordTs"; const string kBroadcastHttpRequest = "kBroadcastHttpRequest"; const string kBroadcastHttpAccess = "kBroadcastHttpAccess"; const string kBroadcastOnGetRtspRealm = "kBroadcastOnGetRtspRealm"; @@ -77,6 +77,8 @@ const string kRtmpDemand = GENERAL_FIELD"rtmp_demand"; const string kTSDemand = GENERAL_FIELD"ts_demand"; const string kFMP4Demand = GENERAL_FIELD"fmp4_demand"; const string kEnableAudio = GENERAL_FIELD"enable_audio"; +const string kCheckNvidiaDev = GENERAL_FIELD"check_nvidia_dev"; +const string kEnableFFmpegLog = GENERAL_FIELD"enable_ffmpeg_log"; const string kWaitTrackReadyMS = GENERAL_FIELD"wait_track_ready_ms"; const string kWaitAddTrackMS = GENERAL_FIELD"wait_add_track_ms"; const string kUnreadyFrameCache = GENERAL_FIELD"unready_frame_cache"; @@ -100,6 +102,8 @@ static onceToken token([](){ mINI::Instance()[kTSDemand] = 0; mINI::Instance()[kFMP4Demand] = 0; mINI::Instance()[kEnableAudio] = 1; + mINI::Instance()[kCheckNvidiaDev] = 1; + mINI::Instance()[kEnableFFmpegLog] = 0; mINI::Instance()[kWaitTrackReadyMS] = 10000; mINI::Instance()[kWaitAddTrackMS] = 3000; mINI::Instance()[kUnreadyFrameCache] = 100; diff --git a/src/Common/config.h b/src/Common/config.h index d49ddc6a..606dfbb0 100644 --- a/src/Common/config.h +++ b/src/Common/config.h @@ -183,6 +183,10 @@ extern const std::string kTSDemand; extern const std::string kFMP4Demand; //转协议是否全局开启或忽略音频 extern const std::string kEnableAudio; +//在docker环境下，不能通过英伟达驱动是否存在来判断是否支持硬件转码 +extern const std::string kCheckNvidiaDev; +//是否开启ffmpeg日志 +extern const std::string kEnableFFmpegLog; //最多等待未初始化的Track 10秒，超时之后会忽略未初始化的Track extern const std::string kWaitTrackReadyMS; //如果直播流只有单Track，最多等待3秒，超时后未收到其他Track的数据，则认为是单Track