amass/ZLMediaKit
1
0
Fork 0
mirror of https://github.com/ZLMediaKit/ZLMediaKit.git synced 2024-11-22 10:40:05 +08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

优化编解码相关代码

Browse Source
This commit is contained in:
ziyue 2022-05-25 15:11:26 +08:00
parent 0739b1ddd2
commit 9736badcea
9 changed files with 630 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
player/test_player.cpp
View File

@ -84,7 +84,7 @@ int main(int argc, char *argv[]) {
});
});
auto delegate = std::make_shared<FrameWriterInterfaceHelper>([decoder](const Frame::Ptr &frame) {
return decoder->inputFrame(frame, false);
return decoder->inputFrame(frame, false, true);
});
videoTrack->addDelegate(delegate);
}
@ -106,7 +106,7 @@ int main(int argc, char *argv[]) {
audio_player->playPCM((const char *) (pcm->get()->data[0]), MIN(len, frame->get()->linesize[0]));
});
auto audio_delegate = std::make_shared<FrameWriterInterfaceHelper>( [decoder](const Frame::Ptr &frame) {
return decoder->inputFrame(frame, false);
return decoder->inputFrame(frame, false, true);
});
audioTrack->addDelegate(audio_delegate);
}

61
src/Codec/H264Encoder.cpp
View File

@ -9,17 +9,14 @@
*/
#ifdef ENABLE_X264
#include "H264Encoder.h"
#include "Util/TimeTicker.h"
using namespace toolkit;
namespace mediakit {
H264Encoder::H264Encoder() {
}
H264Encoder::H264Encoder() {}
H264Encoder::~H264Encoder() {
//* 清除图像区域
@ -39,7 +36,6 @@ H264Encoder::~H264Encoder() {
}
}
/*typedef struct x264_param_t
{
CPU 标志位
@ -212,7 +208,7 @@ Value的值就是fps。
void (*param_free)( void* );
} x264_param_t;*/
bool H264Encoder::init(int iWidth, int iHeight, int iFps) {
bool H264Encoder::init(int iWidth, int iHeight, int iFps, int iBitRate) {
if (_pX264Handle) {
return true;
}
@ -222,7 +218,7 @@ bool H264Encoder::init(int iWidth, int iHeight, int iFps) {
x264_param_default_preset(pX264Param, "ultrafast", "zerolatency");
//* cpuFlags
pX264Param->i_threads = X264_SYNC_LOOKAHEAD_AUTO; //* 取空缓冲区继续使用不死锁的保证.
pX264Param->i_threads = X264_SYNC_LOOKAHEAD_AUTO; //* 取空缓冲区继续使用不死锁的保证.
//* video Properties
pX264Param->i_width = iWidth; //* 宽度.
pX264Param->i_height = iHeight; //* 高度
@ -230,21 +226,21 @@ bool H264Encoder::init(int iWidth, int iHeight, int iFps) {
pX264Param->i_keyint_max = iFps * 3; //ffmpeg:gop_size 关键帧最大间隔
pX264Param->i_keyint_min = iFps * 1; //ffmpeg:keyint_min 关键帧最小间隔
//* Rate control Parameters
pX264Param->rc.i_bitrate = 5000; //* 码率(比特率,单位Kbps)
pX264Param->rc.i_qp_step = 1; //最大的在帧与帧之间进行切变的量化因子的变化量。ffmpeg:max_qdiff
pX264Param->rc.i_qp_min = 10; //ffmpeg:qmin;最小的量化因子。取值范围1-51。建议在10-30之间。
pX264Param->rc.i_qp_max = 41; //ffmpeg:qmax;最大的量化因子。取值范围1-51。建议在10-30之间。
pX264Param->rc.i_bitrate = iBitRate / 1000; //* 码率(比特率,单位Kbps)
pX264Param->rc.i_qp_step = 1; //最大的在帧与帧之间进行切变的量化因子的变化量。ffmpeg:max_qdiff
pX264Param->rc.i_qp_min = 10; //ffmpeg:qmin;最小的量化因子。取值范围1-51。建议在10-30之间。
pX264Param->rc.i_qp_max = 41; //ffmpeg:qmax;最大的量化因子。取值范围1-51。建议在10-30之间。
pX264Param->rc.f_qcompress = 0.6;//ffmpeg:qcompress 量化器压缩比率0-1.越小则比特率越区域固定，但是越高越使量化器参数越固定
pX264Param->analyse.i_me_range = 16; //ffmpeg:me_range 运动侦测的半径
pX264Param->i_frame_reference = 3; //ffmpeg:refsB和P帧向前预测参考的帧数。取值范围1-16。
//该值不影响解码的速度，但是越大解码
//所需的内存越大。这个值在一般情况下
//越大效果越好，但是超过6以后效果就
//不明显了。
pX264Param->analyse.i_me_range = 16; //ffmpeg:me_range 运动侦测的半径
pX264Param->i_frame_reference = 3; //ffmpeg:refsB和P帧向前预测参考的帧数。取值范围1-16。
//该值不影响解码的速度，但是越大解码
//所需的内存越大。这个值在一般情况下
//越大效果越好，但是超过6以后效果就
//不明显了。
pX264Param->analyse.i_trellis = 1; //ffmpeg:trellis
pX264Param->analyse.i_trellis = 1; //ffmpeg:trellis
//pX264Param->analyse.i_me_method=X264_ME_DIA;//ffmpeg:me_method ME_ZERO 运动侦测的方式
pX264Param->rc.f_qblur = 0.5; //ffmpeg:qblur
pX264Param->rc.f_qblur = 0.5; //ffmpeg:qblur
//* bitstream parameters
/*open-GOP
@ -268,7 +264,7 @@ bool H264Encoder::init(int iWidth, int iHeight, int iFps) {
由于B帧压缩性能好于P帧，因此open-GOP在编码性能上稍微优于close-GOP，
但为了兼容性和少一些麻烦，还是把opne-GOP关闭的好。*/
pX264Param->b_open_gop = 0;
pX264Param->i_bframe = 0; //最大B帧数.
pX264Param->i_bframe = 0; //最大B帧数.
pX264Param->i_bframe_pyramid = 0;
pX264Param->i_bframe_adaptive = X264_B_ADAPT_TRELLIS;
//* Log
@ -304,20 +300,19 @@ bool H264Encoder::init(int iWidth, int iHeight, int iFps) {
return true;
}
int H264Encoder::inputData(char* apcYuv[3], int aiYuvLen[3], int64_t i64Pts, H264Frame** ppFrame) {
int H264Encoder::inputData(char *yuv[3], int linesize[3], int64_t cts, H264Frame **out_frame) {
//TimeTicker1(5);
_pPicIn->img.i_stride[0] = aiYuvLen[0];
_pPicIn->img.i_stride[1] = aiYuvLen[1];
_pPicIn->img.i_stride[2] = aiYuvLen[2];
_pPicIn->img.plane[0] = (uint8_t *) apcYuv[0];
_pPicIn->img.plane[1] = (uint8_t *) apcYuv[1];
_pPicIn->img.plane[2] = (uint8_t *) apcYuv[2];
_pPicIn->i_pts = i64Pts;
_pPicIn->img.i_stride[0] = linesize[0];
_pPicIn->img.i_stride[1] = linesize[1];
_pPicIn->img.i_stride[2] = linesize[2];
_pPicIn->img.plane[0] = (uint8_t *) yuv[0];
_pPicIn->img.plane[1] = (uint8_t *) yuv[1];
_pPicIn->img.plane[2] = (uint8_t *) yuv[2];
_pPicIn->i_pts = cts;
int iNal;
x264_nal_t* pNals;
x264_nal_t *pNals;
int iResult = x264_encoder_encode(_pX264Handle, &pNals, &iNal, _pPicIn,
_pPicOut);
int iResult = x264_encoder_encode(_pX264Handle, &pNals, &iNal, _pPicIn, _pPicOut);
if (iResult <= 0) {
return 0;
}
@ -327,7 +322,7 @@ int H264Encoder::inputData(char* apcYuv[3], int aiYuvLen[3], int64_t i64Pts, H26
_aFrames[i].iLength = pNal.i_payload;
_aFrames[i].pucData = pNal.p_payload;
}
*ppFrame = _aFrames;
*out_frame = _aFrames;
return iNal;
}

19
src/Codec/H264Encoder.h
View File

@ -7,13 +7,10 @@
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef CODEC_H264ENCODER_H_
#define CODEC_H264ENCODER_H_
#include <cstdint>
#ifdef __cplusplus
extern "C" {
#endif //__cplusplus
@ -32,14 +29,16 @@ public:
uint8_t *pucData;
} H264Frame;
H264Encoder(void);
virtual ~H264Encoder(void);
bool init(int iWidth, int iHeight, int iFps);
int inputData(char *apcYuv[3], int aiYuvLen[3], int64_t i64Pts, H264Frame **ppFrame);
H264Encoder();
~H264Encoder();
bool init(int iWidth, int iHeight, int iFps, int iBitRate);
int inputData(char *yuv[3], int linesize[3], int64_t cts, H264Frame **out_frame);
private:
x264_t* _pX264Handle = nullptr;
x264_picture_t* _pPicIn = nullptr;
x264_picture_t* _pPicOut = nullptr;
x264_t *_pX264Handle = nullptr;
x264_picture_t *_pPicIn = nullptr;
x264_picture_t *_pPicOut = nullptr;
H264Frame _aFrames[10];
};

836
src/Codec/Transcode.cpp
View File

@ -9,13 +9,20 @@
*/
#if defined(ENABLE_FFMPEG)
#if !defined(_WIN32)
#include <dlfcn.h>
#endif
#include "Util/File.h"
#include "Util/uv_errno.h"
#include "Transcode.h"
#include "Extension/AAC.h"
#define MAX_DELAY_SECOND 3
using namespace std;
using namespace toolkit;
using namespace mediakit;
namespace mediakit {
static string ffmpeg_err(int errnum) {
char errbuf[AV_ERROR_MAX_STRING_SIZE];
@ -23,7 +30,7 @@ static string ffmpeg_err(int errnum) {
return errbuf;
}
std::shared_ptr<AVPacket> alloc_av_packet(){
std::shared_ptr<AVPacket> alloc_av_packet() {
auto pkt = std::shared_ptr<AVPacket>(av_packet_alloc(), [](AVPacket *pkt) {
av_packet_free(&pkt);
});
@ -33,366 +40,146 @@ std::shared_ptr<AVPacket> alloc_av_packet(){
}
//////////////////////////////////////////////////////////////////////////////////////////
template<bool decoder = true, typename ...ARGS>
const AVCodec *getCodec(ARGS ...names);
template<bool decoder = true>
const AVCodec *getCodec(const char *name) {
auto codec = decoder ? avcodec_find_decoder_by_name(name) : avcodec_find_encoder_by_name(name);
if (codec) {
InfoL << (decoder ? "got decoder:" : "got encoder:") << name;
static void on_ffmpeg_log(void *ctx, int level, const char *fmt, va_list args) {
GET_CONFIG(bool, enable_ffmpeg_log, General::kEnableFFmpegLog);
if (!enable_ffmpeg_log) {
return;
}
return codec;
LogLevel lev;
switch (level) {
case AV_LOG_FATAL: lev = LError; break;
case AV_LOG_ERROR: lev = LError; break;
case AV_LOG_WARNING: lev = LWarn; break;
case AV_LOG_INFO: lev = LInfo; break;
case AV_LOG_VERBOSE: lev = LDebug; break;
case AV_LOG_DEBUG: lev = LDebug; break;
case AV_LOG_TRACE: lev = LTrace; break;
default: lev = LTrace; break;
}
LoggerWrapper::printLogV(::toolkit::getLogger(), lev, __FILE__, ctx ? av_default_item_name(ctx) : "NULL", level, fmt, args);
}
template<bool decoder = true>
const AVCodec *getCodec(enum AVCodecID id) {
auto codec = decoder ? avcodec_find_decoder(id) : avcodec_find_encoder(id);
if (codec) {
InfoL << (decoder ? "got decoder:" : "got encoder:") << avcodec_get_name(id);
}
return codec;
static bool setupFFmpeg_l() {
av_log_set_level(AV_LOG_TRACE);
av_log_set_flags(AV_LOG_PRINT_LEVEL);
av_log_set_callback(on_ffmpeg_log);
avcodec_register_all();
return true;
}
template<bool decoder = true, typename First, typename ...ARGS>
const AVCodec *getCodec(First first, ARGS ...names) {
auto codec = getCodec<decoder>(names...);
if (codec) {
return codec;
static void setupFFmpeg() {
static auto flag = setupFFmpeg_l();
}
static bool checkIfSupportedNvidia_l() {
#if !defined(_WIN32)
GET_CONFIG(bool, check_nvidia_dev, General::kCheckNvidiaDev);
if (!check_nvidia_dev) {
return false;
}
return getCodec<decoder>(first);
auto so = dlopen("libnvcuvid.so.1", RTLD_LAZY);
if (!so) {
WarnL << "libnvcuvid.so.1加载失败:" << get_uv_errmsg();
return false;
}
dlclose(so);
bool find_driver = false;
File::scanDir("/dev", [&](const string &path, bool is_dir) {
if (!is_dir && start_with(path, "/dev/nvidia")) {
//找到nvidia的驱动
find_driver = true;
return false;
}
return true;
}, false);
if (!find_driver) {
WarnL << "英伟达硬件编解码器驱动文件 /dev/nvidia* 不存在";
}
return find_driver;
#else
return false;
#endif
}
static bool checkIfSupportedNvidia() {
static auto ret = checkIfSupportedNvidia_l();
return ret;
}
//////////////////////////////////////////////////////////////////////////////////////////
FFmpegFrame::FFmpegFrame(std::shared_ptr<AVFrame> frame) {
if (frame) {
_frame = std::move(frame);
} else {
_frame.reset(av_frame_alloc(), [](AVFrame *ptr) {
av_frame_free(&ptr);
});
}
}
FFmpegFrame::~FFmpegFrame() {
if (_data) {
delete[] _data;
_data = nullptr;
}
}
AVFrame *FFmpegFrame::get() const {
return _frame.get();
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////
FFmpegSwr::FFmpegSwr(AVSampleFormat output, int channel, int channel_layout, int samplerate) {
_target_format = output;
_target_channels = channel;
_target_channel_layout = channel_layout;
_target_samplerate = samplerate;
}
FFmpegSwr::~FFmpegSwr() {
if (_ctx) {
swr_free(&_ctx);
}
}
FFmpegFrame::Ptr FFmpegSwr::inputFrame(const FFmpegFrame::Ptr &frame) {
if (frame->get()->format == _target_format &&
frame->get()->channels == _target_channels &&
frame->get()->channel_layout == (uint64_t)_target_channel_layout &&
frame->get()->sample_rate == _target_samplerate) {
//不转格式
return frame;
}
if (!_ctx) {
_ctx = swr_alloc_set_opts(nullptr, _target_channel_layout, _target_format, _target_samplerate,
frame->get()->channel_layout, (AVSampleFormat) frame->get()->format,
frame->get()->sample_rate, 0, nullptr);
InfoL << "swr_alloc_set_opts:" << av_get_sample_fmt_name((enum AVSampleFormat) frame->get()->format) << " -> "
<< av_get_sample_fmt_name(_target_format);
}
if (_ctx) {
auto out = std::make_shared<FFmpegFrame>();
out->get()->format = _target_format;
out->get()->channel_layout = _target_channel_layout;
out->get()->channels = _target_channels;
out->get()->sample_rate = _target_samplerate;
out->get()->pkt_dts = frame->get()->pkt_dts;
out->get()->pts = frame->get()->pts;
int ret = 0;
if(0 != (ret = swr_convert_frame(_ctx, out->get(), frame->get()))){
WarnL << "swr_convert_frame failed:" << ffmpeg_err(ret);
return nullptr;
}
return out;
}
return nullptr;
}
///////////////////////////////////////////////////////////////////////////
FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track) {
#if (LIBAVCODEC_VERSION_MAJOR < 58)
avcodec_register_all();
#endif
const AVCodec *codec = nullptr;
const AVCodec *codec_default = nullptr;
switch (track->getCodecId()) {
case CodecH264:
codec_default = getCodec(AV_CODEC_ID_H264);
codec = getCodec("libopenh264", AV_CODEC_ID_H264, "h264_videotoolbox", "h264_cuvid");
break;
case CodecH265:
codec_default = getCodec(AV_CODEC_ID_HEVC);
codec = getCodec(AV_CODEC_ID_HEVC, "hevc_videotoolbox", "hevc_cuvid");
break;
case CodecAAC:
codec = getCodec(AV_CODEC_ID_AAC);
break;
case CodecG711A:
codec = getCodec(AV_CODEC_ID_PCM_ALAW);
break;
case CodecG711U:
codec = getCodec(AV_CODEC_ID_PCM_MULAW);
break;
case CodecOpus:
codec = getCodec(AV_CODEC_ID_OPUS);
break;
default: break;
}
if (!codec) {
throw std::runtime_error("未找到解码器");
}
while (true) {
_context.reset(avcodec_alloc_context3(codec), [](AVCodecContext *ctx) {
avcodec_close(ctx);
avcodec_free_context(&ctx);
});
if (!_context) {
throw std::runtime_error("创建解码器失败");
}
//保存AVFrame的引用
#ifdef FF_API_OLD_ENCDEC
_context->refcounted_frames = 1;
#endif
_context->flags |= AV_CODEC_FLAG_LOW_DELAY;
_context->flags2 |= AV_CODEC_FLAG2_FAST;
switch (track->getCodecId()) {
case CodecG711A:
case CodecG711U: {
AudioTrack::Ptr audio = static_pointer_cast<AudioTrack>(track);
_context->channels = audio->getAudioChannel();
_context->sample_rate = audio->getAudioSampleRate();
_context->channel_layout = av_get_default_channel_layout(_context->channels);
break;
}
default:
break;
}
AVDictionary *dict = nullptr;
av_dict_set(&dict, "threads", "auto", 0);
av_dict_set(&dict, "zerolatency", "1", 0);
av_dict_set(&dict, "strict", "-2", 0);
if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) {
/* we do not send complete frames */
_context->flags |= AV_CODEC_FLAG_TRUNCATED;
} else {
// 此时业务层应该需要合帧
_do_merger = true;
}
int ret = avcodec_open2(_context.get(), codec, &dict);
av_dict_free(&dict);
if (ret >= 0) {
//成功
InfoL << "打开解码器成功:" << codec->name;
break;
}
if (codec_default && codec_default != codec) {
//硬件编解码器打开失败，尝试软件的
WarnL << "打开解码器" << codec->name << "失败，原因是:" << ffmpeg_err(ret) << ", 再尝试打开解码器" << codec_default->name;
codec = codec_default;
continue;
}
throw std::runtime_error(StrPrinter << "打开解码器" << codec->name << "失败:" << ffmpeg_err(ret));
}
if (track->getTrackType() == TrackVideo) {
startThread("decoder thread");
}
}
FFmpegDecoder::~FFmpegDecoder() {
stopThread();
}
void FFmpegDecoder::flush() {
while (true) {
auto out_frame = std::make_shared<FFmpegFrame>();
auto ret = avcodec_receive_frame(_context.get(), out_frame->get());
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
if (ret < 0) {
WarnL << "avcodec_receive_frame failed:" << ffmpeg_err(ret);
break;
}
onDecode(out_frame);
}
}
const AVCodecContext *FFmpegDecoder::getContext() const {
return _context.get();
}
bool FFmpegDecoder::inputFrame_l(const Frame::Ptr &frame) {
if (_do_merger) {
return _merger.inputFrame(frame, [&](uint32_t dts, uint32_t pts, const Buffer::Ptr &buffer, bool have_idr) {
decodeFrame(buffer->data(), buffer->size(), dts, pts);
});
}
return decodeFrame(frame->data(), frame->size(), frame->dts(), frame->pts());
}
bool FFmpegDecoder::inputFrame(const Frame::Ptr &frame, bool may_async) {
if (!may_async || !TaskManager::isEnabled()) {
return inputFrame_l(frame);
}
auto frame_cache = Frame::getCacheAbleFrame(frame);
addDecodeTask(frame->keyFrame(), [this, frame_cache]() {
inputFrame_l(frame_cache);
//此处模拟解码太慢导致的主动丢帧
//usleep(100 * 1000);
});
return true;
}
bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint32_t dts, uint32_t pts) {
TimeTicker2(30, TraceL);
auto pkt = alloc_av_packet();
pkt->data = (uint8_t *) data;
pkt->size = size;
pkt->dts = dts;
pkt->pts = pts;
auto ret = avcodec_send_packet(_context.get(), pkt.get());
if (ret < 0) {
if (ret != AVERROR_INVALIDDATA) {
WarnL << "avcodec_send_packet failed:" << ffmpeg_err(ret);
}
return false;
}
while (true) {
auto out_frame = std::make_shared<FFmpegFrame>();
ret = avcodec_receive_frame(_context.get(), out_frame->get());
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
if (ret < 0) {
WarnL << "avcodec_receive_frame failed:" << ffmpeg_err(ret);
break;
}
if (pts - out_frame->get()->pts > MAX_DELAY_SECOND * 1000 && _ticker.createdTime() > 10 * 1000) {
//后面的帧才忽略,防止Track无法ready
WarnL << "解码时，忽略" << MAX_DELAY_SECOND << "秒前的数据:" << pts << " " << out_frame->get()->pts;
continue;
}
onDecode(out_frame);
}
return true;
}
void FFmpegDecoder::setOnDecode(FFmpegDecoder::onDec cb) {
_cb = std::move(cb);
}
void FFmpegDecoder::onDecode(const FFmpegFrame::Ptr &frame) {
if (_cb) {
_cb(frame);
}
}
////////////////////////////////////////////////////////////////////////
void TaskManager::pushExit(){
{
lock_guard<mutex> lck(_task_mtx);
_exit = true;
_task.clear();
_task.emplace_back([](){
throw ThreadExitException();
});
}
_sem.post(10);
}
void TaskManager::addEncodeTask(function<void()> task) {
bool TaskManager::addEncodeTask(function<void()> task) {
{
lock_guard<mutex> lck(_task_mtx);
_task.emplace_back(std::move(task));
if (_task.size() > 30) {
if (_task.size() > _max_task) {
WarnL << "encoder thread task is too more, now drop frame!";
_task.pop_front();
}
}
_sem.post();
return true;
}
void TaskManager::addDecodeTask(bool key_frame, function<void()> task) {
bool TaskManager::addDecodeTask(bool key_frame, function<void()> task) {
{
lock_guard<mutex> lck(_task_mtx);
if (_decode_drop_start) {
if (!key_frame) {
TraceL << "decode thread drop frame";
return;
return false;
}
_decode_drop_start = false;
InfoL << "decode thread stop drop frame";
}
_task.emplace_back(std::move(task));
if (_task.size() > 30) {
if (_task.size() > _max_task) {
_decode_drop_start = true;
WarnL << "decode thread start drop frame";
}
}
_sem.post();
return true;
}
void TaskManager::setMaxTaskSize(size_t size) {
CHECK(size >= 3 && size <= 1000, "async task size limited to 3 ~ 1000, now size is:", size);
_max_task = size;
}
void TaskManager::startThread(const string &name) {
_thread.reset(new thread([this, name]() {
onThreadRun(name);
}), [this](thread *ptr) {
pushExit();
}), [](thread *ptr) {
ptr->join();
delete ptr;
});
}
void TaskManager::stopThread() {
void TaskManager::stopThread(bool drop_task) {
TimeTicker();
if (!_thread) {
return;
}
{
lock_guard<mutex> lck(_task_mtx);
if (drop_task) {
_exit = true;
_task.clear();
}
_task.emplace_back([]() {
throw ThreadExitException();
});
}
_sem.post(10);
_thread = nullptr;
}
TaskManager::~TaskManager() {
stopThread();
stopThread(true);
}
bool TaskManager::isEnabled() const {
@ -423,9 +210,432 @@ void TaskManager::onThreadRun(const string &name) {
} catch (std::exception &ex) {
WarnL << ex.what();
continue;
} catch (...) {
WarnL << "catch one unknown exception";
throw;
}
}
InfoL << name << " exited!";
}
//////////////////////////////////////////////////////////////////////////////////////////
FFmpegFrame::FFmpegFrame(std::shared_ptr<AVFrame> frame) {
if (frame) {
_frame = std::move(frame);
} else {
_frame.reset(av_frame_alloc(), [](AVFrame *ptr) {
av_frame_free(&ptr);
});
}
}
FFmpegFrame::~FFmpegFrame() {
if (_data) {
delete[] _data;
_data = nullptr;
}
}
AVFrame *FFmpegFrame::get() const {
return _frame.get();
}
void FFmpegFrame::fillPicture(AVPixelFormat target_format, int target_width, int target_height) {
assert(_data == nullptr);
_data = new char[avpicture_get_size(target_format, target_width, target_height)];
avpicture_fill((AVPicture *) _frame.get(), (uint8_t *) _data, target_format, target_width, target_height);
}
///////////////////////////////////////////////////////////////////////////
template<bool decoder = true, typename ...ARGS>
AVCodec *getCodec(ARGS ...names);
template<bool decoder = true>
AVCodec *getCodec(const char *name) {
auto codec = decoder ? avcodec_find_decoder_by_name(name) : avcodec_find_encoder_by_name(name);
if (codec) {
InfoL << (decoder ? "got decoder:" : "got encoder:") << name;
} else {
TraceL << (decoder ? "decoder:" : "encoder:") << name << " not found";
}
return codec;
}
template<bool decoder = true>
AVCodec *getCodec(enum AVCodecID id) {
auto codec = decoder ? avcodec_find_decoder(id) : avcodec_find_encoder(id);
if (codec) {
InfoL << (decoder ? "got decoder:" : "got encoder:") << avcodec_get_name(id);
} else {
TraceL << (decoder ? "decoder:" : "encoder:") << avcodec_get_name(id) << " not found";
}
return codec;
}
template<bool decoder = true, typename First, typename ...ARGS>
AVCodec *getCodec(First first, ARGS ...names) {
auto codec = getCodec<decoder>(names...);
if (codec) {
return codec;
}
return getCodec<decoder>(first);
}
FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num) {
setupFFmpeg();
AVCodec *codec = nullptr;
AVCodec *codec_default = nullptr;
switch (track->getCodecId()) {
case CodecH264:
codec_default = getCodec(AV_CODEC_ID_H264);
if (checkIfSupportedNvidia()) {
codec = getCodec("libopenh264", AV_CODEC_ID_H264, "h264_qsv", "h264_videotoolbox", "h264_cuvid", "h264_nvmpi");
} else {
codec = getCodec("libopenh264", AV_CODEC_ID_H264, "h264_qsv", "h264_videotoolbox", "h264_nvmpi");
}
break;
case CodecH265:
codec_default = getCodec(AV_CODEC_ID_HEVC);
if (checkIfSupportedNvidia()) {
codec = getCodec(AV_CODEC_ID_HEVC, "hevc_qsv", "hevc_videotoolbox", "hevc_cuvid", "hevc_nvmpi");
} else {
codec = getCodec(AV_CODEC_ID_HEVC, "hevc_qsv", "hevc_videotoolbox", "hevc_nvmpi");
}
break;
case CodecAAC:
codec = getCodec(AV_CODEC_ID_AAC);
break;
case CodecG711A:
codec = getCodec(AV_CODEC_ID_PCM_ALAW);
break;
case CodecG711U:
codec = getCodec(AV_CODEC_ID_PCM_MULAW);
break;
case CodecOpus:
codec = getCodec(AV_CODEC_ID_OPUS);
break;
case CodecVP8:
codec = getCodec(AV_CODEC_ID_VP8);
break;
case CodecVP9:
codec = getCodec(AV_CODEC_ID_VP9);
break;
default:
break;
}
if (!codec) {
throw std::runtime_error("未找到解码器");
}
while (true) {
_context.reset(avcodec_alloc_context3(codec), [](AVCodecContext *ctx) {
avcodec_free_context(&ctx);
});
if (!_context) {
throw std::runtime_error("创建解码器失败");
}
//保存AVFrame的引用
_context->refcounted_frames = 1;
_context->flags |= AV_CODEC_FLAG_LOW_DELAY;
_context->flags2 |= AV_CODEC_FLAG2_FAST;
if (track->getTrackType() == TrackVideo) {
_context->width = static_pointer_cast<VideoTrack>(track)->getVideoWidth();
_context->height = static_pointer_cast<VideoTrack>(track)->getVideoHeight();
}
switch (track->getCodecId()) {
case CodecG711A:
case CodecG711U: {
AudioTrack::Ptr audio = static_pointer_cast<AudioTrack>(track);
_context->channels = audio->getAudioChannel();
_context->sample_rate = audio->getAudioSampleRate();
_context->channel_layout = av_get_default_channel_layout(_context->channels);
break;
}
default:
break;
}
AVDictionary *dict = nullptr;
if (thread_num <= 0) {
av_dict_set(&dict, "threads", "auto", 0);
} else {
av_dict_set(&dict, "threads", to_string(MIN(thread_num, thread::hardware_concurrency())).data(), 0);
}
av_dict_set(&dict, "zerolatency", "1", 0);
av_dict_set(&dict, "strict", "-2", 0);
if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) {
/* we do not send complete frames */
_context->flags |= AV_CODEC_FLAG_TRUNCATED;
} else {
// 此时业务层应该需要合帧
_do_merger = true;
}
int ret = avcodec_open2(_context.get(), codec, &dict);
av_dict_free(&dict);
if (ret >= 0) {
//成功
InfoL << "打开解码器成功:" << codec->name;
break;
}
if (codec_default && codec_default != codec) {
//硬件编解码器打开失败，尝试软件的
WarnL << "打开解码器" << codec->name << "失败，原因是:" << ffmpeg_err(ret) << ", 再尝试打开解码器" << codec_default->name;
codec = codec_default;
continue;
}
throw std::runtime_error(StrPrinter << "打开解码器" << codec->name << "失败:" << ffmpeg_err(ret));
}
}
FFmpegDecoder::~FFmpegDecoder() {
stopThread(true);
if (_do_merger) {
_merger.inputFrame(nullptr, [&](uint32_t dts, uint32_t pts, const Buffer::Ptr &buffer, bool have_idr) {
decodeFrame(buffer->data(), buffer->size(), dts, pts, false);
});
}
flush();
}
void FFmpegDecoder::flush() {
while (true) {
auto out_frame = std::make_shared<FFmpegFrame>();
auto ret = avcodec_receive_frame(_context.get(), out_frame->get());
if (ret == AVERROR(EAGAIN)) {
avcodec_send_packet(_context.get(), nullptr);
continue;
}
if (ret == AVERROR_EOF) {
break;
}
if (ret < 0) {
WarnL << "avcodec_receive_frame failed:" << ffmpeg_err(ret);
break;
}
onDecode(out_frame);
}
}
const AVCodecContext *FFmpegDecoder::getContext() const {
return _context.get();
}
bool FFmpegDecoder::inputFrame_l(const Frame::Ptr &frame, bool live, bool enable_merge) {
if (_do_merger && enable_merge) {
return _merger.inputFrame(frame, [&](uint32_t dts, uint32_t pts, const Buffer::Ptr &buffer, bool have_idr) {
decodeFrame(buffer->data(), buffer->size(), dts, pts, live);
});
}
return decodeFrame(frame->data(), frame->size(), frame->dts(), frame->pts(), live);
}
bool FFmpegDecoder::inputFrame(const Frame::Ptr &frame, bool live, bool async, bool enable_merge) {
if (async && !TaskManager::isEnabled() && getContext()->codec_type == AVMEDIA_TYPE_VIDEO) {
//开启异步编码，且为视频，尝试启动异步解码线程
startThread("decoder thread");
}
if (!async || !TaskManager::isEnabled()) {
return inputFrame_l(frame, live, enable_merge);
}
auto frame_cache = Frame::getCacheAbleFrame(frame);
return addDecodeTask(frame->keyFrame(), [this, live, frame_cache, enable_merge]() {
inputFrame_l(frame_cache, live, enable_merge);
//此处模拟解码太慢导致的主动丢帧
//usleep(100 * 1000);
});
}
bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint32_t dts, uint32_t pts, bool live) {
TimeTicker2(30, TraceL);
auto pkt = alloc_av_packet();
pkt->data = (uint8_t *) data;
pkt->size = size;
pkt->dts = dts;
pkt->pts = pts;
auto ret = avcodec_send_packet(_context.get(), pkt.get());
if (ret < 0) {
if (ret != AVERROR_INVALIDDATA) {
WarnL << "avcodec_send_packet failed:" << ffmpeg_err(ret);
}
return false;
}
while (true) {
auto out_frame = std::make_shared<FFmpegFrame>();
ret = avcodec_receive_frame(_context.get(), out_frame->get());
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
if (ret < 0) {
WarnL << "avcodec_receive_frame failed:" << ffmpeg_err(ret);
break;
}
if (live && pts - out_frame->get()->pts > MAX_DELAY_SECOND * 1000 && _ticker.createdTime() > 10 * 1000) {
//后面的帧才忽略,防止Track无法ready
WarnL << "解码时，忽略" << MAX_DELAY_SECOND << "秒前的数据:" << pts << " " << out_frame->get()->pts;
continue;
}
onDecode(out_frame);
}
return true;
}
void FFmpegDecoder::setOnDecode(FFmpegDecoder::onDec cb) {
_cb = std::move(cb);
}
void FFmpegDecoder::onDecode(const FFmpegFrame::Ptr &frame) {
if (_cb) {
_cb(frame);
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////
FFmpegSwr::FFmpegSwr(AVSampleFormat output, int channel, int channel_layout, int samplerate) {
_target_format = output;
_target_channels = channel;
_target_channel_layout = channel_layout;
_target_samplerate = samplerate;
}
FFmpegSwr::~FFmpegSwr() {
if (_ctx) {
swr_free(&_ctx);
}
}
FFmpegFrame::Ptr FFmpegSwr::inputFrame(const FFmpegFrame::Ptr &frame) {
if (frame->get()->format == _target_format &&
frame->get()->channels == _target_channels &&
frame->get()->channel_layout == _target_channel_layout &&
frame->get()->sample_rate == _target_samplerate) {
//不转格式
return frame;
}
if (!_ctx) {
_ctx = swr_alloc_set_opts(nullptr, _target_channel_layout, _target_format, _target_samplerate,
frame->get()->channel_layout, (AVSampleFormat) frame->get()->format,
frame->get()->sample_rate, 0, nullptr);
InfoL << "swr_alloc_set_opts:" << av_get_sample_fmt_name((enum AVSampleFormat) frame->get()->format) << " -> "
<< av_get_sample_fmt_name(_target_format);
}
if (_ctx) {
auto out = std::make_shared<FFmpegFrame>();
out->get()->format = _target_format;
out->get()->channel_layout = _target_channel_layout;
out->get()->channels = _target_channels;
out->get()->sample_rate = _target_samplerate;
out->get()->pkt_dts = frame->get()->pkt_dts;
out->get()->pts = frame->get()->pts;
int ret = 0;
if (0 != (ret = swr_convert_frame(_ctx, out->get(), frame->get()))) {
WarnL << "swr_convert_frame failed:" << ffmpeg_err(ret);
return nullptr;
}
return out;
}
return nullptr;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////
FFmpegSws::FFmpegSws(AVPixelFormat output, int width, int height) {
_target_format = output;
_target_width = width;
_target_height = height;
}
FFmpegSws::~FFmpegSws() {
if (_ctx) {
sws_freeContext(_ctx);
_ctx = nullptr;
}
}
int FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame, uint8_t *data) {
TimeTicker2(30, TraceL);
if (!_target_width) {
_target_width = frame->get()->width;
}
if (!_target_height) {
_target_height = frame->get()->height;
}
AVFrame dst;
memset(&dst, 0, sizeof(dst));
avpicture_fill((AVPicture *) &dst, data, _target_format, _target_width, _target_height);
if (!_ctx) {
_ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format,
_target_width, _target_height, _target_format, SWS_FAST_BILINEAR, NULL, NULL, NULL);
InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> "
<< av_get_pix_fmt_name(_target_format);
}
assert(_ctx);
int ret = 0;
if (0 >= (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height, dst.data,
dst.linesize))) {
WarnL << "sws_scale failed:" << ffmpeg_err(ret);
}
return ret;
}
FFmpegFrame::Ptr FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame) {
TimeTicker2(30, TraceL);
if (!_target_width) {
_target_width = frame->get()->width;
}
if (!_target_height) {
_target_height = frame->get()->height;
}
if (frame->get()->format == _target_format && frame->get()->width == _target_width
&& frame->get()->height == _target_height) {
//不转格式
return frame;
}
if (!_ctx) {
_ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format,
_target_width, _target_height, _target_format,
SWS_FAST_BILINEAR, NULL, NULL, NULL);
InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> "
<< av_get_pix_fmt_name(_target_format);
}
if (_ctx) {
auto out = std::make_shared<FFmpegFrame>();
if (!out->get()->data[0]) {
out->fillPicture(_target_format, _target_width, _target_height);
}
int ret = 0;
if (0 == (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height,
out->get()->data, out->get()->linesize))) {
WarnL << "sws_scale failed:" << ffmpeg_err(ret);
return nullptr;
}
out->get()->format = _target_format;
out->get()->width = _target_width;
out->get()->height = _target_height;
out->get()->pkt_dts = frame->get()->pkt_dts;
out->get()->pts = frame->get()->pts;
return out;
}
return nullptr;
}
} //namespace mediakit
#endif//ENABLE_FFMPEG

64
src/Codec/Transcode.h
View File

@ -8,23 +8,29 @@
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef FFMpegDecoder_H_
#define FFMpegDecoder_H_
#ifndef ZLMEDIAKIT_TRANSCODE_H
#define ZLMEDIAKIT_TRANSCODE_H
#if defined(ENABLE_FFMPEG)
#include "Util/TimeTicker.h"
#include "Common/MediaSink.h"
#if defined(ENABLE_FFMPEG)
#ifdef __cplusplus
extern "C" {
#endif
#include "libswscale/swscale.h"
#include "libavutil/avutil.h"
#include "libavutil/pixdesc.h"
#include "libavcodec/avcodec.h"
#include "libswresample/swresample.h"
#include "libavutil/audio_fifo.h"
#ifdef __cplusplus
}
#endif
namespace mediakit {
class FFmpegFrame {
public:
using Ptr = std::shared_ptr<FFmpegFrame>;
@ -33,6 +39,7 @@ public:
~FFmpegFrame();
AVFrame *get() const;
void fillPicture(AVPixelFormat target_format, int target_width, int target_height);
private:
char *_data = nullptr;
@ -45,7 +52,6 @@ public:
FFmpegSwr(AVSampleFormat output, int channel, int channel_layout, int samplerate);
~FFmpegSwr();
FFmpegFrame::Ptr inputFrame(const FFmpegFrame::Ptr &frame);
private:
@ -59,19 +65,19 @@ private:
class TaskManager {
public:
TaskManager() = default;
~TaskManager();
virtual ~TaskManager();
void setMaxTaskSize(size_t size);
void stopThread(bool drop_task);
protected:
void startThread(const std::string &name);
void stopThread();
void addEncodeTask(std::function<void()> task);
void addDecodeTask(bool key_frame, std::function<void()> task);
bool addEncodeTask(std::function<void()> task);
bool addDecodeTask(bool key_frame, std::function<void()> task);
bool isEnabled() const;
private:
void onThreadRun(const std::string &name);
void pushExit();
private:
class ThreadExitException : public std::runtime_error {
@ -83,39 +89,55 @@ private:
private:
bool _decode_drop_start = false;
bool _exit = false;
size_t _max_task = 30;
std::mutex _task_mtx;
toolkit::semaphore _sem;
toolkit::List<std::function<void()> > _task;
std::shared_ptr<std::thread> _thread;
};
class FFmpegDecoder : private TaskManager {
class FFmpegDecoder : public TaskManager {
public:
using Ptr = std::shared_ptr<FFmpegDecoder>;
using onDec = std::function<void(const FFmpegFrame::Ptr &)>;
FFmpegDecoder(const mediakit::Track::Ptr &track);
~FFmpegDecoder();
FFmpegDecoder(const Track::Ptr &track, int thread_num = 2);
~FFmpegDecoder() override;
bool inputFrame(const mediakit::Frame::Ptr &frame, bool may_async = true);
bool inputFrame(const Frame::Ptr &frame, bool live, bool async, bool enable_merge = true);
void setOnDecode(onDec cb);
void flush();
const AVCodecContext *getContext() const;
private:
void onDecode(const FFmpegFrame::Ptr &frame);
bool inputFrame_l(const mediakit::Frame::Ptr &frame);
bool decodeFrame(const char *data, size_t size, uint32_t dts, uint32_t pts);
bool inputFrame_l(const Frame::Ptr &frame, bool live, bool enable_merge);
bool decodeFrame(const char *data, size_t size, uint32_t dts, uint32_t pts, bool live);
private:
bool _do_merger = false;
toolkit::Ticker _ticker;
onDec _cb;
std::shared_ptr<AVCodecContext> _context;
mediakit::FrameMerger _merger { mediakit::FrameMerger::h264_prefix };
FrameMerger _merger{FrameMerger::h264_prefix};
};
class FFmpegSws {
public:
using Ptr = std::shared_ptr<FFmpegSws>;
FFmpegSws(AVPixelFormat output, int width, int height);
~FFmpegSws();
FFmpegFrame::Ptr inputFrame(const FFmpegFrame::Ptr &frame);
int inputFrame(const FFmpegFrame::Ptr &frame, uint8_t *data);
private:
int _target_width;
int _target_height;
SwsContext *_ctx = nullptr;
AVPixelFormat _target_format;
};
}//namespace mediakit
#endif// ENABLE_FFMPEG
#endif /* FFMpegDecoder_H_ */
#endif //ZLMEDIAKIT_TRANSCODE_H

12
src/Common/Device.cpp
View File

@ -28,22 +28,22 @@ using namespace std;
namespace mediakit {
bool DevChannel::inputYUV(char* apcYuv[3], int aiYuvLen[3], uint32_t uiStamp) {
bool DevChannel::inputYUV(char *yuv[3], int linesize[3], uint32_t cts) {
#ifdef ENABLE_X264
//TimeTicker1(50);
if (!_pH264Enc) {
_pH264Enc.reset(new H264Encoder());
if (!_pH264Enc->init(_video->iWidth, _video->iHeight, _video->iFrameRate)) {
if (!_pH264Enc->init(_video->iWidth, _video->iHeight, _video->iFrameRate, _video->iBitRate)) {
_pH264Enc.reset();
WarnL << "H264Encoder init failed!";
}
}
if (_pH264Enc) {
H264Encoder::H264Frame *pOut;
int iFrames = _pH264Enc->inputData(apcYuv, aiYuvLen, uiStamp, &pOut);
H264Encoder::H264Frame *out_frames;
int frames = _pH264Enc->inputData(yuv, linesize, cts, &out_frames);
bool ret = false;
for (int i = 0; i < iFrames; i++) {
ret = inputH264((char *) pOut[i].pucData, pOut[i].iLength, uiStamp) ? true : ret;
for (int i = 0; i < frames; i++) {
ret = inputH264((char *) out_frames[i].pucData, out_frames[i].iLength, cts) ? true : ret;
}
return ret;
}

17
src/Common/Device.h
View File

@ -29,6 +29,7 @@ public:
int iWidth;
int iHeight;
float iFrameRate;
int iBitRate = 2 * 1024 * 1024;
};
class AudioInfo {
@ -104,19 +105,19 @@ public:
/**
* 输入yuv420p视频帧，内部会完成编码并调用inputH264方法
* @param apcYuv
* @param aiYuvLen
* @param uiStamp
* @param yuv yuv420p数据指针
* @param linesize yuv420p数据linesize
* @param cts 采集时间戳，单位毫秒
*/
bool inputYUV(char *apcYuv[3], int aiYuvLen[3], uint32_t uiStamp);
bool inputYUV(char *yuv[3], int linesize[3], uint32_t cts);
/**
* 输入pcm数据，内部会完成编码并调用inputAAC方法
* @param pcData
* @param iDataLen
* @param uiStamp
* @param data pcm数据指针，int16整形
* @param len pcm数据长度
* @param cts 采集时间戳，单位毫秒
*/
bool inputPCM(char *pcData, int iDataLen, uint32_t uiStamp);
bool inputPCM(char *data, int len, uint32_t cts);
private:
MediaOriginType getOriginType(MediaSource &sender) const override;

6
src/Common/config.cpp
View File

@ -42,7 +42,7 @@ bool loadIniConfig(const char *ini_path){
namespace Broadcast {
const string kBroadcastMediaChanged = "kBroadcastMediaChanged";
const string kBroadcastRecordMP4 = "kBroadcastRecordMP4";
const string kBroadcastRecordTs = "kBroadcastRecoredTs";
const string kBroadcastRecordTs = "kBroadcastRecordTs";
const string kBroadcastHttpRequest = "kBroadcastHttpRequest";
const string kBroadcastHttpAccess = "kBroadcastHttpAccess";
const string kBroadcastOnGetRtspRealm = "kBroadcastOnGetRtspRealm";
@ -77,6 +77,8 @@ const string kRtmpDemand = GENERAL_FIELD"rtmp_demand";
const string kTSDemand = GENERAL_FIELD"ts_demand";
const string kFMP4Demand = GENERAL_FIELD"fmp4_demand";
const string kEnableAudio = GENERAL_FIELD"enable_audio";
const string kCheckNvidiaDev = GENERAL_FIELD"check_nvidia_dev";
const string kEnableFFmpegLog = GENERAL_FIELD"enable_ffmpeg_log";
const string kWaitTrackReadyMS = GENERAL_FIELD"wait_track_ready_ms";
const string kWaitAddTrackMS = GENERAL_FIELD"wait_add_track_ms";
const string kUnreadyFrameCache = GENERAL_FIELD"unready_frame_cache";
@ -100,6 +102,8 @@ static onceToken token([](){
mINI::Instance()[kTSDemand] = 0;
mINI::Instance()[kFMP4Demand] = 0;
mINI::Instance()[kEnableAudio] = 1;
mINI::Instance()[kCheckNvidiaDev] = 1;
mINI::Instance()[kEnableFFmpegLog] = 0;
mINI::Instance()[kWaitTrackReadyMS] = 10000;
mINI::Instance()[kWaitAddTrackMS] = 3000;
mINI::Instance()[kUnreadyFrameCache] = 100;

4
src/Common/config.h
View File

@ -183,6 +183,10 @@ extern const std::string kTSDemand;
extern const std::string kFMP4Demand;
//转协议是否全局开启或忽略音频
extern const std::string kEnableAudio;
//在docker环境下，不能通过英伟达驱动是否存在来判断是否支持硬件转码
extern const std::string kCheckNvidiaDev;
//是否开启ffmpeg日志
extern const std::string kEnableFFmpegLog;
//最多等待未初始化的Track 10秒，超时之后会忽略未初始化的Track
extern const std::string kWaitTrackReadyMS;
//如果直播流只有单Track，最多等待3秒，超时后未收到其他Track的数据，则认为是单Track