diff --git a/api/include/mk_transcode.h b/api/include/mk_transcode.h index 2fa71cae..0616e05b 100644 --- a/api/include/mk_transcode.h +++ b/api/include/mk_transcode.h @@ -13,6 +13,7 @@ #include "mk_common.h" #include "mk_track.h" +#include "mk_tcp.h" #ifdef __cplusplus extern "C" { @@ -39,6 +40,16 @@ typedef void(API_CALL *on_mk_decode)(void *user_data, mk_frame_pix frame); */ API_EXPORT mk_decoder API_CALL mk_decoder_create(mk_track track, int thread_num); +/** + * 创建解码器 + * @param track track对象 + * @param thread_num 解码线程数，0时为自动 + * @param codec_name_list 偏好的ffmpeg codec name列表，以NULL结尾，譬如：{"libopenh264", "h264_nvdec", NULL}; + * 在数组中越前，优先级越高;如果指定的codec不存在，或跟mk_track_codec_id类型不匹配时，则使用内部默认codec列表 + * @return 返回解码器对象，NULL代表失败 + */ +API_EXPORT mk_decoder API_CALL mk_decoder_create2(mk_track track, int thread_num, const char *codec_name_list[]); + /** * 销毁解码器 * @param ctx 解码器对象 @@ -96,6 +107,15 @@ API_EXPORT void API_CALL mk_frame_pix_unref(mk_frame_pix frame); */ API_EXPORT mk_frame_pix API_CALL mk_frame_pix_from_av_frame(AVFrame *frame); +/** + * 可无内存拷贝的创建mk_frame_pix对象 + * @param plane_data 多个平面数据, 通过mk_buffer_get_data获取其数据指针 + * @param line_size 平面数据line size + * @param plane 数据平面个数 + * @return mk_frame_pix对象 + */ +API_EXPORT mk_frame_pix API_CALL mk_frame_pix_from_buffer(mk_buffer plane_data[], int line_size[], int plane); + /** * 获取FFmpeg AVFrame对象 * @param frame 解码帧mk_frame_pix @@ -139,13 +159,26 @@ API_EXPORT mk_frame_pix mk_swscale_input_frame2(mk_swscale ctx, mk_frame_pix fra ///////////////////////////////////////////////////////////////////////////////////////////// -API_EXPORT uint8_t** API_CALL mk_get_av_frame_data(AVFrame *frame); -API_EXPORT int* API_CALL mk_get_av_frame_line_size(AVFrame *frame); +API_EXPORT uint8_t **API_CALL mk_get_av_frame_data(AVFrame *frame); +API_EXPORT void API_CALL mk_set_av_frame_data(AVFrame *frame, uint8_t *data, int plane); + +API_EXPORT int *API_CALL mk_get_av_frame_line_size(AVFrame *frame); +API_EXPORT void API_CALL mk_set_av_frame_line_size(AVFrame *frame, int line_size, int plane); + API_EXPORT int64_t API_CALL mk_get_av_frame_dts(AVFrame *frame); +API_EXPORT void API_CALL mk_set_av_frame_dts(AVFrame *frame, int64_t dts); + API_EXPORT int64_t API_CALL mk_get_av_frame_pts(AVFrame *frame); +API_EXPORT void API_CALL mk_set_av_frame_pts(AVFrame *frame, int64_t pts); + API_EXPORT int API_CALL mk_get_av_frame_width(AVFrame *frame); +API_EXPORT void API_CALL mk_set_av_frame_width(AVFrame *frame, int width); + API_EXPORT int API_CALL mk_get_av_frame_height(AVFrame *frame); +API_EXPORT void API_CALL mk_set_av_frame_height(AVFrame *frame, int height); + API_EXPORT int API_CALL mk_get_av_frame_format(AVFrame *frame); +API_EXPORT void API_CALL mk_set_av_frame_format(AVFrame *frame, int format); #ifdef __cplusplus } diff --git a/api/source/mk_transcode.cpp b/api/source/mk_transcode.cpp index c0030549..83981354 100644 --- a/api/source/mk_transcode.cpp +++ b/api/source/mk_transcode.cpp @@ -13,18 +13,33 @@ using namespace mediakit; +std::vector toCodecList(const char *codec_name_list[]) { + std::vector codec_list; + auto i = 0U; + while (codec_name_list[i]) { + codec_list.emplace_back(codec_name_list[i]); + ++i; + } + return codec_list; +} + #ifdef ENABLE_FFMPEG #include "Codec/Transcode.h" API_EXPORT mk_decoder API_CALL mk_decoder_create(mk_track track, int thread_num) { assert(track); - return new FFmpegDecoder(*((Track::Ptr *)track), thread_num); + return new FFmpegDecoder(*((Track::Ptr *) track), thread_num); +} + +API_EXPORT mk_decoder API_CALL mk_decoder_create2(mk_track track, int thread_num, const char *codec_name_list[]) { + assert(track && codec_name_list); + return new FFmpegDecoder(*((Track::Ptr *) track), thread_num, toCodecList(codec_name_list)); } API_EXPORT void API_CALL mk_decoder_release(mk_decoder ctx, int flush_frame) { assert(ctx); - auto decoder = (FFmpegDecoder *)ctx; + auto decoder = (FFmpegDecoder *) ctx; if (flush_frame) { decoder->stopThread(false); } @@ -33,94 +48,142 @@ API_EXPORT void API_CALL mk_decoder_release(mk_decoder ctx, int flush_frame) { API_EXPORT void API_CALL mk_decoder_decode(mk_decoder ctx, mk_frame frame, int async, int enable_merge) { assert(ctx && frame); - ((FFmpegDecoder *)ctx)->inputFrame(*((Frame::Ptr *)frame), false, async, enable_merge); + ((FFmpegDecoder *) ctx)->inputFrame(*((Frame::Ptr *) frame), false, async, enable_merge); } API_EXPORT void API_CALL mk_decoder_set_max_async_frame_size(mk_decoder ctx, size_t size) { assert(ctx && size); - ((FFmpegDecoder *)ctx)->setMaxTaskSize(size); + ((FFmpegDecoder *) ctx)->setMaxTaskSize(size); } API_EXPORT void API_CALL mk_decoder_set_cb(mk_decoder ctx, on_mk_decode cb, void *user_data) { assert(ctx && cb); - ((FFmpegDecoder *)ctx)->setOnDecode([cb, user_data](const FFmpegFrame::Ptr &pix_frame) { - cb(user_data, (mk_frame_pix)&pix_frame); + ((FFmpegDecoder *) ctx)->setOnDecode([cb, user_data](const FFmpegFrame::Ptr &pix_frame) { + cb(user_data, (mk_frame_pix) &pix_frame); }); } API_EXPORT const AVCodecContext *API_CALL mk_decoder_get_context(mk_decoder ctx) { assert(ctx); - return ((FFmpegDecoder *)ctx)->getContext(); + return ((FFmpegDecoder *) ctx)->getContext(); } ///////////////////////////////////////////////////////////////////////////////////////////// API_EXPORT mk_frame_pix API_CALL mk_frame_pix_ref(mk_frame_pix frame) { assert(frame); - return new FFmpegFrame::Ptr(*(FFmpegFrame::Ptr *)frame); + return new FFmpegFrame::Ptr(*(FFmpegFrame::Ptr *) frame); } API_EXPORT mk_frame_pix API_CALL mk_frame_pix_from_av_frame(AVFrame *frame) { assert(frame); - return new FFmpegFrame::Ptr(std::make_shared( - std::shared_ptr(av_frame_clone(frame), [](AVFrame *frame) { av_frame_free(&frame); }))); + return new FFmpegFrame::Ptr(std::make_shared(std::shared_ptr(av_frame_clone(frame), [](AVFrame *frame){ + av_frame_free(&frame); + }))); +} + +API_EXPORT mk_frame_pix API_CALL mk_frame_pix_from_buffer(mk_buffer plane_data[], int line_size[], int plane) { + assert(plane <= AV_NUM_DATA_POINTERS); + std::shared_ptr frame(av_frame_alloc(), [](AVFrame *ptr) { + av_frame_free(&ptr); + }); + std::vector buffer_array; + for (auto i = 0; i < plane; ++i) { + auto buffer = mk_buffer_ref(plane_data[i]); + frame->data[i] = (uint8_t *) mk_buffer_get_data(buffer); + frame->linesize[i] = line_size[i]; + buffer_array.emplace_back(buffer); + } + return new FFmpegFrame::Ptr(new FFmpegFrame(std::move(frame)), [buffer_array](FFmpegFrame *frame) { + for (auto &buffer : buffer_array) { + mk_buffer_unref(buffer); + } + delete frame; + }); } API_EXPORT void API_CALL mk_frame_pix_unref(mk_frame_pix frame) { assert(frame); - delete (FFmpegFrame::Ptr *)frame; + delete (FFmpegFrame::Ptr *) frame; } API_EXPORT AVFrame *API_CALL mk_frame_pix_get_av_frame(mk_frame_pix frame) { assert(frame); - return (*(FFmpegFrame::Ptr *)frame)->get(); + return (*(FFmpegFrame::Ptr *) frame)->get(); } -////////////////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////////////////////////// API_EXPORT mk_swscale mk_swscale_create(int output, int width, int height) { - return new FFmpegSws((AVPixelFormat)output, width, height); + return new FFmpegSws((AVPixelFormat) output, width, height); } API_EXPORT void mk_swscale_release(mk_swscale ctx) { - delete (FFmpegSws *)ctx; + delete (FFmpegSws *) ctx; } API_EXPORT int mk_swscale_input_frame(mk_swscale ctx, mk_frame_pix frame, uint8_t *data) { - return ((FFmpegSws *)ctx)->inputFrame(*(FFmpegFrame::Ptr *)frame, data); + return ((FFmpegSws *) ctx)->inputFrame(*(FFmpegFrame::Ptr *) frame, data); } -API_EXPORT mk_frame_pix mk_swscale_input_frame2(mk_swscale ctx, mk_frame_pix frame) { - return new FFmpegFrame::Ptr(((FFmpegSws *)ctx)->inputFrame(*(FFmpegFrame::Ptr *)frame)); +API_EXPORT mk_frame_pix mk_swscale_input_frame2(mk_swscale ctx, mk_frame_pix frame){ + return new FFmpegFrame::Ptr(((FFmpegSws *) ctx)->inputFrame(*(FFmpegFrame::Ptr *) frame)); } API_EXPORT uint8_t **API_CALL mk_get_av_frame_data(AVFrame *frame) { return frame->data; } +API_EXPORT void API_CALL mk_set_av_frame_data(AVFrame *frame, uint8_t *data, int plane) { + frame->data[plane] = data; +} + API_EXPORT int *API_CALL mk_get_av_frame_line_size(AVFrame *frame) { return frame->linesize; } -API_EXPORT int64_t API_CALL mk_get_av_frame_dts(AVFrame *frame) { +API_EXPORT void API_CALL mk_set_av_frame_line_size(AVFrame *frame, int line_size, int plane) { + frame->linesize[plane] = line_size; +} + +API_EXPORT int64_t API_CALL mk_get_av_frame_dts(AVFrame *frame) { return frame->pkt_dts; } -API_EXPORT int64_t API_CALL mk_get_av_frame_pts(AVFrame *frame) { +API_EXPORT void API_CALL mk_set_av_frame_dts(AVFrame *frame, int64_t dts) { + frame->pkt_dts = dts; +} + +API_EXPORT int64_t API_CALL mk_get_av_frame_pts(AVFrame *frame) { return frame->pts; } +API_EXPORT void API_CALL mk_set_av_frame_pts(AVFrame *frame, int64_t pts) { + frame->pts = pts; +} + API_EXPORT int API_CALL mk_get_av_frame_width(AVFrame *frame) { return frame->width; } +API_EXPORT void API_CALL mk_set_av_frame_width(AVFrame *frame, int width) { + frame->width = width; +} + API_EXPORT int API_CALL mk_get_av_frame_height(AVFrame *frame) { return frame->height; } +API_EXPORT void API_CALL mk_set_av_frame_height(AVFrame *frame, int height) { + frame->height = height; +} + API_EXPORT int API_CALL mk_get_av_frame_format(AVFrame *frame) { return frame->format; } +API_EXPORT void API_CALL mk_set_av_frame_format(AVFrame *frame, int format) { + frame->format = format; +} #endif //ENABLE_FFMPEG \ No newline at end of file diff --git a/src/Codec/Transcode.cpp b/src/Codec/Transcode.cpp index da57389a..eb92015f 100644 --- a/src/Codec/Transcode.cpp +++ b/src/Codec/Transcode.cpp @@ -47,13 +47,13 @@ static void on_ffmpeg_log(void *ctx, int level, const char *fmt, va_list args) { } LogLevel lev; switch (level) { - case AV_LOG_FATAL: + case AV_LOG_FATAL: lev = LError; break; case AV_LOG_ERROR: lev = LError; break; case AV_LOG_WARNING: lev = LWarn; break; case AV_LOG_INFO: lev = LInfo; break; - case AV_LOG_VERBOSE: + case AV_LOG_VERBOSE: lev = LDebug; break; case AV_LOG_DEBUG: lev = LDebug; break; - case AV_LOG_TRACE: + case AV_LOG_TRACE: lev = LTrace; break; default: lev = LTrace; break; } LoggerWrapper::printLogV(::toolkit::getLogger(), lev, __FILE__, ctx ? av_default_item_name(ctx) : "NULL", level, fmt, args); @@ -303,13 +303,31 @@ static inline const AVCodec *getCodec(const std::initializer_list &co return ret; } -FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num) { +template +static inline const AVCodec *getCodecByName(const std::vector &codec_list) { + const AVCodec *ret = nullptr; + for (auto &codec : codec_list) { + ret = getCodec_l(codec.data()); + if (ret) { + return ret; + } + } + return ret; +} + +FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num, const std::vector &codec_name) { setupFFmpeg(); const AVCodec *codec = nullptr; const AVCodec *codec_default = nullptr; + if (!codec_name.empty()) { + codec = getCodecByName(codec_name); + } switch (track->getCodecId()) { case CodecH264: codec_default = getCodec({AV_CODEC_ID_H264}); + if (codec && codec->id == AV_CODEC_ID_H264) { + break; + } if (checkIfSupportedNvidia()) { codec = getCodec({{"libopenh264"}, {AV_CODEC_ID_H264}, {"h264_qsv"}, {"h264_videotoolbox"}, {"h264_cuvid"}, {"h264_nvmpi"}}); } else { @@ -318,6 +336,9 @@ FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num) { break; case CodecH265: codec_default = getCodec({AV_CODEC_ID_HEVC}); + if (codec && codec->id == AV_CODEC_ID_HEVC) { + break; + } if (checkIfSupportedNvidia()) { codec = getCodec({{AV_CODEC_ID_HEVC}, {"hevc_qsv"}, {"hevc_videotoolbox"}, {"hevc_cuvid"}, {"hevc_nvmpi"}}); } else { @@ -325,27 +346,51 @@ FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num) { } break; case CodecAAC: + if (codec && codec->id == AV_CODEC_ID_AAC) { + break; + } codec = getCodec({AV_CODEC_ID_AAC}); break; case CodecG711A: + if (codec && codec->id == AV_CODEC_ID_PCM_ALAW) { + break; + } codec = getCodec({AV_CODEC_ID_PCM_ALAW}); break; case CodecG711U: + if (codec && codec->id == AV_CODEC_ID_PCM_MULAW) { + break; + } codec = getCodec({AV_CODEC_ID_PCM_MULAW}); break; case CodecOpus: + if (codec && codec->id == AV_CODEC_ID_OPUS) { + break; + } codec = getCodec({AV_CODEC_ID_OPUS}); break; + case CodecJPEG: + if (codec && codec->id == AV_CODEC_ID_MJPEG) { + break; + } + codec = getCodec({AV_CODEC_ID_MJPEG}); + break; case CodecVP8: + if (codec && codec->id == AV_CODEC_ID_VP8) { + break; + } codec = getCodec({AV_CODEC_ID_VP8}); break; case CodecVP9: + if (codec && codec->id == AV_CODEC_ID_VP9) { + break; + } codec = getCodec({AV_CODEC_ID_VP9}); break; - default: - break; + default: codec = nullptr; break; } + codec = codec ? codec : codec_default; if (!codec) { throw std::runtime_error("未找到解码器"); } @@ -451,11 +496,11 @@ const AVCodecContext *FFmpegDecoder::getContext() const { bool FFmpegDecoder::inputFrame_l(const Frame::Ptr &frame, bool live, bool enable_merge) { if (_do_merger && enable_merge) { return _merger.inputFrame(frame, [this, live](uint64_t dts, uint64_t pts, const Buffer::Ptr &buffer, bool have_idr) { - decodeFrame(buffer->data(), buffer->size(), dts, pts, live); + decodeFrame(buffer->data(), buffer->size(), dts, pts, live, have_idr); }); } - return decodeFrame(frame->data(), frame->size(), frame->dts(), frame->pts(), live); + return decodeFrame(frame->data(), frame->size(), frame->dts(), frame->pts(), live, frame->keyFrame()); } bool FFmpegDecoder::inputFrame(const Frame::Ptr &frame, bool live, bool async, bool enable_merge) { @@ -476,7 +521,7 @@ bool FFmpegDecoder::inputFrame(const Frame::Ptr &frame, bool live, bool async, b }); } -bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint64_t dts, uint64_t pts, bool live) { +bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint64_t dts, uint64_t pts, bool live, bool key_frame) { TimeTicker2(30, TraceL); auto pkt = alloc_av_packet(); @@ -484,6 +529,9 @@ bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint64_t dts, uin pkt->size = size; pkt->dts = dts; pkt->pts = pts; + if (key_frame) { + pkt->flags |= AV_PKT_FLAG_KEY; + } auto ret = avcodec_send_packet(_context.get(), pkt.get()); if (ret < 0) { @@ -589,68 +637,54 @@ FFmpegSws::~FFmpegSws() { } int FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame, uint8_t *data) { - TimeTicker2(30, TraceL); - if (!_target_width) { - _target_width = frame->get()->width; - } - if (!_target_height) { - _target_height = frame->get()->height; - } - AVFrame dst; - memset(&dst, 0, sizeof(dst)); - av_image_fill_arrays(dst.data, dst.linesize, data, _target_format, _target_width, _target_height,1); - - if (!_ctx) { - _ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format, - _target_width, _target_height, _target_format, SWS_FAST_BILINEAR, NULL, NULL, NULL); - InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> " - << av_get_pix_fmt_name(_target_format); - } - assert(_ctx); - int ret = 0; - if (0 >= (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height, dst.data, - dst.linesize))) { - WarnL << "sws_scale failed:" << ffmpeg_err(ret); - } + int ret; + inputFrame(frame, ret, data); return ret; } FFmpegFrame::Ptr FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame) { - TimeTicker2(30, TraceL); + int ret; + return inputFrame(frame, ret, nullptr); +} - if (!_target_width) { - _target_width = frame->get()->width; - } - if (!_target_height) { - _target_height = frame->get()->height; - } - if (frame->get()->format == _target_format && frame->get()->width == _target_width - && frame->get()->height == _target_height) { +FFmpegFrame::Ptr FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame, int &ret, uint8_t *data) { + ret = -1; + TimeTicker2(30, TraceL); + auto target_width = _target_width ? _target_width : frame->get()->width; + auto target_height = _target_height ? _target_height : frame->get()->height; + if (frame->get()->format == _target_format && frame->get()->width == target_width && frame->get()->height == target_height) { //不转格式 return frame; } + if (_ctx && (_src_width != frame->get()->width || _src_height != frame->get()->height || _src_format != (enum AVPixelFormat) frame->get()->format)) { + //输入分辨率发生变化了 + sws_freeContext(_ctx); + _ctx = nullptr; + } if (!_ctx) { - _ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format, - _target_width, _target_height, _target_format, - SWS_FAST_BILINEAR, NULL, NULL, NULL); - InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> " - << av_get_pix_fmt_name(_target_format); + _src_format = (enum AVPixelFormat) frame->get()->format; + _src_width = frame->get()->width; + _src_height = frame->get()->height; + _ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format, target_width, target_height, _target_format, SWS_FAST_BILINEAR, NULL, NULL, NULL); + InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> " << av_get_pix_fmt_name(_target_format); } if (_ctx) { auto out = std::make_shared(); if (!out->get()->data[0]) { - out->fillPicture(_target_format, _target_width, _target_height); + if (data) { + avpicture_fill((AVPicture *) out->get(), data, _target_format, target_width, target_height); + } else { + out->fillPicture(_target_format, target_width, target_height); + } } - int ret = 0; - if (0 == (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height, - out->get()->data, out->get()->linesize))) { + if (0 >= (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height, out->get()->data, out->get()->linesize))) { WarnL << "sws_scale failed:" << ffmpeg_err(ret); return nullptr; } out->get()->format = _target_format; - out->get()->width = _target_width; - out->get()->height = _target_height; + out->get()->width = target_width; + out->get()->height = target_height; out->get()->pkt_dts = frame->get()->pkt_dts; out->get()->pts = frame->get()->pts; return out; diff --git a/src/Codec/Transcode.h b/src/Codec/Transcode.h index fc4c37d0..8c33b711 100644 --- a/src/Codec/Transcode.h +++ b/src/Codec/Transcode.h @@ -102,7 +102,7 @@ public: using Ptr = std::shared_ptr; using onDec = std::function; - FFmpegDecoder(const Track::Ptr &track, int thread_num = 2); + FFmpegDecoder(const Track::Ptr &track, int thread_num = 2, const std::vector &codec_name = {}); ~FFmpegDecoder() override; bool inputFrame(const Frame::Ptr &frame, bool live, bool async, bool enable_merge = true); @@ -113,7 +113,7 @@ public: private: void onDecode(const FFmpegFrame::Ptr &frame); bool inputFrame_l(const Frame::Ptr &frame, bool live, bool enable_merge); - bool decodeFrame(const char *data, size_t size, uint64_t dts, uint64_t pts, bool live); + bool decodeFrame(const char *data, size_t size, uint64_t dts, uint64_t pts, bool live, bool key_frame); private: bool _do_merger = false; @@ -133,10 +133,16 @@ public: int inputFrame(const FFmpegFrame::Ptr &frame, uint8_t *data); private: - int _target_width; - int _target_height; + FFmpegFrame::Ptr inputFrame(const FFmpegFrame::Ptr &frame, int &ret, uint8_t *data); + +private: + int _target_width = 0; + int _target_height = 0; + int _src_width = 0; + int _src_height = 0; SwsContext *_ctx = nullptr; - AVPixelFormat _target_format; + AVPixelFormat _src_format = AV_PIX_FMT_NONE; + AVPixelFormat _target_format = AV_PIX_FMT_NONE; }; }//namespace mediakit