完善mk_transcode相关接口

api/include/mk_transcode.h

@@ -13,6 +13,7 @@
 
 #include "mk_common.h"
 #include "mk_track.h"
+#include "mk_tcp.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -39,6 +40,16 @@ typedef void(API_CALL *on_mk_decode)(void *user_data, mk_frame_pix frame);
  */
 API_EXPORT mk_decoder API_CALL mk_decoder_create(mk_track track, int thread_num);
 
+/**
+ * 创建解码器
+ * @param track track对象
+ * @param thread_num 解码线程数，0时为自动
+ * @param codec_name_list 偏好的ffmpeg codec name列表，以NULL结尾，譬如：{"libopenh264", "h264_nvdec", NULL};
+ *                        在数组中越前，优先级越高;如果指定的codec不存在，或跟mk_track_codec_id类型不匹配时，则使用内部默认codec列表
+ * @return 返回解码器对象，NULL代表失败
+ */
+API_EXPORT mk_decoder API_CALL mk_decoder_create2(mk_track track, int thread_num, const char *codec_name_list[]);
+
 /**
  * 销毁解码器
  * @param ctx 解码器对象
@@ -96,6 +107,15 @@ API_EXPORT void API_CALL mk_frame_pix_unref(mk_frame_pix frame);
  */
 API_EXPORT mk_frame_pix API_CALL mk_frame_pix_from_av_frame(AVFrame *frame);
 
+/**
+ * 可无内存拷贝的创建mk_frame_pix对象
+ * @param plane_data 多个平面数据, 通过mk_buffer_get_data获取其数据指针
+ * @param line_size 平面数据line size
+ * @param plane 数据平面个数
+ * @return mk_frame_pix对象
+ */
+API_EXPORT mk_frame_pix API_CALL mk_frame_pix_from_buffer(mk_buffer plane_data[], int line_size[], int plane);
+
 /**
  * 获取FFmpeg AVFrame对象
  * @param frame 解码帧mk_frame_pix
@@ -140,12 +160,25 @@ API_EXPORT mk_frame_pix mk_swscale_input_frame2(mk_swscale ctx, mk_frame_pix fra
 /////////////////////////////////////////////////////////////////////////////////////////////
 
 API_EXPORT uint8_t **API_CALL mk_get_av_frame_data(AVFrame *frame);
+API_EXPORT void API_CALL mk_set_av_frame_data(AVFrame *frame, uint8_t *data, int plane);
+
 API_EXPORT int *API_CALL mk_get_av_frame_line_size(AVFrame *frame);
+API_EXPORT void API_CALL mk_set_av_frame_line_size(AVFrame *frame, int line_size, int plane);
+
 API_EXPORT int64_t API_CALL mk_get_av_frame_dts(AVFrame *frame);
+API_EXPORT void API_CALL mk_set_av_frame_dts(AVFrame *frame, int64_t dts);
+
 API_EXPORT int64_t API_CALL mk_get_av_frame_pts(AVFrame *frame);
+API_EXPORT void API_CALL mk_set_av_frame_pts(AVFrame *frame, int64_t pts);
+
 API_EXPORT int API_CALL mk_get_av_frame_width(AVFrame *frame);
+API_EXPORT void API_CALL mk_set_av_frame_width(AVFrame *frame, int width);
+
 API_EXPORT int API_CALL mk_get_av_frame_height(AVFrame *frame);
+API_EXPORT void API_CALL mk_set_av_frame_height(AVFrame *frame, int height);
+
 API_EXPORT int API_CALL mk_get_av_frame_format(AVFrame *frame);
+API_EXPORT void API_CALL mk_set_av_frame_format(AVFrame *frame, int format);
 
 #ifdef __cplusplus
 }

api/source/mk_transcode.cpp

@@ -13,6 +13,16 @@
 
 using namespace mediakit;
 
+std::vector<std::string> toCodecList(const char *codec_name_list[]) {
+    std::vector<std::string> codec_list;
+    auto i = 0U;
+    while (codec_name_list[i]) {
+        codec_list.emplace_back(codec_name_list[i]);
+        ++i;
+    }
+    return codec_list;
+}
+
 #ifdef ENABLE_FFMPEG
 
 #include "Codec/Transcode.h"
@@ -22,6 +32,11 @@ API_EXPORT mk_decoder API_CALL mk_decoder_create(mk_track track, int thread_num)
     return new FFmpegDecoder(*((Track::Ptr *) track), thread_num);
 }
 
+API_EXPORT mk_decoder API_CALL mk_decoder_create2(mk_track track, int thread_num, const char *codec_name_list[]) {
+    assert(track && codec_name_list);
+    return new FFmpegDecoder(*((Track::Ptr *) track), thread_num, toCodecList(codec_name_list));
+}
+
 API_EXPORT void API_CALL mk_decoder_release(mk_decoder ctx, int flush_frame) {
     assert(ctx);
     auto decoder = (FFmpegDecoder *) ctx;
@@ -62,8 +77,29 @@ API_EXPORT mk_frame_pix API_CALL mk_frame_pix_ref(mk_frame_pix frame) {
 
 API_EXPORT mk_frame_pix API_CALL mk_frame_pix_from_av_frame(AVFrame *frame) {
     assert(frame);
-    return new FFmpegFrame::Ptr(std::make_shared<FFmpegFrame>(
+    return new FFmpegFrame::Ptr(std::make_shared<FFmpegFrame>(std::shared_ptr<AVFrame>(av_frame_clone(frame), [](AVFrame *frame){
-        std::shared_ptr<AVFrame>(av_frame_clone(frame), [](AVFrame *frame) { av_frame_free(&frame); })));
+        av_frame_free(&frame);
+    })));
+}
+
+API_EXPORT mk_frame_pix API_CALL mk_frame_pix_from_buffer(mk_buffer plane_data[], int line_size[], int plane) {
+    assert(plane <= AV_NUM_DATA_POINTERS);
+    std::shared_ptr<AVFrame> frame(av_frame_alloc(), [](AVFrame *ptr) {
+        av_frame_free(&ptr);
+    });
+    std::vector<mk_buffer> buffer_array;
+    for (auto i = 0; i < plane; ++i) {
+        auto buffer = mk_buffer_ref(plane_data[i]);
+        frame->data[i] = (uint8_t *) mk_buffer_get_data(buffer);
+        frame->linesize[i] = line_size[i];
+        buffer_array.emplace_back(buffer);
+    }
+    return new FFmpegFrame::Ptr(new FFmpegFrame(std::move(frame)), [buffer_array](FFmpegFrame *frame) {
+        for (auto &buffer : buffer_array) {
+            mk_buffer_unref(buffer);
+        }
+        delete frame;
+    });
 }
 
 API_EXPORT void API_CALL mk_frame_pix_unref(mk_frame_pix frame) {
@@ -76,7 +112,7 @@ API_EXPORT AVFrame *API_CALL mk_frame_pix_get_av_frame(mk_frame_pix frame) {
     return (*(FFmpegFrame::Ptr *) frame)->get();
 }
 
-//////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////////
 
 API_EXPORT mk_swscale mk_swscale_create(int output, int width, int height) {
     return new FFmpegSws((AVPixelFormat) output, width, height);
@@ -98,29 +134,56 @@ API_EXPORT uint8_t **API_CALL mk_get_av_frame_data(AVFrame *frame) {
     return frame->data;
 }
 
+API_EXPORT void API_CALL mk_set_av_frame_data(AVFrame *frame, uint8_t *data, int plane) {
+    frame->data[plane] = data;
+}
+
 API_EXPORT int *API_CALL mk_get_av_frame_line_size(AVFrame *frame) {
     return frame->linesize;
 }
 
+API_EXPORT void API_CALL mk_set_av_frame_line_size(AVFrame *frame, int line_size, int plane) {
+    frame->linesize[plane] = line_size;
+}
+
 API_EXPORT int64_t  API_CALL mk_get_av_frame_dts(AVFrame *frame) {
     return frame->pkt_dts;
 }
 
+API_EXPORT void API_CALL mk_set_av_frame_dts(AVFrame *frame, int64_t dts) {
+    frame->pkt_dts = dts;
+}
+
 API_EXPORT int64_t  API_CALL mk_get_av_frame_pts(AVFrame *frame) {
     return frame->pts;
 }
 
+API_EXPORT void API_CALL mk_set_av_frame_pts(AVFrame *frame, int64_t pts) {
+    frame->pts = pts;
+}
+
 API_EXPORT int API_CALL mk_get_av_frame_width(AVFrame *frame) {
     return frame->width;
 }
 
+API_EXPORT void API_CALL mk_set_av_frame_width(AVFrame *frame, int width) {
+    frame->width = width;
+}
+
 API_EXPORT int API_CALL mk_get_av_frame_height(AVFrame *frame) {
     return frame->height;
 }
 
+API_EXPORT void API_CALL mk_set_av_frame_height(AVFrame *frame, int height) {
+    frame->height = height;
+}
+
 API_EXPORT int API_CALL mk_get_av_frame_format(AVFrame *frame) {
     return frame->format;
 }
 
+API_EXPORT void API_CALL mk_set_av_frame_format(AVFrame *frame, int format) {
+    frame->format = format;
+}
 
 #endif //ENABLE_FFMPEG

src/Codec/Transcode.cpp

@@ -47,13 +47,13 @@ static void on_ffmpeg_log(void *ctx, int level, const char *fmt, va_list args) {
     }
     LogLevel lev;
     switch (level) {
-        case AV_LOG_FATAL:
+        case AV_LOG_FATAL: lev = LError; break;
         case AV_LOG_ERROR: lev = LError; break;
         case AV_LOG_WARNING: lev = LWarn; break;
         case AV_LOG_INFO: lev = LInfo; break;
-        case AV_LOG_VERBOSE:
+        case AV_LOG_VERBOSE: lev = LDebug; break;
         case AV_LOG_DEBUG: lev = LDebug; break;
-        case AV_LOG_TRACE:
+        case AV_LOG_TRACE: lev = LTrace; break;
         default: lev = LTrace; break;
     }
     LoggerWrapper::printLogV(::toolkit::getLogger(), lev, __FILE__, ctx ? av_default_item_name(ctx) : "NULL", level, fmt, args);
@@ -303,13 +303,31 @@ static inline const AVCodec *getCodec(const std::initializer_list<CodecName> &co
     return ret;
 }
 
-FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num) {
+template<bool decoder = true>
+static inline const AVCodec *getCodecByName(const std::vector<std::string> &codec_list) {
+    const AVCodec *ret = nullptr;
+    for (auto &codec : codec_list) {
+        ret = getCodec_l<decoder>(codec.data());
+        if (ret) {
+            return ret;
+        }
+    }
+    return ret;
+}
+
+FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num, const std::vector<std::string> &codec_name) {
     setupFFmpeg();
     const AVCodec *codec = nullptr;
     const AVCodec *codec_default = nullptr;
+    if (!codec_name.empty()) {
+        codec = getCodecByName(codec_name);
+    }
     switch (track->getCodecId()) {
         case CodecH264:
             codec_default = getCodec({AV_CODEC_ID_H264});
+            if (codec && codec->id == AV_CODEC_ID_H264) {
+                break;
+            }
             if (checkIfSupportedNvidia()) {
                 codec = getCodec({{"libopenh264"}, {AV_CODEC_ID_H264}, {"h264_qsv"}, {"h264_videotoolbox"}, {"h264_cuvid"}, {"h264_nvmpi"}});
             } else {
@@ -318,6 +336,9 @@ FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num) {
             break;
         case CodecH265:
             codec_default = getCodec({AV_CODEC_ID_HEVC});
+            if (codec && codec->id == AV_CODEC_ID_HEVC) {
+                break;
+            }
             if (checkIfSupportedNvidia()) {
                 codec = getCodec({{AV_CODEC_ID_HEVC}, {"hevc_qsv"}, {"hevc_videotoolbox"}, {"hevc_cuvid"}, {"hevc_nvmpi"}});
             } else {
@@ -325,27 +346,51 @@ FFmpegDecoder::FFmpegDecoder(const Track::Ptr &track, int thread_num) {
             }
             break;
         case CodecAAC:
+            if (codec && codec->id == AV_CODEC_ID_AAC) {
+                break;
+            }
             codec = getCodec({AV_CODEC_ID_AAC});
             break;
         case CodecG711A:
+            if (codec && codec->id == AV_CODEC_ID_PCM_ALAW) {
+                break;
+            }
             codec = getCodec({AV_CODEC_ID_PCM_ALAW});
             break;
         case CodecG711U:
+            if (codec && codec->id == AV_CODEC_ID_PCM_MULAW) {
+                break;
+            }
             codec = getCodec({AV_CODEC_ID_PCM_MULAW});
             break;
         case CodecOpus:
+            if (codec && codec->id == AV_CODEC_ID_OPUS) {
+                break;
+            }
             codec = getCodec({AV_CODEC_ID_OPUS});
             break;
+        case CodecJPEG:
+            if (codec && codec->id == AV_CODEC_ID_MJPEG) {
+                break;
+            }
+            codec = getCodec({AV_CODEC_ID_MJPEG});
+            break;
         case CodecVP8:
+            if (codec && codec->id == AV_CODEC_ID_VP8) {
+                break;
+            }
             codec = getCodec({AV_CODEC_ID_VP8});
             break;
         case CodecVP9:
-            codec = getCodec({AV_CODEC_ID_VP9});
+            if (codec && codec->id == AV_CODEC_ID_VP9) {
-            break;
-        default:
                 break;
             }
+            codec = getCodec({AV_CODEC_ID_VP9});
+            break;
+        default: codec = nullptr; break;
+    }
 
+    codec = codec ? codec : codec_default;
     if (!codec) {
         throw std::runtime_error("未找到解码器");
     }
@@ -451,11 +496,11 @@ const AVCodecContext *FFmpegDecoder::getContext() const {
 bool FFmpegDecoder::inputFrame_l(const Frame::Ptr &frame, bool live, bool enable_merge) {
     if (_do_merger && enable_merge) {
         return _merger.inputFrame(frame, [this, live](uint64_t dts, uint64_t pts, const Buffer::Ptr &buffer, bool have_idr) {
-            decodeFrame(buffer->data(), buffer->size(), dts, pts, live);
+            decodeFrame(buffer->data(), buffer->size(), dts, pts, live, have_idr);
         });
     }
 
-    return decodeFrame(frame->data(), frame->size(), frame->dts(), frame->pts(), live);
+    return decodeFrame(frame->data(), frame->size(), frame->dts(), frame->pts(), live, frame->keyFrame());
 }
 
 bool FFmpegDecoder::inputFrame(const Frame::Ptr &frame, bool live, bool async, bool enable_merge) {
@@ -476,7 +521,7 @@ bool FFmpegDecoder::inputFrame(const Frame::Ptr &frame, bool live, bool async, b
     });
 }
 
-bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint64_t dts, uint64_t pts, bool live) {
+bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint64_t dts, uint64_t pts, bool live, bool key_frame) {
     TimeTicker2(30, TraceL);
 
     auto pkt = alloc_av_packet();
@@ -484,6 +529,9 @@ bool FFmpegDecoder::decodeFrame(const char *data, size_t size, uint64_t dts, uin
     pkt->size = size;
     pkt->dts = dts;
     pkt->pts = pts;
+    if (key_frame) {
+        pkt->flags |= AV_PKT_FLAG_KEY;
+    }
 
     auto ret = avcodec_send_packet(_context.get(), pkt.get());
     if (ret < 0) {
@@ -589,68 +637,54 @@ FFmpegSws::~FFmpegSws() {
 }
 
 int FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame, uint8_t *data) {
-    TimeTicker2(30, TraceL);
+    int ret;
-    if (!_target_width) {
+    inputFrame(frame, ret, data);
-        _target_width = frame->get()->width;
-    }
-    if (!_target_height) {
-        _target_height = frame->get()->height;
-    }
-    AVFrame dst;
-    memset(&dst, 0, sizeof(dst));
-    av_image_fill_arrays(dst.data, dst.linesize, data,  _target_format, _target_width, _target_height,1);
-
-    if (!_ctx) {
-        _ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format,
-                              _target_width, _target_height, _target_format, SWS_FAST_BILINEAR, NULL, NULL, NULL);
-        InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> "
-              << av_get_pix_fmt_name(_target_format);
-    }
-    assert(_ctx);
-    int ret = 0;
-    if (0 >= (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height, dst.data,
-                              dst.linesize))) {
-        WarnL << "sws_scale failed:" << ffmpeg_err(ret);
-    }
     return ret;
 }
 
 FFmpegFrame::Ptr FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame) {
-    TimeTicker2(30, TraceL);
+    int ret;
+    return inputFrame(frame, ret, nullptr);
+}
 
-    if (!_target_width) {
+FFmpegFrame::Ptr FFmpegSws::inputFrame(const FFmpegFrame::Ptr &frame, int &ret, uint8_t *data) {
-        _target_width = frame->get()->width;
+    ret = -1;
-    }
+    TimeTicker2(30, TraceL);
-    if (!_target_height) {
+    auto target_width = _target_width ? _target_width : frame->get()->width;
-        _target_height = frame->get()->height;
+    auto target_height = _target_height ? _target_height : frame->get()->height;
-    }
+    if (frame->get()->format == _target_format && frame->get()->width == target_width && frame->get()->height == target_height) {
-    if (frame->get()->format == _target_format && frame->get()->width == _target_width
-        && frame->get()->height == _target_height) {
         //不转格式
         return frame;
     }
+    if (_ctx && (_src_width != frame->get()->width || _src_height != frame->get()->height || _src_format != (enum AVPixelFormat) frame->get()->format)) {
+        //输入分辨率发生变化了
+        sws_freeContext(_ctx);
+        _ctx = nullptr;
+    }
     if (!_ctx) {
-        _ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format,
+        _src_format = (enum AVPixelFormat) frame->get()->format;
-                              _target_width, _target_height, _target_format,
+        _src_width = frame->get()->width;
-                              SWS_FAST_BILINEAR, NULL, NULL, NULL);
+        _src_height = frame->get()->height;
-        InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> "
+        _ctx = sws_getContext(frame->get()->width, frame->get()->height, (enum AVPixelFormat) frame->get()->format, target_width, target_height, _target_format, SWS_FAST_BILINEAR, NULL, NULL, NULL);
-              << av_get_pix_fmt_name(_target_format);
+        InfoL << "sws_getContext:" << av_get_pix_fmt_name((enum AVPixelFormat) frame->get()->format) << " -> " << av_get_pix_fmt_name(_target_format);
     }
     if (_ctx) {
         auto out = std::make_shared<FFmpegFrame>();
         if (!out->get()->data[0]) {
-            out->fillPicture(_target_format, _target_width, _target_height);
+            if (data) {
+                avpicture_fill((AVPicture *) out->get(), data, _target_format, target_width, target_height);
+            } else {
+                out->fillPicture(_target_format, target_width, target_height);
             }
-        int ret = 0;
+        }
-        if (0 == (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height,
+        if (0 >= (ret = sws_scale(_ctx, frame->get()->data, frame->get()->linesize, 0, frame->get()->height, out->get()->data, out->get()->linesize))) {
-                                  out->get()->data, out->get()->linesize))) {
             WarnL << "sws_scale failed:" << ffmpeg_err(ret);
             return nullptr;
         }
 
         out->get()->format = _target_format;
-        out->get()->width = _target_width;
+        out->get()->width = target_width;
-        out->get()->height = _target_height;
+        out->get()->height = target_height;
         out->get()->pkt_dts = frame->get()->pkt_dts;
         out->get()->pts = frame->get()->pts;
         return out;

src/Codec/Transcode.h

@@ -102,7 +102,7 @@ public:
     using Ptr = std::shared_ptr<FFmpegDecoder>;
     using onDec = std::function<void(const FFmpegFrame::Ptr &)>;
 
-    FFmpegDecoder(const Track::Ptr &track, int thread_num = 2);
+    FFmpegDecoder(const Track::Ptr &track, int thread_num = 2, const std::vector<std::string> &codec_name = {});
     ~FFmpegDecoder() override;
 
     bool inputFrame(const Frame::Ptr &frame, bool live, bool async, bool enable_merge = true);
@@ -113,7 +113,7 @@ public:
 private:
     void onDecode(const FFmpegFrame::Ptr &frame);
     bool inputFrame_l(const Frame::Ptr &frame, bool live, bool enable_merge);
-    bool decodeFrame(const char *data, size_t size, uint64_t dts, uint64_t pts, bool live);
+    bool decodeFrame(const char *data, size_t size, uint64_t dts, uint64_t pts, bool live, bool key_frame);
 
 private:
     bool _do_merger = false;
@@ -133,10 +133,16 @@ public:
     int inputFrame(const FFmpegFrame::Ptr &frame, uint8_t *data);
 
 private:
-    int _target_width;
+    FFmpegFrame::Ptr inputFrame(const FFmpegFrame::Ptr &frame, int &ret, uint8_t *data);
-    int _target_height;
+
+private:
+    int _target_width = 0;
+    int _target_height = 0;
+    int _src_width = 0;
+    int _src_height = 0;
     SwsContext *_ctx = nullptr;
-    AVPixelFormat _target_format;
+    AVPixelFormat _src_format = AV_PIX_FMT_NONE;
+    AVPixelFormat _target_format = AV_PIX_FMT_NONE;
 };
 
 }//namespace mediakit