From 15297a3ca354d50e83e4113accd8320440095440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=8F=E6=A5=9A?= <771730766@qq.com> Date: Wed, 29 Dec 2021 20:48:15 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=86=85=E5=AD=98malloc?= =?UTF-8?q?=E6=AC=A1=E6=95=B0=E5=88=86=E5=B8=83=E7=BB=9F=E8=AE=A1=20(#1322?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/WebApi.cpp | 62 +++++++++++++++++++---------- src/Common/config.cpp | 81 +++++++++++++++++++++++++++++++++----- src/Record/MP4Recorder.cpp | 2 +- 3 files changed, 114 insertions(+), 31 deletions(-) diff --git a/server/WebApi.cpp b/server/WebApi.cpp index 2b3f61db..7c0006db 100755 --- a/server/WebApi.cpp +++ b/server/WebApi.cpp @@ -212,7 +212,12 @@ static ApiArgsType getAllArgs(const Parser &parser) { } extern uint64_t getTotalMemUsage(); +extern uint64_t getTotalMemBlock(); extern uint64_t getThisThreadMemUsage(); +extern uint64_t getThisThreadMemBlock(); +extern std::vector getBlockTypeSize(); +extern uint64_t getTotalMemBlockByType(int type); +extern uint64_t getThisThreadMemBlockByType(int type) ; static inline void addHttpListener(){ GET_CONFIG(bool, api_debug, API::kApiDebug); @@ -382,45 +387,60 @@ void getStatisticJson(const function &cb) { val["RtmpPacket"] = (Json::UInt64)(ObjectStatistic::count()); #ifdef ENABLE_MEM_DEBUG auto bytes = getTotalMemUsage(); - val["totalMemUsage"] = (Json::UInt64)bytes; - val["totalMemUsageMB"] = (int)(bytes / 1024 / 1024); + val["totalMemUsage"] = (Json::UInt64) bytes; + val["totalMemUsageMB"] = (int) (bytes / 1024 / 1024); + val["totalMemBlock"] = (Json::UInt64) getTotalMemBlock(); + static auto block_type_size = getBlockTypeSize(); + { + int i = 0; + string str; + size_t last = 0; + for (auto sz : block_type_size) { + str.append(to_string(last) + "~" + to_string(sz) + ":" + to_string(getTotalMemBlockByType(i++)) + ";"); + last = sz; + } + str.pop_back(); + val["totalMemBlockTypeCount"] = str; + } auto thread_size = EventPollerPool::Instance().getExecutorSize() + WorkThreadPool::Instance().getExecutorSize(); std::shared_ptr > thread_mem_info = std::make_shared >(thread_size); - std::shared_ptr > thread_mem_total = std::make_shared >(0); - shared_ptr finished(nullptr, [thread_mem_info, cb, obj, thread_mem_total](void *) { - //poller和work线程开辟的内存 + shared_ptr finished(nullptr, [thread_mem_info, cb, obj](void *) { for (auto &val : *thread_mem_info) { (*obj)["threadMem"].append(val); } - - //其他线程申请的内存为总内存减去poller和work线程开辟的内存 - auto bytes = getTotalMemUsage() - *thread_mem_total; - Value val; - val["threadName"] = "other threads"; - val["threadMemUsage"] = (Json::UInt64) bytes; - val["threadMemUsageMB"] = (int) (bytes / 1024 / 1024); - (*obj)["threadMem"].append(val); - //触发回调 cb(*obj); }); auto pos = 0; - auto lam = [&](const TaskExecutor::Ptr &executor) { + auto lam0 = [&](TaskExecutor &executor) { auto &val = (*thread_mem_info)[pos++]; - executor->async([finished, thread_mem_total, &val]() { + executor.async([finished, &val]() { auto bytes = getThisThreadMemUsage(); - *thread_mem_total += bytes; - val["threadName"] = getThreadName(); val["threadMemUsage"] = (Json::UInt64) bytes; - val["threadMemUsageMB"] = (int) (bytes / 1024 / 1024); + val["threadMemUsageMB"] = (Json::UInt64) (bytes / 1024 / 1024); + val["threadMemBlock"] = (Json::UInt64) getThisThreadMemBlock(); + { + int i = 0; + string str; + size_t last = 0; + for (auto sz : block_type_size) { + str.append(to_string(last) + "~" + to_string(sz) + ":" + to_string(getThisThreadMemBlockByType(i++)) + ";"); + last = sz; + } + str.pop_back(); + val["threadMemBlockTypeCount"] = str; + } }); }; - EventPollerPool::Instance().for_each(lam); - WorkThreadPool::Instance().for_each(lam); + auto lam1 = [lam0](const TaskExecutor::Ptr &executor) { + lam0(*executor); + }; + EventPollerPool::Instance().for_each(lam1); + WorkThreadPool::Instance().for_each(lam1); #else cb(*obj); #endif diff --git a/src/Common/config.cpp b/src/Common/config.cpp index 81824dd3..feb336cb 100644 --- a/src/Common/config.cpp +++ b/src/Common/config.cpp @@ -328,13 +328,43 @@ extern "C" { void *__wrap_realloc(void *ptr, size_t c); } +#define BLOCK_TYPES 16 +#define MIN_BLOCK_SIZE 128 + +static int get_mem_block_type(size_t c) { + int ret = 0; + while (c > MIN_BLOCK_SIZE && ret + 1 < BLOCK_TYPES) { + c >>= 1; + ++ret; + } + return ret; +} + +std::vector getBlockTypeSize() { + std::vector ret; + ret.resize(BLOCK_TYPES); + size_t block_size = MIN_BLOCK_SIZE; + for (auto i = 0; i < BLOCK_TYPES; ++i) { + ret[i] = block_size; + block_size <<= 1; + } + return ret; +} + class MemAllocInfo { public: static atomic total_mem_usage; + static atomic total_mem_block; + static atomic total_mem_block_map[BLOCK_TYPES]; + atomic mem_usage{0}; + atomic mem_block{0}; + atomic mem_block_map[BLOCK_TYPES]; }; atomic MemAllocInfo::total_mem_usage{0}; +atomic MemAllocInfo::total_mem_block{0}; +atomic MemAllocInfo::total_mem_block_map[BLOCK_TYPES]; static thread_local MemAllocInfo s_alloc_info; @@ -342,10 +372,28 @@ uint64_t getTotalMemUsage() { return MemAllocInfo::total_mem_usage.load(); } +uint64_t getTotalMemBlock() { + return MemAllocInfo::total_mem_block.load(); +} + +uint64_t getTotalMemBlockByType(int type) { + assert(type < BLOCK_TYPES); + return MemAllocInfo::total_mem_block_map[type].load(); +} + uint64_t getThisThreadMemUsage() { return s_alloc_info.mem_usage.load(); } +uint64_t getThisThreadMemBlock() { + return s_alloc_info.mem_block.load(); +} + +uint64_t getThisThreadMemBlockByType(int type) { + assert(type < BLOCK_TYPES); + return s_alloc_info.mem_block_map[type].load(); +} + #if defined(_WIN32) #pragma pack(push, 1) #endif // defined(_WIN32) @@ -355,6 +403,7 @@ public: static constexpr uint32_t kMagic = 0xFEFDFCFB; uint32_t magic; uint32_t size; + uint8_t type; MemAllocInfo *alloc_info; char ptr; }PACKED; @@ -366,11 +415,29 @@ public: #define MEM_OFFSET (sizeof(MemCookie) - 1) void init_cookie(MemCookie *cookie, size_t c) { + int type = get_mem_block_type(c); MemAllocInfo::total_mem_usage += c; + ++MemAllocInfo::total_mem_block; + ++MemAllocInfo::total_mem_block_map[type]; + s_alloc_info.mem_usage += c; + ++s_alloc_info.mem_block; + ++s_alloc_info.mem_block_map[type]; + cookie->magic = MemCookie::kMagic; cookie->size = c; cookie->alloc_info = &s_alloc_info; + cookie->type = type; +} + +void un_init_cookie(MemCookie *cookie) { + MemAllocInfo::total_mem_usage -= cookie->size; + --MemAllocInfo::total_mem_block; + --MemAllocInfo::total_mem_block_map[cookie->type]; + + cookie->alloc_info->mem_usage -= cookie->size; + --cookie->alloc_info->mem_block; + --cookie->alloc_info->mem_block_map[cookie->type]; } void *__wrap_malloc(size_t c) { @@ -389,10 +456,10 @@ void __wrap_free(void *ptr) { } auto cookie = (MemCookie *) ((char *) ptr - MEM_OFFSET); if (cookie->magic != MemCookie::kMagic) { - throw std::invalid_argument("attempt to free invalid memory"); + __real_free(ptr); + return; } - MemAllocInfo::total_mem_usage -= cookie->size; - cookie->alloc_info->mem_usage -= cookie->size; + un_init_cookie(cookie); __real_free(cookie); } @@ -412,20 +479,16 @@ void *__wrap_realloc(void *ptr, size_t c) { auto cookie = (MemCookie *) ((char *) ptr - MEM_OFFSET); if (cookie->magic != MemCookie::kMagic) { - throw std::invalid_argument("attempt to realloc invalid memory"); + return __real_realloc(ptr, c); } - MemAllocInfo::total_mem_usage -= cookie->size; - cookie->alloc_info->mem_usage -= cookie->size; - + un_init_cookie(cookie); c += MEM_OFFSET; cookie = (MemCookie *) __real_realloc(cookie, c); if (cookie) { init_cookie(cookie, c); return &cookie->ptr; } - - free(cookie); return nullptr; } diff --git a/src/Record/MP4Recorder.cpp b/src/Record/MP4Recorder.cpp index 5d9c1570..5ae7d807 100644 --- a/src/Record/MP4Recorder.cpp +++ b/src/Record/MP4Recorder.cpp @@ -30,6 +30,7 @@ MP4Recorder::MP4Recorder(const string &path, const string &vhost, const string & GET_CONFIG(size_t ,recordSec,Record::kFileSecond); _max_second = max_second ? max_second : recordSec; } + MP4Recorder::~MP4Recorder() { closeFile(); } @@ -61,7 +62,6 @@ void MP4Recorder::createFile() { WarnL << ex.what(); } } - void MP4Recorder::asyncClose() { auto muxer = _muxer; auto full_path_tmp = _full_path_tmp;