添加内存malloc次数分布统计 (#1322)

server/WebApi.cpp

@@ -212,7 +212,12 @@ static ApiArgsType getAllArgs(const Parser &parser) {
 }
 
 extern uint64_t getTotalMemUsage();
+extern uint64_t getTotalMemBlock();
 extern uint64_t getThisThreadMemUsage();
+extern uint64_t getThisThreadMemBlock();
+extern std::vector<size_t> getBlockTypeSize();
+extern uint64_t getTotalMemBlockByType(int type);
+extern uint64_t getThisThreadMemBlockByType(int type) ;
 
 static inline void addHttpListener(){
     GET_CONFIG(bool, api_debug, API::kApiDebug);
@@ -382,45 +387,60 @@ void getStatisticJson(const function<void(Value &val)> &cb) {
     val["RtmpPacket"] = (Json::UInt64)(ObjectStatistic<RtmpPacket>::count());
 #ifdef ENABLE_MEM_DEBUG
     auto bytes = getTotalMemUsage();
-    val["totalMemUsage"] = (Json::UInt64)bytes;
+    val["totalMemUsage"] = (Json::UInt64) bytes;
-    val["totalMemUsageMB"] = (int)(bytes / 1024 / 1024);
+    val["totalMemUsageMB"] = (int) (bytes / 1024 / 1024);
+    val["totalMemBlock"] = (Json::UInt64) getTotalMemBlock();
+    static auto block_type_size = getBlockTypeSize();
+    {
+        int i = 0;
+        string str;
+        size_t last = 0;
+        for (auto sz : block_type_size) {
+            str.append(to_string(last) + "~" + to_string(sz) + ":" + to_string(getTotalMemBlockByType(i++)) + ";");
+            last = sz;
+        }
+        str.pop_back();
+        val["totalMemBlockTypeCount"] = str;
+    }
 
     auto thread_size = EventPollerPool::Instance().getExecutorSize() + WorkThreadPool::Instance().getExecutorSize();
     std::shared_ptr<vector<Value> > thread_mem_info = std::make_shared<vector<Value> >(thread_size);
-    std::shared_ptr<atomic<uint64_t> > thread_mem_total = std::make_shared<atomic<uint64_t> >(0);
 
-    shared_ptr<void> finished(nullptr, [thread_mem_info, cb, obj, thread_mem_total](void *) {
+    shared_ptr<void> finished(nullptr, [thread_mem_info, cb, obj](void *) {
-        //poller和work线程开辟的内存
         for (auto &val : *thread_mem_info) {
             (*obj)["threadMem"].append(val);
         }
-
-        //其他线程申请的内存为总内存减去poller和work线程开辟的内存
-        auto bytes = getTotalMemUsage() - *thread_mem_total;
-        Value val;
-        val["threadName"] = "other threads";
-        val["threadMemUsage"] = (Json::UInt64) bytes;
-        val["threadMemUsageMB"] = (int) (bytes / 1024 / 1024);
-        (*obj)["threadMem"].append(val);
-
         //触发回调
         cb(*obj);
     });
 
     auto pos = 0;
-    auto lam = [&](const TaskExecutor::Ptr &executor) {
+    auto lam0 = [&](TaskExecutor &executor) {
         auto &val = (*thread_mem_info)[pos++];
-        executor->async([finished, thread_mem_total, &val]() {
+        executor.async([finished, &val]() {
             auto bytes = getThisThreadMemUsage();
-            *thread_mem_total += bytes;
-
             val["threadName"] = getThreadName();
             val["threadMemUsage"] = (Json::UInt64) bytes;
-            val["threadMemUsageMB"] = (int) (bytes / 1024 / 1024);
+            val["threadMemUsageMB"] = (Json::UInt64) (bytes / 1024 / 1024);
+            val["threadMemBlock"] = (Json::UInt64) getThisThreadMemBlock();
+            {
+                int i = 0;
+                string str;
+                size_t last = 0;
+                for (auto sz : block_type_size) {
+                    str.append(to_string(last) + "~" + to_string(sz) + ":" + to_string(getThisThreadMemBlockByType(i++)) + ";");
+                    last = sz;
+                }
+                str.pop_back();
+                val["threadMemBlockTypeCount"] = str;
+            }
         });
     };
-    EventPollerPool::Instance().for_each(lam);
+    auto lam1 = [lam0](const TaskExecutor::Ptr &executor) {
-    WorkThreadPool::Instance().for_each(lam);
+        lam0(*executor);
+    };
+    EventPollerPool::Instance().for_each(lam1);
+    WorkThreadPool::Instance().for_each(lam1);
 #else
     cb(*obj);
 #endif

src/Common/config.cpp

@@ -328,13 +328,43 @@ extern "C" {
     void *__wrap_realloc(void *ptr, size_t c);
 }
 
+#define BLOCK_TYPES 16
+#define MIN_BLOCK_SIZE 128
+
+static int get_mem_block_type(size_t c) {
+    int ret = 0;
+    while (c > MIN_BLOCK_SIZE && ret + 1 < BLOCK_TYPES) {
+        c >>= 1;
+        ++ret;
+    }
+    return ret;
+}
+
+std::vector<size_t> getBlockTypeSize() {
+    std::vector<size_t> ret;
+    ret.resize(BLOCK_TYPES);
+    size_t block_size = MIN_BLOCK_SIZE;
+    for (auto i = 0; i < BLOCK_TYPES; ++i) {
+        ret[i] = block_size;
+        block_size <<= 1;
+    }
+    return ret;
+}
+
 class MemAllocInfo {
 public:
     static atomic<uint64_t> total_mem_usage;
+    static atomic<uint64_t> total_mem_block;
+    static atomic<uint64_t> total_mem_block_map[BLOCK_TYPES];
+
     atomic<uint64_t> mem_usage{0};
+    atomic<uint64_t> mem_block{0};
+    atomic<uint64_t> mem_block_map[BLOCK_TYPES];
 };
 
 atomic<uint64_t> MemAllocInfo::total_mem_usage{0};
+atomic<uint64_t> MemAllocInfo::total_mem_block{0};
+atomic<uint64_t> MemAllocInfo::total_mem_block_map[BLOCK_TYPES];
 
 static thread_local MemAllocInfo s_alloc_info;
 
@@ -342,10 +372,28 @@ uint64_t getTotalMemUsage() {
     return MemAllocInfo::total_mem_usage.load();
 }
 
+uint64_t getTotalMemBlock() {
+    return MemAllocInfo::total_mem_block.load();
+}
+
+uint64_t getTotalMemBlockByType(int type) {
+    assert(type < BLOCK_TYPES);
+    return MemAllocInfo::total_mem_block_map[type].load();
+}
+
 uint64_t getThisThreadMemUsage() {
     return s_alloc_info.mem_usage.load();
 }
 
+uint64_t getThisThreadMemBlock() {
+    return s_alloc_info.mem_block.load();
+}
+
+uint64_t getThisThreadMemBlockByType(int type) {
+    assert(type < BLOCK_TYPES);
+    return s_alloc_info.mem_block_map[type].load();
+}
+
 #if defined(_WIN32)
 #pragma pack(push, 1)
 #endif // defined(_WIN32)
@@ -355,6 +403,7 @@ public:
     static constexpr uint32_t kMagic = 0xFEFDFCFB;
     uint32_t magic;
     uint32_t size;
+    uint8_t type;
     MemAllocInfo *alloc_info;
     char ptr;
 }PACKED;
@@ -366,11 +415,29 @@ public:
 #define MEM_OFFSET (sizeof(MemCookie) - 1)
 
 void init_cookie(MemCookie *cookie, size_t c) {
+    int type = get_mem_block_type(c);
     MemAllocInfo::total_mem_usage += c;
+    ++MemAllocInfo::total_mem_block;
+    ++MemAllocInfo::total_mem_block_map[type];
+
     s_alloc_info.mem_usage += c;
+    ++s_alloc_info.mem_block;
+    ++s_alloc_info.mem_block_map[type];
+
     cookie->magic = MemCookie::kMagic;
     cookie->size = c;
     cookie->alloc_info = &s_alloc_info;
+    cookie->type = type;
+}
+
+void un_init_cookie(MemCookie *cookie) {
+    MemAllocInfo::total_mem_usage -= cookie->size;
+    --MemAllocInfo::total_mem_block;
+    --MemAllocInfo::total_mem_block_map[cookie->type];
+
+    cookie->alloc_info->mem_usage -= cookie->size;
+    --cookie->alloc_info->mem_block;
+    --cookie->alloc_info->mem_block_map[cookie->type];
 }
 
 void *__wrap_malloc(size_t c) {
@@ -389,10 +456,10 @@ void __wrap_free(void *ptr) {
     }
     auto cookie = (MemCookie *) ((char *) ptr - MEM_OFFSET);
     if (cookie->magic != MemCookie::kMagic) {
-        throw std::invalid_argument("attempt to free invalid memory");
+        __real_free(ptr);
+        return;
     }
-    MemAllocInfo::total_mem_usage -= cookie->size;
+    un_init_cookie(cookie);
-    cookie->alloc_info->mem_usage -= cookie->size;
     __real_free(cookie);
 }
 
@@ -412,20 +479,16 @@ void *__wrap_realloc(void *ptr, size_t c) {
 
     auto cookie = (MemCookie *) ((char *) ptr - MEM_OFFSET);
     if (cookie->magic != MemCookie::kMagic) {
-        throw std::invalid_argument("attempt to realloc invalid memory");
+        return __real_realloc(ptr, c);
     }
 
-    MemAllocInfo::total_mem_usage -= cookie->size;
+    un_init_cookie(cookie);
-    cookie->alloc_info->mem_usage -= cookie->size;
-
     c += MEM_OFFSET;
     cookie = (MemCookie *) __real_realloc(cookie, c);
     if (cookie) {
         init_cookie(cookie, c);
         return &cookie->ptr;
     }
-
-    free(cookie);
     return nullptr;
 }
 

src/Record/MP4Recorder.cpp

@@ -30,6 +30,7 @@ MP4Recorder::MP4Recorder(const string &path, const string &vhost, const string &
     GET_CONFIG(size_t ,recordSec,Record::kFileSecond);
     _max_second = max_second ? max_second : recordSec;
 }
+
 MP4Recorder::~MP4Recorder() {
     closeFile();
 }
@@ -61,7 +62,6 @@ void MP4Recorder::createFile() {
         WarnL << ex.what();
     }
 }
-
 void MP4Recorder::asyncClose() {
     auto muxer = _muxer;
     auto full_path_tmp = _full_path_tmp;