From 78bb8b9fc7f96ee4e1e4ca41232e915d2bfb69b0 Mon Sep 17 00:00:00 2001 From: luocai Date: Thu, 5 Sep 2024 12:24:05 +0800 Subject: [PATCH] add webrtc aec code. --- .vscode/c_cpp_properties.json | 4 ++- CMakeLists.txt | 1 + Record/CMakeLists.txt | 2 ++ Record/EchoRecord.cpp | 45 ++++++++++++++++++++++++++++--- Record/WebRtcAecm.cpp | 50 +++++++++++++++++++++++++++++++++++ Record/WebRtcAecm.h | 18 +++++++++++++ Record/main.h | 8 ++++-- 7 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 Record/WebRtcAecm.cpp create mode 100644 Record/WebRtcAecm.h diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 68fa7ff..483640f 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -13,7 +13,9 @@ "${workspaceFolder}/build/_deps/kylin-src/Universal", "${workspaceFolder}/build/_deps/kylin-src/HttpProxy" ], - "defines": [], + "defines": [ + "WEBRTC_POSIX" + ], "compilerPath": "/usr/bin/gcc", "cStandard": "c17", "cppStandard": "gnu++17", diff --git a/CMakeLists.txt b/CMakeLists.txt index f3b68f5..202d334 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ FetchContent_Declare(Kylin ) FetchContent_MakeAvailable(Kylin) +add_subdirectory(VocieProcess) add_subdirectory(GatePass) add_subdirectory(Record) add_subdirectory(VoucherVerifyServer) \ No newline at end of file diff --git a/Record/CMakeLists.txt b/Record/CMakeLists.txt index 2b8e391..f4227d0 100644 --- a/Record/CMakeLists.txt +++ b/Record/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(Record main.cpp Recorder.cpp SpeexDsp.h SpeexDsp.cpp Utility.h Utility.cpp + WebRtcAecm.h WebRtcAecm.cpp WebRTCPublisher.h WebRTCPublisher.cpp ) @@ -35,6 +36,7 @@ target_link_directories(Record ) target_link_libraries(Record + PRIVATE VocieProcess PRIVATE asound PRIVATE easymedia PRIVATE drm diff --git a/Record/EchoRecord.cpp b/Record/EchoRecord.cpp index 40454af..66bcc33 100644 --- a/Record/EchoRecord.cpp +++ b/Record/EchoRecord.cpp @@ -1,4 +1,7 @@ #include "BoostLog.h" +#include "SpeexDsp.h" +#include "Utility.h" +#include "WebRtcAecm.h" #include "main.h" #include @@ -14,18 +17,54 @@ void EchoRecordTask::setChannels(int channels) { } } +// underrun occurred pcm播放饥饿 +// 回采信号提前于mic信号，时间差<80ms +// ./Record --echo --vqe=false --channels=2 +// ./Record --echo --vqe=true --channels=2 +// ./Record --echo --vqe=false --channels=1 void EchoRecordTask::run() { + RkAudio::Format format; format.channels = m_channels; - format.period = 64; + format.period = 10; + + m_speex = std::make_shared(); + m_speex->start(format.sampleRate, m_channels, format.period); + + m_farendBuffer.resize(m_channels * sizeof(int16_t) * format.sampleRate / 1000 * format.period); + m_nearendBuffer.resize(m_channels * sizeof(int16_t) * format.sampleRate / 1000 * format.period); + m_webRtcAecm = std::make_shared(); + m_webRtcAecm->start(format.sampleRate, format.channels, format.period); + m_output = std::make_shared(); - if (!m_output->open(sizeof(uint16_t), format.sampleRate, format.channels, format.period, m_vqeEnabled)) { + if (!m_output->open(sizeof(uint16_t), format.sampleRate, 2, format.period, m_vqeEnabled)) { LOG(error) << "audio output open failed."; return; } + m_outBuffer.resize(m_channels * sizeof(int16_t) * format.sampleRate / 1000 * format.period); + m_input = std::make_shared(); - m_input->setDataCallback([this](const RkAudio::Frame &frame) { m_output->write(frame.data, frame.byteSize); }); + m_input->setDataCallback([this](const RkAudio::Frame &frame) { + memcpy(m_nearendBuffer.data(), frame.data, frame.byteSize); + m_webRtcAecm->echoPlayback(reinterpret_cast(m_farendBuffer.data()), m_farendBuffer.size() / 2); + + m_webRtcAecm->echoCancellation(reinterpret_cast(frame.data), reinterpret_cast(m_nearendBuffer.data()), + reinterpret_cast(m_outBuffer.data()), frame.frameSize); + // m_speex->echoPlayback(reinterpret_cast(frame.data)); + // m_speex->echoPlayback(reinterpret_cast(m_buffer.data())); + + // m_speex->echoCapture(reinterpret_cast(frame.data), reinterpret_cast(m_buffer.data())); + + if (m_channels == 2) { + m_output->write(frame.data, frame.byteSize); + } else if (m_channels == 1) { + auto filledData = duplicate(m_outBuffer.data(), m_outBuffer.size()); + m_output->write(filledData.data(), filledData.size()); + } + memcpy(m_farendBuffer.data(), m_outBuffer.data(), m_outBuffer.size()); + // m_output->write(reinterpret_cast(m_buffer.data()), m_buffer.size()); + }); m_input->open(format, m_vqeEnabled); } \ No newline at end of file diff --git a/Record/WebRtcAecm.cpp b/Record/WebRtcAecm.cpp new file mode 100644 index 0000000..07f208b --- /dev/null +++ b/Record/WebRtcAecm.cpp @@ -0,0 +1,50 @@ +#include "WebRtcAecm.h" +#include "BoostLog.h" +#include "modules/audio_processing/aecm/echo_control_mobile.h" + +WebRtcAecm::~WebRtcAecm() { + close(); +} + +void WebRtcAecm::close() { + if (m_hanlde != nullptr) { + webrtc::WebRtcAecm_Free(m_hanlde); + m_hanlde = nullptr; + } +} + +void WebRtcAecm::echoPlayback(const int16_t *play, int samples) { + if (webrtc::WebRtcAecm_BufferFarend(m_hanlde, play, samples) != 0) { + LOG(warning) << "failed to buffer farend signal."; + } +} + +void WebRtcAecm::echoCancellation(const int16_t *nearendNoisy, const int16_t *nearendClean, int16_t *out, int samples) { + if (webrtc::WebRtcAecm_Process(m_hanlde, nearendNoisy, nearendClean, out, samples, 60) != 0) { + LOG(warning) << "WebRtcAecm_Process() failed."; + } +} + +bool WebRtcAecm::start(int sampleRate, int channels, int period) { + close(); + m_hanlde = webrtc::WebRtcAecm_Create(); + if (m_hanlde == nullptr) { + LOG(error) << "WebRtcAecm_Create() failed."; + return false; + } + if (webrtc::WebRtcAecm_Init(m_hanlde, sampleRate) != 0) { + LOG(error) << "WebRtcAecm_Init() failed."; + close(); + return false; + } + + webrtc::AecmConfig config; + config.cngMode = webrtc::AecmTrue; // 启用舒适噪声生成 + config.echoMode = 3; // 回声消除模式，范围是 0-4 + if (WebRtcAecm_set_config(m_hanlde, config) != 0) { + LOG(error) << "WebRtcAecm_set_config() failed."; + close(); + return false; + } + return true; +} \ No newline at end of file diff --git a/Record/WebRtcAecm.h b/Record/WebRtcAecm.h new file mode 100644 index 0000000..aab3558 --- /dev/null +++ b/Record/WebRtcAecm.h @@ -0,0 +1,18 @@ +#ifndef __WEBRTCAECM_H__ +#define __WEBRTCAECM_H__ + +#include + +class WebRtcAecm { +public: + ~WebRtcAecm(); + bool start(int sampleRate, int channels, int period); + void close(); + void echoPlayback(const int16_t *play, int samples); + void echoCancellation(const int16_t *nearendNoisy, const int16_t *nearendClean, int16_t *out, int samples); + +private: + void *m_hanlde = nullptr; +}; + +#endif // __WEBRTCAECM_H__ \ No newline at end of file diff --git a/Record/main.h b/Record/main.h index 6092fba..561f782 100644 --- a/Record/main.h +++ b/Record/main.h @@ -6,7 +6,7 @@ #include class SpeexDsp; - +class WebRtcAecm; class Task { public: virtual void run() = 0; @@ -50,7 +50,11 @@ private: std::shared_ptr m_output; std::shared_ptr m_input; std::shared_ptr m_speex; - std::vector m_buffer; + std::shared_ptr m_webRtcAecm; + std::vector m_outBuffer; + + std::vector m_nearendBuffer; + std::vector m_farendBuffer; }; #endif // __MAIN_H__ \ No newline at end of file