225 lines
9.4 KiB
C
225 lines
9.4 KiB
C
|
/*
|
||
|
|
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
|
||
|
|
*
|
||
|
|
* Use of this source code is governed by a BSD-style license
|
||
|
|
* that can be found in the LICENSE file in the root of the source
|
||
|
|
* tree. An additional intellectual property rights grant can be found
|
||
|
|
* in the file PATENTS. All contributing project authors may
|
||
|
|
* be found in the AUTHORS file in the root of the source tree.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <arm_neon.h>
|
||
|
|
|
||
|
|
#include "common_audio/signal_processing/include/signal_processing_library.h"
|
||
|
|
|
||
|
|
#include "rtc_base/checks.h"
|
||
|
|
|
||
|
|
// NEON intrinsics version of WebRtcSpl_DownsampleFast()
|
||
|
|
// for ARM 32-bit/64-bit platforms.
|
||
|
|
int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in,
|
||
|
|
size_t data_in_length,
|
||
|
|
int16_t* data_out,
|
||
|
|
size_t data_out_length,
|
||
|
|
const int16_t* __restrict coefficients,
|
||
|
|
size_t coefficients_length,
|
||
|
|
int factor,
|
||
|
|
size_t delay) {
|
||
|
|
// Using signed indexes to be able to compute negative i-j that
|
||
|
|
// is used to index data_in.
|
||
|
|
int i = 0;
|
||
|
|
int j = 0;
|
||
|
|
int32_t out_s32 = 0;
|
||
|
|
int endpos = delay + factor * (data_out_length - 1) + 1;
|
||
|
|
size_t res = data_out_length & 0x7;
|
||
|
|
int endpos1 = endpos - factor * res;
|
||
|
|
|
||
|
|
// Return error if any of the running conditions doesn't meet.
|
||
|
|
if (data_out_length == 0 || coefficients_length == 0
|
||
|
|
|| (int)data_in_length < endpos) {
|
||
|
|
return -1;
|
||
|
|
}
|
||
|
|
|
||
|
|
RTC_DCHECK_GE(endpos, 0);
|
||
|
|
RTC_DCHECK_GE(endpos1, 0);
|
||
|
|
|
||
|
|
// First part, unroll the loop 8 times, with 3 subcases
|
||
|
|
// (factor == 2, 4, others).
|
||
|
|
switch (factor) {
|
||
|
|
case 2: {
|
||
|
|
for (i = delay; i < endpos1; i += 16) {
|
||
|
|
// Round value, 0.5 in Q12.
|
||
|
|
int32x4_t out32x4_0 = vdupq_n_s32(2048);
|
||
|
|
int32x4_t out32x4_1 = vdupq_n_s32(2048);
|
||
|
|
|
||
|
|
#if defined(WEBRTC_ARCH_ARM64)
|
||
|
|
// Unroll the loop 2 times.
|
||
|
|
for (j = 0; j < (int)coefficients_length - 1; j += 2) {
|
||
|
|
int32x2_t coeff32 = vld1_dup_s32((int32_t*)&coefficients[j]);
|
||
|
|
int16x4_t coeff16x4 = vreinterpret_s16_s32(coeff32);
|
||
|
|
int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j - 1]);
|
||
|
|
|
||
|
|
// Mul and accumulate low 64-bit data.
|
||
|
|
int16x4_t in16x4_0 = vget_low_s16(in16x8x2.val[0]);
|
||
|
|
int16x4_t in16x4_1 = vget_low_s16(in16x8x2.val[1]);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 1);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_1, coeff16x4, 0);
|
||
|
|
|
||
|
|
// Mul and accumulate high 64-bit data.
|
||
|
|
// TODO: vget_high_s16 need extra cost on ARM64. This could be
|
||
|
|
// replaced by vmlal_high_lane_s16. But for the interface of
|
||
|
|
// vmlal_high_lane_s16, there is a bug in gcc 4.9.
|
||
|
|
// This issue need to be tracked in the future.
|
||
|
|
int16x4_t in16x4_2 = vget_high_s16(in16x8x2.val[0]);
|
||
|
|
int16x4_t in16x4_3 = vget_high_s16(in16x8x2.val[1]);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_2, coeff16x4, 1);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_3, coeff16x4, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
for (; j < (int)coefficients_length; j++) {
|
||
|
|
int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]);
|
||
|
|
int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j]);
|
||
|
|
|
||
|
|
// Mul and accumulate low 64-bit data.
|
||
|
|
int16x4_t in16x4_0 = vget_low_s16(in16x8x2.val[0]);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 0);
|
||
|
|
|
||
|
|
// Mul and accumulate high 64-bit data.
|
||
|
|
// TODO: vget_high_s16 need extra cost on ARM64. This could be
|
||
|
|
// replaced by vmlal_high_lane_s16. But for the interface of
|
||
|
|
// vmlal_high_lane_s16, there is a bug in gcc 4.9.
|
||
|
|
// This issue need to be tracked in the future.
|
||
|
|
int16x4_t in16x4_1 = vget_high_s16(in16x8x2.val[0]);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 0);
|
||
|
|
}
|
||
|
|
#else
|
||
|
|
// On ARMv7, the loop unrolling 2 times results in performance
|
||
|
|
// regression.
|
||
|
|
for (j = 0; j < (int)coefficients_length; j++) {
|
||
|
|
int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]);
|
||
|
|
int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j]);
|
||
|
|
|
||
|
|
// Mul and accumulate.
|
||
|
|
int16x4_t in16x4_0 = vget_low_s16(in16x8x2.val[0]);
|
||
|
|
int16x4_t in16x4_1 = vget_high_s16(in16x8x2.val[0]);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 0);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 0);
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
// Saturate and store the output.
|
||
|
|
int16x4_t out16x4_0 = vqshrn_n_s32(out32x4_0, 12);
|
||
|
|
int16x4_t out16x4_1 = vqshrn_n_s32(out32x4_1, 12);
|
||
|
|
vst1q_s16(data_out, vcombine_s16(out16x4_0, out16x4_1));
|
||
|
|
data_out += 8;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case 4: {
|
||
|
|
for (i = delay; i < endpos1; i += 32) {
|
||
|
|
// Round value, 0.5 in Q12.
|
||
|
|
int32x4_t out32x4_0 = vdupq_n_s32(2048);
|
||
|
|
int32x4_t out32x4_1 = vdupq_n_s32(2048);
|
||
|
|
|
||
|
|
// Unroll the loop 4 times.
|
||
|
|
for (j = 0; j < (int)coefficients_length - 3; j += 4) {
|
||
|
|
int16x4_t coeff16x4 = vld1_s16(&coefficients[j]);
|
||
|
|
int16x8x4_t in16x8x4 = vld4q_s16(&data_in[i - j - 3]);
|
||
|
|
|
||
|
|
// Mul and accumulate low 64-bit data.
|
||
|
|
int16x4_t in16x4_0 = vget_low_s16(in16x8x4.val[0]);
|
||
|
|
int16x4_t in16x4_2 = vget_low_s16(in16x8x4.val[1]);
|
||
|
|
int16x4_t in16x4_4 = vget_low_s16(in16x8x4.val[2]);
|
||
|
|
int16x4_t in16x4_6 = vget_low_s16(in16x8x4.val[3]);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 3);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_2, coeff16x4, 2);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_4, coeff16x4, 1);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_6, coeff16x4, 0);
|
||
|
|
|
||
|
|
// Mul and accumulate high 64-bit data.
|
||
|
|
// TODO: vget_high_s16 need extra cost on ARM64. This could be
|
||
|
|
// replaced by vmlal_high_lane_s16. But for the interface of
|
||
|
|
// vmlal_high_lane_s16, there is a bug in gcc 4.9.
|
||
|
|
// This issue need to be tracked in the future.
|
||
|
|
int16x4_t in16x4_1 = vget_high_s16(in16x8x4.val[0]);
|
||
|
|
int16x4_t in16x4_3 = vget_high_s16(in16x8x4.val[1]);
|
||
|
|
int16x4_t in16x4_5 = vget_high_s16(in16x8x4.val[2]);
|
||
|
|
int16x4_t in16x4_7 = vget_high_s16(in16x8x4.val[3]);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 3);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_3, coeff16x4, 2);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_5, coeff16x4, 1);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_7, coeff16x4, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
for (; j < (int)coefficients_length; j++) {
|
||
|
|
int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]);
|
||
|
|
int16x8x4_t in16x8x4 = vld4q_s16(&data_in[i - j]);
|
||
|
|
|
||
|
|
// Mul and accumulate low 64-bit data.
|
||
|
|
int16x4_t in16x4_0 = vget_low_s16(in16x8x4.val[0]);
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 0);
|
||
|
|
|
||
|
|
// Mul and accumulate high 64-bit data.
|
||
|
|
// TODO: vget_high_s16 need extra cost on ARM64. This could be
|
||
|
|
// replaced by vmlal_high_lane_s16. But for the interface of
|
||
|
|
// vmlal_high_lane_s16, there is a bug in gcc 4.9.
|
||
|
|
// This issue need to be tracked in the future.
|
||
|
|
int16x4_t in16x4_1 = vget_high_s16(in16x8x4.val[0]);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Saturate and store the output.
|
||
|
|
int16x4_t out16x4_0 = vqshrn_n_s32(out32x4_0, 12);
|
||
|
|
int16x4_t out16x4_1 = vqshrn_n_s32(out32x4_1, 12);
|
||
|
|
vst1q_s16(data_out, vcombine_s16(out16x4_0, out16x4_1));
|
||
|
|
data_out += 8;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
default: {
|
||
|
|
for (i = delay; i < endpos1; i += factor * 8) {
|
||
|
|
// Round value, 0.5 in Q12.
|
||
|
|
int32x4_t out32x4_0 = vdupq_n_s32(2048);
|
||
|
|
int32x4_t out32x4_1 = vdupq_n_s32(2048);
|
||
|
|
|
||
|
|
for (j = 0; j < (int)coefficients_length; j++) {
|
||
|
|
int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]);
|
||
|
|
int16x4_t in16x4_0 = vld1_dup_s16(&data_in[i - j]);
|
||
|
|
in16x4_0 = vld1_lane_s16(&data_in[i + factor - j], in16x4_0, 1);
|
||
|
|
in16x4_0 = vld1_lane_s16(&data_in[i + factor * 2 - j], in16x4_0, 2);
|
||
|
|
in16x4_0 = vld1_lane_s16(&data_in[i + factor * 3 - j], in16x4_0, 3);
|
||
|
|
int16x4_t in16x4_1 = vld1_dup_s16(&data_in[i + factor * 4 - j]);
|
||
|
|
in16x4_1 = vld1_lane_s16(&data_in[i + factor * 5 - j], in16x4_1, 1);
|
||
|
|
in16x4_1 = vld1_lane_s16(&data_in[i + factor * 6 - j], in16x4_1, 2);
|
||
|
|
in16x4_1 = vld1_lane_s16(&data_in[i + factor * 7 - j], in16x4_1, 3);
|
||
|
|
|
||
|
|
// Mul and accumulate.
|
||
|
|
out32x4_0 = vmlal_lane_s16(out32x4_0, in16x4_0, coeff16x4, 0);
|
||
|
|
out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_1, coeff16x4, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Saturate and store the output.
|
||
|
|
int16x4_t out16x4_0 = vqshrn_n_s32(out32x4_0, 12);
|
||
|
|
int16x4_t out16x4_1 = vqshrn_n_s32(out32x4_1, 12);
|
||
|
|
vst1q_s16(data_out, vcombine_s16(out16x4_0, out16x4_1));
|
||
|
|
data_out += 8;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Second part, do the rest iterations (if any).
|
||
|
|
for (; i < endpos; i += factor) {
|
||
|
|
out_s32 = 2048; // Round value, 0.5 in Q12.
|
||
|
|
|
||
|
|
for (j = 0; j < (int)coefficients_length; j++) {
|
||
|
|
out_s32 = WebRtc_MulAccumW16(coefficients[j], data_in[i - j], out_s32);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Saturate and store the output.
|
||
|
|
out_s32 >>= 12;
|
||
|
|
*data_out++ = WebRtcSpl_SatW32ToW16(out_s32);
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|