From 2040a9d06f7576b0e0b9739760bf10c58b15e1f7 Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:58:18 +0000 Subject: [PATCH 1/7] AudioStreamConverter: add a multiplier to convert_from The float->integer convert_to already had a volume multiplier; add the matching integer->float gain to convert_from in every ISA (Default, SSE41, AVX2, AVX512, NEON), the dispatcher, and the explicit instantiations. This work also fixed two latent bugs in the SIMD u8 convert_to paths, exposed by testing the multiplier: the gain was applied after the clamp (so a loud signal with gain could scale past the uint8 range instead of clipping to it, matching Default), and the rounding offset was the scaled SCALE instead of the constant 128.5. The u8 convert_from paths scale by (multiplier / 128) and subtract the multiplier - the old constant -1 bias was only correct for a gain of 1. RawAudio2WorkingAudio now takes the multiplier in its constructor and exposes set_multiplier(), mirroring WorkingAudio2RawAudio; it passes the atomic value into convert_from on every block. Tests: the kernel round-trip tests now run with reciprocal multiplier pairs (1.0/1.0, 0.5/2.0, 0.25/4.0) - exact in binary and in-range for the uint8 clamp - and the pipeline Raw->Working->Raw round-trip tests run with 0.5/2.0. --- ...nels_AudioPipeline_RoundTrip_TestInfra.cpp | 40 ++++++++--- ...ernels_AudioPipeline_RoundTrip_TestInfra.h | 9 ++- .../Kernels_AudioPipeline_Tests.cpp | 14 ++-- .../Kernels_RawAudio2WorkingAudio.cpp | 8 ++- .../Kernels_RawAudio2WorkingAudio.h | 5 ++ .../Kernels_AudioStreamConverter.cpp | 28 ++++---- .../Kernels_AudioStreamConverter.h | 2 +- ...udioStreamConverter_Kernels_ARM64_NEON.cpp | 70 +++++++++++-------- ...s_AudioStreamConverter_Kernels_Default.cpp | 33 +++++---- ...els_AudioStreamConverter_Kernels_Default.h | 2 +- ..._AudioStreamConverter_Kernels_x64_AVX2.cpp | 68 ++++++++++-------- ...udioStreamConverter_Kernels_x64_AVX512.cpp | 68 ++++++++++-------- ...AudioStreamConverter_Kernels_x64_SSE41.cpp | 68 ++++++++++-------- ...Kernels_AudioStreamConverter_TestInfra.cpp | 26 +++++-- .../Kernels_AudioStreamConverter_TestInfra.h | 8 ++- 15 files changed, 278 insertions(+), 171 deletions(-) diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp index 90eff6ca37..29f40109a3 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp @@ -4,6 +4,7 @@ * */ +#include #include #include #include @@ -70,7 +71,7 @@ std::string sample_typestr(SampleFormat sample_type){ template -UnitTestResult run_round_trip(SampleFormat sample_type, size_t channels, size_t samples){ +UnitTestResult run_round_trip(SampleFormat sample_type, size_t channels, size_t samples, float from_multiplier){ // Generate random input. AlignedVector input(samples * channels); std::mt19937 generator(42); @@ -98,8 +99,14 @@ UnitTestResult run_round_trip(SampleFormat sample_type, size_t channels, size_t // Small buffer so the sample-size sweep hits multiple chunks. const size_t buffer_size = 256; - RawAudio2WorkingAudio raw2working(sample_type, sample_rate, channels, buffer_size); - WorkingAudio2RawAudio working2raw(sample_type, sample_rate, channels, 1.0f, buffer_size); + RawAudio2WorkingAudio raw2working( + sample_type, sample_rate, channels, + from_multiplier, buffer_size + ); + WorkingAudio2RawAudio working2raw( + sample_type, sample_rate, channels, + 1.0f / from_multiplier, buffer_size + ); RecordingRawAudioListener output(sample_type, sample_rate, channels, samples * channels); raw2working.add_listener(working2raw); working2raw.add_listener(output); @@ -132,26 +139,39 @@ UnitTestResult run_round_trip(SampleFormat sample_type, size_t channels, size_t RoundTripTest::RoundTripTest( SampleFormat sample_type, size_t channels, - size_t samples + size_t samples, + float from_multiplier ) : UnitTest( - "Kernels::AudioPipeline - RoundTrip_" + sample_typestr(sample_type) + - "(" + std::to_string(channels) + "ch x " + std::to_string(samples) + ")" + [&]{ + std::string name = "Kernels::AudioPipeline - RoundTrip_" + sample_typestr(sample_type) + + "(" + std::to_string(channels) + "ch x " + std::to_string(samples) + ")"; + if (from_multiplier != 1.0f){ + char mults[64]; + std::snprintf( + mults, sizeof(mults), " [from %.4g / to %.4g]", + from_multiplier, 1.0f / from_multiplier + ); + name += mults; + } + return name; + }() ) , m_sample_type(sample_type) , m_channels(channels) , m_samples(samples) + , m_from_multiplier(from_multiplier) {} UnitTestResult RoundTripTest::run(Logger& logger, CancellableScope& scope) const{ switch (m_sample_type){ case SampleFormat::UINT8: - return run_round_trip(m_sample_type, m_channels, m_samples); + return run_round_trip(m_sample_type, m_channels, m_samples, m_from_multiplier); case SampleFormat::SINT16: - return run_round_trip(m_sample_type, m_channels, m_samples); + return run_round_trip(m_sample_type, m_channels, m_samples, m_from_multiplier); case SampleFormat::SINT32: - return run_round_trip(m_sample_type, m_channels, m_samples); + return run_round_trip(m_sample_type, m_channels, m_samples, m_from_multiplier); case SampleFormat::FLOAT32: - return run_round_trip(m_sample_type, m_channels, m_samples); + return run_round_trip(m_sample_type, m_channels, m_samples, m_from_multiplier); default: throw InternalProgramError(&logger, PA_CURRENT_FUNCTION, "Unknown sample type."); } diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h index 177046d70f..cb328aecbc 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h @@ -20,15 +20,17 @@ namespace AudioPipeline{ // // Pushes random samples through a RawAudio2WorkingAudio node and a -// WorkingAudio2RawAudio node (with a 1.0f multiplier), then compares the -// resulting raw samples to the input. +// WorkingAudio2RawAudio node (with reciprocal multipliers, so the +// round trip is lossless), then compares the resulting raw samples +// to the input. // class RoundTripTest : public UnitTest{ public: RoundTripTest( SampleFormat sample_type, size_t channels, - size_t samples + size_t samples, + float from_multiplier = 1.0f ); virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override; @@ -38,6 +40,7 @@ class RoundTripTest : public UnitTest{ const SampleFormat m_sample_type; const size_t m_channels; const size_t m_samples; + const float m_from_multiplier; }; diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp index 9c27cc42b7..a8ff4a9463 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp @@ -17,13 +17,17 @@ namespace AudioPipeline{ void add_tests(UnitTestDatabase& database){ - // Raw -> Working -> Raw round-trip. + // Raw -> Working -> Raw round-trip, with reciprocal multipliers on + // the two nodes (exact in binary) to exercise the gain path. + const float from_multipliers[] = { 1.0f, 0.5f }; for (size_t channels = 1; channels <= 4; channels++){ for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add(SampleFormat::UINT8, channels, samples); - database.add(SampleFormat::SINT16, channels, samples); - database.add(SampleFormat::SINT32, channels, samples); - database.add(SampleFormat::FLOAT32, channels, samples); + for (float from_multiplier : from_multipliers){ + database.add(SampleFormat::UINT8, channels, samples, from_multiplier); + database.add(SampleFormat::SINT16, channels, samples, from_multiplier); + database.add(SampleFormat::SINT32, channels, samples, from_multiplier); + database.add(SampleFormat::FLOAT32, channels, samples, from_multiplier); + } } } diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.cpp index 881250bfdc..fff34b7dec 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.cpp @@ -39,7 +39,8 @@ template void RawAudio2WorkingAudio::set_converter(){ m_convert = [this](const void* data, size_t samples){ Kernels::AudioStreamConverter::convert_from( - m_interleaved_floats.data(), (const Type*)data, samples + m_interleaved_floats.data(), (const Type*)data, samples, + m_multiplier.load(std::memory_order_acquire) ); }; } @@ -49,9 +50,11 @@ RawAudio2WorkingAudio::RawAudio2WorkingAudio( SampleFormat sample_type, size_t sample_rate, size_t channels, + float multiplier, size_t buffer_size ) : RawAudioStreamListener(sample_type, sample_rate, channels) + , m_multiplier(multiplier) , m_buffer_size(buffer_size) { // Single-channel float is pass-through. @@ -95,6 +98,9 @@ void RawAudio2WorkingAudio::add_listener(WorkingAudioStreamListener& listener){ void RawAudio2WorkingAudio::remove_listener(WorkingAudioStreamListener& listener){ m_listeners.remove(listener); } +void RawAudio2WorkingAudio::set_multiplier(float multiplier){ + m_multiplier.store(multiplier, std::memory_order_release); +} void RawAudio2WorkingAudio::push_samples(size_t frames, const void* data){ // Single-channel float pass-through. diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h index 9e816043f4..b607245d14 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_Kernels_AudioPipeline_RawAudio2WorkingAudio_H #define PokemonAutomation_Kernels_AudioPipeline_RawAudio2WorkingAudio_H +#include #include #include "Common/Cpp/ListenerSet.h" #include "Common/Cpp/Containers/AlignedVector.h" @@ -25,12 +26,15 @@ class RawAudio2WorkingAudio : public RawAudioStreamListener{ SampleFormat sample_type, size_t sample_rate, size_t channels, + float multiplier, size_t buffer_size = 4096 + 256 // 4k + break super-alignment ); void add_listener(WorkingAudioStreamListener& listener); void remove_listener(WorkingAudioStreamListener& listener); + void set_multiplier(float multiplier); + virtual void push_samples(size_t frames, const void* data) override; @@ -40,6 +44,7 @@ class RawAudio2WorkingAudio : public RawAudioStreamListener{ private: + std::atomic m_multiplier; size_t m_buffer_size; std::function m_convert = nullptr; diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.cpp index 12d931d952..d1c443e25f 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.cpp @@ -15,7 +15,7 @@ namespace AudioStreamConverter{ template -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples); +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_Default(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_Default(UnitTestDatabase& database); @@ -23,7 +23,7 @@ void add_tests_Default(UnitTestDatabase& database); template -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_x64_SSE41(UnitTestDatabase& database); @@ -31,7 +31,7 @@ void add_tests_x64_SSE41(UnitTestDatabase& database); template -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_x64_AVX2(UnitTestDatabase& database); @@ -39,7 +39,7 @@ void add_tests_x64_AVX2(UnitTestDatabase& database); template -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_AVX512(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_x64_AVX512(UnitTestDatabase& database); @@ -47,7 +47,7 @@ void add_tests_x64_AVX512(UnitTestDatabase& database); template -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples); +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_ARM64_NEON(UnitTestDatabase& database); @@ -57,32 +57,32 @@ void add_tests_ARM64_NEON(UnitTestDatabase& database); template -void convert_from(r_ptr native, c_ptr ext, size_t samples){ +void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier){ #ifdef PA_AutoDispatch_x64_17_Skylake if (CPU_CAPABILITY_CURRENT.OK_17_Skylake){ - convert_from_x64_AVX512(native, ext, samples); + convert_from_x64_AVX512(native, ext, samples, multiplier); return; } #endif #ifdef PA_AutoDispatch_x64_13_Haswell if (CPU_CAPABILITY_CURRENT.OK_13_Haswell){ - convert_from_x64_AVX2(native, ext, samples); + convert_from_x64_AVX2(native, ext, samples, multiplier); return; } #endif #ifdef PA_AutoDispatch_x64_08_Nehalem if (CPU_CAPABILITY_CURRENT.OK_08_Nehalem){ - convert_from_x64_SSE41(native, ext, samples); + convert_from_x64_SSE41(native, ext, samples, multiplier); return; } #endif #ifdef PA_AutoDispatch_arm64_20_M1 if (CPU_CAPABILITY_CURRENT.OK_M1){ - convert_from_ARM64_NEON(native, ext, samples); + convert_from_ARM64_NEON(native, ext, samples, multiplier); return; } #endif - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); return; } template @@ -117,9 +117,9 @@ void convert_to(c_ptr native, r_ptr ext, size_t samples, fl -template void convert_from(r_ptr native, c_ptr ext, size_t samples); -template void convert_from(r_ptr native, c_ptr ext, size_t samples); -template void convert_from(r_ptr native, c_ptr ext, size_t samples); +template void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier); +template void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier); +template void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to(c_ptr native, r_ptr ext, size_t samples, float multiplier); template void convert_to(c_ptr native, r_ptr ext, size_t samples, float multiplier); template void convert_to(c_ptr native, r_ptr ext, size_t samples, float multiplier); diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h index b61744010e..b43002fc5a 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h @@ -20,7 +20,7 @@ namespace AudioStreamConverter{ template -void convert_from(r_ptr native, c_ptr ext, size_t samples); +void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to(c_ptr native, r_ptr ext, size_t samples, float multiplier); diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp index 462bbf2095..b6eba1706e 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp @@ -22,26 +22,28 @@ namespace AudioStreamConverter{ -PA_FORCE_INLINE float32x4_t convert_from_u8(uint32x4_t x){ +PA_FORCE_INLINE float32x4_t convert_from_u8(uint32x4_t x, float multiplier){ return vsubq_f32( vmulq_f32( vcvtq_f32_u32(x), - vdupq_n_f32(1.f / 128.f) + vdupq_n_f32(multiplier * (1.f / 128.f)) ), - vdupq_n_f32(1.f) + vdupq_n_f32(multiplier) ); } PA_FORCE_INLINE int32x4_t convert_to_u8(float32x4_t x, float multiplier){ - const float32x4_t SCALE = vdupq_n_f32(multiplier * 128.f); + // Scale first, then clamp: a loud signal with gain clips to the + // uint8 range, it does not scale past it. + x = vmulq_f32(x, vdupq_n_f32(multiplier)); x = vminq_f32(x, vdupq_n_f32(127.0f / 128.0f)); x = vmaxq_f32(x, vdupq_n_f32(-1.0f)); - x = vaddq_f32(vmulq_f32(x, SCALE), SCALE); + x = vaddq_f32(vmulq_f32(x, vdupq_n_f32(128.0f)), vdupq_n_f32(128.5f)); return vcvtq_s32_f32(x); } -PA_FORCE_INLINE float32x4_t convert_from_s16(int32x4_t x){ +PA_FORCE_INLINE float32x4_t convert_from_s16(int32x4_t x, float multiplier){ return vmulq_f32( vcvtq_f32_s32(x), - vdupq_n_f32(1.0f / 32768.0f) + vdupq_n_f32(multiplier * (1.0f / 32768.0f)) ); } PA_FORCE_INLINE int16x4_t convert_to_s16(float32x4_t x, float multiplier){ @@ -50,10 +52,10 @@ PA_FORCE_INLINE int16x4_t convert_to_s16(float32x4_t x, float multiplier){ x = vmaxq_f32(x, vdupq_n_f32(-32768.f)); return vqmovn_s32(vcvtq_s32_f32(x)); } -PA_FORCE_INLINE float32x4_t convert_from_s32(int32x4_t x){ +PA_FORCE_INLINE float32x4_t convert_from_s32(int32x4_t x, float multiplier){ return vmulq_f32( vcvtq_f32_s32(x), - vdupq_n_f32(1.0f / 2147483648.0f) + vdupq_n_f32(multiplier * (1.0f / 2147483648.0f)) ); } PA_FORCE_INLINE int32x4_t convert_to_s32(float32x4_t x, float multiplier){ @@ -66,28 +68,28 @@ PA_FORCE_INLINE int32x4_t convert_to_s32(float32x4_t x, float multiplier){ template -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples); +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, size_t samples, float multiplier); template <> -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 8){ uint8x8_t x = vld1_u8(ext); uint16x8_t x16 = vmovl_u8(x); uint32x4_t x32_0 = vmovl_u16(vget_low_u16(x16)); uint32x4_t x32_1 = vmovl_u16(vget_high_u16(x16)); - vst1q_f32(native + 0, convert_from_u8(x32_0)); - vst1q_f32(native + 4, convert_from_u8(x32_1)); + vst1q_f32(native + 0, convert_from_u8(x32_0, multiplier)); + vst1q_f32(native + 4, convert_from_u8(x32_1, multiplier)); native += 8; ext += 8; samples -= 8; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -110,18 +112,18 @@ void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, siz } } template <> -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ int16x4_t x = vld1_s16(ext); int32x4_t x32 = vmovl_s16(x); - vst1q_f32(native, convert_from_s16(x32)); + vst1q_f32(native, convert_from_s16(x32, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -139,17 +141,17 @@ void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, siz } } template <> -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ int32x4_t x = vld1q_s32(ext); - vst1q_f32(native, convert_from_s32(x)); + vst1q_f32(native, convert_from_s32(x, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -173,37 +175,45 @@ void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, siz template class Test_from_ARM64_NEON : public Test{ public: - Test_from_ARM64_NEON(size_t samples) + Test_from_ARM64_NEON(size_t samples, float from_multiplier, float to_multiplier) : Test( "from_ARM64_NEON", samples, (ConvertFrom)convert_from_ARM64_NEON, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; template class Test_to_ARM64_NEON : public Test{ public: - Test_to_ARM64_NEON(size_t samples) + Test_to_ARM64_NEON(size_t samples, float from_multiplier, float to_multiplier) : Test( "to_ARM64_NEON", samples, (ConvertFrom)convert_from_Default, - (ConvertTo)convert_to_ARM64_NEON + (ConvertTo)convert_to_ARM64_NEON, + from_multiplier, + to_multiplier ) {} }; void add_tests_ARM64_NEON(UnitTestDatabase& database){ + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp index bedff41e66..57aac60bfc 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp @@ -15,7 +15,7 @@ namespace AudioStreamConverter{ template -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples); +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_Default(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_Default(UnitTestDatabase& database); @@ -23,9 +23,9 @@ void add_tests_Default(UnitTestDatabase& database); template <> -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier){ for (size_t c = 0; c < samples; c++){ - native[c] = ((int)ext[c] - 128) * (1.0f / 128.0f); + native[c] = ((int)ext[c] - 128) * (1.0f / 128.0f) * multiplier; } } template <> @@ -38,9 +38,9 @@ void convert_to_Default(c_ptr native, r_ptr ext, size_t } } template <> -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier){ for (size_t c = 0; c < samples; c++){ - native[c] = (float)ext[c] * (1.0f / 32768.0f); + native[c] = (float)ext[c] * (1.0f / 32768.0f) * multiplier; } } template <> @@ -55,9 +55,9 @@ void convert_to_Default(c_ptr native, r_ptr ext, size_t } } template <> -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier){ for (size_t c = 0; c < samples; c++){ - native[c] = (float)ext[c] * (1.0f / 2147483648.0f); + native[c] = (float)ext[c] * (1.0f / 2147483648.0f) * multiplier; } } template <> @@ -82,20 +82,29 @@ void convert_to_Default(c_ptr native, r_ptr ext, size_t template class Test_Default : public Test{ public: - Test_Default(size_t samples) + Test_Default(size_t samples, float from_multiplier, float to_multiplier) : Test( "Default", samples, (ConvertFrom)convert_from_Default, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; void add_tests_Default(UnitTestDatabase& database){ + // The multipliers are reciprocals so the round trip stays in the + // native range (and exact in binary): from scales the floats down, + // to scales them back up. + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h index 6d344895ca..7b55309cbd 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h @@ -24,7 +24,7 @@ namespace AudioStreamConverter{ template -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples); +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_Default(c_ptr native, r_ptr ext, size_t samples, float multiplier); diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp index b9bfeda0e6..ce3c612d97 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp @@ -22,24 +22,26 @@ namespace AudioStreamConverter{ -PA_FORCE_INLINE __m256 convert_from_u8(__m256i x){ +PA_FORCE_INLINE __m256 convert_from_u8(__m256i x, float multiplier){ return _mm256_fmadd_ps( _mm256_cvtepi32_ps(x), - _mm256_set1_ps(1.f / 128.f), - _mm256_set1_ps(-1.f) + _mm256_set1_ps(multiplier * (1.f / 128.f)), + _mm256_set1_ps(-multiplier) ); } PA_FORCE_INLINE __m256i convert_to_u8(__m256 x, float multiplier){ - const __m256 SCALE = _mm256_set1_ps(multiplier * 128.f); + // Scale first, then clamp: a loud signal with gain clips to the + // uint8 range, it does not scale past it. + x = _mm256_mul_ps(x, _mm256_set1_ps(multiplier)); x = _mm256_min_ps(x, _mm256_set1_ps(127.0f / 128.0f)); x = _mm256_max_ps(x, _mm256_set1_ps(-1.0f)); - x = _mm256_fmadd_ps(x, SCALE, SCALE); + x = _mm256_fmadd_ps(x, _mm256_set1_ps(128.0f), _mm256_set1_ps(128.5f)); return _mm256_cvtps_epi32(x); } -PA_FORCE_INLINE __m256 convert_from_s16(__m256i x){ +PA_FORCE_INLINE __m256 convert_from_s16(__m256i x, float multiplier){ return _mm256_mul_ps( _mm256_cvtepi32_ps(x), - _mm256_set1_ps(1.0f / 32768.0f) + _mm256_set1_ps(multiplier * (1.0f / 32768.0f)) ); } PA_FORCE_INLINE __m256i convert_to_s16(__m256 x, float multiplier){ @@ -48,10 +50,10 @@ PA_FORCE_INLINE __m256i convert_to_s16(__m256 x, float multiplier){ x = _mm256_max_ps(x, _mm256_set1_ps(-32768.f)); return _mm256_cvtps_epi32(x); } -PA_FORCE_INLINE __m256 convert_from_s32(__m256i x){ +PA_FORCE_INLINE __m256 convert_from_s32(__m256i x, float multiplier){ return _mm256_mul_ps( _mm256_cvtepi32_ps(x), - _mm256_set1_ps(1.0f / 2147483648.0f) + _mm256_set1_ps(multiplier * (1.0f / 2147483648.0f)) ); } PA_FORCE_INLINE __m256i convert_to_s32(__m256 x, float multiplier){ @@ -64,26 +66,26 @@ PA_FORCE_INLINE __m256i convert_to_s32(__m256 x, float multiplier){ template -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_t samples, float multiplier); template <> -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 8){ __m256i x = _mm256_cvtepu8_epi32( _mm_loadl_epi64((const __m128i*)ext) ); - _mm256_storeu_ps(native, convert_from_u8(x)); + _mm256_storeu_ps(native, convert_from_u8(x, multiplier)); native += 8; ext += 8; samples -= 8; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -112,19 +114,19 @@ void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_ } } template <> -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 8){ __m256i x = _mm256_cvtepi16_epi32( _mm_loadu_si128((const __m128i*)ext) ); - _mm256_storeu_ps(native, convert_from_s16(x)); + _mm256_storeu_ps(native, convert_from_s16(x, multiplier)); native += 8; ext += 8; samples -= 8; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -153,17 +155,17 @@ void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_ } } template <> -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 8){ __m256i x = _mm256_loadu_si256((const __m256i*)ext); - _mm256_storeu_ps(native, convert_from_s32(x)); + _mm256_storeu_ps(native, convert_from_s32(x, multiplier)); native += 8; ext += 8; samples -= 8; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -190,37 +192,45 @@ void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_ template class Test_from_x64_AVX2 : public Test{ public: - Test_from_x64_AVX2(size_t samples) + Test_from_x64_AVX2(size_t samples, float from_multiplier, float to_multiplier) : Test( "from_x64_AVX2", samples, (ConvertFrom)convert_from_x64_AVX2, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; template class Test_to_x64_AVX2 : public Test{ public: - Test_to_x64_AVX2(size_t samples) + Test_to_x64_AVX2(size_t samples, float from_multiplier, float to_multiplier) : Test( "to_x64_AVX2", samples, (ConvertFrom)convert_from_Default, - (ConvertTo)convert_to_x64_AVX2 + (ConvertTo)convert_to_x64_AVX2, + from_multiplier, + to_multiplier ) {} }; void add_tests_x64_AVX2(UnitTestDatabase& database){ + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp index ca95dde97a..08e1671266 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp @@ -22,24 +22,26 @@ namespace AudioStreamConverter{ -PA_FORCE_INLINE __m512 convert_from_u8(__m512i x){ +PA_FORCE_INLINE __m512 convert_from_u8(__m512i x, float multiplier){ return _mm512_fmadd_ps( _mm512_cvtepi32_ps(x), - _mm512_set1_ps(1.f / 128.f), - _mm512_set1_ps(-1.f) + _mm512_set1_ps(multiplier * (1.f / 128.f)), + _mm512_set1_ps(-multiplier) ); } PA_FORCE_INLINE __m512i convert_to_u8(__m512 x, float multiplier){ - const __m512 SCALE = _mm512_set1_ps(multiplier * 128.f); + // Scale first, then clamp: a loud signal with gain clips to the + // uint8 range, it does not scale past it. + x = _mm512_mul_ps(x, _mm512_set1_ps(multiplier)); x = _mm512_min_ps(x, _mm512_set1_ps(127.0f / 128.0f)); x = _mm512_max_ps(x, _mm512_set1_ps(-1.0f)); - x = _mm512_fmadd_ps(x, SCALE, SCALE); + x = _mm512_fmadd_ps(x, _mm512_set1_ps(128.0f), _mm512_set1_ps(128.5f)); return _mm512_cvtps_epi32(x); } -PA_FORCE_INLINE __m512 convert_from_s16(__m512i x){ +PA_FORCE_INLINE __m512 convert_from_s16(__m512i x, float multiplier){ return _mm512_mul_ps( _mm512_cvtepi32_ps(x), - _mm512_set1_ps(1.0f / 32768.0f) + _mm512_set1_ps(multiplier * (1.0f / 32768.0f)) ); } PA_FORCE_INLINE __m512i convert_to_s16(__m512 x, float multiplier){ @@ -48,10 +50,10 @@ PA_FORCE_INLINE __m512i convert_to_s16(__m512 x, float multiplier){ x = _mm512_max_ps(x, _mm512_set1_ps(-32768.f)); return _mm512_cvtps_epi32(x); } -PA_FORCE_INLINE __m512 convert_from_s32(__m512i x){ +PA_FORCE_INLINE __m512 convert_from_s32(__m512i x, float multiplier){ return _mm512_mul_ps( _mm512_cvtepi32_ps(x), - _mm512_set1_ps(1.0f / 2147483648.0f) + _mm512_set1_ps(multiplier * (1.0f / 2147483648.0f)) ); } PA_FORCE_INLINE __m512i convert_to_s32(__m512 x, float multiplier){ @@ -64,7 +66,7 @@ PA_FORCE_INLINE __m512i convert_to_s32(__m512 x, float multiplier){ template -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_AVX512(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_x64_AVX512(UnitTestDatabase& database); @@ -72,12 +74,12 @@ void add_tests_x64_AVX512(UnitTestDatabase& database); template <> -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples > 16){ __m512i x = _mm512_cvtepu8_epi32( _mm_loadu_si128((const __m128i*)ext) ); - _mm512_storeu_ps(native, convert_from_u8(x)); + _mm512_storeu_ps(native, convert_from_u8(x, multiplier)); native += 16; ext += 16; @@ -88,7 +90,7 @@ void convert_from_x64_AVX512(r_ptr native, c_ptr ext, s __m512i i = _mm512_cvtepu8_epi32( _mm_maskz_loadu_epi8(mask, (const __m128i*)ext) ); - _mm512_mask_storeu_ps(native, mask, convert_from_u8(i)); + _mm512_mask_storeu_ps(native, mask, convert_from_u8(i, multiplier)); } } template <> @@ -114,12 +116,12 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz } } template <> -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples > 16){ __m512i x = _mm512_cvtepi16_epi32( _mm256_loadu_si256((const __m256i*)ext) ); - _mm512_storeu_ps(native, convert_from_s16(x)); + _mm512_storeu_ps(native, convert_from_s16(x, multiplier)); native += 16; ext += 16; @@ -130,7 +132,7 @@ void convert_from_x64_AVX512(r_ptr native, c_ptr ext, s __m512i x = _mm512_cvtepi16_epi32( _mm256_maskz_loadu_epi16(mask, ext) ); - _mm512_mask_storeu_ps(native, mask, convert_from_s16(x)); + _mm512_mask_storeu_ps(native, mask, convert_from_s16(x, multiplier)); } } template <> @@ -156,10 +158,10 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz } } template <> -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples > 16){ __m512i x = _mm512_loadu_si512(ext); - _mm512_storeu_ps(native, convert_from_s32(x)); + _mm512_storeu_ps(native, convert_from_s32(x, multiplier)); native += 16; ext += 16; @@ -168,7 +170,7 @@ void convert_from_x64_AVX512(r_ptr native, c_ptr ext, s { __mmask16 mask = ((uint32_t)1 << samples) - 1; __m512i x = _mm512_maskz_loadu_epi32(mask, ext); - _mm512_mask_storeu_ps(native, mask, convert_from_s32(x)); + _mm512_mask_storeu_ps(native, mask, convert_from_s32(x, multiplier)); } } template <> @@ -197,37 +199,45 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz template class Test_from_x64_AVX512 : public Test{ public: - Test_from_x64_AVX512(size_t samples) + Test_from_x64_AVX512(size_t samples, float from_multiplier, float to_multiplier) : Test( "from_x64_AVX512", samples, (ConvertFrom)convert_from_x64_AVX512, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; template class Test_to_x64_AVX512 : public Test{ public: - Test_to_x64_AVX512(size_t samples) + Test_to_x64_AVX512(size_t samples, float from_multiplier, float to_multiplier) : Test( "to_x64_AVX512", samples, (ConvertFrom)convert_from_Default, - (ConvertTo)convert_to_x64_AVX512 + (ConvertTo)convert_to_x64_AVX512, + from_multiplier, + to_multiplier ) {} }; void add_tests_x64_AVX512(UnitTestDatabase& database){ + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp index 42b9c4e3a3..c0cea7eba5 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp @@ -23,26 +23,28 @@ namespace AudioStreamConverter{ -PA_FORCE_INLINE __m128 convert_from_u8(__m128i x){ +PA_FORCE_INLINE __m128 convert_from_u8(__m128i x, float multiplier){ return _mm_add_ps( _mm_mul_ps( _mm_cvtepi32_ps(x), - _mm_set1_ps(1.f / 128.f) + _mm_set1_ps(multiplier * (1.f / 128.f)) ), - _mm_set1_ps(-1.f) + _mm_set1_ps(-multiplier) ); } PA_FORCE_INLINE __m128i convert_to_u8(__m128 x, float multiplier){ - const __m128 SCALE = _mm_set1_ps(multiplier * 128.f); + // Scale first, then clamp: a loud signal with gain clips to the + // uint8 range, it does not scale past it. + x = _mm_mul_ps(x, _mm_set1_ps(multiplier)); x = _mm_min_ps(x, _mm_set1_ps(127.0f / 128.0f)); x = _mm_max_ps(x, _mm_set1_ps(-1.0f)); - x = _mm_add_ps(_mm_mul_ps(x, SCALE), SCALE); + x = _mm_add_ps(_mm_mul_ps(x, _mm_set1_ps(128.0f)), _mm_set1_ps(128.5f)); return _mm_cvtps_epi32(x); } -PA_FORCE_INLINE __m128 convert_from_s16(__m128i x){ +PA_FORCE_INLINE __m128 convert_from_s16(__m128i x, float multiplier){ return _mm_mul_ps( _mm_cvtepi32_ps(x), - _mm_set1_ps(1.0f / 32768.0f) + _mm_set1_ps(multiplier * (1.0f / 32768.0f)) ); } PA_FORCE_INLINE __m128i convert_to_s16(__m128 x, float multiplier){ @@ -51,10 +53,10 @@ PA_FORCE_INLINE __m128i convert_to_s16(__m128 x, float multiplier){ x = _mm_max_ps(x, _mm_set1_ps(-32768.f)); return _mm_cvtps_epi32(x); } -PA_FORCE_INLINE __m128 convert_from_s32(__m128i x){ +PA_FORCE_INLINE __m128 convert_from_s32(__m128i x, float multiplier){ return _mm_mul_ps( _mm_cvtepi32_ps(x), - _mm_set1_ps(1.0f / 2147483648.0f) + _mm_set1_ps(multiplier * (1.0f / 2147483648.0f)) ); } PA_FORCE_INLINE __m128i convert_to_s32(__m128 x, float multiplier){ @@ -67,26 +69,26 @@ PA_FORCE_INLINE __m128i convert_to_s32(__m128 x, float multiplier){ template -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size_t samples, float multiplier); template <> -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ __m128i x = _mm_cvtepu8_epi32( _mm_loadl_epi64((const __m128i*)ext) ); - _mm_storeu_ps(native, convert_from_u8(x)); + _mm_storeu_ps(native, convert_from_u8(x, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -107,19 +109,19 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size } } template <> -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ __m128i x = _mm_cvtepi16_epi32( _mm_loadu_si128((const __m128i*)ext) ); - _mm_storeu_ps(native, convert_from_s16(x)); + _mm_storeu_ps(native, convert_from_s16(x, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -140,17 +142,17 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size } } template <> -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ __m128i x = _mm_loadu_si128((const __m128i*)ext); - _mm_storeu_ps(native, convert_from_s32(x)); + _mm_storeu_ps(native, convert_from_s32(x, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -177,37 +179,45 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size template class Test_from_x64_SSE41 : public Test{ public: - Test_from_x64_SSE41(size_t samples) + Test_from_x64_SSE41(size_t samples, float from_multiplier, float to_multiplier) : Test( "from_x64_SSE41", samples, (ConvertFrom)convert_from_x64_SSE41, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; template class Test_to_x64_SSE41 : public Test{ public: - Test_to_x64_SSE41(size_t samples) + Test_to_x64_SSE41(size_t samples, float from_multiplier, float to_multiplier) : Test( "to_x64_SSE41", samples, (ConvertFrom)convert_from_Default, - (ConvertTo)convert_to_x64_SSE41 + (ConvertTo)convert_to_x64_SSE41, + from_multiplier, + to_multiplier ) {} }; void add_tests_x64_SSE41(UnitTestDatabase& database){ + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp index e2084c0805..f33d166041 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp @@ -6,6 +6,7 @@ #include #include +#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Containers/AlignedVector.tpp" #include "Kernels_AudioStreamConverter_TestInfra.h" @@ -22,15 +23,30 @@ Test::Test( const std::string& arch_str, size_t samples, ConvertFrom convert_from, - ConvertTo convert_to + ConvertTo convert_to, + float from_multiplier, + float to_multiplier ) : UnitTest( - "Kernels::AudioStreamConverter - " + typestr() + "_" + arch_str + - "(" + std::to_string(samples) + ")" + [&]{ + std::string name = "Kernels::AudioStreamConverter - " + typestr() + "_" + arch_str + + "(" + std::to_string(samples) + ")"; + if (from_multiplier != 1.0f || to_multiplier != 1.0f){ + char mults[64]; + std::snprintf( + mults, sizeof(mults), " [from %.4g / to %.4g]", + from_multiplier, to_multiplier + ); + name += mults; + } + return name; + }() ) , m_samples(samples) , m_convert_from(convert_from) , m_convert_to(convert_to) + , m_from_multiplier(from_multiplier) + , m_to_multiplier(to_multiplier) {} template UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ @@ -62,8 +78,8 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ } } - m_convert_from(native.data(), ext0.data(), m_samples); - m_convert_to(native.data(), ext1.data(), m_samples, 1.0f); + m_convert_from(native.data(), ext0.data(), m_samples, m_from_multiplier); + m_convert_to(native.data(), ext1.data(), m_samples, m_to_multiplier); // Check for errors. for (size_t c = 0; c < m_samples; c++){ diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h index 0ae855da91..6a43684f22 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h @@ -20,7 +20,7 @@ namespace AudioStreamConverter{ template -using ConvertFrom = void (*)(r_ptr native, c_ptr ext, size_t samples); +using ConvertFrom = void (*)(r_ptr native, c_ptr ext, size_t samples, float multiplier); template using ConvertTo = void (*)(c_ptr native, r_ptr ext, size_t samples, float multiplier); @@ -42,7 +42,9 @@ class Test : public UnitTest{ const std::string& arch_str, size_t samples, ConvertFrom convert_from, - ConvertTo convert_to + ConvertTo convert_to, + float from_multiplier = 1.0f, + float to_multiplier = 1.0f ); virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override; @@ -53,6 +55,8 @@ class Test : public UnitTest{ size_t m_samples; ConvertFrom m_convert_from; ConvertTo m_convert_to; + float m_from_multiplier; + float m_to_multiplier; }; From 50c33df9588f923442747b76f95788060d4248ad Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:55:38 +0000 Subject: [PATCH 2/7] AudioStreamConverter: build the multiplier test names with PrettyPrint Use tostr_default() instead of a local snprintf buffer, and move the name construction out of a lambda in the member initializer into a private static make_name() method on the test classes. --- ...nels_AudioPipeline_RoundTrip_TestInfra.cpp | 32 +++++++++--------- ...ernels_AudioPipeline_RoundTrip_TestInfra.h | 7 ++++ ...Kernels_AudioStreamConverter_TestInfra.cpp | 33 ++++++++++--------- .../Kernels_AudioStreamConverter_TestInfra.h | 6 ++++ 4 files changed, 46 insertions(+), 32 deletions(-) diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp index 29f40109a3..37f68dbc03 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp @@ -4,12 +4,12 @@ * */ -#include #include #include #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Containers/AlignedVector.tpp" +#include "Common/Cpp/PrettyPrint.h" #include "Kernels_AudioPipeline_RoundTrip_TestInfra.h" #include "Kernels_RawAudio2WorkingAudio.h" #include "Kernels_WorkingAudio2RawAudio.h" @@ -142,26 +142,26 @@ RoundTripTest::RoundTripTest( size_t samples, float from_multiplier ) - : UnitTest( - [&]{ - std::string name = "Kernels::AudioPipeline - RoundTrip_" + sample_typestr(sample_type) + - "(" + std::to_string(channels) + "ch x " + std::to_string(samples) + ")"; - if (from_multiplier != 1.0f){ - char mults[64]; - std::snprintf( - mults, sizeof(mults), " [from %.4g / to %.4g]", - from_multiplier, 1.0f / from_multiplier - ); - name += mults; - } - return name; - }() - ) + : UnitTest(make_name(sample_type, channels, samples, from_multiplier)) , m_sample_type(sample_type) , m_channels(channels) , m_samples(samples) , m_from_multiplier(from_multiplier) {} +std::string RoundTripTest::make_name( + SampleFormat sample_type, + size_t channels, + size_t samples, + float from_multiplier +){ + std::string name = "Kernels::AudioPipeline - RoundTrip_" + sample_typestr(sample_type) + + "(" + std::to_string(channels) + "ch x " + std::to_string(samples) + ")"; + if (from_multiplier != 1.0f){ + name += " [from " + tostr_default(from_multiplier) + + " / to " + tostr_default(1.0f / from_multiplier) + "]"; + } + return name; +} UnitTestResult RoundTripTest::run(Logger& logger, CancellableScope& scope) const{ switch (m_sample_type){ case SampleFormat::UINT8: diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h index cb328aecbc..1f6d57699f 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h @@ -37,6 +37,13 @@ class RoundTripTest : public UnitTest{ private: + static std::string make_name( + SampleFormat sample_type, + size_t channels, + size_t samples, + float from_multiplier + ); + const SampleFormat m_sample_type; const size_t m_channels; const size_t m_samples; diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp index f33d166041..57aff2ca86 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp @@ -6,9 +6,9 @@ #include #include -#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Containers/AlignedVector.tpp" +#include "Common/Cpp/PrettyPrint.h" #include "Kernels_AudioStreamConverter_TestInfra.h" namespace PokemonAutomation{ @@ -27,21 +27,7 @@ Test::Test( float from_multiplier, float to_multiplier ) - : UnitTest( - [&]{ - std::string name = "Kernels::AudioStreamConverter - " + typestr() + "_" + arch_str + - "(" + std::to_string(samples) + ")"; - if (from_multiplier != 1.0f || to_multiplier != 1.0f){ - char mults[64]; - std::snprintf( - mults, sizeof(mults), " [from %.4g / to %.4g]", - from_multiplier, to_multiplier - ); - name += mults; - } - return name; - }() - ) + : UnitTest(make_name(arch_str, samples, from_multiplier, to_multiplier)) , m_samples(samples) , m_convert_from(convert_from) , m_convert_to(convert_to) @@ -92,6 +78,21 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ return true; } template +std::string Test::make_name( + const std::string& arch_str, + size_t samples, + float from_multiplier, + float to_multiplier +){ + std::string name = "Kernels::AudioStreamConverter - " + typestr() + "_" + arch_str + + "(" + std::to_string(samples) + ")"; + if (from_multiplier != 1.0f || to_multiplier != 1.0f){ + name += " [from " + tostr_default(from_multiplier) + + " / to " + tostr_default(to_multiplier) + "]"; + } + return name; +} +template std::string Test::typestr(){ if constexpr (std::is_same_v){ return "u8"; diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h index 6a43684f22..a540ee50c2 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h @@ -52,6 +52,12 @@ class Test : public UnitTest{ static std::string typestr(); private: + static std::string make_name( + const std::string& arch_str, + size_t samples, + float from_multiplier, + float to_multiplier + ); size_t m_samples; ConvertFrom m_convert_from; ConvertTo m_convert_to; From d0f5bca1701d64f0fe33fb5cbfc9b5c6311a341a Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 18 Oct 2026 07:06:21 +0000 Subject: [PATCH 3/7] Add temporary u8 diagnostic test (remove before merge) --- .../Kernels_AudioPipeline_Tests.cpp | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp index a8ff4a9463..b21672c95c 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp @@ -8,6 +8,14 @@ #include "Kernels_AudioPipeline.h" #include "Kernels_AudioPipeline_RoundTrip_TestInfra.h" #include "Kernels_ChannelMixer_TestInfra.h" +#include +#include +#include +#include "Common/Cpp/Containers/AlignedVector.tpp" +#include "Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h" +#include "Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h" +#include "Kernels_RawAudio2WorkingAudio.h" +#include "Kernels_WorkingAudio2RawAudio.h" namespace PokemonAutomation{ namespace Kernels{ @@ -16,6 +24,70 @@ namespace AudioPipeline{ + + +namespace{ + +// TEMPORARY CI DIAGNOSTIC (remove before merge): reports bit-exact +// intermediate values for the smallest u8 round trip. +class DiagU8Test : public UnitTest{ +public: + DiagU8Test() + : UnitTest("Kernels::AudioPipeline - DIAG_U8(1ch x 2)") + {} + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + using Kernels::AudioStreamConverter::convert_from_Default; + using Kernels::AudioStreamConverter::convert_to_Default; + + auto hexf = [](float f){ + uint32_t bits; + std::memcpy(&bits, &f, 4); + char buf[40]; + std::snprintf(buf, sizeof(buf), "%g(0x%08x)", (double)f, bits); + return std::string(buf); + }; + + AlignedVector ext0(2), ext1(2), ext2(2); + AlignedVector native(2), native2(2); + ext0[0] = 95; + ext0[1] = 203; + + convert_from_Default(native.data(), ext0.data(), 2, 1.0f); + convert_to_Default(native.data(), ext1.data(), 2, 1.0f); + Kernels::AudioStreamConverter::convert_from(native2.data(), ext0.data(), 2, 1.0f); + Kernels::AudioStreamConverter::convert_to(native2.data(), ext2.data(), 2, 1.0f); + + // Full pipeline, 2 samples. + RawAudio2WorkingAudio raw2working(SampleFormat::UINT8, 44100, 1, 1.0f, 256); + WorkingAudio2RawAudio working2raw(SampleFormat::UINT8, 44100, 1, 1.0f, 256); + AlignedVector pipeline_out(2); + struct L : RawAudioStreamListener{ + AlignedVector& data; + L(SampleFormat t, size_t r, size_t c, AlignedVector& p) + : RawAudioStreamListener(t, r, c), data(p) + {} + virtual void push_samples(size_t frames, const void* p) override{ + const uint8_t* in = (const uint8_t*)p; + for (size_t i = 0; i < frames; i++){ + data[i] = in[i]; + } + } + } listener(SampleFormat::UINT8, 44100, 1, pipeline_out); + raw2working.add_listener(working2raw); + working2raw.add_listener(listener); + raw2working.push_samples(2, ext0.data()); + + return "DIAG from_default[0]=" + hexf(native[0]) + + " to_default[0]=" + std::to_string(ext1[0]) + + " from_dispatched[0]=" + hexf(native2[0]) + + " to_dispatched[0]=" + std::to_string(ext2[0]) + + " pipeline[0]=" + std::to_string(pipeline_out[0]) + + " pipeline[1]=" + std::to_string(pipeline_out[1]); + } +}; + +} + void add_tests(UnitTestDatabase& database){ // Raw -> Working -> Raw round-trip, with reciprocal multipliers on // the two nodes (exact in binary) to exercise the gain path. @@ -31,6 +103,8 @@ void add_tests(UnitTestDatabase& database){ } } + database.add(); + // ChannelMixer with an identity weight matrix. for (size_t channels = 1; channels <= 4; channels++){ database.add(channels, 1000); From 1099dd1430ca0687e1e3457208154bbedcd93e9b Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 18 Oct 2026 07:24:23 +0000 Subject: [PATCH 4/7] Expand temporary u8 diagnostic (8-sample dispatcher round trip + CPU caps) --- .../Kernels_AudioPipeline_Tests.cpp | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp index b21672c95c..6b2c5cbbe5 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp @@ -16,6 +16,7 @@ #include "Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h" #include "Kernels_RawAudio2WorkingAudio.h" #include "Kernels_WorkingAudio2RawAudio.h" +#include "Common/Cpp/CpuId/CpuId.h" namespace PokemonAutomation{ namespace Kernels{ @@ -77,12 +78,42 @@ class DiagU8Test : public UnitTest{ working2raw.add_listener(listener); raw2working.push_samples(2, ext0.data()); + // 8-sample dispatcher round trip (one full SIMD iteration). + const uint8_t eight[8] = { 0, 255, 95, 203, 7, 250, 128, 33 }; + AlignedVector n8(8), n8b(8); + AlignedVector e8(8); + Kernels::AudioStreamConverter::convert_from(n8.data(), eight, 8, 1.0f); + Kernels::AudioStreamConverter::convert_to(n8.data(), e8.data(), 8, 1.0f); + std::string eightstr = " eight_out="; + for (size_t i = 0; i < 8; i++){ + if (i > 0){ + eightstr += " "; + } + eightstr += std::to_string(e8[i]); + } + std::string n8str = " n8="; + for (size_t i = 0; i < 4; i++){ + if (i > 0){ + n8str += " "; + } + n8str += hexf(n8[i]); + } + +#if defined(PA_ARCH_x86) + std::string capstr = " caps[nehalem=" + std::to_string(CPU_CAPABILITY_CURRENT.OK_08_Nehalem) + + " haswell=" + std::to_string(CPU_CAPABILITY_CURRENT.OK_13_Haswell) + + " skylake=" + std::to_string(CPU_CAPABILITY_CURRENT.OK_17_Skylake) + "]"; +#else + std::string capstr = " caps[non-x86]"; +#endif + return "DIAG from_default[0]=" + hexf(native[0]) + " to_default[0]=" + std::to_string(ext1[0]) + " from_dispatched[0]=" + hexf(native2[0]) + " to_dispatched[0]=" + std::to_string(ext2[0]) + " pipeline[0]=" + std::to_string(pipeline_out[0]) + - " pipeline[1]=" + std::to_string(pipeline_out[1]); + " pipeline[1]=" + std::to_string(pipeline_out[1]) + + n8str + eightstr + capstr; } }; From 0ab456db1958442bdc06728badc32cb562907da5 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 18 Oct 2026 09:50:39 +0000 Subject: [PATCH 5/7] Fix x64 u8/s16/s32 to-conversions: use truncating cvt intrinsics _xmm_cvtps_epi32 / _mm256_cvtps_epi32 / _mm512_cvtps_epi32 are the rounding (CVTPS2DQ) conversions, not the truncating ones; the scalar Default kernels use C casts (truncation). With the new scale-then-clamp u8 path the final values land on k+0.5, where the rounding conversion diverges from the scalar (round(95.5)=96). Use the _cvttps_ (CVTTPS2DQ) truncating intrinsics in all nine x64 to-helpers. Also: - AVX512 u8 to: _mm512_cvtepi32_epi8 (VPMOVQB) saturates int32 lane values above 127, clamping the upper half of the uint8 range. Pack via a 128 bias into signed byte range, then add it back in byte lanes (mod 256). - SSE41 u8/s16 from: the SIMD iterations read a full 128-bit line while only consuming 4 samples, reading past the end of short buffers. Restructure to full-width iterations (16 u8 / 8 s16 samples) with the scalar tail handling the rest. - Remove the temporary DIAG_U8 test. --- .../Kernels_AudioPipeline_Tests.cpp | 106 ------------------ ..._AudioStreamConverter_Kernels_x64_AVX2.cpp | 6 +- ...udioStreamConverter_Kernels_x64_AVX512.cpp | 18 ++- ...AudioStreamConverter_Kernels_x64_SSE41.cpp | 54 +++++---- 4 files changed, 51 insertions(+), 133 deletions(-) diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp index 6b2c5cbbe5..39e3b19ed3 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp @@ -8,15 +8,6 @@ #include "Kernels_AudioPipeline.h" #include "Kernels_AudioPipeline_RoundTrip_TestInfra.h" #include "Kernels_ChannelMixer_TestInfra.h" -#include -#include -#include -#include "Common/Cpp/Containers/AlignedVector.tpp" -#include "Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h" -#include "Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h" -#include "Kernels_RawAudio2WorkingAudio.h" -#include "Kernels_WorkingAudio2RawAudio.h" -#include "Common/Cpp/CpuId/CpuId.h" namespace PokemonAutomation{ namespace Kernels{ @@ -24,101 +15,6 @@ namespace AudioPipeline{ - - - -namespace{ - -// TEMPORARY CI DIAGNOSTIC (remove before merge): reports bit-exact -// intermediate values for the smallest u8 round trip. -class DiagU8Test : public UnitTest{ -public: - DiagU8Test() - : UnitTest("Kernels::AudioPipeline - DIAG_U8(1ch x 2)") - {} - virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ - using Kernels::AudioStreamConverter::convert_from_Default; - using Kernels::AudioStreamConverter::convert_to_Default; - - auto hexf = [](float f){ - uint32_t bits; - std::memcpy(&bits, &f, 4); - char buf[40]; - std::snprintf(buf, sizeof(buf), "%g(0x%08x)", (double)f, bits); - return std::string(buf); - }; - - AlignedVector ext0(2), ext1(2), ext2(2); - AlignedVector native(2), native2(2); - ext0[0] = 95; - ext0[1] = 203; - - convert_from_Default(native.data(), ext0.data(), 2, 1.0f); - convert_to_Default(native.data(), ext1.data(), 2, 1.0f); - Kernels::AudioStreamConverter::convert_from(native2.data(), ext0.data(), 2, 1.0f); - Kernels::AudioStreamConverter::convert_to(native2.data(), ext2.data(), 2, 1.0f); - - // Full pipeline, 2 samples. - RawAudio2WorkingAudio raw2working(SampleFormat::UINT8, 44100, 1, 1.0f, 256); - WorkingAudio2RawAudio working2raw(SampleFormat::UINT8, 44100, 1, 1.0f, 256); - AlignedVector pipeline_out(2); - struct L : RawAudioStreamListener{ - AlignedVector& data; - L(SampleFormat t, size_t r, size_t c, AlignedVector& p) - : RawAudioStreamListener(t, r, c), data(p) - {} - virtual void push_samples(size_t frames, const void* p) override{ - const uint8_t* in = (const uint8_t*)p; - for (size_t i = 0; i < frames; i++){ - data[i] = in[i]; - } - } - } listener(SampleFormat::UINT8, 44100, 1, pipeline_out); - raw2working.add_listener(working2raw); - working2raw.add_listener(listener); - raw2working.push_samples(2, ext0.data()); - - // 8-sample dispatcher round trip (one full SIMD iteration). - const uint8_t eight[8] = { 0, 255, 95, 203, 7, 250, 128, 33 }; - AlignedVector n8(8), n8b(8); - AlignedVector e8(8); - Kernels::AudioStreamConverter::convert_from(n8.data(), eight, 8, 1.0f); - Kernels::AudioStreamConverter::convert_to(n8.data(), e8.data(), 8, 1.0f); - std::string eightstr = " eight_out="; - for (size_t i = 0; i < 8; i++){ - if (i > 0){ - eightstr += " "; - } - eightstr += std::to_string(e8[i]); - } - std::string n8str = " n8="; - for (size_t i = 0; i < 4; i++){ - if (i > 0){ - n8str += " "; - } - n8str += hexf(n8[i]); - } - -#if defined(PA_ARCH_x86) - std::string capstr = " caps[nehalem=" + std::to_string(CPU_CAPABILITY_CURRENT.OK_08_Nehalem) + - " haswell=" + std::to_string(CPU_CAPABILITY_CURRENT.OK_13_Haswell) + - " skylake=" + std::to_string(CPU_CAPABILITY_CURRENT.OK_17_Skylake) + "]"; -#else - std::string capstr = " caps[non-x86]"; -#endif - - return "DIAG from_default[0]=" + hexf(native[0]) + - " to_default[0]=" + std::to_string(ext1[0]) + - " from_dispatched[0]=" + hexf(native2[0]) + - " to_dispatched[0]=" + std::to_string(ext2[0]) + - " pipeline[0]=" + std::to_string(pipeline_out[0]) + - " pipeline[1]=" + std::to_string(pipeline_out[1]) + - n8str + eightstr + capstr; - } -}; - -} - void add_tests(UnitTestDatabase& database){ // Raw -> Working -> Raw round-trip, with reciprocal multipliers on // the two nodes (exact in binary) to exercise the gain path. @@ -134,8 +30,6 @@ void add_tests(UnitTestDatabase& database){ } } - database.add(); - // ChannelMixer with an identity weight matrix. for (size_t channels = 1; channels <= 4; channels++){ database.add(channels, 1000); diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp index ce3c612d97..ac880e782f 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp @@ -36,7 +36,7 @@ PA_FORCE_INLINE __m256i convert_to_u8(__m256 x, float multiplier){ x = _mm256_min_ps(x, _mm256_set1_ps(127.0f / 128.0f)); x = _mm256_max_ps(x, _mm256_set1_ps(-1.0f)); x = _mm256_fmadd_ps(x, _mm256_set1_ps(128.0f), _mm256_set1_ps(128.5f)); - return _mm256_cvtps_epi32(x); + return _mm256_cvttps_epi32(x); } PA_FORCE_INLINE __m256 convert_from_s16(__m256i x, float multiplier){ return _mm256_mul_ps( @@ -48,7 +48,7 @@ PA_FORCE_INLINE __m256i convert_to_s16(__m256 x, float multiplier){ x = _mm256_mul_ps(x, _mm256_set1_ps(multiplier * 32768.f)); x = _mm256_min_ps(x, _mm256_set1_ps(32767.f)); x = _mm256_max_ps(x, _mm256_set1_ps(-32768.f)); - return _mm256_cvtps_epi32(x); + return _mm256_cvttps_epi32(x); } PA_FORCE_INLINE __m256 convert_from_s32(__m256i x, float multiplier){ return _mm256_mul_ps( @@ -60,7 +60,7 @@ PA_FORCE_INLINE __m256i convert_to_s32(__m256 x, float multiplier){ x = _mm256_mul_ps(x, _mm256_set1_ps(multiplier * 2147483648.f)); x = _mm256_min_ps(x, _mm256_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm256_max_ps(x, _mm256_set1_ps(-2147483648.f)); - return _mm256_cvtps_epi32(x); + return _mm256_cvttps_epi32(x); } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp index 08e1671266..0f50b116ec 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp @@ -36,7 +36,15 @@ PA_FORCE_INLINE __m512i convert_to_u8(__m512 x, float multiplier){ x = _mm512_min_ps(x, _mm512_set1_ps(127.0f / 128.0f)); x = _mm512_max_ps(x, _mm512_set1_ps(-1.0f)); x = _mm512_fmadd_ps(x, _mm512_set1_ps(128.0f), _mm512_set1_ps(128.5f)); - return _mm512_cvtps_epi32(x); + return _mm512_cvttps_epi32(x); +} +// VPMOVQB saturates int32 lane values above 127, which would clamp the +// upper half of the uint8 range. Shift the [0, 255] values into signed +// byte range first, then add the 128 back in byte lanes (modulo 256). +PA_FORCE_INLINE __m128i pack_u8_bytes(__m512i x){ + x = _mm512_sub_epi32(x, _mm512_set1_epi32(128)); + __m128i y = _mm512_cvtepi32_epi8(x); + return _mm_add_epi8(y, _mm_set1_epi8(128)); } PA_FORCE_INLINE __m512 convert_from_s16(__m512i x, float multiplier){ return _mm512_mul_ps( @@ -48,7 +56,7 @@ PA_FORCE_INLINE __m512i convert_to_s16(__m512 x, float multiplier){ x = _mm512_mul_ps(x, _mm512_set1_ps(multiplier * 32768.f)); x = _mm512_min_ps(x, _mm512_set1_ps(32767.f)); x = _mm512_max_ps(x, _mm512_set1_ps(-32768.f)); - return _mm512_cvtps_epi32(x); + return _mm512_cvttps_epi32(x); } PA_FORCE_INLINE __m512 convert_from_s32(__m512i x, float multiplier){ return _mm512_mul_ps( @@ -60,7 +68,7 @@ PA_FORCE_INLINE __m512i convert_to_s32(__m512 x, float multiplier){ x = _mm512_mul_ps(x, _mm512_set1_ps(multiplier * 2147483648.f)); x = _mm512_min_ps(x, _mm512_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm512_max_ps(x, _mm512_set1_ps(-2147483648.f)); - return _mm512_cvtps_epi32(x); + return _mm512_cvttps_epi32(x); } @@ -99,7 +107,7 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz __m512 x = _mm512_loadu_ps(native); _mm_storeu_si128( (__m128i*)ext, - _mm512_cvtepi32_epi8(convert_to_u8(x, multiplier)) + pack_u8_bytes(convert_to_u8(x, multiplier)) ); native += 16; @@ -111,7 +119,7 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz __m512 x = _mm512_maskz_loadu_ps(mask, native); _mm_mask_storeu_epi8( (__m128i*)ext, mask, - _mm512_cvtepi32_epi8(convert_to_u8(x, multiplier)) + pack_u8_bytes(convert_to_u8(x, multiplier)) ); } } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp index c0cea7eba5..1161d0bbd8 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp @@ -39,7 +39,7 @@ PA_FORCE_INLINE __m128i convert_to_u8(__m128 x, float multiplier){ x = _mm_min_ps(x, _mm_set1_ps(127.0f / 128.0f)); x = _mm_max_ps(x, _mm_set1_ps(-1.0f)); x = _mm_add_ps(_mm_mul_ps(x, _mm_set1_ps(128.0f)), _mm_set1_ps(128.5f)); - return _mm_cvtps_epi32(x); + return _mm_cvttps_epi32(x); } PA_FORCE_INLINE __m128 convert_from_s16(__m128i x, float multiplier){ return _mm_mul_ps( @@ -51,7 +51,7 @@ PA_FORCE_INLINE __m128i convert_to_s16(__m128 x, float multiplier){ x = _mm_mul_ps(x, _mm_set1_ps(multiplier * 32768.f)); x = _mm_min_ps(x, _mm_set1_ps(32767.f)); x = _mm_max_ps(x, _mm_set1_ps(-32768.f)); - return _mm_cvtps_epi32(x); + return _mm_cvttps_epi32(x); } PA_FORCE_INLINE __m128 convert_from_s32(__m128i x, float multiplier){ return _mm_mul_ps( @@ -63,7 +63,7 @@ PA_FORCE_INLINE __m128i convert_to_s32(__m128 x, float multiplier){ x = _mm_mul_ps(x, _mm_set1_ps(multiplier * 2147483648.f)); x = _mm_min_ps(x, _mm_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm_max_ps(x, _mm_set1_ps(-2147483648.f)); - return _mm_cvtps_epi32(x); + return _mm_cvttps_epi32(x); } @@ -77,15 +77,26 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size_t s template <> void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ - while (samples >= 4){ - __m128i x = _mm_cvtepu8_epi32( - _mm_loadl_epi64((const __m128i*)ext) - ); - _mm_storeu_ps(native, convert_from_u8(x, multiplier)); + // 16 samples per iteration so that the 16-byte load stays in bounds; + // anything smaller is handled by the scalar tail. + while (samples >= 16){ + __m128i x = _mm_loadu_si128((const __m128i*)ext); + _mm_storeu_ps(native, convert_from_u8( + _mm_cvtepu8_epi32(x), multiplier + )); + _mm_storeu_ps(native + 4, convert_from_u8( + _mm_cvtepu8_epi32(_mm_srli_si128(x, 4)), multiplier + )); + _mm_storeu_ps(native + 8, convert_from_u8( + _mm_cvtepu8_epi32(_mm_srli_si128(x, 8)), multiplier + )); + _mm_storeu_ps(native + 12, convert_from_u8( + _mm_cvtepu8_epi32(_mm_srli_si128(x, 12)), multiplier + )); - native += 4; - ext += 4; - samples -= 4; + native += 16; + ext += 16; + samples -= 16; } if (samples > 0){ convert_from_Default(native, ext, samples, multiplier); @@ -110,15 +121,20 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size } template <> void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ - while (samples >= 4){ - __m128i x = _mm_cvtepi16_epi32( - _mm_loadu_si128((const __m128i*)ext) - ); - _mm_storeu_ps(native, convert_from_s16(x, multiplier)); + // 8 samples per iteration: a 4-sample iteration with a 16-byte load + // would read past the end of short buffers. + while (samples >= 8){ + __m128i x = _mm_loadu_si128((const __m128i*)ext); + _mm_storeu_ps(native, convert_from_s16( + _mm_cvtepi16_epi32(x), multiplier + )); + _mm_storeu_ps(native + 4, convert_from_s16( + _mm_cvtepi16_epi32(_mm_srli_si128(x, 8)), multiplier + )); - native += 4; - ext += 4; - samples -= 4; + native += 8; + ext += 8; + samples -= 8; } if (samples > 0){ convert_from_Default(native, ext, samples, multiplier); From 6410f350df2e38e5981cd4bc0ddade8ecc78807d Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:01:50 +0000 Subject: [PATCH 6/7] AVX512 u8 pack: _mm_set1_epi8(-128) instead of 128 GCC declares the argument as char; 128 overflows (-Werror=overflow). --- .../Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp index 0f50b116ec..6ce847d91e 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp @@ -44,7 +44,7 @@ PA_FORCE_INLINE __m512i convert_to_u8(__m512 x, float multiplier){ PA_FORCE_INLINE __m128i pack_u8_bytes(__m512i x){ x = _mm512_sub_epi32(x, _mm512_set1_epi32(128)); __m128i y = _mm512_cvtepi32_epi8(x); - return _mm_add_epi8(y, _mm_set1_epi8(128)); + return _mm_add_epi8(y, _mm_set1_epi8(-128)); } PA_FORCE_INLINE __m512 convert_from_s16(__m512i x, float multiplier){ return _mm512_mul_ps( From 86439ab70368731179df232f0e0e3f8b7d1b4985 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:18:14 +0000 Subject: [PATCH 7/7] Round s16/s32 to half away from zero; exact-width SSE41 loads; add to-vs-Default tests - The Default to-kernels round half away from zero (offset by +/-0.5, then C-cast). The x64 s16/s32 kernels now do the same instead of truncating raw values; the NEON s16/s32 kernels had the same pure truncation and get the offset as well. The u8 kernels already bake the 0.5 into the +128.5 shift, so they only needed the truncating cvt from the previous commit. - SSE41 from: read exactly the bytes of each 4-sample iteration with _mm_loadu_si32 (u8) and _mm_loadl_epi64 (s16) instead of 8/16-byte loads that read past the end of short buffers. - New ToArbitraryTest compares each ISA's to-kernel against Default on random float inputs (including clamps and out-of-range values). The round-trip tests only feed grid-aligned values, which cannot expose rounding differences; this test catches them (verified: it fails against the pre-fix truncating s16 kernel). --- ...udioStreamConverter_Kernels_ARM64_NEON.cpp | 32 ++++++++ ...s_AudioStreamConverter_Kernels_Default.cpp | 17 +++++ ..._AudioStreamConverter_Kernels_x64_AVX2.cpp | 26 +++++++ ...udioStreamConverter_Kernels_x64_AVX512.cpp | 26 +++++++ ...AudioStreamConverter_Kernels_x64_SSE41.cpp | 74 +++++++++++-------- ...Kernels_AudioStreamConverter_TestInfra.cpp | 61 +++++++++++++++ .../Kernels_AudioStreamConverter_TestInfra.h | 24 ++++++ 7 files changed, 228 insertions(+), 32 deletions(-) diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp index b6eba1706e..94afdab4eb 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp @@ -50,6 +50,13 @@ PA_FORCE_INLINE int16x4_t convert_to_s16(float32x4_t x, float multiplier){ x = vmulq_f32(x, vdupq_n_f32(multiplier * 32768.f)); x = vminq_f32(x, vdupq_n_f32(32767.f)); x = vmaxq_f32(x, vdupq_n_f32(-32768.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + x = vaddq_f32(x, vbslq_f32( + vcgtq_f32(x, vdupq_n_f32(0.0f)), + vdupq_n_f32(0.5f), + vdupq_n_f32(-0.5f) + )); return vqmovn_s32(vcvtq_s32_f32(x)); } PA_FORCE_INLINE float32x4_t convert_from_s32(int32x4_t x, float multiplier){ @@ -62,6 +69,13 @@ PA_FORCE_INLINE int32x4_t convert_to_s32(float32x4_t x, float multiplier){ x = vmulq_f32(x, vdupq_n_f32(multiplier * 2147483648.f)); x = vminq_f32(x, vdupq_n_f32(2147483520.f)); // 2^31 - 128 x = vmaxq_f32(x, vdupq_n_f32(-2147483648.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + x = vaddq_f32(x, vbslq_f32( + vcgtq_f32(x, vdupq_n_f32(0.0f)), + vdupq_n_f32(0.5f), + vdupq_n_f32(-0.5f) + )); return vcvtq_s32_f32(x); } @@ -200,6 +214,18 @@ class Test_to_ARM64_NEON : public Test{ ) {} }; +template +class Test_to_arb_ARM64_NEON : public ToArbitraryTest{ +public: + Test_to_arb_ARM64_NEON(size_t samples) + : ToArbitraryTest( + "to_arb_ARM64_NEON", + samples, + (ConvertTo)convert_to_ARM64_NEON + ) + {} +}; + void add_tests_ARM64_NEON(UnitTestDatabase& database){ const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ @@ -215,6 +241,12 @@ void add_tests_ARM64_NEON(UnitTestDatabase& database){ database.add>(samples, from_multiplier, to_multiplier); } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp index 57aac60bfc..64f165075d 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp @@ -93,6 +93,17 @@ class Test_Default : public Test{ ) {} }; +template +class Test_to_arb_Default : public ToArbitraryTest{ +public: + Test_to_arb_Default(size_t samples) + : ToArbitraryTest( + "to_arb_Default", + samples, + (ConvertTo)convert_to_Default + ) + {} +}; void add_tests_Default(UnitTestDatabase& database){ // The multipliers are reciprocals so the round trip stays in the // native range (and exact in binary): from scales the floats down, @@ -106,6 +117,12 @@ void add_tests_Default(UnitTestDatabase& database){ database.add>(samples, from_multiplier, to_multiplier); } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp index ac880e782f..2e8513aa62 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp @@ -48,6 +48,10 @@ PA_FORCE_INLINE __m256i convert_to_s16(__m256 x, float multiplier){ x = _mm256_mul_ps(x, _mm256_set1_ps(multiplier * 32768.f)); x = _mm256_min_ps(x, _mm256_set1_ps(32767.f)); x = _mm256_max_ps(x, _mm256_set1_ps(-32768.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m256 positive = _mm256_cmp_ps(x, _mm256_setzero_ps(), _CMP_GT_OQ); + x = _mm256_add_ps(x, _mm256_blendv_ps(_mm256_set1_ps(-0.5f), _mm256_set1_ps(0.5f), positive)); return _mm256_cvttps_epi32(x); } PA_FORCE_INLINE __m256 convert_from_s32(__m256i x, float multiplier){ @@ -60,6 +64,10 @@ PA_FORCE_INLINE __m256i convert_to_s32(__m256 x, float multiplier){ x = _mm256_mul_ps(x, _mm256_set1_ps(multiplier * 2147483648.f)); x = _mm256_min_ps(x, _mm256_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm256_max_ps(x, _mm256_set1_ps(-2147483648.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m256 positive = _mm256_cmp_ps(x, _mm256_setzero_ps(), _CMP_GT_OQ); + x = _mm256_add_ps(x, _mm256_blendv_ps(_mm256_set1_ps(-0.5f), _mm256_set1_ps(0.5f), positive)); return _mm256_cvttps_epi32(x); } @@ -217,6 +225,18 @@ class Test_to_x64_AVX2 : public Test{ ) {} }; +template +class Test_to_arb_x64_AVX2 : public ToArbitraryTest{ +public: + Test_to_arb_x64_AVX2(size_t samples) + : ToArbitraryTest( + "to_arb_x64_AVX2", + samples, + (ConvertTo)convert_to_x64_AVX2 + ) + {} +}; + void add_tests_x64_AVX2(UnitTestDatabase& database){ const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ @@ -232,6 +252,12 @@ void add_tests_x64_AVX2(UnitTestDatabase& database){ database.add>(samples, from_multiplier, to_multiplier); } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp index 6ce847d91e..e1292f52f6 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp @@ -56,6 +56,10 @@ PA_FORCE_INLINE __m512i convert_to_s16(__m512 x, float multiplier){ x = _mm512_mul_ps(x, _mm512_set1_ps(multiplier * 32768.f)); x = _mm512_min_ps(x, _mm512_set1_ps(32767.f)); x = _mm512_max_ps(x, _mm512_set1_ps(-32768.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m512 positive = _mm512_cmp_ps(x, _mm512_setzero_ps(), _CMP_GT_OQ); + x = _mm512_add_ps(x, _mm512_blendv_ps(_mm512_set1_ps(-0.5f), _mm512_set1_ps(0.5f), positive)); return _mm512_cvttps_epi32(x); } PA_FORCE_INLINE __m512 convert_from_s32(__m512i x, float multiplier){ @@ -68,6 +72,10 @@ PA_FORCE_INLINE __m512i convert_to_s32(__m512 x, float multiplier){ x = _mm512_mul_ps(x, _mm512_set1_ps(multiplier * 2147483648.f)); x = _mm512_min_ps(x, _mm512_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm512_max_ps(x, _mm512_set1_ps(-2147483648.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m512 positive = _mm512_cmp_ps(x, _mm512_setzero_ps(), _CMP_GT_OQ); + x = _mm512_add_ps(x, _mm512_blendv_ps(_mm512_set1_ps(-0.5f), _mm512_set1_ps(0.5f), positive)); return _mm512_cvttps_epi32(x); } @@ -232,6 +240,18 @@ class Test_to_x64_AVX512 : public Test{ ) {} }; +template +class Test_to_arb_x64_AVX512 : public ToArbitraryTest{ +public: + Test_to_arb_x64_AVX512(size_t samples) + : ToArbitraryTest( + "to_arb_x64_AVX512", + samples, + (ConvertTo)convert_to_x64_AVX512 + ) + {} +}; + void add_tests_x64_AVX512(UnitTestDatabase& database){ const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ @@ -247,6 +267,12 @@ void add_tests_x64_AVX512(UnitTestDatabase& database){ database.add>(samples, from_multiplier, to_multiplier); } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp index 1161d0bbd8..71e3c7bcdf 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp @@ -51,6 +51,10 @@ PA_FORCE_INLINE __m128i convert_to_s16(__m128 x, float multiplier){ x = _mm_mul_ps(x, _mm_set1_ps(multiplier * 32768.f)); x = _mm_min_ps(x, _mm_set1_ps(32767.f)); x = _mm_max_ps(x, _mm_set1_ps(-32768.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m128 positive = _mm_cmpgt_ps(x, _mm_setzero_ps()); + x = _mm_add_ps(x, _mm_blendv_ps(_mm_set1_ps(-0.5f), _mm_set1_ps(0.5f), positive)); return _mm_cvttps_epi32(x); } PA_FORCE_INLINE __m128 convert_from_s32(__m128i x, float multiplier){ @@ -63,6 +67,10 @@ PA_FORCE_INLINE __m128i convert_to_s32(__m128 x, float multiplier){ x = _mm_mul_ps(x, _mm_set1_ps(multiplier * 2147483648.f)); x = _mm_min_ps(x, _mm_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm_max_ps(x, _mm_set1_ps(-2147483648.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m128 positive = _mm_cmpgt_ps(x, _mm_setzero_ps()); + x = _mm_add_ps(x, _mm_blendv_ps(_mm_set1_ps(-0.5f), _mm_set1_ps(0.5f), positive)); return _mm_cvttps_epi32(x); } @@ -77,26 +85,15 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size_t s template <> void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ - // 16 samples per iteration so that the 16-byte load stays in bounds; - // anything smaller is handled by the scalar tail. - while (samples >= 16){ - __m128i x = _mm_loadu_si128((const __m128i*)ext); - _mm_storeu_ps(native, convert_from_u8( - _mm_cvtepu8_epi32(x), multiplier - )); - _mm_storeu_ps(native + 4, convert_from_u8( - _mm_cvtepu8_epi32(_mm_srli_si128(x, 4)), multiplier - )); - _mm_storeu_ps(native + 8, convert_from_u8( - _mm_cvtepu8_epi32(_mm_srli_si128(x, 8)), multiplier - )); - _mm_storeu_ps(native + 12, convert_from_u8( - _mm_cvtepu8_epi32(_mm_srli_si128(x, 12)), multiplier - )); + while (samples >= 4){ + // _mm_loadu_si32 reads exactly the 4 bytes of this iteration; + // wider loads would read past the end of short buffers. + __m128i x = _mm_cvtepu8_epi32(_mm_loadu_si32(ext)); + _mm_storeu_ps(native, convert_from_u8(x, multiplier)); - native += 16; - ext += 16; - samples -= 16; + native += 4; + ext += 4; + samples -= 4; } if (samples > 0){ convert_from_Default(native, ext, samples, multiplier); @@ -121,20 +118,15 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size } template <> void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ - // 8 samples per iteration: a 4-sample iteration with a 16-byte load - // would read past the end of short buffers. - while (samples >= 8){ - __m128i x = _mm_loadu_si128((const __m128i*)ext); - _mm_storeu_ps(native, convert_from_s16( - _mm_cvtepi16_epi32(x), multiplier - )); - _mm_storeu_ps(native + 4, convert_from_s16( - _mm_cvtepi16_epi32(_mm_srli_si128(x, 8)), multiplier - )); + while (samples >= 4){ + // _mm_loadl_epi64 reads exactly the 8 bytes of this iteration; + // wider loads would read past the end of short buffers. + __m128i x = _mm_cvtepi16_epi32(_mm_loadl_epi64((const __m128i*)ext)); + _mm_storeu_ps(native, convert_from_s16(x, multiplier)); - native += 8; - ext += 8; - samples -= 8; + native += 4; + ext += 4; + samples -= 4; } if (samples > 0){ convert_from_Default(native, ext, samples, multiplier); @@ -220,6 +212,18 @@ class Test_to_x64_SSE41 : public Test{ ) {} }; +template +class Test_to_arb_x64_SSE41 : public ToArbitraryTest{ +public: + Test_to_arb_x64_SSE41(size_t samples) + : ToArbitraryTest( + "to_arb_x64_SSE41", + samples, + (ConvertTo)convert_to_x64_SSE41 + ) + {} +}; + void add_tests_x64_SSE41(UnitTestDatabase& database){ const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ @@ -235,6 +239,12 @@ void add_tests_x64_SSE41(UnitTestDatabase& database){ database.add>(samples, from_multiplier, to_multiplier); } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp index 57aff2ca86..3e618fd8f4 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp @@ -10,6 +10,7 @@ #include "Common/Cpp/Containers/AlignedVector.tpp" #include "Common/Cpp/PrettyPrint.h" #include "Kernels_AudioStreamConverter_TestInfra.h" +#include "Kernels_AudioStreamConverter_Kernels_Default.h" namespace PokemonAutomation{ namespace Kernels{ @@ -113,6 +114,66 @@ template class Test; +template +ToArbitraryTest::ToArbitraryTest( + const std::string& arch_str, + size_t samples, + ConvertTo convert_to +) + : UnitTest( + "Kernels::AudioStreamConverter - " + Test::typestr() + "_" + arch_str + + "(" + std::to_string(samples) + ")" + ) + , m_samples(samples) + , m_convert_to(convert_to) +{} +template +UnitTestResult ToArbitraryTest::run(Logger& logger, CancellableScope& scope) const{ + if (m_samples < 2){ + throw InternalProgramError(&logger, PA_CURRENT_FUNCTION, "Test size is too small."); + } + + AlignedVector native(m_samples); + AlignedVector ext_ref(m_samples); + AlignedVector ext1(m_samples); + + std::mt19937 generator(42); + std::uniform_real_distribution distribution(-1.0f, 1.0f); + for (size_t c = 0; c < m_samples; c++){ + native[c] = distribution(generator); + } + // Fixed edge values: full scale, just past the clamp, and out of range. + native[0] = -1.0f; + native[1] = 1.0f; + if (m_samples >= 3){ + native[2] = -1.5f; + } + if (m_samples >= 4){ + native[3] = 1.5f; + } + + convert_to_Default(native.data(), ext_ref.data(), m_samples, 1.0f); + m_convert_to(native.data(), ext1.data(), m_samples, 1.0f); + + for (size_t c = 0; c < m_samples; c++){ + if (ext_ref[c] != ext1[c]){ + return "Mismatch at " + std::to_string(c) + " (input " + + tostr_default(native[c]) + "): " + + std::to_string(ext_ref[c]) + " != " + std::to_string(ext1[c]); + } + } + + return true; +} + + +template class ToArbitraryTest; +template class ToArbitraryTest; +template class ToArbitraryTest; + + + + } } } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h index a540ee50c2..1ad8a22156 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h @@ -49,6 +49,7 @@ class Test : public UnitTest{ virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override; +public: static std::string typestr(); private: @@ -68,6 +69,29 @@ class Test : public UnitTest{ +// Compares a `to` kernel against the Default (reference) implementation on +// random float inputs. The round-trip tests only feed grid-aligned values +// (the output of `from`), which cannot expose differences in rounding or +// clamping. +template +class ToArbitraryTest : public UnitTest{ +public: + ToArbitraryTest( + const std::string& arch_str, + size_t samples, + ConvertTo convert_to + ); + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override; + +private: + size_t m_samples; + ConvertTo m_convert_to; +}; + + + + } } }