diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp index 90eff6ca37..37f68dbc03 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.cpp @@ -9,6 +9,7 @@ #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Containers/AlignedVector.tpp" +#include "Common/Cpp/PrettyPrint.h" #include "Kernels_AudioPipeline_RoundTrip_TestInfra.h" #include "Kernels_RawAudio2WorkingAudio.h" #include "Kernels_WorkingAudio2RawAudio.h" @@ -70,7 +71,7 @@ std::string sample_typestr(SampleFormat sample_type){ template -UnitTestResult run_round_trip(SampleFormat sample_type, size_t channels, size_t samples){ +UnitTestResult run_round_trip(SampleFormat sample_type, size_t channels, size_t samples, float from_multiplier){ // Generate random input. AlignedVector input(samples * channels); std::mt19937 generator(42); @@ -98,8 +99,14 @@ UnitTestResult run_round_trip(SampleFormat sample_type, size_t channels, size_t // Small buffer so the sample-size sweep hits multiple chunks. const size_t buffer_size = 256; - RawAudio2WorkingAudio raw2working(sample_type, sample_rate, channels, buffer_size); - WorkingAudio2RawAudio working2raw(sample_type, sample_rate, channels, 1.0f, buffer_size); + RawAudio2WorkingAudio raw2working( + sample_type, sample_rate, channels, + from_multiplier, buffer_size + ); + WorkingAudio2RawAudio working2raw( + sample_type, sample_rate, channels, + 1.0f / from_multiplier, buffer_size + ); RecordingRawAudioListener output(sample_type, sample_rate, channels, samples * channels); raw2working.add_listener(working2raw); working2raw.add_listener(output); @@ -132,26 +139,39 @@ UnitTestResult run_round_trip(SampleFormat sample_type, size_t channels, size_t RoundTripTest::RoundTripTest( SampleFormat sample_type, size_t channels, - size_t samples + size_t samples, + float from_multiplier ) - : UnitTest( - "Kernels::AudioPipeline - RoundTrip_" + sample_typestr(sample_type) + - "(" + std::to_string(channels) + "ch x " + std::to_string(samples) + ")" - ) + : UnitTest(make_name(sample_type, channels, samples, from_multiplier)) , m_sample_type(sample_type) , m_channels(channels) , m_samples(samples) + , m_from_multiplier(from_multiplier) {} +std::string RoundTripTest::make_name( + SampleFormat sample_type, + size_t channels, + size_t samples, + float from_multiplier +){ + std::string name = "Kernels::AudioPipeline - RoundTrip_" + sample_typestr(sample_type) + + "(" + std::to_string(channels) + "ch x " + std::to_string(samples) + ")"; + if (from_multiplier != 1.0f){ + name += " [from " + tostr_default(from_multiplier) + + " / to " + tostr_default(1.0f / from_multiplier) + "]"; + } + return name; +} UnitTestResult RoundTripTest::run(Logger& logger, CancellableScope& scope) const{ switch (m_sample_type){ case SampleFormat::UINT8: - return run_round_trip(m_sample_type, m_channels, m_samples); + return run_round_trip(m_sample_type, m_channels, m_samples, m_from_multiplier); case SampleFormat::SINT16: - return run_round_trip(m_sample_type, m_channels, m_samples); + return run_round_trip(m_sample_type, m_channels, m_samples, m_from_multiplier); case SampleFormat::SINT32: - return run_round_trip(m_sample_type, m_channels, m_samples); + return run_round_trip(m_sample_type, m_channels, m_samples, m_from_multiplier); case SampleFormat::FLOAT32: - return run_round_trip(m_sample_type, m_channels, m_samples); + return run_round_trip(m_sample_type, m_channels, m_samples, m_from_multiplier); default: throw InternalProgramError(&logger, PA_CURRENT_FUNCTION, "Unknown sample type."); } diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h index 177046d70f..1f6d57699f 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_RoundTrip_TestInfra.h @@ -20,24 +20,34 @@ namespace AudioPipeline{ // // Pushes random samples through a RawAudio2WorkingAudio node and a -// WorkingAudio2RawAudio node (with a 1.0f multiplier), then compares the -// resulting raw samples to the input. +// WorkingAudio2RawAudio node (with reciprocal multipliers, so the +// round trip is lossless), then compares the resulting raw samples +// to the input. // class RoundTripTest : public UnitTest{ public: RoundTripTest( SampleFormat sample_type, size_t channels, - size_t samples + size_t samples, + float from_multiplier = 1.0f ); virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override; private: + static std::string make_name( + SampleFormat sample_type, + size_t channels, + size_t samples, + float from_multiplier + ); + const SampleFormat m_sample_type; const size_t m_channels; const size_t m_samples; + const float m_from_multiplier; }; diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp index 9c27cc42b7..39e3b19ed3 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp @@ -15,15 +15,18 @@ namespace AudioPipeline{ - void add_tests(UnitTestDatabase& database){ - // Raw -> Working -> Raw round-trip. + // Raw -> Working -> Raw round-trip, with reciprocal multipliers on + // the two nodes (exact in binary) to exercise the gain path. + const float from_multipliers[] = { 1.0f, 0.5f }; for (size_t channels = 1; channels <= 4; channels++){ for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add(SampleFormat::UINT8, channels, samples); - database.add(SampleFormat::SINT16, channels, samples); - database.add(SampleFormat::SINT32, channels, samples); - database.add(SampleFormat::FLOAT32, channels, samples); + for (float from_multiplier : from_multipliers){ + database.add(SampleFormat::UINT8, channels, samples, from_multiplier); + database.add(SampleFormat::SINT16, channels, samples, from_multiplier); + database.add(SampleFormat::SINT32, channels, samples, from_multiplier); + database.add(SampleFormat::FLOAT32, channels, samples, from_multiplier); + } } } diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.cpp index 881250bfdc..fff34b7dec 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.cpp @@ -39,7 +39,8 @@ template void RawAudio2WorkingAudio::set_converter(){ m_convert = [this](const void* data, size_t samples){ Kernels::AudioStreamConverter::convert_from( - m_interleaved_floats.data(), (const Type*)data, samples + m_interleaved_floats.data(), (const Type*)data, samples, + m_multiplier.load(std::memory_order_acquire) ); }; } @@ -49,9 +50,11 @@ RawAudio2WorkingAudio::RawAudio2WorkingAudio( SampleFormat sample_type, size_t sample_rate, size_t channels, + float multiplier, size_t buffer_size ) : RawAudioStreamListener(sample_type, sample_rate, channels) + , m_multiplier(multiplier) , m_buffer_size(buffer_size) { // Single-channel float is pass-through. @@ -95,6 +98,9 @@ void RawAudio2WorkingAudio::add_listener(WorkingAudioStreamListener& listener){ void RawAudio2WorkingAudio::remove_listener(WorkingAudioStreamListener& listener){ m_listeners.remove(listener); } +void RawAudio2WorkingAudio::set_multiplier(float multiplier){ + m_multiplier.store(multiplier, std::memory_order_release); +} void RawAudio2WorkingAudio::push_samples(size_t frames, const void* data){ // Single-channel float pass-through. diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h index 9e816043f4..b607245d14 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_Kernels_AudioPipeline_RawAudio2WorkingAudio_H #define PokemonAutomation_Kernels_AudioPipeline_RawAudio2WorkingAudio_H +#include #include #include "Common/Cpp/ListenerSet.h" #include "Common/Cpp/Containers/AlignedVector.h" @@ -25,12 +26,15 @@ class RawAudio2WorkingAudio : public RawAudioStreamListener{ SampleFormat sample_type, size_t sample_rate, size_t channels, + float multiplier, size_t buffer_size = 4096 + 256 // 4k + break super-alignment ); void add_listener(WorkingAudioStreamListener& listener); void remove_listener(WorkingAudioStreamListener& listener); + void set_multiplier(float multiplier); + virtual void push_samples(size_t frames, const void* data) override; @@ -40,6 +44,7 @@ class RawAudio2WorkingAudio : public RawAudioStreamListener{ private: + std::atomic m_multiplier; size_t m_buffer_size; std::function m_convert = nullptr; diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.cpp index 12d931d952..d1c443e25f 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.cpp @@ -15,7 +15,7 @@ namespace AudioStreamConverter{ template -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples); +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_Default(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_Default(UnitTestDatabase& database); @@ -23,7 +23,7 @@ void add_tests_Default(UnitTestDatabase& database); template -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_x64_SSE41(UnitTestDatabase& database); @@ -31,7 +31,7 @@ void add_tests_x64_SSE41(UnitTestDatabase& database); template -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_x64_AVX2(UnitTestDatabase& database); @@ -39,7 +39,7 @@ void add_tests_x64_AVX2(UnitTestDatabase& database); template -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_AVX512(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_x64_AVX512(UnitTestDatabase& database); @@ -47,7 +47,7 @@ void add_tests_x64_AVX512(UnitTestDatabase& database); template -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples); +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_ARM64_NEON(UnitTestDatabase& database); @@ -57,32 +57,32 @@ void add_tests_ARM64_NEON(UnitTestDatabase& database); template -void convert_from(r_ptr native, c_ptr ext, size_t samples){ +void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier){ #ifdef PA_AutoDispatch_x64_17_Skylake if (CPU_CAPABILITY_CURRENT.OK_17_Skylake){ - convert_from_x64_AVX512(native, ext, samples); + convert_from_x64_AVX512(native, ext, samples, multiplier); return; } #endif #ifdef PA_AutoDispatch_x64_13_Haswell if (CPU_CAPABILITY_CURRENT.OK_13_Haswell){ - convert_from_x64_AVX2(native, ext, samples); + convert_from_x64_AVX2(native, ext, samples, multiplier); return; } #endif #ifdef PA_AutoDispatch_x64_08_Nehalem if (CPU_CAPABILITY_CURRENT.OK_08_Nehalem){ - convert_from_x64_SSE41(native, ext, samples); + convert_from_x64_SSE41(native, ext, samples, multiplier); return; } #endif #ifdef PA_AutoDispatch_arm64_20_M1 if (CPU_CAPABILITY_CURRENT.OK_M1){ - convert_from_ARM64_NEON(native, ext, samples); + convert_from_ARM64_NEON(native, ext, samples, multiplier); return; } #endif - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); return; } template @@ -117,9 +117,9 @@ void convert_to(c_ptr native, r_ptr ext, size_t samples, fl -template void convert_from(r_ptr native, c_ptr ext, size_t samples); -template void convert_from(r_ptr native, c_ptr ext, size_t samples); -template void convert_from(r_ptr native, c_ptr ext, size_t samples); +template void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier); +template void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier); +template void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to(c_ptr native, r_ptr ext, size_t samples, float multiplier); template void convert_to(c_ptr native, r_ptr ext, size_t samples, float multiplier); template void convert_to(c_ptr native, r_ptr ext, size_t samples, float multiplier); diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h index b61744010e..b43002fc5a 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter.h @@ -20,7 +20,7 @@ namespace AudioStreamConverter{ template -void convert_from(r_ptr native, c_ptr ext, size_t samples); +void convert_from(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to(c_ptr native, r_ptr ext, size_t samples, float multiplier); diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp index 462bbf2095..94afdab4eb 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_ARM64_NEON.cpp @@ -22,72 +22,88 @@ namespace AudioStreamConverter{ -PA_FORCE_INLINE float32x4_t convert_from_u8(uint32x4_t x){ +PA_FORCE_INLINE float32x4_t convert_from_u8(uint32x4_t x, float multiplier){ return vsubq_f32( vmulq_f32( vcvtq_f32_u32(x), - vdupq_n_f32(1.f / 128.f) + vdupq_n_f32(multiplier * (1.f / 128.f)) ), - vdupq_n_f32(1.f) + vdupq_n_f32(multiplier) ); } PA_FORCE_INLINE int32x4_t convert_to_u8(float32x4_t x, float multiplier){ - const float32x4_t SCALE = vdupq_n_f32(multiplier * 128.f); + // Scale first, then clamp: a loud signal with gain clips to the + // uint8 range, it does not scale past it. + x = vmulq_f32(x, vdupq_n_f32(multiplier)); x = vminq_f32(x, vdupq_n_f32(127.0f / 128.0f)); x = vmaxq_f32(x, vdupq_n_f32(-1.0f)); - x = vaddq_f32(vmulq_f32(x, SCALE), SCALE); + x = vaddq_f32(vmulq_f32(x, vdupq_n_f32(128.0f)), vdupq_n_f32(128.5f)); return vcvtq_s32_f32(x); } -PA_FORCE_INLINE float32x4_t convert_from_s16(int32x4_t x){ +PA_FORCE_INLINE float32x4_t convert_from_s16(int32x4_t x, float multiplier){ return vmulq_f32( vcvtq_f32_s32(x), - vdupq_n_f32(1.0f / 32768.0f) + vdupq_n_f32(multiplier * (1.0f / 32768.0f)) ); } PA_FORCE_INLINE int16x4_t convert_to_s16(float32x4_t x, float multiplier){ x = vmulq_f32(x, vdupq_n_f32(multiplier * 32768.f)); x = vminq_f32(x, vdupq_n_f32(32767.f)); x = vmaxq_f32(x, vdupq_n_f32(-32768.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + x = vaddq_f32(x, vbslq_f32( + vcgtq_f32(x, vdupq_n_f32(0.0f)), + vdupq_n_f32(0.5f), + vdupq_n_f32(-0.5f) + )); return vqmovn_s32(vcvtq_s32_f32(x)); } -PA_FORCE_INLINE float32x4_t convert_from_s32(int32x4_t x){ +PA_FORCE_INLINE float32x4_t convert_from_s32(int32x4_t x, float multiplier){ return vmulq_f32( vcvtq_f32_s32(x), - vdupq_n_f32(1.0f / 2147483648.0f) + vdupq_n_f32(multiplier * (1.0f / 2147483648.0f)) ); } PA_FORCE_INLINE int32x4_t convert_to_s32(float32x4_t x, float multiplier){ x = vmulq_f32(x, vdupq_n_f32(multiplier * 2147483648.f)); x = vminq_f32(x, vdupq_n_f32(2147483520.f)); // 2^31 - 128 x = vmaxq_f32(x, vdupq_n_f32(-2147483648.f)); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + x = vaddq_f32(x, vbslq_f32( + vcgtq_f32(x, vdupq_n_f32(0.0f)), + vdupq_n_f32(0.5f), + vdupq_n_f32(-0.5f) + )); return vcvtq_s32_f32(x); } template -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples); +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, size_t samples, float multiplier); template <> -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 8){ uint8x8_t x = vld1_u8(ext); uint16x8_t x16 = vmovl_u8(x); uint32x4_t x32_0 = vmovl_u16(vget_low_u16(x16)); uint32x4_t x32_1 = vmovl_u16(vget_high_u16(x16)); - vst1q_f32(native + 0, convert_from_u8(x32_0)); - vst1q_f32(native + 4, convert_from_u8(x32_1)); + vst1q_f32(native + 0, convert_from_u8(x32_0, multiplier)); + vst1q_f32(native + 4, convert_from_u8(x32_1, multiplier)); native += 8; ext += 8; samples -= 8; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -110,18 +126,18 @@ void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, siz } } template <> -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ int16x4_t x = vld1_s16(ext); int32x4_t x32 = vmovl_s16(x); - vst1q_f32(native, convert_from_s16(x32)); + vst1q_f32(native, convert_from_s16(x32, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -139,17 +155,17 @@ void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, siz } } template <> -void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_ARM64_NEON(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ int32x4_t x = vld1q_s32(ext); - vst1q_f32(native, convert_from_s32(x)); + vst1q_f32(native, convert_from_s32(x, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -173,38 +189,64 @@ void convert_to_ARM64_NEON(c_ptr native, r_ptr ext, siz template class Test_from_ARM64_NEON : public Test{ public: - Test_from_ARM64_NEON(size_t samples) + Test_from_ARM64_NEON(size_t samples, float from_multiplier, float to_multiplier) : Test( "from_ARM64_NEON", samples, (ConvertFrom)convert_from_ARM64_NEON, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; template class Test_to_ARM64_NEON : public Test{ public: - Test_to_ARM64_NEON(size_t samples) + Test_to_ARM64_NEON(size_t samples, float from_multiplier, float to_multiplier) : Test( "to_ARM64_NEON", samples, (ConvertFrom)convert_from_Default, + (ConvertTo)convert_to_ARM64_NEON, + from_multiplier, + to_multiplier + ) + {} +}; +template +class Test_to_arb_ARM64_NEON : public ToArbitraryTest{ +public: + Test_to_arb_ARM64_NEON(size_t samples) + : ToArbitraryTest( + "to_arb_ARM64_NEON", + samples, (ConvertTo)convert_to_ARM64_NEON ) {} }; + void add_tests_ARM64_NEON(UnitTestDatabase& database){ + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp index bedff41e66..64f165075d 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.cpp @@ -15,7 +15,7 @@ namespace AudioStreamConverter{ template -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples); +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_Default(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_Default(UnitTestDatabase& database); @@ -23,9 +23,9 @@ void add_tests_Default(UnitTestDatabase& database); template <> -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier){ for (size_t c = 0; c < samples; c++){ - native[c] = ((int)ext[c] - 128) * (1.0f / 128.0f); + native[c] = ((int)ext[c] - 128) * (1.0f / 128.0f) * multiplier; } } template <> @@ -38,9 +38,9 @@ void convert_to_Default(c_ptr native, r_ptr ext, size_t } } template <> -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier){ for (size_t c = 0; c < samples; c++){ - native[c] = (float)ext[c] * (1.0f / 32768.0f); + native[c] = (float)ext[c] * (1.0f / 32768.0f) * multiplier; } } template <> @@ -55,9 +55,9 @@ void convert_to_Default(c_ptr native, r_ptr ext, size_t } } template <> -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier){ for (size_t c = 0; c < samples; c++){ - native[c] = (float)ext[c] * (1.0f / 2147483648.0f); + native[c] = (float)ext[c] * (1.0f / 2147483648.0f) * multiplier; } } template <> @@ -82,21 +82,47 @@ void convert_to_Default(c_ptr native, r_ptr ext, size_t template class Test_Default : public Test{ public: - Test_Default(size_t samples) + Test_Default(size_t samples, float from_multiplier, float to_multiplier) : Test( "Default", samples, (ConvertFrom)convert_from_Default, + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier + ) + {} +}; +template +class Test_to_arb_Default : public ToArbitraryTest{ +public: + Test_to_arb_Default(size_t samples) + : ToArbitraryTest( + "to_arb_Default", + samples, (ConvertTo)convert_to_Default ) {} }; void add_tests_Default(UnitTestDatabase& database){ + // The multipliers are reciprocals so the round trip stays in the + // native range (and exact in binary): from scales the floats down, + // to scales them back up. + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } + } for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); - database.add>(samples); + database.add>(samples); + database.add>(samples); + database.add>(samples); } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h index 6d344895ca..7b55309cbd 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_Default.h @@ -24,7 +24,7 @@ namespace AudioStreamConverter{ template -void convert_from_Default(r_ptr native, c_ptr ext, size_t samples); +void convert_from_Default(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_Default(c_ptr native, r_ptr ext, size_t samples, float multiplier); diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp index b9bfeda0e6..2e8513aa62 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX2.cpp @@ -22,68 +22,78 @@ namespace AudioStreamConverter{ -PA_FORCE_INLINE __m256 convert_from_u8(__m256i x){ +PA_FORCE_INLINE __m256 convert_from_u8(__m256i x, float multiplier){ return _mm256_fmadd_ps( _mm256_cvtepi32_ps(x), - _mm256_set1_ps(1.f / 128.f), - _mm256_set1_ps(-1.f) + _mm256_set1_ps(multiplier * (1.f / 128.f)), + _mm256_set1_ps(-multiplier) ); } PA_FORCE_INLINE __m256i convert_to_u8(__m256 x, float multiplier){ - const __m256 SCALE = _mm256_set1_ps(multiplier * 128.f); + // Scale first, then clamp: a loud signal with gain clips to the + // uint8 range, it does not scale past it. + x = _mm256_mul_ps(x, _mm256_set1_ps(multiplier)); x = _mm256_min_ps(x, _mm256_set1_ps(127.0f / 128.0f)); x = _mm256_max_ps(x, _mm256_set1_ps(-1.0f)); - x = _mm256_fmadd_ps(x, SCALE, SCALE); - return _mm256_cvtps_epi32(x); + x = _mm256_fmadd_ps(x, _mm256_set1_ps(128.0f), _mm256_set1_ps(128.5f)); + return _mm256_cvttps_epi32(x); } -PA_FORCE_INLINE __m256 convert_from_s16(__m256i x){ +PA_FORCE_INLINE __m256 convert_from_s16(__m256i x, float multiplier){ return _mm256_mul_ps( _mm256_cvtepi32_ps(x), - _mm256_set1_ps(1.0f / 32768.0f) + _mm256_set1_ps(multiplier * (1.0f / 32768.0f)) ); } PA_FORCE_INLINE __m256i convert_to_s16(__m256 x, float multiplier){ x = _mm256_mul_ps(x, _mm256_set1_ps(multiplier * 32768.f)); x = _mm256_min_ps(x, _mm256_set1_ps(32767.f)); x = _mm256_max_ps(x, _mm256_set1_ps(-32768.f)); - return _mm256_cvtps_epi32(x); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m256 positive = _mm256_cmp_ps(x, _mm256_setzero_ps(), _CMP_GT_OQ); + x = _mm256_add_ps(x, _mm256_blendv_ps(_mm256_set1_ps(-0.5f), _mm256_set1_ps(0.5f), positive)); + return _mm256_cvttps_epi32(x); } -PA_FORCE_INLINE __m256 convert_from_s32(__m256i x){ +PA_FORCE_INLINE __m256 convert_from_s32(__m256i x, float multiplier){ return _mm256_mul_ps( _mm256_cvtepi32_ps(x), - _mm256_set1_ps(1.0f / 2147483648.0f) + _mm256_set1_ps(multiplier * (1.0f / 2147483648.0f)) ); } PA_FORCE_INLINE __m256i convert_to_s32(__m256 x, float multiplier){ x = _mm256_mul_ps(x, _mm256_set1_ps(multiplier * 2147483648.f)); x = _mm256_min_ps(x, _mm256_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm256_max_ps(x, _mm256_set1_ps(-2147483648.f)); - return _mm256_cvtps_epi32(x); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m256 positive = _mm256_cmp_ps(x, _mm256_setzero_ps(), _CMP_GT_OQ); + x = _mm256_add_ps(x, _mm256_blendv_ps(_mm256_set1_ps(-0.5f), _mm256_set1_ps(0.5f), positive)); + return _mm256_cvttps_epi32(x); } template -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_t samples, float multiplier); template <> -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 8){ __m256i x = _mm256_cvtepu8_epi32( _mm_loadl_epi64((const __m128i*)ext) ); - _mm256_storeu_ps(native, convert_from_u8(x)); + _mm256_storeu_ps(native, convert_from_u8(x, multiplier)); native += 8; ext += 8; samples -= 8; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -112,19 +122,19 @@ void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_ } } template <> -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 8){ __m256i x = _mm256_cvtepi16_epi32( _mm_loadu_si128((const __m128i*)ext) ); - _mm256_storeu_ps(native, convert_from_s16(x)); + _mm256_storeu_ps(native, convert_from_s16(x, multiplier)); native += 8; ext += 8; samples -= 8; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -153,17 +163,17 @@ void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_ } } template <> -void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX2(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 8){ __m256i x = _mm256_loadu_si256((const __m256i*)ext); - _mm256_storeu_ps(native, convert_from_s32(x)); + _mm256_storeu_ps(native, convert_from_s32(x, multiplier)); native += 8; ext += 8; samples -= 8; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -190,38 +200,64 @@ void convert_to_x64_AVX2(c_ptr native, r_ptr ext, size_ template class Test_from_x64_AVX2 : public Test{ public: - Test_from_x64_AVX2(size_t samples) + Test_from_x64_AVX2(size_t samples, float from_multiplier, float to_multiplier) : Test( "from_x64_AVX2", samples, (ConvertFrom)convert_from_x64_AVX2, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; template class Test_to_x64_AVX2 : public Test{ public: - Test_to_x64_AVX2(size_t samples) + Test_to_x64_AVX2(size_t samples, float from_multiplier, float to_multiplier) : Test( "to_x64_AVX2", samples, (ConvertFrom)convert_from_Default, + (ConvertTo)convert_to_x64_AVX2, + from_multiplier, + to_multiplier + ) + {} +}; +template +class Test_to_arb_x64_AVX2 : public ToArbitraryTest{ +public: + Test_to_arb_x64_AVX2(size_t samples) + : ToArbitraryTest( + "to_arb_x64_AVX2", + samples, (ConvertTo)convert_to_x64_AVX2 ) {} }; + void add_tests_x64_AVX2(UnitTestDatabase& database){ + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp index ca95dde97a..e1292f52f6 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_AVX512.cpp @@ -22,49 +22,67 @@ namespace AudioStreamConverter{ -PA_FORCE_INLINE __m512 convert_from_u8(__m512i x){ +PA_FORCE_INLINE __m512 convert_from_u8(__m512i x, float multiplier){ return _mm512_fmadd_ps( _mm512_cvtepi32_ps(x), - _mm512_set1_ps(1.f / 128.f), - _mm512_set1_ps(-1.f) + _mm512_set1_ps(multiplier * (1.f / 128.f)), + _mm512_set1_ps(-multiplier) ); } PA_FORCE_INLINE __m512i convert_to_u8(__m512 x, float multiplier){ - const __m512 SCALE = _mm512_set1_ps(multiplier * 128.f); + // Scale first, then clamp: a loud signal with gain clips to the + // uint8 range, it does not scale past it. + x = _mm512_mul_ps(x, _mm512_set1_ps(multiplier)); x = _mm512_min_ps(x, _mm512_set1_ps(127.0f / 128.0f)); x = _mm512_max_ps(x, _mm512_set1_ps(-1.0f)); - x = _mm512_fmadd_ps(x, SCALE, SCALE); - return _mm512_cvtps_epi32(x); + x = _mm512_fmadd_ps(x, _mm512_set1_ps(128.0f), _mm512_set1_ps(128.5f)); + return _mm512_cvttps_epi32(x); } -PA_FORCE_INLINE __m512 convert_from_s16(__m512i x){ +// VPMOVQB saturates int32 lane values above 127, which would clamp the +// upper half of the uint8 range. Shift the [0, 255] values into signed +// byte range first, then add the 128 back in byte lanes (modulo 256). +PA_FORCE_INLINE __m128i pack_u8_bytes(__m512i x){ + x = _mm512_sub_epi32(x, _mm512_set1_epi32(128)); + __m128i y = _mm512_cvtepi32_epi8(x); + return _mm_add_epi8(y, _mm_set1_epi8(-128)); +} +PA_FORCE_INLINE __m512 convert_from_s16(__m512i x, float multiplier){ return _mm512_mul_ps( _mm512_cvtepi32_ps(x), - _mm512_set1_ps(1.0f / 32768.0f) + _mm512_set1_ps(multiplier * (1.0f / 32768.0f)) ); } PA_FORCE_INLINE __m512i convert_to_s16(__m512 x, float multiplier){ x = _mm512_mul_ps(x, _mm512_set1_ps(multiplier * 32768.f)); x = _mm512_min_ps(x, _mm512_set1_ps(32767.f)); x = _mm512_max_ps(x, _mm512_set1_ps(-32768.f)); - return _mm512_cvtps_epi32(x); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m512 positive = _mm512_cmp_ps(x, _mm512_setzero_ps(), _CMP_GT_OQ); + x = _mm512_add_ps(x, _mm512_blendv_ps(_mm512_set1_ps(-0.5f), _mm512_set1_ps(0.5f), positive)); + return _mm512_cvttps_epi32(x); } -PA_FORCE_INLINE __m512 convert_from_s32(__m512i x){ +PA_FORCE_INLINE __m512 convert_from_s32(__m512i x, float multiplier){ return _mm512_mul_ps( _mm512_cvtepi32_ps(x), - _mm512_set1_ps(1.0f / 2147483648.0f) + _mm512_set1_ps(multiplier * (1.0f / 2147483648.0f)) ); } PA_FORCE_INLINE __m512i convert_to_s32(__m512 x, float multiplier){ x = _mm512_mul_ps(x, _mm512_set1_ps(multiplier * 2147483648.f)); x = _mm512_min_ps(x, _mm512_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm512_max_ps(x, _mm512_set1_ps(-2147483648.f)); - return _mm512_cvtps_epi32(x); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m512 positive = _mm512_cmp_ps(x, _mm512_setzero_ps(), _CMP_GT_OQ); + x = _mm512_add_ps(x, _mm512_blendv_ps(_mm512_set1_ps(-0.5f), _mm512_set1_ps(0.5f), positive)); + return _mm512_cvttps_epi32(x); } template -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_AVX512(c_ptr native, r_ptr ext, size_t samples, float multiplier); void add_tests_x64_AVX512(UnitTestDatabase& database); @@ -72,12 +90,12 @@ void add_tests_x64_AVX512(UnitTestDatabase& database); template <> -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples > 16){ __m512i x = _mm512_cvtepu8_epi32( _mm_loadu_si128((const __m128i*)ext) ); - _mm512_storeu_ps(native, convert_from_u8(x)); + _mm512_storeu_ps(native, convert_from_u8(x, multiplier)); native += 16; ext += 16; @@ -88,7 +106,7 @@ void convert_from_x64_AVX512(r_ptr native, c_ptr ext, s __m512i i = _mm512_cvtepu8_epi32( _mm_maskz_loadu_epi8(mask, (const __m128i*)ext) ); - _mm512_mask_storeu_ps(native, mask, convert_from_u8(i)); + _mm512_mask_storeu_ps(native, mask, convert_from_u8(i, multiplier)); } } template <> @@ -97,7 +115,7 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz __m512 x = _mm512_loadu_ps(native); _mm_storeu_si128( (__m128i*)ext, - _mm512_cvtepi32_epi8(convert_to_u8(x, multiplier)) + pack_u8_bytes(convert_to_u8(x, multiplier)) ); native += 16; @@ -109,17 +127,17 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz __m512 x = _mm512_maskz_loadu_ps(mask, native); _mm_mask_storeu_epi8( (__m128i*)ext, mask, - _mm512_cvtepi32_epi8(convert_to_u8(x, multiplier)) + pack_u8_bytes(convert_to_u8(x, multiplier)) ); } } template <> -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples > 16){ __m512i x = _mm512_cvtepi16_epi32( _mm256_loadu_si256((const __m256i*)ext) ); - _mm512_storeu_ps(native, convert_from_s16(x)); + _mm512_storeu_ps(native, convert_from_s16(x, multiplier)); native += 16; ext += 16; @@ -130,7 +148,7 @@ void convert_from_x64_AVX512(r_ptr native, c_ptr ext, s __m512i x = _mm512_cvtepi16_epi32( _mm256_maskz_loadu_epi16(mask, ext) ); - _mm512_mask_storeu_ps(native, mask, convert_from_s16(x)); + _mm512_mask_storeu_ps(native, mask, convert_from_s16(x, multiplier)); } } template <> @@ -156,10 +174,10 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz } } template <> -void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_AVX512(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples > 16){ __m512i x = _mm512_loadu_si512(ext); - _mm512_storeu_ps(native, convert_from_s32(x)); + _mm512_storeu_ps(native, convert_from_s32(x, multiplier)); native += 16; ext += 16; @@ -168,7 +186,7 @@ void convert_from_x64_AVX512(r_ptr native, c_ptr ext, s { __mmask16 mask = ((uint32_t)1 << samples) - 1; __m512i x = _mm512_maskz_loadu_epi32(mask, ext); - _mm512_mask_storeu_ps(native, mask, convert_from_s32(x)); + _mm512_mask_storeu_ps(native, mask, convert_from_s32(x, multiplier)); } } template <> @@ -197,38 +215,64 @@ void convert_to_x64_AVX512(c_ptr native, r_ptr ext, siz template class Test_from_x64_AVX512 : public Test{ public: - Test_from_x64_AVX512(size_t samples) + Test_from_x64_AVX512(size_t samples, float from_multiplier, float to_multiplier) : Test( "from_x64_AVX512", samples, (ConvertFrom)convert_from_x64_AVX512, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; template class Test_to_x64_AVX512 : public Test{ public: - Test_to_x64_AVX512(size_t samples) + Test_to_x64_AVX512(size_t samples, float from_multiplier, float to_multiplier) : Test( "to_x64_AVX512", samples, (ConvertFrom)convert_from_Default, + (ConvertTo)convert_to_x64_AVX512, + from_multiplier, + to_multiplier + ) + {} +}; +template +class Test_to_arb_x64_AVX512 : public ToArbitraryTest{ +public: + Test_to_arb_x64_AVX512(size_t samples) + : ToArbitraryTest( + "to_arb_x64_AVX512", + samples, (ConvertTo)convert_to_x64_AVX512 ) {} }; + void add_tests_x64_AVX512(UnitTestDatabase& database){ + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp index 42b9c4e3a3..71e3c7bcdf 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_Kernels_x64_SSE41.cpp @@ -23,70 +23,80 @@ namespace AudioStreamConverter{ -PA_FORCE_INLINE __m128 convert_from_u8(__m128i x){ +PA_FORCE_INLINE __m128 convert_from_u8(__m128i x, float multiplier){ return _mm_add_ps( _mm_mul_ps( _mm_cvtepi32_ps(x), - _mm_set1_ps(1.f / 128.f) + _mm_set1_ps(multiplier * (1.f / 128.f)) ), - _mm_set1_ps(-1.f) + _mm_set1_ps(-multiplier) ); } PA_FORCE_INLINE __m128i convert_to_u8(__m128 x, float multiplier){ - const __m128 SCALE = _mm_set1_ps(multiplier * 128.f); + // Scale first, then clamp: a loud signal with gain clips to the + // uint8 range, it does not scale past it. + x = _mm_mul_ps(x, _mm_set1_ps(multiplier)); x = _mm_min_ps(x, _mm_set1_ps(127.0f / 128.0f)); x = _mm_max_ps(x, _mm_set1_ps(-1.0f)); - x = _mm_add_ps(_mm_mul_ps(x, SCALE), SCALE); - return _mm_cvtps_epi32(x); + x = _mm_add_ps(_mm_mul_ps(x, _mm_set1_ps(128.0f)), _mm_set1_ps(128.5f)); + return _mm_cvttps_epi32(x); } -PA_FORCE_INLINE __m128 convert_from_s16(__m128i x){ +PA_FORCE_INLINE __m128 convert_from_s16(__m128i x, float multiplier){ return _mm_mul_ps( _mm_cvtepi32_ps(x), - _mm_set1_ps(1.0f / 32768.0f) + _mm_set1_ps(multiplier * (1.0f / 32768.0f)) ); } PA_FORCE_INLINE __m128i convert_to_s16(__m128 x, float multiplier){ x = _mm_mul_ps(x, _mm_set1_ps(multiplier * 32768.f)); x = _mm_min_ps(x, _mm_set1_ps(32767.f)); x = _mm_max_ps(x, _mm_set1_ps(-32768.f)); - return _mm_cvtps_epi32(x); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m128 positive = _mm_cmpgt_ps(x, _mm_setzero_ps()); + x = _mm_add_ps(x, _mm_blendv_ps(_mm_set1_ps(-0.5f), _mm_set1_ps(0.5f), positive)); + return _mm_cvttps_epi32(x); } -PA_FORCE_INLINE __m128 convert_from_s32(__m128i x){ +PA_FORCE_INLINE __m128 convert_from_s32(__m128i x, float multiplier){ return _mm_mul_ps( _mm_cvtepi32_ps(x), - _mm_set1_ps(1.0f / 2147483648.0f) + _mm_set1_ps(multiplier * (1.0f / 2147483648.0f)) ); } PA_FORCE_INLINE __m128i convert_to_s32(__m128 x, float multiplier){ x = _mm_mul_ps(x, _mm_set1_ps(multiplier * 2147483648.f)); x = _mm_min_ps(x, _mm_set1_ps(2147483520.f)); // 2^31 - 128 x = _mm_max_ps(x, _mm_set1_ps(-2147483648.f)); - return _mm_cvtps_epi32(x); + // Round half away from zero like the Default path: offset by + // +/-0.5, then truncate. + __m128 positive = _mm_cmpgt_ps(x, _mm_setzero_ps()); + x = _mm_add_ps(x, _mm_blendv_ps(_mm_set1_ps(-0.5f), _mm_set1_ps(0.5f), positive)); + return _mm_cvttps_epi32(x); } template -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples); +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier); template void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size_t samples, float multiplier); template <> -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ - __m128i x = _mm_cvtepu8_epi32( - _mm_loadl_epi64((const __m128i*)ext) - ); - _mm_storeu_ps(native, convert_from_u8(x)); + // _mm_loadu_si32 reads exactly the 4 bytes of this iteration; + // wider loads would read past the end of short buffers. + __m128i x = _mm_cvtepu8_epi32(_mm_loadu_si32(ext)); + _mm_storeu_ps(native, convert_from_u8(x, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -107,19 +117,19 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size } } template <> -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ - __m128i x = _mm_cvtepi16_epi32( - _mm_loadu_si128((const __m128i*)ext) - ); - _mm_storeu_ps(native, convert_from_s16(x)); + // _mm_loadl_epi64 reads exactly the 8 bytes of this iteration; + // wider loads would read past the end of short buffers. + __m128i x = _mm_cvtepi16_epi32(_mm_loadl_epi64((const __m128i*)ext)); + _mm_storeu_ps(native, convert_from_s16(x, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -140,17 +150,17 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size } } template <> -void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples){ +void convert_from_x64_SSE41(r_ptr native, c_ptr ext, size_t samples, float multiplier){ while (samples >= 4){ __m128i x = _mm_loadu_si128((const __m128i*)ext); - _mm_storeu_ps(native, convert_from_s32(x)); + _mm_storeu_ps(native, convert_from_s32(x, multiplier)); native += 4; ext += 4; samples -= 4; } if (samples > 0){ - convert_from_Default(native, ext, samples); + convert_from_Default(native, ext, samples, multiplier); } } template <> @@ -177,38 +187,64 @@ void convert_to_x64_SSE41(c_ptr native, r_ptr ext, size template class Test_from_x64_SSE41 : public Test{ public: - Test_from_x64_SSE41(size_t samples) + Test_from_x64_SSE41(size_t samples, float from_multiplier, float to_multiplier) : Test( "from_x64_SSE41", samples, (ConvertFrom)convert_from_x64_SSE41, - (ConvertTo)convert_to_Default + (ConvertTo)convert_to_Default, + from_multiplier, + to_multiplier ) {} }; template class Test_to_x64_SSE41 : public Test{ public: - Test_to_x64_SSE41(size_t samples) + Test_to_x64_SSE41(size_t samples, float from_multiplier, float to_multiplier) : Test( "to_x64_SSE41", samples, (ConvertFrom)convert_from_Default, + (ConvertTo)convert_to_x64_SSE41, + from_multiplier, + to_multiplier + ) + {} +}; +template +class Test_to_arb_x64_SSE41 : public ToArbitraryTest{ +public: + Test_to_arb_x64_SSE41(size_t samples) + : ToArbitraryTest( + "to_arb_x64_SSE41", + samples, (ConvertTo)convert_to_x64_SSE41 ) {} }; + void add_tests_x64_SSE41(UnitTestDatabase& database){ + const float from_multipliers[] = { 1.0f, 0.5f, 0.25f }; for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ - database.add>(samples); - database.add>(samples); + for (float from_multiplier : from_multipliers){ + float to_multiplier = 1.0f / from_multiplier; + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); - database.add>(samples); - database.add>(samples); + database.add>(samples, from_multiplier, to_multiplier); + database.add>(samples, from_multiplier, to_multiplier); + } } + for (size_t samples = 2; samples < 1000; samples = (size_t)(samples * 1.1) + 1){ + database.add>(samples); + database.add>(samples); + database.add>(samples); + } + } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp index e2084c0805..3e618fd8f4 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.cpp @@ -8,7 +8,9 @@ #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Containers/AlignedVector.tpp" +#include "Common/Cpp/PrettyPrint.h" #include "Kernels_AudioStreamConverter_TestInfra.h" +#include "Kernels_AudioStreamConverter_Kernels_Default.h" namespace PokemonAutomation{ namespace Kernels{ @@ -22,15 +24,16 @@ Test::Test( const std::string& arch_str, size_t samples, ConvertFrom convert_from, - ConvertTo convert_to + ConvertTo convert_to, + float from_multiplier, + float to_multiplier ) - : UnitTest( - "Kernels::AudioStreamConverter - " + typestr() + "_" + arch_str + - "(" + std::to_string(samples) + ")" - ) + : UnitTest(make_name(arch_str, samples, from_multiplier, to_multiplier)) , m_samples(samples) , m_convert_from(convert_from) , m_convert_to(convert_to) + , m_from_multiplier(from_multiplier) + , m_to_multiplier(to_multiplier) {} template UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ @@ -62,8 +65,8 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ } } - m_convert_from(native.data(), ext0.data(), m_samples); - m_convert_to(native.data(), ext1.data(), m_samples, 1.0f); + m_convert_from(native.data(), ext0.data(), m_samples, m_from_multiplier); + m_convert_to(native.data(), ext1.data(), m_samples, m_to_multiplier); // Check for errors. for (size_t c = 0; c < m_samples; c++){ @@ -76,6 +79,21 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ return true; } template +std::string Test::make_name( + const std::string& arch_str, + size_t samples, + float from_multiplier, + float to_multiplier +){ + std::string name = "Kernels::AudioStreamConverter - " + typestr() + "_" + arch_str + + "(" + std::to_string(samples) + ")"; + if (from_multiplier != 1.0f || to_multiplier != 1.0f){ + name += " [from " + tostr_default(from_multiplier) + + " / to " + tostr_default(to_multiplier) + "]"; + } + return name; +} +template std::string Test::typestr(){ if constexpr (std::is_same_v){ return "u8"; @@ -96,6 +114,66 @@ template class Test; +template +ToArbitraryTest::ToArbitraryTest( + const std::string& arch_str, + size_t samples, + ConvertTo convert_to +) + : UnitTest( + "Kernels::AudioStreamConverter - " + Test::typestr() + "_" + arch_str + + "(" + std::to_string(samples) + ")" + ) + , m_samples(samples) + , m_convert_to(convert_to) +{} +template +UnitTestResult ToArbitraryTest::run(Logger& logger, CancellableScope& scope) const{ + if (m_samples < 2){ + throw InternalProgramError(&logger, PA_CURRENT_FUNCTION, "Test size is too small."); + } + + AlignedVector native(m_samples); + AlignedVector ext_ref(m_samples); + AlignedVector ext1(m_samples); + + std::mt19937 generator(42); + std::uniform_real_distribution distribution(-1.0f, 1.0f); + for (size_t c = 0; c < m_samples; c++){ + native[c] = distribution(generator); + } + // Fixed edge values: full scale, just past the clamp, and out of range. + native[0] = -1.0f; + native[1] = 1.0f; + if (m_samples >= 3){ + native[2] = -1.5f; + } + if (m_samples >= 4){ + native[3] = 1.5f; + } + + convert_to_Default(native.data(), ext_ref.data(), m_samples, 1.0f); + m_convert_to(native.data(), ext1.data(), m_samples, 1.0f); + + for (size_t c = 0; c < m_samples; c++){ + if (ext_ref[c] != ext1[c]){ + return "Mismatch at " + std::to_string(c) + " (input " + + tostr_default(native[c]) + "): " + + std::to_string(ext_ref[c]) + " != " + std::to_string(ext1[c]); + } + } + + return true; +} + + +template class ToArbitraryTest; +template class ToArbitraryTest; +template class ToArbitraryTest; + + + + } } } diff --git a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h index 0ae855da91..1ad8a22156 100644 --- a/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h +++ b/SerialPrograms/Source/Kernels/AudioStreamConverter/Kernels_AudioStreamConverter_TestInfra.h @@ -20,7 +20,7 @@ namespace AudioStreamConverter{ template -using ConvertFrom = void (*)(r_ptr native, c_ptr ext, size_t samples); +using ConvertFrom = void (*)(r_ptr native, c_ptr ext, size_t samples, float multiplier); template using ConvertTo = void (*)(c_ptr native, r_ptr ext, size_t samples, float multiplier); @@ -42,17 +42,51 @@ class Test : public UnitTest{ const std::string& arch_str, size_t samples, ConvertFrom convert_from, - ConvertTo convert_to + ConvertTo convert_to, + float from_multiplier = 1.0f, + float to_multiplier = 1.0f ); virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override; +public: static std::string typestr(); private: + static std::string make_name( + const std::string& arch_str, + size_t samples, + float from_multiplier, + float to_multiplier + ); size_t m_samples; ConvertFrom m_convert_from; ConvertTo m_convert_to; + float m_from_multiplier; + float m_to_multiplier; +}; + + + + +// Compares a `to` kernel against the Default (reference) implementation on +// random float inputs. The round-trip tests only feed grid-aligned values +// (the output of `from`), which cannot expose differences in rounding or +// clamping. +template +class ToArbitraryTest : public UnitTest{ +public: + ToArbitraryTest( + const std::string& arch_str, + size_t samples, + ConvertTo convert_to + ); + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override; + +private: + size_t m_samples; + ConvertTo m_convert_to; };