From cdc9893069b4486d252c23d2db6b5cdf971228c3 Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:41:39 +0000 Subject: [PATCH 01/12] Remove the unused Exceptions.h include from Kernels_ScaleInvariantMatrixMatch.cpp It was never used: the file throws no exceptions and includes nothing that needs it. --- .../Kernels_ScaleInvariantMatrixMatch.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.cpp index e53d578e5b..4944d2e7a2 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.cpp @@ -4,7 +4,6 @@ * */ -#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/CpuId/CpuId.h" namespace PokemonAutomation{ From 2aec564988d75c47a1e03482102ce246bfe65ecd Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:42:35 +0000 Subject: [PATCH 02/12] Add unit tests for the ScaleInvariantMatrixMatch kernels Add a TestInfra for the four kernel functions (compute_scale and compute_error, plain and weighted). Each test generates a random width x height matrix pair (and weight matrix for the weighted variants), runs the Default path and the ISA path on identical input, and compares the results. The Default path is the source of truth, so each ISA registers the same (width, height, use_weights) combinations against Default. The sizes cover 1x1 up to 1000x1000, both divisible and undivisible by the vector lengths (4, 8, 16), including the short-row cases that exercise unaligned row prefixes and partial vector tails. The SSE4.1 path back-reads up to 3 floats before a row start, so the matrices are allocated with 16 floats of head padding to keep those reads in-bounds. Scales are not computed exactly by all paths (different addition orders), so results are compared with a tolerance: pass if within 4096 ULPs, or within 1e-4 of the magnitude of the accumulated sum (the absolute part covers near-zero results, where ULPs are far smaller than the accumulated rounding of a multi-million-term sum). The tests are registered per-ISA in each x86 Core file (SSE, AVX2, AVX512), guarded by the same PA_AutoDispatch and CPU_CAPABILITY_CURRENT checks the dispatchers use, so each architecture only runs the tests for the ISAs it can execute. --- .../Source/Kernels/Kernels_Tests.cpp | 2 + .../Kernels_ScaleInvariantMatrixMatch.cpp | 29 +++ .../Kernels_ScaleInvariantMatrixMatch.h | 5 + ...caleInvariantMatrixMatch_Core_x86_AVX2.cpp | 35 ++++ ...leInvariantMatrixMatch_Core_x86_AVX512.cpp | 35 ++++ ...ScaleInvariantMatrixMatch_Core_x86_SSE.cpp | 35 ++++ ...ls_ScaleInvariantMatrixMatch_TestInfra.cpp | 197 ++++++++++++++++++ ...nels_ScaleInvariantMatrixMatch_TestInfra.h | 108 ++++++++++ SerialPrograms/cmake/SourceFiles.cmake | 2 + 9 files changed, 448 insertions(+) create mode 100644 SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp create mode 100644 SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.h diff --git a/SerialPrograms/Source/Kernels/Kernels_Tests.cpp b/SerialPrograms/Source/Kernels/Kernels_Tests.cpp index 869e2ac50a..bcf950ab1e 100644 --- a/SerialPrograms/Source/Kernels/Kernels_Tests.cpp +++ b/SerialPrograms/Source/Kernels/Kernels_Tests.cpp @@ -15,6 +15,7 @@ #include "ImageFilters/Kernels_ImageFilter_Tests.h" #include "ImageScaleBrightness/Kernels_ImageScaleBrightness_Tests.h" #include "ImageStats/Kernels_ImageStats_Tests.h" +#include "ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.h" #include "SpikeConvolution/Kernels_SpikeConvolution.h" #include "Waterfill/Kernels_Waterfill_Tests.h" @@ -34,6 +35,7 @@ void add_tests(UnitTestDatabase& database){ add_tests_ImageFilters(database); add_tests_ImageScaleBrightness(database); add_tests_ImageStats(database); + ScaleInvariantMatrixMatch::add_tests(database); SpikeConvolution::add_tests(database); add_tests_Waterfill(database); } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.cpp index 4944d2e7a2..0ffedea4fb 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.cpp @@ -5,6 +5,7 @@ */ #include "Common/Cpp/CpuId/CpuId.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" namespace PokemonAutomation{ namespace Kernels{ @@ -80,6 +81,10 @@ float compute_error_min4_x86_SSE (size_t width, size_t height, float scale, f float compute_error_min8_x86_AVX2 (size_t width, size_t height, float scale, float const* const* A, float const* const* TW, float const* const* W); float compute_error_min16_x86_AVX512(size_t width, size_t height, float scale, float const* const* A, float const* const* TW, float const* const* W); +void add_tests_x86_SSE(UnitTestDatabase& database); +void add_tests_x86_AVX2(UnitTestDatabase& database); +void add_tests_x86_AVX512(UnitTestDatabase& database); + float compute_error( size_t width, size_t height, float scale, @@ -129,6 +134,30 @@ float compute_error( } +// +// Register the tests for the ISAs this binary can run. Each ISA checks its +// results against the Default path, which is the source of truth, so the +// Default path itself gets no tests. +// +void add_tests(UnitTestDatabase& database){ +#ifdef PA_AutoDispatch_x64_08_Nehalem + if (CPU_CAPABILITY_CURRENT.OK_08_Nehalem){ + add_tests_x86_SSE(database); + } +#endif +#ifdef PA_AutoDispatch_x64_13_Haswell + if (CPU_CAPABILITY_CURRENT.OK_13_Haswell){ + add_tests_x86_AVX2(database); + } +#endif +#ifdef PA_AutoDispatch_x64_17_Skylake + if (CPU_CAPABILITY_CURRENT.OK_17_Skylake){ + add_tests_x86_AVX512(database); + } +#endif +} + + diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.h b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.h index 4c3936f6b2..b68881b454 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.h +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch.h @@ -8,6 +8,9 @@ #define PokemonAutomation_Kernels_ScaleInvariantMatrixMatch_H namespace PokemonAutomation{ + +class UnitTestDatabase; + namespace Kernels{ namespace ScaleInvariantMatrixMatch{ @@ -54,6 +57,8 @@ float compute_error( +void add_tests(UnitTestDatabase& database); + } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp index 88a905f94b..521da4a776 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp @@ -10,6 +10,8 @@ #include "Kernels/Kernels_x64_AVX2.h" #include "Kernels/PartialWordAccess/Kernels_PartialWordAccess_x64_AVX2.h" #include "Kernels_ScaleInvariantMatrixMatch_Routines.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Kernels_ScaleInvariantMatrixMatch_TestInfra.h" namespace PokemonAutomation{ namespace Kernels{ @@ -126,6 +128,39 @@ float compute_error_min8_x86_AVX2( +float compute_scale_Default(size_t width, size_t height, float const* const* A, float const* const* T); +float compute_scale_Default(size_t width, size_t height, float const* const* A, float const* const* TW, float const* const* W); +float compute_error_Default(size_t width, size_t height, float scale, float const* const* A, float const* const* T); +float compute_error_Default(size_t width, size_t height, float scale, float const* const* A, float const* const* TW, float const* const* W); + +class Test_x86_AVX2 : public Test{ +public: + Test_x86_AVX2(size_t width, size_t height, bool use_weights) + : Test( + "x86_AVX2", + width, height, + use_weights, + compute_scale_Default, + compute_scale_min8_x86_AVX2, + compute_scale_Default, + compute_scale_min8_x86_AVX2, + compute_error_Default, + compute_error_min8_x86_AVX2, + compute_error_Default, + compute_error_min8_x86_AVX2 + ) + {} +}; +void add_tests_x86_AVX2(UnitTestDatabase& database){ + for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ + for (bool use_weights : { false, true }){ + database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); + } + } +} + + + } } } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp index 4cbeb2f19f..1c45deae93 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp @@ -8,6 +8,8 @@ #include #include "Kernels_ScaleInvariantMatrixMatch_Routines.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Kernels_ScaleInvariantMatrixMatch_TestInfra.h" namespace PokemonAutomation{ namespace Kernels{ @@ -118,6 +120,39 @@ float compute_error_min16_x86_AVX512( +float compute_scale_Default(size_t width, size_t height, float const* const* A, float const* const* T); +float compute_scale_Default(size_t width, size_t height, float const* const* A, float const* const* TW, float const* const* W); +float compute_error_Default(size_t width, size_t height, float scale, float const* const* A, float const* const* T); +float compute_error_Default(size_t width, size_t height, float scale, float const* const* A, float const* const* TW, float const* const* W); + +class Test_x86_AVX512 : public Test{ +public: + Test_x86_AVX512(size_t width, size_t height, bool use_weights) + : Test( + "x86_AVX512", + width, height, + use_weights, + compute_scale_Default, + compute_scale_min16_x86_AVX512, + compute_scale_Default, + compute_scale_min16_x86_AVX512, + compute_error_Default, + compute_error_min16_x86_AVX512, + compute_error_Default, + compute_error_min16_x86_AVX512 + ) + {} +}; +void add_tests_x86_AVX512(UnitTestDatabase& database){ + for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ + for (bool use_weights : { false, true }){ + database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); + } + } +} + + + } } } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp index 783cbd12a5..a92c88eff6 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp @@ -10,6 +10,8 @@ #include "Kernels/Kernels_x64_SSE41.h" #include "Kernels/PartialWordAccess/Kernels_PartialWordAccess_x64_SSE41.h" #include "Kernels_ScaleInvariantMatrixMatch_Routines.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Kernels_ScaleInvariantMatrixMatch_TestInfra.h" namespace PokemonAutomation{ namespace Kernels{ @@ -120,6 +122,39 @@ float compute_error_min4_x86_SSE( +float compute_scale_Default(size_t width, size_t height, float const* const* A, float const* const* T); +float compute_scale_Default(size_t width, size_t height, float const* const* A, float const* const* TW, float const* const* W); +float compute_error_Default(size_t width, size_t height, float scale, float const* const* A, float const* const* T); +float compute_error_Default(size_t width, size_t height, float scale, float const* const* A, float const* const* TW, float const* const* W); + +class Test_x86_SSE : public Test{ +public: + Test_x86_SSE(size_t width, size_t height, bool use_weights) + : Test( + "x86_SSE", + width, height, + use_weights, + compute_scale_Default, + compute_scale_min4_x86_SSE, + compute_scale_Default, + compute_scale_min4_x86_SSE, + compute_error_Default, + compute_error_min4_x86_SSE, + compute_error_Default, + compute_error_min4_x86_SSE + ) + {} +}; +void add_tests_x86_SSE(UnitTestDatabase& database){ + for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ + for (bool use_weights : { false, true }){ + database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); + } + } +} + + + } } } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp new file mode 100644 index 0000000000..7d87bf4191 --- /dev/null +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp @@ -0,0 +1,197 @@ +/* Scale Invariant Matrix Match Test Infra + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Compiler.h" +#include "Common/Cpp/CancellableScope.h" +#include "Common/Cpp/Containers/AlignedVector.tpp" +#include "Kernels_ScaleInvariantMatrixMatch_TestInfra.h" + +namespace PokemonAutomation{ +namespace Kernels{ +namespace ScaleInvariantMatrixMatch{ + + + +// +// Head padding (in floats) before the first row of the matrices. The +// SSE4.1 implementation back-reads up to 3 floats (VL - 1) before a row +// start to handle unaligned row prefixes, so with this padding those +// reads always stay inside the allocation. +// +static constexpr size_t HEAD_PAD = 16; + +// +// A result passes if it is within 4096 ULPs of the Default result, or +// within 1e-4 of the natural magnitude of the accumulated sum. The +// absolute part covers near-zero results, where ULPs are far smaller +// than the accumulated rounding of a multi-million-term sum. +// +static constexpr uint64_t MAX_ULP_DIFF = 4096; +static constexpr double ABS_TOLERANCE = 1e-4; + +static uint64_t ulp_distance(float a, float b){ + // Monotonic mapping of the float bits to a signed order. + auto U = [](float f) -> uint64_t{ + int32_t i = std::bit_cast(f); + if (i >= 0) + return (uint64_t)(uint32_t)i + 0x80000000ull; + return (uint64_t)(uint32_t)(-i); + }; + uint64_t x = U(a); + uint64_t y = U(b); + return x > y ? x - y : y - x; +} +static bool within_tolerance(float x, float y, double magnitude){ + if (x == y) + return true; + if (ulp_distance(x, y) <= MAX_ULP_DIFF) + return true; + return std::fabs((double)x - (double)y) <= ABS_TOLERANCE * std::fmax(1.0, magnitude); +} + + + +Test::Test( + const std::string& arch_str, + size_t width, size_t height, + bool use_weights, + SimmFnScale scale0, SimmFnScale scale1, + SimmFnScaleW scaleW0, SimmFnScaleW scaleW1, + SimmFnError error0, SimmFnError error1, + SimmFnErrorW errorW0, SimmFnErrorW errorW1 +) + : UnitTest( + "Kernels::ScaleInvariantMatrixMatch - " + arch_str + + "(w=" + std::to_string(width) + ", h=" + std::to_string(height) + + (use_weights ? ", weights)" : ")") + ) + , m_width(width) + , m_height(height) + , m_use_weights(use_weights) + , m_scale0(scale0) + , m_scale1(scale1) + , m_scaleW0(scaleW0) + , m_scaleW1(scaleW1) + , m_error0(error0) + , m_error1(error1) + , m_errorW0(errorW0) + , m_errorW1(errorW1) +{} + +UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ + scope.throw_if_cancelled(); + const size_t n = m_width * m_height; + + // + // Random data with a deterministic seed: A and T in [-1, 1], W in + // [0, 1]. TW = T * W is precomputed with the same float rounding + // that production uses. + // + AlignedVector A(n + HEAD_PAD); + AlignedVector T(n + HEAD_PAD); + AlignedVector rowsA(m_height); + AlignedVector rowsT(m_height); + AlignedVector W; + AlignedVector TW; + AlignedVector rowsW; + AlignedVector rowsTW; + if (m_use_weights){ + W = AlignedVector(n + HEAD_PAD); + TW = AlignedVector(n + HEAD_PAD); + rowsW = AlignedVector(m_height); + rowsTW = AlignedVector(m_height); + } + + std::mt19937 generator((unsigned int)(m_width * 1000003u + m_height * 7919u + (m_use_weights ? 2654435761u : 0u))); + std::uniform_real_distribution data_distribution(-1.0f, 1.0f); + std::uniform_real_distribution weight_distribution(0.0f, 1.0f); + for (size_t c = HEAD_PAD; c < HEAD_PAD + n; c++){ + A[c] = data_distribution(generator); + T[c] = data_distribution(generator); + if (m_use_weights){ + W[c] = weight_distribution(generator); + TW[c] = T[c] * W[c]; + } + } + for (size_t r = 0; r < m_height; r++){ + rowsA[r] = A.data() + HEAD_PAD + r * m_width; + rowsT[r] = T.data() + HEAD_PAD + r * m_width; + if (m_use_weights){ + rowsW[r] = W.data() + HEAD_PAD + r * m_width; + rowsTW[r] = TW.data() + HEAD_PAD + r * m_width; + } + } + + char value0[32]; + char value1[32]; + auto mismatch = [&](const char* fn, float r0, float r1, double magnitude) -> UnitTestResult{ + std::snprintf(value0, sizeof(value0), "%.9g", (double)r0); + std::snprintf(value1, sizeof(value1), "%.9g", (double)r1); + return std::string(fn) + ": default=" + value0 + " isa=" + value1 + + " (" + std::to_string(ulp_distance(r0, r1)) + " ulps, abs " + + std::to_string(std::fabs((double)r0 - (double)r1)) + " vs tolerance " + + std::to_string(ABS_TOLERANCE * std::fmax(1.0, magnitude)) + ")"; + }; + + // + // compute_scale: s = sum(A * T) / sum(A * A). + // + float s0 = m_scale0(m_width, m_height, rowsA.data(), rowsT.data()); + float s1 = m_scale1(m_width, m_height, rowsA.data(), rowsT.data()); + double mag = std::fmax(std::fabs((double)s0), std::fabs((double)s1)); + if (!within_tolerance(s0, s1, mag)) + return mismatch("compute_scale", s0, s1, mag); + + // + // compute_error at the Default scale: sum((s * A - T)^2). + // + double magE = 0.0; + for (size_t c = HEAD_PAD; c < HEAD_PAD + n; c++){ + double m = std::fabs((double)s0 * (double)A[c]) + std::fabs((double)T[c]); + magE += m * m; + } + float e0 = m_error0(m_width, m_height, s0, rowsA.data(), rowsT.data()); + float e1 = m_error1(m_width, m_height, s0, rowsA.data(), rowsT.data()); + if (!within_tolerance(e0, e1, magE)) + return mismatch("compute_error", e0, e1, magE); + + if (m_use_weights){ + // + // Weighted compute_scale: s = sum((A * W) * (T * W)) / sum((A * W)^2). + // + float ws0 = m_scaleW0(m_width, m_height, rowsA.data(), rowsTW.data(), rowsW.data()); + float ws1 = m_scaleW1(m_width, m_height, rowsA.data(), rowsTW.data(), rowsW.data()); + double magW = std::fmax(std::fabs((double)ws0), std::fabs((double)ws1)); + if (!within_tolerance(ws0, ws1, magW)) + return mismatch("compute_scale (weighted)", ws0, ws1, magW); + + // + // Weighted compute_error at the Default weighted scale: + // sum(((s * A) * W - T * W)^2). + // + double magWE = 0.0; + for (size_t c = HEAD_PAD; c < HEAD_PAD + n; c++){ + double m = std::fabs((double)ws0 * (double)A[c]) + std::fabs((double)T[c]); + magWE += (double)W[c] * (double)W[c] * m * m; + } + float we0 = m_errorW0(m_width, m_height, ws0, rowsA.data(), rowsTW.data(), rowsW.data()); + float we1 = m_errorW1(m_width, m_height, ws0, rowsA.data(), rowsTW.data(), rowsW.data()); + if (!within_tolerance(we0, we1, magWE)) + return mismatch("compute_error (weighted)", we0, we1, magWE); + } + + return true; +} + + + +} +} +} diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.h b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.h new file mode 100644 index 0000000000..c902eb242b --- /dev/null +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.h @@ -0,0 +1,108 @@ +/* Scale Invariant Matrix Match Test Infra + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Compiler.h" +#include "Common/Cpp/TestRunners/UnitTest.h" + +#ifndef PokemonAutomation_Kernels_ScaleInvariantMatrixMatch_TestInfra_H +#define PokemonAutomation_Kernels_ScaleInvariantMatrixMatch_TestInfra_H + +namespace PokemonAutomation{ +namespace Kernels{ +namespace ScaleInvariantMatrixMatch{ + + + +// +// Signatures of compute_scale_Default(), compute_scale_min4_x86_SSE(), etc. +// +using SimmFnScale = float (*)(size_t width, size_t height, float const* const* A, float const* const* T); +using SimmFnScaleW = float (*)(size_t width, size_t height, float const* const* A, float const* const* TW, float const* const* W); +using SimmFnError = float (*)(size_t width, size_t height, float scale, float const* const* A, float const* const* T); +using SimmFnErrorW = float (*)(size_t width, size_t height, float scale, float const* const* A, float const* const* TW, float const* const* W); + + + +// +// The width/height combinations the tests cover: 1x1 up to 1000x1000, +// divisible and undivisible by the vector lengths (4, 8, 16), including +// the short-row cases that exercise the unaligned prefix and the partial +// tail of the vector loops. +// +static constexpr size_t kTestSizes[][2] = { + {1, 1}, {2, 1}, {3, 1}, {4, 1}, {5, 1}, {7, 1}, + {8, 1}, {15, 1}, {16, 1}, {17, 1}, + {1, 2}, {2, 3}, {3, 5}, {5, 7}, + {7, 16}, {8, 33}, + {31, 32}, {32, 33}, + {63, 64}, {64, 65}, + {127, 128},{128, 129}, + {255, 256},{256, 257}, + {511, 512},{512, 513}, + {999, 1000},{1000, 1000}, + {1000, 1},{999, 1}, + {1, 1000},{2, 1000},{3, 1000}, + {16, 1000},{17, 1000}, + {256, 1000},{257, 1000}, + {512, 1000},{513, 1000}, + {7, 1000},{8, 1000}, +}; + + + +// +// Runs all four functions (compute_scale / compute_error, plain and +// weighted) twice on identical random data, once with the Default +// implementation and once with the ISA-specific implementation, and +// requires the two results to agree within rounding. The Default path +// is the source of truth. +// +// The results are float sums over up to a million terms accumulated in +// different orders, so an exact comparison is not possible; a result +// passes if it is within 4096 ULPs of the Default result, or within +// 1e-4 of the natural magnitude of the accumulated sum (the latter +// covers near-zero results, where ULPs are far smaller than the +// accumulated rounding). +// +// Warning, this file is compiled in multiple ISA contexts. +// We cannot define the functions here since they will be compiled differently. +// If a higher (AVX512) version wins out in the linker, all versions will use +// it and crash on older CPUs. +// +class Test : public UnitTest{ +public: + Test( + const std::string& arch_str, + size_t width, size_t height, + bool use_weights, + SimmFnScale scale0, SimmFnScale scale1, + SimmFnScaleW scaleW0, SimmFnScaleW scaleW1, + SimmFnError error0, SimmFnError error1, + SimmFnErrorW errorW0, SimmFnErrorW errorW1 + ); + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override; + +private: + size_t m_width; + size_t m_height; + bool m_use_weights; + SimmFnScale m_scale0; + SimmFnScale m_scale1; + SimmFnScaleW m_scaleW0; + SimmFnScaleW m_scaleW1; + SimmFnError m_error0; + SimmFnError m_error1; + SimmFnErrorW m_errorW0; + SimmFnErrorW m_errorW1; +}; + + + +} +} +} +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 715214a137..24ed728d73 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1186,6 +1186,8 @@ file(GLOB LIBRARY_SOURCES Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h + Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp + Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.h Source/Kernels/SpikeConvolution/Kernels_SpikeConvolution.cpp Source/Kernels/SpikeConvolution/Kernels_SpikeConvolution.h Source/Kernels/SpikeConvolution/Kernels_SpikeConvolution_Core_Default.cpp From 0510b368784d9393f2943588ebcecfadf4a47a55 Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:24:15 +0000 Subject: [PATCH 03/12] Fix the A/T swap in the ScaleInvariantMatrixMatch partial-tail loads The plain (unweighted) partial-tail of SumATA2::accumulate() and of SumError::accumulate() called load2_partial_front() with the A and T pointers swapped, so the tail lanes fed T where A was expected: - compute_scale accumulated T*T instead of A*A into sum_A2 for the tail lanes, biasing the scale for any row with a non-zero tail (width not divisible by the vector length); - compute_error accumulated (scale*T - A)^2 instead of (scale*A - T)^2 for the tail lanes. The weighted variants and the unaligned-row prefixes use the correct argument order; only the two plain tails were swapped. The new unit tests (Default as source of truth) catch this: with random data the scale error is on the order of the tail's share of the sum, far outside the 4096-ULP / 1e-4-magnitude tolerance. --- .../Kernels_ScaleInvariantMatrixMatch_Routines.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h index 2ea1cd67e8..4ebf24fe49 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h @@ -98,7 +98,7 @@ struct SumATA2{ if constexpr (VECTOR_LENGTH > 1){ if (length){ vtype a0, t0; - Context::load2_partial_front(length, a0, ptrT, t0, ptrA); + Context::load2_partial_front(length, a0, ptrA, t0, ptrT); sum_at0 = Context::vpma(a0, t0, sum_at0); sum_as0 = Context::vpma(a0, a0, sum_as0); } @@ -271,7 +271,7 @@ struct SumError{ } if (length){ vtype a0, t0; - Context::load2_partial_front(length, a0, ptrT, t0, ptrA); + Context::load2_partial_front(length, a0, ptrA, t0, ptrT); a0 = Context::vpms(scale, a0, t0); sum0 = Context::vpma(a0, a0, sum0); } From 658e0dac1a1dd0b5910fb7821da65868215c56fc Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:38:55 +0000 Subject: [PATCH 04/12] Fix the weighted partial-tail loads to use the advanced pointers The weighted partial-tail loads in SumATA2 and SumError passed the row-start pointers (A, TW, W), which had been shifted back by the unaligned-row prefix and then advanced by a full vector, instead of the current pointers (ptrA, ptrT, ptrW). The tail therefore reloaded the first 'length' elements of the row instead of the last ones. Found by the new unit tests. --- .../Kernels_ScaleInvariantMatrixMatch_Routines.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h index 4ebf24fe49..0f6b22b87e 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h @@ -186,7 +186,7 @@ struct SumATA2{ } if (length){ vtype a0, t0, w0; - Context::load3_partial_front(length, a0, A, t0, TW, w0, W); + Context::load3_partial_front(length, a0, ptrA, t0, ptrT, w0, ptrW); a0 = Context::vmul(a0, w0); sum_as0 = Context::vpma(a0, a0, sum_as0); sum_at0 = Context::vpma(a0, t0, sum_at0); @@ -345,7 +345,7 @@ struct SumError{ } if (length){ vtype a0, t0, w0; - Context::load3_partial_front(length, a0, A, t0, TW, w0, W); + Context::load3_partial_front(length, a0, ptrA, t0, ptrT, w0, ptrW); a0 = Context::vmul(scale, a0); a0 = Context::vpms(a0, w0, t0); sum0 = Context::vpma(a0, a0, sum0); From 7cc8b15091433d77da6c03770798f8332807b15a Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:38:55 +0000 Subject: [PATCH 05/12] Only register SIMM tests at valid widths and pad the test buffers Each ISA kernel requires a minimum row width (4 for SSE4.1, 8 for AVX2, 16 for AVX512): below it, the partial-row loads span past the row end and can read past the end of the allocation. The test registration now skips sizes below the ISA's minimum width. The test matrices also get 15 floats of tail padding after the last row: the SSE4.1 partial-tail load (PSHUFB-based) reads a full 16-byte word that extends up to VL - 1 floats past the row end, which would otherwise run past the allocation on the last row. --- ...caleInvariantMatrixMatch_Core_x86_AVX2.cpp | 5 ++++ ...leInvariantMatrixMatch_Core_x86_AVX512.cpp | 5 ++++ ...ScaleInvariantMatrixMatch_Core_x86_SSE.cpp | 5 ++++ ...ls_ScaleInvariantMatrixMatch_TestInfra.cpp | 23 +++++++++++++------ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp index 521da4a776..132a754d53 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp @@ -152,7 +152,12 @@ class Test_x86_AVX2 : public Test{ {} }; void add_tests_x86_AVX2(UnitTestDatabase& database){ + // The kernel requires a minimum row width (the partial-row loads span up to + // a full vector); sizes below that are not registered for this ISA. for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ + if (kTestSizes[i][0] < 8){ + continue; + } for (bool use_weights : { false, true }){ database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp index 1c45deae93..aab5ab3d70 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp @@ -144,7 +144,12 @@ class Test_x86_AVX512 : public Test{ {} }; void add_tests_x86_AVX512(UnitTestDatabase& database){ + // The kernel requires a minimum row width (the partial-row loads span up to + // a full vector); sizes below that are not registered for this ISA. for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ + if (kTestSizes[i][0] < 16){ + continue; + } for (bool use_weights : { false, true }){ database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp index a92c88eff6..d8f14b3dc3 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp @@ -146,7 +146,12 @@ class Test_x86_SSE : public Test{ {} }; void add_tests_x86_SSE(UnitTestDatabase& database){ + // The kernel requires a minimum row width (the partial-row loads span up to + // a full vector); sizes below that are not registered for this ISA. for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ + if (kTestSizes[i][0] < 4){ + continue; + } for (bool use_weights : { false, true }){ database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp index 7d87bf4191..463141fd88 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp @@ -21,11 +21,20 @@ namespace ScaleInvariantMatrixMatch{ // // Head padding (in floats) before the first row of the matrices. The -// SSE4.1 implementation back-reads up to 3 floats (VL - 1) before a row -// start to handle unaligned row prefixes, so with this padding those -// reads always stay inside the allocation. +// current ISAs do not read before a row start, but the padding keeps the +// first row away from the very start of the allocation in case a future +// ISA does (e.g. an SSE4.1-style back-read of up to VL - 1 floats). // static constexpr size_t HEAD_PAD = 16; +// +// Tail padding after the last row. The SSE4.1 partial-tail load +// (PSHUFB-based `load_f32_no_read_past_end`) always reads a full 16-byte +// word, which extends up to VL - 1 floats past the end of the row even +// though only the first `length` lanes are used. 15 floats (VL - 1 for +// the widest supported ISA) keeps those reads inside the allocation for +// the last row. +// +static constexpr size_t TAIL_PAD = 15; // // A result passes if it is within 4096 ULPs of the Default result, or @@ -94,8 +103,8 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ // [0, 1]. TW = T * W is precomputed with the same float rounding // that production uses. // - AlignedVector A(n + HEAD_PAD); - AlignedVector T(n + HEAD_PAD); + AlignedVector A(n + HEAD_PAD + TAIL_PAD); + AlignedVector T(n + HEAD_PAD + TAIL_PAD); AlignedVector rowsA(m_height); AlignedVector rowsT(m_height); AlignedVector W; @@ -103,8 +112,8 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ AlignedVector rowsW; AlignedVector rowsTW; if (m_use_weights){ - W = AlignedVector(n + HEAD_PAD); - TW = AlignedVector(n + HEAD_PAD); + W = AlignedVector(n + HEAD_PAD + TAIL_PAD); + TW = AlignedVector(n + HEAD_PAD + TAIL_PAD); rowsW = AlignedVector(m_height); rowsTW = AlignedVector(m_height); } From 162c30bc223251abbbc8d0212cca6c53d249a55f Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:44:36 +0000 Subject: [PATCH 06/12] Remove trailing whitespace from the SIMM x86 test registrations --- .../Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp | 2 +- .../Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp | 2 +- .../Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp index 132a754d53..ddce81874c 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp @@ -155,7 +155,7 @@ void add_tests_x86_AVX2(UnitTestDatabase& database){ // The kernel requires a minimum row width (the partial-row loads span up to // a full vector); sizes below that are not registered for this ISA. for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ - if (kTestSizes[i][0] < 8){ + if (kTestSizes[i][0] < 8){ continue; } for (bool use_weights : { false, true }){ diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp index aab5ab3d70..0c36fc5629 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp @@ -147,7 +147,7 @@ void add_tests_x86_AVX512(UnitTestDatabase& database){ // The kernel requires a minimum row width (the partial-row loads span up to // a full vector); sizes below that are not registered for this ISA. for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ - if (kTestSizes[i][0] < 16){ + if (kTestSizes[i][0] < 16){ continue; } for (bool use_weights : { false, true }){ diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp index d8f14b3dc3..adae4ef0ae 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp @@ -149,7 +149,7 @@ void add_tests_x86_SSE(UnitTestDatabase& database){ // The kernel requires a minimum row width (the partial-row loads span up to // a full vector); sizes below that are not registered for this ISA. for (size_t i = 0; i < sizeof(kTestSizes) / sizeof(kTestSizes[0]); i++){ - if (kTestSizes[i][0] < 4){ + if (kTestSizes[i][0] < 4){ continue; } for (bool use_weights : { false, true }){ From 5ada29d7a530043f6bc643ed0ac1f6a963f1db5a Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Sat, 17 Oct 2026 04:57:53 +0000 Subject: [PATCH 07/12] TEMP: add memory diagnostic for the SIMM x86_SSE test failures --- ...ls_ScaleInvariantMatrixMatch_TestInfra.cpp | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp index 463141fd88..bd41c32002 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "Common/Compiler.h" #include "Common/Cpp/CancellableScope.h" @@ -154,6 +155,40 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ // float s0 = m_scale0(m_width, m_height, rowsA.data(), rowsT.data()); float s1 = m_scale1(m_width, m_height, rowsA.data(), rowsT.data()); + + // TEMPORARY DIAGNOSTIC (remove before merge): decode which bytes the + // ISA path reads for the smallest registered size. + if (m_width == 4 && m_height == 1 && !m_use_weights){ + auto hexdump = [](const float* p, int n, const char* label){ + std::cout << label << " [" << (size_t)p << "]: "; + const unsigned char* b = (const unsigned char*)p; + char line[160]; int o = 0; + for (int i = 0; i < n * 4; i++) + o += std::snprintf(line + o, sizeof(line) - o, "%02x ", b[i]); + std::cout << line << std::endl; + }; + std::cout << "[DIAG] " << m_width << "x" << m_height << ": A.data=" << (size_t)A.data() + << " T.data=" << (size_t)T.data() + << " rowsA[0]=" << (size_t)rowsA[0] + << " rowsT[0]=" << (size_t)rowsT[0] + << " A.data%64=" << ((size_t)A.data() % 64) + << " rowsA[0]%16=" << ((size_t)rowsA[0] % 16) + << std::endl; + hexdump(A.data(), 12, "[DIAG] A[0..12)"); + hexdump(T.data(), 12, "[DIAG] T[0..12)"); + float s1_pad = 0.0f, s1_zero = 0.0f; + for (int rep = 0; rep < 2; rep++){ + float fill = (rep == 0) ? -12345.0f : 0.0f; + for (size_t c = 0; c < HEAD_PAD; c++){ A[c] = fill; T[c] = fill; } + for (size_t c = HEAD_PAD + n; c < HEAD_PAD + n + TAIL_PAD; c++){ A[c] = fill; T[c] = fill; } + float s1x = m_scale1(m_width, m_height, rowsA.data(), rowsT.data()); + if (rep == 0){ s1_pad = s1x; }else{ s1_zero = s1x; } + } + std::cout << "[DIAG] s0=" << s0 << " s1=" << s1 + << " s1(pad=-12345)=" << s1_pad + << " s1(pad=0)=" << s1_zero << std::endl; + } + double mag = std::fmax(std::fabs((double)s0), std::fabs((double)s1)); if (!within_tolerance(s0, s1, mag)) return mismatch("compute_scale", s0, s1, mag); From 20c968887dc6b1d6fc333b9b4146ecac6481766f Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Sat, 17 Oct 2026 05:42:42 +0000 Subject: [PATCH 08/12] TEMP: expand SIMM x86 diagnostic (repeat calls, known-data probe) --- ...ls_ScaleInvariantMatrixMatch_TestInfra.cpp | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp index bd41c32002..c5d5d89b87 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp @@ -174,19 +174,21 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ << " A.data%64=" << ((size_t)A.data() % 64) << " rowsA[0]%16=" << ((size_t)rowsA[0] % 16) << std::endl; - hexdump(A.data(), 12, "[DIAG] A[0..12)"); - hexdump(T.data(), 12, "[DIAG] T[0..12)"); - float s1_pad = 0.0f, s1_zero = 0.0f; - for (int rep = 0; rep < 2; rep++){ - float fill = (rep == 0) ? -12345.0f : 0.0f; - for (size_t c = 0; c < HEAD_PAD; c++){ A[c] = fill; T[c] = fill; } - for (size_t c = HEAD_PAD + n; c < HEAD_PAD + n + TAIL_PAD; c++){ A[c] = fill; T[c] = fill; } + hexdump(A.data() + HEAD_PAD, 4, "[DIAG] row A"); + hexdump(T.data() + HEAD_PAD, 4, "[DIAG] row T"); + for (int i = 0; i < 6; i++){ float s1x = m_scale1(m_width, m_height, rowsA.data(), rowsT.data()); - if (rep == 0){ s1_pad = s1x; }else{ s1_zero = s1x; } + std::cout << "[DIAG] s1 repeat#" << i << "=" << s1x << std::endl; } - std::cout << "[DIAG] s0=" << s0 << " s1=" << s1 - << " s1(pad=-12345)=" << s1_pad - << " s1(pad=0)=" << s1_zero << std::endl; + float e0x = m_error0(m_width, m_height, s0, rowsA.data(), rowsT.data()); + float e1x = m_error1(m_width, m_height, s0, rowsA.data(), rowsT.data()); + std::cout << "[DIAG] e0=" << e0x << " e1=" << e1x << std::endl; + alignas(64) float kA[16] = { 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + alignas(64) float kT[16] = { 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + alignas(64) const float* krowsA[1] = { kA + 4 }; + alignas(64) const float* krowsT[1] = { kT + 4 }; + std::cout << "[DIAG] known-data s1 (expect 0.33333333)=" + << m_scale1(4, 1, krowsA, krowsT) << std::endl; } double mag = std::fmax(std::fabs((double)s0), std::fabs((double)s1)); From cd2e551ef3d4e175d83278c18bd40bdfb34971b5 Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Sat, 17 Oct 2026 06:46:56 +0000 Subject: [PATCH 09/12] TEMP: instrument SumATA2 internals in the x86_SSE diagnostic --- ...ScaleInvariantMatrixMatch_Core_x86_SSE.cpp | 27 +++++++++++++++++++ ...ls_ScaleInvariantMatrixMatch_TestInfra.cpp | 6 +++++ 2 files changed, 33 insertions(+) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp index adae4ef0ae..9e4da9b968 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp @@ -94,6 +94,33 @@ float compute_scale_min4_x86_SSE( ){ return compute_scale>(width, height, A, T); } + +// TEMPORARY DIAGNOSTIC (remove before merge) +float simm_diag_scale41(const float* A, const float* T){ + using Ctx = Context_x86_SSE41; + Kernels::print(_mm_set1_ps(42.0f)); // sanity: print() works + SumATA2 sum; + std::cout << "[DIAG2] after ctor sum_AT:" << std::endl; + Kernels::print(sum.sum_AT); + std::cout << "[DIAG2] after ctor sum_A2:" << std::endl; + Kernels::print(sum.sum_A2); + const __m128* pA = (const __m128*)A; + const __m128* pT = (const __m128*)T; + std::cout << "[DIAG2] pA[0]:" << std::endl; + Kernels::print(pA[0]); + std::cout << "[DIAG2] pT[0]:" << std::endl; + Kernels::print(pT[0]); + std::cout << "[DIAG2] manual vpma(pA[0], pT[0], 0):" << std::endl; + Kernels::print(Ctx::vpma(pA[0], pT[0], Ctx::vzero())); + sum.accumulate(4, A, T); + std::cout << "[DIAG2] after accumulate sum_AT:" << std::endl; + Kernels::print(sum.sum_AT); + std::cout << "[DIAG2] after accumulate sum_A2:" << std::endl; + Kernels::print(sum.sum_A2); + float r = sum.scale(); + std::cout << "[DIAG2] scale=" << r << std::endl; + return r; +} float compute_scale_min4_x86_SSE( size_t width, size_t height, float const* const* A, diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp index c5d5d89b87..7ef9e28818 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp @@ -189,6 +189,12 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ alignas(64) const float* krowsT[1] = { kT + 4 }; std::cout << "[DIAG] known-data s1 (expect 0.33333333)=" << m_scale1(4, 1, krowsA, krowsT) << std::endl; + std::cout << "[DIAG] s0 (this run)=" << s0 << std::endl; + float simm_diag_scale41(const float* A, const float* T); + std::cout << "[DIAG] diag on test row:" << std::endl; + simm_diag_scale41(rowsA[0], rowsT[0]); + std::cout << "[DIAG] diag on known data:" << std::endl; + simm_diag_scale41(kA + 4, kT + 4); } double mag = std::fmax(std::fabs((double)s0), std::fabs((double)s1)); From 82b1ad6a650dc35f3e250c7a617d68dd6d5345f3 Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:10:01 +0000 Subject: [PATCH 10/12] Remove the temporary CI diagnostic code from the SIMM tests The diagnostics isolated the root cause of the x86_SSE test failures at -O0; drop them now that the fix is in. --- ...ScaleInvariantMatrixMatch_Core_x86_SSE.cpp | 27 ------------ ...ls_ScaleInvariantMatrixMatch_TestInfra.cpp | 43 ------------------- 2 files changed, 70 deletions(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp index 9e4da9b968..adae4ef0ae 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp @@ -94,33 +94,6 @@ float compute_scale_min4_x86_SSE( ){ return compute_scale>(width, height, A, T); } - -// TEMPORARY DIAGNOSTIC (remove before merge) -float simm_diag_scale41(const float* A, const float* T){ - using Ctx = Context_x86_SSE41; - Kernels::print(_mm_set1_ps(42.0f)); // sanity: print() works - SumATA2 sum; - std::cout << "[DIAG2] after ctor sum_AT:" << std::endl; - Kernels::print(sum.sum_AT); - std::cout << "[DIAG2] after ctor sum_A2:" << std::endl; - Kernels::print(sum.sum_A2); - const __m128* pA = (const __m128*)A; - const __m128* pT = (const __m128*)T; - std::cout << "[DIAG2] pA[0]:" << std::endl; - Kernels::print(pA[0]); - std::cout << "[DIAG2] pT[0]:" << std::endl; - Kernels::print(pT[0]); - std::cout << "[DIAG2] manual vpma(pA[0], pT[0], 0):" << std::endl; - Kernels::print(Ctx::vpma(pA[0], pT[0], Ctx::vzero())); - sum.accumulate(4, A, T); - std::cout << "[DIAG2] after accumulate sum_AT:" << std::endl; - Kernels::print(sum.sum_AT); - std::cout << "[DIAG2] after accumulate sum_A2:" << std::endl; - Kernels::print(sum.sum_A2); - float r = sum.scale(); - std::cout << "[DIAG2] scale=" << r << std::endl; - return r; -} float compute_scale_min4_x86_SSE( size_t width, size_t height, float const* const* A, diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp index 7ef9e28818..463141fd88 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_TestInfra.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include "Common/Compiler.h" #include "Common/Cpp/CancellableScope.h" @@ -155,48 +154,6 @@ UnitTestResult Test::run(Logger& logger, CancellableScope& scope) const{ // float s0 = m_scale0(m_width, m_height, rowsA.data(), rowsT.data()); float s1 = m_scale1(m_width, m_height, rowsA.data(), rowsT.data()); - - // TEMPORARY DIAGNOSTIC (remove before merge): decode which bytes the - // ISA path reads for the smallest registered size. - if (m_width == 4 && m_height == 1 && !m_use_weights){ - auto hexdump = [](const float* p, int n, const char* label){ - std::cout << label << " [" << (size_t)p << "]: "; - const unsigned char* b = (const unsigned char*)p; - char line[160]; int o = 0; - for (int i = 0; i < n * 4; i++) - o += std::snprintf(line + o, sizeof(line) - o, "%02x ", b[i]); - std::cout << line << std::endl; - }; - std::cout << "[DIAG] " << m_width << "x" << m_height << ": A.data=" << (size_t)A.data() - << " T.data=" << (size_t)T.data() - << " rowsA[0]=" << (size_t)rowsA[0] - << " rowsT[0]=" << (size_t)rowsT[0] - << " A.data%64=" << ((size_t)A.data() % 64) - << " rowsA[0]%16=" << ((size_t)rowsA[0] % 16) - << std::endl; - hexdump(A.data() + HEAD_PAD, 4, "[DIAG] row A"); - hexdump(T.data() + HEAD_PAD, 4, "[DIAG] row T"); - for (int i = 0; i < 6; i++){ - float s1x = m_scale1(m_width, m_height, rowsA.data(), rowsT.data()); - std::cout << "[DIAG] s1 repeat#" << i << "=" << s1x << std::endl; - } - float e0x = m_error0(m_width, m_height, s0, rowsA.data(), rowsT.data()); - float e1x = m_error1(m_width, m_height, s0, rowsA.data(), rowsT.data()); - std::cout << "[DIAG] e0=" << e0x << " e1=" << e1x << std::endl; - alignas(64) float kA[16] = { 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - alignas(64) float kT[16] = { 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - alignas(64) const float* krowsA[1] = { kA + 4 }; - alignas(64) const float* krowsT[1] = { kT + 4 }; - std::cout << "[DIAG] known-data s1 (expect 0.33333333)=" - << m_scale1(4, 1, krowsA, krowsT) << std::endl; - std::cout << "[DIAG] s0 (this run)=" << s0 << std::endl; - float simm_diag_scale41(const float* A, const float* T); - std::cout << "[DIAG] diag on test row:" << std::endl; - simm_diag_scale41(rowsA[0], rowsT[0]); - std::cout << "[DIAG] diag on known data:" << std::endl; - simm_diag_scale41(kA + 4, kT + 4); - } - double mag = std::fmax(std::fabs((double)s0), std::fabs((double)s1)); if (!within_tolerance(s0, s1, mag)) return mismatch("compute_scale", s0, s1, mag); From 6d89f56bd515277764eb50c72eea7227ef80a390 Mon Sep 17 00:00:00 2001 From: Gin890 <79114795+Gin890@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:10:01 +0000 Subject: [PATCH 11/12] Zero-initialize the SIMM accumulators in an explicit constructor SumATA2 relied on non-static data member initializers with the implicit default constructor. At -O0 on x86 (gcc on Ubuntu, Apple clang on the Mac Intel runner), those initializers are not applied, so the sum_AT and sum_A2 members kept stale stack contents and the results were machine-dependent garbage (e.g. -8e-34 instead of -0.628, differing per runner while the identical -O2 builds passed). Give SumATA2 an explicit user-defined default constructor that initializes both members from Context::vzero(), and move SumError's sum member into its constructor's member-initializer list. This is the same pattern the CI evidence showed working under -O0 (the SumError error-path results matched the reference bit-for-bit on the failing runners). --- ...Kernels_ScaleInvariantMatrixMatch_Routines.h | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h index 0f6b22b87e..d02feab14c 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Routines.h @@ -22,8 +22,18 @@ struct SumATA2{ using vtype = typename Context::vtype; static constexpr size_t VECTOR_LENGTH = sizeof(vtype) / sizeof(float); - vtype sum_AT = Context::vzero(); - vtype sum_A2 = Context::vzero(); + vtype sum_AT; + vtype sum_A2; + + // An explicit default constructor is required: with only non-static data + // member initializers, the implicit default constructor does not zero the + // accumulators at -O0 on x86, leaving stale stack contents in the lanes + // (observed with gcc and Apple clang; the user-defined constructor of + // SumError initializes correctly under the same conditions). + PA_FORCE_INLINE SumATA2() + : sum_AT(Context::vzero()) + , sum_A2(Context::vzero()) + {} PA_FORCE_INLINE float scale() const{ return Context::vreduce(sum_AT) / Context::vreduce(sum_A2); @@ -205,10 +215,11 @@ struct SumError{ static constexpr size_t VECTOR_LENGTH = sizeof(vtype) / sizeof(float); vtype scale; - vtype sum = Context::vzero(); + vtype sum; PA_FORCE_INLINE SumError(float p_scale) : scale(Context::vset(p_scale)) + , sum(Context::vzero()) {} PA_FORCE_INLINE float sum_sqr() const{ From 3e9bbec0aa1a77128509a1c02eba5cf292b625c8 Mon Sep 17 00:00:00 2001 From: Alexander Yee Date: Sat, 17 Oct 2026 03:03:35 -0700 Subject: [PATCH 12/12] cleanup --- .../Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp | 5 ++--- .../Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp | 5 ++--- .../Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp index ddce81874c..ab431e6ccd 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX2.cpp @@ -158,9 +158,8 @@ void add_tests_x86_AVX2(UnitTestDatabase& database){ if (kTestSizes[i][0] < 8){ continue; } - for (bool use_weights : { false, true }){ - database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); - } + database.add(kTestSizes[i][0], kTestSizes[i][1], false); + database.add(kTestSizes[i][0], kTestSizes[i][1], true); } } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp index 0c36fc5629..ecb9058c25 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_AVX512.cpp @@ -150,9 +150,8 @@ void add_tests_x86_AVX512(UnitTestDatabase& database){ if (kTestSizes[i][0] < 16){ continue; } - for (bool use_weights : { false, true }){ - database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); - } + database.add(kTestSizes[i][0], kTestSizes[i][1], false); + database.add(kTestSizes[i][0], kTestSizes[i][1], true); } } diff --git a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp index adae4ef0ae..042dd0ca12 100644 --- a/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp +++ b/SerialPrograms/Source/Kernels/ScaleInvariantMatrixMatch/Kernels_ScaleInvariantMatrixMatch_Core_x86_SSE.cpp @@ -152,9 +152,8 @@ void add_tests_x86_SSE(UnitTestDatabase& database){ if (kTestSizes[i][0] < 4){ continue; } - for (bool use_weights : { false, true }){ - database.add(kTestSizes[i][0], kTestSizes[i][1], use_weights); - } + database.add(kTestSizes[i][0], kTestSizes[i][1], false); + database.add(kTestSizes[i][0], kTestSizes[i][1], true); } }