From 4fd49538983ab8de3c28a636a9a351a4e99c6dc0 Mon Sep 17 00:00:00 2001 From: Gin <> Date: Sun, 18 Oct 2026 22:56:28 -0700 Subject: [PATCH 1/2] Flush denormal floats to zero for ONNX inference and compute threads The Chinese and Japanese OCR tests ran 15-25x slower on the Mac Intel CI runner than anywhere else: up to 185 s per test, against about 8 s on the AMD Windows runners. English and Korean OCR was just as fast there as elsewhere. The large PaddleOCR server model that Chinese and Japanese share produces denormal (subnormal) floats, and Intel CPUs (the runner is an i7-8700B) handle many operations on them in microcode. AMD and Apple CPUs pay much less, which is why only Mac Intel was slow. ONNX Runtime leaves denormals on unless asked. Turn on flush-to-zero, so the CPU treats denormal inputs and results as zero: - Common/Cpp/Hardware/FloatingPointMode.h: enable_flush_denormals_to_zero() and FlushDenormalsToZeroScope, for x86-64 (MXCSR FTZ and DAZ) and ARM64 (FPCR.FZ). - Every ONNX Runtime Run() call runs inside FlushDenormalsToZeroScope, because ONNX runs part of each inference on the calling thread. The scope restores the thread's previous mode afterwards. - ONNX Runtime's global thread pool uses SetGlobalDenormalAsZero(). - Every GlobalThreadPools thread turns flush-to-zero on when it starts. This also sped up image matching, e.g. PokemonLA::MMOSpriteMatcher. On the Mac Intel runner, the unit test step went from 463 s to 117 s. All unit tests pass on all six CI platforms. Production builds run OCR on the CPU by default (ONNX USE_GPU is off), so Intel users reading Chinese or Japanese text should get the same speedup. Co-Authored-By: Claude Opus 5.5 --- Common/Cpp/Hardware/FloatingPointMode.h | 99 +++++++++++++++++++ .../Tools/GlobalThreadPools.cpp | 9 ++ .../DataLabeling/ML_SegmentAnythingModel.cpp | 3 + .../ML/Inference/ML_PaddleOCRDetector.cpp | 3 + .../ML/Inference/ML_PaddleOCRPipeline.cpp | 5 +- SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp | 4 + .../Source/ML/Models/ML_YOLOv5Model.cpp | 2 + SerialPrograms/cmake/SourceFiles.cmake | 1 + 8 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 Common/Cpp/Hardware/FloatingPointMode.h diff --git a/Common/Cpp/Hardware/FloatingPointMode.h b/Common/Cpp/Hardware/FloatingPointMode.h new file mode 100644 index 0000000000..56cc3d9ca2 --- /dev/null +++ b/Common/Cpp/Hardware/FloatingPointMode.h @@ -0,0 +1,99 @@ +/* Floating-Point Mode + * + * From: https://github.com/PokemonAutomation/ + * + * Turn on flush-to-zero for denormal floats on the current thread. + * + * Denormal (subnormal) floats are nonzero values too small for the normal float + * format: below about 1.2e-38 for float. Intel CPUs handle many operations on one + * in microcode, which is many times slower than normal float math. AMD and Apple + * CPUs pay much less. The large PaddleOCR model for Chinese and Japanese produces + * them: on the Mac Intel CI runner (an i7-8700B), each of those OCR tests took + * 35-185 s, and under 5 s with flush-to-zero. + * + * With flush-to-zero, the CPU treats denormal inputs as zero and rounds denormal + * results to zero. OCR, image matching and model inference don't need values + * that small. + * + * The mode belongs to each thread: + * - x86-64: the FTZ and DAZ bits of MXCSR, which control SSE and AVX math. + * - ARM64 (GCC and Clang): the FZ bit of FPCR, which controls scalar and NEON math. + * - Anywhere else, these functions do nothing. + * + */ + +#ifndef PokemonAutomation_Hardware_FloatingPointMode_H +#define PokemonAutomation_Hardware_FloatingPointMode_H + +#include +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif + +namespace PokemonAutomation{ + + +// Return the current thread's floating-point control register, so it can be +// restored later with `set_floating_point_control()`. +inline uint64_t get_floating_point_control(){ +#if defined(__x86_64__) || defined(_M_X64) + return _mm_getcsr(); +#elif defined(__aarch64__) + uint64_t fpcr; + __asm__ __volatile__("mrs %0, fpcr" : "=r"(fpcr)); + return fpcr; +#else + return 0; +#endif +} + +// Restore a value returned by `get_floating_point_control()`. +inline void set_floating_point_control(uint64_t value){ +#if defined(__x86_64__) || defined(_M_X64) + _mm_setcsr((unsigned int)value); +#elif defined(__aarch64__) + __asm__ __volatile__("msr fpcr, %0" : : "r"(value)); +#else + (void)value; +#endif +} + +// Turn on flush-to-zero for denormal floats on the current thread, until the +// thread ends or the mode is changed again. +inline void enable_flush_denormals_to_zero(){ +#if defined(__x86_64__) || defined(_M_X64) + // 0x8000: FTZ, flush denormal results to zero. + // 0x0040: DAZ, treat denormal inputs as zero. + _mm_setcsr(_mm_getcsr() | 0x8040); +#elif defined(__aarch64__) + // Bit 24: FZ, flush denormal inputs and results to zero. + set_floating_point_control(get_floating_point_control() | ((uint64_t)1 << 24)); +#endif +} + + +// Turn on flush-to-zero on the current thread for this object's lifetime, then +// restore the previous mode. For code that may run on threads this program +// doesn't own, like an ONNX Runtime `Run()` call: ONNX Runtime runs part of +// each inference on the calling thread. +class FlushDenormalsToZeroScope{ +public: + FlushDenormalsToZeroScope(const FlushDenormalsToZeroScope&) = delete; + void operator=(const FlushDenormalsToZeroScope&) = delete; + + FlushDenormalsToZeroScope() + : m_saved(get_floating_point_control()) + { + enable_flush_denormals_to_zero(); + } + ~FlushDenormalsToZeroScope(){ + set_floating_point_control(m_saved); + } + +private: + uint64_t m_saved; +}; + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/Tools/GlobalThreadPools.cpp b/SerialPrograms/Source/CommonFramework/Tools/GlobalThreadPools.cpp index 82b9aefe2b..4d47f87895 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/GlobalThreadPools.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/GlobalThreadPools.cpp @@ -5,6 +5,7 @@ */ #include "Common/Cpp/Concurrency/Backends/ThreadPool_Default.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "CommonFramework/Logging/Logger.h" #include "CommonFramework/Options/Environment/PerformanceOptions.h" #include "GlobalThreadPools.h" @@ -12,12 +13,16 @@ namespace PokemonAutomation{ namespace GlobalThreadPools{ +// Every pool thread turns on flush-to-zero for denormal floats when it starts. +// Denormals are very slow on Intel CPUs, and image matching and ML inference +// on these threads can produce them. See Common/Cpp/Hardware/FloatingPointMode.h. ThreadPool& computation_realtime(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().REALTIME_THREAD_POOL0.PRIORITY.set_on_this_thread(global_logger_tagged()); + enable_flush_denormals_to_zero(); }, 0, PerformanceOptions::instance().REALTIME_THREAD_POOL0.MAX_THREADS ); @@ -27,6 +32,7 @@ ThreadPool& computation_normal(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().NORMAL_THREAD_POOL.PRIORITY.set_on_this_thread(global_logger_tagged()); + enable_flush_denormals_to_zero(); }, 0, PerformanceOptions::instance().NORMAL_THREAD_POOL.MAX_THREADS ); @@ -37,6 +43,7 @@ ThreadPool& unlimited_realtime(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().REALTIME_THREAD_PRIORITY0.set_on_this_thread(global_logger_tagged()); + enable_flush_denormals_to_zero(); }, 0 ); @@ -46,6 +53,7 @@ ThreadPool& unlimited_pivot(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().INFERENCE_PIVOT_PRIORITY0.set_on_this_thread(global_logger_tagged()); + enable_flush_denormals_to_zero(); }, 0 ); @@ -55,6 +63,7 @@ ThreadPool& unlimited_normal(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().NORMAL_THREAD_POOL.PRIORITY.set_on_this_thread(global_logger_tagged()); + enable_flush_denormals_to_zero(); }, 0 ); diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index 06a4f2fbbf..375569e2d5 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -17,6 +17,7 @@ #include #include #include "3rdParty/ONNX/OnnxToolsPA.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" #include "Common/Cpp/Logging/AbstractLogger.h" @@ -65,6 +66,7 @@ void SAMEmbedderSession::run(const cv::Mat& input_image, std::vector& mod const char* input_name_c = input_names[0].data(); const char* output_name_c = output_names[0].data(); auto start = std::chrono::steady_clock::now(); + FlushDenormalsToZeroScope flush_denormals; // Denormals are slow on Intel CPUs (see FloatingPointMode.h). session.Run(run_options, &input_name_c, &input_tensor, 1, &output_name_c, &output_tensor, 1); auto end = std::chrono::steady_clock::now(); auto milliseconds = std::chrono::duration_cast(end - start).count(); @@ -169,6 +171,7 @@ void SAMSession::run( } auto start = std::chrono::steady_clock::now(); + FlushDenormalsToZeroScope flush_denormals; // Denormals are slow on Intel CPUs (see FloatingPointMode.h). session.Run(run_options, input_names_c.data(), input_tensors.data(), SAM_N_INPUT_TENSORS, output_names_c.data(), output_tensors.data(), SAM_N_OUTPUT_TENSORS); auto end = std::chrono::steady_clock::now(); diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp index 3d83bad3c3..d7690d4f02 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp @@ -11,6 +11,7 @@ #include #include #include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "Common/Cpp/Rectangle.tpp" #include "CommonFramework/Logging/Logger.h" #include "CommonFramework/GlobalAutoPaths.h" @@ -279,6 +280,8 @@ std::vector PaddleOCRDetector::detect(const cv::Mat& image_bgr std::vector outputs; try{ + // Denormal floats are very slow on Intel CPUs (see FloatingPointMode.h). + FlushDenormalsToZeroScope flush_denormals; outputs = m_session.Run( Ort::RunOptions{nullptr}, input_names, &input_tensor, 1, diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 94ff02788e..7fa6de3ced 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -10,6 +10,7 @@ #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/GlobalSettingsPanel.h" @@ -385,7 +386,9 @@ std::string PaddleOCRPipeline::recognize_line(const cv::Mat& cropped_image){ m_logger.log("[OCR-DEBUG] Calling m_rec_session.Run() now..."); } - // 5. Run the recognition session + // 5. Run the recognition session. The large recognition models produce denormal + // floats, which are very slow on Intel CPUs (see FloatingPointMode.h). + FlushDenormalsToZeroScope flush_denormals; auto outputs = m_rec_session.Run( Ort::RunOptions{nullptr}, input_names, // char** diff --git a/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp b/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp index 302955ba01..c69ace4a39 100644 --- a/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp +++ b/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp @@ -27,6 +27,10 @@ Ort::ThreadingOptions make_ort_threading_options(){ ret.SetGlobalInterOpNumThreads(threads); } ret.SetGlobalSpinControl(0); + // Flush denormal floats to zero on ONNX Runtime's own threads: the big models produce + // them, and they are very slow on Intel CPUs. ONNX Runtime only supports this on x86; + // see Common/Cpp/Hardware/FloatingPointMode.h. + ret.SetGlobalDenormalAsZero(); return ret; } diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp index e40f4b3ce3..bd0c211525 100644 --- a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp @@ -13,6 +13,7 @@ #include #include #include "3rdParty/ONNX/OnnxToolsPA.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/Logging/Logger.h" #include "ML/Models/ML_ONNXRuntimeHelpers.h" @@ -219,6 +220,7 @@ void YOLOv5Session::run(const cv::Mat& input_image, std::vector(end - start).count(); diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 615f34f3b7..bba787ea5e 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -85,6 +85,7 @@ file(GLOB CORE_LIBRARY_SOURCES ../Common/Cpp/Filesystem/Filesystem_Linux.h ../Common/Cpp/Filesystem/Filesystem_Mac.h ../Common/Cpp/Filesystem/Filesystem_Windows.h + ../Common/Cpp/Hardware/FloatingPointMode.h ../Common/Cpp/Hardware/Hardware.cpp ../Common/Cpp/Hardware/Hardware.h ../Common/Cpp/Hardware/Hardware_arm64_Linux.tpp From 6418a8543df317dd293372cd8745bcc464abb564 Mon Sep 17 00:00:00 2001 From: Gin <> Date: Mon, 19 Oct 2026 08:33:46 -0700 Subject: [PATCH 2/2] Only flush denormals to zero while ONNX Runtime runs a model Review feedback: turning flush-to-zero on for every GlobalThreadPools thread changes floating-point behavior for far more code than needs it. Drop that, so it's only on: - on ONNX Runtime's own thread pool (SetGlobalDenormalAsZero()), and - on the thread calling Run(), only for the length of the call (FlushDenormalsToZeroScope restores the previous mode afterwards). The calling-thread scope is still needed: ONNX Runtime runs part of each inference on the calling thread, which its thread pool option doesn't cover. Co-Authored-By: Claude Opus 5.5 --- Common/Cpp/Hardware/FloatingPointMode.h | 19 +++++++++++++------ .../Tools/GlobalThreadPools.cpp | 9 --------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Common/Cpp/Hardware/FloatingPointMode.h b/Common/Cpp/Hardware/FloatingPointMode.h index 56cc3d9ca2..3295adfc2f 100644 --- a/Common/Cpp/Hardware/FloatingPointMode.h +++ b/Common/Cpp/Hardware/FloatingPointMode.h @@ -12,8 +12,14 @@ * 35-185 s, and under 5 s with flush-to-zero. * * With flush-to-zero, the CPU treats denormal inputs as zero and rounds denormal - * results to zero. OCR, image matching and model inference don't need values - * that small. + * results to zero. Model inference doesn't need values that small. + * + * This program turns it on only while ONNX Runtime runs a model: + * - on ONNX Runtime's own thread pool, with `SetGlobalDenormalAsZero()` in + * ML/Models/ML_OrtEnv.cpp, and + * - on the thread that calls `Run()`, for the length of the call, with + * `FlushDenormalsToZeroScope`. + * All other code keeps the default floating-point behavior. * * The mode belongs to each thread: * - x86-64: the FTZ and DAZ bits of MXCSR, which control SSE and AVX math. @@ -59,7 +65,8 @@ inline void set_floating_point_control(uint64_t value){ } // Turn on flush-to-zero for denormal floats on the current thread, until the -// thread ends or the mode is changed again. +// thread ends or the mode is changed again. Prefer `FlushDenormalsToZeroScope`, +// which restores the previous mode. inline void enable_flush_denormals_to_zero(){ #if defined(__x86_64__) || defined(_M_X64) // 0x8000: FTZ, flush denormal results to zero. @@ -73,9 +80,9 @@ inline void enable_flush_denormals_to_zero(){ // Turn on flush-to-zero on the current thread for this object's lifetime, then -// restore the previous mode. For code that may run on threads this program -// doesn't own, like an ONNX Runtime `Run()` call: ONNX Runtime runs part of -// each inference on the calling thread. +// restore the previous mode. Wrap each ONNX Runtime `Run()` call in one: ONNX +// Runtime runs part of each inference on the calling thread, which can be any +// thread in this program, and its own thread pool doesn't cover that part. class FlushDenormalsToZeroScope{ public: FlushDenormalsToZeroScope(const FlushDenormalsToZeroScope&) = delete; diff --git a/SerialPrograms/Source/CommonFramework/Tools/GlobalThreadPools.cpp b/SerialPrograms/Source/CommonFramework/Tools/GlobalThreadPools.cpp index 4d47f87895..82b9aefe2b 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/GlobalThreadPools.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/GlobalThreadPools.cpp @@ -5,7 +5,6 @@ */ #include "Common/Cpp/Concurrency/Backends/ThreadPool_Default.h" -#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "CommonFramework/Logging/Logger.h" #include "CommonFramework/Options/Environment/PerformanceOptions.h" #include "GlobalThreadPools.h" @@ -13,16 +12,12 @@ namespace PokemonAutomation{ namespace GlobalThreadPools{ -// Every pool thread turns on flush-to-zero for denormal floats when it starts. -// Denormals are very slow on Intel CPUs, and image matching and ML inference -// on these threads can produce them. See Common/Cpp/Hardware/FloatingPointMode.h. ThreadPool& computation_realtime(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().REALTIME_THREAD_POOL0.PRIORITY.set_on_this_thread(global_logger_tagged()); - enable_flush_denormals_to_zero(); }, 0, PerformanceOptions::instance().REALTIME_THREAD_POOL0.MAX_THREADS ); @@ -32,7 +27,6 @@ ThreadPool& computation_normal(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().NORMAL_THREAD_POOL.PRIORITY.set_on_this_thread(global_logger_tagged()); - enable_flush_denormals_to_zero(); }, 0, PerformanceOptions::instance().NORMAL_THREAD_POOL.MAX_THREADS ); @@ -43,7 +37,6 @@ ThreadPool& unlimited_realtime(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().REALTIME_THREAD_PRIORITY0.set_on_this_thread(global_logger_tagged()); - enable_flush_denormals_to_zero(); }, 0 ); @@ -53,7 +46,6 @@ ThreadPool& unlimited_pivot(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().INFERENCE_PIVOT_PRIORITY0.set_on_this_thread(global_logger_tagged()); - enable_flush_denormals_to_zero(); }, 0 ); @@ -63,7 +55,6 @@ ThreadPool& unlimited_normal(){ static ThreadPool_Default runner( [](){ PerformanceOptions::instance().NORMAL_THREAD_POOL.PRIORITY.set_on_this_thread(global_logger_tagged()); - enable_flush_denormals_to_zero(); }, 0 );