Flush denormal floats to zero for ONNX inference and compute threads

Common/Cpp/Hardware/FloatingPointMode.h

@@ -0,0 +1,106 @@
+/*  Floating-Point Mode
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ *  Turn on flush-to-zero for denormal floats on the current thread.
+ *
+ *  Denormal (subnormal) floats are nonzero values too small for the normal float
+ *  format: below about 1.2e-38 for float. Intel CPUs handle many operations on one
+ *  in microcode, which is many times slower than normal float math. AMD and Apple
+ *  CPUs pay much less. The large PaddleOCR model for Chinese and Japanese produces
+ *  them: on the Mac Intel CI runner (an i7-8700B), each of those OCR tests took
+ *  35-185 s, and under 5 s with flush-to-zero.
+ *
+ *  With flush-to-zero, the CPU treats denormal inputs as zero and rounds denormal
+ *  results to zero. Model inference doesn't need values that small.
+ *
+ *  This program turns it on only while ONNX Runtime runs a model:
+ *  - on ONNX Runtime's own thread pool, with `SetGlobalDenormalAsZero()` in
+ *    ML/Models/ML_OrtEnv.cpp, and
+ *  - on the thread that calls `Run()`, for the length of the call, with
+ *    `FlushDenormalsToZeroScope`.
+ *  All other code keeps the default floating-point behavior.
+ *
+ *  The mode belongs to each thread:
+ *  - x86-64: the FTZ and DAZ bits of MXCSR, which control SSE and AVX math.
+ *  - ARM64 (GCC and Clang): the FZ bit of FPCR, which controls scalar and NEON math.
+ *  - Anywhere else, these functions do nothing.
+ *
+ */
+
+#ifndef PokemonAutomation_Hardware_FloatingPointMode_H
+#define PokemonAutomation_Hardware_FloatingPointMode_H
+
+#include <stdint.h>
+#if defined(__x86_64__) || defined(_M_X64)
+#include <xmmintrin.h>
+#endif
+
+namespace PokemonAutomation{
+
+
+// Return the current thread's floating-point control register, so it can be
+// restored later with `set_floating_point_control()`.
+inline uint64_t get_floating_point_control(){
+#if defined(__x86_64__) || defined(_M_X64)
+    return _mm_getcsr();
+#elif defined(__aarch64__)
+    uint64_t fpcr;
+    __asm__ __volatile__("mrs %0, fpcr" : "=r"(fpcr));
+    return fpcr;
+#else
+    return 0;
+#endif
+}
+
+// Restore a value returned by `get_floating_point_control()`.
+inline void set_floating_point_control(uint64_t value){
+#if defined(__x86_64__) || defined(_M_X64)
+    _mm_setcsr((unsigned int)value);
+#elif defined(__aarch64__)
+    __asm__ __volatile__("msr fpcr, %0" : : "r"(value));
+#else
+    (void)value;
+#endif
+}
+
+// Turn on flush-to-zero for denormal floats on the current thread, until the
+// thread ends or the mode is changed again. Prefer `FlushDenormalsToZeroScope`,
+// which restores the previous mode.
+inline void enable_flush_denormals_to_zero(){
+#if defined(__x86_64__) || defined(_M_X64)
+    //  0x8000: FTZ, flush denormal results to zero.
+    //  0x0040: DAZ, treat denormal inputs as zero.
+    _mm_setcsr(_mm_getcsr() | 0x8040);
+#elif defined(__aarch64__)
+    //  Bit 24: FZ, flush denormal inputs and results to zero.
+    set_floating_point_control(get_floating_point_control() | ((uint64_t)1 << 24));
+#endif
+}
+
+
+// Turn on flush-to-zero on the current thread for this object's lifetime, then
+// restore the previous mode. Wrap each ONNX Runtime `Run()` call in one: ONNX
+// Runtime runs part of each inference on the calling thread, which can be any
+// thread in this program, and its own thread pool doesn't cover that part.
+class FlushDenormalsToZeroScope{
+public:
+    FlushDenormalsToZeroScope(const FlushDenormalsToZeroScope&) = delete;
+    void operator=(const FlushDenormalsToZeroScope&) = delete;
+
+    FlushDenormalsToZeroScope()
+        : m_saved(get_floating_point_control())
+    {
+        enable_flush_denormals_to_zero();
+    }
+    ~FlushDenormalsToZeroScope(){
+        set_floating_point_control(m_saved);
+    }
+
+private:
+    uint64_t m_saved;
+};
+
+
+}
+#endif

SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp

@@ -17,6 +17,7 @@
 #include <opencv2/imgcodecs.hpp>
 #include <opencv2/imgproc.hpp>
 #include "3rdParty/ONNX/OnnxToolsPA.h"
+#include "Common/Cpp/Hardware/FloatingPointMode.h"
 #include "Common/Cpp/Exceptions.h"
 #include "Common/Cpp/Filesystem/Filesystem.h"
 #include "Common/Cpp/Logging/AbstractLogger.h"
@@ -65,6 +66,7 @@ void SAMEmbedderSession::run(const cv::Mat& input_image, std::vector<float>& mod
     const char* input_name_c = input_names[0].data();
     const char* output_name_c = output_names[0].data();
     auto start = std::chrono::steady_clock::now();
+    FlushDenormalsToZeroScope flush_denormals;  //  Denormals are slow on Intel CPUs (see FloatingPointMode.h).
     session.Run(run_options, &input_name_c, &input_tensor, 1, &output_name_c, &output_tensor, 1);
     auto end = std::chrono::steady_clock::now();
     auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
@@ -169,6 +171,7 @@ void SAMSession::run(
     }
 
     auto start = std::chrono::steady_clock::now();
+    FlushDenormalsToZeroScope flush_denormals;  //  Denormals are slow on Intel CPUs (see FloatingPointMode.h).
     session.Run(run_options, input_names_c.data(), input_tensors.data(), SAM_N_INPUT_TENSORS,
         output_names_c.data(), output_tensors.data(), SAM_N_OUTPUT_TENSORS);
     auto end = std::chrono::steady_clock::now();

SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp

@@ -11,6 +11,7 @@
 #include <opencv2/core.hpp>
 #include <opencv2/imgproc.hpp>
 #include "Common/Cpp/Exceptions.h"
+#include "Common/Cpp/Hardware/FloatingPointMode.h"
 #include "Common/Cpp/Rectangle.tpp"
 #include "CommonFramework/Logging/Logger.h"
 #include "CommonFramework/GlobalAutoPaths.h"
@@ -279,6 +280,8 @@ std::vector<PaddleOCRTextBox> PaddleOCRDetector::detect(const cv::Mat& image_bgr
 
     std::vector<Ort::Value> outputs;
     try{
+        //  Denormal floats are very slow on Intel CPUs (see FloatingPointMode.h).
+        FlushDenormalsToZeroScope flush_denormals;
         outputs = m_session.Run(
             Ort::RunOptions{nullptr},
             input_names, &input_tensor, 1,

SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp

@@ -10,6 +10,7 @@
 #include <limits>
 #include "Common/Cpp/Exceptions.h"
 #include "Common/Cpp/Filesystem/Filesystem.h"
+#include "Common/Cpp/Hardware/FloatingPointMode.h"
 #include "Common/Cpp/Logging/GlobalLogger.h"
 #include "CommonFramework/GlobalAutoPaths.h"
 #include "CommonFramework/GlobalSettingsPanel.h"
@@ -385,7 +386,9 @@ std::string PaddleOCRPipeline::recognize_line(const cv::Mat& cropped_image){
             m_logger.log("[OCR-DEBUG] Calling m_rec_session.Run() now...");
         }
 
-        // 5. Run the recognition session
+        // 5. Run the recognition session. The large recognition models produce denormal
+        //    floats, which are very slow on Intel CPUs (see FloatingPointMode.h).
+        FlushDenormalsToZeroScope flush_denormals;
         auto outputs = m_rec_session.Run(
             Ort::RunOptions{nullptr}, 
             input_names,   // char** 

SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp

@@ -27,6 +27,10 @@ Ort::ThreadingOptions make_ort_threading_options(){
         ret.SetGlobalInterOpNumThreads(threads);
     }
     ret.SetGlobalSpinControl(0);
+    //  Flush denormal floats to zero on ONNX Runtime's own threads: the big models produce
+    //  them, and they are very slow on Intel CPUs. ONNX Runtime only supports this on x86;
+    //  see Common/Cpp/Hardware/FloatingPointMode.h.
+    ret.SetGlobalDenormalAsZero();
     return ret;
 }
 

SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp

@@ -13,6 +13,7 @@
 #include <opencv2/imgproc.hpp>
 #include <opencv2/dnn.hpp>
 #include "3rdParty/ONNX/OnnxToolsPA.h"
+#include "Common/Cpp/Hardware/FloatingPointMode.h"
 #include "CommonFramework/GlobalAutoPaths.h"
 #include "CommonFramework/Logging/Logger.h"
 #include "ML/Models/ML_ONNXRuntimeHelpers.h"
@@ -219,6 +220,7 @@ void YOLOv5Session::run(const cv::Mat& input_image, std::vector<YOLOv5Session::D
     const char* input_name_c = m_input_names[0].data();
     const char* output_name_c = m_output_names[0].data();
     // auto start = std::chrono::steady_clock::now();
+    FlushDenormalsToZeroScope flush_denormals;  //  Denormals are slow on Intel CPUs (see FloatingPointMode.h).
     m_session.Run(m_run_options, &input_name_c, &input_tensor, 1, &output_name_c, &output_tensor, 1);
     // auto end = std::chrono::steady_clock::now();
     // auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();

SerialPrograms/cmake/SourceFiles.cmake

@@ -85,6 +85,7 @@ file(GLOB CORE_LIBRARY_SOURCES
     ../Common/Cpp/Filesystem/Filesystem_Linux.h
     ../Common/Cpp/Filesystem/Filesystem_Mac.h
     ../Common/Cpp/Filesystem/Filesystem_Windows.h
+    ../Common/Cpp/Hardware/FloatingPointMode.h
     ../Common/Cpp/Hardware/Hardware.cpp
     ../Common/Cpp/Hardware/Hardware.h
     ../Common/Cpp/Hardware/Hardware_arm64_Linux.tpp