diff --git a/Common/Cpp/Hardware/FloatingPointMode.h b/Common/Cpp/Hardware/FloatingPointMode.h new file mode 100644 index 0000000000..3295adfc2f --- /dev/null +++ b/Common/Cpp/Hardware/FloatingPointMode.h @@ -0,0 +1,106 @@ +/* Floating-Point Mode + * + * From: https://github.com/PokemonAutomation/ + * + * Turn on flush-to-zero for denormal floats on the current thread. + * + * Denormal (subnormal) floats are nonzero values too small for the normal float + * format: below about 1.2e-38 for float. Intel CPUs handle many operations on one + * in microcode, which is many times slower than normal float math. AMD and Apple + * CPUs pay much less. The large PaddleOCR model for Chinese and Japanese produces + * them: on the Mac Intel CI runner (an i7-8700B), each of those OCR tests took + * 35-185 s, and under 5 s with flush-to-zero. + * + * With flush-to-zero, the CPU treats denormal inputs as zero and rounds denormal + * results to zero. Model inference doesn't need values that small. + * + * This program turns it on only while ONNX Runtime runs a model: + * - on ONNX Runtime's own thread pool, with `SetGlobalDenormalAsZero()` in + * ML/Models/ML_OrtEnv.cpp, and + * - on the thread that calls `Run()`, for the length of the call, with + * `FlushDenormalsToZeroScope`. + * All other code keeps the default floating-point behavior. + * + * The mode belongs to each thread: + * - x86-64: the FTZ and DAZ bits of MXCSR, which control SSE and AVX math. + * - ARM64 (GCC and Clang): the FZ bit of FPCR, which controls scalar and NEON math. + * - Anywhere else, these functions do nothing. + * + */ + +#ifndef PokemonAutomation_Hardware_FloatingPointMode_H +#define PokemonAutomation_Hardware_FloatingPointMode_H + +#include +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif + +namespace PokemonAutomation{ + + +// Return the current thread's floating-point control register, so it can be +// restored later with `set_floating_point_control()`. +inline uint64_t get_floating_point_control(){ +#if defined(__x86_64__) || defined(_M_X64) + return _mm_getcsr(); +#elif defined(__aarch64__) + uint64_t fpcr; + __asm__ __volatile__("mrs %0, fpcr" : "=r"(fpcr)); + return fpcr; +#else + return 0; +#endif +} + +// Restore a value returned by `get_floating_point_control()`. +inline void set_floating_point_control(uint64_t value){ +#if defined(__x86_64__) || defined(_M_X64) + _mm_setcsr((unsigned int)value); +#elif defined(__aarch64__) + __asm__ __volatile__("msr fpcr, %0" : : "r"(value)); +#else + (void)value; +#endif +} + +// Turn on flush-to-zero for denormal floats on the current thread, until the +// thread ends or the mode is changed again. Prefer `FlushDenormalsToZeroScope`, +// which restores the previous mode. +inline void enable_flush_denormals_to_zero(){ +#if defined(__x86_64__) || defined(_M_X64) + // 0x8000: FTZ, flush denormal results to zero. + // 0x0040: DAZ, treat denormal inputs as zero. + _mm_setcsr(_mm_getcsr() | 0x8040); +#elif defined(__aarch64__) + // Bit 24: FZ, flush denormal inputs and results to zero. + set_floating_point_control(get_floating_point_control() | ((uint64_t)1 << 24)); +#endif +} + + +// Turn on flush-to-zero on the current thread for this object's lifetime, then +// restore the previous mode. Wrap each ONNX Runtime `Run()` call in one: ONNX +// Runtime runs part of each inference on the calling thread, which can be any +// thread in this program, and its own thread pool doesn't cover that part. +class FlushDenormalsToZeroScope{ +public: + FlushDenormalsToZeroScope(const FlushDenormalsToZeroScope&) = delete; + void operator=(const FlushDenormalsToZeroScope&) = delete; + + FlushDenormalsToZeroScope() + : m_saved(get_floating_point_control()) + { + enable_flush_denormals_to_zero(); + } + ~FlushDenormalsToZeroScope(){ + set_floating_point_control(m_saved); + } + +private: + uint64_t m_saved; +}; + + +} +#endif diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index 06a4f2fbbf..375569e2d5 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -17,6 +17,7 @@ #include #include #include "3rdParty/ONNX/OnnxToolsPA.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" #include "Common/Cpp/Logging/AbstractLogger.h" @@ -65,6 +66,7 @@ void SAMEmbedderSession::run(const cv::Mat& input_image, std::vector& mod const char* input_name_c = input_names[0].data(); const char* output_name_c = output_names[0].data(); auto start = std::chrono::steady_clock::now(); + FlushDenormalsToZeroScope flush_denormals; // Denormals are slow on Intel CPUs (see FloatingPointMode.h). session.Run(run_options, &input_name_c, &input_tensor, 1, &output_name_c, &output_tensor, 1); auto end = std::chrono::steady_clock::now(); auto milliseconds = std::chrono::duration_cast(end - start).count(); @@ -169,6 +171,7 @@ void SAMSession::run( } auto start = std::chrono::steady_clock::now(); + FlushDenormalsToZeroScope flush_denormals; // Denormals are slow on Intel CPUs (see FloatingPointMode.h). session.Run(run_options, input_names_c.data(), input_tensors.data(), SAM_N_INPUT_TENSORS, output_names_c.data(), output_tensors.data(), SAM_N_OUTPUT_TENSORS); auto end = std::chrono::steady_clock::now(); diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp index 3d83bad3c3..d7690d4f02 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp @@ -11,6 +11,7 @@ #include #include #include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "Common/Cpp/Rectangle.tpp" #include "CommonFramework/Logging/Logger.h" #include "CommonFramework/GlobalAutoPaths.h" @@ -279,6 +280,8 @@ std::vector PaddleOCRDetector::detect(const cv::Mat& image_bgr std::vector outputs; try{ + // Denormal floats are very slow on Intel CPUs (see FloatingPointMode.h). + FlushDenormalsToZeroScope flush_denormals; outputs = m_session.Run( Ort::RunOptions{nullptr}, input_names, &input_tensor, 1, diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 94ff02788e..7fa6de3ced 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -10,6 +10,7 @@ #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/GlobalSettingsPanel.h" @@ -385,7 +386,9 @@ std::string PaddleOCRPipeline::recognize_line(const cv::Mat& cropped_image){ m_logger.log("[OCR-DEBUG] Calling m_rec_session.Run() now..."); } - // 5. Run the recognition session + // 5. Run the recognition session. The large recognition models produce denormal + // floats, which are very slow on Intel CPUs (see FloatingPointMode.h). + FlushDenormalsToZeroScope flush_denormals; auto outputs = m_rec_session.Run( Ort::RunOptions{nullptr}, input_names, // char** diff --git a/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp b/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp index 302955ba01..c69ace4a39 100644 --- a/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp +++ b/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp @@ -27,6 +27,10 @@ Ort::ThreadingOptions make_ort_threading_options(){ ret.SetGlobalInterOpNumThreads(threads); } ret.SetGlobalSpinControl(0); + // Flush denormal floats to zero on ONNX Runtime's own threads: the big models produce + // them, and they are very slow on Intel CPUs. ONNX Runtime only supports this on x86; + // see Common/Cpp/Hardware/FloatingPointMode.h. + ret.SetGlobalDenormalAsZero(); return ret; } diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp index e40f4b3ce3..bd0c211525 100644 --- a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp @@ -13,6 +13,7 @@ #include #include #include "3rdParty/ONNX/OnnxToolsPA.h" +#include "Common/Cpp/Hardware/FloatingPointMode.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/Logging/Logger.h" #include "ML/Models/ML_ONNXRuntimeHelpers.h" @@ -219,6 +220,7 @@ void YOLOv5Session::run(const cv::Mat& input_image, std::vector(end - start).count(); diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 615f34f3b7..bba787ea5e 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -85,6 +85,7 @@ file(GLOB CORE_LIBRARY_SOURCES ../Common/Cpp/Filesystem/Filesystem_Linux.h ../Common/Cpp/Filesystem/Filesystem_Mac.h ../Common/Cpp/Filesystem/Filesystem_Windows.h + ../Common/Cpp/Hardware/FloatingPointMode.h ../Common/Cpp/Hardware/Hardware.cpp ../Common/Cpp/Hardware/Hardware.h ../Common/Cpp/Hardware/Hardware_arm64_Linux.tpp