Speedup CI and Improve Reproducibility Across Compilers (#3823)

Co-authored-by: John Tramm <jtramm@gmail.com>
Co-authored-by: Paul Romano <paul.k.romano@gmail.com>

Author: 3 people

AGENTS.md

@@ -184,7 +184,7 @@ def test_my_feature():
     harness.main()
 ```
 
-**Workflow**: Create `test.py` and `__init__.py` in `tests/regression_tests/my_test/`, run `pytest --update` to generate reference files (`inputs_true.dat`, `results_true.dat`, etc.), then verify with `pytest` without `--update`. Test results should be generated with a debug build (`-DCMAKE_BUILD_TYPE=Debug`)
+**Workflow**: Create `test.py` and `__init__.py` in `tests/regression_tests/my_test/`, run `pytest --update` to generate reference files (`inputs_true.dat`, `results_true.dat`, etc.), then verify with `pytest` without `--update`. Test results should be generated with `-DOPENMC_ENABLE_STRICT_FP=on` to ensure reproducibility across platforms and optimization levels.
 
 **Critical**: When modifying OpenMC code, regenerate affected test references with `pytest --update` and commit updated reference files.
 

CMakeLists.txt

@@ -38,6 +38,7 @@ option(OPENMC_USE_LIBMESH     "Enable support for libMesh unstructured mesh tall
 option(OPENMC_USE_MPI         "Enable MPI"                                           OFF)
 option(OPENMC_USE_UWUW        "Enable UWUW"                                          OFF)
 option(OPENMC_FORCE_VENDORED_LIBS "Explicitly use submodules defined in 'vendor'"    OFF)
+option(OPENMC_ENABLE_STRICT_FP "Enable strict FP flags to improve test portability"  OFF)
 
 message(STATUS "OPENMC_USE_OPENMP ${OPENMC_USE_OPENMP}")
 message(STATUS "OPENMC_BUILD_TESTS ${OPENMC_BUILD_TESTS}")
@@ -48,6 +49,7 @@ message(STATUS "OPENMC_USE_LIBMESH ${OPENMC_USE_LIBMESH}")
 message(STATUS "OPENMC_USE_MPI ${OPENMC_USE_MPI}")
 message(STATUS "OPENMC_USE_UWUW ${OPENMC_USE_UWUW}")
 message(STATUS "OPENMC_FORCE_VENDORED_LIBS ${OPENMC_FORCE_VENDORED_LIBS}")
+message(STATUS "OPENMC_ENABLE_STRICT_FP ${OPENMC_ENABLE_STRICT_FP}")
 
 # Warnings for deprecated options
 foreach(OLD_OPT IN ITEMS "openmp" "profile" "coverage" "dagmc" "libmesh")
@@ -89,6 +91,19 @@ if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build" FORCE)
 endif()
 
+#===============================================================================
+# When STRICT_FP is enabled, remove NDEBUG from RelWithDebInfo flags so that
+# assert() remains active. CMake normally adds -DNDEBUG for both Release and
+# RelWithDebInfo, which disables C/C++ assert() statements.
+#===============================================================================
+
+if(OPENMC_ENABLE_STRICT_FP)
+  foreach(FLAG_VAR CMAKE_CXX_FLAGS_RELWITHDEBINFO CMAKE_C_FLAGS_RELWITHDEBINFO)
+    string(REPLACE "-DNDEBUG" "" ${FLAG_VAR} "${${FLAG_VAR}}")
+    string(REPLACE "/DNDEBUG" "" ${FLAG_VAR} "${${FLAG_VAR}}")
+  endforeach()
+endif()
+
 #===============================================================================
 # OpenMP for shared-memory parallelism (and GPU support some day!)
 #===============================================================================
@@ -193,6 +208,26 @@ endif()
 # Set compile/link flags based on which compiler is being used
 #===============================================================================
 
+# When OPENMC_ENABLE_STRICT_FP is enabled, disable compiler optimizations that change
+# floating-point results relative to -O0, improving cross-platform and
+# cross-optimization-level reproducibility for regression testing:
+#   -ffp-contract=off    Prevents FMA contraction (fused multiply-add changes rounding)
+#   -fno-builtin         Prevents replacing math function calls (pow, exp, log, etc.)
+#                        with builtin versions that may differ from libm
+# By default (OFF), the compiler is free to use all optimizations for best
+# performance.
+if(OPENMC_ENABLE_STRICT_FP)
+  include(CheckCXXCompilerFlag)
+  check_cxx_compiler_flag(-ffp-contract=off SUPPORTS_FP_CONTRACT_OFF)
+  if(SUPPORTS_FP_CONTRACT_OFF)
+    list(APPEND cxxflags -ffp-contract=off)
+  endif()
+  check_cxx_compiler_flag(-fno-builtin SUPPORTS_NO_BUILTIN)
+  if(SUPPORTS_NO_BUILTIN)
+    list(APPEND cxxflags -fno-builtin)
+  endif()
+endif()
+
 # Skip for Visual Studio which has its own configurations through GUI
 if(NOT MSVC)
 
@@ -539,6 +574,9 @@ endif()
 if (OPENMC_ENABLE_COVERAGE)
   target_compile_definitions(libopenmc PRIVATE COVERAGEBUILD)
 endif()
+if (OPENMC_ENABLE_STRICT_FP)
+  target_compile_definitions(libopenmc PRIVATE OPENMC_ENABLE_STRICT_FP)
+endif()
 
 #===============================================================================
 # openmc executable

docs/source/devguide/tests.rst

@@ -37,6 +37,9 @@ Prerequisites
 - Some tests require `NJOY <https://www.njoy21.io/NJOY2016>`_ to preprocess
   cross section data. The test suite assumes that you have an ``njoy``
   executable available on your :envvar:`PATH`.
+- OpenMC should be compiled with ``-DOPENMC_ENABLE_STRICT_FP=on`` to ensure
+  reproducible floating-point results across platforms and optimization levels.
+  Without this flag, regression tests may not match reference values.
 
 Running Tests
 -------------
@@ -67,9 +70,11 @@ make sure you have satisfied all the prerequisites above. After you have done
 that, consider the following:
 
 - When building OpenMC, make sure you run CMake with
-  ``-DCMAKE_BUILD_TYPE=Debug``. Building with a release build will result in
-  some test failures due to differences in which compiler optimizations are
-  used.
+  ``-DOPENMC_ENABLE_STRICT_FP=on``. This prevents the compiler from applying
+  floating-point optimizations (such as replacing math library calls with
+  builtins or contracting multiply-add into FMA instructions) that can produce
+  bit-level differences across platforms and optimization levels. Any
+  ``CMAKE_BUILD_TYPE`` can be used.
 - Because tallies involve the sum of many floating point numbers, the
   non-associativity of floating point numbers can result in different answers
   especially when the number of threads is high (different order of operations).

docs/source/usersguide/install.rst

@@ -383,6 +383,20 @@ OPENMC_USE_MPI
   options, please see the `FindMPI.cmake documentation
   <https://cmake.org/cmake/help/latest/module/FindMPI.html>`_.
 
+.. _cmake_strict_fp:
+
+OPENMC_ENABLE_STRICT_FP
+  Disables compiler optimizations that change floating-point results relative to
+  unoptimized builds, improving cross-platform and cross-optimization-level
+  reproducibility. This disables FMA contraction (``-ffp-contract=off``) and
+  compiler builtin replacements of math functions like ``pow``, ``exp``, ``log``
+  (``-fno-builtin``). It also keeps C/C++ assertions active by removing the
+  ``-DNDEBUG`` flag from ``RelWithDebInfo`` builds. Without this flag, these
+  optimizations can produce bit-level differences across platforms, compilers,
+  and optimization levels. This option should be used when running the test
+  suite. By default (off), the compiler is free to use all optimizations for
+  best performance. (Default: off)
+
 OPENMC_FORCE_VENDORED_LIBS
   Forces OpenMC to use the submodules located in the vendor directory, as
   opposed to searching the system for already installed versions of those
@@ -415,7 +429,10 @@ Release
 
 RelWithDebInfo
   (Default if no type is specified.) Enable optimization and debug. On most
-  platforms/compilers, this is equivalent to `-O2 -g`.
+  platforms/compilers, this is equivalent to `-O2 -g`. When
+  :ref:`OPENMC_ENABLE_STRICT_FP <cmake_strict_fp>` is enabled, OpenMC removes the
+  ``-DNDEBUG`` flag that CMake normally adds for this build type, so that
+  C/C++ assertions remain active.
 
 Example of configuring for Debug mode:
 

include/openmc/output.h

@@ -10,6 +10,8 @@
 
 namespace openmc {
 
+extern "C" const bool STRICT_FP_ENABLED;
+
 //! \brief Display the main title banner as well as information about the
 //! program developers, version, and date/time which the problem was run.
 void title();

openmc/lib/__init__.py

@@ -49,6 +49,9 @@ def _libmesh_enabled():
 def _uwuw_enabled():
     return c_bool.in_dll(_dll, "UWUW_ENABLED").value
 
+def _strict_fp_enabled():
+    return c_bool.in_dll(_dll, "STRICT_FP_ENABLED").value
+
 
 from .error import *
 from .core import *

src/output.cpp

@@ -44,6 +44,12 @@
 
 namespace openmc {
 
+#ifdef OPENMC_ENABLE_STRICT_FP
+const bool STRICT_FP_ENABLED = true;
+#else
+const bool STRICT_FP_ENABLED = false;
+#endif
+
 //==============================================================================
 
 void title()
@@ -317,6 +323,7 @@ void print_build_info()
   std::string coverage(n);
   std::string mcpl(n);
   std::string uwuw(n);
+  std::string strict_fp(n);
 
 #ifdef PHDF5
   phdf5 = y;
@@ -345,6 +352,9 @@ void print_build_info()
 #ifdef OPENMC_UWUW_ENABLED
   uwuw = y;
 #endif
+#ifdef OPENMC_ENABLE_STRICT_FP
+  strict_fp = y;
+#endif
 
   // Wraps macro variables in quotes
 #define STRINGIFY(x) STRINGIFY2(x)
@@ -363,6 +373,7 @@ void print_build_info()
     fmt::print("Coverage testing:      {}\n", coverage);
     fmt::print("Profiling flags:       {}\n", profiling);
     fmt::print("UWUW support:          {}\n", uwuw);
+    fmt::print("Strict FP:             {}\n", strict_fp);
   }
 }
 

tests/conftest.py

@@ -1,9 +1,51 @@
 import os
+import hashlib
 import pytest
 import openmc
+import openmc.lib
 
 from tests.regression_tests import config as regression_config
 
+# MD5 hash of the official NNDC HDF5 cross_sections.xml file.
+# Generated via: md5sum /path/to/nndc_hdf5/cross_sections.xml
+_NNDC_XS_MD5 = "2d00773012eda670bc9f95d96a31c989"
+
+# Collected during pytest_configure, displayed at start and end of session
+_environment_warnings = []
+
+
+def _check_build_environment():
+    """Check STRICT_FP and cross section data, collecting any warnings."""
+    if not openmc.lib._strict_fp_enabled():
+        _environment_warnings.append(
+            "OpenMC was NOT built with -DOPENMC_ENABLE_STRICT_FP=on. "
+            "Regression test results may not match reference values due to "
+            "compiler floating-point optimizations. Rebuild with "
+            "-DOPENMC_ENABLE_STRICT_FP=on for reproducible results."
+        )
+
+    xs_path = os.environ.get("OPENMC_CROSS_SECTIONS")
+    if not xs_path:
+        _environment_warnings.append(
+            "OPENMC_CROSS_SECTIONS environment variable is not set. "
+            "Regression tests require the NNDC HDF5 cross section data."
+        )
+    elif not os.path.isfile(xs_path):
+        _environment_warnings.append(
+            f"OPENMC_CROSS_SECTIONS ({xs_path}) is not a valid file path. "
+            "Regression tests require the NNDC HDF5 cross section data."
+        )
+    else:
+        with open(xs_path, "rb") as f:
+            md5 = hashlib.md5(f.read()).hexdigest()
+        if md5 != _NNDC_XS_MD5:
+            _environment_warnings.append(
+                f"OPENMC_CROSS_SECTIONS ({xs_path}) does not match the "
+                "official NNDC HDF5 dataset. Regression tests expect the "
+                "NNDC data; results may differ with other cross section "
+                "libraries."
+            )
+
 
 def pytest_addoption(parser):
     parser.addoption('--exe')
@@ -21,6 +63,28 @@ def pytest_configure(config):
         if config.getoption(opt) is not None:
             regression_config[opt] = config.getoption(opt)
 
+    _check_build_environment()
+
+
+def _print_environment_warnings(terminalreporter):
+    """Print environment warnings as a visible section."""
+    if _environment_warnings:
+        terminalreporter.section("OpenMC Environment Warnings")
+        for msg in _environment_warnings:
+            terminalreporter.line(f"WARNING: {msg}", yellow=True)
+        terminalreporter.line("")
+
+
+def pytest_sessionstart(session):
+    """Print environment warnings at the start of the test session."""
+    _print_environment_warnings(session.config.pluginmanager.get_plugin(
+        "terminalreporter"))
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    """Reprint environment warnings at the end so they aren't missed."""
+    _print_environment_warnings(terminalreporter)
+
 
 @pytest.fixture
 def run_in_tmpdir(tmpdir):

tools/ci/gha-install.py

@@ -9,8 +9,8 @@ def install(omp=False, mpi=False, phdf5=False, dagmc=False, libmesh=False):
     os.mkdir('build')
     os.chdir('build')
 
-    # Build in debug mode by default with support for MCPL
-    cmake_cmd = ['cmake', '-DCMAKE_BUILD_TYPE=Debug', '-DOPENMC_USE_MCPL=on']
+    # Build in RelWithDebInfo mode by default with support for MCPL
+    cmake_cmd = ['cmake', '-DCMAKE_BUILD_TYPE=RelWithDebInfo', '-DOPENMC_USE_MCPL=on']
 
     # Turn off OpenMP if specified
     if not omp:
@@ -43,6 +43,9 @@ def install(omp=False, mpi=False, phdf5=False, dagmc=False, libmesh=False):
     # Build in coverage mode for coverage testing
     cmake_cmd.append('-DOPENMC_ENABLE_COVERAGE=on')
 
+    # Enable strict FP for cross-platform reproducibility in CI
+    cmake_cmd.append('-DOPENMC_ENABLE_STRICT_FP=on')
+
     # Build and install
     cmake_cmd.append('..')
     print(' '.join(cmake_cmd))