Merge pull request #1310 from su2code/feature_libROM

Feature libROM for reduced order modelling

Author: jtlau

Common/include/CConfig.hpp

@@ -1123,6 +1123,13 @@ class CConfig {
   INLET_INTERP_TYPE Kind_Inlet_InterpolationType;    /*!brief type of spanwise interpolation data to use for the inlet face. */
   bool PrintInlet_InterpolatedData;               /*!brief option for printing the interpolated data file. */
 
+  /*--- libROM configure options ---*/
+  bool libROM;                              /*!< \brief Toggle saving to libROM. */
+  string libROMbase_FileName;               /*!< \brief Base filename for libROM file saving. */
+  POD_KIND POD_Basis_Gen;                   /*!< \brief Type of POD basis generation (static or incremental). */
+  unsigned short maxBasisDim,               /*!< \brief Maximum number of POD basis dimensions. */
+  rom_save_freq;                            /*!< \brief Frequency of unsteady time steps to save. */
+  
   /* other NEMO configure options*/
   unsigned short nSpecies,                  /*!< \brief No of species present in flow */
   iWall_Catalytic,
@@ -9292,4 +9299,33 @@ class CConfig {
    */
   short FindInterfaceMarker(unsigned short iInterface) const;
 
+  /*!
+   * \brief Get whether or not to save solution data to libROM.
+   * \return True if specified in config file.
+   */
+  bool GetSave_libROM(void) const {return libROM; }
+  
+  /*!
+   * \brief Get the name of the file for libROM to save.
+   * \return Filename prefix for libROM to save to (default: "su2").
+   */
+  string GetlibROMbase_FileName(void) const { return libROMbase_FileName; }
+  
+  /*!
+   * \brief Static or incremental toggle for POD basis generation type.
+   * \return Type of POD generation type
+   */
+  POD_KIND GetKind_PODBasis(void) const { return POD_Basis_Gen; }
+  
+  /*!
+   * \brief Get maximum number of POD basis dimensions (default: 100).
+   * \return Maximum number of POD basis vectors.
+   */
+  unsigned short GetMax_BasisDim(void) const { return maxBasisDim; }
+  
+  /*!
+   * \brief Get frequency of unsteady time steps to save (default: 1).
+   * \return Save frequency for unsteady time steps.
+   */
+  unsigned short GetRom_SaveFreq(void) const { return rom_save_freq; }
 };

Common/include/option_structure.hpp

@@ -2225,6 +2225,18 @@ struct StreamwisePeriodicValues {
   su2double Streamwise_Periodic_InletTemperature;   /*!< \brief Area avg static Temp [K] at the periodic inlet. Used for adaptive outlet heatsink. */
 };
 
+/*!
+ * \brief Type of POD basis generation (for use with libROM)
+ */
+enum class POD_KIND {
+  STATIC,            /*!< \brief Use static SVD for POD basis generation. */
+  INCREMENTAL,       /*!< \brief Use incremental SVD for POD basis generation. */
+};
+static const MapType<std::string, POD_KIND> POD_Map = {
+  MakePair("STATIC_POD",      POD_KIND::STATIC)
+  MakePair("INCREMENTAL_POD", POD_KIND::INCREMENTAL)
+};
+
 #undef MakePair
 /* END_CONFIG_ENUMS */
 

Common/src/CConfig.cpp

@@ -2778,6 +2778,25 @@ void CConfig::SetConfig_Options() {
 
   /* DESCRIPTION: Size of the edge groups colored for thread parallel edge loops (0 forces the reducer strategy). */
   addUnsignedLongOption("EDGE_COLORING_GROUP_SIZE", edgeColorGroupSize, 512);
+  
+  /*--- options that are used for libROM ---*/
+  /*!\par CONFIG_CATEGORY:libROM options \ingroup Config*/
+  
+  /*!\brief SAVE_LIBROM \n DESCRIPTION: Flag for saving data with libROM. */
+  addBoolOption("SAVE_LIBROM", libROM, false);
+  
+  /*!\brief LIBROM_BASE_FILENAME \n DESCRIPTION: Output base file name for libROM   \ingroup Config*/
+  addStringOption("LIBROM_BASE_FILENAME", libROMbase_FileName, string("su2"));
+  
+  /*!\brief BASIS_GENERATION \n DESCRIPTION: Flag for saving data with libROM. */
+  addEnumOption("BASIS_GENERATION", POD_Basis_Gen, POD_Map, POD_KIND::STATIC);
+  
+  /*!\brief MAX_BASIS_DIM \n DESCRIPTION: Maximum number of basis vectors.*/
+  addUnsignedShortOption("MAX_BASIS_DIM", maxBasisDim, 100);
+  
+  /*!\brief MAX_BASIS_DIM \n DESCRIPTION: Maximum number of basis vectors.*/
+  addUnsignedShortOption("ROM_SAVE_FREQ", rom_save_freq, 1);
+  
   /* END_CONFIG_OPTIONS */
 
 }

SU2_CFD/include/solvers/CSolver.hpp

@@ -56,6 +56,12 @@
 #include "../../../Common/include/toolboxes/MMS/CVerificationSolution.hpp"
 #include "../variables/CVariable.hpp"
 
+#ifdef HAVE_LIBROM
+#include "BasisGenerator.h"
+#include "QDEIM.h"
+#include "DEIM.h"
+#endif
+
 using namespace std;
 
 class CSolver {
@@ -196,6 +202,11 @@ class CSolver {
   CVerificationSolution *VerificationSolution; /*!< \brief Verification solution class used within the solver. */
 
   vector<string> fields;
+  
+#ifdef HAVE_LIBROM
+  std::unique_ptr<CAROM::BasisGenerator> u_basis_generator;
+#endif
+  
   /*!
    * \brief Constructor of the class.
    */
@@ -4326,7 +4337,14 @@ class CSolver {
    * \return Struct holding 4 su2doubles.
    */
   virtual StreamwisePeriodicValues GetStreamwisePeriodicValues() const { return StreamwisePeriodicValues(); }
-
+  
+  /*!
+   * \brief Save snapshot or POD data using libROM
+   * \param[in] geometry - Geometrical definition of the problem.
+   * \param[in] config - Definition of the particular problem.
+   * \param[in] converged - Whether or not solution has converged.
+  */
+  void SavelibROM(CGeometry *geometry, CConfig *config, bool converged);
 
 protected:
   /*!

SU2_CFD/src/drivers/CSinglezoneDriver.cpp

@@ -93,6 +93,12 @@ void CSinglezoneDriver::StartSolver() {
     /*--- Output the solution in files. ---*/
 
     Output(TimeIter);
+    
+    /*--- Save iteration solution for libROM ---*/
+    if (config_container[MESH_0]->GetSave_libROM()) {
+      solver_container[ZONE_0][INST_0][MESH_0][FLOW_SOL]->SavelibROM(geometry_container[ZONE_0][INST_0][MESH_0],
+                                                                     config_container[ZONE_0], StopCalc);
+    }
 
     /*--- If the convergence criteria has been met, terminate the simulation. ---*/
 

SU2_CFD/src/solvers/CSolver.cpp

@@ -187,7 +187,6 @@ CSolver::~CSolver(void) {
   delete [] Restart_Data;
 
   delete VerificationSolution;
-
 }
 
 void CSolver::GetPeriodicCommCountAndType(const CConfig* config,
@@ -4212,3 +4211,93 @@ void CSolver::BasicLoadRestart(CGeometry *geometry, const CConfig *config, const
                    string("It could be empty lines at the end of the file."), CURRENT_FUNCTION);
   }
 }
+
+void CSolver::SavelibROM(CGeometry *geometry, CConfig *config, bool converged) {
+  
+#if defined(HAVE_LIBROM) && !defined(CODI_FORWARD_TYPE) && !defined(CODI_REVERSE_TYPE)
+  const bool unsteady            = config->GetTime_Domain();
+  const string filename          = config->GetlibROMbase_FileName();
+  const unsigned long TimeIter   = config->GetTimeIter();
+  const unsigned long nTimeIter  = config->GetnTime_Iter();
+  const int maxBasisDim          = config->GetMax_BasisDim();
+  const int save_freq            = config->GetRom_SaveFreq();
+  int dim = int(nPointDomain * nVar);
+  bool incremental = false;
+
+  if (!u_basis_generator) {
+    
+    /*--- Define SVD basis generator ---*/
+    auto timesteps = static_cast<int>(nTimeIter - TimeIter);
+    CAROM::Options svd_options = CAROM::Options(dim, timesteps, -1,
+                                                false, true).setMaxBasisDimension(int(maxBasisDim));
+    
+    if (config->GetKind_PODBasis() == POD_KIND::STATIC) {
+      if (rank == MASTER_NODE) std::cout << "Creating static basis generator." << std::endl;
+      
+      if (unsteady) {
+        if (rank == MASTER_NODE) std::cout << "Incremental basis generator recommended for unsteady simulations." << std::endl;
+      }
+    }
+    else {
+      if (rank == MASTER_NODE) std::cout << "Creating incremental basis generator." << std::endl;
+      
+      svd_options.setIncrementalSVD(1.0e-3, config->GetDelta_UnstTime(),
+                                    1.0e-2, config->GetDelta_UnstTime()*nTimeIter, true).setDebugMode(false);
+      incremental = true;
+    }
+    
+    u_basis_generator.reset(new CAROM::BasisGenerator(
+      svd_options, incremental,
+      filename));
+    
+    // Save mesh ordering
+    std::ofstream f;
+    f.open(filename + "_mesh_" + to_string(rank) + ".csv");
+      for (unsigned long iPoint = 0; iPoint < nPointDomain; iPoint++) {
+        unsigned long globalPoint = geometry->nodes->GetGlobalIndex(iPoint);
+        auto Coord = geometry->nodes->GetCoord(iPoint);
+        
+        for (unsigned long iDim; iDim < nDim; iDim++) {
+          f << Coord[iDim] << ", ";
+        }
+        f << globalPoint << "\n";
+      }
+    f.close();
+  }
+
+  if (unsteady && (TimeIter % save_freq == 0)) {
+    // give solution and time steps to libROM:
+    su2double dt = config->GetDelta_UnstTime();
+    su2double t =  config->GetCurrent_UnstTime();
+    u_basis_generator->takeSample(const_cast<su2double*>(base_nodes->GetSolution().data()), t, dt);
+  }
+   
+  /*--- End collection of data and save POD ---*/
+  
+  if (converged) {
+  
+    if (!unsteady) {
+       // dt is different for each node, so just use a placeholder dt
+       su2double dt = base_nodes->GetDelta_Time(0);
+       su2double t = dt*TimeIter;
+       u_basis_generator->takeSample(const_cast<su2double*>(base_nodes->GetSolution().data()), t, dt);
+    }
+    
+    if (config->GetKind_PODBasis() == POD_KIND::STATIC) {
+      u_basis_generator->writeSnapshot();
+    }
+    
+    if (rank == MASTER_NODE) std::cout << "Computing SVD" << std::endl;
+    int rom_dim = u_basis_generator->getSpatialBasis()->numColumns();
+    
+    if (rank == MASTER_NODE) std::cout << "Basis dimension: " << rom_dim << std::endl;
+    u_basis_generator->endSamples();
+    
+    if (rank == MASTER_NODE) std::cout << "ROM Sampling ended" << std::endl;
+  }
+  
+#else
+  SU2_MPI::Error("SU2 was not compiled with libROM support.", CURRENT_FUNCTION);
+#endif
+
+}

config_template.cfg

@@ -1692,3 +1692,21 @@ DEFINITION_DV= ( 1, 1.0 | airfoil | 0, 0.05 ); ( 1, 1.0 | airfoil | 0, 0.10 ); (
 %
 % Use combined objective within gradient evaluation: may reduce cost to compute gradients when using the adjoint formulation.
 OPT_COMBINE_OBJECTIVE = NO
+%
+% --------------------- LIBROM PARAMETERS -----------------------%
+% LibROM can be found here: https://github.com/LLNL/libROM
+%
+% Toggle saving to librom (NO, YES)
+SAVE_LIBROM = NO
+%
+% Prefix to the saved libROM files (default: su2)
+LIBROM_BASE_FILENAME = su2
+%
+% Specify POD basis generation algorithm (STATIC_POD, INCREMENTAL_POD) 
+% STATIC_POD recommended for steady problems
+BASIS_GENERATION = STATIC_POD
+%
+% Maximum number of basis vectors to keep (default: 100)
+MAX_BASIS_DIM = 100
+%
+

meson.build

@@ -169,6 +169,21 @@ elif get_option('enable-openblas')
 
 endif
 
+if get_option('enable-librom')
+   
+   assert(get_option('librom_root')!='',
+       'Must specify librom folder (-Dlibrom_root=path/to/libROM)')
+   
+   su2_cpp_args += '-DHAVE_LIBROM'
+
+   librom_root = get_option('librom_root')
+   librom_dep = declare_dependency(include_directories: librom_root,
+                                   link_args: ['-L'+librom_root+'/build', '-lROM'])
+
+   su2_deps += librom_dep
+
+endif
+
 extra_deps = get_option('extra-deps').split(',')
 foreach dep : extra_deps
   if dep != ''
@@ -234,6 +249,7 @@ message('''---------------------------------------------------------------------
          OpenBlas:       @8@
          PaStiX:         @9@
          Mixed Float:    @10@
+         libROM:         @11@
 
          Please be sure to add the $SU2_HOME and $SU2_RUN environment variables,
          and update your $PATH (and $PYTHONPATH if applicable) with $SU2_RUN
@@ -245,10 +261,10 @@ message('''---------------------------------------------------------------------
          export PATH=$PATH:$SU2_RUN
          export PYTHONPATH=$PYTHONPATH:$SU2_RUN
 
-         Use './ninja -C @11@ install' to compile and install SU2
+         Use './ninja -C @12@ install' to compile and install SU2
 '''.format(get_option('prefix')+'/bin', meson.source_root(), get_option('enable-tecio'), get_option('enable-cgns'),
            get_option('enable-autodiff'), get_option('enable-directdiff'), get_option('enable-pywrapper'), get_option('enable-mkl'),
-           get_option('enable-openblas'), get_option('enable-pastix'), get_option('enable-mixedprec'), meson.build_root().split('/')[-1]))
+           get_option('enable-openblas'), get_option('enable-pastix'), get_option('enable-mixedprec'), get_option('enable-librom'), meson.build_root().split('/')[-1]))
 
 if get_option('enable-mpp')
   message(''' To run SU2 with Mutation++ library, add these lines to your .bashrc file:
@@ -258,3 +274,12 @@ if get_option('enable-mpp')
 
           ''')
 endif
+
+if get_option('enable-librom')
+  message(''' To run SU2 with libROM library, add this line to your .bashrc file:
+
+          export LD_LIBRARY_PATH=@0@/build/:$LD_LIBRARY_PATH
+          
+          '''.format(get_option('librom_root')))
+
+endif

meson_options.txt

@@ -19,3 +19,5 @@ option('enable-mixedprec', type : 'boolean', value : false, description: 'use si
 option('extra-deps', type : 'string', value : '', description: 'comma-separated list of extra (custom) dependencies to add for compilation')
 option('enable-mpp',  type : 'boolean', value : false, description: 'enable Mutation++ support')
 option('opdi-backend', type : 'combo', choices : ['auto', 'macro', 'ompt'], value : 'auto', description: 'OpDiLib backend choice')
+option('librom_root', type : 'string', value : '', description: 'libROM base directory')
+option('enable-librom', type : 'boolean', value : false, description: 'enable LLNL libROM support')