Merge pull request #192 from mnlevy1981/support_nvhpc

Add support for nvhpc on ncar machines

Author: mnlevy1981

.gitmodules

@@ -2,7 +2,7 @@
   path = MOM6
   url = https://github.com/NCAR/MOM6.git
   fxDONOTUSEurl = https://github.com/NCAR/MOM6.git
-    fxtag = dev/ncar_240923
+    fxtag = dev/ncar_241003
     fxrequired = AlwaysRequired
 
 [submodule "stochastic_physics"]
@@ -16,6 +16,6 @@
   path = externals/MARBL
   url = https://github.com/marbl-ecosys/MARBL.git
   fxDONOTUSEurl = https://github.com/marbl-ecosys/MARBL.git
-    fxtag = marbl0.47.1
+    fxtag = marbl0.48.0
     fxrequired = AlwaysRequired
 

MOM6

@@ -119,9 +119,11 @@
       <option name="wallclock">00:30:00</option>
     </options>
   </test>
-  <test name="SMS_Ld2_D" grid="TL319_t232" compset="C1850MARBL_JRA">
+  <test name="SMS_Ld1_D" grid="TL319_t232" compset="C1850MARBL_JRA">
     <machines>
+      <machine name="derecho" compiler="intel" category="aux_mom"/>
       <machine name="derecho" compiler="intel" category="aux_mom_MARBL"/>
+      <machine name="derecho" compiler="intel" category="prebeta"/>
     </machines>
     <options>
       <option name="wallclock">00:30:00</option>

cime_config/testdefs/testlist_mom.xml

@@ -2,3 +2,5 @@ intel/
 intel-*/
 gnu/
 gnu-*/
+nvhpc/
+nvhpc-*/

externals/MARBL

@@ -18,6 +18,7 @@ FMS_ROOT=${CESM_ROOT}/libraries/FMS
 # Default compiler
 COMPILER="intel"
 MACHINE="ncar"
+DEBUG=0 # Set to False (REPRO Mode)!
 
 # Parse command line arguments
 while [[ "$#" -gt 0 ]]; do
@@ -28,6 +29,8 @@ while [[ "$#" -gt 0 ]]; do
         --machine) 
             MACHINE="$2"
             shift ;;
+        --debug)
+            DEBUG=1 ;;
         *) 
             echo "Unknown parameter passed: $1"
             echo "Usage: $0 [--compiler <compiler>] [--machine <machine>]"
@@ -58,38 +61,63 @@ case $MACHINE in
         JOBS=4
         ;;
     "ncar")
-        JOBS=36
+        JOBS=32
         ;;
     *)
         echo "Invalid machine type for make -j option: $MACHINE"
         exit 1
         ;;
 esac
 
-if [ -e $1 ]; then
-  BLD_ROOT=${COMPILER}
+if [ "${DEBUG}" == 1 ]; then
+  BLD_ROOT=${COMPILER}-debug
 else
-  BLD_ROOT=$1
+  BLD_ROOT=${COMPILER}
+fi
+
+
+# Load modules for NCAR
+if [ "$MACHINE" == "ncar" ]; then
+  HOST=`hostname`
+  # Load modules if on derecho
+  if [ ! "${HOST:0:5}" == "crhtc" ] && [ ! "${HOST:0:6}" == "casper" ]; then
+    module --force purge
+    . /glade/u/apps/derecho/23.09/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/c645/lmod/lmod/init/sh
+    module load cesmdev/1.0 ncarenv/23.09
+    case $COMPILER in
+      "intel" )
+        module load craype intel/2023.2.1 mkl ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2 esmf/8.6.0
+        ;;
+      "gnu" )
+        module load craype gcc/12.2.0 cray-libsci/23.02.1.1 ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2-debug esmf/8.6.0-debug
+        ;;
+      "nvhpc" )
+        module load craype nvhpc/23.7 ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2 esmf/8.6.0
+        ;;
+      *)
+        echo "Not loading any special modules for ${COMPILER}"
+        ;;
+    esac
+  fi
 fi
 
 # 1) Build FMS
 cd ${INTERFACE_ROOT}/standalone/build
 mkdir -p ${BLD_ROOT}/FMS
 cd ${BLD_ROOT}/FMS
 ${MKMF_ROOT}/list_paths ${FMS_ROOT}/src
-# We need shr_const_mod.F90 and shr_kind_mod.F90 from ${SHR_ROOT}/src
-# to build FMS
+# We need shr_const_mod.F90 and shr_kind_mod.F90 from ${SHR_ROOT}/src to build FMS
 echo "${SHR_ROOT}/src/shr_kind_mod.F90" >> path_names
 echo "${SHR_ROOT}/src/shr_const_mod.F90" >> path_names
 ${MKMF_ROOT}/mkmf -t ${TEMPLATE} -p libfms.a -c "-Duse_libMPI -Duse_netCDF -DSPMD" path_names
-make -j${JOBS} NETCDF=3 REPRO=1 libfms.a
+make -j${JOBS} DEBUG=${DEBUG} libfms.a
 
 # 2) Build MOM6
 cd ${INTERFACE_ROOT}/standalone/build
 mkdir -p ${BLD_ROOT}/MOM6
 cd ${BLD_ROOT}/MOM6
 ${MKMF_ROOT}/list_paths -l ${MOM_ROOT}/{config_src/infra/FMS2,config_src/memory/dynamic_symmetric,config_src/drivers/solo_driver,../externals/MARBL/src,config_src/external,src/{*,*/*}}/
 ${MKMF_ROOT}/mkmf -t ${TEMPLATE} -o '-I../FMS' -p MOM6 -l '-L../FMS -lfms' -c '-Duse_libMPI -Duse_netCDF -DSPMD' path_names
-make -j${JOBS}  NETCDF=3 REPRO=1 MOM6
+make -j${JOBS} DEBUG=${DEBUG} MOM6
 
-echo "Finished build at `date`"
+echo "Finished build at `date`"