Skip to content

Commit f7e8a15

Browse files
mnlevy1981mnlevy1981
committed
Add support for nvhpc on ncar machines
I also added a "--debug" command line argument to build_examples.sh and some module load statements to build_examples.sh for running on the NCAR machines, though that is skipped for casper nodes (including crhtc nodes)
1 parent 3e193bd commit f7e8a15

4 files changed

Lines changed: 271 additions & 7 deletions

File tree

‎standalone/build/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@ intel/
22
intel-*/
33
gnu/
44
gnu-*/
5+
nvhpc/
6+
nvhpc-*/

‎standalone/build/build_examples.sh‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ FMS_ROOT=${CESM_ROOT}/libraries/FMS
1818
# Default compiler
1919
COMPILER="intel"
2020
MACHINE="ncar"
21+
REPRO_OR_DEBUG=REPRO
2122

2223
# Parse command line arguments
2324
while [[ "$#" -gt 0 ]]; do
@@ -28,6 +29,8 @@ while [[ "$#" -gt 0 ]]; do
2829
--machine)
2930
MACHINE="$2"
3031
shift ;;
32+
--debug)
33+
REPRO_OR_DEBUG=DEBUG ;;
3134
*)
3235
echo "Unknown parameter passed: $1"
3336
echo "Usage: $0 [--compiler <compiler>] [--machine <machine>]"
@@ -58,18 +61,43 @@ case $MACHINE in
5861
JOBS=4
5962
;;
6063
"ncar")
61-
JOBS=36
64+
JOBS=32
6265
;;
6366
*)
6467
echo "Invalid machine type for make -j option: $MACHINE"
6568
exit 1
6669
;;
6770
esac
6871

69-
if [ -e $1 ]; then
70-
BLD_ROOT=${COMPILER}
72+
if [ "${REPRO_OR_DEBUG}" == "DEBUG" ]; then
73+
BLD_ROOT=${COMPILER}-debug
7174
else
72-
BLD_ROOT=$1
75+
BLD_ROOT=${COMPILER}
76+
fi
77+
78+
# Load modules for NCAR
79+
if [ "$MACHINE" == "ncar" ]; then
80+
HOST=`hostname`
81+
# Load modules if on derecho
82+
if [ ! "${HOST:0:5}" == "crhtc" ] && [ ! "${HOST:0:6}" == "casper" ]; then
83+
module --force purge
84+
. /glade/u/apps/derecho/23.09/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/c645/lmod/lmod/init/sh
85+
module load cesmdev/1.0 ncarenv/23.09
86+
case $COMPILER in
87+
"intel" )
88+
module load craype intel/2023.2.1 mkl ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2 esmf/8.6.0
89+
;;
90+
"gnu" )
91+
module load craype gcc/12.2.0 cray-libsci/23.02.1.1 ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2-debug esmf/8.6.0-debug
92+
;;
93+
"nvhpc" )
94+
module load craype nvhpc/24.3 ncarcompilers/1.0.0 cmake cray-mpich/8.1.27 netcdf-mpi/4.9.2 parallel-netcdf/1.12.3 parallelio/2.6.2 esmf/8.6.0
95+
;;
96+
*)
97+
echo "Not loading any special modules for ${COMPILER}"
98+
;;
99+
esac
100+
fi
73101
fi
74102

75103
# 1) Build FMS
@@ -82,14 +110,14 @@ ${MKMF_ROOT}/list_paths ${FMS_ROOT}/src
82110
echo "${SHR_ROOT}/src/shr_kind_mod.F90" >> path_names
83111
echo "${SHR_ROOT}/src/shr_const_mod.F90" >> path_names
84112
${MKMF_ROOT}/mkmf -t ${TEMPLATE} -p libfms.a -c "-Duse_libMPI -Duse_netCDF -DSPMD" path_names
85-
make -j${JOBS} NETCDF=3 REPRO=1 libfms.a
113+
make -j${JOBS} NETCDF=3 ${REPRO_OR_DEBUG}=1 libfms.a
86114

87115
# 2) Build MOM6
88116
cd ${INTERFACE_ROOT}/standalone/build
89117
mkdir -p ${BLD_ROOT}/MOM6
90118
cd ${BLD_ROOT}/MOM6
91119
${MKMF_ROOT}/list_paths -l ${MOM_ROOT}/{config_src/infra/FMS2,config_src/memory/dynamic_symmetric,config_src/drivers/solo_driver,../externals/MARBL/src,config_src/external,src/{*,*/*}}/
92120
${MKMF_ROOT}/mkmf -t ${TEMPLATE} -o '-I../FMS' -p MOM6 -l '-L../FMS -lfms' -c '-Duse_libMPI -Duse_netCDF -DSPMD' path_names
93-
make -j${JOBS} NETCDF=3 REPRO=1 MOM6
121+
make -j${JOBS} NETCDF=3 ${REPRO_OR_DEBUG}=1 MOM6
94122

95123
echo "Finished build at `date`"

‎standalone/templates/ncar-intel.mk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ CFLAGS := -D__IFC -sox -traceback -diag-disable=10441
3636
CFLAGS += -I$(NETCDF_PATH)/include
3737
CFLAGS_OPT = -O2 -debug minimal
3838
CFLAGS_OPENMP = -openmp
39-
CFLAGS_DEBUG = -O0 -g -ftrapuv
39+
CFLAGS_DEBUG = -O0 -g -ftrapv
4040

4141
LDFLAGS :=
4242
LDFLAGS_OPENMP := -openmp

‎standalone/templates/ncar-nvhpc.mk‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
# Template for the PGI Compilers
2+
#
3+
# Typical use with mkmf
4+
# mkmf -t ncrc-cray.mk -c"-Duse_libMPI -Duse_netCDF" path_names /usr/local/include
5+
6+
############
7+
# commands #
8+
############
9+
FC = ftn
10+
CC = cc
11+
CXX = cc
12+
LD = ftn $(MAIN_PROGRAM)
13+
14+
############
15+
# flags #
16+
############
17+
18+
DEBUG =
19+
REPRO =
20+
VERBOSE =
21+
OPENMP =
22+
23+
MAKEFLAGS += --jobs=8
24+
25+
INCLUDES := $(shell pkg-config --cflags yaml-0.1)
26+
27+
# Need to use at least GNU Make version 3.81
28+
need := 3.81
29+
ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need))))
30+
ifneq ($(need),$(ok))
31+
$(error Need at least make version $(need). Load module gmake/3.81)
32+
endif
33+
34+
# REPRO, DEBUG and TEST need to be mutually exclusive of each other.
35+
# Make sure the user hasn't supplied two at the same time
36+
ifdef REPRO
37+
ifneq ($(DEBUG),)
38+
$(error Options REPRO and DEBUG cannot be used together)
39+
endif
40+
endif
41+
42+
# Check version of PGI for use of -nofma option
43+
has_nofma := $(shell $(FC) -dryrun -nofma foo.f90 > /dev/null 2>&1; echo $$?)
44+
ifneq ($(has_nofma),0)
45+
NOFMA :=
46+
else
47+
NOFMA := -nofma
48+
endif
49+
50+
# Required Preprocessor Macros:
51+
CPPDEFS += -Duse_netCDF
52+
53+
# Additional Preprocessor Macros needed due to Autotools and CMake
54+
CPPDEFS += -DHAVE_SCHED_GETAFFINITY -DHAVE_GETTID
55+
56+
# Macro for Fortran preprocessor
57+
FPPFLAGS := $(INCLUDES)
58+
# Fortran Compiler flags for the NetCDF library
59+
FPPFLAGS += $(shell nf-config --fflags)
60+
61+
# Base set of Fortran compiler flags
62+
FFLAGS = -g -Mdwarf3 -traceback -i4 -r8 -byteswapio -Mcray=pointer -Mflushz \
63+
-Mnofma -Mdaz -D_F2000
64+
65+
# Flags based on perforance target (production (OPT), reproduction (REPRO), or debug (DEBUG)
66+
FFLAGS_OPT = -O3 -Mvect=nosse -Mnoscalarsse -Mallocatable=95
67+
#FFLAGS_REPRO = -O2 -Mvect=nosse -Mnoscalarsse $(NOFMA)
68+
# NOTE: "REPRO" temporarily uses -O0 due to errors in existing codes.
69+
# Optimization will be restored once the issues have been investigated.
70+
FFLAGS_REPRO = -O0
71+
FFLAGS_DEBUG = -O0 -Ktrap=fp
72+
73+
# Flags to add additional build options
74+
FFLAGS_OPENMP = -mp
75+
FFLAGS_VERBOSE = -v -Minform=inform
76+
77+
# Macro for C preprocessor
78+
CPPFLAGS := $(INCLUDES)
79+
# C Compiler flags for the NetCDF library
80+
CPPFLAGS += $(shell nc-config --cflags)
81+
82+
# Base set of C compiler flags
83+
CFLAGS =
84+
85+
# Flags based on perforance target (production (OPT), reproduction (REPRO), or debug (DEBUG)
86+
CFLAGS_OPT = -O2
87+
CFLAGS_REPRO = -O2
88+
CFLAGS_DEBUG = -O0 -g -traceback -Ktrap=fp
89+
90+
# Flags to add additional build options
91+
CFLAGS_OPENMP = -mp
92+
CFLAGS_VERBOSE = -v -Minform=inform
93+
94+
# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT
95+
# *_TEST will match the production if no new option(s) is(are) to be tested.
96+
FFLAGS_TEST := $(FFLAGS_OPT)
97+
CFLAGS_TEST := $(CFLAGS_OPT)
98+
99+
# Linking flags
100+
LDFLAGS := -byteswapio
101+
LDFLAGS_OPENMP :=
102+
LDFLAGS_VERBOSE := -v
103+
104+
# List of -L library directories to be added to the compile and linking commands
105+
LIBS :=
106+
107+
# Get compile flags based on target macros.
108+
ifdef REPRO
109+
CFLAGS += $(CFLAGS_REPRO)
110+
FFLAGS += $(FFLAGS_REPRO)
111+
else ifdef DEBUG
112+
CFLAGS += $(CFLAGS_DEBUG)
113+
FFLAGS += $(FFLAGS_DEBUG)
114+
else ifdef TEST
115+
CFLAGS += $(CFLAGS_TEST)
116+
FFLAGS += $(FFLAGS_TEST)
117+
else
118+
CFLAGS += $(CFLAGS_OPT)
119+
FFLAGS += $(FFLAGS_OPT)
120+
endif
121+
122+
ifdef OPENMP
123+
CFLAGS += $(CFLAGS_OPENMP)
124+
FFLAGS += $(FFLAGS_OPENMP)
125+
LDFLAGS += $(LDFLAGS_OPENMP)
126+
endif
127+
128+
ifdef VERBOSE
129+
CFLAGS += $(CFLAGS_VERBOSE)
130+
FFLAGS += $(FFLAGS_VERBOSE)
131+
LDFLAGS += $(LDFLAGS_VERBOSE)
132+
endif
133+
134+
ifeq ($(NETCDF),3)
135+
# add the use_LARGEFILE cppdef
136+
ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
137+
CPPDEFS += -Duse_LARGEFILE
138+
endif
139+
# Add netcdf linking
140+
LIBS += $(shell nc-config --libs) $(shell nf-config --flibs)
141+
endif
142+
143+
# These Algebra libraries Add solution to more complex vector matrix model equations
144+
LIBS += -llapack -lblas
145+
LDFLAGS += $(LIBS)
146+
147+
#---------------------------------------------------------------------------
148+
# you should never need to change any lines below.
149+
150+
# see the MIPSPro F90 manual for more details on some of the file extensions
151+
# discussed here.
152+
# this makefile template recognizes fortran sourcefiles with extensions
153+
# .f, .f90, .F, .F90. Given a sourcefile <file>.<ext>, where <ext> is one of
154+
# the above, this provides a number of default actions:
155+
156+
# make <file>.opt create an optimization report
157+
# make <file>.o create an object file
158+
# make <file>.s create an assembly listing
159+
# make <file>.x create an executable file, assuming standalone
160+
# source
161+
# make <file>.i create a preprocessed file (for .F)
162+
# make <file>.i90 create a preprocessed file (for .F90)
163+
164+
# The macro TMPFILES is provided to slate files like the above for removal.
165+
166+
RM = rm -f
167+
TMPFILES = .*.m *.B *.L *.i *.i90 *.l *.s *.mod *.opt
168+
169+
.SUFFIXES: .F .F90 .H .L .T .f .f90 .h .i .i90 .l .o .s .opt .x
170+
171+
.f.L:
172+
$(FC) $(FFLAGS) -c -listing $*.f
173+
.f.opt:
174+
$(FC) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.f
175+
.f.l:
176+
$(FC) $(FFLAGS) -c $(LIST) $*.f
177+
.f.T:
178+
$(FC) $(FFLAGS) -c -cif $*.f
179+
.f.o:
180+
$(FC) $(FFLAGS) -c $*.f
181+
.f.s:
182+
$(FC) $(FFLAGS) -S $*.f
183+
.f.x:
184+
$(FC) $(FFLAGS) -o $*.x $*.f *.o $(LDFLAGS)
185+
.f90.L:
186+
$(FC) $(FFLAGS) -c -listing $*.f90
187+
.f90.opt:
188+
$(FC) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.f90
189+
.f90.l:
190+
$(FC) $(FFLAGS) -c $(LIST) $*.f90
191+
.f90.T:
192+
$(FC) $(FFLAGS) -c -cif $*.f90
193+
.f90.o:
194+
$(FC) $(FFLAGS) -c $*.f90
195+
.f90.s:
196+
$(FC) $(FFLAGS) -c -S $*.f90
197+
.f90.x:
198+
$(FC) $(FFLAGS) -o $*.x $*.f90 *.o $(LDFLAGS)
199+
.F.L:
200+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -listing $*.F
201+
.F.opt:
202+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.F
203+
.F.l:
204+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $(LIST) $*.F
205+
.F.T:
206+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -cif $*.F
207+
.F.f:
208+
$(FC) $(CPPDEFS) $(FPPFLAGS) -EP $*.F > $*.f
209+
.F.i:
210+
$(FC) $(CPPDEFS) $(FPPFLAGS) -P $*.F
211+
.F.o:
212+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $*.F
213+
.F.s:
214+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -S $*.F
215+
.F.x:
216+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -o $*.x $*.F *.o $(LDFLAGS)
217+
.F90.L:
218+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -listing $*.F90
219+
.F90.opt:
220+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.F90
221+
.F90.l:
222+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $(LIST) $*.F90
223+
.F90.T:
224+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -cif $*.F90
225+
.F90.f90:
226+
$(FC) $(CPPDEFS) $(FPPFLAGS) -EP $*.F90 > $*.f90
227+
.F90.i90:
228+
$(FC) $(CPPDEFS) $(FPPFLAGS) -P $*.F90
229+
.F90.o:
230+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $*.F90
231+
.F90.s:
232+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -S $*.F90
233+
.F90.x:
234+
$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -o $*.x $*.F90 *.o $(LDFLAGS)

0 commit comments

Comments
 (0)