Skip to content

Commit dac21b9

Browse files
manishvenumanishvenu
committed
Ensure and Edit Buildable & Runnable versions of each compiler (NVHPC,GNU, INTEL)
Made changes to the intel and nvhpc compiler flags to build properly and run with double_gyre. Unfortunately that meant diverging slightly from the CESM compiler flags, see comments in each template for more information. So they all can be built and run, but each template needs some workshopping to make sure they're good, for example I took out -Ktrap=fp in NVHPC DEBUG which I think I need to confirm can be taken out.
1 parent 1b559ce commit dac21b9

4 files changed

Lines changed: 129 additions & 31 deletions

File tree

‎standalone/templates/ncar-intel_cesm.mk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ MAKEFLAGS += --jobs=8
2020
FC_AUTO_R8 := -r8
2121
FPPFLAGS := -fpp -Wp,-w
2222
FFLAGS := -qno-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source -no-fma -qopt-report -march=core-avx2 $(FC_AUTO_R8)
23-
FFLAGS_DEBUG = -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
23+
FFLAGS_DEBUG = -O0 -g -check uninit -check bounds -check nopointer -fpe0 -check noarg_temp_created # CESM uses -check pointers, that throws an error, changed to nopointer
2424
FFLAGS_REPRO = -O2 -debug minimal
2525

2626
CFLAGS := -qno-opt-dynamic-align -fp-model precise -std=gnu99 -no-fma -qopt-report -march=core-avx2

‎standalone/templates/ncar-nvhpc.mk‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,7 @@ FPPFLAGS := $(INCLUDES)
4747

4848

4949
# Base set of Fortran compiler flags
50-
FFLAGS = -g -Mdwarf3 -traceback -i4 -r8 -byteswapio -Mcray=pointer -Mflushz \
51-
-Mnofma -Mdaz -D_F2000
50+
FFLAGS = -g -Mdwarf3 -traceback -i4 -r8 -byteswapio -Mcray=pointer -Mflushz -Mnofma -Mdaz -D_F2000
5251

5352
# Flags based on perforance target (production (OPT), reproduction (REPRO), or debug (DEBUG)
5453
#FFLAGS_REPRO = -O2 -Mvect=nosse -Mnoscalarsse $(NOFMA)

‎standalone/templates/ncar-nvhpc_cesm.mk‎

Lines changed: 10 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -25,33 +25,17 @@ ifneq ($(need),$(ok))
2525
$(error Need at least make version $(need). Load module gmake/3.81)
2626
endif
2727

28-
29-
# Check version of PGI for use of -nofma option
30-
has_nofma := $(shell $(FC) -dryrun -nofma foo.f90 > /dev/null 2>&1; echo $$?)
31-
ifneq ($(has_nofma),0)
32-
NOFMA :=
33-
else
34-
NOFMA := -nofma
35-
endif
36-
37-
# Required Preprocessor Macros:
38-
CPPDEFS += -Duse_netCDF
39-
40-
# Additional Preprocessor Macros needed due to Autotools and CMake
41-
CPPDEFS += -DHAVE_SCHED_GETAFFINITY -DHAVE_GETTID
42-
4328
# Macro for Fortran preprocessor
4429
FPPFLAGS := $(INCLUDES)
4530

4631

4732
# Base set of Fortran compiler flags
4833
FC_AUTO_R8 = -r8
49-
FFLAGS = $(FC_AUTO_R8) -Mnofma -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee
34+
FFLAGS = $(FC_AUTO_R8) -Mnofma -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee
5035

5136
# Flags based on perforance target (production (OPT), reproduction (REPRO), or debug (DEBUG)
52-
FFLAGS_REPRO = -O -tp=zen3
53-
FFLAGS_DEBUG = -O0 -g -Ktrap=fp -Mbounds -Kieee
54-
37+
FFLAGS_REPRO = -O0 -tp=zen3 # CESM doesn't include the 2, but it looks like we need it. Not quite sure what happens on just -O
38+
FFLAGS_DEBUG = -O0 -g -traceback -Mdwarf3 #-MBounds fails compilation! o does -KTrap=fp, seems like there is a floating point exception in netcdf_io_mod file, which means i'm missing some coompiler flag
5539
# Macro for C preprocessor
5640
CPPFLAGS := $(INCLUDES)
5741

@@ -61,10 +45,10 @@ CFLAGS = -gopt -time -Mnofma
6145

6246
# Flags based on perforance target (production (OPT), reproduction (REPRO), or debug (DEBUG)
6347
CFLAGS_REPRO = -O
64-
CFLAGS_DEBUG = -O0 -g -traceback -Ktrap=fp
48+
CFLAGS_DEBUG =
6549

6650
# Linking flags
67-
LDFLAGS := -byteswapio
51+
LDFLAGS :=
6852

6953
# List of -L library directories to be added to the compile and linking commands
7054
LIBS :=
@@ -84,15 +68,13 @@ endif
8468
FPPFLAGS += $(shell nf-config --fflags)
8569
# C Compiler flags for the NetCDF library
8670
CPPFLAGS += $(shell nc-config --cflags)
87-
# add the use_LARGEFILE cppdef
88-
ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
89-
CPPDEFS += -Duse_LARGEFILE
90-
endif
9171
# Add netcdf linking
92-
LIBS += $(shell nc-config --libs) $(shell nf-config --flibs)
93-
72+
LIBS += $(shell nc-config --libs) $(shell nf-config --flibs)
73+
ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),)
74+
CPPDEFS += -Duse_LARGEFILE
75+
endif
9476
# Additional CPPDEFS Flags
95-
CPPDEFS += -DCNL -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI -DLINUX -DHAVE_GETTID -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI -DLINUX -DHAVE_GETTID -DNDEBUG -DUSE_ESMF_LIB -DHAVE_MPI -DNUOPC_INTERFACE -DPIO2 -DHAVE_SLASHPROC -D_PNETCDF -DESMF_VERSION_MAJOR=8 -DESMF_VERSION_MINOR=6 -DATM_PRESENT -DICE_PRESENT -DLND_PRESENT -DOCN_PRESENT -DROF_PRESENT -DGLC_PRESENT -DWAV_PRESENT -DESP_PRESENT -DMED_PRESENT
77+
CPPDEFS += -DHAVE_SCHED_GETAFFINITY -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI -DLINUX -DHAVE_GETTID -DNDEBUG -DUSE_ESMF_LIB -DHAVE_MPI -DNUOPC_INTERFACE -DPIO2 -DHAVE_SLASHPROC -D_PNETCDF -DESMF_VERSION_MAJOR=8 -DESMF_VERSION_MINOR=6 -DATM_PRESENT -DICE_PRESENT -DLND_PRESENT -DOCN_PRESENT -DROF_PRESENT -DGLC_PRESENT -DWAV_PRESENT -DESP_PRESENT -DMED_PRESENT -DPIO2
9678

9779

9880
# These Algebra libraries Add solution to more complex vector matrix model equations

‎standalone/tmpdir_compiler_outputs_doublegyre/output_nvhpc_cesm.txt‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
NOTE: MPP_DOMAINS_SET_STACK_SIZE: stack size set to 32768.
2+
NOTE: MPP_IO_SET_STACK_SIZE: stack size set to 131072.
3+
NOTE: ======== Model being driven by MOM_driver ========
4+
NOTE: callTree: o Program MOM_main, MOM_driver.F90
5+
NOTE: open_param_file: MOM_input has been opened successfully.
6+
NOTE: open_param_file: MOM_override has been opened successfully.
7+
MOM_in domain decomposition
8+
whalo = 4, ehalo = 4, shalo = 4, nhalo = 4
9+
X-AXIS = 44
10+
Y-AXIS = 40
11+
MOM_inc domain decomposition
12+
whalo = 2, ehalo = 2, shalo = 2, nhalo = 2
13+
X-AXIS = 22
14+
Y-AXIS = 20
15+
NOTE: diag_manager_mod::diag_manager_init: diag_manager is using fms2_io
16+
NOTE: diag_manager_mod::diag_manager_init: prepend_date only supported when diag_manager_init is called with time_init present.
17+
MOM_in domain decomposition
18+
whalo = 4, ehalo = 4, shalo = 4, nhalo = 4
19+
X-AXIS = 44
20+
Y-AXIS = 40
21+
MOM_in domain decomposition
22+
whalo = 2, ehalo = 2, shalo = 2, nhalo = 2
23+
X-AXIS = 22
24+
Y-AXIS = 20
25+
MOM_in domain decomposition
26+
whalo = 4, ehalo = 4, shalo = 4, nhalo = 4
27+
X-AXIS = 44
28+
Y-AXIS = 40
29+
MOM_in domain decomposition
30+
whalo = 2, ehalo = 2, shalo = 2, nhalo = 2
31+
X-AXIS = 22
32+
Y-AXIS = 20
33+
34+
WARNING: Open_file is appending .nc to the filename ./ocean_geometry
35+
36+
MOM_in domain decomposition
37+
whalo = 4, ehalo = 4, shalo = 4, nhalo = 4
38+
X-AXIS = 44
39+
Y-AXIS = 40
40+
MOM_in domain decomposition
41+
whalo = 2, ehalo = 2, shalo = 2, nhalo = 2
42+
X-AXIS = 22
43+
Y-AXIS = 20
44+
45+
WARNING: KVML is a deprecated parameter. Use KV_ML_INVZ2 instead.
46+
47+
MOM_in domain decomposition
48+
whalo = 4, ehalo = 4, shalo = 4, nhalo = 4
49+
X-AXIS = 44
50+
Y-AXIS = 40
51+
MOM_in domain decomposition
52+
whalo = 2, ehalo = 2, shalo = 2, nhalo = 2
53+
X-AXIS = 22
54+
Y-AXIS = 20
55+
56+
WARNING: open_ASCII_file: File ./CPU_stats opened WRITEONLY already exists!
57+
58+
59+
WARNING: open_ASCII_file: File ./ocean.stats opened WRITEONLY already exists!
60+
61+
MOM Day 0.000 0: En 1.423673E-13, MaxCFL 0.00000, Mass 5.288178268008E+18
62+
Total Energy: 4126F9BF42B7CF4E 7.5286363030860736E+05
63+
Total Mass: 5.2881782680077681E+18, Change: 0.0000000000000000E+00 Error: 0.00000E+00 ( 0.0E+00)
64+
MOM Day 1.000 72: En 4.580432E-06, MaxCFL 0.00023, Mass 5.288178268008E+18
65+
Total Energy: 42B607A80A9BFE7D 2.4222139849726488E+13
66+
Total Mass: 5.2881782680077681E+18, Change: -7.4479853272800113E+01 Error: -7.44799E+01 (-1.4E-17)
67+
MOM Day 2.000 144: En 8.958206E-06, MaxCFL 0.00040, Mass 5.288178268008E+18
68+
Total Energy: 42C58AE53354BAAE 4.7372590033269359E+13
69+
Total Mass: 5.2881782680077681E+18, Change: -5.3850339916781394E+01 Error: -5.38503E+01 (-1.0E-17)
70+
MOM Day 3.000 216: En 1.166708E-05, MaxCFL 0.00045, Mass 5.288178268008E+18
71+
Total Energy: 42CC0E8B5CDEE73E 6.1697586478542484E+13
72+
Total Mass: 5.2881782680077681E+18, Change: 8.5736064780355719E+01 Error: 8.57361E+01 ( 1.6E-17)
73+
MOM Day 4.000 288: En 1.542296E-05, MaxCFL 0.00044, Mass 5.288178268008E+18
74+
Total Energy: 42D28B60D587EF95 8.1559337443262328E+13
75+
Total Mass: 5.2881782680077681E+18, Change: -7.4351260657852976E+01 Error: -7.43513E+01 (-1.4E-17)
76+
MOM Day 5.000 360: En 1.826022E-05, MaxCFL 0.00053, Mass 5.288178268008E+18
77+
Total Energy: 42D5F4B97D61AB81 9.6563312821934016E+13
78+
Total Mass: 5.2881782680077681E+18, Change: 7.0123661168087182E+01 Error: 7.01237E+01 ( 1.3E-17)
79+
MOM Day 6.000 432: En 2.397431E-05, MaxCFL 0.00062, Mass 5.288178268008E+18
80+
Total Energy: 42DCD397A5E5D723 1.2678043163631655E+14
81+
Total Mass: 5.2881782680077681E+18, Change: -4.9496049892239796E+01 Error: -4.94960E+01 (-9.4E-18)
82+
MOM Day 7.000 504: En 2.927962E-05, MaxCFL 0.00073, Mass 5.288178268008E+18
83+
Total Energy: 42E19A5109E4E8B8 1.5483585790957375E+14
84+
Total Mass: 5.2881782680077681E+18, Change: 6.3265455880979829E+01 Error: 6.32655E+01 ( 1.2E-17)
85+
MOM Day 8.000 576: En 3.486441E-05, MaxCFL 0.00081, Mass 5.288178268008E+18
86+
Total Energy: 42E4F5D995B9FD39 1.8436920009111378E+14
87+
Total Mass: 5.2881782680077681E+18, Change: 4.5135547774855041E+01 Error: 4.51355E+01 ( 8.5E-18)
88+
MOM Day 9.000 648: En 4.195565E-05, MaxCFL 0.00090, Mass 5.288178268008E+18
89+
Total Energy: 42E9393C9CAD84D0 2.2186896585015050E+14
90+
Total Mass: 5.2881782680077681E+18, Change: 1.0475768039255965E+02 Error: 1.04758E+02 ( 2.0E-17)
91+
MOM Day 10.000 720: En 4.604729E-05, MaxCFL 0.00100, Mass 5.288178268008E+18
92+
Total Energy: 42EBAEF71B9A2C7A 2.4350627232598781E+14
93+
Total Mass: 5.2881782680077681E+18, Change: 2.6968788567848577E+01 Error: 2.69688E+01 ( 5.1E-18)
94+
MOM_in domain decomposition
95+
whalo = 4, ehalo = 4, shalo = 4, nhalo = 4
96+
X-AXIS = 44
97+
Y-AXIS = 40
98+
MOM_in domain decomposition
99+
whalo = 2, ehalo = 2, shalo = 2, nhalo = 2
100+
X-AXIS = 22
101+
Y-AXIS = 20
102+
103+
WARNING: open_ASCII_file: File RESTART/ocean_solo.res opened WRITEONLY already exists!
104+
105+
106+
WARNING: open_ASCII_file: File exitcode opened WRITEONLY already exists!
107+
108+
MPP_DOMAINS_STACK high water mark= 0
109+
110+
Tabulating mpp_clock statistics across 1 PEs...
111+
112+
hits tmin tmax tavg tstd tfrac grain pemin pemax
113+
Total runtime 1 16.680070 16.680070 16.680070 0.000000 1.000 0 0 0
114+
Initialization 1 0.120577 0.120577 0.120577 0.000000 0.007 0 0 0
115+
Main loop 1 16.539006 16.539006 16.539006 0.000000 0.992 0 0 0
116+
Termination 1 0.017994 0.017994 0.017994 0.000000 0.001 0 0 0
117+
MPP_STACK high water mark= 0

0 commit comments

Comments
 (0)