Skip to content

Commit 8ed32fc

Browse files
jbluehjblueh
committed
Add SU2_OMP_SAFE_GLOBAL_ACCESS variants.
Add it to the syntax checker and apply it throughout the code.
1 parent 7cffa8e commit 8ed32fc

15 files changed

Lines changed: 79 additions & 113 deletions

File tree

‎Common/include/basic_types/ad_structure.hpp‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -559,21 +559,13 @@ namespace AD{
559559
FORCEINLINE bool PausePreaccumulation() {
560560
const auto current = PreaccEnabled;
561561
if (!current) return false;
562-
SU2_OMP_BARRIER
563-
SU2_OMP_MASTER
564-
PreaccEnabled = false;
565-
END_SU2_OMP_MASTER
566-
SU2_OMP_BARRIER
562+
SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = false;)
567563
return true;
568564
}
569565

570566
FORCEINLINE void ResumePreaccumulation(bool wasActive) {
571567
if (!wasActive) return;
572-
SU2_OMP_BARRIER
573-
SU2_OMP_MASTER
574-
PreaccEnabled = true;
575-
END_SU2_OMP_MASTER
576-
SU2_OMP_BARRIER
568+
SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = true;)
577569
}
578570

579571
FORCEINLINE void StartNoSharedReading() {

‎Common/include/parallelization/omp_structure.hpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,21 @@ void omp_finalize();
185185

186186
#endif
187187

188+
#define BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS \
189+
SU2_OMP_BARRIER \
190+
SU2_OMP_MASTER
191+
192+
#define END_SU2_OMP_SAFE_GLOBAL_ACCESS \
193+
END_SU2_OMP_MASTER \
194+
SU2_OMP_BARRIER
195+
196+
#define SU2_OMP_SAFE_GLOBAL_ACCESS(...) \
197+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS \
198+
{ \
199+
__VA_ARGS__ \
200+
} \
201+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
202+
188203
/*--- Convenience functions (e.g. to compute chunk sizes). ---*/
189204

190205
/*!

‎Common/src/linear_algebra/CSysSolve.cpp‎

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -214,8 +214,8 @@ unsigned long CSysSolve<ScalarType>::CG_LinSolver(const CSysVector<ScalarType> &
214214
* do this since the working vectors are shared. ---*/
215215

216216
if (!cg_ready) {
217-
SU2_OMP_BARRIER
218-
SU2_OMP_MASTER {
217+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
218+
{
219219
auto nVar = b.GetNVar();
220220
auto nBlk = b.GetNBlk();
221221
auto nBlkDomain = b.GetNBlkDomain();
@@ -227,8 +227,7 @@ unsigned long CSysSolve<ScalarType>::CG_LinSolver(const CSysVector<ScalarType> &
227227

228228
cg_ready = true;
229229
}
230-
END_SU2_OMP_MASTER
231-
SU2_OMP_BARRIER
230+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
232231
}
233232

234233
/*--- Calculate the initial residual, compute norm, and check if system is already solved ---*/
@@ -358,17 +357,16 @@ unsigned long CSysSolve<ScalarType>::FGMRES_LinSolver(const CSysVector<ScalarTyp
358357
/*--- Allocate if not allocated yet ---*/
359358

360359
if (W.size() <= m || (flexible && Z.size() <= m)) {
361-
SU2_OMP_BARRIER
362-
SU2_OMP_MASTER {
360+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
361+
{
363362
W.resize(m+1);
364363
for (auto& w : W) w.Initialize(x.GetNBlk(), x.GetNBlkDomain(), x.GetNVar(), nullptr);
365364
if (flexible) {
366365
Z.resize(m+1);
367366
for (auto& z : Z) z.Initialize(x.GetNBlk(), x.GetNBlkDomain(), x.GetNVar(), nullptr);
368367
}
369368
}
370-
END_SU2_OMP_MASTER
371-
SU2_OMP_BARRIER
369+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
372370
}
373371

374372
/*--- Define various arrays. In parallel, each thread of each rank has and works
@@ -549,8 +547,8 @@ unsigned long CSysSolve<ScalarType>::BCGSTAB_LinSolver(const CSysVector<ScalarTy
549547
/*--- Allocate if not allocated yet ---*/
550548

551549
if (!bcg_ready) {
552-
SU2_OMP_BARRIER
553-
SU2_OMP_MASTER {
550+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
551+
{
554552
auto nVar = b.GetNVar();
555553
auto nBlk = b.GetNBlk();
556554
auto nBlkDomain = b.GetNBlkDomain();
@@ -564,8 +562,7 @@ unsigned long CSysSolve<ScalarType>::BCGSTAB_LinSolver(const CSysVector<ScalarTy
564562

565563
bcg_ready = true;
566564
}
567-
END_SU2_OMP_MASTER
568-
SU2_OMP_BARRIER
565+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
569566
}
570567

571568
/*--- Calculate the initial residual, compute norm, and check if system is already solved ---*/
@@ -715,8 +712,8 @@ unsigned long CSysSolve<ScalarType>::Smoother_LinSolver(const CSysVector<ScalarT
715712
product (A_x), for the latter two this is done only on the first call to the method. ---*/
716713

717714
if (!smooth_ready) {
718-
SU2_OMP_BARRIER
719-
SU2_OMP_MASTER {
715+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
716+
{
720717
auto nVar = b.GetNVar();
721718
auto nBlk = b.GetNBlk();
722719
auto nBlkDomain = b.GetNBlkDomain();
@@ -727,8 +724,7 @@ unsigned long CSysSolve<ScalarType>::Smoother_LinSolver(const CSysVector<ScalarT
727724

728725
smooth_ready = true;
729726
}
730-
END_SU2_OMP_MASTER
731-
SU2_OMP_BARRIER
727+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
732728
}
733729

734730
/*--- Compute the initial residual and check if the system is already solved (if in COMM_FULL mode). ---*/
@@ -972,9 +968,9 @@ unsigned long CSysSolve<ScalarType>::Solve(CSysMatrix<ScalarType> & Jacobian, co
972968
/*--- Start recording if it was stopped for the linear solver ---*/
973969
#ifdef CODI_REVERSE_TYPE
974970
AD::StartRecording();
975-
SU2_OMP_BARRIER
976971

977-
SU2_OMP_MASTER {
972+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
973+
{
978974
AD::SetExtFuncOut(&LinSysSol[0], LinSysSol.GetLocSize());
979975
AD::FuncHelper->addUserData(&LinSysRes);
980976
AD::FuncHelper->addUserData(&LinSysSol);
@@ -983,8 +979,7 @@ unsigned long CSysSolve<ScalarType>::Solve(CSysMatrix<ScalarType> & Jacobian, co
983979
AD::FuncHelper->addUserData(config);
984980
AD::FuncHelper->addUserData(this);
985981
}
986-
END_SU2_OMP_MASTER
987-
SU2_OMP_BARRIER
982+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
988983

989984
AD::FuncHelper->addToTape(CSysSolve_b<ScalarType>::Solve_b);
990985
SU2_OMP_BARRIER

‎SU2_CFD/include/limiters/CLimiterDetails.hpp‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -212,20 +212,18 @@ struct CLimiterDetails<LIMITER::VENKATAKRISHNAN_WANG>
212212
sharedMax(iVar) = max(sharedMax(iVar), localMax(iVar));
213213
}
214214
END_SU2_OMP_CRITICAL
215-
SU2_OMP_BARRIER
216215

217216
/*--- Global reduction. ---*/
218217

219-
SU2_OMP_MASTER
218+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
220219
{
221220
localMin = sharedMin;
222221
SU2_MPI::Allreduce(localMin.data(), sharedMin.data(), varEnd, MPI_DOUBLE, MPI_MIN, SU2_MPI::GetComm());
223222

224223
localMax = sharedMax;
225224
SU2_MPI::Allreduce(localMax.data(), sharedMax.data(), varEnd, MPI_DOUBLE, MPI_MAX, SU2_MPI::GetComm());
226225
}
227-
END_SU2_OMP_MASTER
228-
SU2_OMP_BARRIER
226+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
229227

230228
/*--- Compute eps^2 (each thread has its own copy of it). ---*/
231229

‎SU2_CFD/include/solvers/CFVMFlowSolverBase.hpp‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,7 @@ class CFVMFlowSolverBase : public CSolver {
6161
*/
6262
template<class... Ts>
6363
static void ompMasterAssignBarrier(Ts&&... lhsRhsPairs) {
64-
SU2_OMP_BARRIER
65-
SU2_OMP_MASTER
66-
recursiveAssign(lhsRhsPairs...);
67-
END_SU2_OMP_MASTER
68-
SU2_OMP_BARRIER
64+
SU2_OMP_SAFE_GLOBAL_ACCESS(recursiveAssign(lhsRhsPairs...);)
6965
}
7066

7167
su2double Mach_Inf = 0.0; /*!< \brief Mach number at the infinity. */
@@ -559,16 +555,15 @@ class CFVMFlowSolverBase : public CSolver {
559555
SU2_OMP_CRITICAL
560556
Global_Delta_UnstTimeND = min(Global_Delta_UnstTimeND, glbDtND);
561557
END_SU2_OMP_CRITICAL
562-
SU2_OMP_BARRIER
563558

564-
SU2_OMP_MASTER {
559+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
560+
{
565561
SU2_MPI::Allreduce(&Global_Delta_UnstTimeND, &glbDtND, 1, MPI_DOUBLE, MPI_MIN, SU2_MPI::GetComm());
566562
Global_Delta_UnstTimeND = glbDtND;
567563

568564
config->SetDelta_UnstTimeND(Global_Delta_UnstTimeND);
569565
}
570-
END_SU2_OMP_MASTER
571-
SU2_OMP_BARRIER
566+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
572567
}
573568

574569
/*--- The pseudo local time (explicit integration) cannot be greater than the physical time ---*/

‎SU2_CFD/include/solvers/CFVMFlowSolverBase.inl‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -665,16 +665,15 @@ void CFVMFlowSolverBase<V, R>::ComputeVorticityAndStrainMag(const CConfig& confi
665665
}
666666
END_SU2_OMP_CRITICAL
667667

668-
SU2_OMP_BARRIER
669-
SU2_OMP_MASTER {
668+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
669+
{
670670
su2double MyOmega_Max = Omega_Max;
671671
su2double MyStrainMag_Max = StrainMag_Max;
672672

673673
SU2_MPI::Allreduce(&MyStrainMag_Max, &StrainMag_Max, 1, MPI_DOUBLE, MPI_MAX, SU2_MPI::GetComm());
674674
SU2_MPI::Allreduce(&MyOmega_Max, &Omega_Max, 1, MPI_DOUBLE, MPI_MAX, SU2_MPI::GetComm());
675675
}
676-
END_SU2_OMP_MASTER
677-
SU2_OMP_BARRIER
676+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
678677
}
679678

680679
}

‎SU2_CFD/src/integration/CMultiGridIntegration.cpp‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -227,11 +227,7 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry,
227227
/*--- Temporarily disable implicit integration, for what follows we do not need the Jacobian. ---*/
228228

229229
if (implicit) {
230-
SU2_OMP_BARRIER
231-
SU2_OMP_MASTER
232-
config->SetKind_TimeIntScheme(EULER_EXPLICIT);
233-
END_SU2_OMP_MASTER
234-
SU2_OMP_BARRIER
230+
SU2_OMP_SAFE_GLOBAL_ACCESS(config->SetKind_TimeIntScheme(EULER_EXPLICIT);)
235231
}
236232

237233
/*--- Compute $r_k = P_k + F_k(u_k)$ ---*/
@@ -257,11 +253,7 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry,
257253
/*--- Restore the time integration settings. ---*/
258254

259255
if (implicit) {
260-
SU2_OMP_BARRIER
261-
SU2_OMP_MASTER
262-
config->SetKind_TimeIntScheme(EULER_IMPLICIT);
263-
END_SU2_OMP_MASTER
264-
SU2_OMP_BARRIER
256+
SU2_OMP_SAFE_GLOBAL_ACCESS(config->SetKind_TimeIntScheme(EULER_IMPLICIT);)
265257
}
266258

267259
/*--- Recursive call to MultiGrid_Cycle (this routine). ---*/

‎SU2_CFD/src/integration/CNewtonIntegration.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -162,15 +162,13 @@ void CNewtonIntegration::ComputeFinDiffStep() {
162162

163163
atomicAdd(rmsSol_loc, rmsSol);
164164

165-
SU2_OMP_BARRIER
166-
SU2_OMP_MASTER {
165+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
166+
{
167167
su2double t = rmsSol;
168168
SU2_MPI::Allreduce(&t, &rmsSol, 1, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm());
169169
finDiffStep = finDiffStepND * max(1.0, sqrt(SU2_TYPE::GetValue(rmsSol) / geometry->GetGlobal_nPointDomain()));
170170
}
171-
END_SU2_OMP_MASTER
172-
SU2_OMP_BARRIER
173-
171+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
174172
}
175173

176174
void CNewtonIntegration::MultiGrid_Iteration(CGeometry ****geometry_, CSolver *****solvers_, CNumerics ******numerics_,

‎SU2_CFD/src/solvers/CDiscAdjSolver.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -566,8 +566,8 @@ void CDiscAdjSolver::SetSurface_Sensitivity(CGeometry *geometry, CConfig *config
566566
}
567567
}
568568

569-
SU2_OMP_BARRIER
570-
SU2_OMP_MASTER {
569+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
570+
{
571571
auto local = Sens_Geo;
572572
SU2_MPI::Allreduce(local.data(), Sens_Geo.data(), Sens_Geo.size(), MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm());
573573

@@ -577,9 +577,7 @@ void CDiscAdjSolver::SetSurface_Sensitivity(CGeometry *geometry, CConfig *config
577577
Total_Sens_Geo += x;
578578
}
579579
}
580-
END_SU2_OMP_MASTER
581-
SU2_OMP_BARRIER
582-
580+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
583581
}
584582

585583
void CDiscAdjSolver::Preprocessing(CGeometry *geometry, CSolver **solver_container, CConfig *config, unsigned short iMesh,

‎SU2_CFD/src/solvers/CEulerSolver.cpp‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -343,8 +343,8 @@ CEulerSolver::~CEulerSolver(void) {
343343

344344
void CEulerSolver::InstantiateEdgeNumerics(const CSolver* const* solver_container, const CConfig* config) {
345345

346-
SU2_OMP_BARRIER
347-
SU2_OMP_MASTER {
346+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
347+
{
348348

349349
if (config->Low_Mach_Correction())
350350
SU2_MPI::Error("Low-Mach correction is not supported with vectorization.", CURRENT_FUNCTION);
@@ -359,8 +359,7 @@ void CEulerSolver::InstantiateEdgeNumerics(const CSolver* const* solver_containe
359359
"support vectorization.", CURRENT_FUNCTION);
360360

361361
}
362-
END_SU2_OMP_MASTER
363-
SU2_OMP_BARRIER
362+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
364363
}
365364

366365
void CEulerSolver::InitTurboContainers(CGeometry *geometry, CConfig *config){
@@ -1499,9 +1498,9 @@ void CEulerSolver::CommonPreprocessing(CGeometry *geometry, CSolver **solver_con
14991498

15001499
SU2_OMP_ATOMIC
15011500
ErrorCounter += SetPrimitive_Variables(solver_container, config);
1502-
SU2_OMP_BARRIER
15031501

1504-
SU2_OMP_MASTER { /*--- Ops that are not OpenMP parallel go in this block. ---*/
1502+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
1503+
{ /*--- Ops that are not OpenMP parallel go in this block. ---*/
15051504

15061505
if ((iMesh == MESH_0) && (config->GetComm_Level() == COMM_FULL)) {
15071506
unsigned long tmp = ErrorCounter;
@@ -1528,8 +1527,7 @@ void CEulerSolver::CommonPreprocessing(CGeometry *geometry, CSolver **solver_con
15281527
}
15291528

15301529
}
1531-
END_SU2_OMP_MASTER
1532-
SU2_OMP_BARRIER
1530+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
15331531

15341532
/*--- Artificial dissipation ---*/
15351533

@@ -1927,16 +1925,15 @@ void CEulerSolver::Upwind_Residual(CGeometry *geometry, CSolver **solver_contain
19271925
/*--- Add counter results for all threads. ---*/
19281926
SU2_OMP_ATOMIC
19291927
ErrorCounter += counter_local;
1930-
SU2_OMP_BARRIER
19311928

19321929
/*--- Add counter results for all ranks. ---*/
1933-
SU2_OMP_MASTER {
1930+
BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS
1931+
{
19341932
counter_local = ErrorCounter;
19351933
SU2_MPI::Reduce(&counter_local, &ErrorCounter, 1, MPI_UNSIGNED_LONG, MPI_SUM, MASTER_NODE, SU2_MPI::GetComm());
19361934
config->SetNonphysical_Reconstr(ErrorCounter);
19371935
}
1938-
END_SU2_OMP_MASTER
1939-
SU2_OMP_BARRIER
1936+
END_SU2_OMP_SAFE_GLOBAL_ACCESS
19401937
}
19411938

19421939
}

0 commit comments

Comments
 (0)