Skip to content

Commit ea77689

Browse files
pcarruscagpcarruscag
authored
Merge pull request #1180 from aa-g/feature_mpi_subcomm
Support for MPI communicators which are provided externally
2 parents be378f2 + 3462dda commit ea77689

77 files changed

Lines changed: 874 additions & 870 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Common/include/linear_algebra/CPastixWrapper.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ class CPastixWrapper
9393
* \brief Run the external solver for the task it is currently setup to execute.
9494
*/
9595
void Run() {
96-
dpastix(&state, MPI_COMM_WORLD, nCols, colptr.data(), rowidx.data(), values.data(),
96+
dpastix(&state, SU2_MPI::GetComm(), nCols, colptr.data(), rowidx.data(), values.data(),
9797
loc2glb.data(), perm.data(), NULL, workvec.data(), 1, iparm, dparm);
9898
}
9999

‎Common/include/linear_algebra/CSysVector.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,7 @@ class CSysVector : public VecExpr::CVecExpr<CSysVector<ScalarType>, ScalarType>
315315
SU2_OMP_MASTER {
316316
sum = dotRes;
317317
const auto mpi_type = (sizeof(ScalarType) < sizeof(double)) ? MPI_FLOAT : MPI_DOUBLE;
318-
SelectMPIWrapper<ScalarType>::W::Allreduce(&sum, &dotRes, 1, mpi_type, MPI_SUM, MPI_COMM_WORLD);
318+
SelectMPIWrapper<ScalarType>::W::Allreduce(&sum, &dotRes, 1, mpi_type, MPI_SUM, SU2_MPI::GetComm());
319319
}
320320
}
321321
#endif

‎Common/include/parallelization/mpi_structure.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,17 @@
2727

2828
#include "mpi_structure.hpp"
2929

30+
31+
/* Initialise the MPI Communicator Rank and Size */
3032
int CBaseMPIWrapper::Rank = 0;
3133
int CBaseMPIWrapper::Size = 1;
34+
35+
/* Set the default MPI Communicator */
36+
#ifdef HAVE_MPI
3237
CBaseMPIWrapper::Comm CBaseMPIWrapper::currentComm = MPI_COMM_WORLD;
38+
#else
39+
CBaseMPIWrapper::Comm CBaseMPIWrapper::currentComm = 0; // dummy value
40+
#endif
3341

3442
#ifdef HAVE_MPI
3543
int CBaseMPIWrapper::MinRankError;

‎Common/include/parallelization/mpi_structure.hpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,6 @@ class CMediMPIWrapper : public CBaseMPIWrapper {
467467

468468
#else // HAVE_MPI
469469

470-
#define MPI_COMM_WORLD 0
471470
#define MPI_UNSIGNED_LONG 1
472471
#define MPI_LONG 2
473472
#define MPI_UNSIGNED_SHORT 3

‎Common/include/toolboxes/CQuasiNewtonInvLeastSquares.hpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,11 +95,11 @@ class CQuasiNewtonInvLeastSquares {
9595

9696
su2vector<Scalar> tmp(mat.size());
9797
MPI_Wrapper::Allreduce(mat.data(), tmp.data(), iSample*(iSample+1)/2,
98-
type, MPI_SUM, MPI_COMM_WORLD);
98+
type, MPI_SUM, SU2_MPI::GetComm());
9999
mat = std::move(tmp);
100100

101101
MPI_Wrapper::Allreduce(rhs.data(), sol.data(), iSample,
102-
type, MPI_SUM, MPI_COMM_WORLD);
102+
type, MPI_SUM, SU2_MPI::GetComm());
103103
std::swap(rhs, sol);
104104
}
105105
}

‎Common/src/CConfig.cpp‎

Lines changed: 21 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4953,10 +4953,7 @@ void CConfig::SetMarkers(unsigned short val_software) {
49534953
iMarker_Turbomachinery, iMarker_MixingPlaneInterface;
49544954

49554955
int size = SINGLE_NODE;
4956-
4957-
#ifdef HAVE_MPI
4958-
SU2_MPI::Comm_size(MPI_COMM_WORLD, &size);
4959-
#endif
4956+
SU2_MPI::Comm_size(SU2_MPI::GetComm(), &size);
49604957

49614958
/*--- Compute the total number of markers in the config file ---*/
49624959

@@ -9214,8 +9211,8 @@ void CConfig::SetProfilingCSV(void) {
92149211
int rank = MASTER_NODE;
92159212
int size = SINGLE_NODE;
92169213
#ifdef HAVE_MPI
9217-
SU2_MPI::Comm_rank(MPI_COMM_WORLD, &rank);
9218-
SU2_MPI::Comm_size(MPI_COMM_WORLD, &size);
9214+
SU2_MPI::Comm_rank(SU2_MPI::GetComm(), &rank);
9215+
SU2_MPI::Comm_size(SU2_MPI::GetComm(), &size);
92199216
#endif
92209217

92219218
/*--- Each rank has the same stack trace, so the they have the same
@@ -9299,11 +9296,11 @@ void CConfig::SetProfilingCSV(void) {
92999296
}
93009297

93019298
#ifdef HAVE_MPI
9302-
MPI_Reduce(n_calls, n_calls_red, map_size, MPI_INT, MPI_SUM, MASTER_NODE, MPI_COMM_WORLD);
9303-
MPI_Reduce(l_tot, l_tot_red, map_size, MPI_DOUBLE, MPI_SUM, MASTER_NODE, MPI_COMM_WORLD);
9304-
MPI_Reduce(l_avg, l_avg_red, map_size, MPI_DOUBLE, MPI_SUM, MASTER_NODE, MPI_COMM_WORLD);
9305-
MPI_Reduce(l_min, l_min_red, map_size, MPI_DOUBLE, MPI_MIN, MASTER_NODE, MPI_COMM_WORLD);
9306-
MPI_Reduce(l_max, l_max_red, map_size, MPI_DOUBLE, MPI_MAX, MASTER_NODE, MPI_COMM_WORLD);
9299+
MPI_Reduce(n_calls, n_calls_red, map_size, MPI_INT, MPI_SUM, MASTER_NODE, SU2_MPI::GetComm());
9300+
MPI_Reduce(l_tot, l_tot_red, map_size, MPI_DOUBLE, MPI_SUM, MASTER_NODE, SU2_MPI::GetComm());
9301+
MPI_Reduce(l_avg, l_avg_red, map_size, MPI_DOUBLE, MPI_SUM, MASTER_NODE, SU2_MPI::GetComm());
9302+
MPI_Reduce(l_min, l_min_red, map_size, MPI_DOUBLE, MPI_MIN, MASTER_NODE, SU2_MPI::GetComm());
9303+
MPI_Reduce(l_max, l_max_red, map_size, MPI_DOUBLE, MPI_MAX, MASTER_NODE, SU2_MPI::GetComm());
93079304
#else
93089305
memcpy(n_calls_red, n_calls, map_size*sizeof(int));
93099306
memcpy(l_tot_red, l_tot, map_size*sizeof(double));
@@ -9437,8 +9434,8 @@ void CConfig::GEMMProfilingCSV(void) {
94379434
/* Parallel executable. The profiling data must be sent to the master node.
94389435
First determine the rank and size. */
94399436
int size;
9440-
SU2_MPI::Comm_rank(MPI_COMM_WORLD, &rank);
9441-
SU2_MPI::Comm_size(MPI_COMM_WORLD, &size);
9437+
SU2_MPI::Comm_rank(SU2_MPI::GetComm(), &rank);
9438+
SU2_MPI::Comm_size(SU2_MPI::GetComm(), &size);
94429439

94439440
/* Check for the master node. */
94449441
if(rank == MASTER_NODE) {
@@ -9449,7 +9446,7 @@ void CConfig::GEMMProfilingCSV(void) {
94499446
/* Block until a message from this processor arrives. Determine
94509447
the number of entries in the receive buffers. */
94519448
SU2_MPI::Status status;
9452-
SU2_MPI::Probe(proc, 0, MPI_COMM_WORLD, &status);
9449+
SU2_MPI::Probe(proc, 0, SU2_MPI::GetComm(), &status);
94539450

94549451
int nEntries;
94559452
SU2_MPI::Get_count(&status, MPI_LONG, &nEntries);
@@ -9463,15 +9460,15 @@ void CConfig::GEMMProfilingCSV(void) {
94639460
vector<long> recvBufMNK(3*nEntries);
94649461

94659462
SU2_MPI::Recv(recvBufNCalls.data(), recvBufNCalls.size(),
9466-
MPI_LONG, proc, 0, MPI_COMM_WORLD, &status);
9463+
MPI_LONG, proc, 0, SU2_MPI::GetComm(), &status);
94679464
SU2_MPI::Recv(recvBufTotTime.data(), recvBufTotTime.size(),
9468-
MPI_DOUBLE, proc, 1, MPI_COMM_WORLD, &status);
9465+
MPI_DOUBLE, proc, 1, SU2_MPI::GetComm(), &status);
94699466
SU2_MPI::Recv(recvBufMinTime.data(), recvBufMinTime.size(),
9470-
MPI_DOUBLE, proc, 2, MPI_COMM_WORLD, &status);
9467+
MPI_DOUBLE, proc, 2, SU2_MPI::GetComm(), &status);
94719468
SU2_MPI::Recv(recvBufMaxTime.data(), recvBufMaxTime.size(),
9472-
MPI_DOUBLE, proc, 3, MPI_COMM_WORLD, &status);
9469+
MPI_DOUBLE, proc, 3, SU2_MPI::GetComm(), &status);
94739470
SU2_MPI::Recv(recvBufMNK.data(), recvBufMNK.size(),
9474-
MPI_LONG, proc, 4, MPI_COMM_WORLD, &status);
9471+
MPI_LONG, proc, 4, SU2_MPI::GetComm(), &status);
94759472

94769473
/* Loop over the number of entries. */
94779474
for(int i=0; i<nEntries; ++i) {
@@ -9520,15 +9517,15 @@ void CConfig::GEMMProfilingCSV(void) {
95209517

95219518
/* Send the data to the master node using blocking sends. */
95229519
SU2_MPI::Send(GEMM_Profile_NCalls.data(), GEMM_Profile_NCalls.size(),
9523-
MPI_LONG, MASTER_NODE, 0, MPI_COMM_WORLD);
9520+
MPI_LONG, MASTER_NODE, 0, SU2_MPI::GetComm());
95249521
SU2_MPI::Send(GEMM_Profile_TotTime.data(), GEMM_Profile_TotTime.size(),
9525-
MPI_DOUBLE, MASTER_NODE, 1, MPI_COMM_WORLD);
9522+
MPI_DOUBLE, MASTER_NODE, 1, SU2_MPI::GetComm());
95269523
SU2_MPI::Send(GEMM_Profile_MinTime.data(), GEMM_Profile_MinTime.size(),
9527-
MPI_DOUBLE, MASTER_NODE, 2, MPI_COMM_WORLD);
9524+
MPI_DOUBLE, MASTER_NODE, 2, SU2_MPI::GetComm());
95289525
SU2_MPI::Send(GEMM_Profile_MaxTime.data(), GEMM_Profile_MaxTime.size(),
9529-
MPI_DOUBLE, MASTER_NODE, 3, MPI_COMM_WORLD);
9526+
MPI_DOUBLE, MASTER_NODE, 3, SU2_MPI::GetComm());
95309527
SU2_MPI::Send(sendBufMNK.data(), sendBufMNK.size(),
9531-
MPI_LONG, MASTER_NODE, 4, MPI_COMM_WORLD);
9528+
MPI_LONG, MASTER_NODE, 4, SU2_MPI::GetComm());
95329529
}
95339530

95349531
#endif

‎Common/src/adt/CADTElemClass.cpp‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -75,14 +75,14 @@ CADTElemClass::CADTElemClass(unsigned short val_nDim,
7575
/*--- First determine the number of points per rank and make them
7676
available to all ranks. ---*/
7777
int rank, size;
78-
SU2_MPI::Comm_rank(MPI_COMM_WORLD, &rank);
79-
SU2_MPI::Comm_size(MPI_COMM_WORLD, &size);
78+
SU2_MPI::Comm_rank(SU2_MPI::GetComm(), &rank);
79+
SU2_MPI::Comm_size(SU2_MPI::GetComm(), &size);
8080

8181
vector<int> recvCounts(size), displs(size);
8282
int sizeLocal = (int) val_coor.size();
8383

8484
SU2_MPI::Allgather(&sizeLocal, 1, MPI_INT, recvCounts.data(), 1,
85-
MPI_INT, MPI_COMM_WORLD);
85+
MPI_INT, SU2_MPI::GetComm());
8686
displs[0] = 0;
8787
for(int i=1; i<size; ++i) displs[i] = displs[i-1] + recvCounts[i-1];
8888

@@ -98,14 +98,14 @@ CADTElemClass::CADTElemClass(unsigned short val_nDim,
9898

9999
coorPoints.resize(sizeGlobal);
100100
SU2_MPI::Allgatherv(val_coor.data(), sizeLocal, MPI_DOUBLE, coorPoints.data(),
101-
recvCounts.data(), displs.data(), MPI_DOUBLE, MPI_COMM_WORLD);
101+
recvCounts.data(), displs.data(), MPI_DOUBLE, SU2_MPI::GetComm());
102102

103103
/*--- Determine the number of elements per rank and make them
104104
available to all ranks. ---*/
105105
sizeLocal = (int) val_VTKElem.size();
106106

107107
SU2_MPI::Allgather(&sizeLocal, 1, MPI_INT, recvCounts.data(), 1,
108-
MPI_INT, MPI_COMM_WORLD);
108+
MPI_INT, SU2_MPI::GetComm());
109109
displs[0] = 0;
110110
for(int i=1; i<size; ++i) displs[i] = displs[i-1] + recvCounts[i-1];
111111

@@ -118,13 +118,13 @@ CADTElemClass::CADTElemClass(unsigned short val_nDim,
118118
localElemIDs.resize(sizeGlobal);
119119

120120
SU2_MPI::Allgatherv(val_VTKElem.data(), sizeLocal, MPI_UNSIGNED_SHORT, elemVTK_Type.data(),
121-
recvCounts.data(), displs.data(), MPI_UNSIGNED_SHORT, MPI_COMM_WORLD);
121+
recvCounts.data(), displs.data(), MPI_UNSIGNED_SHORT, SU2_MPI::GetComm());
122122

123123
SU2_MPI::Allgatherv(val_markerID.data(), sizeLocal, MPI_UNSIGNED_SHORT, localMarkers.data(),
124-
recvCounts.data(), displs.data(), MPI_UNSIGNED_SHORT, MPI_COMM_WORLD);
124+
recvCounts.data(), displs.data(), MPI_UNSIGNED_SHORT, SU2_MPI::GetComm());
125125

126126
SU2_MPI::Allgatherv(val_elemID.data(), sizeLocal, MPI_UNSIGNED_LONG, localElemIDs.data(),
127-
recvCounts.data(), displs.data(), MPI_UNSIGNED_LONG, MPI_COMM_WORLD);
127+
recvCounts.data(), displs.data(), MPI_UNSIGNED_LONG, SU2_MPI::GetComm());
128128

129129
/*--- Create the content of ranksOfElems, which stores the original ranks
130130
where the elements come from. ---*/
@@ -140,7 +140,7 @@ CADTElemClass::CADTElemClass(unsigned short val_nDim,
140140
sizeLocal = (int) val_connElem.size();
141141

142142
SU2_MPI::Allgather(&sizeLocal, 1, MPI_INT, recvCounts.data(), 1,
143-
MPI_INT, MPI_COMM_WORLD);
143+
MPI_INT, SU2_MPI::GetComm());
144144
displs[0] = 0;
145145
for(int i=1; i<size; ++i) displs[i] = displs[i-1] + recvCounts[i-1];
146146

@@ -150,14 +150,14 @@ CADTElemClass::CADTElemClass(unsigned short val_nDim,
150150
elemConns.resize(sizeGlobal);
151151

152152
SU2_MPI::Allgatherv(val_connElem.data(), sizeLocal, MPI_UNSIGNED_LONG, elemConns.data(),
153-
recvCounts.data(), displs.data(), MPI_UNSIGNED_LONG, MPI_COMM_WORLD);
153+
recvCounts.data(), displs.data(), MPI_UNSIGNED_LONG, SU2_MPI::GetComm());
154154
}
155155
else {
156156

157157
/*--- A local tree must be built. Copy the data from the arguments into the
158158
member variables and set the ranks to the rank of this processor. ---*/
159159
int rank;
160-
SU2_MPI::Comm_rank(MPI_COMM_WORLD, &rank);
160+
SU2_MPI::Comm_rank(SU2_MPI::GetComm(), &rank);
161161

162162
coorPoints = val_coor;
163163
elemConns = val_connElem;

‎Common/src/adt/CADTPointsOnlyClass.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,14 +52,14 @@ CADTPointsOnlyClass::CADTPointsOnlyClass(unsigned short nDim,
5252
First determine the number of points per rank and store them in such
5353
a way that the info can be used directly in Allgatherv. ---*/
5454
int rank, size;
55-
SU2_MPI::Comm_rank(MPI_COMM_WORLD, &rank);
56-
SU2_MPI::Comm_size(MPI_COMM_WORLD, &size);
55+
SU2_MPI::Comm_rank(SU2_MPI::GetComm(), &rank);
56+
SU2_MPI::Comm_size(SU2_MPI::GetComm(), &size);
5757

5858
vector<int> recvCounts(size), displs(size);
5959
int sizeLocal = (int) nPoints;
6060

6161
SU2_MPI::Allgather(&sizeLocal, 1, MPI_INT, recvCounts.data(), 1,
62-
MPI_INT, MPI_COMM_WORLD);
62+
MPI_INT, SU2_MPI::GetComm());
6363
displs[0] = 0;
6464
for(int i=1; i<size; ++i) displs[i] = displs[i-1] + recvCounts[i-1];
6565

@@ -69,26 +69,26 @@ CADTPointsOnlyClass::CADTPointsOnlyClass(unsigned short nDim,
6969
localPointIDs.resize(sizeGlobal);
7070
SU2_MPI::Allgatherv(pointID, sizeLocal, MPI_UNSIGNED_LONG, localPointIDs.data(),
7171
recvCounts.data(), displs.data(), MPI_UNSIGNED_LONG,
72-
MPI_COMM_WORLD);
72+
SU2_MPI::GetComm());
7373

7474
ranksOfPoints.resize(sizeGlobal);
7575
vector<int> rankLocal(sizeLocal, rank);
7676
SU2_MPI::Allgatherv(rankLocal.data(), sizeLocal, MPI_INT, ranksOfPoints.data(),
77-
recvCounts.data(), displs.data(), MPI_INT, MPI_COMM_WORLD);
77+
recvCounts.data(), displs.data(), MPI_INT, SU2_MPI::GetComm());
7878

7979
/*--- Gather the coordinates of the points on all ranks. ---*/
8080
for(int i=0; i<size; ++i) {recvCounts[i] *= nDim; displs[i] *= nDim;}
8181

8282
coorPoints.resize(nDim*sizeGlobal);
8383
SU2_MPI::Allgatherv(coor, nDim*sizeLocal, MPI_DOUBLE, coorPoints.data(),
84-
recvCounts.data(), displs.data(), MPI_DOUBLE, MPI_COMM_WORLD);
84+
recvCounts.data(), displs.data(), MPI_DOUBLE, SU2_MPI::GetComm());
8585
}
8686
else {
8787

8888
/*--- A local tree must be built. Copy the coordinates and point IDs and
8989
set the ranks to the rank of this processor. ---*/
9090
int rank;
91-
SU2_MPI::Comm_rank(MPI_COMM_WORLD, &rank);
91+
SU2_MPI::Comm_rank(SU2_MPI::GetComm(), &rank);
9292

9393
coorPoints.assign(coor, coor + nDim*nPoints);
9494
localPointIDs.assign(pointID, pointID + nPoints);

0 commit comments

Comments
 (0)