Skip to content

Commit 0829072

Browse files
bigfootedbigfooted
authored
Fix writing of multiple inlet profiles when using MPI (#1999)
* fix mpi communication for multiple inlet profiles
1 parent b159ff8 commit 0829072

3 files changed

Lines changed: 68 additions & 29 deletions

File tree

‎Common/include/option_structure.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ const unsigned int EXIT_DIVERGENCE = 2; /*!< \brief Exit code (divergence). */
7474

7575
const unsigned int MAX_PARAMETERS = 10; /*!< \brief Maximum number of parameters for a design variable definition. */
7676
const unsigned int MAX_NUMBER_PERIODIC = 10; /*!< \brief Maximum number of periodic boundary conditions. */
77-
const unsigned int MAX_STRING_SIZE = 200; /*!< \brief Maximum number of domains. */
77+
const unsigned int MAX_STRING_SIZE = 400; /*!< \brief Maximum size of a generic string. */
7878
const unsigned int MAX_NUMBER_FFD = 15; /*!< \brief Maximum number of FFDBoxes for the FFD. */
7979
enum: unsigned int{MAX_SOLS = 13}; /*!< \brief Maximum number of solutions at the same time (dimension of solution container array). */
8080
const unsigned int MAX_TERMS = 7; /*!< \brief Maximum number of terms in the numerical equations (dimension of solver container array). */

‎SU2_CFD/include/CMarkerProfileReaderFVM.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ class CMarkerProfileReaderFVM {
7171

7272
vector<unsigned long> numberOfRowsInProfile; /*!< \brief Auxiliary structure for holding the number of rows for a particular marker in a profile file. */
7373
vector<unsigned long> numberOfColumnsInProfile; /*!< \brief Auxiliary structure for holding the number of columns for a particular marker in a profile file. */
74+
vector<string> totalColumnNames; /*!< \brief Names of the columns for the profile, one for each inlet marker. */
75+
vector<string> totalColumnValues; /*!< \brief Initial values for the profile, constructed from MARKER_INLET. */
7476

7577
vector<vector<passivedouble> > profileData; /*!< \brief Auxiliary structure for holding the data values from a profile file. */
7678
vector<vector<vector<su2double> > > profileCoords; /*!< \brief Data structure for holding the merged inlet boundary coordinates from all ranks. */

‎SU2_CFD/src/CMarkerProfileReaderFVM.cpp‎

Lines changed: 65 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,9 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
207207
unsigned long index, iChar;
208208

209209
char str_buf[MAX_STRING_SIZE];
210+
char name_buf[MAX_STRING_SIZE];
211+
char value_buf[MAX_STRING_SIZE];
212+
210213
vector<string> Marker_Tags;
211214

212215
vector<unsigned long> nRowCum_Counter;
@@ -219,6 +222,7 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
219222
nLocalPoint = 0; numberOfProfiles = 0;
220223
for (iMarker = 0; iMarker < config->GetnMarker_All(); iMarker++) {
221224
if (config->GetMarker_All_KindBC(iMarker) == markerType) {
225+
222226
numberOfProfiles++;
223227
for (iVertex = 0; iVertex < geometry->nVertex[iMarker]; iVertex++) {
224228
iPoint = geometry->vertex[iMarker][iVertex]->GetNode();
@@ -238,7 +242,8 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
238242
SU2_MPI::Gather(&Buffer_Send_nPoin, 1, MPI_UNSIGNED_LONG,
239243
Buffer_Recv_nPoin, 1, MPI_UNSIGNED_LONG, MASTER_NODE, SU2_MPI::GetComm());
240244
SU2_MPI::Allreduce(&nLocalPoint, &MaxLocalPoint, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm());
241-
SU2_MPI::Allreduce(&numberOfProfiles, &maxProfiles, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm());
245+
SU2_MPI::Reduce(&numberOfProfiles, &maxProfiles, 1, MPI_UNSIGNED_LONG, MPI_SUM, MASTER_NODE,
246+
SU2_MPI::GetComm());
242247

243248
/*--- Send and Recv buffers. ---*/
244249

@@ -254,14 +259,23 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
254259
char *Buffer_Send_Str = new char[MaxLocalPoint*MAX_STRING_SIZE];
255260
char *Buffer_Recv_Str = nullptr;
256261

262+
char *Buffer_Send_Name = new char[MaxLocalPoint*MAX_STRING_SIZE];
263+
char *Buffer_Recv_Name = nullptr;
264+
265+
char *Buffer_Send_Value = new char[MaxLocalPoint*MAX_STRING_SIZE];
266+
char *Buffer_Recv_Value = nullptr;
267+
257268
/*--- Prepare the receive buffers in the master node only. ---*/
258269

259270
if (rank == MASTER_NODE) {
260271

261272
Buffer_Recv_X = new su2double[nProcessor*MaxLocalPoint];
262273
Buffer_Recv_Y = new su2double[nProcessor*MaxLocalPoint];
263274
if (dimension == 3) Buffer_Recv_Z = new su2double[nProcessor*MaxLocalPoint];
275+
264276
Buffer_Recv_Str = new char[nProcessor*MaxLocalPoint*MAX_STRING_SIZE];
277+
Buffer_Recv_Name = new char[nProcessor*MaxLocalPoint*MAX_STRING_SIZE];
278+
Buffer_Recv_Value = new char[nProcessor*MaxLocalPoint*MAX_STRING_SIZE];
265279

266280
/*--- Sum total number of nodes to be written and allocate arrays ---*/
267281

@@ -275,6 +289,9 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
275289
profileCoords[iMarker].resize(dimension);
276290
}
277291

292+
totalColumnNames.resize(maxProfiles);
293+
totalColumnValues.resize(maxProfiles);
294+
278295
}
279296

280297
/*--- Main communication routine. Loop over each coordinate and perform
@@ -316,6 +333,16 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
316333
SPRINTF(&Buffer_Send_Str[jPoint*MAX_STRING_SIZE], "%s",
317334
config->GetMarker_All_TagBound(iMarker).c_str());
318335

336+
/*--- Store the column names ---*/
337+
338+
SPRINTF(&Buffer_Send_Name[jPoint*MAX_STRING_SIZE], "%s",
339+
columnNames[iMarker].c_str());
340+
341+
/*--- Store the column values ---*/
342+
343+
SPRINTF(&Buffer_Send_Value[jPoint*MAX_STRING_SIZE], "%s",
344+
columnValues[iMarker].c_str());
345+
319346
/*--- Increment jPoint as the counter. We need this because iPoint
320347
may include halo nodes that we skip over during this loop. ---*/
321348

@@ -339,11 +366,18 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
339366
SU2_MPI::Gather(Buffer_Send_Str, (int)MaxLocalPoint*MAX_STRING_SIZE, MPI_CHAR,
340367
Buffer_Recv_Str, (int)MaxLocalPoint*MAX_STRING_SIZE, MPI_CHAR, MASTER_NODE, SU2_MPI::GetComm());
341368

369+
SU2_MPI::Gather(Buffer_Send_Name, (int)MaxLocalPoint*MAX_STRING_SIZE, MPI_CHAR,
370+
Buffer_Recv_Name, (int)MaxLocalPoint*MAX_STRING_SIZE, MPI_CHAR, MASTER_NODE, SU2_MPI::GetComm());
371+
SU2_MPI::Gather(Buffer_Send_Value, (int)MaxLocalPoint*MAX_STRING_SIZE, MPI_CHAR,
372+
Buffer_Recv_Value, (int)MaxLocalPoint*MAX_STRING_SIZE, MPI_CHAR, MASTER_NODE, SU2_MPI::GetComm());
373+
342374
/*--- The master node unpacks and sorts this variable by marker tag. ---*/
343375

344376
if (rank == MASTER_NODE) {
345377

346378
profileTags.clear();
379+
totalColumnNames.clear();
380+
totalColumnValues.clear();
347381

348382
/*--- First, parse the marker tags to count how many total profile markers
349383
we have now on the master. ---*/
@@ -353,30 +387,34 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
353387
index = (iProcessor*MaxLocalPoint + iPoint)*MAX_STRING_SIZE;
354388
for (iChar = 0; iChar < MAX_STRING_SIZE; iChar++) {
355389
str_buf[iChar] = Buffer_Recv_Str[index + iChar];
390+
name_buf[iChar] = Buffer_Recv_Name[index + iChar];
391+
value_buf[iChar] = Buffer_Recv_Value[index + iChar];
356392
}
393+
357394
Marker_Tags.push_back(str_buf);
358-
profileTags.push_back(str_buf);
359-
}
360-
}
361395

362-
/*--- Remove the duplicate profile marker strings. From 1 per point to 1 per marker. ---*/
396+
/*--- only add if not already in the list ---*/
397+
if (std::find(profileTags.begin(), profileTags.end(), str_buf) == profileTags.end()) {
398+
profileTags.push_back(str_buf);
399+
totalColumnNames.push_back(name_buf);
400+
totalColumnValues.push_back(value_buf);
401+
}
363402

364-
profileTags.erase(unique(profileTags.begin(),
365-
profileTags.end()),
366-
profileTags.end());
403+
}
367404

368-
/*--- Store the unique number of markers for writing later. ---*/
405+
}
369406

370407
numberOfProfiles = profileTags.size();
371408

372409
/*--- Count the number of rows (nodes) per marker. ---*/
373410

374-
numberOfRowsInProfile.resize(numberOfProfiles,0.0);
411+
numberOfRowsInProfile.resize(maxProfiles,0.0);
375412
jPoint = 0;
376413
for (iProcessor = 0; iProcessor < nProcessor; iProcessor++) {
377414
for (iPoint = 0; iPoint < Buffer_Recv_nPoin[iProcessor]; iPoint++) {
378-
for (iMarker = 0; iMarker < numberOfProfiles; iMarker++) {
415+
for (iMarker = 0; iMarker < maxProfiles; iMarker++) {
379416
if (profileTags[iMarker] == Marker_Tags[jPoint]) {
417+
380418
numberOfRowsInProfile[iMarker]++;
381419
}
382420
}
@@ -389,7 +427,7 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
389427
jPoint = 0; kPoint = 0;
390428
for (iProcessor = 0; iProcessor < nProcessor; iProcessor++) {
391429
for (iPoint = 0; iPoint < Buffer_Recv_nPoin[iProcessor]; iPoint++) {
392-
for (iMarker = 0; iMarker < numberOfProfiles; iMarker++) {
430+
for (iMarker = 0; iMarker < maxProfiles; iMarker++) {
393431

394432
if (profileTags[iMarker] == Marker_Tags[kPoint]) {
395433

@@ -415,22 +453,26 @@ void CMarkerProfileReaderFVM::MergeProfileMarkers() {
415453
jPoint = (iProcessor+1)*MaxLocalPoint;
416454

417455
}
456+
418457
}
419458

420459
/*--- Immediately release the temporary data buffers. ---*/
421-
422460
delete [] Buffer_Send_X;
423461
delete [] Buffer_Send_Y;
424462
delete [] Buffer_Send_Z;
425463
delete [] Buffer_Send_Str;
464+
delete [] Buffer_Send_Name;
465+
delete [] Buffer_Send_Value;
466+
426467
if (rank == MASTER_NODE) {
427468
delete [] Buffer_Recv_X;
428469
delete [] Buffer_Recv_Y;
429470
delete [] Buffer_Recv_Z;
430471
delete [] Buffer_Recv_nPoin;
431472
delete [] Buffer_Recv_Str;
473+
delete [] Buffer_Recv_Name;
474+
delete [] Buffer_Recv_Value;
432475
}
433-
434476
}
435477

436478
void CMarkerProfileReaderFVM::WriteMarkerProfileTemplate() {
@@ -448,43 +490,38 @@ void CMarkerProfileReaderFVM::WriteMarkerProfileTemplate() {
448490
if (rank == MASTER_NODE) {
449491

450492
ofstream node_file("example_"+filename);
451-
452493
node_file << "NMARK= " << numberOfProfiles << endl;
453494

454-
unsigned short iMarkerCounter = 0;
455-
for (unsigned short iMarker = 0; iMarker < config->GetnMarker_All(); iMarker++) {
456-
if (config->GetMarker_All_KindBC(iMarker) == markerType) {
495+
for (unsigned long iMarker = 0; iMarker < numberOfProfiles; iMarker++) {
457496

458497
/*--- Access the default data for this marker. ---*/
459-
460-
string Marker_Tag = profileTags[iMarkerCounter];
498+
string Marker_Tag = profileTags[iMarker];
461499

462500
/*--- Header information for this marker. ---*/
463501

464502
node_file << "MARKER_TAG= " << Marker_Tag << endl;
465-
node_file << "NROW=" << numberOfRowsInProfile[iMarkerCounter] << endl;
503+
node_file << "NROW=" << numberOfRowsInProfile[iMarker] << endl;
466504
node_file << "NCOL=" << nColumns << endl;
467505

506+
468507
/*--- header line (names of the columns) --- */
469-
node_file << columnNames[iMarkerCounter] << endl;
508+
node_file << totalColumnNames[iMarker] << endl;
470509

471510
node_file << setprecision(15);
472511
node_file << std::scientific;
473512

474513
/*--- Loop over the data structure and write the coords and vars. ---*/
475514

476-
for (unsigned long iPoint = 0; iPoint < numberOfRowsInProfile[iMarkerCounter]; iPoint++) {
477-
515+
for (unsigned long iPoint = 0; iPoint < numberOfRowsInProfile[iMarker]; iPoint++) {
478516
for (unsigned short iDim = 0; iDim < dimension; iDim++) {
479-
node_file << profileCoords[iMarkerCounter][iDim][iPoint] << "\t";
517+
node_file << profileCoords[iMarker][iDim][iPoint] << "\t";
480518
}
481519

482-
node_file << columnValues[iMarkerCounter] << endl;
520+
node_file << totalColumnValues[iMarker] << endl;
483521

484522
}
485523

486-
iMarkerCounter++;
487-
}
524+
488525
} // iMarker
489526
node_file.close();
490527

0 commit comments

Comments
 (0)