reduce size of gradient for output, print global tape memory stats

Author: pcarruscag

Common/src/geometry/CPhysicalGeometry.cpp

@@ -7662,10 +7662,9 @@ void CPhysicalGeometry::SetBoundControlVolume(const CConfig *config, unsigned sh
         vertex[iMarker][iVertex]->SetZeroValues();
   }
 
-  SU2_OMP_MASTER {
-
   /*--- Loop over all the boundary elements ---*/
 
+  SU2_OMP_FOR_DYN(1)
   for (unsigned short iMarker = 0; iMarker < nMarker; iMarker++) {
     for (unsigned long iElem = 0; iElem < nElem_Bound[iMarker]; iElem++) {
 
@@ -7726,7 +7725,6 @@ void CPhysicalGeometry::SetBoundControlVolume(const CConfig *config, unsigned sh
       AD::EndPreacc();
     }
   }
-  } SU2_OMP_BARRIER
 
   /*--- Check if there is a normal with null area ---*/
 

SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp

@@ -564,11 +564,20 @@ void CDiscAdjMultizoneDriver::SetRecording(unsigned short kind_recording, Kind_T
     }
   }
 
-  if (rank == MASTER_NODE) {
-    if(kind_recording != NONE && config_container[record_zone]->GetWrt_AD_Statistics()) {
-      AD::PrintStatistics();
+  if (kind_recording != NONE && driver_config->GetWrt_AD_Statistics()) {
+    if (rank == MASTER_NODE) AD::PrintStatistics();
+#ifdef CODI_REVERSE_TYPE
+    if (size > SINGLE_NODE) {
+      su2double myMem = AD::globalTape.getTapeValues().getUsedMemorySize(), totMem = 0.0;
+      SU2_MPI::Allreduce(&myMem, &totMem, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+      if (rank == MASTER_NODE) {
+        cout << "MPI\n";
+        cout << "-------------------------------------\n";
+        cout << "  Total memory used      :  " << totMem << " MB\n";
+        cout << "-------------------------------------\n" << endl;
+      }
     }
-    cout << "-------------------------------------------------------------------------\n" << endl;
+#endif
   }
 
   AD::StopRecording();

SU2_CFD/src/drivers/CDiscAdjSinglezoneDriver.cpp

@@ -291,9 +291,20 @@ void CDiscAdjSinglezoneDriver::SetRecording(unsigned short kind_recording){
 
   SetObjFunction();
 
-  if (rank == MASTER_NODE && kind_recording != NONE && config_container[ZONE_0]->GetWrt_AD_Statistics()) {
-    AD::PrintStatistics();
-    cout << "-------------------------------------------------------------------------\n" << endl;
+  if (kind_recording != NONE && config_container[ZONE_0]->GetWrt_AD_Statistics()) {
+    if (rank == MASTER_NODE) AD::PrintStatistics();
+#ifdef CODI_REVERSE_TYPE
+    if (size > SINGLE_NODE) {
+      su2double myMem = AD::globalTape.getTapeValues().getUsedMemorySize(), totMem = 0.0;
+      SU2_MPI::Allreduce(&myMem, &totMem, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+      if (rank == MASTER_NODE) {
+        cout << "MPI\n";
+        cout << "-------------------------------------\n";
+        cout << "  Total memory used      :  " << totMem << " MB\n";
+        cout << "-------------------------------------\n" << endl;
+      }
+    }
+#endif
   }
 
   AD::StopRecording();

SU2_CFD/src/solvers/CNSSolver.cpp

@@ -110,7 +110,11 @@ void CNSSolver::Preprocessing(CGeometry *geometry, CSolver **solver_container, C
 
   CommonPreprocessing(geometry, solver_container, config, iMesh, iRKStep, RunTime_EqSystem, Output);
 
-  /*--- Compute gradient for MUSCL reconstruction. ---*/
+  /*--- Compute gradient for MUSCL reconstruction, for output (i.e. the
+   turbulence solver) only density and velocity are needed ---*/
+
+  const auto nPrimVarGrad_bak = nPrimVarGrad;
+  if (Output) nPrimVarGrad = 1+nDim;
 
   if (config->GetReconstructionGradientRequired() && (iMesh == MESH_0)) {
     switch (config->GetKind_Gradient_Method_Recon()) {
@@ -132,6 +136,8 @@ void CNSSolver::Preprocessing(CGeometry *geometry, CSolver **solver_container, C
     SetPrimitive_Gradient_LS(geometry, config);
   }
 
+  nPrimVarGrad = nPrimVarGrad_bak;
+
   /*--- Compute the limiter in case we need it in the turbulence model or to limit the
    *    viscous terms (check this logic with JST and 2nd order turbulence model) ---*/
 
@@ -150,21 +156,17 @@ void CNSSolver::Preprocessing(CGeometry *geometry, CSolver **solver_container, C
 
   nodes->SetVorticity_StrainMag();
 
+  /*--- Min and Max are not really differentiable ---*/
+  const bool wasActive = AD::BeginPassive();
+
   su2double strainMax = 0.0, omegaMax = 0.0;
 
   SU2_OMP(for schedule(static,omp_chunk_size) nowait)
   for (unsigned long iPoint = 0; iPoint < nPoint; iPoint++) {
-
-    su2double StrainMag = nodes->GetStrainMag(iPoint);
-    const su2double* Vorticity = nodes->GetVorticity(iPoint);
-    su2double Omega = sqrt(Vorticity[0]*Vorticity[0]+ Vorticity[1]*Vorticity[1]+ Vorticity[2]*Vorticity[2]);
-
-    strainMax = max(strainMax, StrainMag);
-    omegaMax = max(omegaMax, Omega);
-
+    strainMax = max(strainMax, nodes->GetStrainMag(iPoint));
+    omegaMax = max(omegaMax, GeometryToolbox::Norm(3, nodes->GetVorticity(iPoint)));
   }
-  SU2_OMP_CRITICAL
-  {
+  SU2_OMP_CRITICAL {
     StrainMag_Max = max(StrainMag_Max, strainMax);
     Omega_Max = max(Omega_Max, omegaMax);
   }
@@ -182,6 +184,8 @@ void CNSSolver::Preprocessing(CGeometry *geometry, CSolver **solver_container, C
     SU2_OMP_BARRIER
   }
 
+  AD::EndPassive(wasActive);
+
   /*--- Compute the TauWall from the wall functions ---*/
 
   if (wall_functions) {