[ET Trac] [Einstein Toolkit] #434: Keep track of masked-out volume in CarpetMask

[Einstein Toolkit]

#434: Keep track of masked-out volume in CarpetMask
--------------------------+-------------------------------------------------
  Reporter:  eschnett     |       Owner:  eschnett  
      Type:  enhancement  |      Status:  review    
  Priority:  blocker      |   Milestone:  ET_2012_11
 Component:  Carpet       |     Version:            
Resolution:               |    Keywords:            
--------------------------+-------------------------------------------------
Changes (by eschnett):

  * priority:  minor => blocker
  * milestone:  ET_2013_05 => ET_2012_11


Comment:

 I confirm that my previous patch has no effect, since ghost zones were
 already correctly excluded.

 I see the OpenMP error; a global variable is modified by all patches
 simultaneously. A reduction clause is missing.

 The segfault seems to occur in an I/O statement; a certain "reserve"
 function fails. This looks like memory management. I don't know where
 string buffers are used -- this may be a timer issue. Raising level since
 this may be a blocker.

 New patch for OpenMP issue:
 {{{
 $ git diff .
 diff --git a/Carpet/CarpetMask/src/mask_excluded.cc
 b/Carpet/CarpetMask/src/mask
 index aec5ffa..8787ef1 100644
 --- a/Carpet/CarpetMask/src/mask_excluded.cc
 +++ b/Carpet/CarpetMask/src/mask_excluded.cc
 @@ -62,7 +62,8 @@ namespace CarpetMask {

          bool const exterior = exclude_exterior[n];

 -#pragma omp parallel
 +        CCTK_REAL local_excised = 0.0;
 +#pragma omp parallel reduction (+: local_excised)
          CCTK_LOOP3_ALL(CarpetExcludedSetup, cctkGH, i,j,k) {
            int const ind = CCTK_GFINDEX3D (cctkGH, i, j, k);

 @@ -75,11 +76,12 @@ namespace CarpetMask {
                dx2 + dy2 + dz2 <= r2)
            {
              // Tally up the weight we are removing
 -            * excised_cells += cell_volume * factor * BCNT(iweight[ind]);
 +            local_excised += cell_volume * factor * BCNT(iweight[ind]);
              iweight[ind] = 0;
            }

          } CCTK_ENDLOOP3_ALL(CarpetExcludedSetup);
 +        * excised_cells += local_excised;

        } // if r>=0
      }   // for n
 diff --git a/Carpet/CarpetMask/src/mask_surface.cc
 b/Carpet/CarpetMask/src/mask_
 index f2a592b..044d1de 100644
 --- a/Carpet/CarpetMask/src/mask_surface.cc
 +++ b/Carpet/CarpetMask/src/mask_surface.cc
 @@ -129,7 +129,8 @@ namespace CarpetMask {
              }
            }

 -#pragma omp parallel
 +        CCTK_REAL local_excised = 0.0;
 +#pragma omp parallel reduction (+: local_excised)
            CCTK_LOOP3_ALL(CarpetSurfaceSetup, cctkGH, i,j,k) {
              int const ind = CCTK_GFINDEX3D (cctkGH, i, j, k);

 @@ -142,7 +143,7 @@ namespace CarpetMask {
              if (rho < 1.0e-12) {
                // Always excise the surface origin
                // Tally up the weight we are removing
 -              * excised_cells += cell_volume * factor *
 BCNT(iweight[ind]);
 +              local_excised += cell_volume * factor * BCNT(iweight[ind]);
                iweight[ind] = 0;
              } else {
                CCTK_REAL theta =
 @@ -188,11 +189,12 @@ namespace CarpetMask {
                  sf_radius[a + maxntheta * (b + maxnphi * sn)];
                if (rho <= dr * shrink_factor) {
                  // Tally up the weight we are removing
 -                * excised_cells += cell_volume * factor *
 BCNT(iweight[ind]);
 +                local_excised += cell_volume * factor *
 BCNT(iweight[ind]);
                  iweight[ind] = 0;
                }
              }
            } CCTK_ENDLOOP3_ALL(CarpetSurfaceSetup);
 +          * excised_cells += local_excised;

          } else {

 }}}

-- 
Ticket URL: <https://trac.einsteintoolkit.org/ticket/434#comment:19>
Einstein Toolkit <http://einsteintoolkit.org>
The Einstein Toolkit