User: rhaas Date: 2012/06/05 03:30 PM

Modified: /trunk/ interface.ccl, param.ccl, schedule.ccl /trunk/src/ make.code.defn

Log: GRHydro: Schedule AtmosphereReset in LAST Mol_PostStep

This saves some syncs. This also gets rid of the akward parameter "disable_sync_and_tmunu_after_mol".

Original patch by Christian Reisswig. Adapted to not rely on MoL support by Roland Haas. Bug fixed by Christian Reisswig.

File Changes:

Directory: /trunk/src/ ======================

File [modified]: make.code.defn Delta lines: +1 -0 =================================================================== --- trunk/src/make.code.defn 2012-06-05 20:29:03 UTC (rev 342) +++ trunk/src/make.code.defn 2012-06-05 20:30:33 UTC (rev 343) @@ -63,6 +63,7 @@ GRHydro_PPMMReconstruct_drv.F90\ GRHydro_ENOReconstruct_drv.F90\ GRHydro_PPMReconstruct_drv.F90\ + GRHydro_LastMoLPostStep.c\ GRHydro_TVDReconstruct_drv.F90

Directory: /trunk/ ==================

File [modified]: interface.ccl Delta lines: +1 -0 =================================================================== --- trunk/interface.ccl 2012-06-05 20:29:03 UTC (rev 342) +++ trunk/interface.ccl 2012-06-05 20:30:33 UTC (rev 343) @@ -476,6 +476,7 @@ private:

int GRHydro_reflevel type = SCALAR tags='checkpoint="no"' "Refinement level GRHydro is working on right now" +int InLastMoLPostStep type = SCALAR tags='checkpoint="no"' "Flag to indicate if we are currently in the last MoL_PostStep"

real GRHydro_con_bext type = GF Timelevels = 1 tags='Prolongation="None" checkpoint="no"' {

File [modified]: param.ccl Delta lines: +4 -6 =================================================================== --- trunk/param.ccl 2012-06-05 20:29:03 UTC (rev 342) +++ trunk/param.ccl 2012-06-05 20:30:33 UTC (rev 343) @@ -592,14 +592,12 @@ 0: :: "Greater or equal to 0" } 0

-# The following parameter is to reduce the number of syncs/boundary calls done -# it will lead to correct results when set to yes ONLY when the simulation does not use -# an atmosphere at all. -boolean disable_sync_and_tmunu_after_MoL "Do not sync/prolongate and reset Tmunu after MoL integration is done" STEERABLE=ALWAYS -{ -} "no"

+############### Optimization options ################################################### + + + # This can improve the performance on many cores, but may be dangerous # (it seems to work in simple testcases, but morde complicated settings may fail) # If you encounter strange behavior, switch this OFF

File [modified]: schedule.ccl Delta lines: +117 -75 =================================================================== --- trunk/schedule.ccl 2012-06-05 20:29:03 UTC (rev 342) +++ trunk/schedule.ccl 2012-06-05 20:30:33 UTC (rev 343) @@ -67,6 +67,7 @@ STORAGE:evolve_Y_e STORAGE:evolve_temper STORAGE:GRHydro_reflevel +STORAGE:InLastMoLPostStep STORAGE:densrhs STORAGE:taurhs STORAGE:srhs @@ -1000,45 +1001,45 @@

if (!sync_conserved_only) { -schedule GRHydro_Boundaries IN HydroBase_Select_Boundaries AS GRHydro_Bound -{ - LANG: Fortran - OPTIONS: LEVEL - SYNC: dens - SYNC: tau - SYNC: scon - SYNC: HydroBase::w_lorentz - SYNC: HydroBase::rho - SYNC: HydroBase::press - SYNC: HydroBase::eps - SYNC: HydroBase::vel - SYNC: Bcons - SYNC: HydroBase::Bvec - SYNC: psidc - SYNC: GRHydro_cons_tracers - SYNC: GRHydro_tracers - SYNC: hydrobase::temperature - SYNC: hydrobase::entropy - SYNC: hydrobase::Y_e - SYNC: Y_e_con - SYNC: lvel - SYNC: lBvec -} "Select GRHydro boundary conditions" + schedule GRHydro_Boundaries IN HydroBase_Select_Boundaries AS GRHydro_Bound + { + LANG: Fortran + OPTIONS: LEVEL + SYNC: dens + SYNC: tau + SYNC: scon + SYNC: HydroBase::w_lorentz + SYNC: HydroBase::rho + SYNC: HydroBase::press + SYNC: HydroBase::eps + SYNC: HydroBase::vel + SYNC: Bcons + SYNC: HydroBase::Bvec + SYNC: psidc + SYNC: GRHydro_cons_tracers + SYNC: GRHydro_tracers + SYNC: hydrobase::temperature + SYNC: hydrobase::entropy + SYNC: hydrobase::Y_e + SYNC: Y_e_con + SYNC: lvel + SYNC: lBvec + } "Select GRHydro boundary conditions" } else { -schedule GRHydro_Boundaries IN HydroBase_Select_Boundaries AS GRHydro_Bound -{ - LANG: Fortran - OPTIONS: LEVEL - SYNC: dens - SYNC: tau - SYNC: scon - SYNC: Bcons - SYNC: psidc - SYNC: GRHydro_cons_tracers - SYNC: Y_e_con -} "Select GRHydro boundary conditions" + schedule GRHydro_Boundaries IN HydroBase_Select_Boundaries AS GRHydro_Bound + { + LANG: Fortran + OPTIONS: LEVEL + SYNC: dens + SYNC: tau + SYNC: scon + SYNC: Bcons + SYNC: psidc + SYNC: GRHydro_cons_tracers + SYNC: Y_e_con + } "Select GRHydro boundary conditions"

# after a regrid Cactus relies on the boundary thorns to fill in the outer # and symmetry boundaries (we need valid data for the initial guess in Con2Prim) @@ -1049,36 +1050,63 @@ { } "Apply boundary conditions to primitives"

- # This is necessary, otherwise we are inconsistent in atmosphere treatment of conserved and primitive variables - if(!disable_sync_and_tmunu_after_MoL) { - schedule group GRHydro_PrimitiveBoundaries IN CCTK_EVOL AFTER (MoL_Evolution,GRHydro_AtmosphereReset,sync_GRHydro_C2P_failed) BEFORE HydroBase_Boundaries - { - } "Apply boundary conditions to primitives" + # This is necessary to provide initial guesses for Con2Prim after atmo reset (which happens only at last MoL post-step)! + schedule group GRHydro_PrimitiveBoundaries IN HydroBase_PostStep BEFORE HydroBase_Boundaries IF GRHydro::InLastMoLPostStep + { + } "Apply boundary conditions to primitives" + + + if(CCTK_IsImplementationActive("Coordinates")) { + + # With MP, we only need to sync primitives in LOCAL tensor base!! + schedule GRHydro_SelectPrimitiveBoundaries IN GRHydro_PrimitiveBoundaries + { + LANG: Fortran + OPTIONS: LEVEL + # do a sync to be very sure we have proper initial guesses everywhere +# SYNC: HydroBase::w_lorentz # not used as initial guess and hence not needed! +# SYNC: HydroBase::press # not used as initial guess and hence not needed! +# SYNC: HydroBase::vel +# SYNC: HydroBase::Bvec +# SYNC: hydrobase::entropy # not used as initial guess and hence not needed! +# SYNC: hydrobase::Y_e # not used as initial guess and hence not needed! +# SYNC: GRHydro_tracers # not used as initial guess and hence not needed! + SYNC: HydroBase::rho + SYNC: HydroBase::eps + SYNC: hydrobase::temperature + SYNC: lvel + SYNC: lBvec + } "Select primitive variables for boudary conditions" } + else + { + # Without MP, we only need to sync primitives in GLOBAL tensor base!! + schedule GRHydro_SelectPrimitiveBoundaries IN GRHydro_PrimitiveBoundaries + { + LANG: Fortran + OPTIONS: LEVEL + # do a sync to be very sure we have proper initial guesses everywhere +# SYNC: HydroBase::w_lorentz # not used as initial guess and hence not needed! +# SYNC: HydroBase::press # not used as initial guess and hence not needed! +# SYNC: hydrobase::entropy # not used as initial guess and hence not needed! +# SYNC: hydrobase::Y_e # not used as initial guess and hence not needed! +# SYNC: lvel +# SYNC: lBvec +# SYNC: GRHydro_tracers # not used as initial guess and hence not needed! + SYNC: HydroBase::rho + SYNC: HydroBase::eps + SYNC: HydroBase::vel + SYNC: HydroBase::Bvec + SYNC: hydrobase::temperature + } "Select primitive variables for boudary conditions"

- schedule GRHydro_SelectPrimitiveBoundaries IN GRHydro_PrimitiveBoundaries - { - LANG: Fortran - OPTIONS: LEVEL - # do a sync to be very sure we have proper initial guesses everywhere - SYNC: HydroBase::w_lorentz - SYNC: HydroBase::rho - SYNC: HydroBase::press - SYNC: HydroBase::eps - SYNC: HydroBase::vel - SYNC: HydroBase::Bvec - SYNC: hydrobase::temperature - SYNC: hydrobase::entropy - SYNC: hydrobase::Y_e - SYNC: lvel - SYNC: lBvec - SYNC: GRHydro_tracers - } "Select primitive variables for boudary conditions" + } + schedule group ApplyBCs AS GRHydro_ApplyPrimitiveBCs in GRHydro_PrimitiveBoundaries AFTER GRHydro_SelectPrimitiveBoundaries { } "Apply boundary conditions to primitive variables"

-} +} # sync_conserved_only

############################################################ ### Compute first differences of rho for mesh refinement ### @@ -1100,37 +1128,51 @@ # } "Compute relative differences in rho"

################################################################ +### Some actions are only required in the very last MoL sub- ### +### step but cannot be moved into PseudoEvolution since they ### +### affect other routines in PostStep. A prime example is ### +### AmosphereReset which forms an operator split RHS. ### +################################################################ + +schedule GRHydro_SetLastMoLPostStep IN MoL_PostStep +{ + LANG: C + OPTIONS: level +} "Set grid scalar InLastMoLPostStep if this is the last MoL PostStep call" +# there used to be a comment along the lines "You did not see this abuse of +# MoL. Nothing to see here. Move along." Consider such a comment to be present +# again. +schedule GRHydro_ClearLastMoLPostStep IN MoL_Step AFTER MoL_PostStep +{ + LANG: C + OPTIONS: level +} "Reset InLastMoLPostStep to zero" +schedule GRHydro_ClearLastMoLPostStep IN CCTK_WRAGH +{ + LANG: C + OPTIONS: global-early +} "Initialize InLastMoLPostStep to zero" + +################################################################ ### If a point has been marked as requiring resetting to the ### ### atmosphere, this is where we do it. ### ### ### -### This is logically still part of the MoL evolution step ### -### since it is part of evolving the primitives. The final ### -### SetTmunu should be before PseudoEvolution so that eg. ### -### constraints use the correct stress energy tensor. ### +### This is executed in the last MoL_PosStep ### ################################################################

if (CCTK_Equals(Bvec_evolution_method,"GRHydro")) { - schedule GRHydro_AtmosphereResetM AT EVOL AFTER MoL_Evolution + schedule GRHydro_AtmosphereResetM IN MoL_PostStep AFTER ADMBase_SetADMVars AFTER GRHydro_SetLastMoLPostStep BEFORE HydroBase_PostStep IF GRHydro::InLastMoLPostStep { LANG: Fortran } "Reset the atmosphere - MHD version" } else { - schedule GRHydro_AtmosphereReset AT EVOL AFTER MoL_Evolution + schedule GRHydro_AtmosphereReset IN MoL_PostStep AFTER ADMBase_SetADMVars BEFORE HydroBase_PostStep IF GRHydro::InLastMoLPostStep { LANG: Fortran } "Reset the atmosphere" }

-if(!disable_sync_and_tmunu_after_MoL) { - schedule group HydroBase_Boundaries AT EVOL AFTER MoL_Evolution AFTER (GRHydro_AtmosphereReset GRHydro_AtmosphereResetM) - { - } "HydroBase Boundary conditions group" - - schedule GROUP SetTmunu AT EVOL AFTER HydroBase_Boundaries BEFORE MoL_PseudoEvolution - { - } "Calculate the stress-energy tensor" -}

} # end of if (CCTK_EQUALS(hydrobase::evolution_method, "GRHydro"))