Hi all
I'm currently trying to run a BHNS simulation with initial data generated by Kadath elliptic solver (Fuka branch).
After initial data import, I'm getting an error from THC:
*WARNING level 0 from host nid00462 process 0 while executing schedule bin HydroBase_Select_Boundaries, routine THC_Core::THC_SelectBC in thorn THC_Core, file /lustre/tetyda/home/topolski/Cactus/arrangements/THCCore/THC_Core/src/thc_select_bc.cc:76: -> Failed to select the BCs*
The simulation uses the following thorns NOT included with the official release of ET:
HRSCCore Boost KadathImporter THCBase/CPPUtils, THCBase/FDCore ,THCBase/HRSCCore THCExtra/EOS_Barotropic, THCExtra/EOS_Thermal, THCExtra/EOS_Thermal_Idealgas
Here's the parameter file. If I can provide more info, please let me know (also, if to upload it to a somewhat more convenient place).
Best regards Konrad
# The following is a modification of the ETK Gallery par file to utilize # Kadath initial data for a binary system of BH-NS
################################################################################ # Active thorns ################################################################################
ActiveThorns = " ADMBase ML_ADMConstraints ADMCoupling ADMMacros AEILocalInterp AHFinderDirect Boundary Carpet CarpetIOASCII CarpetIOBasic CarpetIOHDF5 CarpetIOScalar CarpetInterp CarpetInterp2 CarpetLib CarpetReduce CarpetRegrid2 CarpetTracker CartGrid3D CoordBase Coordinates CoordinatesSymmetry GlobalDerivative hwloc IOUtil InitBase Interpolate2 QuasiLocalMeasures KadathImporter LocalInterp LoopControl MoL NaNChecker PunctureTracker Slab SpaceMask SphericalSurface StaticConformal SummationByParts SymBase SystemStatistics SystemTopology TerminationTrigger TensorTypes Time TmunuBase Vectors ML_BSSN ML_BSSN_Helper NewRad GenericFD WeylScal4 Multipole ADMDerivatives CarpetSlab CoordGauge Dissipation EOS_Barotropic EOS_Thermal EOS_Thermal_Idealgas HydroBase HRSCCore ML_ADMConstraints THC_Core TimerReport BLAS LAPACK GSL Boost CPPUtils FDCore HDF5 MPI Timers CycleClock TGRtensor zlib Fortran VolumeIntegrals_GRMHD CarpetMask ReflectionSymmetry # currently turned off #GRHydro #PizzaBase #PizzaIDBase #PizzaNumUtils #NSTracker #Hydro_Analysis "
################################################################################ # Grid structure ################################################################################
#------------------------------------------------------------------------------ # Diagnostic parameters: #------------------------------------------------------------------------------ Carpet::output_timers_every = 0 Carpet::storage_verbose = "no" Carpet::verbose = "no" Carpet::veryverbose = "no" Carpet::grid_structure_filename = "carpet-grid-structure" Carpet::grid_coordinates_filename = "carpet-grid-coordinates"
CarpetLib::output_bboxes = "no"
CarpetMask::verbose = "no" CarpetReduce::verbose = "no" CarpetRegrid2::verbose = "no" CarpetRegrid2::veryverbose = "no" CarpetTracker::verbose = "no"
TimerReport::out_every = 4096 TimerReport::out_filename = "TimerReport" TimerReport::output_all_timers = "yes" TimerReport::output_all_timers_together = "yes" TimerReport::output_all_timers_readable = "yes" TimerReport::n_top_timers = 40
QuasiLocalMeasures::verbose = "no" SphericalSurface::verbose = "no"
Carpet::domain_from_multipatch = yes CartGrid3D::type = "multipatch" CartGrid3D::set_coordinate_ranges_on = "all maps" Coordinates::coordinate_system = "Thornburg04" Coordinates::h_cartesian = 1.5574825174818182 Coordinates::h_radial = 1.5574825174818182
Coordinates::sphere_inner_radius = 51.3969230769 Coordinates::sphere_outer_radius = 2194.15912088 Coordinates::n_angular = 28
Driver::ghost_size = 5 Coordinates::patch_boundary_size = 5 Coordinates::additional_overlap_size = 3 Coordinates::outer_boundary_size = 5
CoordinatesSymmetry::reflection_z = yes CoordinatesSymmetry::stagger = no Coordinates::symmetry = "+z bitant" Coordinates::additional_symmetry_size = 1 Coordinates::verbose = no
Time::timestep_method = "given" Time::timestep = 0.3504335664334091 Carpet::time_refinement_factors = "[1,1,2,4,8,16,32,64,128,256]"
# General Carpet parameters: Carpet::enable_all_storage = "no" Carpet::use_buffer_zones = "yes" Carpet::schedule_barriers = "no"
Carpet::poison_new_timelevels = "yes" Carpet::check_for_poison = "no"
Carpet::init_3_timelevels = "no" Carpet::init_fill_timelevels = "yes"
CarpetLib::poison_new_memory = "yes" CarpetLib::poison_value = 114 CarpetLib::check_bboxes = "no" CarpetLib::interleave_communications = "yes" CarpetLib::combine_sends = "yes"
CarpetInterp::tree_search = "yes" CarpetInterp::check_tree_search = "no"
CarpetRegrid2::freeze_unaligned_levels = "yes" CarpetRegrid2::freeze_unaligned_parent_levels = "yes" CarpetRegrid2::ensure_proper_nesting = "yes" CarpetRegrid2::snap_to_coarse = "yes" CarpetRegrid2::symmetry_rotating180 = "no"
############################################################################### # SpaceMask - only needed for GRHydro ###############################################################################
#SpaceMask::use_mask = "yes"
################################################################################ # Mesh refinement ################################################################################
Carpet::max_refinement_levels = 9 CarpetRegrid2::num_centres = 2 CarpetRegrid2::num_levels_1 = 7 CarpetRegrid2::position_x_1 = +17.37737 CarpetRegrid2::radius_1 = [0,21.267692,10.633846,5.316923,2.658462,1.329231,0.664615] CarpetRegrid2::num_levels_2 = 7 CarpetRegrid2::position_x_2 = -13.76685 CarpetRegrid2::radius_2 = [0,17.132308,8.566154,4.283077,2.141538,1.070769,0.535385]
Carpet::prolongation_order_space = 5 Carpet::prolongation_order_time = 2
CarpetRegrid2::regrid_every = 128
################################################################################ # Initial data ################################################################################
HydroBase::timelevels = 3 HydroBase::prolongation_type ="None" ADMBase::metric_type = "physical"
ADMBase::initial_data = "Kadath" ADMBase::initial_lapse = "Kadath" ADMBase::initial_shift = "Kadath" ADMBase::initial_dtlapse = "Kadath" ADMBase::initial_dtshift = "Kadath" HydroBase::initial_hydro = "Kadath"
# Kadath BNS file KadathImporter::type = "BHNS" #PizzaIDBase::eos_file = "gam2.polytrope" # Geometric unit system for initial data, specified by length unit. # use CACTUS units #PizzaBase::length_unit = 1476.7161818921163 KadathImporter::filename = "/home/topolski/Cactus/BHNS_par/BHNS.info" KadathImporter::puncture_lapse = yes
InitBase::initial_data_setup_method = "init_single_level"
# ============================================================================ # Templated hydrodynamics code # ============================================================================= HydroBase::evolution_method = "THCode" HydroBase::initial_entropy = "THCode"
THC_Core::physics = "GRHD" THC_Core::eos_type = "ideal" THC_Core::bc_type = "none" THC_Core::atmo_rho = 1e-13
HRSCCore::scheme = "FV" HRSCCore::reconstruction = "MP5" HRSCCore::riemann_solver = "HLLE"
THC_Core::c2a_BH_alp = 0.15 THC_Core::c2a_rho_strict = 2.0e-5 THC_Core::c2a_set_to_nan_on_failure = "no" THC_Core::c2a_fix_conservatives = "yes" THC_Core::c2a_kill_on_failure = "no"
EOS_Thermal::evol_eos_name = "IdealGas" EOS_Thermal_IdealGas::index_n = 1 EOS_Thermal_IdealGas::eps_min = 1e-8
################################################################################ # Evolution and boundary ################################################################################
GenericFD::jacobian_group = "Coordinates::jacobian" GenericFD::jacobian_derivative_group = "Coordinates::jacobian2" GenericFD::jacobian_identity_map = 0
ADMBase::evolution_method = "ML_BSSN" ADMBase::lapse_evolution_method = "ML_BSSN" ADMBase::shift_evolution_method = "ML_BSSN" ADMBase::dtlapse_evolution_method = "ML_BSSN" ADMBase::dtshift_evolution_method = "ML_BSSN"
ADMBase::lapse_timelevels = 3 ADMBase::shift_timelevels = 3 ADMBase::metric_timelevels = 3
TmunuBase::prolongation_type = "none" TmunuBase::stress_energy_storage = "yes" TmunuBase::stress_energy_at_RHS = "yes" TmunuBase::support_old_CalcTmunu_mechanism = "no"
ML_BSSN::timelevels = 3 ML_BSSN::fdOrder = 8 THC_Core::fd_order = 4
# 1+log slicing requires harmonicN = 1 and harmonicF = 2.0 ML_BSSN::harmonicN = 1 ML_BSSN::harmonicF = 2.0
ML_BSSN::ShiftGammaCoeff = 0.75 ML_BSSN::evolveA = 1 ML_BSSN::evolveB = 1 ML_BSSN::AlphaDriver = 0.0 ML_BSSN::BetaDriver = 0.7 # ~ 1/M ML_BSSN::advectLapse = 1 ML_BSSN::advectShift = 1
ML_BSSN::MinimumLapse = 1.0e-8
# conformalaMethod = 1 for W, 0 for phi ML_BSSN::conformalMethod = 1
# We apply dissipation using GlobalDerivaitive so disable it here ML_BSSN::epsDiss = 0.0
ML_BSSN::dt_lapse_shift_method = "noLapseShiftAdvection"
ML_BSSN::initial_boundary_condition = "extrapolate-gammas" ML_BSSN::rhs_boundary_condition = "NewRad" Boundary::radpower = 2
ML_BSSN::ML_log_confac_bound = "none" ML_BSSN::ML_metric_bound = "none" ML_BSSN::ML_Gamma_bound = "none" ML_BSSN::ML_trace_curv_bound = "none" ML_BSSN::ML_curv_bound = "none" ML_BSSN::ML_lapse_bound = "none" ML_BSSN::ML_dtlapse_bound = "none" ML_BSSN::ML_shift_bound = "none" ML_BSSN::ML_dtshift_bound = "none"
Dissipation::order = 5 Dissipation::epsdis = 0.1 Dissipation::vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_metric ML_BSSN::ML_curv ML_BSSN::ML_trace_curv ML_BSSN::ML_Gamma ML_BSSN::ML_lapse ML_BSSN::ML_shift "
################################################################################ # BH and NS tracking (currently using VolumeIntegrals_GRMHD for tracking the # neutron star, will switch to NSTracker some time in the future) # BH is at x>0, NS at x<0 ################################################################################
CarpetTracker::surface [0] = 0 CarpetTracker::surface [1] = 1 #PunctureTracker::track [0] = yes #PunctureTracker::initial_x [0] = -13.76685 #PunctureTracker::which_surface_to_store_info[0] = 0 PunctureTracker::track [0] = yes PunctureTracker::initial_x [0] = +17.37737 PunctureTracker::which_surface_to_store_info[0] = 1
VolumeIntegrals_GRMHD::verbose = 1 VolumeIntegrals_GRMHD::VolIntegral_out_every = 32 VolumeIntegrals_GRMHD::NumIntegrals = 4 # Integrate the entire volume with an integrand of 1. (used for testing/validation purposes only). VolumeIntegrals_GRMHD::Integration_quantity_keyword[1] = "one" # To compute the center of mass in an integration volume originally centered at (x,y,z) = (-13.76685,0,0) #with a coordinate radius of 13.5. Also use the center of mass integral result to set the ZEROTH AMR center. VolumeIntegrals_GRMHD::Integration_quantity_keyword[2] = "centerofmass" VolumeIntegrals_GRMHD::volintegral_sphere__center_x_initial [2] = -13.76685 VolumeIntegrals_GRMHD::volintegral_inside_sphere__radius [2] = 9.10722 VolumeIntegrals_GRMHD::amr_centre__tracks__volintegral_inside_sphere[2] = 0 # Same as above, except use the integrand=1 (for validation purposes, to ensure the integration volume is approximately 4/3*pi*13.5^3). VolumeIntegrals_GRMHD::Integration_quantity_keyword[3] = "one" VolumeIntegrals_GRMHD::volintegral_sphere__center_x_initial [3] = -13.76685 VolumeIntegrals_GRMHD::volintegral_inside_sphere__radius [3] = 9.10722 # Perform rest-mass integrals over entire volume. VolumeIntegrals_GRMHD::Integration_quantity_keyword[4] = "restmass" VolumeIntegrals_GRMHD::enable_file_output = 1 VolumeIntegrals_GRMHD::outVolIntegral_dir = "volume_integration" VolumeIntegrals_GRMHD::CoM_integrand_GAMMA_SPEED_LIMIT = 1e4
######################## # Setting up NS Tracker and Hydro_Analysis #########################
#Hydro_Analysis::Hydro_Analysis_comp_rho_max = "true" #Hydro_Analysis::Hydro_Analysis_rho_max_loc_only_positive_x = "true" #Hydro_Analysis::Hydro_Analysis_comp_rho_max_origin_distance = "yes" #Hydro_Analysis::Hydro_Analysis_average_multiple_maxima_locations = "yes" #Hydro_Analysis::Hydro_Analysis_interpolator_name = "Lagrange polynomial interpolation (tensor product)"
#NSTracker::NSTracker_SF_Name = "Lefthand NS" #NSTracker::NSTracker_SF_Index = 0 #NSTracker::NSTracker_max_distance = 3 #NSTracker::NSTracker_tracked_location = "Hydro_Analysis::Hydro_Analysis_rho_max_loc" # location of maximum #position_x[NSTracker_SF_Index] = Hydro_Analysis_rho_max_loc #position_x[NSTracker_SF_Index_Opposite] = - Hydro_Analysis_rho_max_loc #NSTracker::NSTracker_tracked_location = "Hydro_Analysis::Hydro_Analysis_rho_core_center_volume_weighted" # center of mass #NSTracker::NSTracker_stop_time = -1
################################################################################ # Spatial finite differencing ################################################################################
SummationByParts::order = 8
# Drop order instead of using upwinded stencils, only for advection derivatives SummationByParts::sbp_upwind_deriv = no
SummationByParts::sbp_1st_deriv = yes SummationByParts::sbp_2nd_deriv = no SummationByParts::onesided_interpatch_boundaries = no SummationByParts::onesided_outer_boundaries = yes SummationByParts::use_dissipation = no GlobalDerivative::use_dissipation = yes SummationByParts::scale_with_h = yes SummationByParts::dissipation_type = "Kreiss-Oliger" SummationByParts::epsdis = 0.15
# Because dt/dx is half that of the other levels we also need half the dissipation GlobalDerivative::epsdis_for_level [0] = 0.075
# Variables for dissipation SummationByParts::vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_metric ML_BSSN::ML_trace_curv ML_BSSN::ML_curv ML_BSSN::ML_Gamma ML_BSSN::ML_lapse ML_BSSN::ML_shift ML_BSSN::ML_dtlapse ML_BSSN::ML_dtshift "
################################################################################ # Time integration ################################################################################
MoL::ODE_Method = "rk4" MoL::MoL_Intermediate_Steps = 4 MoL::MoL_Num_Scratch_Levels = 1
################################################################################ # Interpolation ################################################################################
# Use 5-th order interpatch interpolation on the Llama grid Interpolate::interpolator_order = 5
################################################################################ # Psi4 computation ################################################################################
WeylScal4::fdOrder = 8 WeylScal4::calc_scalars = "psis" WeylScal4::calc_invariants = "always"
################################################################################ # Psi4 mode decomposition ################################################################################
# Radii are chosen to be evenly spaced in 1/r as that is the variable # extrapolation is performed in Multipole::nradii = 7 Multipole::radius[0] = 100 Multipole::radius[1] = 115 Multipole::radius[2] = 136 Multipole::radius[3] = 167 Multipole::radius[4] = 214 Multipole::radius[5] = 300 Multipole::radius[6] = 500 Multipole::ntheta = 120 Multipole::nphi = 240 Multipole::variables = "WeylScal4::Psi4r{sw=-2 cmplx='WeylScal4::Psi4i' name='psi4'}" Multipole::out_every = 256 Multipole::l_max = 8 Multipole::output_hdf5 = yes
# Disable ASCII output to avoid creating a large number of files Multipole::output_ascii = no
################################################################################ # Apparent Horizons ################################################################################
AHFinderDirect::N_horizons = 2 AHFinderDirect::find_every = 256 AHFinderDirect::output_h_every = 0 AHFinderDirect::max_Newton_iterations__initial = 50 AHFinderDirect::max_Newton_iterations__subsequent = 50 AHFinderDirect::max_allowable_Theta_growth_iterations = 10 AHFinderDirect::max_allowable_Theta_nonshrink_iterations = 10 AHFinderDirect::geometry_interpolator_name = "Lagrange polynomial interpolation" AHFinderDirect::geometry_interpolator_pars = "order=4" AHFinderDirect::surface_interpolator_name = "Lagrange polynomial interpolation" AHFinderDirect::surface_interpolator_pars = "order=4" AHFinderDirect::verbose_level = "physics details" AHFinderDirect::move_origins = yes
AHFinderDirect::origin_x [1] = +17.37737 AHFinderDirect::initial_guess__coord_sphere__x_center[1] = +17.37737 AHFinderDirect::initial_guess__coord_sphere__radius [1] = 0.73 AHFinderDirect::which_surface_to_store_info [1] = 2 AHFinderDirect::set_mask_for_individual_horizon [1] = no AHFinderDirect::reset_horizon_after_not_finding [1] = no AHFinderDirect::track_origin_from_grid_scalar [1] = yes AHFinderDirect::track_origin_source_x [1] = "PunctureTracker::pt_loc_x[0]" AHFinderDirect::track_origin_source_y [1] = "PunctureTracker::pt_loc_y[0]" AHFinderDirect::track_origin_source_z [1] = "PunctureTracker::pt_loc_z[0]" AHFinderDirect::max_allowable_horizon_radius [1] = 3
#AHFinderDirect::origin_x [2] = -5.53846153846 #AHFinderDirect::initial_guess__coord_sphere__x_center[2] = -5.53846153846 #AHFinderDirect::initial_guess__coord_sphere__radius [2] = 0.535384615385 #AHFinderDirect::which_surface_to_store_info [2] = 3 #AHFinderDirect::set_mask_for_individual_horizon [2] = no #AHFinderDirect::reset_horizon_after_not_finding [2] = no #AHFinderDirect::track_origin_from_grid_scalar [2] = yes #AHFinderDirect::track_origin_source_x [2] = "PunctureTracker::pt_loc_x[1]" #AHFinderDirect::track_origin_source_y [2] = "PunctureTracker::pt_loc_y[1]" #AHFinderDirect::track_origin_source_z [2] = "PunctureTracker::pt_loc_z[1]" #AHFinderDirect::max_allowable_horizon_radius [2] = 3
AHFinderDirect::origin_x [2] = 0 AHFinderDirect::find_after_individual [2] = 371902 AHFinderDirect::initial_guess__coord_sphere__x_center[2] = 0 AHFinderDirect::initial_guess__coord_sphere__radius [2] = 1.67 AHFinderDirect::which_surface_to_store_info [2] = 4 AHFinderDirect::set_mask_for_individual_horizon [2] = no AHFinderDirect::max_allowable_horizon_radius [2] = 6
################################################################################ # Spherical surfaces ################################################################################
SphericalSurface::nsurfaces = 4 SphericalSurface::maxntheta = 66 SphericalSurface::maxnphi = 124
# Surfaces 0 and 1 are used by PunctureTracker
# Horizon 1 SphericalSurface::ntheta [2] = 41 SphericalSurface::nphi [2] = 80 SphericalSurface::nghoststheta [2] = 2 SphericalSurface::nghostsphi [2] = 2
# Horizon 2 #SphericalSurface::ntheta [3] = 41 #SphericalSurface::nphi [3] = 80 #SphericalSurface::nghoststheta [3] = 2 #SphericalSurface::nghostsphi [3] = 2
# Common horizon SphericalSurface::ntheta [3] = 41 SphericalSurface::nphi [3] = 80 SphericalSurface::nghoststheta [3] = 2 SphericalSurface::nghostsphi [3] = 2
################################################################################ # Isolated Horizons ################################################################################
QuasiLocalMeasures::veryverbose = no QuasiLocalMeasures::interpolator = "Lagrange polynomial interpolation" QuasiLocalMeasures::interpolator_options = "order=4" QuasiLocalMeasures::spatial_order = 4 QuasiLocalMeasures::num_surfaces = 3 QuasiLocalMeasures::surface_index [0] = 2 QuasiLocalMeasures::surface_index [1] = 3 #QuasiLocalMeasures::surface_index [2] = 4 QuasiLocalMeasures::output_vtk_every = 0
################################################################################ # Correctness checking ################################################################################
NaNChecker::check_every = 256 NanChecker::check_after = 0 NaNChecker::report_max = 10 NaNChecker::verbose = "all" NaNChecker::action_if_found = terminate NaNChecker::out_NaNmask = yes NaNChecker::check_vars = " ML_BSSN::ML_log_confac HydroBase::w_lorentz ADMBase::curv ADMBase::metric ADMBase::lapse ADMBase::shift HydroBase::rho HydroBase::eps HydroBase::press HydroBase::vel "
################################################################################ # Timers ################################################################################
Carpet::output_timer_tree_every = 1024 Carpet::output_initialise_timer_tree = yes
################################################################################ # Output ################################################################################
IO::out_dir = "data" CarpetIOScalar::all_reductions_in_one_file = "no" CarpetIOScalar::one_file_per_group = "no" CarpetIOASCII::one_file_per_group = "no" CarpetIOHDF5::one_file_per_group = "no"
IOBasic::outInfo_every = 1 IOBasic::outInfo_reductions = "minimum maximum" IOBasic::outInfo_vars = " ML_BSSN::ML_Ham ML_BSSN::ML_log_confac Carpet::physical_time_per_hour SystemStatistics::maxrss_mb SystemStatistics::swap_used_mb Carpet::physical_time_per_hour HydroBase::rho #GRHydro::dens{reductions = 'sum maximum'} HydroBase::w_lorentz "
IOScalar::outScalar_every = 256 IOScalar::outScalar_reductions = "minimum maximum average" IOScalar::outScalar_vars = " SystemStatistics::process_memory_mb ML_BSSN::ML_Ham{ reductions = 'maximum' } "
IOASCII::out0D_every = 256 IOASCII::out0D_vars = " Carpet::physical_time_per_hour Carpet::timing PunctureTracker::pt_loc QuasiLocalMeasures::qlm_scalars{out_every = 256} "
IOASCII::out1D_every = 0 IOASCII::out1D_d = no IOASCII::out1D_x = yes IOASCII::out1D_y = no IOASCII::out1D_z = yes IOASCII::out1D_vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_trace_curv WeylScal4::Psi4r "
IOASCII::out2D_every = 0 IOASCII::out2D_vars = " "
IOHDF5::out_every = 0 IOHDF5::out_vars = " Grid::Coordinates{out_every=1000000000 refinement_levels={0}} ML_BSSN::ML_log_confac WeylScal4::Psi4r WeylScal4::Psi4i WeylScal4::curvIr{refinement_levels={3 5}} WeylScal4::curvIi{refinement_levels={3 5}} WeylScal4::curvJr{refinement_levels={3 5}} WeylScal4::curvJi{refinement_levels={3 5}} "
IOHDF5::out2D_every = 64 IOHDF5::out2D_vars = " Grid::Coordinates{out_every=1000000000 refinement_levels={0}} ML_BSSN::ML_Ham ML_BSSN::alpha ML_BSSN::phi ML_BSSN::trK WeylScal4::Psi4r WeylScal4::Psi4i HydroBase::rho HydroBase::press HydroBase::w_lorentz "
################################################################################ # Checkpointing and recovery ################################################################################
#CarpetIOHDF5::checkpoint = yes IOHDF5::checkpoint = "yes" IO::checkpoint_ID = "yes" IO::recover = "autoprobe" IO::checkpoint_every_walltime_hours = 1.5 IO::checkpoint_keep=2 IO::checkpoint_on_terminate = "yes" IO::out_mode = "onefile" IO::checkpoint_dir = "../checkpoints" IO::recover_dir = "../checkpoints" IO::abort_on_io_errors = yes CarpetIOHDF5::open_one_input_file_at_a_time = yes CarpetIOHDF5::compression_level = 0
################################################################################ # Run termination ################################################################################
TerminationTrigger::max_walltime = 48 # Trigger termination 30 minutes before the walltime is reached TerminationTrigger::on_remaining_walltime = 30 TerminationTrigger::output_remtime_every_minutes = 30 TerminationTrigger::termination_from_file = yes TerminationTrigger::termination_file = "terminate.txt" TerminationTrigger::create_termination_file = yes
Cactus::terminate = time Cactus::cctk_final_time = 1900.0
Konrad
You need to look at the file and the line number given in the error message. Nearby there will be a call to a function that selects boundary conditions. This call will specify certain grid functions, boundary conditions, or boundary sizes. Some of these will be wrong, e.g. the grid functions might not exist or might not have storage, or they might not have enough boundary points, or the boundary condition might not exist (might be provided by a thorn that is not active) etc. It might be worthwhile to add a printf statement to the code that outputs the respective information. It could also be that the boundary conditions are selected multiple times, but are never actually applied; this would then be a scheduling error.
For further help it would probably be necessary to share the respective thorn.
-erik
On Fri, Aug 13, 2021 at 4:57 AM Konrad Topolski k.topolski2@student.uw.edu.pl wrote:
Hi all
I'm currently trying to run a BHNS simulation with initial data generated by Kadath elliptic solver (Fuka branch).
After initial data import, I'm getting an error from THC: WARNING level 0 from host nid00462 process 0 while executing schedule bin HydroBase_Select_Boundaries, routine THC_Core::THC_SelectBC in thorn THC_Core, file /lustre/tetyda/home/topolski/Cactus/arrangements/THCCore/THC_Core/src/thc_select_bc.cc:76: -> Failed to select the BCs
The simulation uses the following thorns NOT included with the official release of ET:
HRSCCore Boost KadathImporter THCBase/CPPUtils, THCBase/FDCore ,THCBase/HRSCCore THCExtra/EOS_Barotropic, THCExtra/EOS_Thermal, THCExtra/EOS_Thermal_Idealgas
Here's the parameter file. If I can provide more info, please let me know (also, if to upload it to a somewhat more convenient place).
Best regards Konrad
# The following is a modification of the ETK Gallery par file to utilize # Kadath initial data for a binary system of BH-NS
################################################################################ # Active thorns ################################################################################
ActiveThorns = " ADMBase ML_ADMConstraints ADMCoupling ADMMacros AEILocalInterp AHFinderDirect Boundary Carpet CarpetIOASCII CarpetIOBasic CarpetIOHDF5 CarpetIOScalar CarpetInterp CarpetInterp2 CarpetLib CarpetReduce CarpetRegrid2 CarpetTracker CartGrid3D CoordBase Coordinates CoordinatesSymmetry GlobalDerivative hwloc IOUtil InitBase Interpolate2 QuasiLocalMeasures KadathImporter LocalInterp LoopControl MoL NaNChecker PunctureTracker Slab SpaceMask SphericalSurface StaticConformal SummationByParts SymBase SystemStatistics SystemTopology TerminationTrigger TensorTypes Time TmunuBase Vectors ML_BSSN ML_BSSN_Helper NewRad GenericFD WeylScal4 Multipole ADMDerivatives CarpetSlab CoordGauge Dissipation EOS_Barotropic EOS_Thermal EOS_Thermal_Idealgas HydroBase HRSCCore ML_ADMConstraints THC_Core TimerReport BLAS LAPACK GSL Boost CPPUtils FDCore HDF5 MPI Timers CycleClock TGRtensor zlib Fortran VolumeIntegrals_GRMHD CarpetMask ReflectionSymmetry # currently turned off #GRHydro #PizzaBase #PizzaIDBase #PizzaNumUtils #NSTracker #Hydro_Analysis "
################################################################################ # Grid structure ################################################################################
#------------------------------------------------------------------------------ # Diagnostic parameters: #------------------------------------------------------------------------------ Carpet::output_timers_every = 0 Carpet::storage_verbose = "no" Carpet::verbose = "no" Carpet::veryverbose = "no" Carpet::grid_structure_filename = "carpet-grid-structure" Carpet::grid_coordinates_filename = "carpet-grid-coordinates"
CarpetLib::output_bboxes = "no"
CarpetMask::verbose = "no" CarpetReduce::verbose = "no" CarpetRegrid2::verbose = "no" CarpetRegrid2::veryverbose = "no" CarpetTracker::verbose = "no"
TimerReport::out_every = 4096 TimerReport::out_filename = "TimerReport" TimerReport::output_all_timers = "yes" TimerReport::output_all_timers_together = "yes" TimerReport::output_all_timers_readable = "yes" TimerReport::n_top_timers = 40
QuasiLocalMeasures::verbose = "no" SphericalSurface::verbose = "no"
Carpet::domain_from_multipatch = yes CartGrid3D::type = "multipatch" CartGrid3D::set_coordinate_ranges_on = "all maps" Coordinates::coordinate_system = "Thornburg04" Coordinates::h_cartesian = 1.5574825174818182 Coordinates::h_radial = 1.5574825174818182
Coordinates::sphere_inner_radius = 51.3969230769 Coordinates::sphere_outer_radius = 2194.15912088 Coordinates::n_angular = 28
Driver::ghost_size = 5 Coordinates::patch_boundary_size = 5 Coordinates::additional_overlap_size = 3 Coordinates::outer_boundary_size = 5
CoordinatesSymmetry::reflection_z = yes CoordinatesSymmetry::stagger = no Coordinates::symmetry = "+z bitant" Coordinates::additional_symmetry_size = 1 Coordinates::verbose = no
Time::timestep_method = "given" Time::timestep = 0.3504335664334091 Carpet::time_refinement_factors = "[1,1,2,4,8,16,32,64,128,256]"
# General Carpet parameters: Carpet::enable_all_storage = "no" Carpet::use_buffer_zones = "yes" Carpet::schedule_barriers = "no"
Carpet::poison_new_timelevels = "yes" Carpet::check_for_poison = "no"
Carpet::init_3_timelevels = "no" Carpet::init_fill_timelevels = "yes"
CarpetLib::poison_new_memory = "yes" CarpetLib::poison_value = 114 CarpetLib::check_bboxes = "no" CarpetLib::interleave_communications = "yes" CarpetLib::combine_sends = "yes"
CarpetInterp::tree_search = "yes" CarpetInterp::check_tree_search = "no"
CarpetRegrid2::freeze_unaligned_levels = "yes" CarpetRegrid2::freeze_unaligned_parent_levels = "yes" CarpetRegrid2::ensure_proper_nesting = "yes" CarpetRegrid2::snap_to_coarse = "yes" CarpetRegrid2::symmetry_rotating180 = "no"
############################################################################### # SpaceMask - only needed for GRHydro ###############################################################################
#SpaceMask::use_mask = "yes"
################################################################################ # Mesh refinement ################################################################################
Carpet::max_refinement_levels = 9 CarpetRegrid2::num_centres = 2 CarpetRegrid2::num_levels_1 = 7 CarpetRegrid2::position_x_1 = +17.37737 CarpetRegrid2::radius_1 = [0,21.267692,10.633846,5.316923,2.658462,1.329231,0.664615] CarpetRegrid2::num_levels_2 = 7 CarpetRegrid2::position_x_2 = -13.76685 CarpetRegrid2::radius_2 = [0,17.132308,8.566154,4.283077,2.141538,1.070769,0.535385]
Carpet::prolongation_order_space = 5 Carpet::prolongation_order_time = 2
CarpetRegrid2::regrid_every = 128
################################################################################ # Initial data ################################################################################
HydroBase::timelevels = 3 HydroBase::prolongation_type ="None" ADMBase::metric_type = "physical"
ADMBase::initial_data = "Kadath" ADMBase::initial_lapse = "Kadath" ADMBase::initial_shift = "Kadath" ADMBase::initial_dtlapse = "Kadath" ADMBase::initial_dtshift = "Kadath" HydroBase::initial_hydro = "Kadath"
# Kadath BNS file KadathImporter::type = "BHNS" #PizzaIDBase::eos_file = "gam2.polytrope" # Geometric unit system for initial data, specified by length unit. # use CACTUS units #PizzaBase::length_unit = 1476.7161818921163 KadathImporter::filename = "/home/topolski/Cactus/BHNS_par/BHNS.info" KadathImporter::puncture_lapse = yes
InitBase::initial_data_setup_method = "init_single_level"
# ============================================================================ # Templated hydrodynamics code # ============================================================================= HydroBase::evolution_method = "THCode" HydroBase::initial_entropy = "THCode"
THC_Core::physics = "GRHD" THC_Core::eos_type = "ideal" THC_Core::bc_type = "none" THC_Core::atmo_rho = 1e-13
HRSCCore::scheme = "FV" HRSCCore::reconstruction = "MP5" HRSCCore::riemann_solver = "HLLE"
THC_Core::c2a_BH_alp = 0.15 THC_Core::c2a_rho_strict = 2.0e-5 THC_Core::c2a_set_to_nan_on_failure = "no" THC_Core::c2a_fix_conservatives = "yes" THC_Core::c2a_kill_on_failure = "no"
EOS_Thermal::evol_eos_name = "IdealGas" EOS_Thermal_IdealGas::index_n = 1 EOS_Thermal_IdealGas::eps_min = 1e-8
################################################################################ # Evolution and boundary ################################################################################
GenericFD::jacobian_group = "Coordinates::jacobian" GenericFD::jacobian_derivative_group = "Coordinates::jacobian2" GenericFD::jacobian_identity_map = 0
ADMBase::evolution_method = "ML_BSSN" ADMBase::lapse_evolution_method = "ML_BSSN" ADMBase::shift_evolution_method = "ML_BSSN" ADMBase::dtlapse_evolution_method = "ML_BSSN" ADMBase::dtshift_evolution_method = "ML_BSSN"
ADMBase::lapse_timelevels = 3 ADMBase::shift_timelevels = 3 ADMBase::metric_timelevels = 3
TmunuBase::prolongation_type = "none" TmunuBase::stress_energy_storage = "yes" TmunuBase::stress_energy_at_RHS = "yes" TmunuBase::support_old_CalcTmunu_mechanism = "no"
ML_BSSN::timelevels = 3 ML_BSSN::fdOrder = 8 THC_Core::fd_order = 4
# 1+log slicing requires harmonicN = 1 and harmonicF = 2.0 ML_BSSN::harmonicN = 1 ML_BSSN::harmonicF = 2.0
ML_BSSN::ShiftGammaCoeff = 0.75 ML_BSSN::evolveA = 1 ML_BSSN::evolveB = 1 ML_BSSN::AlphaDriver = 0.0 ML_BSSN::BetaDriver = 0.7 # ~ 1/M ML_BSSN::advectLapse = 1 ML_BSSN::advectShift = 1
ML_BSSN::MinimumLapse = 1.0e-8
# conformalaMethod = 1 for W, 0 for phi ML_BSSN::conformalMethod = 1
# We apply dissipation using GlobalDerivaitive so disable it here ML_BSSN::epsDiss = 0.0
ML_BSSN::dt_lapse_shift_method = "noLapseShiftAdvection"
ML_BSSN::initial_boundary_condition = "extrapolate-gammas" ML_BSSN::rhs_boundary_condition = "NewRad" Boundary::radpower = 2
ML_BSSN::ML_log_confac_bound = "none" ML_BSSN::ML_metric_bound = "none" ML_BSSN::ML_Gamma_bound = "none" ML_BSSN::ML_trace_curv_bound = "none" ML_BSSN::ML_curv_bound = "none" ML_BSSN::ML_lapse_bound = "none" ML_BSSN::ML_dtlapse_bound = "none" ML_BSSN::ML_shift_bound = "none" ML_BSSN::ML_dtshift_bound = "none"
Dissipation::order = 5 Dissipation::epsdis = 0.1 Dissipation::vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_metric ML_BSSN::ML_curv ML_BSSN::ML_trace_curv ML_BSSN::ML_Gamma ML_BSSN::ML_lapse ML_BSSN::ML_shift "
################################################################################ # BH and NS tracking (currently using VolumeIntegrals_GRMHD for tracking the # neutron star, will switch to NSTracker some time in the future) # BH is at x>0, NS at x<0 ################################################################################
CarpetTracker::surface [0] = 0 CarpetTracker::surface [1] = 1 #PunctureTracker::track [0] = yes #PunctureTracker::initial_x [0] = -13.76685 #PunctureTracker::which_surface_to_store_info[0] = 0 PunctureTracker::track [0] = yes PunctureTracker::initial_x [0] = +17.37737 PunctureTracker::which_surface_to_store_info[0] = 1
VolumeIntegrals_GRMHD::verbose = 1 VolumeIntegrals_GRMHD::VolIntegral_out_every = 32 VolumeIntegrals_GRMHD::NumIntegrals = 4 # Integrate the entire volume with an integrand of 1. (used for testing/validation purposes only). VolumeIntegrals_GRMHD::Integration_quantity_keyword[1] = "one" # To compute the center of mass in an integration volume originally centered at (x,y,z) = (-13.76685,0,0) #with a coordinate radius of 13.5. Also use the center of mass integral result to set the ZEROTH AMR center. VolumeIntegrals_GRMHD::Integration_quantity_keyword[2] = "centerofmass" VolumeIntegrals_GRMHD::volintegral_sphere__center_x_initial [2] = -13.76685 VolumeIntegrals_GRMHD::volintegral_inside_sphere__radius [2] = 9.10722 VolumeIntegrals_GRMHD::amr_centre__tracks__volintegral_inside_sphere[2] = 0 # Same as above, except use the integrand=1 (for validation purposes, to ensure the integration volume is approximately 4/3*pi*13.5^3). VolumeIntegrals_GRMHD::Integration_quantity_keyword[3] = "one" VolumeIntegrals_GRMHD::volintegral_sphere__center_x_initial [3] = -13.76685 VolumeIntegrals_GRMHD::volintegral_inside_sphere__radius [3] = 9.10722 # Perform rest-mass integrals over entire volume. VolumeIntegrals_GRMHD::Integration_quantity_keyword[4] = "restmass" VolumeIntegrals_GRMHD::enable_file_output = 1 VolumeIntegrals_GRMHD::outVolIntegral_dir = "volume_integration" VolumeIntegrals_GRMHD::CoM_integrand_GAMMA_SPEED_LIMIT = 1e4
######################## # Setting up NS Tracker and Hydro_Analysis #########################
#Hydro_Analysis::Hydro_Analysis_comp_rho_max = "true" #Hydro_Analysis::Hydro_Analysis_rho_max_loc_only_positive_x = "true" #Hydro_Analysis::Hydro_Analysis_comp_rho_max_origin_distance = "yes" #Hydro_Analysis::Hydro_Analysis_average_multiple_maxima_locations = "yes" #Hydro_Analysis::Hydro_Analysis_interpolator_name = "Lagrange polynomial interpolation (tensor product)"
#NSTracker::NSTracker_SF_Name = "Lefthand NS" #NSTracker::NSTracker_SF_Index = 0 #NSTracker::NSTracker_max_distance = 3 #NSTracker::NSTracker_tracked_location = "Hydro_Analysis::Hydro_Analysis_rho_max_loc" # location of maximum #position_x[NSTracker_SF_Index] = Hydro_Analysis_rho_max_loc #position_x[NSTracker_SF_Index_Opposite] = - Hydro_Analysis_rho_max_loc #NSTracker::NSTracker_tracked_location = "Hydro_Analysis::Hydro_Analysis_rho_core_center_volume_weighted" # center of mass #NSTracker::NSTracker_stop_time = -1
################################################################################ # Spatial finite differencing ################################################################################
SummationByParts::order = 8
# Drop order instead of using upwinded stencils, only for advection derivatives SummationByParts::sbp_upwind_deriv = no
SummationByParts::sbp_1st_deriv = yes SummationByParts::sbp_2nd_deriv = no SummationByParts::onesided_interpatch_boundaries = no SummationByParts::onesided_outer_boundaries = yes SummationByParts::use_dissipation = no GlobalDerivative::use_dissipation = yes SummationByParts::scale_with_h = yes SummationByParts::dissipation_type = "Kreiss-Oliger" SummationByParts::epsdis = 0.15
# Because dt/dx is half that of the other levels we also need half the dissipation GlobalDerivative::epsdis_for_level [0] = 0.075
# Variables for dissipation SummationByParts::vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_metric ML_BSSN::ML_trace_curv ML_BSSN::ML_curv ML_BSSN::ML_Gamma ML_BSSN::ML_lapse ML_BSSN::ML_shift ML_BSSN::ML_dtlapse ML_BSSN::ML_dtshift "
################################################################################ # Time integration ################################################################################
MoL::ODE_Method = "rk4" MoL::MoL_Intermediate_Steps = 4 MoL::MoL_Num_Scratch_Levels = 1
################################################################################ # Interpolation ################################################################################
# Use 5-th order interpatch interpolation on the Llama grid Interpolate::interpolator_order = 5
################################################################################ # Psi4 computation ################################################################################
WeylScal4::fdOrder = 8 WeylScal4::calc_scalars = "psis" WeylScal4::calc_invariants = "always"
################################################################################ # Psi4 mode decomposition ################################################################################
# Radii are chosen to be evenly spaced in 1/r as that is the variable # extrapolation is performed in Multipole::nradii = 7 Multipole::radius[0] = 100 Multipole::radius[1] = 115 Multipole::radius[2] = 136 Multipole::radius[3] = 167 Multipole::radius[4] = 214 Multipole::radius[5] = 300 Multipole::radius[6] = 500 Multipole::ntheta = 120 Multipole::nphi = 240 Multipole::variables = "WeylScal4::Psi4r{sw=-2 cmplx='WeylScal4::Psi4i' name='psi4'}" Multipole::out_every = 256 Multipole::l_max = 8 Multipole::output_hdf5 = yes
# Disable ASCII output to avoid creating a large number of files Multipole::output_ascii = no
################################################################################ # Apparent Horizons ################################################################################
AHFinderDirect::N_horizons = 2 AHFinderDirect::find_every = 256 AHFinderDirect::output_h_every = 0 AHFinderDirect::max_Newton_iterations__initial = 50 AHFinderDirect::max_Newton_iterations__subsequent = 50 AHFinderDirect::max_allowable_Theta_growth_iterations = 10 AHFinderDirect::max_allowable_Theta_nonshrink_iterations = 10 AHFinderDirect::geometry_interpolator_name = "Lagrange polynomial interpolation" AHFinderDirect::geometry_interpolator_pars = "order=4" AHFinderDirect::surface_interpolator_name = "Lagrange polynomial interpolation" AHFinderDirect::surface_interpolator_pars = "order=4" AHFinderDirect::verbose_level = "physics details" AHFinderDirect::move_origins = yes
AHFinderDirect::origin_x [1] = +17.37737 AHFinderDirect::initial_guess__coord_sphere__x_center[1] = +17.37737 AHFinderDirect::initial_guess__coord_sphere__radius [1] = 0.73 AHFinderDirect::which_surface_to_store_info [1] = 2 AHFinderDirect::set_mask_for_individual_horizon [1] = no AHFinderDirect::reset_horizon_after_not_finding [1] = no AHFinderDirect::track_origin_from_grid_scalar [1] = yes AHFinderDirect::track_origin_source_x [1] = "PunctureTracker::pt_loc_x[0]" AHFinderDirect::track_origin_source_y [1] = "PunctureTracker::pt_loc_y[0]" AHFinderDirect::track_origin_source_z [1] = "PunctureTracker::pt_loc_z[0]" AHFinderDirect::max_allowable_horizon_radius [1] = 3
#AHFinderDirect::origin_x [2] = -5.53846153846 #AHFinderDirect::initial_guess__coord_sphere__x_center[2] = -5.53846153846 #AHFinderDirect::initial_guess__coord_sphere__radius [2] = 0.535384615385 #AHFinderDirect::which_surface_to_store_info [2] = 3 #AHFinderDirect::set_mask_for_individual_horizon [2] = no #AHFinderDirect::reset_horizon_after_not_finding [2] = no #AHFinderDirect::track_origin_from_grid_scalar [2] = yes #AHFinderDirect::track_origin_source_x [2] = "PunctureTracker::pt_loc_x[1]" #AHFinderDirect::track_origin_source_y [2] = "PunctureTracker::pt_loc_y[1]" #AHFinderDirect::track_origin_source_z [2] = "PunctureTracker::pt_loc_z[1]" #AHFinderDirect::max_allowable_horizon_radius [2] = 3
AHFinderDirect::origin_x [2] = 0 AHFinderDirect::find_after_individual [2] = 371902 AHFinderDirect::initial_guess__coord_sphere__x_center[2] = 0 AHFinderDirect::initial_guess__coord_sphere__radius [2] = 1.67 AHFinderDirect::which_surface_to_store_info [2] = 4 AHFinderDirect::set_mask_for_individual_horizon [2] = no AHFinderDirect::max_allowable_horizon_radius [2] = 6
################################################################################ # Spherical surfaces ################################################################################
SphericalSurface::nsurfaces = 4 SphericalSurface::maxntheta = 66 SphericalSurface::maxnphi = 124
# Surfaces 0 and 1 are used by PunctureTracker
# Horizon 1 SphericalSurface::ntheta [2] = 41 SphericalSurface::nphi [2] = 80 SphericalSurface::nghoststheta [2] = 2 SphericalSurface::nghostsphi [2] = 2
# Horizon 2 #SphericalSurface::ntheta [3] = 41 #SphericalSurface::nphi [3] = 80 #SphericalSurface::nghoststheta [3] = 2 #SphericalSurface::nghostsphi [3] = 2
# Common horizon SphericalSurface::ntheta [3] = 41 SphericalSurface::nphi [3] = 80 SphericalSurface::nghoststheta [3] = 2 SphericalSurface::nghostsphi [3] = 2
################################################################################ # Isolated Horizons ################################################################################
QuasiLocalMeasures::veryverbose = no QuasiLocalMeasures::interpolator = "Lagrange polynomial interpolation" QuasiLocalMeasures::interpolator_options = "order=4" QuasiLocalMeasures::spatial_order = 4 QuasiLocalMeasures::num_surfaces = 3 QuasiLocalMeasures::surface_index [0] = 2 QuasiLocalMeasures::surface_index [1] = 3 #QuasiLocalMeasures::surface_index [2] = 4 QuasiLocalMeasures::output_vtk_every = 0
################################################################################ # Correctness checking ################################################################################
NaNChecker::check_every = 256 NanChecker::check_after = 0 NaNChecker::report_max = 10 NaNChecker::verbose = "all" NaNChecker::action_if_found = terminate NaNChecker::out_NaNmask = yes NaNChecker::check_vars = " ML_BSSN::ML_log_confac HydroBase::w_lorentz ADMBase::curv ADMBase::metric ADMBase::lapse ADMBase::shift HydroBase::rho HydroBase::eps HydroBase::press HydroBase::vel "
################################################################################ # Timers ################################################################################
Carpet::output_timer_tree_every = 1024 Carpet::output_initialise_timer_tree = yes
################################################################################ # Output ################################################################################
IO::out_dir = "data" CarpetIOScalar::all_reductions_in_one_file = "no" CarpetIOScalar::one_file_per_group = "no" CarpetIOASCII::one_file_per_group = "no" CarpetIOHDF5::one_file_per_group = "no"
IOBasic::outInfo_every = 1 IOBasic::outInfo_reductions = "minimum maximum" IOBasic::outInfo_vars = " ML_BSSN::ML_Ham ML_BSSN::ML_log_confac Carpet::physical_time_per_hour SystemStatistics::maxrss_mb SystemStatistics::swap_used_mb Carpet::physical_time_per_hour HydroBase::rho #GRHydro::dens{reductions = 'sum maximum'} HydroBase::w_lorentz "
IOScalar::outScalar_every = 256 IOScalar::outScalar_reductions = "minimum maximum average" IOScalar::outScalar_vars = " SystemStatistics::process_memory_mb ML_BSSN::ML_Ham{ reductions = 'maximum' } "
IOASCII::out0D_every = 256 IOASCII::out0D_vars = " Carpet::physical_time_per_hour Carpet::timing PunctureTracker::pt_loc QuasiLocalMeasures::qlm_scalars{out_every = 256} "
IOASCII::out1D_every = 0 IOASCII::out1D_d = no IOASCII::out1D_x = yes IOASCII::out1D_y = no IOASCII::out1D_z = yes IOASCII::out1D_vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_trace_curv WeylScal4::Psi4r "
IOASCII::out2D_every = 0 IOASCII::out2D_vars = " "
IOHDF5::out_every = 0 IOHDF5::out_vars = " Grid::Coordinates{out_every=1000000000 refinement_levels={0}} ML_BSSN::ML_log_confac WeylScal4::Psi4r WeylScal4::Psi4i WeylScal4::curvIr{refinement_levels={3 5}} WeylScal4::curvIi{refinement_levels={3 5}} WeylScal4::curvJr{refinement_levels={3 5}} WeylScal4::curvJi{refinement_levels={3 5}} "
IOHDF5::out2D_every = 64 IOHDF5::out2D_vars = " Grid::Coordinates{out_every=1000000000 refinement_levels={0}} ML_BSSN::ML_Ham ML_BSSN::alpha ML_BSSN::phi ML_BSSN::trK WeylScal4::Psi4r WeylScal4::Psi4i HydroBase::rho HydroBase::press HydroBase::w_lorentz "
################################################################################ # Checkpointing and recovery ################################################################################
#CarpetIOHDF5::checkpoint = yes IOHDF5::checkpoint = "yes" IO::checkpoint_ID = "yes" IO::recover = "autoprobe" IO::checkpoint_every_walltime_hours = 1.5 IO::checkpoint_keep=2 IO::checkpoint_on_terminate = "yes" IO::out_mode = "onefile" IO::checkpoint_dir = "../checkpoints" IO::recover_dir = "../checkpoints" IO::abort_on_io_errors = yes CarpetIOHDF5::open_one_input_file_at_a_time = yes CarpetIOHDF5::compression_level = 0
################################################################################ # Run termination ################################################################################
TerminationTrigger::max_walltime = 48 # Trigger termination 30 minutes before the walltime is reached TerminationTrigger::on_remaining_walltime = 30 TerminationTrigger::output_remtime_every_minutes = 30 TerminationTrigger::termination_from_file = yes TerminationTrigger::termination_file = "terminate.txt" TerminationTrigger::create_termination_file = yes
Cactus::terminate = time Cactus::cctk_final_time = 1900.0
Users mailing list Users@einsteintoolkit.org http://lists.einsteintoolkit.org/mailman/listinfo/users
Hi Konrad,
I have never seen the code fail at that point. I suspect that your initial data thorn might be scheduling the BCs in a bin in which cctk_nghostzones is not available. Does your parfile work if you replace the NS-BH initial data with some trivial initial data from THC_InitialData?
Best,
David
On Aug 13, 2021, at 4:57 AM, Konrad Topolski k.topolski2@student.uw.edu.pl wrote:
Hi all
I'm currently trying to run a BHNS simulation with initial data generated by Kadath elliptic solver (Fuka branch).
After initial data import, I'm getting an error from THC: WARNING level 0 from host nid00462 process 0 while executing schedule bin HydroBase_Select_Boundaries, routine THC_Core::THC_SelectBC in thorn THC_Core, file /lustre/tetyda/home/topolski/Cactus/arrangements/THCCore/THC_Core/src/thc_select_bc.cc:76: -> Failed to select the BCs
The simulation uses the following thorns NOT included with the official release of ET:
HRSCCore Boost KadathImporter THCBase/CPPUtils, THCBase/FDCore ,THCBase/HRSCCore THCExtra/EOS_Barotropic, THCExtra/EOS_Thermal, THCExtra/EOS_Thermal_Idealgas
Here's the parameter file. If I can provide more info, please let me know (also, if to upload it to a somewhat more convenient place).
Best regards Konrad
# The following is a modification of the ETK Gallery par file to utilize # Kadath initial data for a binary system of BH-NS
################################################################################ # Active thorns ################################################################################
ActiveThorns = " ADMBase ML_ADMConstraints ADMCoupling ADMMacros AEILocalInterp AHFinderDirect Boundary Carpet CarpetIOASCII CarpetIOBasic CarpetIOHDF5 CarpetIOScalar CarpetInterp CarpetInterp2 CarpetLib CarpetReduce CarpetRegrid2 CarpetTracker CartGrid3D CoordBase Coordinates CoordinatesSymmetry GlobalDerivative hwloc IOUtil InitBase Interpolate2 QuasiLocalMeasures KadathImporter LocalInterp LoopControl MoL NaNChecker PunctureTracker Slab SpaceMask SphericalSurface StaticConformal SummationByParts SymBase SystemStatistics SystemTopology TerminationTrigger TensorTypes Time TmunuBase Vectors ML_BSSN ML_BSSN_Helper NewRad GenericFD WeylScal4 Multipole ADMDerivatives CarpetSlab CoordGauge Dissipation EOS_Barotropic EOS_Thermal EOS_Thermal_Idealgas HydroBase HRSCCore ML_ADMConstraints THC_Core TimerReport BLAS LAPACK GSL Boost CPPUtils FDCore HDF5 MPI Timers CycleClock TGRtensor zlib Fortran VolumeIntegrals_GRMHD CarpetMask ReflectionSymmetry # currently turned off #GRHydro #PizzaBase #PizzaIDBase #PizzaNumUtils #NSTracker #Hydro_Analysis "
################################################################################ # Grid structure ################################################################################
#------------------------------------------------------------------------------ # Diagnostic parameters: #------------------------------------------------------------------------------ Carpet::output_timers_every = 0 Carpet::storage_verbose = "no" Carpet::verbose = "no" Carpet::veryverbose = "no" Carpet::grid_structure_filename = "carpet-grid-structure" Carpet::grid_coordinates_filename = "carpet-grid-coordinates"
CarpetLib::output_bboxes = "no"
CarpetMask::verbose = "no" CarpetReduce::verbose = "no" CarpetRegrid2::verbose = "no" CarpetRegrid2::veryverbose = "no" CarpetTracker::verbose = "no"
TimerReport::out_every = 4096 TimerReport::out_filename = "TimerReport" TimerReport::output_all_timers = "yes" TimerReport::output_all_timers_together = "yes" TimerReport::output_all_timers_readable = "yes" TimerReport::n_top_timers = 40
QuasiLocalMeasures::verbose = "no" SphericalSurface::verbose = "no"
Carpet::domain_from_multipatch = yes CartGrid3D::type = "multipatch" CartGrid3D::set_coordinate_ranges_on = "all maps" Coordinates::coordinate_system = "Thornburg04" Coordinates::h_cartesian = 1.5574825174818182 Coordinates::h_radial = 1.5574825174818182
Coordinates::sphere_inner_radius = 51.3969230769 Coordinates::sphere_outer_radius = 2194.15912088 Coordinates::n_angular = 28
Driver::ghost_size = 5 Coordinates::patch_boundary_size = 5 Coordinates::additional_overlap_size = 3 Coordinates::outer_boundary_size = 5
CoordinatesSymmetry::reflection_z = yes CoordinatesSymmetry::stagger = no Coordinates::symmetry = "+z bitant" Coordinates::additional_symmetry_size = 1 Coordinates::verbose = no
Time::timestep_method = "given" Time::timestep = 0.3504335664334091 Carpet::time_refinement_factors = "[1,1,2,4,8,16,32,64,128,256]"
# General Carpet parameters: Carpet::enable_all_storage = "no" Carpet::use_buffer_zones = "yes" Carpet::schedule_barriers = "no"
Carpet::poison_new_timelevels = "yes" Carpet::check_for_poison = "no"
Carpet::init_3_timelevels = "no" Carpet::init_fill_timelevels = "yes"
CarpetLib::poison_new_memory = "yes" CarpetLib::poison_value = 114 CarpetLib::check_bboxes = "no" CarpetLib::interleave_communications = "yes" CarpetLib::combine_sends = "yes"
CarpetInterp::tree_search = "yes" CarpetInterp::check_tree_search = "no"
CarpetRegrid2::freeze_unaligned_levels = "yes" CarpetRegrid2::freeze_unaligned_parent_levels = "yes" CarpetRegrid2::ensure_proper_nesting = "yes" CarpetRegrid2::snap_to_coarse = "yes" CarpetRegrid2::symmetry_rotating180 = "no"
############################################################################### # SpaceMask - only needed for GRHydro ###############################################################################
#SpaceMask::use_mask = "yes"
################################################################################ # Mesh refinement ################################################################################
Carpet::max_refinement_levels = 9 CarpetRegrid2::num_centres = 2 CarpetRegrid2::num_levels_1 = 7 CarpetRegrid2::position_x_1 = +17.37737 CarpetRegrid2::radius_1 = [0,21.267692,10.633846,5.316923,2.658462,1.329231,0.664615] CarpetRegrid2::num_levels_2 = 7 CarpetRegrid2::position_x_2 = -13.76685 CarpetRegrid2::radius_2 = [0,17.132308,8.566154,4.283077,2.141538,1.070769,0.535385]
Carpet::prolongation_order_space = 5 Carpet::prolongation_order_time = 2
CarpetRegrid2::regrid_every = 128
################################################################################ # Initial data ################################################################################
HydroBase::timelevels = 3 HydroBase::prolongation_type ="None" ADMBase::metric_type = "physical"
ADMBase::initial_data = "Kadath" ADMBase::initial_lapse = "Kadath" ADMBase::initial_shift = "Kadath" ADMBase::initial_dtlapse = "Kadath" ADMBase::initial_dtshift = "Kadath" HydroBase::initial_hydro = "Kadath"
# Kadath BNS file KadathImporter::type = "BHNS" #PizzaIDBase::eos_file = "gam2.polytrope" # Geometric unit system for initial data, specified by length unit. # use CACTUS units #PizzaBase::length_unit = 1476.7161818921163 KadathImporter::filename = "/home/topolski/Cactus/BHNS_par/BHNS.info" KadathImporter::puncture_lapse = yes
InitBase::initial_data_setup_method = "init_single_level"
# ============================================================================ # Templated hydrodynamics code # ============================================================================= HydroBase::evolution_method = "THCode" HydroBase::initial_entropy = "THCode"
THC_Core::physics = "GRHD" THC_Core::eos_type = "ideal" THC_Core::bc_type = "none" THC_Core::atmo_rho = 1e-13
HRSCCore::scheme = "FV" HRSCCore::reconstruction = "MP5" HRSCCore::riemann_solver = "HLLE"
THC_Core::c2a_BH_alp = 0.15 THC_Core::c2a_rho_strict = 2.0e-5 THC_Core::c2a_set_to_nan_on_failure = "no" THC_Core::c2a_fix_conservatives = "yes" THC_Core::c2a_kill_on_failure = "no"
EOS_Thermal::evol_eos_name = "IdealGas" EOS_Thermal_IdealGas::index_n = 1 EOS_Thermal_IdealGas::eps_min = 1e-8
################################################################################ # Evolution and boundary ################################################################################
GenericFD::jacobian_group = "Coordinates::jacobian" GenericFD::jacobian_derivative_group = "Coordinates::jacobian2" GenericFD::jacobian_identity_map = 0
ADMBase::evolution_method = "ML_BSSN" ADMBase::lapse_evolution_method = "ML_BSSN" ADMBase::shift_evolution_method = "ML_BSSN" ADMBase::dtlapse_evolution_method = "ML_BSSN" ADMBase::dtshift_evolution_method = "ML_BSSN"
ADMBase::lapse_timelevels = 3 ADMBase::shift_timelevels = 3 ADMBase::metric_timelevels = 3
TmunuBase::prolongation_type = "none" TmunuBase::stress_energy_storage = "yes" TmunuBase::stress_energy_at_RHS = "yes" TmunuBase::support_old_CalcTmunu_mechanism = "no"
ML_BSSN::timelevels = 3 ML_BSSN::fdOrder = 8 THC_Core::fd_order = 4
# 1+log slicing requires harmonicN = 1 and harmonicF = 2.0 ML_BSSN::harmonicN = 1 ML_BSSN::harmonicF = 2.0
ML_BSSN::ShiftGammaCoeff = 0.75 ML_BSSN::evolveA = 1 ML_BSSN::evolveB = 1 ML_BSSN::AlphaDriver = 0.0 ML_BSSN::BetaDriver = 0.7 # ~ 1/M ML_BSSN::advectLapse = 1 ML_BSSN::advectShift = 1
ML_BSSN::MinimumLapse = 1.0e-8
# conformalaMethod = 1 for W, 0 for phi ML_BSSN::conformalMethod = 1
# We apply dissipation using GlobalDerivaitive so disable it here ML_BSSN::epsDiss = 0.0
ML_BSSN::dt_lapse_shift_method = "noLapseShiftAdvection"
ML_BSSN::initial_boundary_condition = "extrapolate-gammas" ML_BSSN::rhs_boundary_condition = "NewRad" Boundary::radpower = 2
ML_BSSN::ML_log_confac_bound = "none" ML_BSSN::ML_metric_bound = "none" ML_BSSN::ML_Gamma_bound = "none" ML_BSSN::ML_trace_curv_bound = "none" ML_BSSN::ML_curv_bound = "none" ML_BSSN::ML_lapse_bound = "none" ML_BSSN::ML_dtlapse_bound = "none" ML_BSSN::ML_shift_bound = "none" ML_BSSN::ML_dtshift_bound = "none"
Dissipation::order = 5 Dissipation::epsdis = 0.1 Dissipation::vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_metric ML_BSSN::ML_curv ML_BSSN::ML_trace_curv ML_BSSN::ML_Gamma ML_BSSN::ML_lapse ML_BSSN::ML_shift "
################################################################################ # BH and NS tracking (currently using VolumeIntegrals_GRMHD for tracking the # neutron star, will switch to NSTracker some time in the future) # BH is at x>0, NS at x<0 ################################################################################
CarpetTracker::surface [0] = 0 CarpetTracker::surface [1] = 1 #PunctureTracker::track [0] = yes #PunctureTracker::initial_x [0] = -13.76685 #PunctureTracker::which_surface_to_store_info[0] = 0 PunctureTracker::track [0] = yes PunctureTracker::initial_x [0] = +17.37737 PunctureTracker::which_surface_to_store_info[0] = 1
VolumeIntegrals_GRMHD::verbose = 1 VolumeIntegrals_GRMHD::VolIntegral_out_every = 32 VolumeIntegrals_GRMHD::NumIntegrals = 4 # Integrate the entire volume with an integrand of 1. (used for testing/validation purposes only). VolumeIntegrals_GRMHD::Integration_quantity_keyword[1] = "one" # To compute the center of mass in an integration volume originally centered at (x,y,z) = (-13.76685,0,0) #with a coordinate radius of 13.5. Also use the center of mass integral result to set the ZEROTH AMR center. VolumeIntegrals_GRMHD::Integration_quantity_keyword[2] = "centerofmass" VolumeIntegrals_GRMHD::volintegral_sphere__center_x_initial [2] = -13.76685 VolumeIntegrals_GRMHD::volintegral_inside_sphere__radius [2] = 9.10722 VolumeIntegrals_GRMHD::amr_centre__tracks__volintegral_inside_sphere[2] = 0 # Same as above, except use the integrand=1 (for validation purposes, to ensure the integration volume is approximately 4/3*pi*13.5^3). VolumeIntegrals_GRMHD::Integration_quantity_keyword[3] = "one" VolumeIntegrals_GRMHD::volintegral_sphere__center_x_initial [3] = -13.76685 VolumeIntegrals_GRMHD::volintegral_inside_sphere__radius [3] = 9.10722 # Perform rest-mass integrals over entire volume. VolumeIntegrals_GRMHD::Integration_quantity_keyword[4] = "restmass" VolumeIntegrals_GRMHD::enable_file_output = 1 VolumeIntegrals_GRMHD::outVolIntegral_dir = "volume_integration" VolumeIntegrals_GRMHD::CoM_integrand_GAMMA_SPEED_LIMIT = 1e4
######################## # Setting up NS Tracker and Hydro_Analysis #########################
#Hydro_Analysis::Hydro_Analysis_comp_rho_max = "true" #Hydro_Analysis::Hydro_Analysis_rho_max_loc_only_positive_x = "true" #Hydro_Analysis::Hydro_Analysis_comp_rho_max_origin_distance = "yes" #Hydro_Analysis::Hydro_Analysis_average_multiple_maxima_locations = "yes" #Hydro_Analysis::Hydro_Analysis_interpolator_name = "Lagrange polynomial interpolation (tensor product)"
#NSTracker::NSTracker_SF_Name = "Lefthand NS" #NSTracker::NSTracker_SF_Index = 0 #NSTracker::NSTracker_max_distance = 3 #NSTracker::NSTracker_tracked_location = "Hydro_Analysis::Hydro_Analysis_rho_max_loc" # location of maximum #position_x[NSTracker_SF_Index] = Hydro_Analysis_rho_max_loc #position_x[NSTracker_SF_Index_Opposite] = - Hydro_Analysis_rho_max_loc #NSTracker::NSTracker_tracked_location = "Hydro_Analysis::Hydro_Analysis_rho_core_center_volume_weighted" # center of mass #NSTracker::NSTracker_stop_time = -1
################################################################################ # Spatial finite differencing ################################################################################
SummationByParts::order = 8
# Drop order instead of using upwinded stencils, only for advection derivatives SummationByParts::sbp_upwind_deriv = no
SummationByParts::sbp_1st_deriv = yes SummationByParts::sbp_2nd_deriv = no SummationByParts::onesided_interpatch_boundaries = no SummationByParts::onesided_outer_boundaries = yes SummationByParts::use_dissipation = no GlobalDerivative::use_dissipation = yes SummationByParts::scale_with_h = yes SummationByParts::dissipation_type = "Kreiss-Oliger" SummationByParts::epsdis = 0.15
# Because dt/dx is half that of the other levels we also need half the dissipation GlobalDerivative::epsdis_for_level [0] = 0.075
# Variables for dissipation SummationByParts::vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_metric ML_BSSN::ML_trace_curv ML_BSSN::ML_curv ML_BSSN::ML_Gamma ML_BSSN::ML_lapse ML_BSSN::ML_shift ML_BSSN::ML_dtlapse ML_BSSN::ML_dtshift "
################################################################################ # Time integration ################################################################################
MoL::ODE_Method = "rk4" MoL::MoL_Intermediate_Steps = 4 MoL::MoL_Num_Scratch_Levels = 1
################################################################################ # Interpolation ################################################################################
# Use 5-th order interpatch interpolation on the Llama grid Interpolate::interpolator_order = 5
################################################################################ # Psi4 computation ################################################################################
WeylScal4::fdOrder = 8 WeylScal4::calc_scalars = "psis" WeylScal4::calc_invariants = "always"
################################################################################ # Psi4 mode decomposition ################################################################################
# Radii are chosen to be evenly spaced in 1/r as that is the variable # extrapolation is performed in Multipole::nradii = 7 Multipole::radius[0] = 100 Multipole::radius[1] = 115 Multipole::radius[2] = 136 Multipole::radius[3] = 167 Multipole::radius[4] = 214 Multipole::radius[5] = 300 Multipole::radius[6] = 500 Multipole::ntheta = 120 Multipole::nphi = 240 Multipole::variables = "WeylScal4::Psi4r{sw=-2 cmplx='WeylScal4::Psi4i' name='psi4'}" Multipole::out_every = 256 Multipole::l_max = 8 Multipole::output_hdf5 = yes
# Disable ASCII output to avoid creating a large number of files Multipole::output_ascii = no
################################################################################ # Apparent Horizons ################################################################################
AHFinderDirect::N_horizons = 2 AHFinderDirect::find_every = 256 AHFinderDirect::output_h_every = 0 AHFinderDirect::max_Newton_iterations__initial = 50 AHFinderDirect::max_Newton_iterations__subsequent = 50 AHFinderDirect::max_allowable_Theta_growth_iterations = 10 AHFinderDirect::max_allowable_Theta_nonshrink_iterations = 10 AHFinderDirect::geometry_interpolator_name = "Lagrange polynomial interpolation" AHFinderDirect::geometry_interpolator_pars = "order=4" AHFinderDirect::surface_interpolator_name = "Lagrange polynomial interpolation" AHFinderDirect::surface_interpolator_pars = "order=4" AHFinderDirect::verbose_level = "physics details" AHFinderDirect::move_origins = yes
AHFinderDirect::origin_x [1] = +17.37737 AHFinderDirect::initial_guess__coord_sphere__x_center[1] = +17.37737 AHFinderDirect::initial_guess__coord_sphere__radius [1] = 0.73 AHFinderDirect::which_surface_to_store_info [1] = 2 AHFinderDirect::set_mask_for_individual_horizon [1] = no AHFinderDirect::reset_horizon_after_not_finding [1] = no AHFinderDirect::track_origin_from_grid_scalar [1] = yes AHFinderDirect::track_origin_source_x [1] = "PunctureTracker::pt_loc_x[0]" AHFinderDirect::track_origin_source_y [1] = "PunctureTracker::pt_loc_y[0]" AHFinderDirect::track_origin_source_z [1] = "PunctureTracker::pt_loc_z[0]" AHFinderDirect::max_allowable_horizon_radius [1] = 3
#AHFinderDirect::origin_x [2] = -5.53846153846 #AHFinderDirect::initial_guess__coord_sphere__x_center[2] = -5.53846153846 #AHFinderDirect::initial_guess__coord_sphere__radius [2] = 0.535384615385 #AHFinderDirect::which_surface_to_store_info [2] = 3 #AHFinderDirect::set_mask_for_individual_horizon [2] = no #AHFinderDirect::reset_horizon_after_not_finding [2] = no #AHFinderDirect::track_origin_from_grid_scalar [2] = yes #AHFinderDirect::track_origin_source_x [2] = "PunctureTracker::pt_loc_x[1]" #AHFinderDirect::track_origin_source_y [2] = "PunctureTracker::pt_loc_y[1]" #AHFinderDirect::track_origin_source_z [2] = "PunctureTracker::pt_loc_z[1]" #AHFinderDirect::max_allowable_horizon_radius [2] = 3
AHFinderDirect::origin_x [2] = 0 AHFinderDirect::find_after_individual [2] = 371902 AHFinderDirect::initial_guess__coord_sphere__x_center[2] = 0 AHFinderDirect::initial_guess__coord_sphere__radius [2] = 1.67 AHFinderDirect::which_surface_to_store_info [2] = 4 AHFinderDirect::set_mask_for_individual_horizon [2] = no AHFinderDirect::max_allowable_horizon_radius [2] = 6
################################################################################ # Spherical surfaces ################################################################################
SphericalSurface::nsurfaces = 4 SphericalSurface::maxntheta = 66 SphericalSurface::maxnphi = 124
# Surfaces 0 and 1 are used by PunctureTracker
# Horizon 1 SphericalSurface::ntheta [2] = 41 SphericalSurface::nphi [2] = 80 SphericalSurface::nghoststheta [2] = 2 SphericalSurface::nghostsphi [2] = 2
# Horizon 2 #SphericalSurface::ntheta [3] = 41 #SphericalSurface::nphi [3] = 80 #SphericalSurface::nghoststheta [3] = 2 #SphericalSurface::nghostsphi [3] = 2
# Common horizon SphericalSurface::ntheta [3] = 41 SphericalSurface::nphi [3] = 80 SphericalSurface::nghoststheta [3] = 2 SphericalSurface::nghostsphi [3] = 2
################################################################################ # Isolated Horizons ################################################################################
QuasiLocalMeasures::veryverbose = no QuasiLocalMeasures::interpolator = "Lagrange polynomial interpolation" QuasiLocalMeasures::interpolator_options = "order=4" QuasiLocalMeasures::spatial_order = 4 QuasiLocalMeasures::num_surfaces = 3 QuasiLocalMeasures::surface_index [0] = 2 QuasiLocalMeasures::surface_index [1] = 3 #QuasiLocalMeasures::surface_index [2] = 4 QuasiLocalMeasures::output_vtk_every = 0
################################################################################ # Correctness checking ################################################################################
NaNChecker::check_every = 256 NanChecker::check_after = 0 NaNChecker::report_max = 10 NaNChecker::verbose = "all" NaNChecker::action_if_found = terminate NaNChecker::out_NaNmask = yes NaNChecker::check_vars = " ML_BSSN::ML_log_confac HydroBase::w_lorentz ADMBase::curv ADMBase::metric ADMBase::lapse ADMBase::shift HydroBase::rho HydroBase::eps HydroBase::press HydroBase::vel "
################################################################################ # Timers ################################################################################
Carpet::output_timer_tree_every = 1024 Carpet::output_initialise_timer_tree = yes
################################################################################ # Output ################################################################################
IO::out_dir = "data" CarpetIOScalar::all_reductions_in_one_file = "no" CarpetIOScalar::one_file_per_group = "no" CarpetIOASCII::one_file_per_group = "no" CarpetIOHDF5::one_file_per_group = "no"
IOBasic::outInfo_every = 1 IOBasic::outInfo_reductions = "minimum maximum" IOBasic::outInfo_vars = " ML_BSSN::ML_Ham ML_BSSN::ML_log_confac Carpet::physical_time_per_hour SystemStatistics::maxrss_mb SystemStatistics::swap_used_mb Carpet::physical_time_per_hour HydroBase::rho #GRHydro::dens{reductions = 'sum maximum'} HydroBase::w_lorentz "
IOScalar::outScalar_every = 256 IOScalar::outScalar_reductions = "minimum maximum average" IOScalar::outScalar_vars = " SystemStatistics::process_memory_mb ML_BSSN::ML_Ham{ reductions = 'maximum' } "
IOASCII::out0D_every = 256 IOASCII::out0D_vars = " Carpet::physical_time_per_hour Carpet::timing PunctureTracker::pt_loc QuasiLocalMeasures::qlm_scalars{out_every = 256} "
IOASCII::out1D_every = 0 IOASCII::out1D_d = no IOASCII::out1D_x = yes IOASCII::out1D_y = no IOASCII::out1D_z = yes IOASCII::out1D_vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_trace_curv WeylScal4::Psi4r "
IOASCII::out2D_every = 0 IOASCII::out2D_vars = " "
IOHDF5::out_every = 0 IOHDF5::out_vars = " Grid::Coordinates{out_every=1000000000 refinement_levels={0}} ML_BSSN::ML_log_confac WeylScal4::Psi4r WeylScal4::Psi4i WeylScal4::curvIr{refinement_levels={3 5}} WeylScal4::curvIi{refinement_levels={3 5}} WeylScal4::curvJr{refinement_levels={3 5}} WeylScal4::curvJi{refinement_levels={3 5}} "
IOHDF5::out2D_every = 64 IOHDF5::out2D_vars = " Grid::Coordinates{out_every=1000000000 refinement_levels={0}} ML_BSSN::ML_Ham ML_BSSN::alpha ML_BSSN::phi ML_BSSN::trK WeylScal4::Psi4r WeylScal4::Psi4i HydroBase::rho HydroBase::press HydroBase::w_lorentz "
################################################################################ # Checkpointing and recovery ################################################################################
#CarpetIOHDF5::checkpoint = yes IOHDF5::checkpoint = "yes" IO::checkpoint_ID = "yes" IO::recover = "autoprobe" IO::checkpoint_every_walltime_hours = 1.5 IO::checkpoint_keep=2 IO::checkpoint_on_terminate = "yes" IO::out_mode = "onefile" IO::checkpoint_dir = "../checkpoints" IO::recover_dir = "../checkpoints" IO::abort_on_io_errors = yes CarpetIOHDF5::open_one_input_file_at_a_time = yes CarpetIOHDF5::compression_level = 0
################################################################################ # Run termination ################################################################################
TerminationTrigger::max_walltime = 48 # Trigger termination 30 minutes before the walltime is reached TerminationTrigger::on_remaining_walltime = 30 TerminationTrigger::output_remtime_every_minutes = 30 TerminationTrigger::termination_from_file = yes TerminationTrigger::termination_file = "terminate.txt" TerminationTrigger::create_termination_file = yes
Cactus::terminate = time Cactus::cctk_final_time = 1900.0
Users mailing list Users@einsteintoolkit.org http://lists.einsteintoolkit.org/mailman/listinfo/users
Hi David
Changing the initial data (i.e. not using Kadath at all) from: #ADMBase::metric_type = "physical" #ADMBase::initial_data = "Kadath" #ADMBase::initial_lapse = "Kadath" #ADMBase::initial_shift = "Kadath" #ADMBase::initial_dtlapse = "Kadath" #ADMBase::initial_dtshift = "Kadath" #HydroBase::initial_hydro = "Kadath" # Kadath BHNS file #KadathImporter::type = "BHNS" #KadathImporter::filename = "/home/topolski/Cactus/BHNS_par/BHNS.info" #KadathImporter::puncture_lapse = yes
to: THC_InitialData::id_type = "shocktube" THC_InitialData::shocktube_case = "blast_wave" THC_InitialData::shocktube_dir = "x"
did not change the result and the error is still the same. In the ISSUES file of one of the THC thorns it says that Intel compilers do not pass all the tests - and I did compile with Intel - but I think it's unlikely to cause this problem.
The single NS initial data and its parameter file supplied with Kadath works fine and I took the THC part from there. This leads me to believe (as Erik points out too) I messed something up in grid structure / carpet parameters - I tried to follow the gallery example file for GW150914.
Best Konrad
pt., 13 sie 2021 o 17:38 Radice, David dur566@psu.edu napisał(a):
Hi Konrad,
I have never seen the code fail at that point. I suspect that your initial data thorn might be scheduling the BCs in a bin in which cctk_nghostzones is not available. Does your parfile work if you replace the NS-BH initial data with some trivial initial data from THC_InitialData?
Best,
David
On Aug 13, 2021, at 4:57 AM, Konrad Topolski <
k.topolski2@student.uw.edu.pl> wrote:
Hi all
I'm currently trying to run a BHNS simulation with initial data
generated by Kadath elliptic solver (Fuka branch).
After initial data import, I'm getting an error from THC: WARNING level 0 from host nid00462 process 0 while executing schedule bin HydroBase_Select_Boundaries, routine
THC_Core::THC_SelectBC
in thorn THC_Core, file
/lustre/tetyda/home/topolski/Cactus/arrangements/THCCore/THC_Core/src/thc_select_bc.cc:76:
-> Failed to select the BCs
The simulation uses the following thorns NOT included with the official
release of ET:
HRSCCore Boost KadathImporter THCBase/CPPUtils, THCBase/FDCore ,THCBase/HRSCCore THCExtra/EOS_Barotropic, THCExtra/EOS_Thermal,
THCExtra/EOS_Thermal_Idealgas
Here's the parameter file. If I can provide more info, please let me
know (also, if to upload it to a somewhat more convenient place).
Best regards Konrad
# The following is a modification of the ETK Gallery par file to utilize # Kadath initial data for a binary system of BH-NS
################################################################################
# Active thorns
################################################################################
ActiveThorns = " ADMBase ML_ADMConstraints ADMCoupling ADMMacros AEILocalInterp AHFinderDirect Boundary Carpet CarpetIOASCII CarpetIOBasic CarpetIOHDF5 CarpetIOScalar CarpetInterp CarpetInterp2 CarpetLib CarpetReduce CarpetRegrid2 CarpetTracker CartGrid3D CoordBase Coordinates CoordinatesSymmetry GlobalDerivative hwloc IOUtil InitBase Interpolate2 QuasiLocalMeasures KadathImporter LocalInterp LoopControl MoL NaNChecker PunctureTracker Slab SpaceMask SphericalSurface StaticConformal SummationByParts SymBase SystemStatistics SystemTopology TerminationTrigger TensorTypes Time TmunuBase Vectors ML_BSSN ML_BSSN_Helper NewRad GenericFD WeylScal4 Multipole ADMDerivatives CarpetSlab CoordGauge Dissipation EOS_Barotropic EOS_Thermal EOS_Thermal_Idealgas HydroBase HRSCCore ML_ADMConstraints THC_Core TimerReport BLAS LAPACK GSL Boost CPPUtils FDCore HDF5 MPI Timers CycleClock TGRtensor zlib Fortran VolumeIntegrals_GRMHD CarpetMask ReflectionSymmetry # currently turned off #GRHydro #PizzaBase #PizzaIDBase #PizzaNumUtils #NSTracker #Hydro_Analysis "
################################################################################
# Grid structure
################################################################################
#------------------------------------------------------------------------------
# Diagnostic parameters:
#------------------------------------------------------------------------------
Carpet::output_timers_every = 0 Carpet::storage_verbose = "no" Carpet::verbose = "no" Carpet::veryverbose = "no" Carpet::grid_structure_filename = "carpet-grid-structure" Carpet::grid_coordinates_filename = "carpet-grid-coordinates"
CarpetLib::output_bboxes = "no"
CarpetMask::verbose = "no" CarpetReduce::verbose = "no" CarpetRegrid2::verbose = "no" CarpetRegrid2::veryverbose = "no" CarpetTracker::verbose = "no"
TimerReport::out_every = 4096 TimerReport::out_filename = "TimerReport" TimerReport::output_all_timers = "yes" TimerReport::output_all_timers_together = "yes" TimerReport::output_all_timers_readable = "yes" TimerReport::n_top_timers = 40
QuasiLocalMeasures::verbose = "no" SphericalSurface::verbose = "no"
Carpet::domain_from_multipatch = yes CartGrid3D::type = "multipatch" CartGrid3D::set_coordinate_ranges_on = "all maps" Coordinates::coordinate_system = "Thornburg04" Coordinates::h_cartesian = 1.5574825174818182 Coordinates::h_radial = 1.5574825174818182
Coordinates::sphere_inner_radius = 51.3969230769 Coordinates::sphere_outer_radius = 2194.15912088 Coordinates::n_angular = 28
Driver::ghost_size = 5 Coordinates::patch_boundary_size = 5 Coordinates::additional_overlap_size = 3 Coordinates::outer_boundary_size = 5
CoordinatesSymmetry::reflection_z = yes CoordinatesSymmetry::stagger = no Coordinates::symmetry = "+z bitant" Coordinates::additional_symmetry_size = 1 Coordinates::verbose = no
Time::timestep_method = "given" Time::timestep = 0.3504335664334091 Carpet::time_refinement_factors = "[1,1,2,4,8,16,32,64,128,256]"
# General Carpet parameters: Carpet::enable_all_storage = "no" Carpet::use_buffer_zones = "yes" Carpet::schedule_barriers = "no"
Carpet::poison_new_timelevels = "yes" Carpet::check_for_poison = "no"
Carpet::init_3_timelevels = "no" Carpet::init_fill_timelevels = "yes"
CarpetLib::poison_new_memory = "yes" CarpetLib::poison_value = 114 CarpetLib::check_bboxes = "no" CarpetLib::interleave_communications = "yes" CarpetLib::combine_sends = "yes"
CarpetInterp::tree_search = "yes" CarpetInterp::check_tree_search = "no"
CarpetRegrid2::freeze_unaligned_levels = "yes" CarpetRegrid2::freeze_unaligned_parent_levels = "yes" CarpetRegrid2::ensure_proper_nesting = "yes" CarpetRegrid2::snap_to_coarse = "yes" CarpetRegrid2::symmetry_rotating180 = "no"
###############################################################################
# SpaceMask - only needed for GRHydro
###############################################################################
#SpaceMask::use_mask = "yes"
################################################################################
# Mesh refinement
################################################################################
Carpet::max_refinement_levels = 9 CarpetRegrid2::num_centres = 2 CarpetRegrid2::num_levels_1 = 7 CarpetRegrid2::position_x_1 = +17.37737 CarpetRegrid2::radius_1 =
[0,21.267692,10.633846,5.316923,2.658462,1.329231,0.664615]
CarpetRegrid2::num_levels_2 = 7 CarpetRegrid2::position_x_2 = -13.76685 CarpetRegrid2::radius_2 =
[0,17.132308,8.566154,4.283077,2.141538,1.070769,0.535385]
Carpet::prolongation_order_space = 5 Carpet::prolongation_order_time = 2
CarpetRegrid2::regrid_every = 128
################################################################################
# Initial data
################################################################################
HydroBase::timelevels = 3 HydroBase::prolongation_type ="None" ADMBase::metric_type = "physical"
ADMBase::initial_data = "Kadath" ADMBase::initial_lapse = "Kadath" ADMBase::initial_shift = "Kadath" ADMBase::initial_dtlapse = "Kadath" ADMBase::initial_dtshift = "Kadath" HydroBase::initial_hydro = "Kadath"
# Kadath BNS file KadathImporter::type = "BHNS" #PizzaIDBase::eos_file = "gam2.polytrope" # Geometric unit system for initial data, specified by length unit. # use CACTUS units #PizzaBase::length_unit = 1476.7161818921163 KadathImporter::filename = "/home/topolski/Cactus/BHNS_par/BHNS.info" KadathImporter::puncture_lapse = yes
InitBase::initial_data_setup_method = "init_single_level"
#
============================================================================
# Templated hydrodynamics code #
=============================================================================
HydroBase::evolution_method = "THCode" HydroBase::initial_entropy = "THCode"
THC_Core::physics = "GRHD" THC_Core::eos_type = "ideal" THC_Core::bc_type = "none" THC_Core::atmo_rho = 1e-13
HRSCCore::scheme = "FV" HRSCCore::reconstruction = "MP5" HRSCCore::riemann_solver = "HLLE"
THC_Core::c2a_BH_alp = 0.15 THC_Core::c2a_rho_strict = 2.0e-5 THC_Core::c2a_set_to_nan_on_failure = "no" THC_Core::c2a_fix_conservatives = "yes" THC_Core::c2a_kill_on_failure = "no"
EOS_Thermal::evol_eos_name = "IdealGas" EOS_Thermal_IdealGas::index_n = 1 EOS_Thermal_IdealGas::eps_min = 1e-8
################################################################################
# Evolution and boundary
################################################################################
GenericFD::jacobian_group = "Coordinates::jacobian" GenericFD::jacobian_derivative_group = "Coordinates::jacobian2" GenericFD::jacobian_identity_map = 0
ADMBase::evolution_method = "ML_BSSN" ADMBase::lapse_evolution_method = "ML_BSSN" ADMBase::shift_evolution_method = "ML_BSSN" ADMBase::dtlapse_evolution_method = "ML_BSSN" ADMBase::dtshift_evolution_method = "ML_BSSN"
ADMBase::lapse_timelevels = 3 ADMBase::shift_timelevels = 3 ADMBase::metric_timelevels = 3
TmunuBase::prolongation_type = "none" TmunuBase::stress_energy_storage = "yes" TmunuBase::stress_energy_at_RHS = "yes" TmunuBase::support_old_CalcTmunu_mechanism = "no"
ML_BSSN::timelevels = 3 ML_BSSN::fdOrder = 8 THC_Core::fd_order = 4
# 1+log slicing requires harmonicN = 1 and harmonicF = 2.0 ML_BSSN::harmonicN = 1 ML_BSSN::harmonicF = 2.0
ML_BSSN::ShiftGammaCoeff = 0.75 ML_BSSN::evolveA = 1 ML_BSSN::evolveB = 1 ML_BSSN::AlphaDriver = 0.0 ML_BSSN::BetaDriver = 0.7 # ~ 1/M ML_BSSN::advectLapse = 1 ML_BSSN::advectShift = 1
ML_BSSN::MinimumLapse = 1.0e-8
# conformalaMethod = 1 for W, 0 for phi ML_BSSN::conformalMethod = 1
# We apply dissipation using GlobalDerivaitive so disable it here ML_BSSN::epsDiss = 0.0
ML_BSSN::dt_lapse_shift_method = "noLapseShiftAdvection"
ML_BSSN::initial_boundary_condition = "extrapolate-gammas" ML_BSSN::rhs_boundary_condition = "NewRad" Boundary::radpower = 2
ML_BSSN::ML_log_confac_bound = "none" ML_BSSN::ML_metric_bound = "none" ML_BSSN::ML_Gamma_bound = "none" ML_BSSN::ML_trace_curv_bound = "none" ML_BSSN::ML_curv_bound = "none" ML_BSSN::ML_lapse_bound = "none" ML_BSSN::ML_dtlapse_bound = "none" ML_BSSN::ML_shift_bound = "none" ML_BSSN::ML_dtshift_bound = "none"
Dissipation::order = 5 Dissipation::epsdis = 0.1 Dissipation::vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_metric ML_BSSN::ML_curv ML_BSSN::ML_trace_curv ML_BSSN::ML_Gamma ML_BSSN::ML_lapse ML_BSSN::ML_shift "
################################################################################
# BH and NS tracking (currently using VolumeIntegrals_GRMHD for tracking
the
# neutron star, will switch to NSTracker some time in the future) # BH is at x>0, NS at x<0
################################################################################
CarpetTracker::surface [0] = 0 CarpetTracker::surface [1] = 1 #PunctureTracker::track [0] = yes #PunctureTracker::initial_x [0] = -13.76685 #PunctureTracker::which_surface_to_store_info[0] = 0 PunctureTracker::track [0] = yes PunctureTracker::initial_x [0] = +17.37737 PunctureTracker::which_surface_to_store_info[0] = 1
VolumeIntegrals_GRMHD::verbose = 1 VolumeIntegrals_GRMHD::VolIntegral_out_every = 32 VolumeIntegrals_GRMHD::NumIntegrals = 4 # Integrate the entire volume with an integrand of 1. (used for
testing/validation purposes only).
VolumeIntegrals_GRMHD::Integration_quantity_keyword[1] = "one" # To compute the center of mass in an integration volume originally
centered at (x,y,z) = (-13.76685,0,0)
#with a coordinate radius of 13.5. Also use the center of mass integral
result to set the ZEROTH AMR center.
VolumeIntegrals_GRMHD::Integration_quantity_keyword[2] = "centerofmass" VolumeIntegrals_GRMHD::volintegral_sphere__center_x_initial [2]
= -13.76685
VolumeIntegrals_GRMHD::volintegral_inside_sphere__radius [2]
= 9.10722
VolumeIntegrals_GRMHD::amr_centre__tracks__volintegral_inside_sphere[2]
= 0
# Same as above, except use the integrand=1 (for validation purposes, to
ensure the integration volume is approximately 4/3*pi*13.5^3).
VolumeIntegrals_GRMHD::Integration_quantity_keyword[3] = "one" VolumeIntegrals_GRMHD::volintegral_sphere__center_x_initial [3]
= -13.76685
VolumeIntegrals_GRMHD::volintegral_inside_sphere__radius [3]
= 9.10722
# Perform rest-mass integrals over entire volume. VolumeIntegrals_GRMHD::Integration_quantity_keyword[4] = "restmass" VolumeIntegrals_GRMHD::enable_file_output = 1 VolumeIntegrals_GRMHD::outVolIntegral_dir = "volume_integration" VolumeIntegrals_GRMHD::CoM_integrand_GAMMA_SPEED_LIMIT = 1e4
######################## # Setting up NS Tracker and Hydro_Analysis #########################
#Hydro_Analysis::Hydro_Analysis_comp_rho_max = "true" #Hydro_Analysis::Hydro_Analysis_rho_max_loc_only_positive_x = "true" #Hydro_Analysis::Hydro_Analysis_comp_rho_max_origin_distance = "yes" #Hydro_Analysis::Hydro_Analysis_average_multiple_maxima_locations = "yes" #Hydro_Analysis::Hydro_Analysis_interpolator_name = "Lagrange polynomial
interpolation (tensor product)"
#NSTracker::NSTracker_SF_Name = "Lefthand NS" #NSTracker::NSTracker_SF_Index = 0 #NSTracker::NSTracker_max_distance = 3 #NSTracker::NSTracker_tracked_location =
"Hydro_Analysis::Hydro_Analysis_rho_max_loc" # location of maximum
#position_x[NSTracker_SF_Index] = Hydro_Analysis_rho_max_loc #position_x[NSTracker_SF_Index_Opposite] = - Hydro_Analysis_rho_max_loc #NSTracker::NSTracker_tracked_location =
"Hydro_Analysis::Hydro_Analysis_rho_core_center_volume_weighted" # center of mass
#NSTracker::NSTracker_stop_time = -1
################################################################################
# Spatial finite differencing
################################################################################
SummationByParts::order = 8
# Drop order instead of using upwinded stencils, only for advection
derivatives
SummationByParts::sbp_upwind_deriv = no
SummationByParts::sbp_1st_deriv = yes SummationByParts::sbp_2nd_deriv = no SummationByParts::onesided_interpatch_boundaries = no SummationByParts::onesided_outer_boundaries = yes SummationByParts::use_dissipation = no GlobalDerivative::use_dissipation = yes SummationByParts::scale_with_h = yes SummationByParts::dissipation_type = "Kreiss-Oliger" SummationByParts::epsdis = 0.15
# Because dt/dx is half that of the other levels we also need half the
dissipation
GlobalDerivative::epsdis_for_level [0] = 0.075
# Variables for dissipation SummationByParts::vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_metric ML_BSSN::ML_trace_curv ML_BSSN::ML_curv ML_BSSN::ML_Gamma ML_BSSN::ML_lapse ML_BSSN::ML_shift ML_BSSN::ML_dtlapse ML_BSSN::ML_dtshift "
################################################################################
# Time integration
################################################################################
MoL::ODE_Method = "rk4" MoL::MoL_Intermediate_Steps = 4 MoL::MoL_Num_Scratch_Levels = 1
################################################################################
# Interpolation
################################################################################
# Use 5-th order interpatch interpolation on the Llama grid Interpolate::interpolator_order = 5
################################################################################
# Psi4 computation
################################################################################
WeylScal4::fdOrder = 8 WeylScal4::calc_scalars = "psis" WeylScal4::calc_invariants = "always"
################################################################################
# Psi4 mode decomposition
################################################################################
# Radii are chosen to be evenly spaced in 1/r as that is the variable # extrapolation is performed in Multipole::nradii = 7 Multipole::radius[0] = 100 Multipole::radius[1] = 115 Multipole::radius[2] = 136 Multipole::radius[3] = 167 Multipole::radius[4] = 214 Multipole::radius[5] = 300 Multipole::radius[6] = 500 Multipole::ntheta = 120 Multipole::nphi = 240 Multipole::variables = "WeylScal4::Psi4r{sw=-2
cmplx='WeylScal4::Psi4i' name='psi4'}"
Multipole::out_every = 256 Multipole::l_max = 8 Multipole::output_hdf5 = yes
# Disable ASCII output to avoid creating a large number of files Multipole::output_ascii = no
################################################################################
# Apparent Horizons
################################################################################
AHFinderDirect::N_horizons = 2 AHFinderDirect::find_every = 256 AHFinderDirect::output_h_every = 0 AHFinderDirect::max_Newton_iterations__initial = 50 AHFinderDirect::max_Newton_iterations__subsequent = 50 AHFinderDirect::max_allowable_Theta_growth_iterations = 10 AHFinderDirect::max_allowable_Theta_nonshrink_iterations = 10 AHFinderDirect::geometry_interpolator_name = "Lagrange
polynomial interpolation"
AHFinderDirect::geometry_interpolator_pars = "order=4" AHFinderDirect::surface_interpolator_name = "Lagrange
polynomial interpolation"
AHFinderDirect::surface_interpolator_pars = "order=4" AHFinderDirect::verbose_level = "physics
details"
AHFinderDirect::move_origins = yes
AHFinderDirect::origin_x [1] = +17.37737 AHFinderDirect::initial_guess__coord_sphere__x_center[1] = +17.37737 AHFinderDirect::initial_guess__coord_sphere__radius [1] = 0.73 AHFinderDirect::which_surface_to_store_info [1] = 2 AHFinderDirect::set_mask_for_individual_horizon [1] = no AHFinderDirect::reset_horizon_after_not_finding [1] = no AHFinderDirect::track_origin_from_grid_scalar [1] = yes AHFinderDirect::track_origin_source_x [1] =
"PunctureTracker::pt_loc_x[0]"
AHFinderDirect::track_origin_source_y [1] =
"PunctureTracker::pt_loc_y[0]"
AHFinderDirect::track_origin_source_z [1] =
"PunctureTracker::pt_loc_z[0]"
AHFinderDirect::max_allowable_horizon_radius [1] = 3
#AHFinderDirect::origin_x [2] =
-5.53846153846
#AHFinderDirect::initial_guess__coord_sphere__x_center[2] =
-5.53846153846
#AHFinderDirect::initial_guess__coord_sphere__radius [2] =
0.535384615385
#AHFinderDirect::which_surface_to_store_info [2] = 3 #AHFinderDirect::set_mask_for_individual_horizon [2] = no #AHFinderDirect::reset_horizon_after_not_finding [2] = no #AHFinderDirect::track_origin_from_grid_scalar [2] = yes #AHFinderDirect::track_origin_source_x [2] =
"PunctureTracker::pt_loc_x[1]"
#AHFinderDirect::track_origin_source_y [2] =
"PunctureTracker::pt_loc_y[1]"
#AHFinderDirect::track_origin_source_z [2] =
"PunctureTracker::pt_loc_z[1]"
#AHFinderDirect::max_allowable_horizon_radius [2] = 3
AHFinderDirect::origin_x [2] = 0 AHFinderDirect::find_after_individual [2] = 371902 AHFinderDirect::initial_guess__coord_sphere__x_center[2] = 0 AHFinderDirect::initial_guess__coord_sphere__radius [2] = 1.67 AHFinderDirect::which_surface_to_store_info [2] = 4 AHFinderDirect::set_mask_for_individual_horizon [2] = no AHFinderDirect::max_allowable_horizon_radius [2] = 6
################################################################################
# Spherical surfaces
################################################################################
SphericalSurface::nsurfaces = 4 SphericalSurface::maxntheta = 66 SphericalSurface::maxnphi = 124
# Surfaces 0 and 1 are used by PunctureTracker
# Horizon 1 SphericalSurface::ntheta [2] = 41 SphericalSurface::nphi [2] = 80 SphericalSurface::nghoststheta [2] = 2 SphericalSurface::nghostsphi [2] = 2
# Horizon 2 #SphericalSurface::ntheta [3] = 41 #SphericalSurface::nphi [3] = 80 #SphericalSurface::nghoststheta [3] = 2 #SphericalSurface::nghostsphi [3] = 2
# Common horizon SphericalSurface::ntheta [3] = 41 SphericalSurface::nphi [3] = 80 SphericalSurface::nghoststheta [3] = 2 SphericalSurface::nghostsphi [3] = 2
################################################################################
# Isolated Horizons
################################################################################
QuasiLocalMeasures::veryverbose = no QuasiLocalMeasures::interpolator = "Lagrange polynomial
interpolation"
QuasiLocalMeasures::interpolator_options = "order=4" QuasiLocalMeasures::spatial_order = 4 QuasiLocalMeasures::num_surfaces = 3 QuasiLocalMeasures::surface_index [0] = 2 QuasiLocalMeasures::surface_index [1] = 3 #QuasiLocalMeasures::surface_index [2] = 4 QuasiLocalMeasures::output_vtk_every = 0
################################################################################
# Correctness checking
################################################################################
NaNChecker::check_every = 256 NanChecker::check_after = 0 NaNChecker::report_max = 10 NaNChecker::verbose = "all" NaNChecker::action_if_found = terminate NaNChecker::out_NaNmask = yes NaNChecker::check_vars = " ML_BSSN::ML_log_confac HydroBase::w_lorentz ADMBase::curv ADMBase::metric ADMBase::lapse ADMBase::shift HydroBase::rho HydroBase::eps HydroBase::press HydroBase::vel "
################################################################################
# Timers
################################################################################
Carpet::output_timer_tree_every = 1024 Carpet::output_initialise_timer_tree = yes
################################################################################
# Output
################################################################################
IO::out_dir = "data" CarpetIOScalar::all_reductions_in_one_file = "no" CarpetIOScalar::one_file_per_group = "no" CarpetIOASCII::one_file_per_group = "no" CarpetIOHDF5::one_file_per_group = "no"
IOBasic::outInfo_every = 1 IOBasic::outInfo_reductions = "minimum maximum" IOBasic::outInfo_vars = " ML_BSSN::ML_Ham ML_BSSN::ML_log_confac Carpet::physical_time_per_hour SystemStatistics::maxrss_mb SystemStatistics::swap_used_mb Carpet::physical_time_per_hour HydroBase::rho #GRHydro::dens{reductions = 'sum maximum'} HydroBase::w_lorentz "
IOScalar::outScalar_every = 256 IOScalar::outScalar_reductions = "minimum maximum average" IOScalar::outScalar_vars = " SystemStatistics::process_memory_mb ML_BSSN::ML_Ham{ reductions = 'maximum' } "
IOASCII::out0D_every = 256 IOASCII::out0D_vars = " Carpet::physical_time_per_hour Carpet::timing PunctureTracker::pt_loc QuasiLocalMeasures::qlm_scalars{out_every = 256} "
IOASCII::out1D_every = 0 IOASCII::out1D_d = no IOASCII::out1D_x = yes IOASCII::out1D_y = no IOASCII::out1D_z = yes IOASCII::out1D_vars = " ML_BSSN::ML_log_confac ML_BSSN::ML_trace_curv WeylScal4::Psi4r "
IOASCII::out2D_every = 0 IOASCII::out2D_vars = " "
IOHDF5::out_every = 0 IOHDF5::out_vars = " Grid::Coordinates{out_every=1000000000 refinement_levels={0}} ML_BSSN::ML_log_confac WeylScal4::Psi4r WeylScal4::Psi4i WeylScal4::curvIr{refinement_levels={3 5}} WeylScal4::curvIi{refinement_levels={3 5}} WeylScal4::curvJr{refinement_levels={3 5}} WeylScal4::curvJi{refinement_levels={3 5}} "
IOHDF5::out2D_every = 64 IOHDF5::out2D_vars = " Grid::Coordinates{out_every=1000000000 refinement_levels={0}} ML_BSSN::ML_Ham ML_BSSN::alpha ML_BSSN::phi ML_BSSN::trK WeylScal4::Psi4r WeylScal4::Psi4i HydroBase::rho HydroBase::press HydroBase::w_lorentz "
################################################################################
# Checkpointing and recovery
################################################################################
#CarpetIOHDF5::checkpoint = yes IOHDF5::checkpoint = "yes" IO::checkpoint_ID = "yes" IO::recover = "autoprobe" IO::checkpoint_every_walltime_hours = 1.5 IO::checkpoint_keep=2 IO::checkpoint_on_terminate = "yes" IO::out_mode = "onefile" IO::checkpoint_dir = "../checkpoints" IO::recover_dir = "../checkpoints" IO::abort_on_io_errors = yes CarpetIOHDF5::open_one_input_file_at_a_time = yes CarpetIOHDF5::compression_level = 0
################################################################################
# Run termination
################################################################################
TerminationTrigger::max_walltime = 48 # Trigger termination 30 minutes before the walltime is reached TerminationTrigger::on_remaining_walltime = 30 TerminationTrigger::output_remtime_every_minutes = 30 TerminationTrigger::termination_from_file = yes TerminationTrigger::termination_file = "terminate.txt" TerminationTrigger::create_termination_file = yes
Cactus::terminate = time Cactus::cctk_final_time = 1900.0
Users mailing list Users@einsteintoolkit.org http://lists.einsteintoolkit.org/mailman/listinfo/users
users@lists.einsteintoolkit.org
-
Erik Schnetter -
Konrad Topolski -
Radice, David