#! /usr/bin/perl -w use strict; # User options my $version = '3'; # Increase this after every release my $path = "bench"; # output directory my $memory = 500.0e+6; # bytes per core my $walltime = 300.0; # seconds my @levels_list = (9); # (1, 9); # count my @multipatch_list = (0); # 0 or 1 my @hydro_list = (0); # (0, 1); # 0 or 1 my $mincores = 12; # probably cores per node my $maxcores = 1020; # 4; # some large upper limit my @threads_list = (1); # (1, 2, 4); # good numbers of threads my @run_list = (5); # identifier to distinguish benchmark series # Loop over user parameters my @cores_list = (); foreach my $power (0..30) { my $cores = $mincores * (1 << $power); last if $cores>$maxcores; push @cores_list, $cores; } mkdir $path; open SUBMIT, ">", "$path/submit.sh"; print SUBMIT "#! /bin/bash\n"; print SUBMIT "\n"; print SUBMIT "# This script contains Simulation Factory commands to submit all benchmarks\n"; print SUBMIT "\n"; foreach our $hydro (@hydro_list) { foreach our $multipatch (@multipatch_list) { next if $hydro && $multipatch; foreach our $levels (@levels_list) { foreach our $cores (@cores_list) { # Basic settings my $maxlevels = 10; my $minsize = $hydro ? 10.0 : 1.0; # interior of finest grid my $numvars = $hydro ? 35 : 25; # BSSN / GRHydro my $timelevels = 5; # RK4 my $bytes_per_var = 8; # double my $ghosts = 3; # 4th order my $memory_overhead = 1.5; # estimate my $amr_overhead = 1.0 + 0.5 * ($levels-1); # estimate my $cpu_efficiency = 0.1; # estimate my $cpu_frequency = 2.0e+9; # probably my $ops_per_cycle = 4; # probably my $ops_per_point = $hydro ? 10000 : 5000; # BSSN / GRHydro my $steps_per_iter = 4; # RK4 my $buffers = $ghosts * ($steps_per_iter - 1); # Determine grid size # Bytes per core per grid point our $bytes_per_point = 1.0 * $timelevels * $numvars * $bytes_per_var * $memory_overhead / $cores; sub memory($); sub memory($) { my ($gridsize) = @_; # TODO: take additional overlap points into account my $npoints = ($gridsize / $cores**(1.0/3.0) + 2*$ghosts)**3 * $cores; return $bytes_per_point * $npoints * ($levels + 6*$multipatch); } our $gridsize = $buffers + 1; my $real_memory = 0.0; while (($real_memory = memory $gridsize) < $memory) { ++ $gridsize; } # Determine iterations our $seconds_per_point = 1.0 * $ops_per_point * $steps_per_iter * $amr_overhead / ($cpu_efficiency * $ops_per_cycle * $cpu_frequency) / $cores; sub walltime($); sub walltime($) { my ($iteration) = @_; # TODO: take additional overlap points into account my $npoints = $gridsize**3; my $count = 0; foreach my $level (0..$levels-1) { if ($iteration % (1 << ($maxlevels - $level - 1)) == 0) { $count += $npoints; if ($multipatch && $level==0) { $count += 6*$npoints; } } } return $seconds_per_point * $count; } our $iterations = 0; my $real_walltime = 0.0; while (($real_walltime += walltime $iterations) < $walltime) { ++ $iterations; } ++ $iterations; # Dependent parameters my $filename = "bench". "-".($hydro ? "tov" : "minkowski"). "-".($multipatch ? "mp" : "amr"). "-".sprintf("lev%02d",$levels). "-".sprintf("grid%06d",$gridsize). "-".sprintf("cores%06d",$cores). "-".sprintf("iter%06d",$iterations); foreach my $threads (@threads_list) { if ($cores % $threads == 0) { foreach my $run (@run_list) { print SUBMIT "./bin/sim submit $filename-proc".sprintf("%06d",$cores)."-thr".sprintf("%04d",$threads)."-run".sprintf("%04d",$run)." --parfile=$path/$filename.par --procs=$cores --num-threads=$threads --walltime=0:59:0\n"; } } } print "\n"; print "\n"; print "\n"; print "Benchmark parameters:\n"; print "\n"; print " Physics: ".($hydro ? "TOV" : "Minkowski")."\n"; print " Grid: ".($multipatch ? "multi-block" : "AMR")."\n"; print " Cores: $cores\n"; print " Total grid points: $gridsize^3\n"; print " Iterations: $iterations\n"; print "\n"; print " Memory requested [bytes]: $memory\n"; print " Memory used [bytes]: $real_memory (".sprintf("%.1f",100.0*$real_memory/$memory)."%)\n"; print " Walltime requested [seconds]: $walltime\n"; print " Walltime real [seconds]: $real_walltime (".sprintf("%.1f",100.0*$real_walltime/$walltime)."%)\n"; print "\n"; print " Parameter file name: $filename.par\n"; open FILE, ">", "$path/$filename.par"; sub p($); sub p($) { my ($line) = @_; print FILE "$line\n"; } p 'Cactus::cctk_run_title = "Einstein Toolkit Benchmark version '.($version).'"'; p ''; p '# Target requirements:'; p '# Physics: '.($hydro ? 'TOV' : 'Minkowski'); p '# Refinement levels: '.($levels); p '# Memory per core: '.($memory).' bytes'; p '# Wall time: '.($walltime).' seconds'; p '# Number of cores: '.($cores); p '# Estimated actual requirements:'; p '# Memory per core: '.($real_memory).' bytes'; p '# Wall time: '.($real_walltime).' seconds'; p '# Specifications:'; p '# Grid size: '.($gridsize).'^3'; p '# Iterations: '.($iterations); p ''; p ''; p ''; p 'Cactus::cctk_full_warnings = yes'; p 'Cactus::highlight_warning_messages = no'; p ''; p 'Cactus::cctk_itlast = '.($iterations); p ''; p ''; p ''; p 'ActiveThorns = "IOUtil"'; p ''; p 'IO::out_dir = $parfile'; p ''; p ''; if ($hydro) { p ''; p 'ActiveThorns = "Constants"'; } p ''; p 'ActiveThorns = "GenericFD"'; p ''; p 'ActiveThorns = "InitBase"'; p ''; p 'ActiveThorns = "LoopControl"'; p ''; p 'LoopControl::legacy_init = no'; p 'LoopControl::use_random_restart_hill_climbing = no'; p ''; p ''; p ''; p 'ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetInterp2 CarpetReduce"'; p ''; p 'Carpet::verbose = no'; p 'Carpet::veryverbose = no'; p 'Carpet::schedule_barriers = yes # no'; p 'Carpet::storage_verbose = no'; p 'Carpet::timers_verbose = no'; p 'CarpetLib::output_bboxes = no'; p ''; if (! $multipatch) { p 'Carpet::domain_from_coordbase = yes'; } else { p 'Carpet::domain_from_multipatch = yes'; } p 'Carpet::max_refinement_levels = '.($maxlevels); p ''; p 'driver::ghost_size = '.($ghosts); p 'Carpet::use_buffer_zones = yes'; p ''; p 'Carpet::prolongation_order_space = 5'; p 'Carpet::prolongation_order_time = 2'; p ''; p 'Carpet::init_fill_timelevels = yes'; p ''; p 'Carpet::poison_new_timelevels = yes'; p 'CarpetLib::poison_new_memory = yes'; p ''; p 'Carpet::output_timers_every = '.($iterations).' # 0'; p 'CarpetLib::print_timestats_every = '.($iterations).' # 0'; p 'CarpetLib::print_memstats_every = 0'; p ''; p ''; p ''; if (! $multipatch) { p 'ActiveThorns = "Boundary CartGrid3D CoordBase ReflectionSymmetry SymBase"'; p ''; p 'CoordBase::domainsize = "minmax"'; p 'CoordBase::spacing = "numcells"'; p ''; p 'CoordBase::xmin = 0.0'; p 'CoordBase::ymin = 0.0'; p 'CoordBase::zmin = 0.0'; # Add buffers explicitly to coarse grid extent, so that all levels # have the same number of active grid points my $coarsesize = 1.0 * $minsize / ($gridsize - $buffers) * $gridsize * (1 << ($levels - 1)); p 'CoordBase::xmax = '.($coarsesize); p 'CoordBase::ymax = '.($coarsesize); p 'CoordBase::zmax = '.($coarsesize); p 'CoordBase::ncells_x = '.($gridsize); p 'CoordBase::ncells_y = '.($gridsize); p 'CoordBase::ncells_z = '.($gridsize); p ''; p 'CoordBase::boundary_shiftout_x_lower = 1'; p 'CoordBase::boundary_shiftout_y_lower = 1'; p 'CoordBase::boundary_shiftout_z_lower = 1'; p 'CoordBase::boundary_size_x_lower = 3'; p 'CoordBase::boundary_size_y_lower = 3'; p 'CoordBase::boundary_size_z_lower = 3'; p 'CoordBase::boundary_size_x_upper = 3'; p 'CoordBase::boundary_size_y_upper = 3'; p 'CoordBase::boundary_size_z_upper = 3'; p ''; p 'CartGrid3D::type = "coordbase"'; p ''; p 'ReflectionSymmetry::reflection_x = yes'; p 'ReflectionSymmetry::reflection_y = yes'; p 'ReflectionSymmetry::reflection_z = yes'; p 'ReflectionSymmetry::avoid_origin_x = no'; p 'ReflectionSymmetry::avoid_origin_y = no'; p 'ReflectionSymmetry::avoid_origin_z = no'; } else { p 'ActiveThorns = "Boundary CartGrid3D CoordBase Coordinates Interpolate2 SymBase"'; p ''; my $coarsesize = 1.0 * $minsize / ($gridsize - $buffers) * $gridsize * (1 << ($levels - 1)); p 'Coordinates::coordinate_system = "Thornburg04"'; p 'Coordinates::h_cartesian = '.(1.0*$coarsesize/($gridsize-1)); p 'Coordinates::h_radial = '.(2.0*$coarsesize/($gridsize-1)); p ''; p 'Coordinates::sphere_inner_radius = '.(0.5*$coarsesize); p 'Coordinates::sphere_outer_radius = '.(2.5*$coarsesize); p 'Coordinates::n_angular = '.($gridsize-1); p ''; p 'Coordinates::patch_boundary_size = 3'; p 'Coordinates::additional_overlap_size = 3'; p 'Coordinates::outer_boundary_size = 3'; p ''; p 'Interpolate::interpolator_order = 4'; p 'CarpetInterp::tree_search = yes'; p 'CarpetInterp::check_tree_search = no'; p ''; p 'CartGrid3D::type = "multipatch"'; p 'CartGrid3D::set_coordinate_ranges_on = "all maps"'; } p ''; p ''; p ''; p 'ActiveThorns = "CarpetRegrid2"'; p ''; p 'CarpetRegrid2::regrid_every = 0'; p 'CarpetRegrid2::num_centres = 1'; p ''; p 'CarpetRegrid2::num_levels_1 = '.($levels); foreach my $level (1..$levels-1) { my $levelsize = 1.0 * $minsize * (1 << ($levels - $level - 1)); p 'CarpetRegrid2::radius_1['.($level).'] = '.($levelsize); } p ''; p ''; p ''; p 'ActiveThorns = "SphericalSurface"'; p ''; p ''; p ''; p 'ActiveThorns = "MoL Time"'; p ''; p 'MoL::ODE_Method = "RK4"'; p 'MoL::MoL_Intermediate_Steps = 4'; p 'MoL::MoL_Num_Scratch_Levels = 1'; p ''; p 'Time::dtfac = 0.25'; p ''; p ''; p ''; p 'ActiveThorns = "ADMBase ADMCoupling ADMMacros CoordGauge SpaceMask StaticConformal TmunuBase"'; if ($hydro) { p ''; p ''; p ''; p 'Activethorns = "HydroBase"'; p ''; p 'HydroBase::timelevels = 3'; p ''; p 'TmunuBase::stress_energy_storage = yes'; p 'TmunuBase::stress_energy_at_RHS = yes'; p 'TmunuBase::timelevels = 1'; p 'TmunuBase::prolongation_type = "none"'; p 'TmunuBase::support_old_CalcTmunu_mechanism = no'; p ''; p 'SpaceMask::use_mask = yes'; } p ''; p ''; p ''; my $bssn = $multipatch ? 'ML_BSSN_MP' : 'ML_BSSN'; p 'ActiveThorns = "'.($bssn).' '.($bssn).'_Helper NewRad"'; if (! $hydro) { p ''; p 'ADMBase::initial_data = "Cartesian Minkowski"'; p 'ADMBase::initial_lapse = "one"'; p 'ADMBase::initial_shift = "zero"'; p 'ADMBase::initial_dtlapse = "zero"'; p 'ADMBase::initial_dtshift = "zero"'; } p ''; p 'ADMBase::evolution_method = "'.($bssn).'"'; p 'ADMBase::lapse_evolution_method = "'.($bssn).'"'; p 'ADMBase::shift_evolution_method = "'.($bssn).'"'; p 'ADMBase::dtlapse_evolution_method = "'.($bssn).'"'; p 'ADMBase::dtshift_evolution_method = "'.($bssn).'"'; p ''; p ''.($bssn).'::timelevels = 3'; p ''; p ''.($bssn).'::harmonicF = 2.0 # 1+log'; p ''.($bssn).'::harmonicN = 1 # 1+log'; p ''.($bssn).'::ShiftGammaCoeff = 0.75'; p ''.($bssn).'::LapseACoeff = 1'; p ''.($bssn).'::ShiftBCoeff = 1'; p ''.($bssn).'::AlphaDriver = 1.0'; p ''.($bssn).'::BetaDriver = 1.0'; p ''.($bssn).'::LapseAdvectionCoeff = 1.0'; p ''.($bssn).'::ShiftAdvectionCoeff = 1.0'; if ($hydro) { p ''.($bssn).'::UseSpatialBetaDriver = yes'; p ''.($bssn).'::SpatialShiftGammaCoeffRadius = 20.0'; } p ''; p ''.($bssn).'::EpsDiss = 0.2'; p ''; p ''.($bssn).'::my_initial_boundary_condition = "extrapolate-gammas"'; p ''.($bssn).'::my_rhs_boundary_condition = "NewRad"'; p 'Boundary::radpower = 2'; p ''; p ''.($bssn).'::MinimumLapse = 1.0e-8'; if ($hydro) { p ''; p ''; p ''; p 'ActiveThorns = "EOS_Omni GRHydro"'; p ''; p 'HydroBase::evolution_method = "GRHydro"'; p ''; p 'EOS_Omni::poly_gamma = 2.0'; p 'EOS_Omni::poly_k = 100.0'; p ''; p 'GRHydro::Riemann_solver = "Marquina"'; p 'GRHydro::GRHydro_EOS_type = "Polytype"'; p 'GRHydro::GRHydro_EOS_table = "2D_Polytrope"'; p 'GRHydro::recon_method = "ppm"'; p 'GRHydro::GRHydro_stencil = 3'; p 'GRHydro::bound = "none"'; p 'GRHydro::rho_abs_min = 1.e-10'; p ''; p ''; p ''; p 'ActiveThorns = "TOVSolver"'; p ''; p 'ADMBase::initial_data = "TOV"'; p 'ADMBase::initial_lapse = "TOV"'; p 'ADMBase::initial_shift = "zero"'; p 'ADMBase::initial_dtlapse = "zero"'; p 'ADMBase::initial_dtshift = "zero"'; p ''; p 'TOVSolver::TOV_Rho_Central[0] = 1.28e-3'; p 'TOVSolver::TOV_Gamma[0] = 2.0'; p 'TOVSolver::TOV_K[0] = 100.0'; } p ''; p ''; p ''; p 'ActiveThorns = "CarpetIOBasic"'; p ''; p 'IOBasic::outInfo_every = 1'; p 'IOBasic::outInfo_reductions = "norm2"'; p 'IOBasic::outInfo_vars = "'; p ' Carpet::physical_time_per_hour'; p ' Carpet::local_grid_points_per_second'; p '"'; p ''; p ''; p ''; p 'ActiveThorns = "CarpetIOScalar"'; p ''; p 'IOScalar::one_file_per_group = yes'; p ''; p 'IOScalar::outScalar_every = '.($iterations); p 'IOScalar::outScalar_vars = "'; p ' ADMBase::lapse'; if ($hydro) { p ' HydroBase::rho'; } p '"'; p ''; p ''; p ''; p 'ActiveThorns = "CarpetIOASCII"'; p ''; p 'IOASCII::one_file_per_group = yes'; p ''; p 'IOASCII::out3D_ghosts = no'; p ''; p 'IOASCII::out0D_every = '.($iterations); p 'IOASCII::out0D_vars = "'; p ' Carpet::timing'; p '"'; p ''; p 'IOASCII::out1D_every = '.($iterations); p 'IOASCII::out1D_d = no'; p 'IOASCII::out1D_vars = "'; if ($multipatch) { p ' grid::coordinates'; } p ' ADMBase::lapse'; p ' '.($bssn).'::ML_lapse'; if ($hydro) { p ' HydroBase::rho'; p ' GRHydro::dens'; } p '"'; p ''; p ''; p ''; p 'ActiveThorns = "NaNChecker"'; p ''; p 'NaNChecker::check_every = 1'; p 'NaNChecker::action_if_found = "terminate"'; p 'NaNChecker::check_vars = "'; p ' ADMBase::lapse'; if ($hydro) { p ' HydroBase::rho'; } p '"'; p ''; p ''; p ''; p 'ActiveThorns = "Formaline"'; p ''; p ''; p ''; p 'ActiveThorns = "TimerReport"'; p ''; p 'TimerReport::out_every = '.($iterations); p '#TimerReport::out_filename = "TimerReport"'; p 'TimerReport::output_all_timers_together = yes'; p 'TimerReport::output_all_timers_readable = yes'; p 'TimerReport::n_top_timers = 100'; close FILE; } # End of loops over user parameters } } } close SUBMIT;