#!  /bin/bash


# This machine description file is used internally by simfactory as a template
# during the sim setup and sim setup-silent commands
# Edit at your own risk
# Machine description
#last tested 1-11-2022
#last edited 23-10-2022
#tested by spandan spandan19@iiserb.ac.in

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
cat ${PBS_NODEFILE} > SIMFACTORY/NODES

echo "Environment:"
export CACTUS_NUM_PROCS=@NUM_PROCS@
export CACTUS_NUM_THREADS=@NUM_THREADS@
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@
env | sort > SIMFACTORY/ENVIRONMENT

echo "Job setup:"
echo "   Allocated:"
echo "      Nodes:                      @NODES@"
echo "      Cores per node:             @PPN@"
echo "   Running:"
echo "      MPI processes:              @NUM_PROCS@"
echo "      OpenMP threads per process: @NUM_THREADS@"
echo "      MPI processes per node:     @NODE_PROCS@"
echo "      OpenMP threads per core:    @(1.0*(@NODE_PROCS@*@NUM_THREADS@)/@PPN@)@"
echo "      OpenMP threads per node:    @(@NODE_PROCS@*@NUM_THREADS@)@"

echo "Starting:"
#echo "showing all modules in the current node ============"
#module avail
#locate 3.1.4
#echo "showing all modules ========================="

export CACTUS_STARTTIME=$(date +%s)
CPUS=`cat $PBS_NODEFILE | wc -l`
DATE=`date +%c`
echo Job started at $DATE

#if [ @RUNDEBUG@ -eq 0 ]; then
    # TODO: check if one needs to manually prune the hostfile so that running
    # with 1 taks per node works otherwise OpenMPI is sometimes picky with
    # ranks and sockets
#    mpirun -np @NUM_PROCS@ --map-by numa:PE=@NUM_THREADS@ @EXECUTABLE@ -L 3 @PARFILE@
#else
#    gdb --args @EXECUTABLE@ -L 3 @PARFILE@
#fi

time mpiexec -n @NUM_PROCS@ -npernode @(@PPN_USED@ / @NUM_THREADS@)@ @EXECUTABLE@ -L 3 @PARFILE@


echo "Stopping:"
date
DATE=`date +%c`
echo Job finished at $DATE