#! /bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date

echo "Environment:"
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@
export CACTUS_NUM_PROCS=@NUM_PROCS@
export CACTUS_NUM_THREADS=@NUM_THREADS@
env > SIMFACTORY/ENVIRONMENT

export MPI_PROCESS_PER_NODE=@(1.0*@NUM_PROCS@/@NODES@)@
export MPI_PROCESS=@NUM_PROCS@

echo "Job setup:"
echo "   Allocated:"
echo "      Nodes:                      @NODES@"
echo "      Cores per node:             @PPN@"
echo "   SLURM setting"
echo "      SLURM_NNODES :  ${SLURM_NNODES}"
echo "      SLURM_NPROCS :  ${SLURM_NPROCS}"
echo "      SLURM_NTASKS :  ${SLURM_NTASKS}"
echo "      SLURM_CPUS_ON_NODE  :  ${SLURM_CPUS_ON_NODE}"
echo "      SLURM_CPUS_PER_TASK :  ${SLURM_CPUS_PER_TASK}"
echo "      SLURM_TASKS_PER_NODE:  ${SLURM_TASKS_PER_NODE}"
echo "   Running:"
echo "      MPI processes:              @NUM_PROCS@"
echo "      OpenMP threads per process: @NUM_THREADS@"
echo "      MPI processes per node:     @(1.0*@NUM_PROCS@/@NODES@)@"
echo "      OpenMP threads per core:    @(1.0*(@NUM_PROCS@*@NUM_THREADS@)/(@NODES@*@PPN@))@"
echo "      OpenMP threads per node:    @PPN_USED@"

export I_MPI_DEBUG=5

###ulimit -c unlimited
###export decfort_dump_flag=TRUE

echo "Starting:"
 
mpirun @EXECUTABLE@ -L 1 @PARFILE@  

echo "Stopping:"
date

echo "Done."