#!/bin/bash

echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors


cd @RUNDIR@-active

echo "Checking:"
pwd
hostname
date
cat ${PBS_NODEFILE} > SIMFACTORY/NODES

echo "Environment:"
export PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:$PATH
export LD_LIBRARY_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:$LD_LIBRARY_PATH
export C_INCLUDE_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:$C_INCLUDE_PATH

export PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:$PATH
export LD_LIBRARY_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:$LD_LIBRARY_PATH
export C_INCLUDE_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:$C_INCLUDE_PATH

export CACTUS_NUM_PROCS=@NUM_PROCS@
export CACTUS_NUM_THREADS=@NUM_THREADS@
export MP_SINGLE_THREAD=no
export OMP_NUM_THREADS=@NUM_THREADS@
##export MP_TASK_AFFINITY=core:$OMP_NUM_TRHEADS
#### PBS debugging information

env | sort > SIMFACTORY/ENVIRONMENT

echo "Job setup:"
echo "   Allocated:"
echo "      Nodes:                      @NODES@"
echo "      Cores per node:             @PPN@"
echo "   Running:"
echo "      MPI processes:              @NUM_PROCS@"
echo "      OpenMP threads per process: @NUM_THREADS@"
echo "      MPI processes per node:     @(1.0*@NUM_PROCS@/@NODES@)@"
echo "      OpenMP threads per core:    @(1.0*(@NUM_PROCS@*@NUM_THREADS@)/(@NODES@*@PPN@))@"
echo "      OpenMP threads per node:    @PPN_USED@"

ulimit -Sa
ulimit -Ha
ulimit -s unlimited

echo "Starting:"
echo "Chris test:"
export CACTUS_STARTTIME=$(date +%s)

time mpirun --mca orte_base_help_aggregate 0 -np @NUM_PROCS@ @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date

echo "Done."