#!/bin/bash

set -x
set -e

cd @SIMULATION_DIR@

# Environment setup
source /opt/rh/devtoolset-8/enable

module purge
module load lamod/cmake/3.17
module load lamod/fftw/gnu/3.3.8
module load lamod/openmpi/gnu/4.1.0
module load libraries/gsl/2.6_gnu
module load libraries/hdf5/1.10.5_gnu

echo "Environment diagnostics:"
date
hostname
env
ldd @EXECUTABLE@ | grep -E "lapack|blas|openssl|stdc++|gfortran"

# Set runtime parameters
export CACTUS_NUM_PROCS=64
export CACTUS_NUM_THREADS=1
export OMP_NUM_THREADS=1
export GMON_OUT_PREFIX=gmon.out
env | sort > SIMFACTORY/ENVIRONMENT

echo "Starting simulation at $(date)"
export CACTUS_STARTTIME=$(date +%s)

mpirun -np $CACTUS_NUM_PROCS -x LD_LIBRARY_PATH @EXECUTABLE@ -L 3 @PARFILE@
#srun --mpi=pmi2 --export=ALL,LD_LIBRARY_PATH @EXECUTABLE@ -L 3 @PARFILE@

echo "Simulation finished at $(date)"
touch segment.done