#!/bin/sh -l
#SBATCH -J tovs1
#SBATCH -p general
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=3
#SBATCH --cpus-per-task=4
#SBATCH --mem=48G

#SBATCH --time=00:15:00

# Setup modules
module purge

module load gnu
module load xalt
module load gsl
module load openblas
module load netlib-blas-lapack
module load openmpi
module load fftw
module load hdf5

module list

# Setup variables
cd /N/u/warrenjs/Quartz/ET/Cactus
pwd

export CACTUS_NUM_PROCS=3
export CACTUS_NUM_THREADS=4

export OMP_PROC_BIND=true
export OMP_PLACES=cores
export OMP_NUM_THREADS=4

host=$(hostname)
echo ${host}

# Setup machine file on compute node
cp ./simfactory/mdb/machines/h1.quartz.uits.iu.edu.ini ./simfactory/mdb/machines/${host}.ini

echo ".ini copied"

sed -i -e "s/h1.quartz.uits.iu.edu/${host}/g" ./simfactory/mdb/machines/${host}.ini

echo "Machine file created for "${host}


# Build executable
./simfactory/bin/sim build --mdbkey make 'make -j2' --thornlist=../einsteintoolkit.th --optionlist=generic.cfg --runscript=quartz.run --submitscript=generic.sub


echo "Executable made; preparing to run"


# Run simulation
./simfactory/bin/sim create-run TOV --parfile=par/static_tov.par --ppn=12 --procs=12 --num-threads=4 --ppn-used=12 --num-smt=1

# --ppn             cores per node
# --procs           threads
# --num-threads     threads per process
# --ppn-used        threads per node
# --num-smt         threads per core