#!/bin/sh

echo "Running apollo.run"
echo ""
echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors

echo ""

echo "Checking:"
pwd
hostname
date

echo ""

echo "Environment:"
export CACTUS_NUM_PROCS=@NUM_PROCS@
export CACTUS_NUM_THREADS=@NUM_THREADS@
export OMP_NUM_THREADS=@NUM_THREADS@
export GMON_OUT_PREFIX=gmon.out
export MPIDIR=/cm/shared/apps/mpich/ge/gcc/64/3.0.4

echo "Job setup:"
echo "   Allocated:"
echo "      Nodes:                      @NODES@"
echo "      Cores per node:             @PPN@"
echo "   Running:"
echo "      MPI processes:              @NUM_PROCS@"
echo "      OpenMP threads per process: @NUM_THREADS@"
echo "      OpenMP threads per node:    @PPN_USED@"

echo ""

#if [ ! -z "$MPI_NODEFILE" ]; then
#    cat ${MPI_NODEFILE} > SIMFACTORY/NODES
#    export MPI_NODEFILE=${TMPDIR}/machines
#
#    uniq ${MPI_NODEFILE} > PROC_NODES
#
#    for node in $(cat PROC_NODES); do
#        for (( proc=0; $proc<@(@PPN_USED@/@NUM_THREADS@)@; proc=$proc+1)); do
#            echo ${node}
#        done
#    done > ${MPI_NODEFILE}
#fi

env | sort > SIMFACTORY/ENVIRONMENT

echo ""
export CACTUS_STARTTIME=$(date +%s)
echo ""

mpirun -np @NUM_PROCS@ -cpus-per-proc @NUM_THREADS@ @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date