#! /bin/bash

echo "Preparing:"
set -euxo pipefail

cd @RUNDIR@-active

echo 'Job setup:'
echo '   Allocated:'
echo '      Nodes:                      @NODES@'
echo '      Cores per node:             @PPN@'
echo '   Running:'
echo '      MPI processes:              @NUM_PROCS@'
echo '      OpenMP threads per process: @NUM_THREADS@'
echo '      MPI processes per node:     @NODE_PROCS@'
echo '      OpenMP threads per core:    @NUM_SMT@'
echo '      OpenMP threads per node:    @PPN_USED@'

echo "Checking:"
date
env
hostname
pwd

scontrol show hostnames
hostfile="SIMFACTORY/NODES"
scontrol show hostnames |
    awk '{ print $1, "slots=@NODE_PROCS@"; }' >"$hostfile" 

/sbin/ifconfig || true

VIEW="@SOURCEDIR@/view-cuda"
"${VIEW}/bin/ompi_info" >SIMFACTORY/ompi_info.txt &

HPCTOOLKIT_DIR="${VIEW}/../../spack/opt/spack/linux-ubuntu18.04-skylake_avx512/gcc-10.2.0/hpctoolkit-2021.03.01-ed7bgg5yoc2uh56xwtbeinoialkbv4r5"
"${HPCTOOLKIT_DIR}/bin/hpcrun" -L >SIMFACXTORY/hpcrun.txt &

case @NUM_SMT@ in
    (1)
        case @NUM_THREADS@ in
            (1) bind_to=core; map_by=ppr:1:core;;
            (2) bind_to=socket; map_by=ppr:8:socket;;
            (4) bind_to=socket; map_by=ppr:4:socket;;
            (8) bind_to=socket; map_by=ppr:2:socket;;
            (16) bind_to=socket; map_by=ppr:1:socket;;
            (*) bind_to=none; map_by=node;;
        esac;;
    (2)
        case @NUM_THREADS@ in
            (1) bind_to=core; map_by=ppr:2:core;;
            (2) bind_to=core; map_by=ppr:1:core;;
            (4) bind_to=socket; map_by=ppr:8:socket;;
            (8) bind_to=socket; map_by=ppr:4:socket;;
            (16) bind_to=socket; map_by=ppr:2:socket;;
            (32) bind_to=socket; map_by=ppr:1:socket;;
            (*) bind_to=none; map_by=node;;
        esac;;
    (*)
        bind_to=none; map_by=node;;
esac

echo "Environment:"
export 'SIMULATION_ID=@SIMULATION_ID@'
export 'CACTUS_MAX_MEMORY=@(@MEMORY@ * 1024)@' # Byte
export 'CACTUS_NUM_PROCS=@NUM_PROCS@'
export 'CACTUS_NUM_THREADS=@NUM_THREADS@'
export 'CACTUS_SET_THREAD_BINDINGS=1'
export 'CXX_MAX_TASKS=500'
# export 'GLIBCXX_FORCE_NEW=1'
export 'GMON_OUT_PREFIX=gmon.out'
export 'OMP_DISPLAY_ENV=FALSE'  # false, true
export 'OMP_MAX_TASKS=500'
export 'OMP_NUM_THREADS=@NUM_THREADS@'
export 'OMP_PLACES=cores'       # threads, cores, sockets
export 'OMP_PROC_BIND=FALSE'    # false, true, master, close, spread
export 'OMP_STACKSIZE=8192'     # kByte
env | sort >'SIMFACTORY/ENVIRONMENT'

# cat >runperf.sh <<'EOF'
# #!/bin/bash
# exec "$@"
# # which perf
# # perf --version
# # exec perf record -g -o "perf.data.$OMPI_COMM_WORLD_RANK" "$@"
# # exec perf record -g -e L1-icache-load-misses -o "perf.data.$OMPI_COMM_WORLD_RANK" "$@"
# # exec perf record -g -e L1-dcache-load-misses -o "perf.data.$OMPI_COMM_WORLD_RANK" "$@"
# # exec perf record -g -e LLC-load-misses -o "perf.data.$OMPI_COMM_WORLD_RANK" "$@"
# EOF
# chmod a+x runperf.sh
# sleep 10                        # GPFS is sometimes slow

# ulimit -c unlimited

echo "Starting:"
date
export CACTUS_STARTTIME=$(date +%s)

time                                            \
    "${VIEW}/bin/mpiexec"                       \
    --prefix "${VIEW}"                          \
    --mca oob_tcp_if_include eth0               \
    --mca btl_tcp_if_include eth0               \
    --hostfile "${hostfile}"                    \
    --n @NUM_PROCS@                             \
    --map-by "${map_by}"                        \
    --display-map                               \
    --bind-to "${bind_to}"                      \
    --report-bindings                           \
    "${HPCTOOLKIT_DIR}/bin/hpcrun"              \
    -o hpctoolkit-measurements                  \
    -e REALTIME                                 \
    -e gpu=nvidia                               \
    -t                                          \
    "@EXECUTABLE@"                              \
    -L 3                                        \
    "@PARFILE@"                                 \
    >stdout.txt                                 \
    2>stderr.txt

echo "Stopping:"
date

# echo time "${HPCTOOLKIT_DIR}/bin/hpcstruct" -j $(nproc) --gpucfg yes "@EXECUTABLE@"
# echo time "${HPCTOOLKIT_DIR}/bin/hpcstruct" -j $(nproc) --gpucfg yes hpctoolkit-measurements
# echo time "${VIEW}/bin/mpiexec" --n $(nproc) "${HPCTOOLKIT_DIR}/bin/hpcprof" --structure $(basename "@EXECUTABLE@").hpcstruct -o hpctoolkit-database hpctoolkit-measurements

echo time "${HPCTOOLKIT_DIR}/bin/hpcstruct" -j $(nproc) --gpucfg yes "@EXECUTABLE@"
echo time "${HPCTOOLKIT_DIR}/bin/hpcstruct" --gpucfg yes hpctoolkit-measurements
echo time "${HPCTOOLKIT_DIR}/bin/hpcprof" --structure $(basename "@EXECUTABLE@").hpcstruct -o hpctoolkit-database hpctoolkit-measurements

echo "Done."