#!/bin/bash
#PBS -N nsns2
#PBS -j oe
#PBS -V
#PBS -v TEMP=/scratch1
#PBS -o job_nsns2.out
#PBS -e job_nsns2.err
#PBS -l select=2:ncpus=16
#PBS -l walltime=24:10:00
#PBS -q p-queue

export OMP_NUM_THREADS=1
cd /home2/mallick/ET/Cactus
#module load /home2/mallick/ET/Cactus/openmpi-x86_64
#cd /home2/mallick/ET/Cactus
#mkdir /scratch/mallick/sam/$PBS_JOBID
#cp * /scratch/mallick/sam/$PBS_JOBID
#cd /scratch/mallick/sam/$PBS_JOBID

CPUS=`cat $PBS_NODEFILE | wc -l`
DATE=`date +%c`
echo Job started at $DATE
#export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib
module load /home2/mallick/ET/Cactus/openmpi-x86_64

#/home2/mallick/ET/Cactus/simfactory/bin/sim setup-silent

#time /opt/intel/impi/4.1.0.024/intel64/bin/mpiexec.hydra 

#time /opt/mpi/openmpi/1.6.2/intel/bin/mpirun
/home2/mallick/ET/Cactus/simfactory/bin/sim whoami

time mpirun /home2/mallick/ET/Cactus/simfactory/bin/sim create-submit nsns30 --basedir=/home2/mallick/simulations --procs=32 --ppn=16 --num-threads=1 --num-smt=1 --ppn-used=16 --parfile /home2/mallick/ET/Cactus/parfile/nsns_vlr_mass_diff.par --walltime=24:00:00

DATE=`date +%c`
echo Job finished at $DATE

#cp * $PBS_O_WORKDIR
#rm -rf /scratch/sam/$PBS_JOBID