+ set -e
+ cd /mnt/lustre/users/cstevens/CCM/simulations/mctest/output-0000-active
+ echo Checking:
+ pwd
+ hostname
+ date
+ cat /var/spool/PBS/aux/451664.sched01
+ echo Environment:
+ export PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/libs/hdf5/1.8.16/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/libs/openmpi/1.8.8-intel16.0.1/bin:/apps/chpc/bio/anaconda2/bin:/apps/chpc/bio/lib/gnu/gsl/2.1/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../clck_latest/bin/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../itac_latest/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../advisor_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../inspector_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../vtune_amplifier_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../debugger_2016/gdb/intel64_mic/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/intel64/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/bin/intel64:/apps/chpc/compmech/gnuplot-dumb-5.0.3/bin:/apps/chpc/bio/git/2.8.1/bin:/opt/pbs/default/sbin/:/opt/pbs/default/bin/:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ibutils/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/3.2.10/bin:/opt/dell/srvadmin/bin:/home/cstevens/.local/bin:/home/cstevens/bin:/home/cstevens/.local/bin:/home/cstevens/bin
+ PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/libs/hdf5/1.8.16/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/libs/openmpi/1.8.8-intel16.0.1/bin:/apps/chpc/bio/anaconda2/bin:/apps/chpc/bio/lib/gnu/gsl/2.1/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../clck_latest/bin/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../itac_latest/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../advisor_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../inspector_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../vtune_amplifier_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../debugger_2016/gdb/intel64_mic/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/intel64/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/bin/intel64:/apps/chpc/compmech/gnuplot-dumb-5.0.3/bin:/apps/chpc/bio/git/2.8.1/bin:/opt/pbs/default/sbin/:/opt/pbs/default/bin/:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ibutils/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/3.2.10/bin:/opt/dell/srvadmin/bin:/home/cstevens/.local/bin:/home/cstevens/bin:/home/cstevens/.local/bin:/home/cstevens/bin
+ export LD_LIBRARY_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/libs/hdf5/1.8.16/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/libs/openmpi/1.8.8-intel16.0.1/lib:/apps/chpc/bio/lib/gnu/gsl/2.1/lib:/apps/libs/zlib/1.2.8/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../clck_latest/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../itac_latest/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../inspector_xe/lib64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../vtune_amplifier_xe/lib64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/../compiler/lib/intel64_lin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/../tbb/lib/intel64_lin/gcc4.4:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/lib/intel64_lin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../debugger_2016/libipt/intel64/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/tbb/lib/intel64/gcc4.4:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mkl/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/ipp/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/mic/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/intel64/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/compiler/lib/intel64:/apps/chpc/compmech/gnuplot-dumb-5.0.3/lib:/apps/chpc/bio/git/2.8.1/lib64:/opt/pbs/default/lib/
+ LD_LIBRARY_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/libs/hdf5/1.8.16/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/libs/openmpi/1.8.8-intel16.0.1/lib:/apps/chpc/bio/lib/gnu/gsl/2.1/lib:/apps/libs/zlib/1.2.8/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../clck_latest/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../itac_latest/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../inspector_xe/lib64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../vtune_amplifier_xe/lib64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/../compiler/lib/intel64_lin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/../tbb/lib/intel64_lin/gcc4.4:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/lib/intel64_lin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../debugger_2016/libipt/intel64/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/tbb/lib/intel64/gcc4.4:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mkl/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/ipp/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/mic/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/intel64/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/compiler/lib/intel64:/apps/chpc/compmech/gnuplot-dumb-5.0.3/lib:/apps/chpc/bio/git/2.8.1/lib64:/opt/pbs/default/lib/
+ export C_INCLUDE_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:
+ C_INCLUDE_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:
+ export PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/libs/hdf5/1.8.16/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/libs/openmpi/1.8.8-intel16.0.1/bin:/apps/chpc/bio/anaconda2/bin:/apps/chpc/bio/lib/gnu/gsl/2.1/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../clck_latest/bin/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../itac_latest/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../advisor_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../inspector_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../vtune_amplifier_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../debugger_2016/gdb/intel64_mic/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/intel64/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/bin/intel64:/apps/chpc/compmech/gnuplot-dumb-5.0.3/bin:/apps/chpc/bio/git/2.8.1/bin:/opt/pbs/default/sbin/:/opt/pbs/default/bin/:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ibutils/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/3.2.10/bin:/opt/dell/srvadmin/bin:/home/cstevens/.local/bin:/home/cstevens/bin:/home/cstevens/.local/bin:/home/cstevens/bin
+ PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/libs/hdf5/1.8.16/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/bin:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/bin:/apps/libs/openmpi/1.8.8-intel16.0.1/bin:/apps/chpc/bio/anaconda2/bin:/apps/chpc/bio/lib/gnu/gsl/2.1/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../clck_latest/bin/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../itac_latest/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../advisor_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../inspector_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../vtune_amplifier_xe/bin64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../debugger_2016/gdb/intel64_mic/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/intel64/bin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/bin/intel64:/apps/chpc/compmech/gnuplot-dumb-5.0.3/bin:/apps/chpc/bio/git/2.8.1/bin:/opt/pbs/default/sbin/:/opt/pbs/default/bin/:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ibutils/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/3.2.10/bin:/opt/dell/srvadmin/bin:/home/cstevens/.local/bin:/home/cstevens/bin:/home/cstevens/.local/bin:/home/cstevens/bin
+ export LD_LIBRARY_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/libs/hdf5/1.8.16/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/libs/openmpi/1.8.8-intel16.0.1/lib:/apps/chpc/bio/lib/gnu/gsl/2.1/lib:/apps/libs/zlib/1.2.8/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../clck_latest/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../itac_latest/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../inspector_xe/lib64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../vtune_amplifier_xe/lib64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/../compiler/lib/intel64_lin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/../tbb/lib/intel64_lin/gcc4.4:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/lib/intel64_lin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../debugger_2016/libipt/intel64/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/tbb/lib/intel64/gcc4.4:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mkl/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/ipp/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/mic/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/intel64/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/compiler/lib/intel64:/apps/chpc/compmech/gnuplot-dumb-5.0.3/lib:/apps/chpc/bio/git/2.8.1/lib64:/opt/pbs/default/lib/
+ LD_LIBRARY_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/libs/hdf5/1.8.16/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/lib:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/lib:/apps/libs/openmpi/1.8.8-intel16.0.1/lib:/apps/chpc/bio/lib/gnu/gsl/2.1/lib:/apps/libs/zlib/1.2.8/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../clck_latest/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../itac_latest/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../inspector_xe/lib64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../vtune_amplifier_xe/lib64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/../compiler/lib/intel64_lin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/../tbb/lib/intel64_lin/gcc4.4:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/daal/lib/intel64_lin:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/../../debugger_2016/libipt/intel64/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/tbb/lib/intel64/gcc4.4:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mkl/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/ipp/lib/intel64:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/mic/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/mpi/intel64/lib:/apps/compilers/intel/parallel_studio_xe_2016/compilers_and_libraries/linux/compiler/lib/intel64:/apps/chpc/compmech/gnuplot-dumb-5.0.3/lib:/apps/chpc/bio/git/2.8.1/lib64:/opt/pbs/default/lib/
+ export C_INCLUDE_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:
+ C_INCLUDE_PATH=/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/fftw-3.3.5-build/include:/apps/chpc/astro/utils/intel/v16-openmpi188/gsl-1.16-build/include:
+ export CACTUS_NUM_PROCS=20
+ CACTUS_NUM_PROCS=20
+ export CACTUS_NUM_THREADS=12
+ CACTUS_NUM_THREADS=12
+ export MP_SINGLE_THREAD=no
+ MP_SINGLE_THREAD=no
+ export OMP_NUM_THREADS=12
+ OMP_NUM_THREADS=12
+ env
+ sort
+ echo 'Job setup:'
+ echo '   Allocated:'
+ echo '      Nodes:                      10'
+ echo '      Cores per node:             24'
+ echo '   Running:'
+ echo '      MPI processes:              20'
+ echo '      OpenMP threads per process: 12'
+ echo '      MPI processes per node:     2.0'
+ echo '      OpenMP threads per core:    1.0'
+ echo '      OpenMP threads per node:    24'
+ ulimit -Sa
+ ulimit -Ha
+ ulimit -s unlimited
+ echo Starting:
+ echo 'Chris test:'
++ date +%s
+ export CACTUS_STARTTIME=1494859191
+ CACTUS_STARTTIME=1494859191
+ mpirun --mca orte_base_help_aggregate 0 -np 20 /mnt/lustre/users/cstevens/CCM/simulations/mctest/SIMFACTORY/exe/cactus_mclachlantest_mpidebug -L 3 /mnt/lustre/users/cstevens/CCM/simulations/mctest/output-0000/qc0-mclachlan.par