+ set -e
+ cd /scratch/ettest34/simulations/tov_collapse/output-0000-active
+ echo Checking:
+ pwd
+ hostname
+ date
+ cat /var/spool/torque/aux//653521.qb2
+ echo Environment:
+ export GMON_OUT_PREFIX=gmon.out
+ GMON_OUT_PREFIX=gmon.out
+ export OMP_NUM_THREADS=8
+ OMP_NUM_THREADS=8
+ export MPICHDIR=/usr/local/packages/mvapich2/1.6rc1/intel-11.1
+ MPICHDIR=/usr/local/packages/mvapich2/1.6rc1/intel-11.1
+ export MPD_NODEFILE=mpd_nodefile
+ MPD_NODEFILE=mpd_nodefile
+ export MV2_SRQ_SIZE=4000
+ MV2_SRQ_SIZE=4000
+ export MPI_NODEFILE=mpi_nodefile
+ MPI_NODEFILE=mpi_nodefile
+ env
+ sort
+ echo Starting:
+ uniq /var/spool/torque/aux//653521.qb2
++ cat mpd_nodefile
+ for node in '$(cat ${MPD_NODEFILE})'
+ (( proc=0 ))
+ (( 0<1 ))
+ echo qb575
+ (( proc=0+1 ))
+ (( 1<1 ))
+ for node in '$(cat ${MPD_NODEFILE})'
+ (( proc=0 ))
+ (( 0<1 ))
+ echo qb538
+ (( proc=0+1 ))
+ (( 1<1 ))
+ for node in '$(cat ${MPD_NODEFILE})'
+ (( proc=0 ))
+ (( 0<1 ))
+ echo qb535
+ (( proc=0+1 ))
+ (( 1<1 ))
+ for node in '$(cat ${MPD_NODEFILE})'
+ (( proc=0 ))
+ (( 0<1 ))
+ echo qb533
+ (( proc=0+1 ))
+ (( 1<1 ))
++ date +%s
+ export CACTUS_STARTTIME=1350514554
+ CACTUS_STARTTIME=1350514554
+ '[' 0 -eq 0 ']'
+ /usr/local/packages/mvapich2/1.6rc1/intel-11.1/bin/mpirun_rsh -np 4 -hostfile mpi_nodefile /bin/env MV2_ENABLE_AFFINITY=0 OMP_NUM_THREADS=8 /scratch/ettest34/simulations/tov_collapse/SIMFACTORY/exe/cactus_sim -L 3 /scratch/ettest34/simulations/tov_collapse/output-0000/tov_collapse_vc_l3.par
WARNING level 1 in thorn GRHydro processor 0 host qb575
  (line 2004 of GRHydro_Con2Prim.F90): 
  -> Con2Prim failed; stopping the code.
WARNING level 1 in thorn GRHydro processor 0 host qb575
  (line 2005 of GRHydro_Con2Prim.F90): 
  -> Even with mesh refinement, this point is not restricted from a finer level, so this is really an error
WARNING level 1 in thorn GRHydro processor 0 host qb575
  (line 2007 of GRHydro_Con2Prim.F90): 
  ->         on carpet reflevel:  4
WARNING level 1 in thorn GRHydro processor 0 host qb575
  (line 2010 of GRHydro_Con2Prim.F90): 
  ->       xyz location:     3.400000        2.600000        2.600000
WARNING level 1 in thorn GRHydro processor 0 host qb575
  (line 2012 of GRHydro_Con2Prim.F90): 
  ->             radius:     5.007994
WARNING level 0 in thorn GRHydro processor 0 host qb575
  (line 2015 of GRHydro_Con2Prim.F90): 
  ->         velocities:              NaN             NaN             NaN
cactus_sim: /home/ettest34/Cactus/arrangements/Carpet/Carpet/src/helpers.cc:272: int Carpet::Abort(const _cGH *, int): Assertion `0' failed.
Rank 0 with PID 16125 received signal6
Writing backtrace to tov_collapse_vc_l3/backtrace.0.txt
WARNING level 1 in thorn GRHydro processor 1 host qb538
  (line 2004 of GRHydro_Con2Prim.F90): 
  -> Con2Prim failed; stopping the code.
WARNING level 1 in thorn GRHydro processor 1 host qb538
  (line 2005 of GRHydro_Con2Prim.F90): 
  -> Even with mesh refinement, this point is not restricted from a finer level, so this is really an error
WARNING level 1 in thorn GRHydro processor 1 host qb538
  (line 2007 of GRHydro_Con2Prim.F90): 
  ->         on carpet reflevel:  4
WARNING level 1 in thorn GRHydro processor 1 host qb538
  (line 2010 of GRHydro_Con2Prim.F90): 
  ->       xyz location:     3.800000        3.200000        4.200000
WARNING level 1 in thorn GRHydro processor 1 host qb538
  (line 2012 of GRHydro_Con2Prim.F90): 
  ->             radius:     6.505382
WARNING level 0 in thorn GRHydro processor 1 host qb538
  (line 2015 of GRHydro_Con2Prim.F90): 
  ->         velocities:              NaN             NaN             NaNWARNING level 1 in thorn GRHydro processor 2 host qb535
  (line 2004 of GRHydro_Con2Prim.F90): 
  ->
cactus_sim: /home/ettest34/Cactus/arrangements/Carpet/Carpet/src/helpers.cc:272: int Carpet::Abort(const _cGH *, int): Assertion `0' failed.
Rank 1 with PID 30195  Con2Prim failed; stopping the code.
WARNING level 1 in thorn GRHydro processor 2 host qb535
  (line 2005 of GRHydro_Con2Prim.F90): 
  -> Even with mesh refinement, this point is not restricted from a finer level, so this is really an error
WARNING level 1 in thorn GRHydro processor 2 host qb535
  (line 2007 of GRHydro_Con2Prim.F90): 
  ->received signal6
         on carpet reflevel:  4
WARNING level 1 in thorn GRHydro processor 2 host qb535
  (line 2010 of GRHydro_Con2Prim.F90): 
  ->       xyz location:     4.200000        3.000000        3.600000
Writing backtrace to tov_collapse_vc_l3/backtrace.1.txt
WARNING level 1 in thorn GRHydro processor 2 host qb535
  (line 2012 of GRHydro_Con2Prim.F90): 
  ->             radius:     6.292853
WARNING level 0 in thorn GRHydro processor 2 host qb535
  (line 2015 of GRHydro_Con2Prim.F90): 
  ->         velocities:              NaN             NaN             NaN
cactus_sim: /home/ettest34/Cactus/arrangements/Carpet/Carpet/src/helpers.cc:272: int Carpet::Abort(const _cGH *, int): Assertion `0' failed.
Rank 2 with PID 16637 received signal6
Writing backtrace to tov_collapse_vc_l3/backtrace.2.txt
MPI process (rank: 2) terminated unexpectedly on qb535
Exit code -5 signaled from qb538
MPI process (rank: 1) terminated unexpectedly on qb538
MPI process (rank: 0) terminated unexpectedly on qb575