Backtrace from rank 1 pid 378130:
1. CarpetLib::signal_handler(int)   [/N/scratch/warrenjs/data/TOV/SIMFACTORY/
exe/cactus_sim(_ZN9CarpetLib14signal_handlerEi+0x9d) [0xaf47cd]]
2. /lib64/libc.so.6(+0x37400) [0x14d0dbe85400]
3. /N/soft/rhel8/mvapich/gnu/2.3.5/lib/libmpi.so.12(MPIDI_CH3_iStartMsgv+0x8b
e) [0x14d0dd6c05de]
4. /N/soft/rhel8/mvapich/gnu/2.3.5/lib/libmpi.so.12(MPIDI_CH3_EagerContigSend
+0x83) [0x14d0dd6b1903]
5. /N/soft/rhel8/mvapich/gnu/2.3.5/lib/libmpi.so.12(MPID_Send+0x123) [0x14d0d
d6b8f13]
6. /N/soft/rhel8/mvapich/gnu/2.3.5/lib/libmpi.so.12(+0x6919c) [0x14d0dd5b619c
]
7. /N/soft/rhel8/mvapich/gnu/2.3.5/lib/libmpi.so.12(MPIR_Reduce_intra+0x185)
[0x14d0dd5b6ef5]
8. /N/soft/rhel8/mvapich/gnu/2.3.5/lib/libmpi.so.12(MPIR_Reduce_impl+0x51) [0
x14d0dd5b6ce1]
9. /N/soft/rhel8/mvapich/gnu/2.3.5/lib/libmpi.so.12(MPI_Reduce+0x3cb) [0x14d0
dd5ba74b]
10. CarpetReduce::Finalise(_cGH const*, int, int, void*, int, void const*, vo
id const*, CarpetReduce::reduction const*)   [/N/scratch/warrenjs/data/TOV/SI
MFACTORY/exe/cactus_sim(_ZN12CarpetReduce8FinaliseEPK4_cGHiiPviPKvS5_PKNS_9re
ductionE+0x429) [0xc4f239]]
11. CarpetReduce::ReduceGVs(_cGH const*, int, int, int, void*, int, int const
*, CarpetReduce::reduction const*, int)   [/N/scratch/warrenjs/data/TOV/SIMFA
CTORY/exe/cactus_sim(_ZN12CarpetReduce9ReduceGVsEPK4_cGHiiiPviPKiPKNS_9reduct
ionEi+0x797) [0xc51157]]
12. CarpetReduce::iaverage_GVs(_cGH const*, int, int, int, void*, int, int co
nst*)   [/N/scratch/warrenjs/data/TOV/SIMFACTORY/exe/cactus_sim(_ZN12CarpetRe
duce12iaverage_GVsEPK4_cGHiiiPviPKi+0x1e) [0xc5307e]]
13. /N/scratch/warrenjs/data/TOV/SIMFACTORY/exe/cactus_sim(CCTK_Reduce+0xf1)
[0x1f153b1]
14. /N/scratch/warrenjs/data/TOV/SIMFACTORY/exe/cactus_sim() [0xac6dc5]
15. /N/scratch/warrenjs/data/TOV/SIMFACTORY/exe/cactus_sim() [0xac87c3]
16. /N/scratch/warrenjs/data/TOV/SIMFACTORY/exe/cactus_sim() [0xac89f4]
17. Carpet::OutputGH(_cGH const*)   [/N/scratch/warrenjs/data/TOV/SIMFACTORY/
exe/cactus_sim(_ZN6Carpet8OutputGHEPK4_cGH+0xbb) [0xa0ccfb]]
18. /N/scratch/warrenjs/data/TOV/SIMFACTORY/exe/cactus_sim() [0xa08115]
19. Carpet::Initialise(tFleshConfig*)   [/N/scratch/warrenjs/data/TOV/SIMFACT
ORY/exe/cactus_sim(_ZN6Carpet10InitialiseEP12tFleshConfig+0x1552) [0xa0af72]]
20. /N/scratch/warrenjs/data/TOV/SIMFACTORY/exe/cactus_sim(main+0x33) [0x6098
43]
21. /lib64/libc.so.6(__libc_start_main+0xf3) [0x14d0dbe71493]
22. /N/scratch/warrenjs/data/TOV/SIMFACTORY/exe/cactus_sim(_start+0x2e) [0x60
f78e]

The hexadecimal addresses in this backtrace can also be interpreted
with a debugger (e.g. gdb), or with the 'addr2line' (or 'gaddr2line')
command line tool: 'addr2line -e cactus_sim <address>'.