[ET Trac] [Einstein Toolkit] #2008: NaNs when running static tov on >40 cores

[Einstein Toolkit]

#2008: NaNs when running static tov on >40 cores
------------------------------------+---------------------------------------
  Reporter:  allgwy001@…            |       Owner:  knarf     
      Type:  defect                 |      Status:  assigned  
  Priority:  major                  |   Milestone:            
 Component:  Carpet                 |     Version:  ET_2016_05
Resolution:                         |    Keywords:            
------------------------------------+---------------------------------------

Comment (by allgw001@…):

 Details about the build:


 {{{
 zypper in libnuma-devel
 wget
 https://bitbucket.org/einsteintoolkit/manifest/raw/ET_2016_11/einsteintoolkit.th

 remove ExternalLibraries/OpenBLAS, ExternalLibraries/OpenCL,
 Carpet/ReductionTest and CactusTest/TestFortranCrayPointers
 add Llama/WaveExtractCPM and EinsteinAnalysis/ADMConstraints

 curl -kLO
 https://raw.githubusercontent.com/gridaphobe/CRL/ET_2016_11/GetComponents
 chmod a+x GetComponents
 ./GetComponents --parallel einsteintoolkit.th
 module load compilers/gcc530 mpi/openmpi-1.10.1

 cd Cactus
 cp simfactory/mdb/optionlists/centos.cfg ./sles.cfg
 Add MPI_DIR=/opt/exp_soft/openmpi-1.10.1/

 make ET-config options=sles.cfg THORNLIST=thornlists/einsteintoolkit.th
 make -j 64 ET
 }}}

 where sles.cfg contains:

 {{{
 # SLES
 #
 # Whenever this version string changes, the application is configured
 # and rebuilt from scratch
 VERSION = 2016-05-06

 CPP = cpp
 FPP = cpp
 CC  = gcc
 CXX = g++
 F77 = gfortran
 F90 = gfortran

 CPPFLAGS = -DMPICH_IGNORE_CXX_SEEK
 FPPFLAGS = -traditional

 CFLAGS   =  -g3 -std=gnu99 -rdynamic -Wunused-variable
 CXXFLAGS =  -g3 -std=c++0x -rdynamic -Wunused-variable -fno-inline
 F77FLAGS =  -g3 -fcray-pointer -ffixed-line-length-none
 F90FLAGS =  -g3 -fcray-pointer -ffixed-line-length-none

 LIBS = numa gfortran

 LIBDIRS =

 MPI_DIR=/opt/exp_soft/openmpi-1.10.1/

 DEBUG           = no
 CPP_DEBUG_FLAGS = -DCARPET_DEBUG
 FPP_DEBUG_FLAGS = -DCARPET_DEBUG
 C_DEBUG_FLAGS   = -O0
 CXX_DEBUG_FLAGS = -O0
 F77_DEBUG_FLAGS = -O0 -ffixed-line-length-none
 F90_DEBUG_FLAGS = -O0 -ffree-line-length-none
 }}}
 My PBS script:
 {{{
 #PBS -q UCTlong
 #PBS -N static_tov_mod_2
 #PBS -m abe
 #PBS -M allgwy001 at myuct.ac.za
 #PBS -l nodes=1:ppn=44:series600
 #PBS -o /home/allgwy001/Output/static_tov_mod_2
 #PBS -e /home/allgwy001/Output/static_tov_mod_2
 module load software/EinsteinToolkit
 cd /home/allgwy001/Output
 export OMP_NUM_THREADS=1
 mpirun -np 44 -hostfile $PBS_NODEFILE -v cactus_ET
 /home/allgwy001/Parameter_Files/static_tov_mod_2.par
 }}}
 I simply ran using
 {{{
 qsub <PBS script>
 }}}

-- 
Ticket URL: <https://trac.einsteintoolkit.org/ticket/2008#comment:9>
Einstein Toolkit <http://einsteintoolkit.org>
The Einstein Toolkit