[leonardo-dcgp1]

# last tested on: August 2025
# last tested by: Panagiotis Iosif <panagiotis.iosif@units.it>

# Machine description
nickname        = leonardo
name            = LEONARDO
location        = CINECA, Casalecchio di Reno (BO), Italy
status          = production
description     = The New Italian Tier-1 cluster for industrial and public research

# Access to this machine
hostname        = login.leonardo.cineca.it
rsynccmd        = rsync
sshcmd          = ssh
envsetup        = <<EOT
    module load profile/base &&
    # Load explicitly also: GCC and openmpi
    # Not sure if this explicitness is needed. Test and see if anything complains
    module load gcc/12.2.0 &&
    module load openmpi/4.1.6--gcc--12.2.0-cuda-12.2 &&
    # --- GSL ---
    # old gsl
    # module load gsl/2.7.1--gcc--12.2.0
    # updated gsl
    module load gsl/2.7.1--gcc--12.2.0-spack0.22 &&
    # --- HDF5 ---
    # old hdf5
    # module load hdf5/1.14.3--openmpi--4.1.6--gcc--12.2.0
    module load hdf5/1.14.3--openmpi--4.1.6--gcc--12.2.0-spack0.22 &&
    # --- FFTW3 ---
    # old fftw3
    #module load fftw/3.3.10--openmpi--4.1.6--gcc--12.2.0
    # updated fftw3
    module load fftw/3.3.10--openmpi--4.1.6--gcc--12.2.0-spack0.22 &&
    # --- Boost ---
    # old boost
    # module load boost/1.83.0--openmpi--4.1.6--gcc--12.2.0
    # updated boost
    module load boost/1.85.0--openmpi--4.1.6--gcc--12.2.0 &&
    # --- Netlib Scalapack ---
    # old netlib-scalapack
    # module load netlib-scalapack/2.2.0--openmpi--4.1.6--gcc--12.2.0
    # updated netlib-scalapack
    module load netlib-scalapack/2.2.0--openmpi--4.1.6--gcc--12.2.0-spack0.22 &&
    # --- OpenBLAS ---
    module load openblas/0.3.26--gcc--12.2.0
EOT

aliaspattern    = ^(login0[1257])(\.leonardo\.cineca\.it)?$


# Source tree management
sourcebasedir   = /leonardo/home/userexternal/@USER@
disabled-thorns = <<EOT
EOT
# The following thorns were disabled in the original file by Bruno.
# Removed for now and will revisit if needed.
# ------------------------
# ExternalLibraries/ADIOS2
# ExternalLibraries/AMReX
# ExternalLibraries/Silo
# GRHayLET/GRHayLHDX
# GRHayLET/GRHayLIDX
# CarpetX/Algo
# CarpetX/PDESolvers
# CarpetX/PoissonX
# CarpetX/ADMBaseX
# CarpetX/Arith
# CarpetX/BoxInBox
# CarpetX/CarpetX
# CarpetX/CoordinatesX
# CarpetX/Derivs
# CarpetX/ErrorEstimator
# CarpetX/FluxWaveToyX
# CarpetX/HydroBaseX
# CarpetX/Loop
# CarpetX/ODESolvers
# CarpetX/SIMDWaveToyX
# CarpetX/SpacetimeWaveToyX
# CarpetX/StaggeredWaveToyX
# CarpetX/TestArrayGroup
# CarpetX/TestBoundaries
# CarpetX/TestBoxInBox
# CarpetX/TestDerivs
# CarpetX/TestInterpolate
# CarpetX/TestNorms
# CarpetX/TestODESolvers
# CarpetX/TestODESolvers2
# CarpetX/TestOutput
# CarpetX/TestProlongate
# CarpetX/TestSymmetries
# CarpetX/TmunuBaseX
# CarpetX/WaveToyX
# ExternalLibraries/PAPI
# THCExtra/WeakRates
# ------------------------
enabled-thorns = <<EOT
EOT
optionlist      = leonardo-dcgp1.cfg
submitscript    = leonardo-dcgp1.sub
runscript       = leonardo-dcgp1.run
makejobs        = 4
make            = make -j@MAKEJOBS@

#-----------------------------------------------------------
# 
# Model: Atos BullSequana X2140 three-node CPU blade
#  
# Nodes: 1536
# Processors: 56 cores sockets Intel Sapphire Rapids
#             2 x Intel CPU, TDB 350W
# Cores: 112 cores/node, 172032 cores in total
# Accelerators: NONE
# RAM: 512 (16 x 32) GB DDR5 4800 MHz
# Internal Network: NVIDIA Mellanox HDR DragonFly++ 200Gb/s
# 	   	    2 x NVIDIA HDR 2×100 Gb/s cards
#		    1x Nvidia HDR100 100 Gb/s card
# Disk Space: - of local scratch
# Peak Performance: 9 Pflops/s
#
# --------------------------------------------------------

# Simulation management
    # In general, for 'basedir', it is advised to use a 
    # '$SCRATCH/simulations' directory.
    # For Leonardo this is: 
    #     /leonardo_scratch/large/userexternal/@USER@/simulations
    # For now will use '$HOME' for my simulations.
    # Will amend if needed depending on size of simulations.
    # NOTE: see https://docs.hpc.cineca.it/hpc/hpc_data_storage.html
    # for $HOME, $SCRATCH and $WORK sizes and attributes.
basedir         = /leonardo/home/userexternal/@USER@/simulations
#-------------------------------
# Cluster characteristics
#-------------------------------
cpu             = 172032
cpufreq         = 2.6
ppn             = 112
spn             = 2
max-num-threads = 56
num-threads     = 56
memory          = 494000
nodes           = 1536
min-ppn         = 112
allocation      = NO_ALLOCATION
queue           = @QUEUE@
maxwalltime     = 24:00:00
#-------------------------------
# Queueing system options
#-------------------------------
submit          = sbatch @SUBMITSCRIPT@
getstatus       = squeue -j @JOB_ID@ 
stop            = scancel @JOB_ID@
submitpattern   = 'Submitted batch job (\d+)'
statuspattern   = '@JOB_ID@ '
queuedpattern   = ' PD '
runningpattern  = ' (CF|CG|R|TO) '
holdingpattern  = ' S '
exechost        = hostname
exechostpattern = (.*)
stdout          = cat @SIMULATION_NAME@.out
stderr          = cat @SIMULATION_NAME@.err
stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err

# 'qstat' is PBS-specific command. 
# Leonardo uses Slurm.
# Replaced with best guess, after checking other ini files using Slurm.
# see also: https://docs.einsteintoolkit.org/et-docs/Configuring_a_new_machine#Machine_definition
# exechost        = qstat -f @JOB_ID@ 
# exechostpattern = exec_host = (\w+)/