[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::Creating simulation illinois11_wB_540_test
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::Simulation directory: /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::Simulation Properties:
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::[properties]
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::machine         = cori
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::simulationid    = simulation-illinois11_wB_540_test-cori-cori.nersc.gov-chlin-2019.05.07-06.37.29-47713
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::sourcedir       = /global/homes/c/chlin/et/Cactus
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::configuration   = sim
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::configid        = config-sim-cori05.nersc.gov-global-u1-c-chlin-et-Cactus
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::buildid         = build-sim-cori05.nersc.gov-chlin-2019.04.30-05.59.13-37486
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::testsuite       = False
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::executable      = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/exe/cactus_sim
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::optionlist      = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/cfg/OptionList
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::submitscript    = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/run/SubmitScript
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::runscript       = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/run/RunScript
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::parfile         = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/par/illinois11_wB.par
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::
[LOG:2019-05-07 06:37:29] self.create(simulationName, parfile)::Simulation illinois11_wB_540_test created
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::Restart for simulation illinois11_wB_540_test created with restart id 0, long restart id 0000
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::Prepping for submission
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::No previous walltime available to be used, using walltime 0:30:00
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::Defined substituion properties for submission
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::{'SIMULATION_ID': 'simulation-illinois11_wB_540_test-cori-cori.nersc.gov-chlin-2019.05.07-06.37.29-47713', 'NODE_PROCS': 2, 'PPN_USED': 32, 'PPN': 32, 'ALLOCATION': 'm2775', 'WALLTIME_HH': '00', 'CPUFREQ': '2.3', 'USER': 'chlin', 'RUNDIR': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000', 'NODES': 2, 'SIMULATION_NAME': 'illinois11_wB_540_test', 'WALLTIME': '0:30:00', 'NUM_THREADS': 16, 'EXECUTABLE': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/exe/cactus_sim', 'PROCS_REQUESTED': 64, 'EMAIL': 'chlin', 'RESTART_ID': 0, 'CHAINED_JOB_ID': '', 'FROM_RESTART_COMMAND': '', 'NUM_SMT': 1, 'WALLTIME_SECONDS': 1800, 'SIMFACTORY': '/global/u1/c/chlin/et/Cactus/repos/simfactory2/bin/sim', 'PROCS': 64, 'SUBMITSCRIPT': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000/SIMFACTORY/SubmitScript', 'WALLTIME_HOURS': 0.5, 'WALLTIME_MM': '30', 'PARFILE': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000/illinois11_wB.par', 'WALLTIME_SS': '00', 'QUEUE': 'debug', 'CONFIGURATION': 'sim', 'SOURCEDIR': '/global/homes/c/chlin/et/Cactus', 'HOSTNAME': 'cori.nersc.gov', 'NUM_PROCS': 4, 'SCRIPTFILE': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000/SIMFACTORY/SubmitScript', 'MEMORY': '131072', 'WALLTIME_MINUTES': 30, 'SHORT_SIMULATION_NAME': 'illinois11_wB_5'}
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::self.Properties: /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000/SIMFACTORY/properties.ini
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::[properties]
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::machine         = cori
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::simulationid    = simulation-illinois11_wB_540_test-cori-cori.nersc.gov-chlin-2019.05.07-06.37.29-47713
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::sourcedir       = /global/homes/c/chlin/et/Cactus
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::configuration   = sim
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::configid        = config-sim-cori05.nersc.gov-global-u1-c-chlin-et-Cactus
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::buildid         = build-sim-cori05.nersc.gov-chlin-2019.04.30-05.59.13-37486
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::testsuite       = False
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::executable      = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/exe/cactus_sim
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::optionlist      = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/cfg/OptionList
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::submitscript    = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/run/SubmitScript
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::runscript       = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/run/RunScript
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::parfile         = /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/par/illinois11_wB.par
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::chainedjobid    = -1
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::ppn             = 32
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::procsrequested  = 64
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::allocation      = m2775
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::user            = chlin
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::numsmt          = 1
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::walltime        = 0:30:00
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::numprocs        = 4
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::nodeprocs       = 2
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::numthreads      = 16
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::hostname        = cori.nersc.gov
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::ppnused         = 32
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::queue           = debug
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::cpufreq         = 2.3
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::procs           = 64
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::memory          = 131072
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::nodes           = 2
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::pbsSimulationName= illinois11_wB_5
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::saving substituted submitscript contents to: /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000/SIMFACTORY/SubmitScript
[LOG:2019-05-07 06:37:29] self.submit(submitScript)::Executing submission command: sbatch /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000/SIMFACTORY/SubmitScript
[LOG:2019-05-07 06:37:29] self.makeActive()::Simulation illinois11_wB_540_test with restart-id 0 has been made active
[LOG:2019-05-07 06:37:34] job_id = self.extractJobId(output)::received raw output: Submitted batch job 21169123
[LOG:2019-05-07 06:37:34] job_id = self.extractJobId(output)::
[LOG:2019-05-07 06:37:34] job_id = self.extractJobId(output)::using submitRegex: Submitted batch job (\d+)
[LOG:2019-05-07 06:37:34] self.submit(submitScript)::After searching raw output, it was determined that the job_id is: 21169123
[LOG:2019-05-07 06:37:34] self.submit(submitScript)::Simulation illinois11_wB_540_test, with restart id 0, and job id 21169123 has been submitted
[LOG:2019-05-07 06:46:10] self.load(simulationName, restartId)::For simulation illinois11_wB_540_test, loaded restart id 0, long restart id 0000
[LOG:2019-05-07 06:46:10] self.run()::Prepping for execution/run
[LOG:2019-05-07 06:46:10] checkpointing = self.PrepareCheckpointing(recover_id)::PrepareCheckpointing: max_restart_id: -1
[LOG:2019-05-07 06:46:10] self.run()::Defined substitution properties for execution/run
[LOG:2019-05-07 06:46:10] self.run()::{'SIMULATION_ID': 'simulation-illinois11_wB_540_test-cori-cori.nersc.gov-chlin-2019.05.07-06.37.29-47713', 'NODE_PROCS': '2', 'PPN_USED': '32', 'PPN': '32', 'WALLTIME_HH': '00', 'CPUFREQ': '2.3', 'USER': 'chlin', 'RUNDIR': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000', 'NODES': '2', 'SIMULATION_NAME': 'illinois11_wB_540_test', 'WALLTIME': '0:30:00', 'NUM_THREADS': '16', 'EXECUTABLE': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/exe/cactus_sim', 'PROCS_REQUESTED': '64', 'RESTART_ID': 0, 'NUM_SMT': '1', 'WALLTIME_SECONDS': 1800, 'CONFIGURATION': 'sim', 'PROCS': '64', 'SUBMITSCRIPT': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/run/SubmitScript', 'WALLTIME_MM': '30', 'MACHINE': 'cori', 'PARFILE': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000/illinois11_wB.par', 'WALLTIME_SS': '00', 'WALLTIME_HOURS': 0.5, 'SOURCEDIR': '/global/homes/c/chlin/et/Cactus', 'HOSTNAME': 'cori.nersc.gov', 'RUNDEBUG': 0, 'NUM_PROCS': '4', 'SCRIPTFILE': '/global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/SIMFACTORY/run/SubmitScript', 'MEMORY': '131072', 'WALLTIME_MINUTES': 30, 'SHORT_SIMULATION_NAME': 'illinois11_wB_5'}
[LOG:2019-05-07 06:46:10] self.run()::Executing run command: /global/cscratch1/sd/chlin/simulations2/illinois11_wB_540_test/output-0000/SIMFACTORY/RunScript