Note
Go to the end to download the full example code.
Basic example on how to use the CCC cluster using multi-tasks¶
Basic example
When you’re running hundreds or thousands of jobs, automation is a necessity.
This is where hopla can help you.
A simple example of how to use hopla on a CCC cluster. Please check
the user guide for a more in depth presentation of all
functionalities.
Imports¶
import hopla
import numpy as np
from pprint import pprint
Executor Context¶
executor = hopla.Executor(
cluster="ccc",
folder="/tmp/hopla",
queue="rome",
image="/tmp/hopla/my-docker-img.tar",
walltime=1,
project_id="genXXX",
backend="flux",
)
Submit Jobs¶
chunks = np.array_split(range(1, 11), 3)
jobs = [
executor.submit([hopla.DelayedSubmission("sleep", k) for k in c])
for c in chunks
]
pprint(jobs)
print(jobs[0].delayed_submission)
[DelayedCCCJob(
job_id=1,
submission_id=None,
_hub=n4h00001rs,
image_name=/tmp/hopla/my-docker-img.tar,
),
DelayedCCCJob(
job_id=2,
submission_id=None,
_hub=n4h00001rs,
image_name=/tmp/hopla/my-docker-img.tar,
),
DelayedCCCJob(
job_id=3,
submission_id=None,
_hub=n4h00001rs,
image_name=/tmp/hopla/my-docker-img.tar,
)]
[DelayedSubmission(
command=sleep 1,
execution_parameters=,
), DelayedSubmission(
command=sleep 2,
execution_parameters=,
), DelayedSubmission(
command=sleep 3,
execution_parameters=,
), DelayedSubmission(
command=sleep 4,
execution_parameters=,
)]
Generate a batch¶
/home/runner/work/hopla/hopla/examples/plot_ccc_flux_multi_tasks.py:56: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
jobs[0].generate_batch()
|Traceback (most recent call last):
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
| self.import_image()
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
| stdout = subprocess.check_output(cmd)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
| return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
| with Popen(*popenargs, **kwargs) as process:
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
| self._execute_child(args, executable, preexec_fn, close_fds,
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
| raise child_exception_type(errno_num, err_msg, err_filename)
|FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'
JobPaths(
flux_dir=/tmp/hopla/logs/1_flux,
job_id=1,
joblib_file=/tmp/hopla/submissions/1_joblib_script.py,
log_folder=/tmp/hopla/logs,
oneshot_dir=/tmp/hopla/logs/1_oneshot,
oneshot_file=/tmp/hopla/submissions/1_oneshot_script.sh,
stderr=/tmp/hopla/logs/1_log.err,
stdout=/tmp/hopla/logs/1_log.out,
submission_file=/tmp/hopla/submissions/1_submission.sh,
submission_folder=/tmp/hopla/submissions,
task_file=/tmp/hopla/submissions/1_tasks.txt,
worker_file=/tmp/hopla/submissions/worker.sh,
)
#!/bin/bash
# Parameters
#MSUB -q rome
#MSUB -Q long
#MSUB -m workflash,scratch,work
#MSUB -T 3600
#MSUB -n 1
#MSUB -E "--gres=gpu:0"
#MSUB -M 2000
#MSUB -r hopla
#MSUB -e /tmp/hopla/logs/1_log.err
#MSUB -o /tmp/hopla/logs/1_log.out
#MSUB -A genXXX
#MSUB -F
# Environment
echo $SLURM_JOB_ID
echo $HOSTNAME
# Command
cd /tmp/hopla/logs/1_flux
ccc_mprun -D -v -B /tmp/hopla/submissions/1_tasks.txt
# Flux WIP
flux job wait --all
flux jobs -a
flux jobs -o "{status}:{returncode}"
flux jobs --filter=failed --format="{id} {exception.type} {exception.note}"
# End
echo "HOPLASAY-DONE"
1-1 . /tmp/hopla/submissions/worker.sh pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar -- sleep 1
1-1 . /tmp/hopla/submissions/worker.sh pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar -- sleep 2
1-1 . /tmp/hopla/submissions/worker.sh pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar -- sleep 3
1-1 . /tmp/hopla/submissions/worker.sh pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar -- sleep 4
Start Jobs¶
We can’t execute the code on the CI since the CCC infrastructure is not available.
from hopla.config import Config
with Config(dryrun=True, delay_s=3):
executor(max_jobs=2)
print(executor.report)
CCC_MSUB: 0%| | 0/3 [00:00<?, ?it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
self.generate_batch()
|Traceback (most recent call last):
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
| self.import_image()
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
| stdout = subprocess.check_output(cmd)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
| return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
| with Popen(*popenargs, **kwargs) as process:
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
| self._execute_child(args, executable, preexec_fn, close_fds,
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
| raise child_exception_type(errno_num, err_msg, err_filename)
|FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'
[command] ccc_msub /tmp/hopla/submissions/1_submission.sh
CCC_MSUB: 33%|███▎ | 1/3 [00:00<00:00, 249.50it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
self.generate_batch()
|Traceback (most recent call last):
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
| self.import_image()
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
| stdout = subprocess.check_output(cmd)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
| return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
| with Popen(*popenargs, **kwargs) as process:
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
| self._execute_child(args, executable, preexec_fn, close_fds,
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
| raise child_exception_type(errno_num, err_msg, err_filename)
|FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'
[command] ccc_msub /tmp/hopla/submissions/2_submission.sh
CCC_MSUB: 67%|██████▋ | 2/3 [00:00<00:00, 286.91it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
self.generate_batch()
|Traceback (most recent call last):
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
| self.import_image()
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
| stdout = subprocess.check_output(cmd)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
| return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
| with Popen(*popenargs, **kwargs) as process:
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
| self._execute_child(args, executable, preexec_fn, close_fds,
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
| raise child_exception_type(errno_num, err_msg, err_filename)
|FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'
[command] ccc_msub /tmp/hopla/submissions/3_submission.sh
CCC_MSUB: 100%|██████████| 3/3 [00:03<00:00, 1.00s/it]
CCC_MSUB: 100%|██████████| 3/3 [00:03<00:00, 1.00s/it]
CCC_MSUB: 100%|██████████| 3/3 [00:06<00:00, 2.00s/it]
----------------------------------------
DelayedCCCJob<job_id=1>exitcode: failure
DelayedCCCJob<job_id=1>submission: /tmp/hopla/submissions/1_submission.sh
DelayedCCCJob<job_id=1>stdout: none
DelayedCCCJob<job_id=1>stderr: none
DelayedCCCJob<job_id=1>number_of_tasks: 4
DelayedCCCJob<job_id=1>failed_tasks: []
DelayedCCCJob<job_id=1>running_tasks: 0
DelayedCCCJob<job_id=1>logdir: /tmp/hopla/logs/1_flux
----------------------------------------
DelayedCCCJob<job_id=2>exitcode: failure
DelayedCCCJob<job_id=2>submission: /tmp/hopla/submissions/2_submission.sh
DelayedCCCJob<job_id=2>stdout: none
DelayedCCCJob<job_id=2>stderr: none
DelayedCCCJob<job_id=2>number_of_tasks: 3
DelayedCCCJob<job_id=2>failed_tasks: []
DelayedCCCJob<job_id=2>running_tasks: 0
DelayedCCCJob<job_id=2>logdir: /tmp/hopla/logs/2_flux
----------------------------------------
DelayedCCCJob<job_id=3>exitcode: failure
DelayedCCCJob<job_id=3>submission: /tmp/hopla/submissions/3_submission.sh
DelayedCCCJob<job_id=3>stdout: none
DelayedCCCJob<job_id=3>stderr: none
DelayedCCCJob<job_id=3>number_of_tasks: 3
DelayedCCCJob<job_id=3>failed_tasks: []
DelayedCCCJob<job_id=3>running_tasks: 0
DelayedCCCJob<job_id=3>logdir: /tmp/hopla/logs/3_flux
Total running time of the script: (0 minutes 6.247 seconds)
Estimated memory usage: 109 MB