Note
Go to the end to download the full example code.
Basic example on how to use the CCC cluster using multi-tasks¶
Basic example
When you’re running hundreds or thousands of jobs, automation is a necessity.
This is where hopla can help you.
A simple example of how to use hopla on a CCC cluster. Please check
the user guide for a more in depth presentation of all
functionalities.
Imports¶
import hopla
import numpy as np
from pprint import pprint
Executor Context¶
executor = hopla.Executor(
cluster="ccc",
folder="/tmp/hopla",
queue="rome",
image="/tmp/hopla/my-docker-img.tar",
walltime=1,
project_id="genXXX",
backend="joblib",
)
Submit Jobs¶
chunks = np.array_split(range(1, 11), 3)
jobs = [
executor.submit([hopla.DelayedSubmission("sleep", k) for k in c])
for c in chunks
]
pprint(jobs)
print(jobs[0].delayed_submission)
[DelayedCCCJob(
job_id=1,
submission_id=None,
_hub=n4h00001rs,
image_name=/tmp/hopla/my-docker-img.tar,
),
DelayedCCCJob(
job_id=2,
submission_id=None,
_hub=n4h00001rs,
image_name=/tmp/hopla/my-docker-img.tar,
),
DelayedCCCJob(
job_id=3,
submission_id=None,
_hub=n4h00001rs,
image_name=/tmp/hopla/my-docker-img.tar,
)]
[DelayedSubmission(
command=sleep 1,
execution_parameters=,
), DelayedSubmission(
command=sleep 2,
execution_parameters=,
), DelayedSubmission(
command=sleep 3,
execution_parameters=,
), DelayedSubmission(
command=sleep 4,
execution_parameters=,
)]
Generate a batch¶
/home/runner/work/hopla/hopla/examples/plot_ccc_joblib_multi_tasks.py:56: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
jobs[0].generate_batch()
|Traceback (most recent call last):
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
| self.import_image()
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
| stdout = subprocess.check_output(cmd)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
| return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
| with Popen(*popenargs, **kwargs) as process:
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
| self._execute_child(args, executable, preexec_fn, close_fds,
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
| raise child_exception_type(errno_num, err_msg, err_filename)
|FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'
JobPaths(
flux_dir=/tmp/hopla/logs/1_flux,
job_id=1,
joblib_file=/tmp/hopla/submissions/1_joblib_script.py,
log_folder=/tmp/hopla/logs,
oneshot_dir=/tmp/hopla/logs/1_oneshot,
oneshot_file=/tmp/hopla/submissions/1_oneshot_script.sh,
stderr=/tmp/hopla/logs/1_log.err,
stdout=/tmp/hopla/logs/1_log.out,
submission_file=/tmp/hopla/submissions/1_submission.sh,
submission_folder=/tmp/hopla/submissions,
task_file=/tmp/hopla/submissions/1_tasks.txt,
worker_file=/tmp/hopla/submissions/worker.sh,
)
#!/bin/bash
# Parameters
#MSUB -q rome
#MSUB -Q long
#MSUB -m workflash,scratch,work
#MSUB -T 3600
#MSUB -n 1
#MSUB -c 1
#MSUB -E "--gres=gpu:0"
#MSUB -M 2000
#MSUB -r hopla
#MSUB -e /tmp/hopla/logs/1_log.err
#MSUB -o /tmp/hopla/logs/1_log.out
#MSUB -A genXXX
#MSUB -F #use the Flux plug
# Environment
echo $SLURM_JOB_ID
echo $HOSTNAME
module load python3/3.12
# Command
python /tmp/hopla/submissions/1_joblib_script.py
echo "HOPLASAY-DONE"
##########################################################################
# Hopla - Copyright (C) AGrigis, 2015 - 2025
# Distributed under the terms of the CeCILL-B license, as published by
# the CEA-CNRS-INRIA. Refer to the LICENSE file or to
# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
# for details.
##########################################################################
import sys
import subprocess
from joblib import Parallel, delayed
def run_command(cmd):
"""
Run a single command line string.
Returns a dictionary with command, status code, stdout, stderr.
"""
try:
result = subprocess.run(
cmd,
shell=True,
capture_output=True,
text=True
)
return {
"command": cmd,
"returncode": result.returncode,
"stdout": result.stdout.strip(),
"stderr": result.stderr.strip()
}
except Exception as e:
return {
"command": cmd,
"returncode": -1,
"stdout": "",
"stderr": str(e)
}
if __name__ == "__main__":
commands = [
'pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar -- sleep 1',
'pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar -- sleep 2',
'pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar -- sleep 3',
'pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar -- sleep 4',
]
n_jobs = 1
results = Parallel(n_jobs=n_jobs)(
delayed(run_command)(cmd) for cmd in commands
)
for item in results:
print("="*40)
print(f"Command : {item['command']}")
print(f"Returncode: {item['returncode']}")
print(f"Stdout : {item['stdout']}")
print(f"Stderr : {item['stderr']}")
if any(item["returncode"] != 0 for item in results):
sys.exit(1)
else:
sys.exit(0)
Start Jobs¶
We can’t execute the code on the CI since the CCC infrastructure is not available.
from hopla.config import Config
with Config(dryrun=True, delay_s=3):
executor(max_jobs=2)
print(executor.report)
CCC_MSUB: 0%| | 0/3 [00:00<?, ?it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
self.generate_batch()
|Traceback (most recent call last):
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
| self.import_image()
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
| stdout = subprocess.check_output(cmd)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
| return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
| with Popen(*popenargs, **kwargs) as process:
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
| self._execute_child(args, executable, preexec_fn, close_fds,
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
| raise child_exception_type(errno_num, err_msg, err_filename)
|FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'
[command] ccc_msub /tmp/hopla/submissions/1_submission.sh
CCC_MSUB: 33%|███▎ | 1/3 [00:00<00:00, 278.84it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
self.generate_batch()
|Traceback (most recent call last):
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
| self.import_image()
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
| stdout = subprocess.check_output(cmd)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
| return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
| with Popen(*popenargs, **kwargs) as process:
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
| self._execute_child(args, executable, preexec_fn, close_fds,
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
| raise child_exception_type(errno_num, err_msg, err_filename)
|FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'
[command] ccc_msub /tmp/hopla/submissions/2_submission.sh
CCC_MSUB: 67%|██████▋ | 2/3 [00:00<00:00, 323.43it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
self.generate_batch()
|Traceback (most recent call last):
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
| self.import_image()
| File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
| stdout = subprocess.check_output(cmd)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
| return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
| with Popen(*popenargs, **kwargs) as process:
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
| self._execute_child(args, executable, preexec_fn, close_fds,
| File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
| raise child_exception_type(errno_num, err_msg, err_filename)
|FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'
[command] ccc_msub /tmp/hopla/submissions/3_submission.sh
CCC_MSUB: 100%|██████████| 3/3 [00:03<00:00, 1.00s/it]
CCC_MSUB: 100%|██████████| 3/3 [00:03<00:00, 1.00s/it]
CCC_MSUB: 100%|██████████| 3/3 [00:06<00:00, 2.00s/it]
----------------------------------------
DelayedCCCJob<job_id=1>exitcode: failure
DelayedCCCJob<job_id=1>submission: /tmp/hopla/submissions/1_submission.sh
DelayedCCCJob<job_id=1>stdout: none
DelayedCCCJob<job_id=1>stderr: none
----------------------------------------
DelayedCCCJob<job_id=2>exitcode: failure
DelayedCCCJob<job_id=2>submission: /tmp/hopla/submissions/2_submission.sh
DelayedCCCJob<job_id=2>stdout: none
DelayedCCCJob<job_id=2>stderr: none
----------------------------------------
DelayedCCCJob<job_id=3>exitcode: failure
DelayedCCCJob<job_id=3>submission: /tmp/hopla/submissions/3_submission.sh
DelayedCCCJob<job_id=3>stdout: none
DelayedCCCJob<job_id=3>stderr: none
Total running time of the script: (0 minutes 6.241 seconds)
Estimated memory usage: 109 MB