Note

Go to the end to download the full example code.

Basic example on how to use the CCC cluster using multi-tasks

Basic example

When you’re running hundreds or thousands of jobs, automation is a necessity. This is where hopla can help you.

A simple example of how to use hopla on a CCC cluster. Please check the user guide for a more in depth presentation of all functionalities.

Imports

import hopla
import numpy as np
from pprint import pprint

Executor Context

executor = hopla.Executor(
    cluster="ccc",
    folder="/tmp/hopla",
    queue="rome",
    image="/tmp/hopla/my-docker-img.tar",
    walltime=1,
    project_id="genXXX",
    backend="oneshot",
)

Submit Jobs

chunks = np.array_split(range(1, 11), 3)
jobs = [
    executor.submit([hopla.DelayedSubmission("sleep", k) for k in c])
    for c in chunks
]
pprint(jobs)
print(jobs[0].delayed_submission)

[DelayedCCCJob(
  job_id=1,
  submission_id=None,
  _hub=n4h00001rs,
  image_name=/tmp/hopla/my-docker-img.tar,
),
 DelayedCCCJob(
  job_id=2,
  submission_id=None,
  _hub=n4h00001rs,
  image_name=/tmp/hopla/my-docker-img.tar,
),
 DelayedCCCJob(
  job_id=3,
  submission_id=None,
  _hub=n4h00001rs,
  image_name=/tmp/hopla/my-docker-img.tar,
)]
[DelayedSubmission(
  command=sleep 1,
  execution_parameters=,
), DelayedSubmission(
  command=sleep 2,
  execution_parameters=,
), DelayedSubmission(
  command=sleep 3,
  execution_parameters=,
), DelayedSubmission(
  command=sleep 4,
  execution_parameters=,
)]

Generate a batch

jobs[0].generate_batch()
print(jobs[0].paths)
batch = jobs[0].paths.submission_file
with open(batch) as of:
    print(of.read())
script = jobs[0].paths.submission_file
with open(script) as of:
    print(of.read())

/home/runner/work/hopla/hopla/examples/plot_ccc_oneshot_multi_tasks.py:56: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
  jobs[0].generate_batch()
   |Traceback (most recent call last):
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
   |    self.import_image()
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
   |    stdout = subprocess.check_output(cmd)
   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
   |    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
   |    with Popen(*popenargs, **kwargs) as process:
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
   |    self._execute_child(args, executable, preexec_fn, close_fds,
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
   |    raise child_exception_type(errno_num, err_msg, err_filename)
   |FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'

JobPaths(
  flux_dir=/tmp/hopla/logs/1_flux,
  job_id=1,
  joblib_file=/tmp/hopla/submissions/1_joblib_script.py,
  log_folder=/tmp/hopla/logs,
  oneshot_dir=/tmp/hopla/logs/1_oneshot,
  oneshot_file=/tmp/hopla/submissions/1_oneshot_script.sh,
  stderr=/tmp/hopla/logs/1_log.err,
  stdout=/tmp/hopla/logs/1_log.out,
  submission_file=/tmp/hopla/submissions/1_submission.sh,
  submission_folder=/tmp/hopla/submissions,
  task_file=/tmp/hopla/submissions/1_tasks.txt,
  worker_file=/tmp/hopla/submissions/worker.sh,
)
#!/bin/bash

# Parameters
#MSUB -q rome
#MSUB -Q long
#MSUB -m workflash,scratch,work
#MSUB -T 3600
#MSUB -n 1
#MSUB -c 1
#MSUB -E "--gres=gpu:0"
#MSUB -M 2000
#MSUB -r hopla
#MSUB -e /tmp/hopla/logs/1_log.err
#MSUB -o /tmp/hopla/logs/1_log.out
#MSUB -A genXXX
#MSUB -F #use the Flux plug

# Environment
echo $SLURM_JOB_ID
echo $HOSTNAME


# Command
pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar  /bin/bash -- -c '/bin/bash /tmp/hopla/submissions/1_oneshot_script.sh'
echo "HOPLASAY-DONE"

#!/bin/bash

# Parameters
#MSUB -q rome
#MSUB -Q long
#MSUB -m workflash,scratch,work
#MSUB -T 3600
#MSUB -n 1
#MSUB -c 1
#MSUB -E "--gres=gpu:0"
#MSUB -M 2000
#MSUB -r hopla
#MSUB -e /tmp/hopla/logs/1_log.err
#MSUB -o /tmp/hopla/logs/1_log.out
#MSUB -A genXXX
#MSUB -F #use the Flux plug

# Environment
echo $SLURM_JOB_ID
echo $HOSTNAME


# Command
pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar  /bin/bash -- -c '/bin/bash /tmp/hopla/submissions/1_oneshot_script.sh'
echo "HOPLASAY-DONE"

Start Jobs

We can’t execute the code on the CI since the CCC infrastructure is not available.

from hopla.config import Config

with Config(dryrun=True, delay_s=3):
    executor(max_jobs=2)
    print(executor.report)

CCC_MSUB:   0%|          | 0/3 [00:00<?, ?it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
  self.generate_batch()
   |Traceback (most recent call last):
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
   |    self.import_image()
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
   |    stdout = subprocess.check_output(cmd)
   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
   |    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
   |    with Popen(*popenargs, **kwargs) as process:
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
   |    self._execute_child(args, executable, preexec_fn, close_fds,
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
   |    raise child_exception_type(errno_num, err_msg, err_filename)
   |FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'

[command] ccc_msub /tmp/hopla/submissions/1_submission.sh

CCC_MSUB:  33%|███▎      | 1/3 [00:00<00:00, 264.32it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
  self.generate_batch()
   |Traceback (most recent call last):
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
   |    self.import_image()
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
   |    stdout = subprocess.check_output(cmd)
   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
   |    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
   |    with Popen(*popenargs, **kwargs) as process:
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
   |    self._execute_child(args, executable, preexec_fn, close_fds,
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
   |    raise child_exception_type(errno_num, err_msg, err_filename)
   |FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'

[command] ccc_msub /tmp/hopla/submissions/2_submission.sh

CCC_MSUB:  67%|██████▋   | 2/3 [00:00<00:00, 303.08it/s]/home/runner/work/hopla/hopla/doc/../hopla/utils.py:372: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
  self.generate_batch()
   |Traceback (most recent call last):
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 152, in generate_batch
   |    self.import_image()
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 256, in import_image
   |    stdout = subprocess.check_output(cmd)
   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
   |    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
   |    with Popen(*popenargs, **kwargs) as process:
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
   |    self._execute_child(args, executable, preexec_fn, close_fds,
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
   |    raise child_exception_type(errno_num, err_msg, err_filename)
   |FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'

[command] ccc_msub /tmp/hopla/submissions/3_submission.sh

CCC_MSUB: 100%|██████████| 3/3 [00:03<00:00,  1.00s/it]
CCC_MSUB: 100%|██████████| 3/3 [00:03<00:00,  1.00s/it]
CCC_MSUB: 100%|██████████| 3/3 [00:06<00:00,  2.00s/it]
----------------------------------------
DelayedCCCJob<job_id=1>exitcode: failure
DelayedCCCJob<job_id=1>submission: /tmp/hopla/submissions/1_submission.sh
DelayedCCCJob<job_id=1>stdout: none
DelayedCCCJob<job_id=1>stderr: none
DelayedCCCJob<job_id=1>number_of_tasks: 4
DelayedCCCJob<job_id=1>failed_tasks: 0
DelayedCCCJob<job_id=1>running_tasks: 0
DelayedCCCJob<job_id=1>logdir: /tmp/hopla/logs/1_oneshot
----------------------------------------
DelayedCCCJob<job_id=2>exitcode: failure
DelayedCCCJob<job_id=2>submission: /tmp/hopla/submissions/2_submission.sh
DelayedCCCJob<job_id=2>stdout: none
DelayedCCCJob<job_id=2>stderr: none
DelayedCCCJob<job_id=2>number_of_tasks: 3
DelayedCCCJob<job_id=2>failed_tasks: 0
DelayedCCCJob<job_id=2>running_tasks: 0
DelayedCCCJob<job_id=2>logdir: /tmp/hopla/logs/2_oneshot
----------------------------------------
DelayedCCCJob<job_id=3>exitcode: failure
DelayedCCCJob<job_id=3>submission: /tmp/hopla/submissions/3_submission.sh
DelayedCCCJob<job_id=3>stdout: none
DelayedCCCJob<job_id=3>stderr: none
DelayedCCCJob<job_id=3>number_of_tasks: 3
DelayedCCCJob<job_id=3>failed_tasks: 0
DelayedCCCJob<job_id=3>running_tasks: 0
DelayedCCCJob<job_id=3>logdir: /tmp/hopla/logs/3_oneshot

Total running time of the script: (0 minutes 6.239 seconds)

Estimated memory usage: 109 MB

Gallery generated by Sphinx-Gallery