Basic example on how to use the CCC cluster using multi-tasks

Basic example

When you’re running hundreds or thousands of jobs, automation is a necessity. This is where hopla can help you.

A simple example of how to use hopla on a CCC cluster. Please check the user guide for a more in depth presentation of all functionalities.

Imports

import hopla
import numpy as np
from pprint import pprint

Executor Context

executor = hopla.Executor(
    cluster="ccc",
    folder="/tmp/hopla",
    queue="rome",
    image="/tmp/hopla/my-docker-img.tar",
    walltime=1,
    project_id="genXXX",
)

Submit Jobs

chunks = np.array_split(range(1, 11), 3)
jobs = [
    executor.submit([hopla.DelayedSubmission("sleep", k) for k in c])
    for c in chunks
]
pprint(jobs)
print(jobs[0].delayed_submission)

[DelayedCCCJob(
  job_id=1,
  submission_id=None,
  _hub=n4h00001rs,
  image_name=/tmp/hopla/my-docker-img.tar,
),
 DelayedCCCJob(
  job_id=2,
  submission_id=None,
  _hub=n4h00001rs,
  image_name=/tmp/hopla/my-docker-img.tar,
),
 DelayedCCCJob(
  job_id=3,
  submission_id=None,
  _hub=n4h00001rs,
  image_name=/tmp/hopla/my-docker-img.tar,
)]
[DelayedSubmission(
  command=sleep 1,
  execution_parameters=,
), DelayedSubmission(
  command=sleep 2,
  execution_parameters=,
), DelayedSubmission(
  command=sleep 3,
  execution_parameters=,
), DelayedSubmission(
  command=sleep 4,
  execution_parameters=,
)]

Generate a batch

jobs[0].generate_batch()
print(jobs[0].paths)
batch = jobs[0].paths.submission_file
with open(batch) as of:
    print(of.read())
tasks = jobs[0].paths.task_file
with open(tasks) as of:
    print(of.read())

/home/runner/work/hopla/hopla/examples/plot_ccc_multi_tasks.py:55: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
  jobs[0].generate_batch()
   |Traceback (most recent call last):
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 125, in generate_batch
   |    self.import_image()
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 175, in import_image
   |    stdout = subprocess.check_output(cmd)
   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
   |    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
   |    with Popen(*popenargs, **kwargs) as process:
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
   |    self._execute_child(args, executable, preexec_fn, close_fds,
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
   |    raise child_exception_type(errno_num, err_msg, err_filename)
   |FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'

JobPaths(
  flux_dir=/tmp/hopla/logs/1_flux,
  job_id=1,
  log_folder=/tmp/hopla/logs,
  stderr=/tmp/hopla/logs/1_log.err,
  stdout=/tmp/hopla/logs/1_log.out,
  submission_file=/tmp/hopla/submissions/1_submission.sh,
  submission_folder=/tmp/hopla/submissions,
  task_file=/tmp/hopla/submissions/1_tasks.txt,
  worker_file=/tmp/hopla/submissions/worker.sh,
)
#!/bin/bash

# Parameters
#MSUB -q rome
#MSUB -Q long
#MSUB -m workflash,scratch,work
#MSUB -T 3600
#MSUB -n 1
#MSUB -E "--gres=gpu:0"
#MSUB -M 2000
#MSUB -r hopla
#MSUB -e /tmp/hopla/logs/1_log.err
#MSUB -o /tmp/hopla/logs/1_log.out
#MSUB -A genXXX
#MSUB -F

# Environment
echo $SLURM_JOB_ID
echo $HOSTNAME


# Command
cd /tmp/hopla/logs/1_flux
ccc_mprun -D -v -B /tmp/hopla/submissions/1_tasks.txt

# Flux WIP
flux job wait --all
flux jobs -a
flux jobs -o "{status}:{returncode}"
flux jobs --filter=failed --format="{id} {exception.type} {exception.note}"

# End
echo "HOPLASAY-DONE"

1-1 . /tmp/hopla/submissions/worker.sh pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar  -- sleep 1
1-1 . /tmp/hopla/submissions/worker.sh pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar  -- sleep 2
1-1 . /tmp/hopla/submissions/worker.sh pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar  -- sleep 3
1-1 . /tmp/hopla/submissions/worker.sh pcocc-rs run n4h00001rs:/tmp/hopla/my-docker-img.tar  -- sleep 4

Start Jobs

We can’t execute the code on the CI since the CCC infrastructure is not available.

from hopla.config import Config

with Config(dryrun=True, delay_s=3):
    executor(max_jobs=2)

CCC_MSUB:   0%|          | 0/3 [00:00<?, ?it/s]
/home/runner/work/hopla/hopla/doc/../hopla/utils.py:354: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
  self.generate_batch()
   |Traceback (most recent call last):
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 125, in generate_batch
   |    self.import_image()
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 175, in import_image
   |    stdout = subprocess.check_output(cmd)
   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
   |    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
   |    with Popen(*popenargs, **kwargs) as process:
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
   |    self._execute_child(args, executable, preexec_fn, close_fds,
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
   |    raise child_exception_type(errno_num, err_msg, err_filename)
   |FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'

[command] ccc_msub /tmp/hopla/submissions/1_submission.sh

/home/runner/work/hopla/hopla/doc/../hopla/utils.py:354: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
  self.generate_batch()
   |Traceback (most recent call last):
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 125, in generate_batch
   |    self.import_image()
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 175, in import_image
   |    stdout = subprocess.check_output(cmd)
   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
   |    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
   |    with Popen(*popenargs, **kwargs) as process:
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
   |    self._execute_child(args, executable, preexec_fn, close_fds,
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
   |    raise child_exception_type(errno_num, err_msg, err_filename)
   |FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'

[command] ccc_msub /tmp/hopla/submissions/2_submission.sh

/home/runner/work/hopla/hopla/doc/../hopla/utils.py:354: UserWarning: Can't import image: /tmp/hopla/my-docker-img.tar
  self.generate_batch()
   |Traceback (most recent call last):
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 125, in generate_batch
   |    self.import_image()
   |  File "/home/runner/work/hopla/hopla/doc/../hopla/ccc.py", line 175, in import_image
   |    stdout = subprocess.check_output(cmd)
   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 466, in check_output
   |    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 548, in run
   |    with Popen(*popenargs, **kwargs) as process:
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1026, in __init__
   |    self._execute_child(args, executable, preexec_fn, close_fds,
   |  File "/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/subprocess.py", line 1955, in _execute_child
   |    raise child_exception_type(errno_num, err_msg, err_filename)
   |FileNotFoundError: [Errno 2] No such file or directory: 'pcocc-rs'

[command] ccc_msub /tmp/hopla/submissions/3_submission.sh

CCC_MSUB: 4it [00:03,  1.33it/s]
CCC_MSUB: 4it [00:06,  1.50s/it]

Estimated memory usage: 108 MB