utils/cluster.py

import os
import sys
import stat
import shutil
import subprocess

from loguru import logger

GPUS = {
    'v100-v16': ('\"Tesla V100-PCIE-16GB\"', 'tesla', 16000),
    'v100-p32': ('\"Tesla V100-PCIE-32GB\"', 'tesla', 32000),
    'v100-s32': ('\"Tesla V100-SXM2-32GB\"', 'tesla', 32000),
    'v100-p16': ('\"Tesla P100-PCIE-16GB\"', 'tesla', 16000),
}

def get_gpus(min_mem=10000, arch=('tesla', 'quadro', 'rtx')):
    gpu_names = []
    for k, (gpu_name, gpu_arch, gpu_mem) in GPUS.items():
        if gpu_mem >= min_mem and gpu_arch in arch:
            gpu_names.append(gpu_name)

    assert len(gpu_names) > 0, 'Suitable GPU model could not be found'

    return gpu_names


def execute_task_on_cluster(
        script,
        exp_name,
        output_dir,
        condor_dir,
        cfg_file,
        num_exp=1,
        exp_opts=None,
        bid_amount=10,
        num_workers=2,
        memory=64000,
        gpu_min_mem=10000,
        gpu_arch=('tesla', 'quadro', 'rtx'),
        num_gpus=1
):
    # copy config to a new experiment directory and source from there.
    # this makes sure the correct config is copied even if you change the config file
    # after starting the experiment and before the first job is submitted
    temp_config_dir = os.path.join(os.path.dirname(condor_dir), 'temp_configs', exp_name)
    os.makedirs(temp_config_dir, exist_ok=True)
    new_cfg_file = os.path.join(temp_config_dir, 'config.yaml')
    shutil.copy(src=cfg_file, dst=new_cfg_file)

    gpus = get_gpus(min_mem=gpu_min_mem, arch=gpu_arch)

    gpus = ' || '.join([f'CUDADeviceName=={x}' for x in gpus])

    condor_log_dir = os.path.join(condor_dir, 'condorlog', exp_name)
    os.makedirs(condor_log_dir, exist_ok=True)
    submission = f'executable = {condor_log_dir}/{exp_name}_run.sh\n' \
                 'arguments = $(Process) $(Cluster)\n' \
                 f'error = {condor_log_dir}/{exp_name}_$(Cluster).$(Process).err\n' \
                 f'output = {condor_log_dir}/{exp_name}_$(Cluster).$(Process).out\n' \
                 f'log = {condor_log_dir}/{exp_name}_$(Cluster).$(Process).log\n' \
                 f'request_memory = {memory}\n' \
                 f'request_cpus={int(num_workers)}\n' \
                 f'request_gpus={num_gpus}\n' \
                 f'requirements={gpus}\n' \
                 f'+MaxRunningPrice = 500\n' \
                 f'queue {num_exp}'
                 # f'request_cpus={int(num_workers/2)}\n' \
                 # f'+RunningPriceExceededAction = \"kill\"\n' \
    print('<<< Condor Submission >>> ')
    print(submission)

    with open(f'{condor_log_dir}/{exp_name}_submit.sub', 'w') as f:
        f.write(submission)

    # output_dir = os.path.join(output_dir, exp_name)
    logger.info(f'The logs for this experiments can be found under: {condor_log_dir}')
    logger.info(f'The outputs for this experiments can be found under: {output_dir}')
    ## This is the trick. Notice there is no --cluster here
    bash = 'export PYTHONBUFFERED=1\n export PATH=$PATH\n ' \
           f'{sys.executable} {script} --cfg {new_cfg_file} --cfg_id $1'

    if exp_opts is not None:
        bash += ' --opts '
        for opt in exp_opts:
            bash += f'{opt} '
        bash += 'SYSTEM.CLUSTER_NODE $2.$1'
    else:
        bash += ' --opts SYSTEM.CLUSTER_NODE $2.$1'

    executable_path = f'{condor_log_dir}/{exp_name}_run.sh'

    with open(executable_path, 'w') as f:
        f.write(bash)

    os.chmod(executable_path, stat.S_IRWXU)

    cmd = ['condor_submit_bid', f'{bid_amount}', f'{condor_log_dir}/{exp_name}_submit.sub']
    logger.info('Executing ' + ' '.join(cmd))
    subprocess.call(cmd)