SCRIPTS SLURM collect_prun_batch_slurmΒΆ

# -*-Python-*-
# Created by bgriers at 04 Jul 2018  10:30

"""
This script gathers the code output from our batch jobs

"""

runids = list(root['OUTPUTS']['SLURM']['RUN_DB'].keys())

# Print job info
for runid in runids:
    if 'job' in root['OUTPUTS']['SLURM']['RUN_DB'][runid]:
        print('Job for {}'.format(runid))
        print(' > has ID {}'.format(root['OUTPUTS']['SLURM']['RUN_DB'][runid]['job'].jobID))
        print(' > has qstat {}'.format(root['OUTPUTS']['SLURM']['RUN_DB'][runid]['job'].qstat()))
        print(' > has remotedir {}'.format(root['OUTPUTS']['SLURM']['RUN_DB'][runid]['job'].remotedir))

        qstat = root['OUTPUTS']['SLURM']['RUN_DB'][runid]['job'].qstat()
        if qstat == 'R':
            printw('Job {} is still running...'.format(runid))
            # Here you can downsync an in-progress run if your code writes output as it goes
        elif qstat is None:
            printi('Job {} appears to be finished.... downsyncing'.format(runid))
            remotedir = root['OUTPUTS']['SLURM']['RUN_DB'][runid]['job'].remotedir
            printi(' > from {}'.format(remotedir))
            OMFITx.executable(root, (), ['pwd.txt'], executable='', clean='local', remotedir=remotedir)
            try:
                pwd = OMFITascii('./pwd.txt')
            except OMFITexception:
                printe('Cannot load pwd.txt as OMFITascii')
            if isinstance(pwd, OMFITascii):
                root['OUTPUTS']['SLURM']['RUN_DB'][runid]['pwd'] = pwd
        else:
            printe('Cannot determine status of job')
    else:
        printw('Job info for {} not available.  Was it even run with job manager?'.format(runid))