#!/usr/bin/env python3
#
# rm-merged
#

import sys
import re

import os
import logging
from pprint import pprint
import subprocess as sp
from pathlib import Path
from functools import partial
# from subprocess import run, PIPE
from zipfile import ZipFile, ZIP_BZIP2


from tqdm import tqdm

run = partial(sp.Popen, stdout=sp.PIPE, encoding='utf-8')


def argparser():
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument("-o", "--output",
                        default=None,
                        help="Output file")
    parser.add_argument("--ignore",
                        action='store_true',
                        help='this option is ignored')
    parser.add_argument('masterjob',
                        nargs='+',)
    return parser


def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    args = argparser().parse_args(argv)

    env = os.environ.copy()

    wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
    gbbox_exe, *_ = wh.communicate()

    module = 'AliRoot/latest'
    if wh.returncode != 0:
        ali_work = os.environ.get("ALIBUILD_WORK_DIR", "$HOME/alice/sw")
        ali_arch = os.environ.get("ALIBUILD_ARCHITECTURE", "ubuntu1904_x86-64")
        module_dir = os.path.expandvars(f'{ali_work}/MODULES/{ali_arch}')

        logging.warning("Not in python environment; attempting to load %s from %s", module, module_dir)

        os.environ['MODULEPATH'] = ':'.join([module_dir, *os.environ.get('MODULEPATH', '').split(':')])
        cmd = run(['modulecmd', 'python', 'load', module], stderr=sp.DEVNULL)
        cmd_out, *_ = cmd.communicate()
        exec(cmd_out)

        wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
        if wh.wait() > 0:
            print("failed to load modules")
            return 1

    for masterjob in args.masterjob:
        remove_sources_files(masterjob)

    return 0


def remove_sources_files(masterjob, jdl_archive_file=None):
    print('removing source files for masterjob', masterjob)
    cmd = run(['gbbox', 'ps', '-As', f'-id={masterjob}'])
    cmd_out, *_ = cmd.communicate()
    jl = JobList(cmd_out)

    for pid in tqdm(jl.done()):
        ps_proc = run(['alien_ps', '-jdl', pid])
        jdl, *_ = ps_proc.communicate()

        m = re.search(r"InputData\s=\s+{([^}]+)}", jdl, re.M)
        if not m:
            continue

        input_dirs = [Path(a.group(1)).parent
                      for a in re.finditer("LF:([^,]+)", m.group(1))]

        rmdir_proc = run(['alien_rmdir', *input_dirs])
        rmdir_proc.wait()

        kill_proc = run(['alien_kill', pid])
        kill_proc.wait()
        if jdl_archive_file:
            save_jdl_to_archive(jdl, jdl_archive_file)


def save_jdl_to_archive(jdl, output):

    jdl_archive = Path(output)
    if jdl_archive.suffix == '.bz2':
        compression = ZIP_BZIP2
    else:
        compression = None

    with ZipFile(jdl_archive, 'a', compression=compression) as zfile:
        for pid in jl.running():
            subjobs = JobList.FromCommand('gbbox ps -As -id=%s' % pid)
            subjobs_done = set(subjobs.done())
            saved_pids = set(zfile.namelist())

            missing_pids = subjobs_done - saved_pids
            for pid in tqdm(missing_pids):
                # run.jobs()

                cmd = run(['alien_ps', '-jdl', pid])
                jdl_txt, *_ = cmd.communicate()
                # for pid, jjdl in jdl_txt:
                zfile.writestr('%s/jdl' % pid, jdl_txt)

                cmd = run(['alien_ps', '-trace', pid])
                trace_txt, *_ = cmd.communicate()
                # for pid, jjdl in trace_txt:
                zfile.writestr('%s/trace' % pid, trace_txt)

                cmd = run(['alien_kill', pid])
                cmd.wait()

    return
    print("Loading processes")
    cmd = run(['gbbox', 'ps', '-s', '-X'])
    cmd_out, *_ = cmd.communicate()
    print(cmd_out)
    # print('---')
    # print(os.environ['PATH'])
    # print('---')
    # print(os.environ['PATH'])

    ls = sp.Popen(['ls', '-l', '--color=always'], stdout=sp.PIPE)  # capture_output=True)
    # ls = sp.Popen(['ls', '-l'], stdout=sp.PIPE)  #, capture_output=True)
    print(ls)


    ls_out, _ = ls.communicate()
    print(ls_out.decode())


class JobList:

    REGEX = re.compile(r"""
                        (?P<user>[a-z]+)
                        \s+
                        (?P<splitjob>-)?
                        (?P<pid>[0-9]+)
                        \s+
                        (?P<status>[A-Z_]+)
                        \s+
                        (?P<runtime>[0-9:]+)?
                        \s+
                        (?P<path>\S+)
                       """, re.X)

    def __init__(self, text):

        self.jobs = [row.groupdict()
                     for row in self.REGEX.finditer(text)]

    def running(self):
        return [j['pid']
                for j in self.jobs
                if j['status'] in ('RS', 'R')]

    def done(self):
        return [j['pid']
                for j in self.jobs
                if j['status'] in ('D', 'D_WARN')]

    @classmethod
    def FromCommand(cls, args):
        cmd = run(args.split())
        cmd_out, *_ = cmd.communicate()
        print(cmd_out)
        return cls(cmd_out)


if __name__ == "__main__":
    logging.basicConfig(level=logging.ERROR)
    exit(main())

32c08f278b16f5d13c6c2666da6cd41721cb3979
<mask token>


def argparser():
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument('-o', '--output', default=None, help='Output file')
    parser.add_argument('--ignore', action='store_true', help=
        'this option is ignored')
    parser.add_argument('masterjob', nargs='+')
    return parser


def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    args = argparser().parse_args(argv)
    env = os.environ.copy()
    wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
    gbbox_exe, *_ = wh.communicate()
    module = 'AliRoot/latest'
    if wh.returncode != 0:
        ali_work = os.environ.get('ALIBUILD_WORK_DIR', '$HOME/alice/sw')
        ali_arch = os.environ.get('ALIBUILD_ARCHITECTURE', 'ubuntu1904_x86-64')
        module_dir = os.path.expandvars(f'{ali_work}/MODULES/{ali_arch}')
        logging.warning(
            'Not in python environment; attempting to load %s from %s',
            module, module_dir)
        os.environ['MODULEPATH'] = ':'.join([module_dir, *os.environ.get(
            'MODULEPATH', '').split(':')])
        cmd = run(['modulecmd', 'python', 'load', module], stderr=sp.DEVNULL)
        cmd_out, *_ = cmd.communicate()
        exec(cmd_out)
        wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
        if wh.wait() > 0:
            print('failed to load modules')
            return 1
    for masterjob in args.masterjob:
        remove_sources_files(masterjob)
    return 0


<mask token>


class JobList:
    REGEX = re.compile(
        """
                        (?P<user>[a-z]+)
                        \\s+
                        (?P<splitjob>-)?
                        (?P<pid>[0-9]+)
                        \\s+
                        (?P<status>[A-Z_]+)
                        \\s+
                        (?P<runtime>[0-9:]+)?
                        \\s+
                        (?P<path>\\S+)
                       """
        , re.X)

    def __init__(self, text):
        self.jobs = [row.groupdict() for row in self.REGEX.finditer(text)]

    def running(self):
        return [j['pid'] for j in self.jobs if j['status'] in ('RS', 'R')]

    def done(self):
        return [j['pid'] for j in self.jobs if j['status'] in ('D', 'D_WARN')]

    @classmethod
    def FromCommand(cls, args):
        cmd = run(args.split())
        cmd_out, *_ = cmd.communicate()
        print(cmd_out)
        return cls(cmd_out)


<mask token>

<mask token>


def argparser():
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument('-o', '--output', default=None, help='Output file')
    parser.add_argument('--ignore', action='store_true', help=
        'this option is ignored')
    parser.add_argument('masterjob', nargs='+')
    return parser


def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    args = argparser().parse_args(argv)
    env = os.environ.copy()
    wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
    gbbox_exe, *_ = wh.communicate()
    module = 'AliRoot/latest'
    if wh.returncode != 0:
        ali_work = os.environ.get('ALIBUILD_WORK_DIR', '$HOME/alice/sw')
        ali_arch = os.environ.get('ALIBUILD_ARCHITECTURE', 'ubuntu1904_x86-64')
        module_dir = os.path.expandvars(f'{ali_work}/MODULES/{ali_arch}')
        logging.warning(
            'Not in python environment; attempting to load %s from %s',
            module, module_dir)
        os.environ['MODULEPATH'] = ':'.join([module_dir, *os.environ.get(
            'MODULEPATH', '').split(':')])
        cmd = run(['modulecmd', 'python', 'load', module], stderr=sp.DEVNULL)
        cmd_out, *_ = cmd.communicate()
        exec(cmd_out)
        wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
        if wh.wait() > 0:
            print('failed to load modules')
            return 1
    for masterjob in args.masterjob:
        remove_sources_files(masterjob)
    return 0


def remove_sources_files(masterjob, jdl_archive_file=None):
    print('removing source files for masterjob', masterjob)
    cmd = run(['gbbox', 'ps', '-As', f'-id={masterjob}'])
    cmd_out, *_ = cmd.communicate()
    jl = JobList(cmd_out)
    for pid in tqdm(jl.done()):
        ps_proc = run(['alien_ps', '-jdl', pid])
        jdl, *_ = ps_proc.communicate()
        m = re.search('InputData\\s=\\s+{([^}]+)}', jdl, re.M)
        if not m:
            continue
        input_dirs = [Path(a.group(1)).parent for a in re.finditer(
            'LF:([^,]+)', m.group(1))]
        rmdir_proc = run(['alien_rmdir', *input_dirs])
        rmdir_proc.wait()
        kill_proc = run(['alien_kill', pid])
        kill_proc.wait()
        if jdl_archive_file:
            save_jdl_to_archive(jdl, jdl_archive_file)


def save_jdl_to_archive(jdl, output):
    jdl_archive = Path(output)
    if jdl_archive.suffix == '.bz2':
        compression = ZIP_BZIP2
    else:
        compression = None
    with ZipFile(jdl_archive, 'a', compression=compression) as zfile:
        for pid in jl.running():
            subjobs = JobList.FromCommand('gbbox ps -As -id=%s' % pid)
            subjobs_done = set(subjobs.done())
            saved_pids = set(zfile.namelist())
            missing_pids = subjobs_done - saved_pids
            for pid in tqdm(missing_pids):
                cmd = run(['alien_ps', '-jdl', pid])
                jdl_txt, *_ = cmd.communicate()
                zfile.writestr('%s/jdl' % pid, jdl_txt)
                cmd = run(['alien_ps', '-trace', pid])
                trace_txt, *_ = cmd.communicate()
                zfile.writestr('%s/trace' % pid, trace_txt)
                cmd = run(['alien_kill', pid])
                cmd.wait()
    return
    print('Loading processes')
    cmd = run(['gbbox', 'ps', '-s', '-X'])
    cmd_out, *_ = cmd.communicate()
    print(cmd_out)
    ls = sp.Popen(['ls', '-l', '--color=always'], stdout=sp.PIPE)
    print(ls)
    ls_out, _ = ls.communicate()
    print(ls_out.decode())


class JobList:
    REGEX = re.compile(
        """
                        (?P<user>[a-z]+)
                        \\s+
                        (?P<splitjob>-)?
                        (?P<pid>[0-9]+)
                        \\s+
                        (?P<status>[A-Z_]+)
                        \\s+
                        (?P<runtime>[0-9:]+)?
                        \\s+
                        (?P<path>\\S+)
                       """
        , re.X)

    def __init__(self, text):
        self.jobs = [row.groupdict() for row in self.REGEX.finditer(text)]

    def running(self):
        return [j['pid'] for j in self.jobs if j['status'] in ('RS', 'R')]

    def done(self):
        return [j['pid'] for j in self.jobs if j['status'] in ('D', 'D_WARN')]

    @classmethod
    def FromCommand(cls, args):
        cmd = run(args.split())
        cmd_out, *_ = cmd.communicate()
        print(cmd_out)
        return cls(cmd_out)


if __name__ == '__main__':
    logging.basicConfig(level=logging.ERROR)
    exit(main())

<mask token>
run = partial(sp.Popen, stdout=sp.PIPE, encoding='utf-8')


def argparser():
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument('-o', '--output', default=None, help='Output file')
    parser.add_argument('--ignore', action='store_true', help=
        'this option is ignored')
    parser.add_argument('masterjob', nargs='+')
    return parser


def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    args = argparser().parse_args(argv)
    env = os.environ.copy()
    wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
    gbbox_exe, *_ = wh.communicate()
    module = 'AliRoot/latest'
    if wh.returncode != 0:
        ali_work = os.environ.get('ALIBUILD_WORK_DIR', '$HOME/alice/sw')
        ali_arch = os.environ.get('ALIBUILD_ARCHITECTURE', 'ubuntu1904_x86-64')
        module_dir = os.path.expandvars(f'{ali_work}/MODULES/{ali_arch}')
        logging.warning(
            'Not in python environment; attempting to load %s from %s',
            module, module_dir)
        os.environ['MODULEPATH'] = ':'.join([module_dir, *os.environ.get(
            'MODULEPATH', '').split(':')])
        cmd = run(['modulecmd', 'python', 'load', module], stderr=sp.DEVNULL)
        cmd_out, *_ = cmd.communicate()
        exec(cmd_out)
        wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
        if wh.wait() > 0:
            print('failed to load modules')
            return 1
    for masterjob in args.masterjob:
        remove_sources_files(masterjob)
    return 0


def remove_sources_files(masterjob, jdl_archive_file=None):
    print('removing source files for masterjob', masterjob)
    cmd = run(['gbbox', 'ps', '-As', f'-id={masterjob}'])
    cmd_out, *_ = cmd.communicate()
    jl = JobList(cmd_out)
    for pid in tqdm(jl.done()):
        ps_proc = run(['alien_ps', '-jdl', pid])
        jdl, *_ = ps_proc.communicate()
        m = re.search('InputData\\s=\\s+{([^}]+)}', jdl, re.M)
        if not m:
            continue
        input_dirs = [Path(a.group(1)).parent for a in re.finditer(
            'LF:([^,]+)', m.group(1))]
        rmdir_proc = run(['alien_rmdir', *input_dirs])
        rmdir_proc.wait()
        kill_proc = run(['alien_kill', pid])
        kill_proc.wait()
        if jdl_archive_file:
            save_jdl_to_archive(jdl, jdl_archive_file)


def save_jdl_to_archive(jdl, output):
    jdl_archive = Path(output)
    if jdl_archive.suffix == '.bz2':
        compression = ZIP_BZIP2
    else:
        compression = None
    with ZipFile(jdl_archive, 'a', compression=compression) as zfile:
        for pid in jl.running():
            subjobs = JobList.FromCommand('gbbox ps -As -id=%s' % pid)
            subjobs_done = set(subjobs.done())
            saved_pids = set(zfile.namelist())
            missing_pids = subjobs_done - saved_pids
            for pid in tqdm(missing_pids):
                cmd = run(['alien_ps', '-jdl', pid])
                jdl_txt, *_ = cmd.communicate()
                zfile.writestr('%s/jdl' % pid, jdl_txt)
                cmd = run(['alien_ps', '-trace', pid])
                trace_txt, *_ = cmd.communicate()
                zfile.writestr('%s/trace' % pid, trace_txt)
                cmd = run(['alien_kill', pid])
                cmd.wait()
    return
    print('Loading processes')
    cmd = run(['gbbox', 'ps', '-s', '-X'])
    cmd_out, *_ = cmd.communicate()
    print(cmd_out)
    ls = sp.Popen(['ls', '-l', '--color=always'], stdout=sp.PIPE)
    print(ls)
    ls_out, _ = ls.communicate()
    print(ls_out.decode())


class JobList:
    REGEX = re.compile(
        """
                        (?P<user>[a-z]+)
                        \\s+
                        (?P<splitjob>-)?
                        (?P<pid>[0-9]+)
                        \\s+
                        (?P<status>[A-Z_]+)
                        \\s+
                        (?P<runtime>[0-9:]+)?
                        \\s+
                        (?P<path>\\S+)
                       """
        , re.X)

    def __init__(self, text):
        self.jobs = [row.groupdict() for row in self.REGEX.finditer(text)]

    def running(self):
        return [j['pid'] for j in self.jobs if j['status'] in ('RS', 'R')]

    def done(self):
        return [j['pid'] for j in self.jobs if j['status'] in ('D', 'D_WARN')]

    @classmethod
    def FromCommand(cls, args):
        cmd = run(args.split())
        cmd_out, *_ = cmd.communicate()
        print(cmd_out)
        return cls(cmd_out)


if __name__ == '__main__':
    logging.basicConfig(level=logging.ERROR)
    exit(main())

import sys
import re
import os
import logging
from pprint import pprint
import subprocess as sp
from pathlib import Path
from functools import partial
from zipfile import ZipFile, ZIP_BZIP2
from tqdm import tqdm
run = partial(sp.Popen, stdout=sp.PIPE, encoding='utf-8')


def argparser():
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument('-o', '--output', default=None, help='Output file')
    parser.add_argument('--ignore', action='store_true', help=
        'this option is ignored')
    parser.add_argument('masterjob', nargs='+')
    return parser


def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    args = argparser().parse_args(argv)
    env = os.environ.copy()
    wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
    gbbox_exe, *_ = wh.communicate()
    module = 'AliRoot/latest'
    if wh.returncode != 0:
        ali_work = os.environ.get('ALIBUILD_WORK_DIR', '$HOME/alice/sw')
        ali_arch = os.environ.get('ALIBUILD_ARCHITECTURE', 'ubuntu1904_x86-64')
        module_dir = os.path.expandvars(f'{ali_work}/MODULES/{ali_arch}')
        logging.warning(
            'Not in python environment; attempting to load %s from %s',
            module, module_dir)
        os.environ['MODULEPATH'] = ':'.join([module_dir, *os.environ.get(
            'MODULEPATH', '').split(':')])
        cmd = run(['modulecmd', 'python', 'load', module], stderr=sp.DEVNULL)
        cmd_out, *_ = cmd.communicate()
        exec(cmd_out)
        wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
        if wh.wait() > 0:
            print('failed to load modules')
            return 1
    for masterjob in args.masterjob:
        remove_sources_files(masterjob)
    return 0


def remove_sources_files(masterjob, jdl_archive_file=None):
    print('removing source files for masterjob', masterjob)
    cmd = run(['gbbox', 'ps', '-As', f'-id={masterjob}'])
    cmd_out, *_ = cmd.communicate()
    jl = JobList(cmd_out)
    for pid in tqdm(jl.done()):
        ps_proc = run(['alien_ps', '-jdl', pid])
        jdl, *_ = ps_proc.communicate()
        m = re.search('InputData\\s=\\s+{([^}]+)}', jdl, re.M)
        if not m:
            continue
        input_dirs = [Path(a.group(1)).parent for a in re.finditer(
            'LF:([^,]+)', m.group(1))]
        rmdir_proc = run(['alien_rmdir', *input_dirs])
        rmdir_proc.wait()
        kill_proc = run(['alien_kill', pid])
        kill_proc.wait()
        if jdl_archive_file:
            save_jdl_to_archive(jdl, jdl_archive_file)


def save_jdl_to_archive(jdl, output):
    jdl_archive = Path(output)
    if jdl_archive.suffix == '.bz2':
        compression = ZIP_BZIP2
    else:
        compression = None
    with ZipFile(jdl_archive, 'a', compression=compression) as zfile:
        for pid in jl.running():
            subjobs = JobList.FromCommand('gbbox ps -As -id=%s' % pid)
            subjobs_done = set(subjobs.done())
            saved_pids = set(zfile.namelist())
            missing_pids = subjobs_done - saved_pids
            for pid in tqdm(missing_pids):
                cmd = run(['alien_ps', '-jdl', pid])
                jdl_txt, *_ = cmd.communicate()
                zfile.writestr('%s/jdl' % pid, jdl_txt)
                cmd = run(['alien_ps', '-trace', pid])
                trace_txt, *_ = cmd.communicate()
                zfile.writestr('%s/trace' % pid, trace_txt)
                cmd = run(['alien_kill', pid])
                cmd.wait()
    return
    print('Loading processes')
    cmd = run(['gbbox', 'ps', '-s', '-X'])
    cmd_out, *_ = cmd.communicate()
    print(cmd_out)
    ls = sp.Popen(['ls', '-l', '--color=always'], stdout=sp.PIPE)
    print(ls)
    ls_out, _ = ls.communicate()
    print(ls_out.decode())


class JobList:
    REGEX = re.compile(
        """
                        (?P<user>[a-z]+)
                        \\s+
                        (?P<splitjob>-)?
                        (?P<pid>[0-9]+)
                        \\s+
                        (?P<status>[A-Z_]+)
                        \\s+
                        (?P<runtime>[0-9:]+)?
                        \\s+
                        (?P<path>\\S+)
                       """
        , re.X)

    def __init__(self, text):
        self.jobs = [row.groupdict() for row in self.REGEX.finditer(text)]

    def running(self):
        return [j['pid'] for j in self.jobs if j['status'] in ('RS', 'R')]

    def done(self):
        return [j['pid'] for j in self.jobs if j['status'] in ('D', 'D_WARN')]

    @classmethod
    def FromCommand(cls, args):
        cmd = run(args.split())
        cmd_out, *_ = cmd.communicate()
        print(cmd_out)
        return cls(cmd_out)


if __name__ == '__main__':
    logging.basicConfig(level=logging.ERROR)
    exit(main())

#!/usr/bin/env python3
#
# rm-merged
#

import sys
import re

import os
import logging
from pprint import pprint
import subprocess as sp
from pathlib import Path
from functools import partial
# from subprocess import run, PIPE
from zipfile import ZipFile, ZIP_BZIP2


from tqdm import tqdm

run = partial(sp.Popen, stdout=sp.PIPE, encoding='utf-8')


def argparser():
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument("-o", "--output",
                        default=None,
                        help="Output file")
    parser.add_argument("--ignore",
                        action='store_true',
                        help='this option is ignored')
    parser.add_argument('masterjob',
                        nargs='+',)
    return parser


def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    args = argparser().parse_args(argv)

    env = os.environ.copy()

    wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
    gbbox_exe, *_ = wh.communicate()

    module = 'AliRoot/latest'
    if wh.returncode != 0:
        ali_work = os.environ.get("ALIBUILD_WORK_DIR", "$HOME/alice/sw")
        ali_arch = os.environ.get("ALIBUILD_ARCHITECTURE", "ubuntu1904_x86-64")
        module_dir = os.path.expandvars(f'{ali_work}/MODULES/{ali_arch}')

        logging.warning("Not in python environment; attempting to load %s from %s", module, module_dir)

        os.environ['MODULEPATH'] = ':'.join([module_dir, *os.environ.get('MODULEPATH', '').split(':')])
        cmd = run(['modulecmd', 'python', 'load', module], stderr=sp.DEVNULL)
        cmd_out, *_ = cmd.communicate()
        exec(cmd_out)

        wh = sp.Popen(['/usr/bin/which', 'gbbox'], stdout=sp.PIPE)
        if wh.wait() > 0:
            print("failed to load modules")
            return 1

    for masterjob in args.masterjob:
        remove_sources_files(masterjob)

    return 0


def remove_sources_files(masterjob, jdl_archive_file=None):
    print('removing source files for masterjob', masterjob)
    cmd = run(['gbbox', 'ps', '-As', f'-id={masterjob}'])
    cmd_out, *_ = cmd.communicate()
    jl = JobList(cmd_out)

    for pid in tqdm(jl.done()):
        ps_proc = run(['alien_ps', '-jdl', pid])
        jdl, *_ = ps_proc.communicate()

        m = re.search(r"InputData\s=\s+{([^}]+)}", jdl, re.M)
        if not m:
            continue

        input_dirs = [Path(a.group(1)).parent
                      for a in re.finditer("LF:([^,]+)", m.group(1))]

        rmdir_proc = run(['alien_rmdir', *input_dirs])
        rmdir_proc.wait()

        kill_proc = run(['alien_kill', pid])
        kill_proc.wait()
        if jdl_archive_file:
            save_jdl_to_archive(jdl, jdl_archive_file)


def save_jdl_to_archive(jdl, output):

    jdl_archive = Path(output)
    if jdl_archive.suffix == '.bz2':
        compression = ZIP_BZIP2
    else:
        compression = None

    with ZipFile(jdl_archive, 'a', compression=compression) as zfile:
        for pid in jl.running():
            subjobs = JobList.FromCommand('gbbox ps -As -id=%s' % pid)
            subjobs_done = set(subjobs.done())
            saved_pids = set(zfile.namelist())

            missing_pids = subjobs_done - saved_pids
            for pid in tqdm(missing_pids):
                # run.jobs()

                cmd = run(['alien_ps', '-jdl', pid])
                jdl_txt, *_ = cmd.communicate()
                # for pid, jjdl in jdl_txt:
                zfile.writestr('%s/jdl' % pid, jdl_txt)

                cmd = run(['alien_ps', '-trace', pid])
                trace_txt, *_ = cmd.communicate()
                # for pid, jjdl in trace_txt:
                zfile.writestr('%s/trace' % pid, trace_txt)

                cmd = run(['alien_kill', pid])
                cmd.wait()

    return
    print("Loading processes")
    cmd = run(['gbbox', 'ps', '-s', '-X'])
    cmd_out, *_ = cmd.communicate()
    print(cmd_out)
    # print('---')
    # print(os.environ['PATH'])
    # print('---')
    # print(os.environ['PATH'])

    ls = sp.Popen(['ls', '-l', '--color=always'], stdout=sp.PIPE)  # capture_output=True)
    # ls = sp.Popen(['ls', '-l'], stdout=sp.PIPE)  #, capture_output=True)
    print(ls)


    ls_out, _ = ls.communicate()
    print(ls_out.decode())


class JobList:

    REGEX = re.compile(r"""
                        (?P<user>[a-z]+)
                        \s+
                        (?P<splitjob>-)?
                        (?P<pid>[0-9]+)
                        \s+
                        (?P<status>[A-Z_]+)
                        \s+
                        (?P<runtime>[0-9:]+)?
                        \s+
                        (?P<path>\S+)
                       """, re.X)

    def __init__(self, text):

        self.jobs = [row.groupdict()
                     for row in self.REGEX.finditer(text)]

    def running(self):
        return [j['pid']
                for j in self.jobs
                if j['status'] in ('RS', 'R')]

    def done(self):
        return [j['pid']
                for j in self.jobs
                if j['status'] in ('D', 'D_WARN')]

    @classmethod
    def FromCommand(cls, args):
        cmd = run(args.split())
        cmd_out, *_ = cmd.communicate()
        print(cmd_out)
        return cls(cmd_out)


if __name__ == "__main__":
    logging.basicConfig(level=logging.ERROR)
    exit(main())
