def clean():
    proj()
    print ". cleaning up build and dist"
    try:
        sh.rm("-r", sh.glob("dist/*"), sh.glob("build/*"))
    except:
        print ".. already clean"

    def reduce_python(self):
        print("Reduce python")
        oldpwd = os.getcwd()
        try:
            print("Remove files unlikely to be used")
            os.chdir(join(self.ctx.dist_dir, "root", "python"))
            sh.rm("-rf", "share")
            sh.rm("-rf", "bin")
            os.chdir(join(self.ctx.dist_dir, "root", "python", "lib"))
            sh.rm("-rf", "pkgconfig")
            sh.rm("libpython2.7.a")
            os.chdir(join(self.ctx.dist_dir, "root", "python", "lib", "python2.7"))
            sh.find(".", "-iname", "*.pyc", "-exec", "rm", "{}", ";")
            sh.find(".", "-iname", "*.py", "-exec", "rm", "{}", ";")
            #sh.find(".", "-iname", "test*", "-exec", "rm", "-rf", "{}", ";")
            sh.rm("-rf", "wsgiref", "bsddb", "curses", "idlelib", "hotshot")
            sh.rm("-rf", sh.glob("lib*"))

            # now create the zip.
            print("Create a python27.zip")
            sh.rm("config/libpython2.7.a")
            sh.rm("config/python.o")
            sh.rm("config/config.c.in")
            sh.rm("config/makesetup")
            sh.rm("config/install-sh")
            sh.mv("config", "..")
            sh.mv("site-packages", "..")
            sh.zip("-r", "../python27.zip", sh.glob("*"))
            sh.rm("-rf", sh.glob("*"))
            sh.mv("../config", ".")
            sh.mv("../site-packages", ".")
        finally:
            os.chdir(oldpwd)

def save_data(last_submitted, sim_data, sim_type):
    qstat = sh.Command("qstat")
    grep = sh.Command("grep")

    are_on_queue = True

    while are_on_queue:
        try:
            grep(qstat(), str(last_submitted))
            print "{0} submitted last still on queue," \
                  " waiting 20 min from {1}".format(last_submitted, datetime.datetime.now().time())
            time.sleep(20 * 60)
        except:
            are_on_queue = False

    tar = sh.Command("tar")

    for name, tar_pattern in iterate_over_folders(sim_data, [create_mini_tar_names], sim_type):
        tar("-czpf", name, sh.glob(tar_pattern))

    for name, tar_pattern in iterate_over_folders(sim_data, [create_full_tar_names], sim_type):
        try:
            tar("-czpf", name, sh.glob(tar_pattern))
        except:
            pass

    for name, tar_pattern in iterate_over_folders(sim_data, [create_pics_tar_names], sim_type):
        try:
            tar("-czpf", name, sh.glob(tar_pattern))
        except:
            pass

    def reduce_python(self):
        logger.info("Reduce python")
        oldpwd = os.getcwd()
        try:
            logger.info("Remove files unlikely to be used")
            os.chdir(join(self.ctx.dist_dir, "root", "python3"))
            # os.execve("/bin/bash", ["/bin/bash"], env=os.environ)
            sh.rm("-rf", "bin", "share")

            # platform binaries and configuration
            os.chdir(join(
                self.ctx.dist_dir, "root", "python3", "lib",
                "python3.7", "config-3.7m-darwin"))
            sh.rm("libpython3.7m.a")
            sh.rm("python.o")
            sh.rm("config.c.in")
            sh.rm("makesetup")
            sh.rm("install-sh")

            # cleanup pkgconfig and compiled lib
            os.chdir(join(self.ctx.dist_dir, "root", "python3", "lib"))
            sh.rm("-rf", "pkgconfig")
            sh.rm("-f", "libpython3.7m.a")

            # cleanup python libraries
            os.chdir(join(
                self.ctx.dist_dir, "root", "python3", "lib", "python3.7"))
            sh.rm("-rf", "wsgiref", "curses", "idlelib", "lib2to3",
                  "ensurepip", "turtledemo", "lib-dynload", "venv",
                  "pydoc_data")
            sh.find(".", "-path", "*/test*/*", "-delete")
            sh.find(".", "-name", "*.exe", "-type", "f", "-delete")
            sh.find(".", "-name", "test*", "-type", "d", "-delete")
            sh.find(".", "-iname", "*.pyc", "-delete")
            sh.find(".", "-path", "*/__pycache__/*", "-delete")
            sh.find(".", "-name", "__pycache__", "-type", "d", "-delete")

            # now precompile to Python bytecode
            hostpython = sh.Command(self.ctx.hostpython)
            shprint(hostpython, "-m", "compileall", "-f", "-b")
            # sh.find(".", "-iname", "*.py", "-delete")

            # some pycache are recreated after compileall
            sh.find(".", "-path", "*/__pycache__/*", "-delete")
            sh.find(".", "-name", "__pycache__", "-type", "d", "-delete")

            # create the lib zip
            logger.info("Create a python3.7.zip")
            sh.mv("config-3.7m-darwin", "..")
            sh.mv("site-packages", "..")
            sh.zip("-r", "../python37.zip", sh.glob("*"))
            sh.rm("-rf", sh.glob("*"))
            sh.mv("../config-3.7m-darwin", ".")
            sh.mv("../site-packages", ".")
        finally:
            os.chdir(oldpwd)

def clean():
    """clean up generated files"""
    proj()
    print ". cleaning up build and dist"
    try:
        sh.rm("-r",
            sh.glob("dist/*"),
            sh.glob("build/*")
        )
    except Exception, err:
        print ".. already clean: %s" % err

 def split(self):
     # Call Roche binary #
     barcode_file = TmpFile.from_string(self.barcode_text)
     sh.sfffile("-s", "barcodes_keyword", "-mcf", barcode_file.path, self.path)
     # Check result #
     produced_files = set(sh.glob('454Reads.*.sff'))
     expected_files = set(['454Reads.%s.sff' % (sample.name.upper()) for sample in self.sample_links.values()])
     assert produced_files == expected_files
     # Make piece objects #
     self.pieces = [SamplePiece(p, self) for p in sh.glob('454Reads.*.sff')]
     for piece in self.pieces: piece.rename()
     # Cleanup #
     barcode_file.remove()

 def split(self):
     # Call Roche binary #
     handle = tempfile.NamedTemporaryFile(delete=False)
     handle.write(self.barcode_file)
     handle.close()
     sh.sfffile("-s", "barcodes_keyword", "-mcf", handle.name, self.path)
     # Check result #
     produced_files = set(sh.glob('454Reads.*.sff'))
     expected_files = set(['454Reads.%s.sff' % (sample.name.upper()) for sample in self.sample_links.values()])
     assert produced_files == expected_files
     # Make piece objects #
     self.pieces = [SamplePiece(p, self) for p in sh.glob('454Reads.*.sff')]
     for piece in self.pieces: piece.rename()
     os.remove(handle.name)

    def post_syslog(self, message, response):
        output = status.tar_syslog_files(
            "/run/shm/syslog-%s.tar.gz" %
            (datetime.datetime.now().strftime("%Y%m%d%H%M")))
        headers = message.data.get("headers", {})
        r = requests.post(
            message.data["url"],
            files={output: open(output, "rb")},
            headers=headers,
            verify=False
        )

        if r.status_code != requests.codes.ok:
            return response(
                code=r.status_code,
                data={"message": "Can't upload config."}
            )

        sh.rm("-rf", sh.glob("/run/shm/syslog-*.tar.gz"))
        resp = r.json()
        if "url" not in resp:
            return response(
                code=500, data={"message": "Can't get file link."})

        return response(data={"url": resp["url"]})

def runtime_assets():
        rt_cfg = dict(
            themes=dict(
                path="lib/swatch/*.css",
                sub_data=lambda x: x.split(".")[1],
                sub_text=lambda x: x
            ),
            code_themes=dict(
                path="lib/cm/theme/*.css",
                sub_data=lambda x: os.path.basename(x)[0:-4],
                sub_text=lambda x: " ".join(x.split("-")).title()
            ),
            examples=dict(
                path="blockml/*.xml",
                sub_data=lambda x: os.path.basename(x)[0:-4],
                sub_text=lambda x: " ".join(x.split("_")).title()
            )
        )
        
        result = {}
        
        for thing, cfg in rt_cfg.items():
            result[thing] = sorted([
                (cfg["sub_text"](cfg["sub_data"](path)), cfg["sub_data"](path))
                for path in sh.glob(cfg["path"])
            ], key=lambda x: x[0].upper())
        return result

def asset_links(asset_type):
    template = """
                <li><a href="#%(thing)s" data-blockd3-%(thing)s="%(file)s">
                    %(text)s
                </a></li>"""
    cfg = dict(
        THEMES=dict(
            path="lib/swatch/*.css",
            thing="theme",
            sub_data=lambda x: x.split(".")[1],
            sub_text=lambda x: x
        ),
        EXAMPLES=dict(
            path="blockml/*.xml",
            thing="example",
            sub_data=lambda x: os.path.basename(x)[0:-4],
            sub_text=lambda x: " ".join(x.split("_")).title()
        )
    )[asset_type]
    return "\n".join([
        template % {
            "file": cfg["sub_data"](path),
            "thing": cfg["thing"],
            "text": cfg["sub_text"](cfg["sub_data"](path))
        }
        for path in sh.glob(cfg["path"])
    ])

def testinfra(testinfra_dir,
              debug=False,
              env=os.environ.copy(),
              out=logger.warning,
              err=logger.error,
              **kwargs):
    """
    Runs testinfra against specified ansible inventory file

    :param inventory: Path to ansible inventory file
    :param testinfra_dir: Path to the testinfra tests
    :param debug: Pass debug flag to testinfra
    :param env: Environment to pass to underlying sh call
    :param out: Function to process STDOUT for underlying sh call
    :param err: Function to process STDERR for underlying sh call
    :return: sh response object
    """
    kwargs['debug'] = debug
    kwargs['_env'] = env
    kwargs['_out'] = out
    kwargs['_err'] = err

    if 'HOME' not in kwargs['_env']:
        kwargs['_env']['HOME'] = os.path.expanduser('~')

    tests = '{}/test_*.py'.format(testinfra_dir)
    tests_glob = sh.glob(tests)

    return sh.testinfra(tests_glob, **kwargs)

def set_current(timestamp):
    """
    Set an app directory to the currently live app
    by creating a symlink as specified in config
    """

    app_path = path.join(install_parent, timestamp)

    log(
        "Linking live path '{live}' to app dir: {app_dir}".format(
            app_dir=app_path, live=live_link_path
        )
    )

    run(sh.rm, live_link_path, force=True)
    run(sh.ln, app_path, live_link_path, symbolic=True)

    site_to_enable = path.join(sites_available_dir, timestamp)

    site_links = sh.glob(path.join(sites_enabled_dir, '*'))

    # Delete existing site links
    run(sh.rm, site_links, f=True)

    # Add our link into sites-enabled
    run(sh.ln, site_to_enable, sites_enabled_path, s=True)

    # Restart apache
    restart()

def testinfra(inventory,
              testinfra_dir,
              debug=False,
              env=None,
              out=print_stdout,
              err=print_stderr):
    """
    Runs testinfra against specified ansible inventory file

    :param inventory: Path to ansible inventory file
    :param testinfra_dir: Path to the testinfra tests
    :param debug: Pass debug flag to testinfra
    :param env: Environment to pass to underlying sh call
    :param out: Function to process STDOUT for underlying sh call
    :param err: Function to process STDERR for underlying sh call
    :return: sh response object
    """
    kwargs = {
        '_env': env,
        '_out': out,
        '_err': err,
        'debug': debug,
        'ansible_inventory': inventory,
        'sudo': True,
        'connection': 'ansible',
        'n': 3
    }

    if 'HOME' not in kwargs['_env']:
        kwargs['_env']['HOME'] = os.path.expanduser('~')

    tests = '{}/test_*.py'.format(testinfra_dir)
    tests_glob = sh.glob(tests)

    return sh.testinfra(tests_glob, **kwargs)

    def search(self, package, path):
        '''Looks for package in files in path using zgrep shell binary.'''

        try:
            log_lines = sh.zgrep(package, glob(path))
        except sh.ErrorReturnCode_1 as e:      # buffer overflown??
            # don't know why this happens when using sh.
            log_lines = e.stdout

        log_lines = log_lines.split("\n")
        # get all but the last line -> get rid of last '' empty line
        log_lines = log_lines[:-1]

        for line in log_lines:
            #logger.debug("Following line was found:\n%s" % line)
            logger.debug("Following line containing metapackage was found "
                         "in package manager's log files:")
            print(line)
            self.installation_lines.append(line)

        if not self.installation_lines:
            logger.info("zgrep could not find in logs any info that ",
                        "can be used to uninstall the package.",
                        "Exiting...")
            sys.exit()
        else:
            logger.info("Search results from zgrep where collected.")

        self._check()
        return self.main_line

def favicon():
    """generate the favicon... ugly"""
    proj()
    print(". generating favicons...")
    sizes = [16, 32, 64, 128]

    tmp_file = lambda size: "/tmp/favicon-%s.png" % size

    for size in sizes:
        print("... %sx%s" % (size, size))
        sh.convert(
            "design/logo.svg",
            "-resize",
            "%sx%s" % (size, size),
            tmp_file(size))

    print(".. generating bundle")

    sh.convert(
        *[tmp_file(size) for size in sizes] + [
            "-colors", 256,
            "static/img/favicon.ico"
        ]
    )

    print(".. cleaning up")
    sh.rm(sh.glob("/tmp/favicon-*.png"))

def test_piping():
    from sh import sort, du, glob, wc, ls

    # sort this directory by biggest file
    print sort(du(glob('*'), '-sb'), '-rn')

    # print the number of folders and files in /etc
    print wc(ls('/etc', '-l'), '-l')

def test_glob_expansion():
    # TODO: error
    import sh

    # this will not work
    sh.ls('*.py')

    sh.ls(sh.glob('*.py'))

    def modified(self):
        print "purging cache for {slug}".format(slug=self.slug)

        if s.WMS_CACHE_DB.exists(self.slug):
            cached_filenames = s.WMS_CACHE_DB.smembers(self.slug)
            for filename in cached_filenames:
                sh.rm('-rf', sh.glob(filename+"*"))
            s.WMS_CACHE_DB.srem(self.slug, cached_filenames)

def clean_up(files, email_name, options):

    log.info("Preparing to clean up")
    mv = sh.mv
    move_tups = []

    stat_glob = options.dir+"*.stat/*"
    stat_dir_glob = options.dir+"*.stat/"
    stat_dest = options.output+"stat/"

    count_glob = options.dir+"*.genes.results"
    isoform_glob = options.dir+"*isoforms.results"
    count_dest = options.output+"counts/"

    log.info("Moving data files")
    
    if not options.no_bam:
        gbam_glob = options.dir+"*.genome.sorted.bam"
        gbam_dest = options.output+"bams/"
        move_tups.append((gbam_glob, gbam_dest))

    if not options.no_bam and not options.no_wig:
        wig_glob = options.output+"*.wig"
        bw_glob = options.output+"*.bw"
        vis_dest = options.output+"vis/"
        move_tups.append((wig_glob, vis_dest))
        move_tups.append((bw_glob, vis_dest))

    for glob, dest in move_tups:
        mv(sh.glob(glob), dest)

    mv(sh.glob(count_glob), count_dest)
    mv(sh.glob(isoform_glob), count_dest)
    mv(sh.glob(stat_glob), stat_dest)
    
    log.info("Deleting junk files")
    report_files = [files[0], files[1], log.handlers[0].baseFilename]
    subject = "RSEM/deseq2 pipeline"

    if options.exp_name != "rsem-deseq":
        subject += " - {}".format(options.exp_name)

    log.info("Sending report")
    send_report(list(options.to), subject, report_files)
    log.info("Run complete! Congradulations!")

def assert_git_notes(hgsha1s):
    gitrepo = os.getcwd()
    sh.cd(".git/refs/notes")
    notes_refs = sh.ls(sh.glob("hg-*")).stdout.splitlines()
    sh.cd(gitrepo)
    sh.git.notes("--ref=hg", "merge", *notes_refs)
    output = sh.git.log(pretty="format:%N", notes="hg").stdout
    notes = [line for line in output.splitlines() if line]
    assert notes == hgsha1s