def action(file_path):
     if re.match('|'.join(INCLUDE_PATTERNS), file_path, re.M | re.I):
         if not re.match('|'.join(EXCLUDE_PATTERNS),
                         file_path,
                         re.M | re.I):
             print 'Formatting file {}'.format(file_path)
             run_cmd(FORMAT_CMD.format(file_path))

def get_adenylation_domains(fasta, known=None, lagging_strand=False):
    adenylation_domains = []

    fasta_seqs = []
    for fs in SeqIO.parse(fasta, 'fasta'):
        revcom=False
        seq = str(fs.seq)
        pepseq, rf = get_pepseq(seq)
        if rf < 0 == lagging_strand:
            revcom=True
            seq = utils.reverse_complement(seq)
        fasta_seqs.append({'id': fs.id, 'seq': seq, 'pepseq': pepseq, 'rf': rf})
    for fs in fasta_seqs:
        utils.run_cmd([hmmsearch, '--domtblout', 'dump', os.path.abspath('lib/AMP-binding.hmm'), '-'],
                  '>header\n' + pepseq)
        with open('dump') as f:
            out = f.read()
        res_stream = StringIO(out)
        os.remove('dump')
        results = list(SearchIO.parse(res_stream, 'hmmsearch3-domtab'))

        for result in results:
            for i, hsp in enumerate(result.hsps, 1):
                s = hsp.hit_start
                e = hsp.hit_end

                adenylation_domains.append((AdenylationDomain(fs['seq'][s*3:e*3], known, '{}_{}'.format(fs['id'], i), revcom), s, e))

    return adenylation_domains

def clone_vm(output_path, game_hash, base_vm, name, remote):
    assert re.match(r'[a-zA-Z0-9_-]+\Z', name)
    status(game_hash, "Creating VM: {}".format(name), remote)
    basepath = gamepath(output_path, game_hash)
    run_cmd(['VBoxManage', 'clonevm', base_vm, '--name', name, '--basefolder',
             basepath], "clonevm{}".format(name))
    return os.path.join(basepath, name)

def guestmount(what, where):
    pidfile = os.tempnam(None, 'guestmountpid')
    run_cmd(['sudo', 'guestmount', '--pid-file', pidfile, '-a', what, '-i',
             where], "guestmount")
    assert os.path.isfile(pidfile)
    logging.info('guestmount pid file %s', pidfile)
    return pidfile

def deploy_app(request):
    app_id = request.GET.get('app_id')
    app = App.objects.get(id=app_id)
    site_name = app.site.name
    wd = app.wd
    
    config = _get_config(wd)
    print config

    out = ''

    if (config['type'] == 'static'):
        # try:
        out += _update('starting in', run_cmd('pwd', wd=wd, echo=True))
        out += _update('update from repo', run_cmd('git pull origin master', wd=wd, echo=True))
        out += _update("post_update hooks", run_cmd('sh post_update.sh', wd=wd, echo=True))
        out += _update('deploy built site', run_cmd('cp -R %s/* %s' % (
            config['build_dir'], config['dest_dir']
        ), wd=wd, echo=True))
        # except Exception, e:
        #             exc_type, exc_value, exc_traceback = sys.exc_info()
        #             formatted_lines = traceback.format_exc().splitlines()
        #                 
        #             return HttpResponseServerError("<pre>%s\nERROR: %s\n\n%s</pre>" % (out, exc_value, formatted_lines))
    
    do = Deploy.objects.create(app=app, deploy_id='placeholder', output=out, complete=True)
    do.save()
    
    return HttpResponse("<pre>%s\nDEPLOYED: <a href='%s'>%s</a></pre>" % (out, config['url'], config['url']))

def gen_flash_algo():
    run_cmd([FROMELF, '--bin', ALGO_ELF_PATH, '-o', TMP_DIR_W_TERM])
    try:
        flash_info = FlashInfo(DEV_INFO_PATH)
        ALGO_START = flash_info.get_algo_start()
    except IOError, e:
        print repr(e), e
        ALGO_START = 0x20000000

def convert_file(page_image, page_djvu, bitonal=False, quality=48, force_convert=False):
    """
    Convert an image to DJVU with the given parameters:
    
    page_image      : image filename to convert
    page_djvu       : djvu file of the converted image (target)
    bitonal         : bitonal djvu output?
    quality         : decibel quality 16-50
    force_convert   : always run through imagemagick
    """

    directory = os.path.dirname(page_image)

    #create temporary DJVU file
    if bitonal:
        tempPpm  = os.path.join(directory, 'IMG-DJVU-CONVERTER-TEMP-FILE.pbm')
    else:
        tempPpm  = os.path.join(directory, 'IMG-DJVU-CONVERTER-TEMP-FILE.ppm')

    root, ext = os.path.splitext( page_image )

    if not bitonal and ext.lower() in ['.jpg', '.jpeg']:

        if force_convert:
            cmd = ['convert', page_image, tempPpm]
            utils.run_cmd(cmd)
            file = tempPpm
        else:
            file = page_image

        #convert jpg to a temp djvu file
        cmd = ['c44', '-decibel', str(quality), file, page_djvu]
        
        utils.run_cmd(cmd)

    elif bitonal and ext.lower() in ['.tiff','.tif']:
        #convert jpg to a temp djvu file
        cmd = ['cjb2', page_image, page_djvu]
        
        utils.run_cmd(cmd)

    else: #image needs converting

        cmd = ['convert', page_image, tempPpm]
        #print cmd
        utils.run_cmd(cmd)

        if bitonal:
            cmd = ['cjb2', tempPpm, page_djvu]
        else:
            cmd = ['c44', '-decibel', str(quality), tempPpm, page_djvu]
            
        utils.run_cmd(cmd)


    #Remove any leftover temporary files
    if os.path.exists(tempPpm):
        os.remove(tempPpm)

def run_job(hadoop_bin, job_conf, date_str):
    """ 根据任务配置job，运行日期date_str执行mapreduce任务 """
    output_path = _parse_template(job_conf['mapreduce']['-output'][0], DATE=date_str)
    try:
        logging.info('Delete directory %s' %output_path)
        cmd = '%s fs -rmr %s' %(hadoop_bin, output_path)
        logging.info('Delete dir:%s' %cmd)
        utils.run_cmd(cmd)
    except Exception:
        logging.warning('Delete directory %s failed' %output_path)
    cmd_str =  make_mapreduce_cmd(hadoop_bin, job_conf, DATE=date_str)
    utils.run_cmd(cmd_str)

    def create(self, x, y, z, radius):
        log.info('Creating spherical seed NAME={} MNI=({},{},{}) RADIUS={}mm'.format(self.name,x,y,z,radius))

        self.file = os.path.join(self.dir, '%s_%dmm.nii.gz' % (self.name, radius,))
        cmd = '3dUndump -prefix {ofile} -xyz -orient LPI -master {std} -srad {rad} <(echo \'{x} {y} {z}\')' \
                .format(ofile = self.file,
                        std   = mri_standard,
                        rad   = radius,
                        x     = x,
                        y     = y,
                        z     = z)

        run_cmd(cmd) # blocking

def gen_flash_algo(filename_path, dev_info_path, algo_bin_path):

    output_file = os.path.splitext(filename_path)[0]
    output_file += ".txt"

    run_cmd([FROMELF, '--bin', filename_path, '-o', TMP_DIR_W_TERM])
    try:
        flash_info = FlashInfo(DEV_INFO_PATH)
        ALGO_START = flash_info.get_algo_start()
        flash_info.printInfo()
    except IOError, e:
        print "No flash info file found: %s, using default start address." % DEV_INFO_PATH
        ALGO_START = 0x20000000

 def test_SUID_sandbox(self):
     log_file = self.new_temp_file('/tmp/chrome.log')
     url = cm.BASE_TEST_URL + "font-face/font-face-names.html"
     unexpected_strs = ('FATAL:browser_main_loop.cc', 'Running without the SUID sandbox')
     expected_strs = ('FPLOG',)
     
     cmd='timeout 10 xvfb-run --auto-servernum %s --disable-setuid-sandbox --enable-logging=stderr --v=1 --vmodule=frame=1 \
         --user-data-dir=/tmp/temp_profile%s --disk-cache-dir=/tmp/tmp_cache%s %s 2>&1 | tee %s' %\
         (cm.CHROME_MOD_BINARY, ut.rand_str(), ut.rand_str(), url, log_file)
     
     ut.run_cmd(cmd)
     
     self.assert_all_patterns_not_in_file(log_file, unexpected_strs)
     self.assert_all_patterns_in_file(log_file, expected_strs)

    def tar_scm(self, args, should_succeed=True):
        # simulate new temporary outdir for each tar_scm invocation
        mkfreshdir(self.outdir)

        # osc launches source services with cwd as pkg dir
        # (see run_source_services() in osc/core.py)
        print("chdir to pkgdir: %s" % self.pkgdir)
        os.chdir(self.pkgdir)

        cmdargs = args + ['--outdir', self.outdir]
        quotedargs = ["'%s'" % arg for arg in cmdargs]
        cmdstr = 'python2 %s %s 2>&1' % \
                 (self.tar_scm_bin(), " ".join(quotedargs))
        print
        print ">>>>>>>>>>>"
        print "Running", cmdstr
        print
        (stdout, stderr, ret) = run_cmd(cmdstr)
        if stdout:
            print "--v-v-- begin STDOUT from tar_scm --v-v--"
            print stdout,
            print "--^-^-- end   STDOUT from tar_scm --^-^--"
        if stderr:
            print "\n"
            print "--v-v-- begin STDERR from tar_scm --v-v--"
            print stderr,
            print "--^-^-- end   STDERR from tar_scm --^-^--"
        succeeded = ret == 0
        self.assertEqual(succeeded, should_succeed,
                         "expected tar_scm to " +
                         ("succeed" if should_succeed else "fail"))
        return (stdout, stderr, ret)

def append_page(page, main_file):
    """
    Appends a DjVu page to a main file
    
    page        : single page djvu file
    main_file   : file to append to
    """

    if os.path.exists(main_file):
        #Add the djvu file to the collated file
        cmd = ['djvm','-i', main_file, page]
    else:
        # Create the collated file
        cmd = ['djvm', '-c', main_file, page]

    utils.run_cmd(cmd)

def crawl_worker(agent_cfg, url_tuple):
    """Crawl given url. Will work in parallel. Cannot be class method."""
    MAX_SLEEP_BEFORE_JOB = 10 # prevent starting all parallel processes at the same instance
    sleep(random() * MAX_SLEEP_BEFORE_JOB) # sleep for a while
    
    try:
        idx, url = url_tuple
        idx = str(idx)
        
        stdout_log =  os.path.join(agent_cfg['job_dir'], fu.get_out_filename_from_url(url, str(idx), '.txt'))
       
        if not url[:5] in ('data:', 'http:', 'https', 'file:'):
            url = 'http://' + url
        
        proxy_opt = mitm.init_mitmproxy(stdout_log[:-4], agent_cfg['timeout'], agent_cfg['mitm_proxy_logs']) if agent_cfg['use_mitm_proxy'] else ""
        
        if not 'chrome_clicker' in agent_cfg['type']:
            cmd = get_visit_cmd(agent_cfg, proxy_opt, stdout_log, url)
            wl_log.info('>> %s (%s) %s' % (url, idx, cmd))
            status, output = ut.run_cmd(cmd) # Run the command
            if status and status != ERR_CMD_TIMEDOUT:
                wl_log.critical('Error while visiting %s(%s) w/ command: %s: (%s) %s' % (url, idx, cmd, status, output))
            else:
                wl_log.info(' >> ok %s (%s)' % (url, idx))
            
        else:
            cr.crawl_url(agent_cfg['type'], url, proxy_opt)
            
        sleep(2) # this will make sure mitmdump is timed out before we start to process the network dump
        if agent_cfg['post_visit_func']: # this pluggable function will parse the logs and do whatever we want
            agent_cfg['post_visit_func'](stdout_log, crawl_id=agent_cfg['crawl_id'])
            
    except Exception as exc:
        wl_log.critical('Exception in worker function %s %s' % (url_tuple, exc))

    def execute_commands(self, base_infname, base_outfname, n_procs):
        # ----------------------------------------------------------------------------------------
        def get_outfname(iproc):
            return self.subworkdir(iproc, n_procs) + '/' + base_outfname
        # ----------------------------------------------------------------------------------------
        def get_cmd_str(iproc):
            return self.get_vdjalign_cmd_str(self.subworkdir(iproc, n_procs), base_infname, base_outfname, n_procs)

        # start all procs for the first time
        procs, n_tries = [], []
        for iproc in range(n_procs):
            procs.append(utils.run_cmd(get_cmd_str(iproc), self.subworkdir(iproc, n_procs)))
            n_tries.append(1)
            time.sleep(0.1)

        # keep looping over the procs until they're all done
        while procs.count(None) != len(procs):  # we set each proc to None when it finishes
            for iproc in range(n_procs):
                if procs[iproc] is None:  # already finished
                    continue
                if procs[iproc].poll() is not None:  # it's finished
                    utils.finish_process(iproc, procs, n_tries, self.subworkdir(iproc, n_procs), get_outfname(iproc), get_cmd_str(iproc))
            sys.stdout.flush()
            time.sleep(1)

        for iproc in range(n_procs):
            os.remove(self.subworkdir(iproc, n_procs) + '/' + base_infname)

        sys.stdout.flush()

 def _run_cmd(self, *cmd):
     cmd = map(str, cmd)
     if self.compose_files is not None:
         for filename in reversed(self.compose_files):
             cmd[1:1] = ["-f", filename]
     if self.project_name is not None:
         cmd[1:1] = ["-p", self.project_name]
     return utils.run_cmd(cmd, workdir=self.dir)

 def radio_current_track(self):
     title = ''
     mpc_state = run_cmd('mpc')
     if '[playing]' in mpc_state:
         title = mpc_state.split('\n')[0]
         if '[SomaFM]: ' in title:
             title = title.split('[SomaFM]: ')[1]
     return title

def guestunmount(mntdir, guestmount_pidfile):
    run_cmd(['sudo', 'guestunmount', mntdir], "guestunmount")
    logging.info('Waiting for guestmount (pidfile %s) to exit...',
                 guestmount_pidfile)
    sleepcount = 0
    while sleepcount < 100:
        try:
            subprocess.check_output(['pgrep', '-F', guestmount_pidfile],
                                    stderr=subprocess.STDOUT)
        except Exception, e:
            # If the return code was non-zero it raises a CalledProcessError.
            # The CalledProcessError object will have the return code in the
            # returncode attribute and any output in the output attribute.
            break
        sleepcount += 1
        if sleepcount % 10 == 0:
            logging.info('    still sleeping (count=%d)...', sleepcount)
        time.sleep(1)

    def __init__(self, *args, **kwargs):
        self.volume = self.volume_from_mixer()
        self.state_read()

        mpc_state = run_cmd('mpc')
        if '[playing]' in mpc_state:
            self.state['radio_on'] = True
        else:
            self.state['radio_on'] = False

def snapshot_overlay(underlay, overlay, out_file, vmin=.2, vmax=.7, auto_coords=False):
    # first, generate a combined volume (overlay and underlay)
    tmp_vol = tempfile.mktemp(suffix='.nii.gz')
    cmd = 'overlay 0 0 {underlay} -a {overlay} {vmin} {vmax} {output}' \
            .format( underlay = underlay,
                     overlay  = overlay,
                     vmin      = vmin,
                     vmax      = vmax,
                     output   = tmp_vol)
    run_cmd(cmd)

    # second, produce snapshot from combined volume
    if auto_coords:
        sx,sy,sz = image_center_of_gravity(overlay)

        # temp image for each direction
        tmp_x,tmp_y,tmp_z = [tempfile.mktemp(suffix='.png') for x in range(3)]

        # produce images
        cmd='slicer {input} -x -{x} {tx} -y -{y} {ty} -z -{z} {tz} -t' \
                .format(input = tmp_vol,
                        x     = sx,
                        tx    = tmp_x,
                        y     = sy,
                        ty    = tmp_y,
                        z     = sz,
                        tz    = tmp_z)
        run_cmd(cmd)

        # combine views into single image
        tmp_img = tempfile.mktemp(suffix='.png')
        cmd='montage -tile 3x1 -geometry +0+0 {} {} {} {}'.format(tmp_x,tmp_y,tmp_z, tmp_img)
        run_cmd(cmd)

    else:
        # take axial, sagittal, coronal views at center slice
        tmp_img = tempfile.mktemp(suffix='.png')
        cmd = 'slicer {input} -a {output} -t'.format(input=tmp_vol, output=tmp_img)
        run_cmd(cmd)

    # scale the image (default is too small)
    cmd='convert {input} -scale 300% -trim {output}'.format(input=tmp_img, output=out_file)
    run_cmd(cmd)