def main_exonerate(ref_fa,refseq_pr,exonerate_path,thread,exon2align_gff,index_s=0,index_e=0):
    '''
    * refseq_pr: all protein seqeunces of the organism
    * path: path to store splited protein sequences.
    '''
    if not os.path.exists(exonerate_path): os.mkdir(exonerate_path)
    # 1) split file
    os.chdir(exonerate_path)
    if os.listdir(path) != []:
        split_fa(refseq_pr,100,exonerate_path)
    # 2) run exonerate for each file
    faFiles = natsorted(glob.glob('file*.fa'))
    if index_e == 0:
        faFiles = faFiles[index_s:]
    else:
        faFiles = faFiles[index_s:index_e]
    pool = mp.Pool(processes=int(thread))
    for f in faFiles:
        out = f[:-2]+'gff'
        pool.apply_async(exonerate,args=(ref_fa,f,out))
    pool.close()
    pool.join()
    # 3) merge the gff files
    exonerate_gff = 'exonerate.gff'
    if not os.path.exists(exonerate_gff):
        gff_fns = natsorted(glob.glob('file*.gff'))
        exonerate2gff(gff_fns,exonerate_gff)

def find_idle_busy_slaves(parser, args, show_idle):
    parser.add_option(
        '-b', '--builder', dest='builders', action='append', default=[],
        help='Builders to filter on')
    parser.add_option(
        '-s', '--slave', dest='slaves', action='append', default=[],
        help='Slaves to filter on')
    options, args, buildbot = parser.parse_args(args)
    if args:
        parser.error('Unrecognized parameters: %s' % ' '.join(args))
    if not options.builders:
        options.builders = buildbot.builders.keys
    for builder in options.builders:
        builder = buildbot.builders[builder]
        if options.slaves:
            # Only the subset of slaves connected to the builder.
            slaves = list(set(options.slaves).intersection(set(builder.slaves.names)))
            if not slaves:
                continue
        else:
            slaves = builder.slaves.names
        busy_slaves = [build.slave.name for build in builder.current_builds]
        if show_idle:
            slaves = natsorted(set(slaves) - set(busy_slaves))
        else:
            slaves = natsorted(set(slaves) & set(busy_slaves))
        if options.quiet:
            for slave in slaves:
                print slave
        else:
            if slaves:
                print 'Builder %s: %s' % (builder.name, ', '.join(slaves))
    return 0

    def _get_list_of_files(self,path):
        """
        Go through each subdirectory of `path`, and choose one file from each to use in our hash.
        Continue to increase self.iter, so we use a different 'slot' of randomness each time.
        """
        chosen_files = []

        # Get a list of all subdirectories
        directories = []
        for root, dirs, files in natsort.natsorted(os.walk(path, topdown=False)):
            for name in dirs:
                if name[:1] is not '.':
                    directories.append(os.path.join(root, name))
                    directories = natsort.natsorted(directories)

        # Go through each directory in the list, and choose one file from each.
        # Add this file to our master list of robotparts.
        for directory in directories:
            files_in_dir = []
            for imagefile in natsort.natsorted(os.listdir(directory)):
                files_in_dir.append(os.path.join(directory,imagefile))
                files_in_dir = natsort.natsorted(files_in_dir)

            # Use some of our hash bits to choose which file
            element_in_list = self.hasharray[self.iter] % len(files_in_dir)
            chosen_files.append(files_in_dir[element_in_list])
            self.iter += 1

        return chosen_files

def scenarios_comms(paths):
    subdirs = natsorted(map_paths(paths))
    for i, subdir in enumerate(natsorted(subdirs)):
        title = os.path.basename(subdir)
        sources = npz_in_dir(subdir)
        log.info("{0:%}:{1}:{2}/{3}".format(float(i) / float(len(subdirs)), title, memory(), swapsize()))
        yield (subdir, generate_sources(sources, comms_only=True))

def write_excel_data(dev_data, norm_to_ctrl, norm_to_mean):
    """Write data into a file"""

    # Define excel directory
    xls_dir = "./excel"

    # Change directory to EXPROOTPATH
    os.chdir(EXPROOTPATH)

    # Check to see if excel directory exists and if it doesn't make it
    try:
        os.makedirs(xls_dir)
    except OSError:
        if not os.path.isdir(xls_dir):
            raise

    # Reorder
    dev_data = dev_data.reorder_levels(['device', 'interval', 'well'])
    norm_to_ctrl = norm_to_ctrl.stack().unstack(-4).reorder_levels(['device', 'interval', 2]) #.sort_index(0)
    norm_to_mean = norm_to_mean.stack().unstack(-4).reorder_levels(['device', 'interval', 2])

    # Sort
    dev_data = dev_data.reindex(index=natsorted(dev_data.index))
    norm_to_ctrl = norm_to_ctrl.reindex(index=natsorted(norm_to_ctrl.index))
    norm_to_mean = norm_to_mean.reindex(index=natsorted(norm_to_mean.index))

    # Create the Excel Workbook
    writer = pd.ExcelWriter(xls_dir+"/"+'data.xlsx', engine='xlsxwriter')

    # Write the data to the Excel Workbook
    dev_data.to_excel(writer, sheet_name='Raw_Device_Data')
    norm_to_ctrl.to_excel(writer, sheet_name='Ratio_to_Control')
    norm_to_mean.to_excel(writer, sheet_name='Ratio_to_Control_2')

def get_metadata(path, band_name, git_root):
    try:
        with open(os.path.join(path, 'description'), 'r') as desc:
            description = desc.read()
    except Exception:
        description = ''
    metadata = {'name': band_name,
                'description': description,
                'albums': [],
                'git_root': git_root}
    album_paths = natsort.natsorted(os.listdir(path))
    for album_name in filter(lambda a: filter_album_names(path, a), album_paths):
        album_path = os.path.join(path, album_name)
        try:
            with open(os.path.join(album_path, 'description'), 'r') as desc:
                album_description = desc.read()
        except Exception:
            album_description = 'Shorts are comfy and easy to wear!'
        tracks = []
        track_number = 1
        track_paths = natsort.natsorted(os.listdir(album_path))
        for track in filter(filter_tracks, track_paths):
            track_name = clean_track_name(track)
            tracks.append({'number': track_number,
                           'name': track_name,
                           'path': os.path.join(album_path, track)})
            track_number += 1
        metadata['albums'].append({'name': album_name,
                                   'path': album_path,
                                   'tracks': tracks,
                                   'description': album_description})
    return metadata

def listFiles():
    """ Lists all available Charmm topologies and parameter files

    Examples
    --------
    >>> from htmd.builder import charmm
    >>> charmm.listFiles()             # doctest: +ELLIPSIS
    ---- Topologies files list...

    """
    from natsort import natsorted
    charmmdir = path.join(home(), 'builder', 'charmmfiles', '')  # maybe just lookup current module?
    topos = natsorted(glob(path.join(charmmdir, 'top', '*.rtf')))
    params = natsorted(glob(path.join(charmmdir, 'par', '*.prm')))
    streams = natsorted(glob(path.join(charmmdir, 'str', '*', '*.str')))
    print('---- Topologies files list: ' + path.join(charmmdir, 'top', '') + ' ----')
    for t in topos:
        t = t.replace(charmmdir, '')
        print(t)
    print('---- Parameters files list: ' + path.join(charmmdir, 'par', '') + ' ----')
    for p in params:
        p = p.replace(charmmdir, '')
        print(p)
    print('---- Stream files list: ' + path.join(charmmdir, 'str', '') + ' ----')
    for s in streams:
        s = s.replace(charmmdir, '')
        print(s)

def test_natsorted_with_LOCALE_and_mixed_input_returns_sorted_results_without_error():
    load_locale('en_US')
    a = ['0', 'Á', '2', 'Z']
    assert natsorted(a) == ['0', '2', 'Z', 'Á']
    a = ['2', 'ä', 'b', 1.5, 3]
    assert natsorted(a, alg=ns.LOCALE) == [1.5, '2', 3, 'ä', 'b']
    locale.setlocale(locale.LC_ALL, str(''))

def test_natsorted_returns_sorted_list_with_mixed_type_input_and_does_not_raise_TypeError_on_Python3():
    # You can mix types with natsorted.  This can get around the new
    # 'unorderable types' issue with Python 3.
    a = [6, 4.5, '7', '2.5', 'a']
    assert natsorted(a) == ['2.5', 4.5, 6, '7', 'a']
    a = [46, '5a5b2', 'af5', '5a5-4']
    assert natsorted(a) == ['5a5-4', '5a5b2', 46, 'af5']

 def render_listing(in_name, out_name, folders=[], files=[]):
     if in_name:
         with open(in_name, 'r') as fd:
             try:
                 lexer = get_lexer_for_filename(in_name)
             except:
                 lexer = TextLexer()
             code = highlight(fd.read(), lexer,
                              HtmlFormatter(cssclass='code',
                                            linenos="table", nowrap=False,
                                            lineanchors=utils.slugify(in_name),
                                            anchorlinenos=True))
         title = os.path.basename(in_name)
     else:
         code = ''
         title = ''
     crumbs = utils.get_crumbs(os.path.relpath(out_name,
                                               kw['output_folder']),
                               is_file=True)
     context = {
         'code': code,
         'title': title,
         'crumbs': crumbs,
         'lang': kw['default_lang'],
         'folders': natsort.natsorted(folders),
         'files': natsort.natsorted(files),
         'description': title,
     }
     self.site.render_template('listing.tmpl', out_name,
                               context)

def metadata_stats_from_sample_and_prep_templates(st, pt):
    """Print out summary statistics for the sample and prep templates

    Parameters
    ----------
    st : SampleTemplate
        Initialized SampleTemplate to use for the metadat stats.
    pt : PrepTemplate
        Initialized PrepTemplate to use for the metadat stats.

    Returns
    -------
    dict
        Dictionary object where the keys are the names of the metadata
        categories and the keys are tuples where the first element is the name
        of a metadata value in category and the second element is the number of
        times that value was seen.
    """
    df = metadata_map_from_sample_and_prep_templates(st, pt)
    out = {}

    for column in natsorted(df.columns):
        counts = df[column].value_counts()

        # get a pandas series of the value-count pairs
        out[column] = [(key, counts[key]) for key in natsorted(counts.index)]

    return out

    def __GetOBSDatasetName(self, band):
        self.refBand = dict()
        self.emisBand = dict()
        self.refBandname = dict()
        self.emisBandname = dict()

        for band in self.OrbitInfo.BandsType:
            if self.OrbitInfo.BandsType[band] == 'REF':
                self.refBand[band] = self.OrbitInfo.BandsType[band]
            else:
                self.emisBand[band] = self.OrbitInfo.BandsType[band]

        self.refBand = natsorted(self.refBand, alg=ns.IGNORECASE)
        self.emisBand = natsorted(self.emisBand, alg=ns.IGNORECASE)

        refNum = 0
        for refband in self.refBand:
            self.refBandname[refband] = refNum
            refNum = refNum + 1

        emisNum = 0
        for emisband in self.emisBand:
            self.emisBandname[emisband] = emisNum
            emisNum = emisNum + 1

        return self.refBandname, self.emisBandname

def get_cover (filepath):
   path = root + '/' + filepath

   if os.path.isdir(path):
      files = quick(os.listdir(path))
      image = path + '/' + cover_cleaner(files)

      with open(image, 'rb') as file_:
         cover = file_.read()
         return cover

   elif os.path.isfile(path):
      filetype = path.split('.')[-1]

      if filetype == 'zip':
         with ZipFile(path) as archive:
            files = natsorted(archive.namelist())
            image = cover_cleaner(files)

            with archive.open(image) as file_:
               cover = file_.read()
               return cover

      elif filetype == 'rar':
         with rarfile.RarFile(path) as archive:
            files = natsorted(archive.namelist())
            image = cover_cleaner(files)

            with archive.open(image) as file_:
               cover = file_.read()
               return cover

def fpkm_from_htseq(bam_path,ruv_path,exn_file):
    """
    This function calculates fpkm from the htseq-count results.
    * bam_path: pathway that has bam files. Used to get total mapped reads.
    * ruv_path: pathway that has ruvseq corrected count data.
    * exn_file: 6 columns. including ['chr','start','end','geneid','traccess','strand'].
    output file that ends with .fpkm.
    """
    os.chdir(bam_path)
    bams = [f for f in os.listdir(bam_path) if f.endswith('.bam')]
    bams = natsorted(bams)
    # 1. get total count
    totalCount = []
    for b in bams:
        bamHandle = pysam.AlignmentFile(b,'rb')
        totalCount.append(bamHandle.mapped)
    # 2. get rna_obj
    rna_df = pd.read_csv(exn_file,sep='\t',header=0,low_memory=False)
    rna_obj = trpr(rna_df)
    # 3. get length for each gene
    os.chdir(ruv_path)
    norm_count_files = [f for f in os.listdir(ruv_path) if f.endswith('.txt')]
    norm_count_files = natsorted(norm_count_files)
    for fn,total in zip(norm_count_files,totalCount):
        df = pd.read_csv(fn,sep=' ',header=None,names=['geneid','count'],index_col=0,low_memory=False)
        df['len'] = df.index.map(lambda x: rna_obj.get_gene_trpr_len(x,multi_chrom='Y'))
        df['fpkm'] = df['count']/float(total)/df['len']*10**9
        df['fpkm'].ix[:-20].to_csv(fn[:-3]+'fpkm.txt',sep='\t')

def get_page (filepath, pagenum):
   path = root + '/' + filepath

   if os.path.isdir(path):
      files = natsorted(os.listdir(path))
      files = pages_cleaner(files)

      image = path + '/' + files[pagenum - 1]
      with open(image, 'rb') as file_:
         page = file_.read()
         return page

   elif os.path.isfile(path):
      filetype = path.split('.')[-1]

      if filetype == 'zip':
         with ZipFile(path) as archive:
            files = natsorted(archive.namelist())
            files = pages_cleaner(files)

            image = files[pagenum - 1]
            with archive.open(image) as file_:
               page = file_.read()
               return page

      elif filetype == 'rar':
         with rarfile.RarFile(path) as archive:
            files = natsorted(archive.namelist())
            files = pages_cleaner(files)

            image = files[pagenum - 1]
            with archive.open(image) as file_:
               page = file_.read()
               return page

    def get_sheet_values(self, sheetname, includeEmptyCells=True):
        """
        Returns the values from the sheet name specified.

        Arguments:
                |  Sheet Name (string)                 | The selected sheet that the cell values will be returned from.                                                              |
                |  Include Empty Cells (default=True)  | The empty cells will be included by default. To deactivate and only return cells with values, pass 'False' in the variable. |
        Example:

        | *Keywords*           |  *Parameters*                                      |
        | Open Excel           |  C:\\Python27\\ExcelRobotTest\\ExcelRobotTest.xls  |
        | Get Sheet Values     |  TestSheet1                                        |

        """
        my_sheet_index = self.sheetNames.index(sheetname)
        sheet = self.wb.sheet_by_index(my_sheet_index)
        data = {}
        for row_index in range(sheet.nrows):
            for col_index in range(sheet.ncols):
                cell = cellname(row_index, col_index)
                value = sheet.cell(row_index, col_index).value
                data[cell] = value
        if includeEmptyCells is True:
            sortedData = natsort.natsorted(data.items(), key=itemgetter(0))
            return sortedData
        else:
            data = dict([(k, v) for (k, v) in data.items() if v])
            OrderedData = natsort.natsorted(data.items(), key=itemgetter(0))
            return OrderedData

def stats_from_df(df):
    """Create a dictionary of summary statistics for a sample or prep template

    Parameters
    ----------
    t : SampleTemplate or PrepTemplate
        Sample or prep template object to summarize

    Returns
    -------
    dict
        Dictionary object where the keys are the names of the metadata
        categories and the keys are tuples where the first element is the name
        of a metadata value in category and the second element is the number of
        times that value was seen.
    """
    out = {}

    for column in natsorted(df.columns):
        counts = df[column].value_counts()

        # get a pandas series of the value-count pairs
        out[column] = [(key, counts[key]) for key in natsorted(counts.index)]

    return out

    def get_all_tags(self):
        """ 
        Return a tuple of lists ([common_tags], [anti_tags], [organisational_tags]) all tags of all tasks of this course 
        Since this is an heavy procedure, we use a cache to cache results. Cache should be updated when a task is modified.
        """
        
        if self._all_tags_cache != None:
            return self._all_tags_cache
    
        tag_list_common = set()
        tag_list_misconception = set()
        tag_list_org = set()

        tasks = self.get_tasks()
        for id, task in tasks.items():
            for tag in task.get_tags()[0]:
                tag_list_common.add(tag)
            for tag in task.get_tags()[1]:
                tag_list_misconception.add(tag)
            for tag in task.get_tags()[2]:
                tag_list_org.add(tag)
        
        tag_list_common = natsorted(tag_list_common, key=lambda y: y.get_name().lower())
        tag_list_misconception = natsorted(tag_list_misconception, key=lambda y: y.get_name().lower())
        tag_list_org = natsorted(tag_list_org, key=lambda y: y.get_name().lower())
             
        self._all_tags_cache = (list(tag_list_common), list(tag_list_misconception), list(tag_list_org))
        return self._all_tags_cache

def find_idle_busy_slaves(parser, args, show_idle):
    parser.add_option("-b", "--builder", dest="builders", action="append", default=[], help="Builders to filter on")
    parser.add_option("-s", "--slave", dest="slaves", action="append", default=[], help="Slaves to filter on")
    options, args, buildbot = parser.parse_args(args)
    if args:
        parser.error("Unrecognized parameters: %s" % " ".join(args))
    if not options.builders:
        options.builders = buildbot.builders.keys
    for builder in options.builders:
        builder = buildbot.builders[builder]
        if options.slaves:
            # Only the subset of slaves connected to the builder.
            slaves = list(set(options.slaves).intersection(set(builder.slaves.names)))
            if not slaves:
                continue
        else:
            slaves = builder.slaves.names
        busy_slaves = [build.slave.name for build in builder.current_builds]
        if show_idle:
            slaves = natsorted(set(slaves) - set(busy_slaves))
        else:
            slaves = natsorted(set(slaves) & set(busy_slaves))
        if options.quiet:
            for slave in slaves:
                print slave
        else:
            if slaves:
                print "Builder %s: %s" % (builder.name, ", ".join(slaves))
    return 0

    def get_all_tags_names_as_list(self, admin=False, language="en"):
        """ Computes and cache two list containing all tags name sorted by natural order on name """

        if admin:
            if self._all_tags_cache_list_admin != {} and language in self._all_tags_cache_list_admin:
                return self._all_tags_cache_list_admin[language] #Cache hit
        else:
            if self._all_tags_cache_list != {} and language in self._all_tags_cache_list:
                return self._all_tags_cache_list[language] #Cache hit
                        
        #Cache miss, computes everything
        s_stud = set()
        s_admin = set()
        (common, _, org) = self.get_all_tags()
        for tag in common + org:
            # Is tag_name_with_translation correct by doing that like that ?
            tag_name_with_translation = self.gettext(language, tag.get_name()) if tag.get_name() else ""
            s_admin.add(tag_name_with_translation) 
            if tag.is_visible_for_student():
                s_stud.add(tag_name_with_translation) 
        self._all_tags_cache_list_admin[language] = natsorted(s_admin, key=lambda y: y.lower())
        self._all_tags_cache_list[language] = natsorted(s_stud, key=lambda y: y.lower())
        
        if admin:
            return self._all_tags_cache_list_admin[language]
        return self._all_tags_cache_list[language]