def split(self):
		from Bio import SeqIO
		from utils import ensure_dir

		# Count the sequences

		print("Counting sequences...")
		n_seqs = 0
		handle = open(self.sauce_filepath, "r")
		for record in SeqIO.parse(handle, "fasta"):
			n_seqs += 1
		handle.close()
		chunk_size = int(n_seqs / self.n_chunks)
		print("...There are ["+str(n_seqs)+"] sequences")

		# Iterate through chunks of sequences.
		# For each sequence, write a single fasta file in the right location.
		n_seqs = 0
		handle = open(self.sauce_filepath, "r")
		for record in SeqIO.parse(handle, "fasta"):
			chunk_n = int(n_seqs / chunk_size)
			chunk_dir = self.target_dirpath+"/chunk_"+str(chunk_n)+"/"+record.id
			ensure_dir(chunk_dir+"/")

			f = open(chunk_dir+"/seq.fasta", "w")
			SeqIO.write(record, f, "fasta")
			f.close()

			n_seqs += 1

			if n_seqs % 100 == 0:
				print("["+str(n_seqs)+"] fasta files written")

		handle.close()		

def submit_emmy_experiment(nt, N, tgs, is_dp, procs, ts, kernel, outfile, target_dir):
    import os
    import subprocess
    from string import Template
    from utils import ensure_dir

    job_template = Template(
        """export OMP_NUM_THREADS=10; export I_MPI_PIN_DOMAIN=socket; export KMP_AFFINITY=granularity=fine,scatter; mpirun_rrze -np $procs -npernode 2 -- $exec_path --n-tests 2 --disable-source-point --npx 1 --npy $procs --npz 1 --nx $N --ny $N --nz $N  --verbose 1 --halo-concatenate 1 --target-ts $ts --wavefront 1 --nt $nt --target-kernel $kernel --thread-group-size $tgs | tee $outfile"""
    )

    target_dir = os.path.join(os.path.abspath("."), target_dir)
    ensure_dir(target_dir)
    outpath = os.path.join(target_dir, outfile)

    if is_dp == 1:
        exec_path = os.path.join(os.path.abspath("."), "build_dp/mwd_kernel")
    else:
        exec_path = os.path.join(os.path.abspath("."), "build/mwd_kernel")

    job_cmd = job_template.substitute(
        nt=nt,
        N=N,
        tgs=tgs,
        procs=procs,
        ts=ts,
        kernel=kernel,
        outfile=outpath,
        exec_path=exec_path,
        target_dir=target_dir,
    )

    print job_cmd
    sts = subprocess.call(job_cmd, shell=True)

    def run(self):
        self.info("Starting up... ")
        self.info("PID is %s" % os.getpid())
        self.info("TEMP DIR is '%s'" % self.temp_dir)
        if self.tags:
            self.info("Tags are: %s" % document_pretty_string(self.tags))
        else:
            self.info("No tags configured")

        ensure_dir(self._temp_dir)
        self._update_pid_file()
        # Start the command server
        self._start_command_server()

        # start the backup processor
        self._backup_processor.start()

        # start the restore processor
        self._restore_processor.start()

        # start the backup processor
        self._backup_processor.join()

        # start the restore processor
        self._restore_processor.join()

        self.info("Engine completed")
        self._pre_shutdown()

def main():
    app_settings = settings.get_app_settings()
    api = TwitterAPI.TwitterAPI(** app_settings)

    store_data = store.get_data()

    search_term = 'vnnlp'
    query = {'screen_name': search_term}

    filename = 'user_timeline/{}.yaml'.format(search_term)

    utils.ensure_dir(filename)

    if 'user_timeline' in store_data and 'max_id' in store_data['user_timeline']:
        query['max_id'] = store_data['user_timeline']['max_id'] - 1

    max_id = None
    try:
        with open(filename, 'a') as output_file:
            r = TwitterAPI.TwitterPager(api, 'statuses/user_timeline', query)

            for tweet in r.get_iterator():
                yaml.dump([tweet], output_file, default_flow_style=False)
                if 'id' in tweet:
                    max_id = tweet['id']

    except KeyboardInterrupt:
        pass

    if not 'user_timeline' in store_data:
        store_data['user_timeline'] = {}

    store_data['user_timeline']['max_id'] = max_id
    store.store_data(store_data)

def process_audio_poly(wavdir, outdir, tol, ssm_read_pk, read_pk, n_jobs=4,
                       tonnetz=False):
    utils.ensure_dir(outdir)
    files = glob.glob(os.path.join(wavdir, "*.wav"))
    Parallel(n_jobs=n_jobs)(delayed(process_piece)(
        wav, outdir, tol, ssm_read_pk, read_pk, tonnetz)
        for wav in files)

def simple_file_logger(name, log_file_name):
    lgr = logging.getLogger(name)

    if lgr.handlers:
        return lgr
    lgr.propagate = False

    log_dir = resolve_path(mbs_config.MBS_LOG_PATH)
    ensure_dir(log_dir)

    lgr.setLevel(logging.INFO)

    formatter = logging.Formatter("%(levelname)8s | %(asctime)s | %(message)s")

    logfile = os.path.join(log_dir, log_file_name)
    fh = TimedRotatingFileHandler(logfile, backupCount=10, when="midnight")

    fh.setFormatter(formatter)
    # add the handler to the root logger
    lgr.addHandler(fh)

    if LOG_TO_STDOUT:
        sh = logging.StreamHandler(sys.stdout)
        sh.setFormatter(formatter)
        lgr.addHandler(sh)

    return lgr

def copy_image_subset(source_dir, target_dir, subset_inds):
    frames_list = os.listdir(source_dir)
    frames_list.sort()
    utils.ensure_dir(target_dir, True)
    for i in subset_inds:
        if os.path.exists(os.path.join(source_dir, frames_list[i])):
            shutil.copy2(os.path.join(source_dir, frames_list[i]), os.path.join(target_dir, frames_list[i]))

 def __init__(self, cfg):
     self.cfg = cfg
     self.out_fn = self.cfg.get("machine", "ext_definitions")
     ensure_dir(os.path.dirname(self.out_fn))
     dep_map_fn = cfg.get("deps", "dep_map")
     self.read_dep_map(dep_map_fn)
     self.lemmatizer = Lemmatizer(cfg)

def submit_emmy_experiment(nwf, tgs, wf, is_dp, th, ts, tb, kernel, outfile, target_dir):
    import os
    import subprocess
    from string import Template
    from utils import ensure_dir    

    job_template=Template(
"""export OMP_NUM_THREADS=$thn; likwid-perfctr -m -s 0x03 -g MEM -C S0:0-$th $exec_path --n-tests 2 --disable-source-point --npx 1 --npy 1 --npz 1 --nx 960 --ny 960 --nz 960  --verbose 1 --target-ts $ts --wavefront $wf --nt 100 --target-kernel $kernel --t-dim $tb --thread-group-size $tgs --num-wavefronts $nwf --cache-size 0 | tee $outfile""")
#"""export OMP_NUM_THREADS=$thn; numactl -N 0 $exec_path --n-tests 2 --disable-source-point --npx 1 --npy 1 --npz 1 --nx 480 --ny 480 --nz 480  --verbose 1 --target-ts $ts --wavefront $wf --nt 100 --target-kernel $kernel --t-dim $tb --thread-group-size $tgs | tee $outfile""")


    target_dir = os.path.join(os.path.abspath("."),target_dir)
    ensure_dir(target_dir)
    outpath = os.path.join(target_dir,outfile)

    if(is_dp==1):
        exec_path = os.path.join(os.path.abspath("."),"build_dp/mwd_kernel")
    else:
        exec_path = os.path.join(os.path.abspath("."),"build/mwd_kernel")


    job_cmd = job_template.substitute(nwf=nwf, tgs=tgs, wf=wf, th=(th-1), thn=th, ts=ts, tb=tb, kernel=kernel, outfile=outpath, exec_path=exec_path, target_dir=target_dir)

    print job_cmd
    sts = subprocess.call(job_cmd, shell=True)

def recursive_execute_gmms_on_machine(gmm_stats_path, dest_path, machine):
    for f in os.listdir(gmm_stats_path):
        gmm_stats_file = os.path.join(gmm_stats_path, f)
        logger.debug('extract_i_vectors: ' + gmm_stats_file)

        if utils.is_hdf5_file(gmm_stats_file):

            stripped_filename = utils.strip_filename(f)
            dest_file = os.path.join(dest_path, stripped_filename)
            dest_file += '.hdf5'

            utils.ensure_dir(dest_path)

            # load the GMM stats file
            gmm_stats = bob.machine.GMMStats(bob.io.HDF5File(gmm_stats_file))

            # extract i-vector
            output = machine.forward(gmm_stats)

            # save them!
            bob.io.save(output, dest_file)

            print 'savin ivec ' + dest_file


        elif os.path.isdir(gmm_stats_file):
            new_path = os.path.join(gmm_stats_path, f)
            new_dest_dir = os.path.join(dest_path, f)
            extract_i_vectors(new_path, new_dest_dir, machine)

        else:
            logger.info('Unknown file type: ' + str(gmm_stats_file))
            continue

def recursively_extract_features(path, dest_dir, wl=20, ws=10, nf=24, nceps=19, fmin=0., fmax=4000., d_w=2, pre=0.97,
                                 mel=True):
    for f in os.listdir(path):
        f_location = os.path.join(path, f)
        if utils.is_wav_file(f_location):
            logger.debug('Extract features: ' + f_location)
            stripped_filename = utils.strip_filename(f)
            dest_file = os.path.join(dest_dir, stripped_filename)
            dest_file += '.hdf5'

            mfcc = feature_extraction(f_location, wl, ws, nf, nceps, fmin, fmax, d_w, pre, mel)
            utils.ensure_dir(dest_dir)

            bob.io.save(mfcc, dest_file)
            logger.debug('savin mfcc to ' + dest_file)


        elif os.path.isdir(f_location):
            logger.debug('Extract features: ' + f_location)
            new_path = os.path.join(path, f)
            new_dest_dir = os.path.join(dest_dir, f)
            recursively_extract_features(new_path, new_dest_dir)

        else:
            logger.info('Unknown file type: ' + str(f_location))
            continue

def compute_gmm_sufficient_statistics(ubm, src_dir, dest_dir):
    for f in os.listdir(src_dir):
        f_location = os.path.join(src_dir, f)
        logger.debug('compute_gmm_sufficient_statistics: ' + f_location)

        if utils.is_hdf5_file(f_location):

            stripped_filename = utils.strip_filename(f)
            dest_file = os.path.join(dest_dir, stripped_filename)
            dest_file += '.hdf5'

            feature = bob.io.load(f_location)
            gmm_stats = bob.machine.GMMStats(ubm.dim_c, ubm.dim_d)
            ubm.acc_statistics(feature, gmm_stats)

            utils.ensure_dir(dest_dir)

            gmm_stats.save(bob.io.HDF5File(dest_file, 'w'))
            logger.debug('savin gmm_stats to ' + dest_file)


        elif os.path.isdir(f_location):
            new_path = os.path.join(src_dir, f)
            new_dest_dir = os.path.join(dest_dir, f)
            compute_gmm_sufficient_statistics(ubm, new_path, new_dest_dir)

        else:
            logger.info('Unknown file type: ' + str(f_location))
            continue

def extract_i_vectors(gmm_stats_path, ivec_dir, ivec_machine):
    for f in os.listdir(gmm_stats_path):
        f_location = os.path.join(gmm_stats_path, f)
        logger.debug('extract_i_vectors: ' + f_location)

        if utils.is_hdf5_file(f_location):

            stripped_filename = utils.strip_filename(f)
            dest_file = os.path.join(ivec_dir, stripped_filename)
            dest_file += '.hdf5'

            utils.ensure_dir(ivec_dir)

            gmm_stats_to_ivec(f_location, dest_file, ivec_machine)

            logger.debug('saving ivec ' + dest_file)


        elif os.path.isdir(f_location):
            new_path = os.path.join(gmm_stats_path, f)
            new_dest_dir = os.path.join(ivec_dir, f)
            extract_i_vectors(new_path, new_dest_dir, ivec_machine)

        else:
            logger.info('Unknown file type: ' + str(f_location))
            continue

    def parse_sentences(self, entries):

        os.chdir('magyarlanc')

        self.tmp_dir = '../' + self.cfg.get('data', 'tmp_dir')
        ensure_dir(self.tmp_dir)

        with NamedTemporaryFile(dir=self.tmp_dir, delete=False) as in_file:
            json.dump(entries, in_file)
            in_file_name = in_file.name

        with NamedTemporaryFile(dir=self.tmp_dir, delete=False) as out_file:
            out_file_name = out_file.name
            success = self.run_magyarlanc(in_file_name, out_file_name)

        if success:
            print 'magyarlanc ok'
        else:
            print 'magyarlanc nem ok'

#        with open(out_file_name) as out_file:
#            new_entries = json.load(out_file)

        new_entries = self.parse_output(out_file_name)

        os.chdir('..')

        return new_entries

    def get_vienna_layout(self, data):

        temp_folder = "/tmp/"+str(uuid.uuid4())
        ensure_dir(temp_folder)
        dot_bracket_filepath = temp_folder+"/dotbracket.txt"

        f = open(dot_bracket_filepath, "w")
        f.write(data["sequence"]+"\n"+data["structure"]+"\n")
        f.close()

        # change to tmp folder
        os.chdir(temp_folder)

        # use RNAplot CLI to generate the xrna tab delimited file
        os.system("RNAplot -o xrna < "+dot_bracket_filepath)

        # get the coords out by parsing the file
        coords = []
        with open(temp_folder+"/rna.ss") as f:
            for line in f:
                line = line.strip()
                if line == "" or line[0] == "#":
                    continue

                bits = line.split()
                x = float(bits[2])
                y = float(bits[3])
                coords.append([x, y])

        os.system("rm -rf "+temp_folder)

        return coords

def get_logger():
    global logger, _logging_level

    if logger:
        return logger

    logger = logging.getLogger("MongoctlLogger")

    log_file_name="mongoctl.log"
    conf_dir = mongoctl_globals.DEFAULT_CONF_ROOT
    log_dir = utils.resolve_path(os.path.join(conf_dir, LOG_DIR))
    utils.ensure_dir(log_dir)


    logger.setLevel(logging.DEBUG)
    formatter = logging.Formatter("%(levelname)8s | %(asctime)s | %(message)s")
    logfile = os.path.join(log_dir, log_file_name)
    fh = TimedRotatingFileHandler(logfile, backupCount=50, when="midnight")

    fh.setFormatter(formatter)
    fh.setLevel(logging.DEBUG)
    # add the handler to the root logger
    logging.getLogger().addHandler(fh)

    global _log_to_stdout
    if _log_to_stdout:
        sh = logging.StreamHandler(sys.stdout)
        std_formatter = logging.Formatter("%(message)s")
        sh.setFormatter(std_formatter)
        sh.setLevel(_logging_level)
        logging.getLogger().addHandler(sh)

    return logger

def database_ish():
	ensure_dir(os.getcwd()  + "/build/")
	initialise()
	image_store = create_engine(direc)
	Session = sessionmaker(bind=image_store)
	session = Session()
	return session

 def print_edges(self, edge_fn):
     ensure_dir(os.path.dirname(edge_fn))
     logging.info("printing edges to {0}".format(edge_fn))
     with open(edge_fn, 'w') as f:
         for c, i in enumerate(self.coocc[0]):
             j = self.coocc[1][c]
             k = self.coocc[2][c]
             f.write("{0}\t{1}\t{2}\n".format(i, j, k))

 def __init__(self, cfg):
     self.cfg = cfg
     self.text_to_4lang = TextTo4lang(cfg)
     self.graph_dir = self.cfg.get("qa", "graph_dir")
     self.dep_dir = self.cfg.get("qa", "deps_dir")
     ensure_dir(self.graph_dir)
     ensure_dir(self.dep_dir)
     self.word_similarity = WordSimilarity(cfg)

 def get_file(self, filename):
     pkg = utils.Package.from_filename(filename)
     if pkg.filename in self.packages:
         return self.packages[pkg.filename]
     directory = os.path.join('packages', pkg.name.lower())
     utils.ensure_dir(directory)
     filepath = os.path.join(directory, pkg.filename)
     self.packages[pkg.filename] = utils.PackageReadWriter(filepath)
     return  self.packages[pkg.filename]