def get_kmer_densities(path, kmer):
    mC_trans = string.maketrans("C", "E")
    hmC_trans = string.maketrans("C", "O")
    c_density = KmerHdpDistribution(path, kmer)
    mc_density = KmerHdpDistribution(path, kmer.translate(mC_trans))
    hmc_density = KmerHdpDistribution(path, kmer.translate(hmC_trans))
    return c_density, mc_density, hmc_density

def build_sample_ids_transtable():
    """Build translation table for sample ids being MIENS compliant"""
    all_chars = "".join([chr(i) for i in range(128)])
    valid_sample_id_chars = letters + digits + "."
    non_valid_sample_id_chars = all_chars.translate(maketrans("", ""), valid_sample_id_chars)
    trans_table = maketrans(non_valid_sample_id_chars, "." * len(non_valid_sample_id_chars))
    return trans_table

def solve_partial(w, h, pattern, answer, fix_num, solve_num, reset_loop_count=RESET_LOOP_COUNT):
    trans_str_wall = answer[:fix_num]
    trans_table_wall = string.maketrans(trans_str_wall, 
                                        '=' * len(trans_str_wall))
    trans_str_asta = answer[fix_num + solve_num:-1].replace('=', '')
    trans_table_asta = string.maketrans(trans_str_asta,
                                        '*' * len(trans_str_asta))
    pattern_rep = pattern.translate(trans_table_wall)
    pattern_rep = pattern_rep.translate(trans_table_asta)
    answer_rep = answer.translate(trans_table_wall)
    answer_rep = answer_rep.translate(trans_table_asta)

    ####### debug #######
    print '--------- pattern_rep'
    print_pattern(w, h, pattern_rep)
    print '--------- answer_rep'
    print_pattern(w, h, answer_rep)
    ####### debug #######

    move = solve_all(w, h, pattern_rep, answer_rep, reset_loop_count)

    ####### debug #######
    if move:
        pattern_work = create_pattern(w, h, pattern, move)
        print '--------- succeeded'
        print_pattern(w, h, pattern_work)
    else:
        print '--------- not succeeded'
    ####### debug #######
    return move

def build_anagrams():
    '''
    build a python dict of sorted-word and the set of words
    Using brown corpus and a file containing words from multiple corpora
    strip all punctuations, retain hyphens, replace underscores with space
    '''
    punctuations_replace = '#"(){}[]<>.+,/:;[email protected]_|~-'
    punctuations_remove = '!$\'%&\\*^`'
    # 1. Brown Corpus
    for word in nltk.corpus.brown.words():
        w = str(word)
        # polish just the way tokens were
        w_list = w.translate(string.maketrans(punctuations_replace,' '*len(punctuations_replace)), punctuations_remove).strip().lower().split()
        for each_w in w_list:
            # add the word to redis with key as a sorted word
            wam[''.join(sorted(each_w))].add(each_w)
    # 2. Wordnet
    for word in nltk.wordnet.wordnet.words():
        w = str(word)
        # polish just the way tokens were
        w_list = w.translate(string.maketrans(punctuations_replace,' '*len(punctuations_replace)), punctuations_remove).strip().lower().split()
        for each_w in w_list:
            # add the word to redis with key as a sorted word
            wam[''.join(sorted(each_w))].add(each_w)
    # 3. Other corpora
    with open(BIG_WL, 'r') as f:
        for line in f:
            w = str(line).strip()
            # polish just the way tokens were
            w_list = w.translate(string.maketrans(punctuations_replace,' '*len(punctuations_replace)), punctuations_remove).strip().lower().split()
            for each_w in w_list:
                # add the word to redis with key as a sorted word
                wam[''.join(sorted(each_w))].add(each_w)

 def test_the_reverse_complement(self):
     """Check obj.reverse_complement() method."""
     mapping = ""
     for example1 in self._examples:
         if isinstance(example1, MutableSeq):
             continue
         try:
             comp = example1.reverse_complement()
         except ValueError as e:
             self.assertEqual(str(e), "Proteins do not have complements!")
             continue
         str1 = str(example1)
         # This only does the unambiguous cases
         if any(("U" in str1, "u" in str1, example1.alphabet == generic_rna)):
             mapping = maketrans("ACGUacgu", "UGCAugca")
         elif any(("T" in str1, "t" in str1, example1.alphabet == generic_dna,
                  example1.alphabet == generic_nucleotide)):
             mapping = maketrans("ACGTacgt", "TGCAtgca")
         elif "A" not in str1 and "a" not in str1:
             mapping = maketrans("CGcg", "GCgc")
         else:
             # TODO - look at alphabet?
             continue
         self.assertEqual(str1.translate(mapping)[::-1], str(comp))
         self.assertEqual(comp.alphabet, example1.alphabet)

def reverseComplement(seq, rna=False):
    if rna:
        complements = string.maketrans("AUCGN", "UAGCN")
        return convertToRNA(seq).translate(complements)[::-1]
    else:
        complements = string.maketrans("ATCGN", "TAGCN")
        return convertToDNA(seq).translate(complements)[::-1]

def html_for_url_node(node):
    if not re.match("javascript:", node["url"]):
        linkURL = sanitize(node["url"])
        keysURL = linkURL
        ktrspaces = "                            "
        ktrtable = string.maketrans("[email protected]#$%^&*()_+-=`~;:'\",<.>/?\\|", ktrspaces)
        keysURL = str(keysURL).translate(ktrtable, "").lower()
        keysURL = str(keysURL).translate(None, "[email protected]#$%^&*()_+-=`~;:'\",<.>/?\\|").lower()
        #
        tags = sanitize(node["name"])
        # tags= node['name'] Check for UTF-8 etc...
        # print "TAGS: ",tags
        # tags = tags.translate(None,'[email protected]#$%^&*()_+-=`~;:\'",<.>/?\\|')
        # tags.translate(None,'[email protected]#$%^&*()_+-=`~;:\'",<.>/?\\|')
        # trtable =                          '                              '
        trspaces = "                            "
        trtable = string.maketrans("[email protected]#$%^&*()_+-=`~;:'\",<.>/?\\|", trspaces)
        # tags = str(tags).translate(trtable,'[email protected]#$%^&*()_+-=`~;:\'",<.>/?\\|')
        tags = str(tags).translate(trtable, "").lower()
        tags = str(tags).translate(None, "[email protected]#$%^&*()_+-=`~;:'\",<.>/?\\|").lower()
        #
        allTags = tags + " " + keysURL
        print "# '", sanitize(node["url"]), "'", allTags
        # print '# \'',sanitize(node['url']),'\'', tags
        return '<dt><a href="%s">%s</a>\n' % (sanitize(node["url"]), sanitize(node["name"]))
    else:
        return ""

def scrape_links_and_wordlistify(links, lower=False, verbose=1):
    import nltk
    import requests
    import string
    raw = ''
    wordlist = {}
    for site in links:
        try:
            if verbose == 1:
                print '[+] fetching data from: ', site
            if site.find('http://pastebin.com/') == 0:
                raw = requests.get(site.replace('http://pastebin.com/', 'http://pastebin.com/raw.php?i=')).content
            else:
                raw = requests.get(site).content
            if lower == False:
                l = string.translate(nltk.clean_html(raw), string.maketrans(string.punctuation, ' ' * 32)).split()
                freq_an(l, wordlist)
            else:
                l = string.lower(nltk.clean_html(raw))
                l = string.translate(l, string.maketrans(string.punctuation, ' ' * 32)).split()
                freq_an(l, wordlist)
        except:
            if verbose == 1:
                print '[-] Skipping url: ', site
    return wordlist

	def generateCounts(self):
		wordCounts = {}
		hashtagCounts = {}

		for tweet in self.trainSet:
			hashtags = []
			for word in tweet.split():
				if word.startswith('#') and len(word) > 2:
					word = word.lower().translate(string.maketrans("",""), string.punctuation) # remove punctuation
					hashtags.append(word)
					if word not in wordCounts:
						wordCounts[word] = 1
					else:
						wordCounts[word] += 1
				else:
					if '@' in word:
						continue
					if word in self.stopWords:
						continue
					word = word.lower().translate(string.maketrans("",""), string.punctuation) # remove punctuation
					if word not in wordCounts:
						wordCounts[word] = 1
					else:
						wordCounts[word] += 1

			for hashtag in hashtags:
				if hashtag not in hashtagCounts:
					hashtagCounts[hashtag] = 1.0
				else:
					hashtagCounts[hashtag] += 1.0

		return wordCounts, hashtagCounts

def ROT13(test_string):
    """Return a encrypted string offset by 13


    Encrytion does not impact spaces or special characters"""


    shifted_cypherlower = string.ascii_lowercase[13:] + string.ascii_lowercase[:13]
    shifted_cyperupper = string.ascii_uppercase[13:] + string.ascii_uppercase[:13]
    transtable_lower = string.maketrans(string.ascii_lowercase, shifted_cypherlower)
    #print transtable_lower 
    transtable_upper = string.maketrans(string.ascii_uppercase, shifted_cyperupper)
    #print transtable_upper

    
    encrypted_text = []

    for letter in test_string:
        if letter.islower():
            encrypted_text.append(letter.translate(transtable_lower))
        else:
            encrypted_text.append(letter.translate(transtable_upper))
        final_encryption =  "".join(encrypted_text)

    return final_encryption

 def __init__(self): 
     self.currentTroll = random.randint(0, 10)
     self.trollCounter = 0
     self.trolls = [
         #aradia
         { 'prefix': '', 'replace': lambda x: x.translate(string.maketrans('oo', '00')) },
         #terezi
         { 'prefix': '', 'replace': lambda x: x.upper().translate(string.maketrans('AIE', '413')) },
         #tavros
         { 'prefix': '', 'replace': lambda x: x.title().swapcase() },
         #sollux
         { 'prefix': '', 'replace': lambda x: x.replace('s', '2').replace('S', '2').replace('i', 'ii').replace('I', 'II') },
         #karkat
         { 'prefix': '', 'replace': lambda x: x.upper() },
         #nepeta
         { 'prefix': ':33 <', 'replace': lambda x: x.replace('ee', '33').replace('EE', '33') },
         #kanaya
         { 'prefix': '', 'replace': lambda x: x.capitalize() },
         #vriska
         { 'prefix': '', 'replace': lambda x: x.translate(string.maketrans('bB', '88')).replace('ate', '8') },
         #equius
         { 'prefix': 'D -->', 'replace': lambda x: x.translate(string.maketrans('xX', '%%')) },
         #gamzee TODO need a full func
         #eridan
         { 'prefix': '', 'replace': lambda x: x.replace('w', 'ww').replace('v', 'vv').replace('W', 'WW').replace('V', 'VV') },
         #feferi
         { 'prefix': '', 'replace': lambda x: x.replace('h', ')(').replace('H', ')(').replace('E', '-E') },
     ]

 def __flatten(dic, prefix):
     '''
     recursively pass through a dict and flatten it\'s "internal" dicts
     '''
     results = {}
     if dic is not None:
         try:
             for key in dic.keys():
                 if type(dic[key]) in [float, int]:
                     results["%s.%s" % (
                         prefix,
                         str(key).translate(string.maketrans(".", "_"))
                     )] = dic[key]
                 elif type(dic[key] in [dict]):
                     results.update(
                         GraphiteUploaderPlugin.__flatten(
                             dic[key],
                             "%s.%s" % (
                                 prefix,
                                 key.translate(string.maketrans(".", "_"))
                             )
                         )
                     )
         except AttributeError:
             pass
     return results

def tm():

    import sys
    import nltk
    import string

    input_file_name = raw_input("Please enter the input file name: ")
    input_path = raw_input("Please enter the input path: ")
    output_file_name = raw_input("Please enter the output file name: ")
    print '\nPlease note that the above entered filename would be used as',
    print 'a prefix for the entire set of documents to be generated.\n'
    output_path = raw_input("Please enter the output path: ")

    with open (input_path + '\\' + input_file_name + '.txt','r') as f:

        para = []
        data = f.read()
        selected=0
        notselect=0
        sentences = data.split("\n\n")

        print "Total # of paragraphs",len(sentences)

        for x in xrange(len(sentences)):
            cond = sentences[x].endswith(".")
            if cond:
                cnt = sentences[x].count(".")
            else:
                cnt= sentences[x].count(".")+1

            if cnt >5:
                #print "paragraph ",x+1,"is selected"
                selected+=1
                sentences[x] = '@'+sentences[x].lower()
                sentences[x] = sentences[x].translate(string.maketrans("",""),string.digits)
                sentences[x] = sentences[x].translate(string.maketrans("",""),string.punctuation)
                tokens = nltk.word_tokenize(sentences[x])
                lemma  = nltk.WordNetLemmatizer()
                porter = nltk.PorterStemmer()

                afewwords = [lemma.lemmatize(i) for i in tokens]
                afewwords = [porter.stem(i) for i in tokens]

                sentences[x] = ' '.join(afewwords)
                para.append(sentences[x])

                filename = output_path + '\\' + output_file_name + str(selected) + '.txt'
                w = open(filename,'w')
                w.write(''.join(para))
                w.close()
                para = []
            else:
                #print "paragraph ",x+1,"is not selected"
                notselect+=1
            #print "cnt - ", cnt
        #print"\n"

        print "# of paragraphs selected",selected
        print "# of paragraphs not selected",notselect
    f.close()

 def checkData(self,row):
     jobTime = row[4]
     jobTerm = row[5]
     w1={'full time':1,'part time':-1,'':0}    #this dictionary corresponds to time feature
     w2={'permanent':0,'contract':1,'':-1}     #this dictionary corresponds to term feature
     
     if jobTime == '' or jobTerm == '':
         s=row[2].lower()          
         s=s.translate(string.maketrans("‘’‚“”„†‡‰‹›!“#$%&‘()™*+,-�./0123456789:;<=>[email protected][\]_`{|}~–—΅Ά£¤¥¦§¨©�«¬®―°±²³΄µ¶·ΈΉΊ»Ό½ΎΏήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ�€Άβ—�ο�'","                                                                                                                                "))
         if jobTime=='':
             if ('full time' in s and 'part time' in s) or ('full time' not in s and 'part time' not in s):
                 word1=''
             else:
                 if 'full time' in s:      #searching full time in description
                     word1='full time'
                 else:
                     word1='part time'
         else:
             word1=jobTime.translate(string.maketrans("_"," ")) #removing underscore from time feature value
             
         if jobTerm=='':
             if ('permanent' in s and 'contract' in s) or ('permanent' not in s and 'contract' not in s):
                 word2=''
             else:
                 if 'permanent' in s:      #searching permanent in description
                     word2='permanent'
                 else:
                     word2='contract'
         else: word2=jobTerm.translate(string.maketrans("_"," "))   #removing underscore from term feature value
     
     else:
         word1=jobTime.translate(string.maketrans("_"," "))
         word2=jobTerm.translate(string.maketrans("_"," "))
         
     return [word1,w1[word1],word2,w2[word2]]

def url_sign( uri_path, params, client_id, signing_key ):
    signing_key = signing_key.translate(string.maketrans('-_', '+/'))
    padding_factor = ( 4 - len( signing_key ) % 4 ) % 4
    signing_key += "=" * padding_factor
    binary_key = base64.b64decode(unicode(signing_key).translate(dict(zip(map(ord, u'-_'), u'+/'))))

    # construct URI for signing
    uri_path_params = uri_path + '?'
    first = True
    for k in params.keys():
        if not first:
            uri_path_params += '&'
        else:
            first = False
        uri_path_params = "%(base)s%(key)s=%(value)s" % {
                                                        'base':uri_path_params,
                                                        'key':k,
                                                        'value':urllib.quote_plus(str(params[k]))
                                                        }
    uri_path_params += '&client=' + client_id

    # Sign
    digest = hmac.new(binary_key, uri_path_params, hashlib.sha1).digest()
    digest = base64.b64encode( digest )
    digest = digest.translate(string.maketrans('+/', '-_'))
    return "%s&sig=%s" % ( uri_path_params, digest.rstrip('=') )

	def generateHashtagSpecificVocabulary(self):
		wordsMappedToHashtags = {}

		for tweet in self.trainSet:
			words = []
			hashtags = []
			for word in tweet.split():
				if word.startswith('#') and len(word) > 2:
					word = word.lower().translate(string.maketrans("",""), string.punctuation) # remove punctuation
					hashtags.append(word)
					words.append(word)
				else:
					if '@' in word:
						continue
					if word in self.stopWords:
						continue
					word = word.lower().translate(string.maketrans("",""), string.punctuation) # remove punctuation
					words.append(word)

			for hashtag in hashtags:
				if hashtag not in wordsMappedToHashtags:
					wordsMappedToHashtags[hashtag] = {}
				for word in words:
					if word not in wordsMappedToHashtags[hashtag]:
						wordsMappedToHashtags[hashtag][word] = 1.0
					else:
						wordsMappedToHashtags[hashtag][word] += 1.0

		return wordsMappedToHashtags

 def checkData(self,row):
     jobTime = row[4]
     jobTerm = row[5]
     w1={'full time':1,'part time':-1,'':0}    #this dictionary corresponds to time feature
     w2={'permanent':0,'contract':1,'':-1}     #this dictionary corresponds to term feature
     
     if jobTime == '' or jobTerm == '':
         s=row[2].lower()          
         s=s.translate(string.maketrans("‘’‚“”„†‡‰‹›!“#$%&‘()™*+,-˜./0123456789:;<=>[email protected][\]_`{|}~–—¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿Þßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ€¢â—ªïž'","                                                                                                                                "))
         if jobTime=='':
             if ('full time' in s and 'part time' in s) or ('full time' not in s and 'part time' not in s):
                 word1=''
             else:
                 if 'full time' in s:      #searching full time in description
                     word1='full time'
                 else:
                     word1='part time'
         else:
             word1=jobTime.translate(string.maketrans("_"," ")) #removing underscore from time feature value
             
         if jobTerm=='':
             if ('permanent' in s and 'contract' in s) or ('permanent' not in s and 'contract' not in s):
                 word2=''
             else:
                 if 'permanent' in s:      #searching permanent in description
                     word2='permanent'
                 else:
                     word2='contract'
         else: word2=jobTerm.translate(string.maketrans("_"," "))   #removing underscore from term feature value
     
     else:
         word1=jobTime.translate(string.maketrans("_"," "))
         word2=jobTerm.translate(string.maketrans("_"," "))
         
     return [word1,w1[word1],word2,w2[word2]]

def make_build_alignment(c_alns, mc_alns, hmc_alns, strand, threshold, max_assignments):
    # translation tables for methylation
    C_trans_table = string.maketrans("C", "C")
    mC_trans_table = string.maketrans("C", "E")
    hmC_trans_table = string.maketrans("C", "O")

    C_table = collect_assignments(c_alns, strand, threshold, max_assignments, C_trans_table)
    mC_table = collect_assignments(mc_alns, strand, threshold, max_assignments, mC_trans_table)
    hmC_table = collect_assignments(hmc_alns, strand, threshold, max_assignments, hmC_trans_table)

    nb_c_assignments = C_table.shape[0] if C_table is not None else "None"
    nb_mc_assignments = mC_table.shape[0] if mC_table is not None else "None"
    nb_hmc_assignments = hmC_table.shape[0] if hmC_table is not None else "None"

    print("[buildAlignments] NOTICE: I found {C} C-assignments, {mC} mC-assignments, and {hmC} hmC-assignments "
          "for strand {strand}"
          "".format(C=nb_c_assignments, mC=nb_mc_assignments, hmC=nb_hmc_assignments, strand=strand),
          file=sys.stderr)
    tables = []

    for table in (C_table, mC_table, hmC_table):
        if table is None:
            continue
        else:
            tables.append(table)

    return pd.concat(tables)

def hashword(plaintext):
	"""
	Munge a plaintext word into something else. Hopefully, the result
	will have some mnemonic value.
	"""
	# get a list of random bytes. A byte will be randomly picked from
	# this list when needed.
	rb = getrandomlist()
	# 0.25 chance of case being swapped
	if rb[rb[0]] < 64:
		plaintext = string.swapcase(plaintext)
	# 0.50 chance of vowels being translated one of two ways.
	if rb[rb[2]] > 127:
		plaintext = string.translate(plaintext, 
			string.maketrans('aeiou AEIOU', '@3!0& 4#10%'))
	else:
		plaintext = string.translate(plaintext, 
			string.maketrans('aeiou AEIOU', '^#1$~ $3!0&'))
	# 0.4 chance of some additional consonant translation
	if rb[rb[4]] < 102:
		plaintext = string.translate(plaintext, 
			string.maketrans('cglt CGLT', '(<1+ (<1+'))
	# if word is short, add some digits
	if len(plaintext) < 5:
		plaintext = plaintext + `rb[5]`
	# 0.2 chance of some more digits appended
	if rb[rb[3]] < 51:
		plaintext = plaintext + `rb[205]`
	return plaintext

 def checkreserveduser(user):
   if os.path.isdir(os.path.join(config.datadir, user)):
     return
   if user.lower() != user:
     errorexit('Username should be lowercase.')
   normalized = user.lower()
   if os.path.isdir(os.path.join(config.datadir, normalized)):
     errorexit('Username is reserved.')
   normalized = user.lower()
   if normalized != user and os.path.isdir(
         os.path.join(config.datadir, normalized)):
     errorexit('Username is reserved.')
   normalizedi = normalized.translate(string.maketrans('013456789', 'oieasbtbg'))
   if normalized != normalizedi and os.path.isdir(
         os.path.join(config.datadir, normalizedi)):
     errorexit('Username is reserved.')
   normalizedl = normalized.translate(string.maketrans('013456789', 'oleasbtbg'))
   if normalizedl != normalized and os.path.isdir(
         os.path.join(config.datadir, normalizedl)):
     errorexit('Username is reserved.')
   with open(os.path.join(progpath, 'bad-words.txt')) as f:
     badwords = f.read().splitlines()
   if any(word in badwords for word in [normalized, normalizedi, normalizedl]):
     errorexit('Username is reserved.')
   return