def getData(self, url):
		
		data = Data()

		try:
			req = urllib2.Request(url, headers={'User-Agent' : "Magic Browser"})
			request = urllib2.urlopen(req)
			mime = request.info().getheader('Content-Type')		
			code = request.code

			print(colored('[' + mime + '] ' + url, 'yellow'))

			if code is 200:
				if 'text/html' in mime:

					html = request.read()
					data = self.parse(html, url)
					
				else:
					#ANALYSIS TYPE
					data.url = url
					data.type = mime

			elif code is 400:
				data.broke = True

		except UnicodeEncodeError as e :

			print(colored(e, 'red'))
			data.broke = True

		return data 

def hash_cracker(dict_file_name, hash_file_name):
    found = 0
    file_dict = open(dict_file_name, "r")
    passwords = file_dict.read().split('\n')
    file_hash = open(hash_file_name, "r")
    enc_hashes = file_hash.read().split('\n')
    print enc_hashes
    for pwd in passwords:
        try:
            for rec in enc_hashes:
                # print pwd.split(':')[1]
                test = sha1(pwd.split(':')[1] + rec.split(':')[1]).hexdigest()
                if pwd.split(':')[1] == 'admin':
                    print rec.split(':')[0]
                    print test
                if test == rec.split(':')[0]:
                    print colored("[-] Password Found for password_sha " + rec.split(':')[0] + " : " + pwd)
                    found = 1
                    break
                else:
                    pass
                    if found == 1:
                        break
        except:
            pass

def server():
	global connection
	global board
	global newTiles
	global WAIT
	# Create a TCP/IP socket
	server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	# Bind the socket to the port
	if int(sys.argv[1]) == 1:
		server_address = ('localhost', PORT_NUMBER_2)
	else:
		server_address = ('localhost', PORT_NUMBER_1)
	print >>sys.stderr, 'starting up on %s port %s' % server_address
	server_sock.bind(server_address)
	# Listen for incoming connections
	server_sock.listen(1)

	while True:
	    # Wait for a connection
	    print >>sys.stderr, 'waiting for a connection'
	    connection, client_address = server_sock.accept()
	    try:
	        print >>sys.stderr, 'connection from', client_address
	        state = "NORMAL"
	        # Receive the data in small chunks and retransmit it
	        while True:
	            data = connection.recv(1000)
	            # print >>sys.stderr, 'received "%s"' % data
	            if data:
	            	if data == "BOARD" or data == "INITIAL" or data == "NEW_TILES":
	            		state = data
	            		print "STATE: %s"%state
	            	else:
		            	if state == "BOARD":
		            		board = pickle.loads(data)
		            		print colored("The other player has made a move",'red')
		            		printBoard()
		            		print "received board"
		            		WAIT = False
		            	elif state == "INITIAL":
		            		board = pickle.loads(data)
		            		print "received initial"
		            		printBoard()
		            	elif state == "NEW_TILES":
		            		newTiles = pickle.loads(data)
		            		print newTiles
		            		print "received new tiles"
		            	state = "NORMAL"
		            	print "STATE: NORMAL"
	            else:
	                break
	    except KeyboardInterrupt:
	    	print "Closed connection on server"
	    	connection.close()
	    	break     
	    finally:
	        # Clean up the connection
	    	print "Closed connection on server"
	        connection.close()
	        break

    def process(self, msg, kwargs):
        if self.extra is None:
            return u'{}'.format(msg), kwargs

        if 'module' in self.extra.keys():
            if len(self.extra['module']) > 8:
                self.extra['module'] = self.extra['module'][:8] + '...'

        #If the logger is being called when hooking the 'options' module function
        if len(self.extra) == 1 and ('module' in self.extra.keys()):
            return u'{:<64} {}'.format(colored(self.extra['module'], 'cyan', attrs=['bold']), msg), kwargs

        #If the logger is being called from CMEServer
        if len(self.extra) == 2 and ('module' in self.extra.keys()) and ('host' in self.extra.keys()):
            return u'{:<24} {:<39} {}'.format(colored(self.extra['module'], 'cyan', attrs=['bold']), self.extra['host'], msg), kwargs

        #If the logger is being called from a protocol
        if 'module' in self.extra.keys():
            module_name = colored(self.extra['module'], 'cyan', attrs=['bold'])
        else:
            module_name = colored(self.extra['protocol'], 'blue', attrs=['bold'])

        return u'{:<24} {:<15} {:<6} {:<16} {}'.format(module_name,
                                                    self.extra['host'],
                                                    self.extra['port'],
                                                    self.extra['hostname'].decode('utf-8') if self.extra['hostname'] else 'NONE',
                                                    msg), kwargs

def one(expected, champs, display, shuffle = False):
    global count, total
    vm = VM()
    vm.add(champs)

    if shuffle:
        vm.shuffle()

    if display:
        util.tprint('Battle', '%s / %s' % (colored('%d' % (count), 'yellow'), colored('%d' % (total), 'yellow')))
        util.tprint('Champions', ', '.join([colored(champ.champion.name, 'blue', attrs = ['bold']) for champ in vm.champions]))
    count += 1
    winner = vm.launch(stdout = PIPE)
    report = None
    if winner:
        if display:
            if winner.champion.filename == expected.filename:
                color = 'green'
            else:
                color = 'red'
            util.tprint('Winner', colored(winner.champion.name, color, attrs = ['bold']))
        report = Report(champs, winner.champion)

    if display:
        print ''
    return report

def main(username):
    google_api = vault.get_key('google_api')
    if google_api != None:
        API_SERVICE_NAME = "youtube"
        API_VERSION = "v3"
        max_results = 50
        video_ids = []
        service = build(API_SERVICE_NAME, API_VERSION,
                        developerKey=google_api)
        channel_id = find_channel_by_username(service,
                                              part='snippet',
                                              maxResults=max_results,
                                              q=username)
        if channel_id is not None:
            channel_details = get_channel_details(username,
                                                  service,
                                                  part='snippet,contentDetails,statistics',
                                                  id=channel_id['Channel ID'])

            channel_analysis = analyze_activity(service,
                                                part='snippet,contentDetails',
                                                channelId=channel_id['Channel ID'],
                                                maxResults=max_results)
            return [ channel_id, channel_details, channel_analysis ]
        else:
            return [ colored(style.BOLD +'[!] Error: Channel not found for ' +
                             username + '\n' + style.END, 'red') ]
    else:
        return [ colored(style.BOLD + '[!] Error: No Google API key found. Skipping' +
                         style.END, 'red') ]

def list_letter(aset, vb=False, output=False):
    '''list words by letter, from a-z (ingore case).'''

    conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
    curs = conn.cursor()
    try:
        if not vb:
            curs.execute('SELECT name, pr FROM Word WHERE aset = "%s"' % aset)
        else:
            curs.execute('SELECT expl, pr FROM Word WHERE aset = "%s"' % aset)
    except Exception as e:
        print(colored('something\'s wrong, catlog is from A to Z', 'red'))
        print(e)
    else:
        if not output:
            print(colored(format(aset, '-^40s'), 'green'))
        else:
            print(format(aset, '-^40s'))

        for line in curs.fetchall():
            expl = line[0]
            pr = line[1]
            print('\n' + '=' * 40 + '\n')
            if not output:
                print(colored('★ ' * pr, 'red', ), colored('☆ ' * (5 - pr), 'yellow'), sep='')
                colorful_print(expl)
            else:
                print('★ ' * pr + '☆ ' * (5 - pr))
                normal_print(expl)
    finally:
        curs.close()
        conn.close()

def bye():
    global flag
    global op_stop
    flag = False
    op_stop = True
    print termcolor.colored("Bye", "cyan")
    print termcolor.colored("有任何建议欢迎与我联系: [email protected]", "cyan")

    def display_time(self):
        length = len(self.TITLE)
        while True:
            if self.q == 1:  # 退出
                break
            if self.song_time >= 0 and self.douban.playingsong:
                minute = int(self.song_time) / 60
                sec = int(self.song_time) % 60
                show_time = str(minute).zfill(2) + ':' + str(sec).zfill(2)

                self.volume = self.get_volume()  # 获取音量
                self.TITLE = self.TITLE[:length - 1] + '  ' + self.douban.playingsong['kbps'] + 'kbps  ' + colored(show_time, 'cyan') + '  rate: ' + colored(self.rate[int(round(self.douban.playingsong['rating_avg'])) - 1], 'red') + '  vol: '
                if self.is_muted:
                    self.TITLE += '✖'
                else:
                    self.TITLE += self.volume.strip() + '%'
                if self.loop:
                    self.TITLE += '  ' + colored('↺', 'red')
                else:
                    self.TITLE += '  ' + colored('→', 'red')
                self.TITLE += '\r'
                self.display()
                if not self.pause:
                    self.song_time -= 1
            else:
                self.TITLE = self.TITLE[:length]
            time.sleep(1)

 def __init__(self):
   # Load the trained models
   print(colored('Loading CLF models','magenta'))
   self.topicHasher = texthasher.safe_load(TEXT_VECTORIZER_PATH,stop_words=[])
   self.tagHasher   = taghasher.safe_load(TAG_HASHER_PATH,n_feature=256)
   self.clf         = cluster.safe_load(CLF_PATH,method=None,n_features=256)
   print(colored('[DONE] ','green'),'CLF models loaded')

def loadsession():
    global session
    try:
        session.cookies.load(ignore_discard="true")
    except:
        termcolor.colored("加载异常", "red")
        pass

def pty_based_auth(auth_backend, pty):
    """
    Show a username/password login prompt.
    Return username when authentication succeeded.
    """
    tries = 0
    while True:
        # Password authentication required for this session?
        sys.stdout.write('\033[2J\033[0;0H') # Clear screen
        sys.stdout.write(colored('Please authenticate\r\n\r\n', 'cyan'))

        if tries:
            sys.stdout.write(colored('  Authentication failed, try again\r\n', 'red'))

        try:
            console = Console(pty)
            username = console.input('Username', False)
            password = console.input('Password', True)
        except NoInput:
            raise NotAuthenticated

        if auth_backend.authenticate(username, password):
            sys.stdout.write(colored(' ' * 40 + 'Authentication successful\r\n\r\n', 'green'))
            return username
        else:
            tries += 1
            if tries == 3:
                raise NotAuthenticated

def show_telemetry(tel):
    output = [] 

    for k, v in tel.iteritems():
        if k == 'register':
            color = 'blue'

        elif k == 'density':
            color = 'red'

        elif k == 'harmonicity':
            color = 'green'

        elif k == 'roughness':
            color = 'cyan'

        elif k == 'pace':
            color = 'yellow'

        else:
            color = 'white'

        if k == 'name':
            output += [ colored(' '.join(v), color) ]
        else:
            output += [ colored('%s: %.2f' % (k[:3], v), color) ]

    output = ' | '.join(output)
    dsp.log(output)

def search_replace_with_prompt(fpath, txt1, txt2, force=False):
	""" Search and replace all txt1 by txt2 in the file with confirmation"""

	from termcolor import colored
	with open(fpath, 'r') as f:
		content = f.readlines()

	tmp = []
	for c in content:
		if c.find(txt1) != -1:
			print fpath
			print  colored(txt1, 'red').join(c[:-1].split(txt1))
			a = ''
			if force:
				c = c.replace(txt1, txt2)
			else:
				while a.lower() not in ['y', 'n', 'skip']:
					a = raw_input('Do you want to Change [y/n/skip]?')
				if a.lower() == 'y':
					c = c.replace(txt1, txt2)
				elif a.lower() == 'skip':
					return 'skip'
		tmp.append(c)

	with open(fpath, 'w') as f:
		f.write(''.join(tmp))
	print colored('Updated', 'green')

    def play(self):
        self.lrc_dict = {}  # 歌词清空
        if not self.loop:
            self.douban.get_song()
        if self.is_muted:  # 静音状态
            subprocess.Popen('echo "mute {mute}" > {fifo}'.format(fifo=self.mplayer_controller, mute=1), shell=True, stdin=subprocess.PIPE)
        song = self.douban.playingsong
        self.song_time = song['length']
        # 是否是红心歌曲
        if song['like'] == 1:
            love = self.love
        else:
            love = ''
        title = colored(song['title'], 'green')
        albumtitle = colored(song['albumtitle'], 'yellow')
        artist = colored(song['artist'], 'white')
        self.SUFFIX_SELECTED = (love + ' ' + title + ' • ' + albumtitle + ' • ' + artist + ' ' + song['public_time']).replace('\\', '')

        cmd = 'mplayer -slave -input file={fifo} {song_url} >/dev/null 2>&1'
        self.p = subprocess.Popen(cmd.format(fifo=self.mplayer_controller, song_url=song['url']), shell=True, stdin=subprocess.PIPE)  # subprocess.PIPE防止继承父进程
        self.pause = False
        self.display()
        self.notifySend()
        if self.lrc_display:  # 获取歌词
            self.lrc_dict = self.douban.get_lrc()
            if not self.lrc_dict:  # 歌词获取失败,关闭歌词界面
                self.lrc_display = 0
        self.start = 1

 def informer(self,_type,msg,debug=True,log=False,color='green',type_color="cyan",attrs=["bold"]):
     if debug :print '[ '+colored(_type,type_color,attrs=attrs)+' ]'" :"+colored(msg,color,attrs=attrs)
     if log:
         try:
             self.log_file.write(msg)
         except Exception,e:
             print colored(e,'red','on_white',attrs=['bold'])

def xap_doc_sim_for_pmids(pmids,n_terms=100):
	enq=xapian.Enquire(database)
	rset=xapian.RSet()
	for a in pmids:
		try: 
			database.get_document(a)
		except xapian.DocNotFoundError:
			continue
		rset.add_document(a)
	if rset.empty():
		print colored("No documents for set %s"%(str(pmids)),'red')
		return {},[]
	eset=enq.get_eset(n_terms,rset) # Number of terms in the eset
	# q=xapian.Query(xapian.Query.OP_OR,eset.begin(),eset.end())
	terms=[x[0] for x in eset.items]
	print ";".join(terms[:10])
	q=xapian.Query(xapian.Query.OP_OR,terms)
	enq.set_query(q)
	res=enq.get_mset(0,database.get_doccount())
	weighted_docs=[(r.docid,r.weight) for r in res]
	#normalize the weights to [0,1]
	max_w=max([x[1] for x in weighted_docs])

	weighted_docs=[(x[0],x[1]*1.0/max_w) for x in weighted_docs]

	sorted_results=dict(weighted_docs)
	return sorted_results,terms

def search_database(word):
    '''offline search.'''

    conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
    curs = conn.cursor()
    curs.execute(r'SELECT expl, pr FROM Word WHERE name LIKE "%s%%"' % word)
    res = curs.fetchall()
    if res:
        print(colored(word + ' 在数据库中存在', 'white', 'on_green'))
        print()
        print(colored('★ ' * res[0][1], 'red'), colored('☆ ' * (5 - res[0][1]), 'yellow'), sep='')
        colorful_print(res[0][0])
    else:
        print(colored(word + ' 不在本地，从有道词典查询', 'white', 'on_red'))
        search_online(word)
        input_msg = '若存入本地，请输入优先级(1~5) ，否则 Enter 跳过\n>>> '
        if sys.version_info[0] == 2:
            add_in_db_pr = raw_input(input_msg)
        else:
            add_in_db_pr = input(input_msg)

        if add_in_db_pr and add_in_db_pr.isdigit():
            if(int(add_in_db_pr) >= 1 and int(add_in_db_pr) <= 5):
                add_word(word, int(add_in_db_pr))
                print(colored('单词 {word} 已加入数据库中'.format(word=word), 'white', 'on_red'))
    curs.close()
    conn.close()

def remap_keys(old_filename, new_filename, output_folder):
    """
    Create a script to remap bibtex keys from one .bib file to another.

    This function uses the edit distance on filenames generated from a bitex entry
    to compare entries together, and greedily matches old entries to new entries.
    """
    old_db = utils.read_bib_file(old_filename, homogenize=False)
    new_db = utils.read_bib_file(new_filename, homogenize=False)
    old_list = {}
    new_list = {}
    subst = {}
    for entry in new_db.entries:
        name = nomenclature.gen_filename(entry)
        new_list[name] = entry['ID']
    for entry in old_db.entries:
        name = nomenclature.gen_filename(entry)
        if name in new_list.keys():
            subst[entry['ID']] = new_list[name]
            del new_list[name]
        else:
            old_list[name] = entry['ID']
    for name, bibkey in new_list.items():
        match, score = utils.most_similar_filename(name, old_list.keys())
        if score < 0.90:
            print(termcolor.colored("Warning: potentially incorrect substitution:", 'yellow', attrs=["bold"]))
            print(termcolor.colored('-', 'red') + match)
            print(termcolor.colored('+', 'green') + name)
        subst[old_list[match]] = bibkey
    utils.write_remap_script(subst, output_folder)

def list_latest(limit, vb=False, output=False):
    '''list words by latest time you add to database.'''

    conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
    curs = conn.cursor()
    try:
        if not vb:
            curs.execute('SELECT name, pr, addtime FROM Word ORDER by datetime(addtime) DESC LIMIT  %d' % limit)
        else:
            curs.execute('SELECT expl, pr, addtime FROM Word ORDER by datetime(addtime) DESC LIMIT  %d' % limit)
    except Exception as e:
        print(e)
        print(colored('something\'s wrong, please set the limit', 'red'))
    else:
        for line in curs.fetchall():
            expl = line[0]
            pr = line[1]
            print('\n' + '=' * 40 + '\n')
            if not output:
                print(colored('★ ' * pr, 'red'), colored('☆ ' * (5 - pr), 'yellow'), sep='')
                colorful_print(expl)
            else:
                print('★ ' * pr + '☆ ' * (5 - pr))
                normal_print(expl)
    finally:
        curs.close()
        conn.close()