def get_tags(limit):
    tagset = {}
    try:
        for code in Code.where():
            tag = code.tags
            if not tag:
                continue
            ts = tag.split(",")
            for t in ts:
                if tagset.has_key(t):
                    tagset[t] += 1
                else:
                    tagset[t] = 1

        for post in Post.where(status=1):
            tag = post.tags
            if not tag:
                continue
            ts = tag.split(",")
            for t in ts:
                if tagset.has_key(t):
                    tagset[t] += 1
                else:
                    tagset[t] = 1       
        sort_tags = sorted( tagset.items(),key=lambda d:d[1],reverse=True)
        print sort_tags
        return sort_tags[:limit]
    except Exception,e:
        logger.error("get tags error,%s"%e)
        raise e

def package(pkg_name, option="-y"):

	if pkg_name:
		command_line_str = "yum " + option + " install " + pkg_name
		if os.system(command_line_str)!=0:
			logger.error("exec: %s error" % (command_line_str))
			sys.exit(1)

    def get_qty(self):
        qty_res = [p.text for p in self.s.select(
            settings.WELL['qty_css_selector'])]

        try:
            # Extract product attributes: |Size|QTY|Weight|Unit_Price|Price
            qty_regex_result = re.findall(
                settings.WELL['qty_regex'], qty_res[0])
            assert(len(qty_regex_result) == 1)

            # Strip unnecessary stuff
            qty_regex_result[0] = re.sub(
                r"swatch_stock_list\[0\]='.*'; ", "", qty_regex_result[0])
            qty_regex_results = qty_regex_result[0].split(" ")

            # Second element is package quantity
            self.pkg_qty = qty_regex_results[1]

            try:
                # Calculate unit price
                self.unit_price = "%.2f" % (
                    (float)((self.current_price).strip('$')) / (float)(self.pkg_qty))
                # Reconvert to string by adding the $ sign
                self.unit_price = "$" + self.unit_price

            except ValueError as err:
                logger.error("Unable calculating price: %s" %
                             (err), exc_info=True)

        except (AssertionError, IndexError) as err:
            logger.error("Unable to get qty: %s" % (err), exc_info=True)

 def fetch_result(self, cond={}):
     try:
         for item in self.db.Result.find(cond):
             yield item
     except errors.ServerSelectionTimeoutError as e:
         logger.error("timed out to fetch {0}".format(table))
         raise exceptions.ConnectionError("connecting timeout")

 def fetch(self, table, cond):
     try:
         for item in getattr(self.db, table).find(cond):
             yield item
     except errors.ServerSelectionTimeoutError as e:
         logger.error("timed out to fetch {0}".format(table))
         raise exceptions.ConnectionError("connecting timeout")

    def parse_data(self):
        """Parse info for each product"""
        # Extract price information
        regular_price = self.s.select(
            settings.TOYSRUS['regular_price_css_selector'])

        try:
            # Sometimes regular price is lower than current
            assert(len(regular_price) <= 1)
            current_price = self.s.select(
                settings.TOYSRUS['current_price_css_selector'])
            assert(len(current_price) == 1)

            if(regular_price):
                self.regular_price = regular_price[0].text.strip()
            self.current_price = current_price[0].text.strip()
        except AssertionError as err:
            logger.error("Unable to get price: %s" % (err), exc_info=True)

        # Extract upc info
        upc = self.s.select(settings.TOYSRUS['upc_css_selector'])
        try:
            assert(len(upc) == 1)
            self.upc = upc[0].text.strip()
        except AssertionError as err:
            logger.error("Unable to get UPC code: %s" % (err), exc_info=True)

        # Extract qty and unit price info
        self.get_qty()

        # Extract size information
        self.get_size()

def sitemap_data():
    urlset = []
    try:
        for post in Post.where(status=1):
            lastmod_tmp = datetime.datetime.strptime(post.modified,"%Y-%m-%d %H:%M:%S")
            lastmod = lastmod_tmp.strftime("%Y-%m-%dT%H:%M:%SZ")
            url = dict(loc="http://www.talkincode.org/news/post/view/%s"%post.id,
                       lastmod=lastmod,chgfreq="daily")
            urlset.append(url)


        for code in Code.where():
            lastmod_tmp = datetime.datetime.strptime(code.create_time,"%Y-%m-%d %H:%M:%S")
            lastmod = lastmod_tmp.strftime("%Y-%m-%dT%H:%M:%SZ")
            url = dict(loc="http://www.talkincode.org/code/view/%s"%code.id,
                       lastmod=lastmod,chgfreq="monthly")
            urlset.append(url)       

        for proj in Project.where():
            lastmod_tmp = datetime.datetime.strptime(proj.created,"%Y-%m-%d %H:%M:%S")
            lastmod = lastmod_tmp.strftime("%Y-%m-%dT%H:%M:%SZ")
            url = dict(loc="http://www.talkincode.org/open/proj/view/%s"%proj.id,
                       lastmod=lastmod,chgfreq="monthly")
            urlset.append(url)                             
          
        return urlset
    except Exception,e:
        logger.error("login error,%s"%e)
        return []

	def parse_blocks(self):		
		lineno, interval, intervals = 0, {}, []

		def block_ends(interval, intervals):
			interval['lineno'] = lineno
			intervals.append(interval)

		# iterator for MULTILINES_PATTERN
		mp_iter = CycleIterator(self.__pack(TextgridBlocksParser.PATTERN_KEYS, TextgridBlocksParser.MULTILINES_PATTERN))
		# iterator for BLOCK_PATTERN
		bp_iter = CycleIterator(self.__pack(TextgridBlocksParser.PATTERN_KEYS, TextgridBlocksParser.BLOCK_PATTERN))
		line_pattern = bp_iter.next()

		for line in self.lines:
			lineno += 1

			# always try to match the begining pattern at first to avoid missing a normal block 
			# therefore, reset the block parsing once a line was matched to the begining pattern
			# but unmatched to the current one.
			if not bp_iter.begins() and self.__match(bp_iter.head(), line):
				logger.error('unable to parse line %d, ignored' % (lineno-1))
				interval, line_pattern = {}, bp_iter.reset()

			# to match the pattern one by one until it ends
			if self.__match(line_pattern, line):
				self.__update(interval, line_pattern, line)

				# if the end of block was matched
				# block ends here for most situation
				if bp_iter.ends():
					block_ends(interval, intervals)
					interval = {}

			# when a text existed in multiple lines
			elif bp_iter.ends():
					# match the begining of text in multi-lines
					if self.__match(mp_iter.head(), line):
						self.__update(interval, mp_iter.head(), line)
						continue # should not to call the next block pattern

					# match the pattern of end line
					# block also may end here for multiple lines
					elif self.__match(mp_iter.tail(), line): 
						self.__update(interval, mp_iter.tail(), line, append=True)
						block_ends(interval, intervals)
						interval = {}

					# match the pattern without quotes
					else:
						# append the middle part of the text
						self.__update(interval, mp_iter.index(1), line, append=True)
						continue
			else:
				# does not match anything
				# logger.error('unable to parse line %d, ignored' % (lineno-1))
				continue
			
			line_pattern = bp_iter.next()	# match the next pattern

		return intervals

def get_tests_groups_from_jenkins(runner_name, build_number, distros):
    runner_build = Build(runner_name, build_number)
    res = {}
    for b in runner_build.build_data['subBuilds']:

        if b['result'] is None:
            logger.debug("Skipping '{0}' job (build #{1}) because it's still "
                         "running...".format(b['jobName'], b['buildNumber'],))
            continue

        # Get the test group from the console of the job
        z = Build(b['jobName'], b['buildNumber'])
        console = z.get_job_console()
        groups = [keyword.split('=')[1]
                  for line in console
                  for keyword in line.split()
                  if 'run_tests.py' in line and '--group=' in keyword]
        if not groups:
            logger.error("No test group found in console of the job {0}/{1}"
                         .format(b['jobName'], b['buildNumber']))
            continue
        # Use the last group (there can be several groups in upgrade jobs)
        test_group = groups[-1]

        # Get the job suffix
        job_name = b['jobName']
        for distro in distros:
            if distro in job_name:
                sep = '.' + distro + '.'
                job_suffix = job_name.split(sep)[-1]
                break
        else:
            job_suffix = job_name.split('.')[-1]
        res[job_suffix] = test_group
    return res

    def get_check_create_test_run(self, plan, cases):
        plan = self.project.plans.get(plan.id)
        suite_cases = self.suite.cases()
        run_name = self.get_run_name()
        runs = plan.runs.find_all(name=run_name)
        run = self.check_need_create_run(plan,
                                         runs,
                                         suite_cases)

        if run is None:
            logger.info('Run not found in plan "{}", create: "{}"'.format(
                plan.name, run_name))

            # Create new test run with cases from test suite
            suite_cases = self.get_suite_cases()

            if not suite_cases:
                logger.error('Empty test cases set.')
                return None

            # suite_cases = self.suite.cases.find(type_id=type_ids[0])
            run = Run(name=run_name,
                      description=self.run_description,
                      suite_id=self.suite.id,
                      milestone_id=self.milestone.id,
                      config_ids=[],
                      case_ids=[x.id for x in suite_cases]
                      )
            plan.add_run(run)
            logger.debug('Run created "{}"'.format(run_name))
        return run

def r_file(filename, mode=None, content="", action=""):

	is_exists_file = os.path.exists(filename)

	if action == "create":

		if is_exists_file:

			try:
				os.remove(filename)
				os.mknod(filename)
				logger.info("Create File Ok.")

				with open(filename , 'w+') as f:
					f.write(content)

			except OSError, e:
				logger.error("filename: %s " % (filename) + str(e) )
				sys.exit(1)
		else:

			try:
				os.mknod(filename)
				logger.info("Create File Ok.")

				with open(filename , 'w+') as f:
					f.write(content)

			except OSError, e:
				logger.error("filename: %s" % (filename) + str(e))
				sys.exit(1)

def weater(lat, lng, timezone, unit='celsius', weather_key_=weather_key):
    res = {}
    try:
        owm = pyowm.OWM(weather_key_)
        observation = owm.three_hours_forecast_at_coords(lat, lng)
    except Exception, e:
        logger.error("weater {0}".format(e.message))

	def exec_dcfg(self):

		# first exec support script to install system tables.
		try:
			logger.info("============MYSQL DEFAULT CONFIG===========")
			logger.info("install system db.")
			os.chdir(mysql_home)
			exec_command('./scripts/mysql_install_db --user=mysql')

			logger.info("copy boot script to correct directory.")
			exec_command('cp ' + mysql_boot_script + ' /etc/init.d/')

			# sed config
			exec_command('sed -i -e "46s/basedir=/basedir=\/opt\/magima\/mysql/g" /etc/init.d/mysql.server')

			exec_command('sed -i -e "47s/datadir=/datadir=\/opt\/magima\/mysql\/data/g" /etc/init.d/mysql.server')

			exec_command("/etc/init.d/mysql.server start")
			exec_command("/etc/init.d/mysql.server status")
			exec_command("/etc/init.d/mysql.server stop")
			logger.info("==============TOMCAT DEFAULT CONFIG==============")
			logger.info("copy tomcat bootscript to /etc/init.d/")
			exec_command("cp " + tomcat_bootstrap + " /etc/init.d/tomcat6")
			exec_command("sudo /etc/init.d/tomcat6 start")
			exec_command("sudo /etc/init.d/tomcat6 status")
			exec_command("sudo /etc/init.d/tomcat6 stop")

		except OSError , oserr:
			logger.error("os error: %s " % str(oserr))
			sys.exit(1)

def apply_transactions(transactions, auto=False):
    ''' Apply renaming transactions.
    apply_transactions(transactions)
    transactions = [(old_path, new_path),(old_path),(new_path),...]
    Manual review of transactions is required.
    '''
    if auto:
        logger.warning('Auto is On. No confirmation required.')
    print('='*30)
    if not transactions:
        logger.debug('NO TRANSACTIONS')
        sys.exit('No Transactions to apply.')
        return

    for t in transactions:
        print('[{}] > [{}]'.format(t[0].name, t[1].name))
    print('{} Transactions to apply. Renaming...'.format(len(transactions)))
    count = 0
    if auto or input('EXECUTE ? [y]\n>') == 'y':
        for src, dst in transactions:
            try:
                src.rename(dst)
            except:
                logger.error(sys.exc_info()[0].__name__)
                logger.error('Could not rename: [{}]>[{}]'.format(src, dst))
            else:
                logger.debug('[{}] renamed to [{}]'.format(src, dst))
                count += 1

        print('{} folders renamed.'.format(count))

    def discover_functions(self):
	self.self_cursor.execute("SELECT pro_oid,func_name,id FROM function_name WHERE {0}={1} AND alive".format(self.sub_fk,self.id))
	local_funcs=self.self_cursor.fetchall()
	try:
	    self.prod_cursor.execute("""SELECT p.oid AS pro_oid,p.proname AS funcname,p.proretset,t.typname,l.lanname
FROM pg_proc p
LEFT JOIN pg_namespace n ON n.oid = p.pronamespace
JOIN pg_type t ON p.prorettype=t.oid
JOIN pg_language l ON p.prolang=l.oid
WHERE (p.prolang <> (12)::oid)
AND n.oid=(SELECT oid FROM pg_namespace WHERE nspname='public')""")
	except Exception as e:
	    logger.error("Cannot execute function discovery query: {0}".format(e.pgerror))
	    return
	prod_funcs=self.prod_cursor.fetchall()
	for l_func in local_funcs:
	    for p_func in prod_funcs:
		if l_func[0]==p_func[0] and l_func[1]==p_func[1]:
		    break
	    else:
		logger.info("Retired function {0} in schema {1}".format(l_func[1],self.db_fields['sch_name']))
		old_func=FunctionName(l_func[2])
#		old_func.populate()
		old_func.retire()
	for p_func in  prod_funcs:
	    for l_func in local_funcs:
		if p_func[0]==l_func[0] and p_func[1]==l_func[1]:
		    break
	    else:
		logger.info("Created new function: {0} in schema {1}".format(p_func[1],self.db_fields['sch_name']))
		new_func=FunctionName()
		new_func.set_fields(sn_id=self.id,pro_oid=p_func[0],func_name=p_func[1],proretset=p_func[2],prorettype=p_func[3],prolang=p_func[4])
		new_func.create()
		new_func.truncate()

    def get(self):
        uid = self.current_user
        type_ = self.get_argument('type', None)
        if not type_:
            self.set_status(400)
            result = dict(code=40011, msg=u'缺少type参数')
            return self.jsonify(result)

        keep_info = self.keep_map(type_)

        key = "uid:{}:keep:{}".format(uid, type_)
        times = rdb.incr(key)
        if times == 1:
            rdb.expire(key, get_to_tomorrow())
        else:
            logger.warning('have try times {}'.format(times))
            result = dict(code=40010, msg=u'每天只能{}一次哦!'.format(keep_info['name']))
            return self.jsonify(result)

        try:
            row = Pet.keep(uid=uid, score=keep_info['score'])
            logger.info('keep pet {}'.format(row))
        except Exception, e:
            self.set_status(500)
            logger.error('keep pet error {}'.format(e))
            result = dict(code=40012, msg=u'更新服务器错误, 请稍后重试!')
            return self.jsonify(result)

    def discover_tables(self):
	self.self_cursor.execute("SELECT obj_oid,tbl_name,id FROM table_name WHERE {0}={1} AND alive".format(self.sub_fk,self.id))
	local_tbls=self.self_cursor.fetchall()
	try:
	    self.prod_cursor.execute("""SELECT r.oid,r.relname,
CASE WHEN h.inhrelid IS NULL THEN 'f'::boolean ELSE 't'::boolean END AS has_parent
FROM pg_class r
LEFT JOIN pg_inherits h ON r.oid=h.inhrelid
WHERE r.relkind='r'
AND r.relnamespace=(SELECT oid FROM pg_namespace WHERE nspname='public')""")
	except Exception as e:
	    logger.error("Cannot execute tables discovery query: {0}".format(e.pgerror))
	    return
	prod_tbls=self.prod_cursor.fetchall()
	for l_table in local_tbls:
	    for p_table in prod_tbls:
		if l_table[0]==p_table[0] and l_table[1]==p_table[1]:
		    break
	    else:
		logger.info("Retired table {0} in schema {1}".format(l_table[1],self.db_fields['sch_name']))
		old_table=TableName(l_table[2])
#		old_table.populate()
		old_table.retire()
	for p_table in  prod_tbls:
	    for l_table in local_tbls:
		if p_table[0]==l_table[0] and p_table[1]==l_table[1]:
		    break
	    else:
		logger.info("Created new table: {0} in schema {1}".format(p_table[1],self.db_fields['sch_name']))
		new_table=TableName()
		new_table.set_fields(sn_id=self.id,tbl_name=p_table[1],obj_oid=p_table[0],has_parent=p_table[2])
		new_table.create()
		new_table.truncate()

    def discover_indexes(self):
	self.self_cursor.execute("SELECT obj_oid,idx_name,id FROM index_name WHERE tn_id={0} AND alive".format(self.id))
	local_idxs=self.self_cursor.fetchall()
	try:
	    self.prod_cursor.execute("""SELECT i.indexrelid,c.relname,i.indisunique,i.indisprimary
FROM pg_index i
JOIN pg_class c ON i.indexrelid=c.oid
WHERE i.indrelid={0}""".format(self.db_fields['obj_oid']))
	except Exception as e:
	    logger.error("Cannot execute index discovery query: {0}".format(e.pgerror))
	    return
	prod_idxs=self.prod_cursor.fetchall()
	for l_idx in local_idxs:
	    for p_idx in prod_idxs:
		if l_idx[0]==p_idx[0] and l_idx[1]==p_idx[1]:
		    break
	    else:
		logger.info("Retired index {0} in table {1}".format(l_idx[1],self.db_fields['tbl_name']))
		old_idx=IndexName(l_idx[2])
		old_idx.retire()
	for p_idx in prod_idxs:
	    for l_idx in local_idxs:
		if l_idx[0]==p_idx[0] and l_idx[1]==p_idx[1]:
		    break
	    else:
		logger.info("Create new index {0} in table {1}".format(p_idx[1],self.db_fields['tbl_name']))
		new_index=IndexName()
		new_index.set_fields(tn_id=self.id,obj_oid=p_idx[0],idx_name=p_idx[1],is_unique=p_idx[2],is_primary=p_idx[3])
		new_index.create()
		new_index.truncate()

 def saveUserInfo(self, appid, username, password):
     """
     将用户注册信息入库
     :param appid:
     :param username:
     :param password:
     :return: 保存成功返回True : False
     """
     sql = """
         BEGIN;
         INSERT INTO userinfo (appid_id, username, createtime, updatetime) VALUES (%s, %s, %s, %s);
         INSERT INTO localauth (userid_id, password) VALUES (%s, %s);
         COMMIT;
     """
     insert_userinfo = "INSERT INTO userinfo (appid_id, username, createtime, updatetime) VALUES (%s, %s, %s, %s)"
     insert_localauth = "INSERT INTO localauth (userid_id, password) VALUES (%s, %s)"
     now = str(int(time.time()))
     try:
         userid_id = self.db.insert(insert_userinfo, appid, username, now, now)
     except Exception as e:
         logger.error(str(e))
         return False
     else:
         try:
             self.db.insert(insert_localauth, userid_id, utils.encodePassword(password))
         except Exception as e:
             logger.error(str(e))
             return False
     return True

	def process(self, raw_data):
		data, name = raw_data
		try:
			# cipher_text must be a multiple of 16, fill the margin with 0
			data = data if len(data)%16==0 else ''.join([data, '0'*(16-len(data)%16)])
			return (AESDecipherer.decipherer.decrypt(data), name)
		except ValueError, e:
			logger.error(u'unable to decrypt %s by the means of AES' % name)