def get_potential_addresses(address):
    """
    Get permutations of as many potential addresses as we can make out of
    the given address.

    Permutations include the address with all punctuation removed, the
    address with directions abbreviated, and the address with street types
    abbreviated (as they are in the City of New Orleans' addresses
    shapefile).
    """
    addresses = set()
    addresses.add(address.lower())

    # Try removing punctuation from the address
    for address in list(addresses):
        addresses.add(re.subn('[^\s\w]', '', address)[0])

    #  Try abbreviating directions
    for address in list(addresses):
        abbreviated = address
        for direction, abbr in directions.items():
            abbreviated = re.subn(r'\b%s\b' % direction, abbr, abbreviated)[0]
        addresses.add(abbreviated)

    # Abbreviate street types
    for address in list(addresses):
        for street_type, abbr in street_types.items():
            abbreviated = re.subn(r'\b%s\b' % street_type, abbr, address)[0]
            addresses.add(abbreviated)

    return tuple(addresses)

	def html2utf8(self,in_html):
		in_html = (re.subn(r'<(script).*?</\1>(?s)', '', in_html)[0])
		in_html = (re.subn(r'<(style).*?</\1>(?s)', '', in_html)[0])
		entitydict = {}

		entities = re.finditer('&([^#][A-Za-z]{1,5}?);', in_html)
		for x in entities:
			key = x.group(0)
			if key not in entitydict:
				entitydict[key] = htmlentitydefs.name2codepoint[x.group(1)]

		entities = re.finditer('&#x([0-9A-Fa-f]{2,2}?);', in_html)
		for x in entities:
			key = x.group(0)
			if key not in entitydict:
				entitydict[key] = "%d" % int(key[3:5], 16)

		entities = re.finditer('&#(\d{1,5}?);', in_html)
		for x in entities:
			key = x.group(0)
			if key not in entitydict:
				entitydict[key] = x.group(1)

		if re.search("charset=utf-8", in_html):
			for key, codepoint in iteritems(entitydict):
				in_html = in_html.replace(key, unichr(int(codepoint)))
			self.inhtml = in_html.encode('utf8')
			return

		for key, codepoint in iteritems(entitydict):
			in_html = in_html.replace(key, unichr(int(codepoint)).encode('latin-1', 'ignore'))
		self.inhtml = in_html.decode('latin-1').encode('utf8')

 def alter_path(self, path, **varargs):
     '''Modify the path passed so that it can be executed on the remote host
     
     path = path to modify
     regexp_substitution - if true, exclude \g<...> from substitution
     '''
     regexp_substitution = varargs.get("regexp_substitution", False)
     for mapping in self.mappings:
         local_directory = mapping.local_directory.value
         remote_directory = mapping.remote_directory.value
         if regexp_substitution:
             local_directory = local_directory.replace("\\", "\\\\")
             remote_directory = remote_directory.replace("\\", "\\\\")
         
         if sys.platform.startswith('win'):
             # Windows is case-insentitve so do case-insensitve mapping
             if path.upper().startswith(local_directory.upper()):
                 path = (remote_directory +
                         path[len(local_directory):])
         else:
             if path.startswith(local_directory):
                 path = (remote_directory +
                         path[len(local_directory):])
     if self.remote_host_is_windows.value:
         path = path.replace('/','\\')
     elif (regexp_substitution):
         path = re.subn('\\\\\\\\','/',path)[0]
         path = re.subn('\\\\(?!g\\<[^>]*\\>)','/',path)[0]
     else:
         path = path.replace('\\','/')
     return path

def retrieveEspnGameIdsForDay(date, root_url):
    """

    """
    url = root_url + date
    url = "http://scores.espn.go.com/nba/scoreboard?date=20150202"
    scores_page = soupypages.soupFromUrl(url)
    if scores_page['pass']:
        scores_page = scores_page['soup']
        game_data = scores_page.body.findAll(id="main-container")[0].find('div', class_='scoreboards').attrs['data-data']
        game_data = re.subn('true', "'true'", game_data)[0]
        game_data = re.subn('false', "'false'", game_data)[0]
        game_data = eval(game_data)
        game_ids = [game['id'] for game in game_data['events']]
        return game_ids
    
##    scores_page = makePage(url, hdr=True)
##    if scores_page['pass']:
##        game_ids = key_phrase.findall(scores_page['page'])
##        return game_ids
    
    else:
        err_msg = scores_page['err']
        if type(err_msg) == str:
            print("Failed to retrieve scores page for %s. Error: %s" %\
                  (date, err_msg))
        else:
            print("Failed to retrieve scores page for %s. Unknown error." %\
                  date)
        return []

def replace(inp):
    result = []
    for f in inp:
        f = f.decode("utf-8")
        
        #f = re.sub(r"([^\\])([_|\^])", r"\1\\\2", f) # do not require backslash in front of ^ or _
        #f = re.sub(r"^([_|\^])", r"\\\1", f)
        
        # escape combining marks with a space after the backslash
        for c in combiningmarks:
            offset = 0
            for s in re.finditer(c[0], f):
                f = f[:s.start()+1+offset] + " " + f[s.start()+1+offset:]
                offset += 1
            
        for r in replacements:
            f = re.sub(r[0], r[1], f)

        # expand groups of subscripts: \_{01234}    
        offset = 0
        #for s in re.finditer(r"\\_\{[^\}]+\}", f):
        for s in re.finditer(ur"_\{[0-9\+-=\(\)<>\-aeoxjhklmnpstiruv\u03B2\u03B3\u03C1\u03C6\u03C7]+\}", f):
            newstring,n = re.subn(ur"([0-9\+-=\(\)<>\-aeoxjhklmnpstiruv\u03B2\u03B3\u03C1\u03C6\u03C7])", r"_\1", s.group(0)[2:-1])
            f = f[:s.start()+offset] + newstring + f[s.end()+offset:]
            offset += n*2 - (n + 3)
            
            
        # expand groups of superscripts: \^{01234}    
        offset = 0
        #for s in re.finditer(r"\\\^\{[^\}]+\}", f):
        for s in re.finditer(ur"\^\{[0-9\+-=\(\)<>ABDEGHIJKLMNOPRTUWabcdefghijklmnoprstuvwxyz\u03B2\u03B3\u03B4\u03C6\u03C7\u222B]+\}", f):
            newstring,n = re.subn(ur"([0-9\+-=\(\)<>ABDEGHIJKLMNOPRTUWabcdefghijklmnoprstuvwxyz\u03B2\u03B3\u03B4\u03C6\u03C7\u222B])", r"^\1", s.group(0)[2:-1])
            f = f[:s.start()+offset] + newstring + f[s.end()+offset:]
            offset += n*2 - (n + 3)

    def set_new_patches(self, fns):
        self.wipe_patches()
        if not fns:
            return
        apply_method = self.patches_apply_method()
        ps = ""
        pa = ""
        for i, pfn in enumerate(fns, start=1):
            ps += "Patch%04d: %s\n" % (i, pfn)
            if apply_method == "rpm":
                pa += "%%patch%04d -p1\n" % i
        # PatchXXX: lines after Source0 / #patches_base=
        self._txt, n = re.subn(self.RE_AFTER_PATCHES_BASE, r"\g<1>%s\n" % ps, self.txt, count=1)

        if n != 1:
            for m in re.finditer(self.RE_AFTER_SOURCES, self.txt):
                pass
            if not m:
                raise exception.SpecFileParseError(spec_fn=self.fn, error="Failed to append PatchXXXX: lines")
            i = m.end()
            startnl, endnl = "", ""
            if self._txt[i - 2] != "\n":
                startnl += "\n"
            if self._txt[i] != "\n":
                endnl += "\n"
            self._txt = self._txt[:i] + startnl + ps + endnl + self._txt[i:]
        # %patchXXX -p1 lines after "%setup" if needed
        if apply_method == "rpm":
            self._txt, n = re.subn(r"((?:^|\n)%setup[^\n]*\n)\s*", r"\g<1>\n%s\n" % pa, self.txt)
            if n == 0:
                raise exception.SpecFileParseError(
                    spec_fn=self.fn, error="Failed to append %patchXXXX lines after %setup"
                )

 def alter_path(self, path, **varargs):
     '''Modify the path passed so that it can be executed on the remote host
     
     path = path to modify
     regexp_substitution - if true, exclude \g<...> from substitution
     '''
     for mapping in self.mappings:
         if sys.platform.startswith('win'):
             # Windows is case-insentitve so do case-insensitve mapping
             if path.upper().startswith(mapping.local_directory.value.upper()):
                 path = (mapping.remote_directory.value +
                         path[len(mapping.local_directory.value):])
         else:
             if path.startswith(mapping.local_directory.value):
                 path = (mapping.remote_directory.value +
                         path[len(mapping.local_directory.value):])
     if self.remote_host_is_windows.value:
         path = path.replace('/','\\')
     elif (varargs.has_key("regexp_substitution") and
              varargs["regexp_substitution"]):
         path = re.subn('\\\\\\\\','/',path)[0]
         path = re.subn('\\\\(?!g\\<[^>]*\\>)','/',path)[0]
     else:
         path = path.replace('\\','/')
     return path

def chop_end(seqs, model_end, min_modelpositions):	
	chop_seq_dict = dict()
	
	refseq = seqs[refseq_ID]
	model_pos = 0
	model_map = dict()
	for i in range(len(refseq.seq)):
		if refseq.seq[i] != ".":
			model_pos += 1
			model_map[model_pos] = i

	seq_end = model_map[model_end]
	
	for seqid in seqs.keys():
		seq = seqs[seqid]
		## slice from the end
		seqstr = str(seq.seq)[0: seq_end]
		
		if seqid != refseq_ID:
		# check if the upper case chars( model position) pass the minimum requirement
			if len(re.subn(upper_regex, "", seqstr)[0]) >= min_modelpositions:
				chop_seq_dict[seqid] =  seqstr
				
			else:	
				print "%s length %s  %s" % (seq.id, len(re.subn(upper_regex, "", seqstr)[0]), re.subn(upper_regex, "", seqstr)[0])
				pass
		else:
			chop_seq_dict[seqid] = seqstr
	return chop_seq_dict

def fixrow(data, key, methoddict, verify_rules, equationdict, originaldata):
    """function that i wrote to fix rows that were causing problems after results were created.
    after i did this it was clearly a problem because i had no way of knowing what caused the error.
    because of this i reworked ther interpolation script to allow for fixes in there rather than after the fact"""
    originaldata = {k: v for k, v in chain(originaldata['blockdata'].items(), originaldata['tractdata'].items())}
    def subber(matchobject):
        return '{:d}'.format(originaldata[matchobject.group(0)])

    for table, method in methoddict.items():
        testdict = verify_rules[str(method)]
        tabledata = {k.split('_')[-1][2:]: v for k, v in data.items() if table in k}
        for total, components in sorted(testdict.items(), key = lambda x: int(x[0])):
            result, valtotal, valcomps = testrule(tabledata, total, components)
            if result == 'mult':
                totalname = '{0}_av{1}'.format(table, total)
                if table == 'ds185_dt31':
                    print key, totalname, 'total', re.subn(FINDVARIABLES, subber, equationdict[totalname]['equation'])[0], valtotal
                valcomps = {k: v for k, v in valcomps.iteritems() if v == valtotal}
                lengthcomps = len(valcomps)
                for val in valcomps:
                    tabledata[val] = float(valtotal) / lengthcomps
                    fullname = '{0}_av{1}'.format(table, val)
                    if table == 'ds185_dt31':
                        print key, fullname, 'value', re.subn(FINDVARIABLES, subber, equationdict[fullname]['equation'])[0], tabledata[val]
                    data[fullname] = float(valtotal) / lengthcomps
    return data

def error_027_mnemonic_codes(text):
    """Fix some cases and return (new_text, replacements_count) tuple."""
    (text, ignored) = ignore(text, r"https?://\S+")
    (text, count1) = re.subn(r"–", "–", text)
    (text, count2) = re.subn(r" ", " ", text)
    text = deignore(text, ignored)
    return (text, count1 + count2)

def error_032_link_two_pipes(text):
    """Fix some cases and return (new_text, replacements_count) tuple."""
    (text, ignored) = ignore(text, r"\[\[\s*:?\s*{}.*?\]\]".format(IMAGE))
    (text, count1) = re.subn(r"\[\[([^|\[\]\n]+)\|\|([^|\[\]\n]+)\]\]", "[[\\1|\\2]]", text)
    (text, count2) = re.subn(r"\[\[([^|\[\]\n]+)\|([^|\[\]\n]+)\|\]\]", "[[\\1|\\2]]", text)
    text = deignore(text, ignored)
    return (text, count1 + count2)

def build_template_tags(blocks):
    content = StringIO()
    block_template = '''
    {%% block %(name)s %%}
        %(comment)s

        Default:
            %(default_value)s

        %(blocks)s
    {%% endblock %(name)s %%}

    '''

    for block in blocks:
        subblocks = re.subn("\n", "\n\t", build_template_tags(block.blocks))

        if subblocks:
            subblocks = subblocks[0]

        content.write(block_template % {
            'name': block.name,
            'comment': block.comment,
            'default_value': re.subn("\n", "\n\t", str(block.default_value))[0],
            'blocks': subblocks,
        })

    return content.getvalue()

 def shell_remote_execution(self, tgt, arg, arg1):
     ''' Shell command execution with parameters '''
     if arg in ['disk.usage', 'network.interfaces', 'grains.items', 'test.ping', 'state.running', 'status.meminfo',
                'status.uptime', 'service.get_all']:
         params = {'client': 'local', 'tgt': tgt, 'fun': arg, 'expr_form': 'list'}
     elif arg == 'cmd.run':
         params = {'client': 'local', 'tgt': tgt, 'fun': arg, 'arg1': arg1, 'arg2': 'runas=weihu','expr_form': 'list'}
         obj = urllib.urlencode(params)
         obj, number = re.subn("arg\d", 'arg', obj)
         content = self.postRequest(obj)
         ret = content['return'][0]
         return ret
     elif arg in ['cp.get_file', 'cp.get_dir', 'cp.get_url']:
         a = arg1.split()
         a1 = a[0]
         a2 = a[1]
         params = {'client': 'local', 'tgt': tgt, 'fun': arg, 'arg1': a1, 'arg2': a2, 'expr_form': 'list'}
         obj = urllib.urlencode(params)
         obj, number = re.subn("arg\d", 'arg', obj)
         content = self.postRequest(obj)
         ret = content['return'][0]
         return ret
     else:
         params = {'client': 'local', 'tgt': tgt, 'fun': arg, 'arg': arg1, 'expr_form': 'list'}
     obj = urllib.urlencode(params)
     content = self.postRequest(obj)
     ret = content['return'][0]
     return ret

def getDefaultEditionContent():
    '''获取标准版文件内容

    Returns:
        标准版文件内容
    '''
    partContent = ''
    for fileName in devPartFileNameList:
        partContent += open(devDirName + os.sep + fileName + '.js', 'r', encoding = encoding).read() + '\n\n'
    content = open(devDirName + os.sep + devKFOLFileName + userScriptExt, 'r', encoding = encoding).read()
    match = re.search('/\*\s*\{PartFileContent\}\s*\*/', content, flags=re.S | re.I)
    if match:
        content = content.replace(match.group(0), partContent)
    else:
        raise Exception('未找到{PartFileContent}占位符')
    content, num  = re.subn(r'(// @description.+?)(// @include)',
                     r'\g<1>'
                     '// @updateURL   https://greasyfork.org/scripts/8615-kf-online%E5%8A%A9%E6%89%8B/code/KF%20Online%E5%8A%A9%E6%89%8B.meta.js\n'
                     '// @downloadURL https://greasyfork.org/scripts/8615-kf-online%E5%8A%A9%E6%89%8B/code/KF%20Online%E5%8A%A9%E6%89%8B.user.js\n'
                     r'\g<2>',
                     content,
                     count=1,
                     flags=re.S | re.I)
    if num == 0: raise NoFoundReplaceStringError('标准版', 1)
    content, num  = re.subn(r'// @require.+?(// @version)', '\g<1>', content, count=1, flags=re.S | re.I)
    if num == 0: raise NoFoundReplaceStringError('标准版', 2)
    return content

	def subst(self, replace, replace_with):
		"re search and replace on the current node"
		nodes = self.current_nodes[:]
		check = len(nodes) == 1
		a = self.current_attrib
		if a:
			new, num = re.subn(replace, replace_with, a.value)
			if not num:
				raise Beep
			a = self.model.set_attrib(nodes[0], a.name, new)
			self.move_to(nodes[0], a)
		else:
			self.move_to([])
			final = []
			for n in nodes:
				if n.nodeType == Node.TEXT_NODE:
					old = n.data.replace('\n', ' ')
					new, num = re.subn(replace, replace_with, old)
					if check and not num:
						self.move_to(n)
						raise Beep
					self.model.set_data(n, new)
					final.append(n)
				elif n.nodeType == Node.ELEMENT_NODE:
					old = str(n.nodeName)
					new, num = re.subn(replace, replace_with, old)
					if check and not num:
						self.move_to(n)
						raise Beep
					new_ns, x = self.model.split_qname(n, new)
					final.append(self.model.set_name(n, new_ns, new))
				else:
					self.move_to(n)
					raise Beep
			self.move_to(final)

 def _clean_xml(self, line):
     line2 = re.subn('"', '"', line)
     line3 = re.subn('&', '&', line2[0])
     line4 = re.subn("'", ''', line3[0])
     line5 = re.subn('<', '&lt;', line4[0])
     line6 = re.subn('>', '&gt;', line5[0])
     return line6[0]            

	def geturllist(self,url):	#将文档中包含的url通通加入到url列表中等待访问
		tmpurl=re.findall('<a\s+[^>]*[\s]*href[\s]*=[\s]*[\'\"]([^>^\'^\"]*)[\'\"][\s]*[^>]*>',str(self.gethtmldom(url)))#匹配文档中的url
		domain=re.findall(r'^(?:https?://)?[\w\.]*',self.url);#获取该网页的域名
		for x in tmpurl:
			x=re.subn('^\s*','',x)[0]	#将url中的开头空白字符去掉
			x=re.subn(r'\\','/',x)[0]	#所有\换成/
			if re.match('http',x,re.I):	#如果该连接是以http开头的，丢进待访问列表
				sitedomain=re.findall(r'^(?:https?://)?[\w\.]*',x)
				if self.allnet!=0 or sitedomain[0]==domain[0]:
					self.urllist.append(x)
				else:
					pass
			elif re.match('^//',x,re.I):#如果该连接是以//开头，加上http丢入列表
				x='http:'+x
				sitedomain=re.findall(r'^(?:https?://)?[\w\.]*',x)
				if self.allnet!=0 or sitedomain[0]==domain[0]:
					self.urllist.append(x)
				else:
					pass
			elif len(re.findall('^#',x))!=0:#如果是锚点就丢弃
				pass
			else:					#不然加上判断是不是绝对路径
				if len(re.findall('^/',x))!=0:	#如果是绝对路径就加上域名丢进等待列表
					x=domain[0]+x
					self.urllist.append(x)
				else:									
					if url==domain[0]:
						x=url+'/'+x
						self.urllist.append(x);
					else:
						x=re.subn(r'(/[^/]*)$','/'+x,url)[0]
						self.urllist.append(x)
		return self.urllist

def cache_customevent_trend(ids=[]):
    """
    Runs the rawdata gatherer for the specific custom events.
    Intended to be run mainly but the BibSched daemon interface.

    For a specific id, all possible timespans' rawdata is gathered.

    @param ids: The custom event ids that are subject to caching.
    @type ids: []
    """
    args = {}
    timespans = _get_timespans()

    for id in ids:
        args['id'] = id
        args['cols'] = []

        for i in range(len(timespans)):
            # Get timespans parameters
            args['timespan'] = timespans[i][0]
            args.update({ 't_start': timespans[i][2], 't_end': timespans[i][3], 'granularity': timespans[i][4],
                          't_format': timespans[i][5], 'xtic_format': timespans[i][6] })

            # Create unique filename for this combination of parameters
            filename = "webstat_customevent_%(id)s_%(timespan)s" \
                        % { 'id': re.subn("[^\w]", "_", id)[0], 'timespan': re.subn("[^\w]", "_", args['timespan'])[0] }

            # Create closure of gatherer function in case cache needs to be refreshed
            gatherer = lambda: get_customevent_trend(args)

            # Get data file from cache, ALWAYS REFRESH DATA!
            _get_file_using_cache(filename, gatherer, True).read()

    return True

    def __init__(cls, name, bases, attrs):
        cls.registry = copy.deepcopy(getattr(cls, 'registry',
                                             {'mappers': {}, 'types': {}}))

        def add_mapper(f, ff, fft):
            log.debug("{} adding mapper {} for {} ({})",
                      cls.__name__, f, ff, fft)
            if f in cls.registry:
                log.warning("Overwriting {} for class {} with {} "
                            "(previous value: {})", f, name, ff,
                            cls.registry['mappers'][f])
            cls.registry['mappers'][f] = ff
            cls.registry['types'][ff] = fft

        # get form_field mappers from dunder string
        form_field_mappers = getattr(cls, '__form_fields__', {})
        for field, (form_field, form_field_type) in form_field_mappers.items():
            add_mapper(field, form_field, form_field_type)

        for key, val in attrs.items():
            try:
                form_field, form_field_type = getattr(val, 'form_field')
            except AttributeError:
                pass  # most attributes are not a form_field mapper
            else:
                field, n = re.subn(re_frender, '', key)
                assert n == 1  # only then is it a field renderer
                add_mapper(field, form_field, form_field_type)

            # get fields that need finalization
            if getattr(val, 'needs_finalization', False):
                field, n = re.subn(re_frender, '', key)
                assert n == 1  # only then is it a field renderer
                cls._to_finalize = getattr(cls, '_to_finalize', []) + [field]

 def set_new_patches(self, fns):
     self.wipe_patches()
     if not fns:
         return
     apply_method = self.patches_apply_method()
     ps = ''
     pa = ''
     for i, pfn in enumerate(fns, start=1):
         ps += "Patch%04d: %s\n" % (i, pfn)
         if apply_method == 'rpm':
             pa += "%%patch%04d -p1\n" % i
     ## PatchXXX: lines after Source0 / #patches_base=
     self._txt, n = re.subn(
         self.RE_AFTER_PATCHES_BASE,
         r'\g<1>%s\n' % ps, self.txt, count=1)
     if n != 1:
         self._txt, n = re.subn(
             self.RE_AFTER_SOURCES,
             r'\g<1>%s\n' % ps, self.txt, count=1)
     if n != 1:
         raise Exception("SpecFileParseError: Failed to append PatchXXXX: lines")
     ## %patchXXX -p1 lines after "%setup" if needed
     if apply_method == 'rpm':
         self._txt, n = re.subn(
             r'((?:^|\n)%setup[^\n]*\n)\s*',
             r'\g<1>\n%s\n' % pa, self.txt)
         if n == 0:
             raise Exception("SpecFileParseError: Failed to append %patchXXXX lines after %setup")