def makeconfig(infp, outfp, modules, with_ifdef=0):
    m1 = re.compile('-- ADDMODULE MARKER 1 --')
    m2 = re.compile('-- ADDMODULE MARKER 2 --')
    while 1:
        line = infp.readline()
        if not line: break
        outfp.write(line)
        if m1 and m1.search(line):
            m1 = None
            for mod in modules:
                if mod in never:
                    continue
                if with_ifdef:
                    outfp.write("#ifndef init%s\n"%mod)
                outfp.write('extern void init%s(void);\n' % mod)
                if with_ifdef:
                    outfp.write("#endif\n")
        elif m2 and m2.search(line):
            m2 = None
            for mod in modules:
                if mod in never:
                    continue
                outfp.write('\t{"%s", init%s},\n' %
                            (mod, mod))
    if m1:
        sys.stderr.write('MARKER 1 never found\n')
    elif m2:
        sys.stderr.write('MARKER 2 never found\n')

def catchweibo():
	c = 3362
	# c是爬虫起始页
	for i in range(6906):
		pn = (i+c)
		url = 'http://weibo.cn/1767797335/profile?filter=0&page='+str(pn)	
		#上面地址是你要爬的人的微薄url，用weibo.cn是因为那个地方进去访问限制少	
		print url
		req = urllib2.Request(url)
		req.add_header("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36")
		req.add_header("Cookie", "_T_WM=edf4469bb5245a50aa32006460daa5ae; _T_WL=1; _WEIBO_UID=5638019231; SUB=_2A254gp9aDeTxGeNI6FoR8SfOyD2IHXVbjCESrDV6PUJbrdAKLXOnkW1HSRVVWhAfa6SQUOfsMJvV5z1nWg..; gsid_CTandWM=4u3Fdd4a1W8HT0Rlp91lUnEHN3J")
		#上面这行修改自己的cookie，每个cookie大概能爬1000页左右，如果只有一个帐号就隔一个小时之后再爬
		try:
			res = urllib2.urlopen(req)
			print 'ok1'
		except:
			print 'error open'
			continue
		html = res.read()
		print html

		reg1 = re.compile(r'(<div class="c" id="M_[\d\D]*?)<div class="s"></div>')
		reg2 = re.compile(r'<span class="ct">(.*?)&nbsp;')
		yuanchuang = reg1.findall(html)
		# atime = reg2.findall(html)
		if not yuanchuang:
			print 'reg none'
			c = c-1
			continue
		for j in range(0, len(yuanchuang)):
			print len(yuanchuang)
			print yuanchuang[j]
			print '\n'
			fout.write(yuanchuang[j]+'\n'+'\n<br><br>')

    def acquire(self, testname, buf, status, command):
        # record failures based on exit status
        if status:
            self.failures.append("Exit %s: %s" % (status, command))

        # scan test log for magical tokens
        # see also: http://hg.mozilla.org/automation/logparser/
        passre = re.compile("^TEST-(PASS|EXPECTED-FAIL).*")
        failre = re.compile("^TEST-UNEXPECTED-.*")
        tback = re.compile("^Traceback.*")
        excpt = re.compile("^Exception:.*")

        self.text[testname] = []

        for line in buf:
            print line
            if passre.match(line):
                self.passes.append(line)
            elif failre.match(line):
                self.failures.append(line)
            elif tback.match(line):
                self.failures.append(line)
            elif excpt.match(line):
                self.failures.append(line)
            else:
                self.info.append(line)
            self.text[testname].append(line)

def fromUrl( streamUrl ):
    Log( "Channel.py fromUrl ..." )
    """
    Two types of valid stream URLs:

    hdhomerun://<device-id>-<tuner>/ch<physical-channel>-<program-number>

    hdhomerun://<device-id>-<tuner>/tuner<tuner>?channel=<modulation>:<frequency>&program=<program-number>

    """

    channel = Channel()
    
    urlRe = re.compile( r'^\s*hdhomerun\:\/\/([\w\-]+)\-(\d+)\/tuner(\d+)\?channel\=([^\:]+)\:(.+)\&program\=(.+)$' )
    reMatch = urlRe.match( streamUrl )
    if reMatch:
        deviceId = reMatch.group(1)
        tunerId1 = reMatch.group(2)
        tunerId2 = reMatch.group(3)
        channel.Modulation = reMatch.group(4)
        channel.Frequency = reMatch.group(5)
        channel.ProgramNumber = reMatch.group(6)
        return channel

    urlRe = re.compile( r'^\s*hdhomerun\:\/\/([\w\-]+)\-(\d+)\/ch([^\-]+)-(\w+)$' )
    reMatch = urlRe.match( streamUrl )
    if reMatch:
        deviceId = reMatch.group(1)
        tunerId1 = reMatch.group(2)
        channel.PhysicalChannel = reMatch.group(3)
        channel.ProgramNumber = reMatch.group(4)
        return channel

    return None

def __cut_internal(sentence,HMM=True):
    if not ( type(sentence) is unicode):
        try:
            sentence = sentence.decode('utf-8')
        except:
            sentence = sentence.decode('gbk','ignore')
    re_han, re_skip = re.compile(ur"([\u4E00-\u9FA5a-zA-Z0-9+#&\._]+)"), re.compile(ur"(\r\n|\s)")
    re_eng,re_num = re.compile(ur"[a-zA-Z0-9]+"), re.compile(ur"[\.0-9]+")
    blocks = re_han.split(sentence)
    if HMM:
        __cut_blk = __cut_DAG
    else:
        __cut_blk = __cut_DAG_NO_HMM

    for blk in blocks:
        if re_han.match(blk):
            for word in __cut_blk(blk):
                yield word
        else:
            tmp = re_skip.split(blk)
            for x in tmp:
                if re_skip.match(x):
                    yield pair(x,'x')
                else:
                    for xx in x:
                        if re_num.match(xx):
                            yield pair(xx,'m')
                        elif re_eng.match(x):
                            yield pair(xx,'eng')
                        else:
                            yield pair(xx,'x')

def interact(line, stdin, process):
  global tot_sec
  global VIDEO_PAUSED
  global omx_stdin
  global omx_process

  omx_stdin = stdin
  omx_process = process

  # video regexp
  video_curr_rexp = re.compile(r'V :\s*([\d.]+).*')
  video_total_rexp = re.compile(r'Length : *([\d.]+)*')

  # get current video time
  curr = video_curr_rexp.search(line)

  if curr and tot_sec:
    pts = curr.group(1)
    sec = int(pts.split(".")[0]) / 1000000
    print(sec, tot_sec)
    # stop video to last seconds
    if tot_sec == sec and VIDEO_PAUSED == False:
      VIDEO_PAUSED = True
      stdin.put('p')
      print("---- PAUSE ----")

  else:
    len = video_total_rexp.search(line)

    if len:
      tot_pts = len.group(1)
      tot_sec = (int(tot_pts) / 1000) - 11

def clean(sentence):
    """
    Takes the tweet as an input, cleans it using the regular expression
    defined and explained below and returns the cleaned string.

    All the "stop words" are removed by using the below list of regexs. Of the
    following the regex r"http[s]*://\S+", selects all the links in the sentence.
    r" q.\d+", selects the strings like q.1653 from the sentence.
    r"[#@]\w+", selects the @ mentions and hashtags in the sentence.
    r"[^A-Za-z0-9]", selects all the special characters in the sentence.
    r"\w+[-']\w+" selects all the words with "-" or "'" in between them.
    """

    common = [r"\bi\b", r"\bi[nfs]\b", r"\bo[nfr]\b", r"\ba\b", r"\ba[nts]\b",
              r"^i", r"\bother\b", r"\bhe\b", r"\bhave\b", r"\bus\b",
              r"\b[gdtsn]o\b", r"\bnot\b", r"\b[wb]e\b", r"\byour[s]*\b",
              r"\bwhich\b", r"\bthat\b", r"\bha[sd]\b", r"\band\b", r"\bby\b",
              r"\bthe[y]*\b", r"\b[t]*his\b", r"\bit[s]*\b", r"\bfor\b", r"\byou\b",
              r"\bwill\b", r"\bg[eo]t\b", r"\bbut\b", r"\bour\b", r"\bwas\b",
              r"\bcan\b", r"\balso\b", r"\byet\b", r"\bafter\b", r"\bwith\b",
              r"\bthem\b", r"\bdid\b", r"\bare\b", r"\bfrom\b", r"http[s]*://\S+",
              r" q.\d+", r"[#@]\w+", r"[^A-Za-z0-9]", r"\w+[-']\w+"]

    pattern = r"(" + r"|".join(common) + r")"
    p = re.compile(pattern)

    sentence = p.sub(" ", sentence)

    p = re.compile("  +")
    sentence = p.sub(" ", sentence).strip()

    return sentence

    def get_ticket_tumbers(cls, build):
        """Extract ticket ids from the changeset of a Jenkins build"""
        items = build.get_changeset_items()
        ticket_numbers = []
        regex = re.compile(cls.TICKET_REGEX)

        for entry in items:
            message = entry["msg"]
            print("-- found message: ", message)

            noissue = re.compile(r"#noissue")
            if not noissue.search(message):
                match = regex.search(message)
                if match is None:
                    print(
                        "found malformed message in build: ",
                        build.get_number(), "\n",
                        "with message: ",
                        message
                    )
                else:
                    ticket = match.group(1)
                    if ticket not in ticket_numbers:
                        ticket_numbers.append(ticket)

        return ticket_numbers

    def add(self, irc, msg, args, channel, regexp, action):
        """[<channel>] <regexp> <action>

        Associates <regexp> with <action>.  <channel> is only
        necessary if the message isn't sent on the channel
        itself.  Action is echoed upon regexp match, with variables $1, $2, 
        etc. being interpolated from the regexp match groups."""
        if not self._checkManageCapabilities(irc, msg, channel):
            capabilities = self.registryValue('requireManageCapability')
            irc.errorNoCapability(capabilities, Raise=True)
        db = self.getDb(channel)
        cursor = db.cursor()
        cursor.execute("SELECT id, usage_count, locked FROM triggers WHERE regexp=?", (regexp,))
        results = cursor.fetchall()
        if len(results) != 0:
            (id, usage_count, locked) = map(int, results[0])
        else:
            locked = 0
            usage_count = 0
        if not locked:
            try:
                re.compile(regexp)
            except Exception, e:
                irc.error('Invalid python regexp: %s' % (e,))
                return
            if ircdb.users.hasUser(msg.prefix):
                name = ircdb.users.getUser(msg.prefix).name
            else:
                name = msg.nick
            cursor.execute("""INSERT INTO triggers VALUES
                              (NULL, ?, ?, ?, ?, ?, ?)""",
                            (regexp, name, int(time.time()), usage_count, action, locked,))
            db.commit()
            irc.replySuccess()

def parse_replace_hook(s):
    """
        Returns a (pattern, regex, replacement) tuple.

        The general form for a replacement hook is as follows:

            /patt/regex/replacement

        The first character specifies the separator. Example:

            :~q:foo:bar

        If only two clauses are specified, the pattern is set to match
        universally (i.e. ".*"). Example:

            /foo/bar/

        Clauses are parsed from left to right. Extra separators are taken to be
        part of the final clause. For instance, the replacement clause below is
        "foo/bar/":

            /one/two/foo/bar/

        Checks that pattern and regex are both well-formed. Raises
        ParseException on error.
    """
    patt, regex, replacement = _parse_hook(s)
    try:
        re.compile(regex)
    except re.error, e:
        raise ParseException("Malformed replacement regex: %s"%str(e.message))

    def __load_book_menu (self, lines) :
	r1 = re.compile(u'^\s*目\s*录\s*$')
	r2 = re.compile(u'^\s*([^·…]+)\s*[·．…]{2,}\s*([l\d]+)\s*$')
	menus = {}
	start = False
	not_match = 0
	for line in lines :
	    words = line.decode(self.default_coding)
	    words.strip('\n')
	    if re.match(r1, words) :
		start = True
		continue
	    elif start :
		m = re.match(r2, words)
		if m :
		    title = m.group(1)
		    page  = m.group(2)
		    page  = page.replace('l', '1')
		    page  = int(page.encode(self.default_coding))
		    menus[page] = self.__get_simple_string(title)
		    not_match = 0
		else :
		    not_match += 1
		    if not_match > 10 :
			break
	
	return menus

def _translate(version, rules, standard):
    """Translate Python version into Debian one.

    >>> _translate('1.C2betac', ['s/c//gi'], None)
    '1.2beta'
    >>> _translate('5-fooa1.2beta3-fooD',
    ...     ['s/^/1:/', 's/-foo//g', 's:([A-Z]):+$1:'], 'PEP386')
    '1:5~a1.2~beta3+D'
    >>> _translate('x.y.x.z', ['tr/xy/ab/', 'y,z,Z,'], None)
    'a.b.a.Z'
    """
    for rule in rules:
        # uscan supports s, tr and y operations
        if rule.startswith(('tr', 'y')):
            # Note: no support for escaped separator in the pattern
            pos = 1 if rule.startswith('y') else 2
            tmp = rule[pos + 1:].split(rule[pos])
            version = version.translate(str.maketrans(tmp[0], tmp[1]))
        elif rule.startswith('s'):
            # uscan supports: g, u and x flags
            tmp = rule[2:].split(rule[1])
            pattern = re.compile(tmp[0])
            count = 1
            if tmp[2:]:
                flags = tmp[2]
                if 'g' in flags:
                    count = 0
                if 'i' in flags:
                    pattern = re.compile(tmp[0], re.I)
            version = pattern.sub(_pl2py(tmp[1]), version, count)
        else:
            log.warn('unknown rule ignored: %s', rule)
    if standard == 'PEP386':
        version = PRE_VER_RE.sub(r'~\g<1>', version)
    return version

def Episodes(url, name):
    # try:
    link = GetContentMob(url)
    newlink = "".join(link.splitlines()).replace("\t", "")
    match = re.compile(
        '<td style="text-align:justify" class="movieepisode"><strong>' + name + "</strong>(.+?)</td>"
    ).findall(newlink)
    mirrors = re.compile("<a [^>]*href=[\"']?([^>^\"^']+)[\"']?[^>]*>(.+?)</a>").findall(match[0])

    if len(mirrors) >= 1:
        i = 1
        for mcontent in mirrors:
            vLinktemp, vLinkName = mcontent
            vLink = ""
            j = 1
            k = 1
            for mlink in mirrors:
                vLink1, vLinkName1 = mlink
                if j >= i:
                    if i == len(mirrors) or j == len(mirrors) or k == 12:
                        vLink += viddomain + vLink1 + "+++" + vLinkName1
                    else:
                        vLink += viddomain + vLink1 + "+++" + vLinkName1 + "***"
                    if k % 12 == 0:
                        break
                    k += 1
                j += 1
            i += 1
            # addLink("tập:  " + RemoveHTML(vLinkName).strip(),mobileurl+"/"+vLink,3,'',"")
            addLink("Tập:  " + RemoveHTML(vLinkName).strip(), vLink, 3, "", "")
            print vLink

def init_db(db_url):
    regex = re.compile('^mongodb:\\/\\/(.*?):(.*?)@(.*?):([0-9]+)\\/(.*)$')
    match = regex.match(db_url)

    if not match:
        regex = re.compile('^mongodb:\\/\\/(.*?)\\/(.*)$')
        match = regex.match(db_url)

        username = None
        password = None
        host = match.group(1)
        port = None
        db_name = match.group(2)
    else:
        username = match.group(1)
        password = match.group(2)
        host = match.group(3)
        port = int(match.group(4))
        db_name = match.group(5)

    conn = mongoengine.connect(db_name,
            host=host,
            port=port,
            username=username,
            password=password)

    return conn[db_name]

def LISTTV4(murl):
        main.addDir('Search Rlsmix','rlsmix',136,"%s/art/search.png"%selfAddon.getAddonInfo("path"))
        main.addLink('[COLOR red]First turbobit Link could be HD[/COLOR]','',"%s/art/tvb.png"%selfAddon.getAddonInfo("path"))
        urllist=['http://www.rlsmix.net/category/tv-shows/','http://www.rlsmix.net/category/tv-shows/page/2/','http://www.rlsmix.net/category/tv-shows/page/3/','http://www.rlsmix.net/category/tv-shows/page/4/','http://www.rlsmix.net/category/tv-shows/page/5/','http://www.rlsmix.net/category/tv-shows/page/6/','http://www.rlsmix.net/category/tv-shows/page/7/','http://www.rlsmix.net/category/tv-shows/page/8/','http://www.rlsmix.net/category/tv-shows/page/9/','http://www.rlsmix.net/category/tv-shows/page/10/']
        dialogWait = xbmcgui.DialogProgress()
        ret = dialogWait.create('Please wait until Show list is cached.')
        totalLinks = 10
        loadedLinks = 0
        remaining_display = 'Pages loaded :: [B]'+str(loadedLinks)+' / '+str(totalLinks)+'[/B].'
        dialogWait.update(0,'[B]Will load instantly from now on[/B]',remaining_display)
        for murl in urllist:
                link=main.OPENURL(murl)
                link=link.replace('\r','').replace('\n','').replace('\t','').replace(' ','')
                match=re.compile('<h1 class="titles"><a href="(.+?)" title="Permanent Link to (.+?)">.+?src="http://uppix.net/(.+?)"').findall(link)
                for url,name,thumb in match:
                        match2=re.compile('TV Round Up').findall(name)
                        name=name.replace('\xc2\xa0','').replace('" ','').replace(' "','').replace('"','').replace("&#039;","'").replace("&amp;","and").replace("&#8217;","'").replace("amp;","and").replace("#8211;","-")
                        if len(match2)==0:
                            main.addDir(name,url,62,'http://uppix.net/'+thumb)
                
                loadedLinks = loadedLinks + 1
                percent = (loadedLinks * 100)/totalLinks
                remaining_display = 'Pages loaded :: [B]'+str(loadedLinks)+' / '+str(totalLinks)+'[/B].'
                dialogWait.update(percent,'[B]Will load instantly from now on[/B]',remaining_display)
                if (dialogWait.iscanceled()):
                        return False   
        dialogWait.close()
        del dialogWait
        main.GA("TV","Rlsmix")

def normalizeTitle(title):
  # remove leading whitespace and underscores
  title = title.strip(' _')
  # replace sequences of whitespace and underscore chars with a single space
  title = re.compile(r'[\s_]+').sub(' ', title)

  m = re.compile(r'([^:]*):(\s*)(\S(?:.*))').match(title)
  if m:
      prefix = m.group(1)
      if m.group(2):
          optionalWhitespace = ' '
      else:
          optionalWhitespace = ''
      rest = m.group(3)

      ns = prefix.capitalize()
      if ns in acceptedNamespaces:
          # If the prefix designates a known namespace, then it might be
          # followed by optional whitespace that should be removed to get
          # the canonical page name
          # (e.g., "Category:  Births" should become "Category:Births").
          title = ns + ":" + rest.capitalize()
      else:
          # No namespace, just capitalize first letter.
	  # If the part before the colon is not a known namespace, then we must
          # not remove the space after the colon (if any), e.g.,
          # "3001: The_Final_Odyssey" != "3001:The_Final_Odyssey".
          # However, to get the canonical page name we must contract multiple
          # spaces into one, because
          # "3001:   The_Final_Odyssey" != "3001: The_Final_Odyssey".
          title = prefix.capitalize() + ":" + optionalWhitespace + rest
  else:
      # no namespace, just capitalize first letter
      title = title.capitalize();
  return title

    def find_bug_ids(self, ctx):
        '''find valid bug ids that are referred to in changeset
        comments and that do not already have references to this
        changeset.'''

        if bugzilla._bug_re is None:
            bugzilla._bug_re = re.compile(
                self.ui.config('bugzilla', 'regexp', bugzilla._default_bug_re),
                re.IGNORECASE)
            bugzilla._split_re = re.compile(r'\D+')
        start = 0
        ids = set()
        while True:
            m = bugzilla._bug_re.search(ctx.description(), start)
            if not m:
                break
            start = m.end()
            for id in bugzilla._split_re.split(m.group(1)):
                if not id: continue
                ids.add(int(id))
        if ids:
            ids = self.filter_real_bug_ids(ids)
        if ids:
            ids = self.filter_unknown_bug_ids(ctx.node(), ids)
        return ids

 def __init__(self, **kwargs):
     for k, v in kwargs.items():
         if hasattr(self, k):
             setattr(self, k, v)
     self.keys = AttrDictSimple(
         name=("Name", SENTENCE_SYMBOLS_RE, "Name " +
               SENTENCE_SYMBOLS_WARNING),
         ident=("Identifier", re.compile(r"^[a-zA-Z][a-zA-Z\d_]{2,}$"),
                "Identifier must be 3 or more alphanumeric characters"
                " (underscore allowed)."),
         desc=("Description", SENTENCE_SYMBOLS_RE, "Description " +
               SENTENCE_SYMBOLS_WARNING),
         prefix=("Prefix", re.compile(r"^[a-zA-Z][a-zA-Z\d_]{2,4}$"),
                 "Prefix must be 3 to 5 alphanumeric characters"
                 " (underscores allowed)."),
         domain=("Domain",
                 re.compile(r"^([a-zA-Z0-9]+(\-[a-zA-Z0-9]+)*\.)*"
                            "[a-zA-Z0-9]+(\-[a-zA-Z0-9]+)*"
                            "\.[a-zA-Z]{2,4}$"),
                 "Domain must be a valid domain name."),
         mimetype=("MIME type", re.compile(r"^[a-zA-Z0-9]+"
                                           "\/[a-zA-Z0-9\-]+$"),
                   "Please use alphanumeric characters and dashes in the"
                   " format: application/x-firebreath"),
         disable_gui=("has no UI", re.compile(r"^true$|false$"),
                      "Please enter valid input: true or false"),
     )

def parse_stat_str(s):
    print 'Parsing %s: ' % s
    d = {'mode': 'NOTPARSED', 'zxid': 'NOTPARSED'}
    d['mode'] = re.compile('.*Mode:\s(.*)').search(s).group(1)
    d['zxid'] = re.compile('.*Zxid:\s(.*)').search(s).group(1)
    print 'Parsed %s: ' % d
    return d

    def loginWithCheckCode(self):
        checkcode = raw_input('请输入验证码')
        self.post['TPL_checkcode'] = checkcode
        self.postData = urllib.urlencode(self.post)
        try:
            request = urllib2.Request(self.loginURL,self.postData,self.loginHeaders)
            response = self.opener.open(request)
            content = response.read().decode('gbk')
            pattern = re.compile(u'\u9a8c\u8bc1\u7801\u9519\u8bef',re.S)
            result = re.search(pattern,content)
            if result:
                print u"验证码输入错误"
                return False
            else:
                tokenPattern = re.compile('id="J_HToken" value="(.*?)"')
                tokenMatch = re.search(tokenPattern,content)
                if tokenMatch:
                    print u"验证码输入正确"
                    print tokenMatch.group(1)
                    return tokenMatch.group(1)
                else:
                    print u"J_Token"
                    return False

        except urllib2.HTTPError,e:
            print u"出错",e.reason
            return False