def match(path):
        for pat, type, property, default_title in patterns:
            m = web.re_compile('^' + pat).match(path)
            if m:
                prefix = m.group()
                extra = web.lstrips(path, prefix)
                tokens = extra.split("/", 2)

                # `extra` starts with "/". So first token is always empty.
                middle = web.listget(tokens, 1, "")
                suffix = web.listget(tokens, 2, "")
                if suffix:
                    suffix = "/" + suffix

                return type, property, default_title, prefix, middle, suffix
        return None, None, None, None, None, None

def batchInsertPreNf(market=None,brand=None):
    ##dbconn.query("insert into prenaifen (name,duration,weight,spec,brand,series,sellto,age,duan,pack,place,price,img) select name,duration,weight,spec,brand,series,sellto,age,duan,pack,place,price,img from naifenitem where brand = $brand and market = $market",vars=dict(brand=brand,market=market))
    res = dbconn.query(u"select * from prenaifen where brand = $brand",vars=dict(brand=brand))
    for r in res:
        res1  = dbconn.query(u"select * from naifenitem where name = $name and market=$market",vars=dict(name=r.name,market=market))
        item = web.listget(res1,0,None)
        dbconn.insert("prmatch",prid=r.ID,itemid=item.itemid,market=market)

def make_question(url, category):
    """Create the random question from wikipedia."""
    try:
        article = lib.dnl(url, referer='http://en.wikipedia.org/')
        xml = lib.parse_xml(article)
    except:
        return False
        raise 'Either can\'t download the wikipedia article or can\'t parse it.'
    
    answer = xml.findtext('.//h1').encode('utf-8')
    snippet = ''
    for p in xml.xpath('//div[@id="bodyContent"]/p'):
        snippet += etree.tostring(p, method='text', encoding='utf-8').strip() + ' '
#        snippet += lib.strip_tags(etree.tostring(p)) + '\n'
    snippet = snippet[0:snippet_size].replace('\n', '<br />').decode('utf-8')
    snippet = re.sub('\[\d+\]', '', snippet)
    
    snippet_secret = snippet
    for a in answer.split():
        if a[-1] == 's': a = a[:-1]
        p = re.compile(r'\b%ss?\b' % re.escape(a), re.I)
        snippet_secret = p.sub('<strong class="depleted">' + '?' * len(a) + '</strong>', snippet_secret)
    
    image_url = web.listget(
        xml.xpath('//img[@class="thumbimage"]//@src'), 0, '')
    
    return web.storage(
        dict(answer=answer, snippet_secret=snippet_secret, category=category,
        snippet=snippet, wiki_url=url, image_url=image_url))

def already_voted(module_id, user_ip):
    return web.listget(
        db.select('votes',
            vars = dict(ip=user_ip, id=module_id),
            what = 'count(vote)', 
            where = 'ip = $ip and module_id = $id',
            group = 'module_id having count(module_id) > 0'), 0, False)

def get_module(id):
    """Get a specific module by its id."""
    return web.listget(
        db.select('modules', 
            vars = dict(id=id), 
            what = 'id, title, description, author, author_affiliation, title_url, \
                    author_email, render_inline, screenshot, calculated_vote, url',
            where='id = $id'), 0, False)

def get_random_wiki_url(category):
    """Use yahoo to get a random wikipedia url."""
    wiki_url = ''
    if categories.yahoo_queries.has_key(category):
        query = categories.get_yahoo_query(category)
        wiki_url = web.listget(
            lib.yahoo_search(query, random.randint(0, question_range), 1), 0, {}).get('url', '')
    return wiki_url

def get_special_question(ID_special_question):
    special_question_row = web.listget(list(db.select('special_questions',dict(ID_special_question=ID_special_question),where='ID_special_question=$ID_special_question')), 0, default=None)
    if special_question_row is None:
        return None
    else:
        ID_special_question=special_question_row['ID_special_question']
        description_special_question=special_question_row['Description']
        question_list = [Question(row['Description'],row['Answer']) for row in list(db.select('questions',dict(ID_special_question=ID_special_question),where='ID_special_question=$ID_special_question'))]
        return SpecialQuestion(description_special_question,question_list)

 def GET(self):
     data = web.input()
     naifenid = data.naifenid
     r = web.listget(dbconn.query("""
         select n.itemid,n.market from prmatch m
         join naifenitem n
         on m.itemid = n.itemid and m.market = n.market
         where m.naifenid = $naifenid
     """,vars=dict(naifenid=naifenid)),0,None)
     if r:
         return web.seeother(getItemUrl(r.itemid,r.market))

def get_related(module_id): 
    """Get a similar module."""
    module_tags = [t.tag for t in tags.get_tags(module_id)]
    
    m = web.listget(
        db.select('tags', 
            vars  = dict(id=module_id, tags=module_tags),
            what  = 'module_id',
            where = 'module_id != $id and %s' % web.sqlors("tag = ", module_tags),
            group = 'module_id having count(module_id) > 1',
            order = 'rand()'), 0, False)
    
    return m and get_module(m.module_id)

    def GET(self, img_id):
        img = image.get_img_by_imgid(img_id)
        author = web.listget(users.get_users_by_id(img.userID) , 0, {})

        fav_user = image.GetFavImageByImageId(img_id).list()
        fav_user_ids = []
        for i in xrange(len(fav_user)):
            fav_user_ids += str(fav_user[i].user_id).split()

        usernnames = []
        for i in xrange(len(fav_user_ids)):
            usernnames += users.get_users_by_id(fav_user_ids[i])

        return view.base03(view.photo_fans(img_id, img, usernnames, author), user, siteName, 2)

def update_calculated_vote(module_id):
    min_votes = 5
    r = web.listget(
        db.select('votes',
            vars = dict(id=module_id),
            what = 'sum(vote) / count(module_id) as calculated_vote', 
            where = 'module_id = $id',
            group = 'module_id having count(module_id) > %s' % min_votes), 0, False)
    
    if r:
        db.update('modules', 
            vars = dict(id=module_id),
            where='id = $id',    
            calculated_vote=r.calculated_vote)

def fix_dirs(data):
    data = list(data)
    
    keys = set()
    for id, author, t, comment, docs in data:
        keys.update(doc['key'] for doc in docs if web.listget(doc['key'].split("/"), 1) not in skip_dirs)

    dirnames = set(os.path.dirname(k) for k in keys)

    def f(doc):
        if doc['key'] in dirnames:
            doc['key'] = doc['key'].rstrip("/") + "/index"
        return doc
        
    return _map_docs(f, data)

def main():
    data = read()
    data = fix_lang(data)
    data = fix_dirs(data)
    data = sorted(data, key=lambda d: d[2])
    
    spammers = [email.strip() for email in open("data/spammers.txt")]
    spamedits = [id.strip() for id in open("data/spamedits.txt")]

    skip_dirs = "user permission macros templates type user group usergroup".split()
    
    system("rm -rf build && mkdir build && cd build && git init")
    
    for id, author, t, comment, docs in data:
        if "http://" in comment and "webpy.org" not in comment:
            print "ignoring changeset: %s" % str((id, author, t, comment))
            continue

        author = re.sub(" +", " ", author)
        if author.strip() in spammers or id in spamedits:
            print "ignoring spam: %s" % str((id, author, t, comment))
            continue
            
        bad_keys = ['\xe0\xb1\x86recentchanges.md', ').md']
        
        print "** processing changeset", id, t
        added = False
        for doc in docs:
            key = doc['key'][1:].encode('utf-8') + ".md"
            if doc['type']['key'] == '/type/page' and web.listget(doc['key'].split("/"), 1) not in skip_dirs and key not in bad_keys:
                title = doc.get('title', '')
                body = doc.get("body", {})
                if isinstance(body, dict):
                    body = body.get("value", "").replace("\r\n", "\n")
                    
                write("build/" + key, title, body)
                system("cd build && git add '%s'" % key)
                added = True
                
        if author == "annonymous":
            author = "anonymous <[email protected]>"

        if added:
            system("cd build && git commit -m %s --date='%s' --author='%s'" % (simplejson.dumps(comment), git_date(t), author))
            
    system("cp -r _layouts static build")
    system("cd build && git add _layouts static && git commit -m 'Added layouts and static file'")

 def get(self, query):
     """
     Get values from the cache given a query. 
     
     The variable query is a string or an object with a unique representation.
     An object with a unique representation must implement instance variable "uniq".
     """
     query = get_unique_repr(query)
     key = self._make_key(query)
     r = self.db.select('cache', vars=dict(key=key), where='_key = $key', limit=1)
     r = web.listget(r, 0)
     
     value = None
     if r:
         if not r.sticky:
             self.db.query('update cache set datetime=now() where _key = $key', vars=dict(key=key))
         value = pickle.loads(base64.b64decode(r.value))
     return value

    def _split(self, path):
        """Splits the path as required by the get_readable_path.
        
            >>> _split = ReadableUrlProcessor()._split

            >>> _split('/b/OL1M')
            ('/b/OL1M', '', '')
            >>> _split('/b/OL1M/foo')
            ('/b/OL1M', 'foo', '')
            >>> _split('/b/OL1M/foo/cover')
            ('/b/OL1M', 'foo', '/cover')
            >>> _split('/b/OL1M/foo/cover/bar')
            ('/b/OL1M', 'foo', '/cover/bar')
        """
        tokens = path.split('/', 4)

        prefix = '/'.join(tokens[:3])
        middle = web.listget(tokens, 3, '')
        suffix = '/'.join(tokens[4:])

        if suffix:
            suffix = '/' + suffix

        return prefix, middle, suffix

def get_just_added_comment(uid, pid):
    return web.listget(
        db.select('_post_comments', vars = dict(pid=pid, uid = uid), where='pid = $pid and uid = $uid', order='id DESC', limit = 1), 0, {})

def getdp(dpid):
    return web.listget(dbconn.query("select * from danping where id = $dpid",vars=dict(dpid=dpid)),0,None)

def getsp(spid):
    return web.listget(dbconn.query("select * from formalitem where id=$spid",vars=dict(spid=spid)),0,None)

def get_user_by_email(email):
    result = db.select('users', vars=dict(email=email), where=web.db.sqlwhere({'email': email, 'is_active': '1'})).list()
    return web.listget(result, 0, {})

def hasNfitem(nf):
    return web.listget(dbconn.query("select * from naifenitem where itemid=$itemid and market=$market",vars=dict(itemid=nf.itemid,market=nf.market)),0,None)