def index():
    """
    Lists all articles, separated by category. This method maps to the front
    page.
    """
    # Create a dictionary `files` that separates articles by category.
    for file_ in listdir(ARTICLE_DIR):
        if isfile(ARTICLE_DIR + file_) and file_ != 'empty':
            files = dict(Miscellaneous=[])
            break
        files = dict()

    for file_ in listdir(ARTICLE_DIR):
        if isdir(os.path.join(ARTICLE_DIR, file_)):
            files[file_] = []
            for f in listdir(os.path.join(ARTICLE_DIR, file_)):
                if f.endswith('.md'):
                    with open(os.path.join(ARTICLE_DIR, file_, f), 'r') as f_open:
                        title=f_open.readline()
                        files[file_].append(dict(file_=f, slug=slugify(title), title=title.decode('utf-8')))
        else:
            if file_.endswith('.md'):
                with open(os.path.join(ARTICLE_DIR, file_), 'r') as f_open:
                    title=f_open.readline()
                    files['Miscellaneous'].append(dict(file_=file_, slug=slugify(title), title=title.decode('utf-8')))

    blurb = open('pages/front.md', 'r').read()

    return render_template('index.html', files=files, blurb=blurb)

 def fetch(self):
     categories = []
     zendesk_categories = self.req.get_items(model.Category)
     for zendesk_category in zendesk_categories:
         category_filename = utils.slugify(zendesk_category['name'])
         category = model.Category(zendesk_category['name'], zendesk_category['description'], category_filename)
         print('Category %s created' % category.name)
         category.meta = zendesk_category
         zendesk_sections = self.req.get_items(model.Section, category)
         categories.append(category)
         for zendesk_section in zendesk_sections:
             section_filename = utils.slugify(zendesk_section['name'])
             section = model.Section(category, zendesk_section['name'],
                                     zendesk_section['description'], section_filename)
             print('Section %s created' % section.name)
             section.meta = zendesk_section
             zendesk_articles = self.req.get_items(model.Article, section)
             category.sections.append(section)
             for zendesk_article in zendesk_articles:
                 body = html2text.html2text(zendesk_article.get('body', ''))
                 article_filename = utils.slugify(zendesk_article['title'])
                 article = model.Article(section, zendesk_article['title'], body, article_filename)
                 print('Article %s created' % article.name)
                 article.meta = zendesk_article
                 section.articles.append(article)
     return categories

 def create_question(self, title, description, userprofile, tags):
     if self.exists(slug=slugify(title)):
         raise QuestionAlreadyExistsException
     question = Question(title=title, slug=slugify(title),
                         description=description, raised_by=userprofile)
     question.save()
     question.tags.add(*tags)
     return question 

 def find_or_create(cls, continent, realm, name):
     key_name = cls.key_name( continent, realm, name )
     c = cls.get_by_key_name( key_name )
     if c:
         return c
     c = Character( key_name = key_name, continent = continent, realm = realm, name = name )
     c.urltoken = slugify(name)
     c.realm_urltoken = slugify(realm)
     return c

    def __init__(self, file_path, extra_data={}):
        self.text_file = TextFile(file_path)
        self.title = self.text_file.title
        self.extra_data = extra_data
        self.filename, self.ext = path.splitext(self.text_file.file_path)
        self.slug = self.text_file.headers.get('slug') or slugify(self.title) or \
                                                          slugify(self.filename)
        self.body = self.markup()

        super(Post, self).__init__(self.slug)
        self.headers = self.text_file.headers

 def post(self):
     assert self.creator
     name = self.read('name')
     if not (name and name.strip()): raise PostingError("Please enter a name for your Boragle") 
     slug = utils.slugify(self.read('url'))
     if slug == '': slug = utils.slugify(name)
     if Boragle.find_by_slug(slug): raise PostingError('This url is already in use.')
     new_boragle = Boragle(name = self.read('name'),
             slugs = [slug],
             desc = self.read('desc'),
             creator = self.creator)
     new_boragle.put()
     self.redirect(new_boragle.url)

def new_job():
    if not g.site.domain == g.user:
        abort(403)

    j = Job()
    if request.method == "POST":
        portfolio = Portfolio.objects.get(site=g.site.domain)
        job_name = request.form.get("name")
        slugs = [__j.slug for __j in Job.objects.filter(site=g.site.domain)]
        counter = 1
        slug = slugify(job_name)
        __slug = slug
        while __slug in slugs:
            counter += 1
            __slug = "%s_%d" % (slug, counter)
        j.slug = __slug
        j.name = job_name
        j.site = g.site.domain
        j.categories = [ c.strip() for c in request.form.get("categories").split(",") ]
        j.intro = request.form.get("intro")
        j.description = request.form.get("description")
        j.slides = []
        texts = request.form.getlist("text")
        image_urls = request.form.getlist("image_url")
        captions = request.form.getlist("caption")
        caption_links = request.form.getlist("caption_link")
        for text, image_url, caption, caption_link in zip(texts, image_urls, captions, caption_links):
            if text or image_url:
                j.slides.append(Slide(text=text, image_url=image_url, caption=caption, caption_link=caption_link))
        j.save()
        portfolio.jobs.append(j)
        portfolio.save()
        return redirect(url_for(".job", slug=j.slug))
    return render_template("edit_job.html", job=j)

    def new_post(post_pages, is_post=True):
        # Guess where we should put this
        for path, _, _, use_in_rss in post_pages:
            if use_in_rss == is_post:
                break
        else:
            path = post_pages[0][0]

        print "Creating New Post"
        print "-----------------\n"
        title = raw_input("Enter title: ").decode(sys.stdin.encoding)
        slug = utils.slugify(title)
        data = u"\n".join([title, slug, datetime.datetime.now().strftime("%Y/%m/%d %H:%M")])
        output_path = os.path.dirname(path)
        meta_path = os.path.join(output_path, slug + ".meta")
        pattern = os.path.basename(path)
        if pattern.startswith("*."):
            suffix = pattern[1:]
        else:
            suffix = ".txt"
        txt_path = os.path.join(output_path, slug + suffix)

        if os.path.isfile(meta_path) or os.path.isfile(txt_path):
            print "The title already exists!"
            exit()

        with codecs.open(meta_path, "wb+", "utf8") as fd:
            fd.write(data)
        with codecs.open(txt_path, "wb+", "utf8") as fd:
            fd.write(u"Write your post here.")
        print "Your post's metadata is at: ", meta_path
        print "Your post's text is at: ", txt_path

 def test_college_valid_creation(self):
     data = {'name':'SVPCET'}
     College.objects.create_college(name=data['name'])
     college = College.objects.latest()
     self.assertTrue(college)
     self.assertEquals(college.name, data['name'])
     self.assertEquals(college.slug, slugify(data['name']))

 def test_company_valid_creation(self):
     data = {'name':'Infosys'}
     Company.objects.create_company(**data)
     company = Company.objects.latest()
     self.assertTrue(company)
     self.assertEquals(company.name, data['name'])
     self.assertEquals(company.slug, slugify(data['name']))

def save_aviso(form):
    entity=exists_entity(Aviso, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.texto=form.texto
        entity.ativo=form.ativo
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.texto=form.texto
        entity.ativo=form.ativo
        db.put(entity)
    else:
        while find_slug(Aviso, slug):
            slug = versionate(slug)
        db.put(Aviso(
            titulo=form.titulo,
            slug=slug,
            data_publicacao = datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S'),
            author = users.get_current_user(),
            texto=form.texto,
            ativo=form.ativo))

def save_foto(form):
    entity=exists_entity(Foto, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.thumb=form.thumb
        entity.foto=form.foto
        entity.status=form.status
        entity.link_acao_foto=form.link_acao_foto
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.thumb=form.thumb
        entity.foto=form.foto
        entity.status=form.status
        entity.link_acao_foto=form.link_acao_foto
        db.put(entity)
    else:
        while find_slug(Foto, slug):
            slug = versionate(slug)
        db.put(Foto(
            titulo=form.titulo,
            slug=slug,
            data_publicacao = datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S'),
            thumb = form.thumb,
            foto = form.foto,
            status = form.status,
            link_acao_foto=form.link_acao_foto))

def process_item_fn(row):
    links = row.xpath('.//a[contains(@class,"title")]')
    for link in links:
        try:
            votes = row.xpath('.//div[contains(@class, "score likes")]')[0].text_content().strip()
            row_link = (
                link.attrib["href"]
                if link.attrib["href"].startswith("http")
                else "http://reddit.com" + link.attrib["href"]
            )
            if int(votes) < 20:
                return False
            comment_a = row.xpath('.//a[contains(text(), "comment")]')[0]
            comments = comment_a.text.split()[0]
            comments = "0" if "comment" in comments else comments
            title = normalize(link.text_content())
            tagline = row.xpath('.//p[@class="tagline"]')[0].text_content().split("by")
            date = row.xpath(".//time/@datetime")[0]
            author = tagline[1].split()[0]
            return {
                "_id": slugify(title),
                "title": title,
                "author": author,
                "likes": {"at": datetime.datetime.now().isoformat()[:19], "n": int(votes)},
                "comments": comments,
                "date": date,
                "url": row_link,
                "description": "",
                "comment_link": comment_a.attrib["href"],
            }
        except ValueError as e:
            print("reddit error", e)
    return False

 def create_company(self, name):
     slug = slugify(name)
     if self.exists(slug=slug):
         raise CompanyAlreadyExistsException
     company = Company(name=name, slug=slug)
     company.save()
     return company

def process_item_fn(row, conf):
    links = row.xpath('.//a[contains(@class,"title")]')
    for link in links:
        try:
            votes = row.xpath('.//div[contains(@class, "score likes")]')[0].text_content().strip()
            row_link = link.attrib['href'] if link.attrib['href'].startswith(
                'http') else 'http://reddit.com' + link.attrib['href']
            if int(votes) < conf['reddit_minimum_votes']:
                return False
            comment_a = row.xpath('.//a[contains(text(), "comment")]')[0]
            comments = comment_a.text.split()[0]
            comments = '0' if 'comment' in comments else comments
            title = normalize(link.text_content())
            tagline = row.xpath('.//p[@class="tagline"]')[0].text_content().split('by')
            date = row.xpath('.//time/@datetime')[0]
            author = tagline[1].split()[0]
            return {'_id': slugify(title),
                    'title': title,
                    'author': author,
                    'likes': {'at': datetime.datetime.now().isoformat()[:19], 'n': int(votes)},
                    'comments': comments,
                    'date': date,
                    'url': row_link,
                    'description': '',
                    'comment_link': comment_a.attrib['href']}
        except ValueError as e:
            print('reddit error', e)
    return False

 def __init__(self, title=None, created_at=None):
     if title:
         self.title = title
         self.slug = slugify(title)
     if created_at:
         self.created_at = created_at
         self.updated_at = created_at

def show(slug):
    """
    Search the `articles` directory for an article whose slug matches the URL
    parameter. When we find the article, render it.
    """
    article = None

    # Searching articles ..
    for file_ in listdir(ARTICLE_DIR):
        if file_.endswith(EXTENSIONS):
            with open(os.path.join(ARTICLE_DIR, file_), 'r') as f:
                if slug == slugify(f.readline()):
                    article = os.path.join(ARTICLE_DIR, file_)
                    break

    # If we didn't find the article, it doesn't exist.
    if not article:
        article = os.path.join(PAGES_DIR, 'article-404.md')

    with open(article, 'r') as f:
        lines = f.read().split('\n')
        # Title should be the first line of the file. 
        title = lines.pop(0).strip().decode('utf8')
        # Category should be second.
        category = lines.pop(0).strip().decode('utf8')
        # The rest is the article itself.
        source = '\n'.join(lines).decode('utf8')
        
    return render_template('show.html', article=dict(title=title, source=source))

def read_player_line(line):
    n1 = line.index('"')
    n2 = line.index('"', n1 + 1)
    nick = line[n1 + 1 : n2]
    p = Player()
    setattr(p, "nick", nick)
    setattr(p, "slug_nick", slugify(p.nick))
    attribs = line[n2 + 2 :].split(" ")
    for attr in attribs[2:]:
        attr = attr.split(":")
        if len(attr) == 2:
            key, val = attr
            if key in _LOG_FLOAT_PROPERTIES:
                setattr(p, key, float(val))
            else:
                assert key in _LOG_INT_PROPERTIES
                setattr(p, key, int(val))
        elif len(attr) == 5:
            weapon, shots, hits, kills, deaths = attr
            try:
                hitrate = float(hits) / float(shots)
            except ZeroDivisionError:
                hitrate = 0.0
            setattr(
                p, weapon, dict(shots=int(shots), hits=int(hits), kills=int(kills), deaths=int(deaths), hitrate=hitrate)
            )
        else:
            print attr
    for prop in _LOG_INT_PROPERTIES:
        if not hasattr(p, prop):
            setattr(p, prop, 0)
    for prop in _LOG_FLOAT_PROPERTIES:
        if not hasattr(p, prop):
            setattr(p, prop, 0.0)
    return p

def show(slug):
    """
    Search the `articles` directory for an article whose slug matches the URL
    parameter. When we find the article, render it.
    """
    # Find the right article
    for file_ in listdir(ARTICLE_DIR):
        if file_.endswith(EXTENSIONS):
            with open(os.path.join(ARTICLE_DIR, file_), 'r') as f:
                if slug == slugify(f.readline()):
                    article = os.path.join(ARTICLE_DIR, file_)
                    break

    # Now that we've found the right article, let's process it.
    with open(article, 'r') as f:
        lines = f.read().split('\n')
        
        # We don't need title or category, but it's easier to explicitly state
        # why we're popping the first two lines.
        title = lines.pop(0).strip() # Title should appear on the first line
        category = lines.pop(0).strip() # Category should appear on the second

        source = '\n'.join(lines).decode('utf8')
        
    return render_template('show.html', article=dict(source=source))

 def __init__(self, dflog_path, messaging=None, **time_kwargs):
     #calculate the beginning of the week from logfile name for ublox timestamps
     week_epoch=utils.logpath2dt(dflog_path)
     self.read_dflog(dflog_path, epoch=week_epoch)
     self.set_dtype_from_fmt()
     self.flight_name=utils.slugify(dflog_path.split('/')[-1])
     self.messaging=messaging