def run(self):
        if micawber is None:
            msg = req_missing(['micawber'], 'use the media directive', optional=True)
            return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')]

        providers = micawber.bootstrap_basic()
        return [nodes.raw('', micawber.parse_text(" ".join(self.arguments), providers), format='html')]

    def import_posts(self, zipfile, names):
        """Import all posts."""
        out_folder = 'posts'
        providers = micawber.bootstrap_basic()
        for name in names:
            with zipfile.open(name, 'r') as post_f:
                data = json.load(post_f)
                title = data['title']

                slug = utils.slugify(title)

                if not slug:  # should never happen
                    LOGGER.error("Error converting post:", title)
                    return

                description = ''
                post_date = dateutil.parser.parse(data["published"])
                content = data["object"]["content"]

                for obj in data["object"].get("attachments", []):
                    content += '\n<div> {} </div>\n'.format(micawber.parse_text(obj["url"], providers))

                tags = []
                self.write_metadata(os.path.join(self.output_folder, out_folder, slug + '.meta'), title, slug, post_date, description, tags)
                self.write_content(
                    os.path.join(self.output_folder, out_folder, slug + '.html'),
                    content)

def _build_provider_list():
    """
    Construct the provider registry, using the app settings.
    """
    registry = None
    if appsettings.FLUENT_OEMBED_SOURCE == 'basic':
        registry = bootstrap_basic()
    elif appsettings.FLUENT_OEMBED_SOURCE == 'embedly':
        params = {}
        if appsettings.MICAWBER_EMBEDLY_KEY:
            params['key'] = appsettings.MICAWBER_EMBEDLY_KEY
        registry = bootstrap_embedly(**params)
    elif appsettings.FLUENT_OEMBED_SOURCE == 'noembed':
        registry = bootstrap_noembed(nowrap=1)
    elif appsettings.FLUENT_OEMBED_SOURCE == 'list':
        # Fill list manually in the settings, e.g. to have a fixed set of supported secure providers.
        registry = ProviderRegistry()
        for regex, provider in appsettings.FLUENT_OEMBED_PROVIDER_LIST:
            registry.register(regex, Provider(provider))
    else:
        raise ImproperlyConfigured("Invalid value of FLUENT_OEMBED_SOURCE, only 'basic', 'list', 'noembed' or 'embedly' is supported.")

    # Add any extra providers defined in the settings
    for regex, provider in appsettings.FLUENT_OEMBED_EXTRA_PROVIDERS:
        registry.register(regex, Provider(provider))

    return registry

def media_create_tags(request, match, instance=None,  template_name='media_form.html'):

    if request.method == 'POST':
        form = MediaForm(request.POST, instance=instance if instance is not None else Media())

        if form.is_valid():
            providers = micawber.bootstrap_basic()
            try:
                provider_obj = providers.request(form.cleaned_data['url'])
            except micawber.exceptions.ProviderNotFoundException as msg:
                return render(request, template_name, {'object_form': form, 'error_message': msg})
            except micawber.exceptions.ProviderException as msg:
                return render(request, template_name, {'object_form': form, 'error_message': "Embedding disabled by request"})
            media = form.save(commit=False)
            media.title = provider_obj.get('title') if len(provider_obj.get('title')) < 200 else provider_obj.get('title')[:200]
            media.media_type = provider_obj['type']
            media.thumbnail_url = provider_obj.get('thumbnail_url', provider_obj.get('url'))
            media.submitted_by = request.user if instance is None else instance.submitted_by
            media.blog_type = 1  # media
            media.save()
            media.tags.clear()
            for tag in form.cleaned_data['tags']:
                media.tags.add(tag.lower())

            return redirect(media)
    else:
        if match:
            tag_list = ['"%s"' % tag.replace('-', ' ') for tag in match.split('|')]
            form = MediaForm(initial={'tags': ','.join(tag_list)})
        else:
            form = MediaForm(instance=instance)
    return render(request, template_name, {'object_form': form})

def bookmark(ctx, url):
    providers = micawber.bootstrap_basic()

    bookmark = {
        'url': url
    }

    try:
        oembed = providers.request(url)
        soup = BeautifulSoup(oembed['html'], 'html.parser')
    except:
        oembed = {}
        content = '<!-- TODO Write something here. -->'

    if oembed.get('type') == 'video':
        iframe_src = soup.iframe.get('src')
        content = ' include c_video.html iframe_url="{0}" '.format(iframe_src)
        bookmark = oembed

    if oembed.get('type') == 'photo':
        oembed['image_src'] = soup.img.get('src')
        content = (' include c_image.html src="{image_src}" '
                   'alt="{title}" ')

    post = Post(content, title=ctx.obj['TITLE'], layout='post_bookmark',
                category='bookmark', date=ctx.obj['NOW'].isoformat(),
                tags=ctx.obj.get('TAGS'), bookmark=bookmark)

    write_post('bookmark', ctx.obj['NOW'], ctx.obj['SLUG'], post)

def getIframeForVideo(videoUrl):
    providers = micawber.bootstrap_basic()
    try:
        embed = providers.request(videoUrl, maxwidth=580, maxheight=377)
    except (InvalidResponseException,
            ProviderException,
            ProviderNotFoundException):
        return None
    return embed['html']

def get_title_with_oembed(url):
    providers = micawber.bootstrap_basic()
    try:
        res = providers.request(url)
    except micawber.ProviderException as e:
        logger.info(e)
        return ''

    return '{} - {}'.format(res.get('title'), res.get('provider_name'))

def getInformationsForVideo(videoUrl):
    providers = micawber.bootstrap_basic()
    try:
        embed = providers.request(videoUrl)
    except (InvalidResponseException,
            ProviderException,
            ProviderNotFoundException):
        return {'title': 'Vidéo introuvable',
                'thumb': ''}
    return {'title': embed['title'],
            'thumb': embed['thumbnail_url']}

    def run(self):
        if micawber is None:
            msg = (
                "To use the media directive you need to install "
                "the micawber module."
            )
            utils.logger.WARN(msg)
            return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')]

        providers = micawber.bootstrap_basic()
        return [nodes.raw('', micawber.parse_text(" ".join(self.arguments), providers), format='html')]

 def test_oembed_services(self):
   '''
   A test various oembed services and a simple howto use micawber to discover oembed services (soundcloud, vimeo etc) from urls
   Note: it doesn't work with ISSUU
   '''
   import micawber
   mic = micawber.bootstrap_basic()
   you = mic.request('https://www.youtube.com/watch?v=GGyLP6R4HTE') # youtube
   vim = mic.request('http://vimeo.com/17081933') # vimeo
   try:
     nothing =  mic.request('http://en.wikipedia.org/wiki/The_Boat_Race_2000')
   except micawber.exceptions.ProviderNotFoundException, e:
     pass

    def run(self):
        if micawber is None:
            msg = "To use the media directive, isntall micawber first."
            return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')]

        url = " ".join(self.arguments)
        providers = micawber.bootstrap_basic()
        data = providers.request(url)
        html = '<h3>{}</h3>{}'.format(
            data['title'],
            micawber.parse_text(url, providers)
        )
        return [nodes.raw('', html, format='html')]

 def html_content(self):
     app = current_app._get_current_object()
     # hilite = CodeHiLiteExtension(linenums=False, css_class='highlight')
     extras = ExtraExtension()
     #TODO why can I not add CodeHiLiteExtension????
     # markdown_content = markdown(self.body, extensions=[hilite, extras])
     markdown_content = markdown(self.body, extensions=[extras])
     oembed_providers = bootstrap_basic(OEmbedCache())
     oembed_content = parse_html(
         markdown_content,
         oembed_providers,
         urlize_all=True,
         maxwidth=app.config['SITE_WIDTH'])
     return Markup(oembed_content)

    def save(self, *args, **kwargs):
        providers = micawber.bootstrap_basic()
        providers.register(WISTIA_REGEX, micawber.Provider("http://fast.wistia.com/oembed/"))
        
        oembed_response = providers.request(self.media_url,maxwidth=600, maxheight=500, width=600)
        self.embed_code=oembed_response["html"]
        
        if oembed_response["type"]=="photo":
            self.media_type = IMAGE
            
            #request the image again, add full-size link to the embed code
            second_oembed_response = providers.request(self.media_url)
            link_text = '<div class="viewfullsize-link"><a href="%s">View full-size image</a></div>' % second_oembed_response["url"]
            self.embed_code = "".join([self.embed_code,link_text])
            
        elif oembed_response["type"]=="video":
            self.media_type = VIDEO
            
        elif oembed_response["type"]=="rich":
            self.media_type = RICH
            
        
        try:
            if "thumbnail_url" in oembed_response:
                self.thumbnail_url=oembed_response["thumbnail_url"]
                
            elif oembed_response["type"]=="photo":
                self.thumbnail_url = oembed_response["url"]
            else:
                self.thumbnail_url = os.path.join(settings.STATIC_URL,"images/rich.png")

        except:
            self.thumbnail_url=""
        
        '''
        TODO some hacky Wistia-specific additions here
        '''
        if re.match(WISTIA_REGEX,self.media_url):
            self.thumbnail_url = "%s?image_crop_resized=260x180" % self.thumbnail_url.split("?")[0]
            
            if self.media_type == IMAGE:
                self.embed_code = re.sub("image_crop_resized","image_resize",self.embed_code)
                
            elif self.media_type == RICH:
                
                if oembed_response["url"]:
                    link_text = '<div class="viewfullsize-link"><a href="%s">Download this file</a></div>' % oembed_response["url"]
                    self.embed_code = "".join([self.embed_code,link_text])

        super(MediaArtefact, self).save(*args, **kwargs)

def index():
	#import shit
	providers = micawber.bootstrap_basic()
	add_oembed_filters(app, providers)
	#connect to db
	db_obj = connect_db('data.db')
	#assign cursor and connection to variables
	cur = db_obj[0]
	con = db_obj[1]
	
	#get list of all threads		
	cur.execute('SELECT * FROM thread;')
	threads = cur.fetchall()
	#get original post belonging to each thread
	cur.execute('SELECT * FROM post WHERE id = 1 ;')
	posts = cur.fetchall()

	if request.method == 'POST':
		
		#grab form data
		subject = request.form['subjectLine']
		comment = request.form['comment']
		embed = request.form['embed']

		if subject == "" or comment == "" or embed =="":
			return "need javascript"
		
		#get highest threadID
		cur.execute('SELECT MAX(id) FROM thread;')
		#current post is 1 higher
		threadID = cur.fetchone()[0] + 1
		thread = [threadID, subject]
		#adds thread to the database by passing it the item
		cur.execute('insert into thread values(?,?)',thread)

		#create original post
		post = [1, threadID, strftime("%I:%M %B %d, %Y"), comment, embed] 
		cur.execute('insert into post values(?,?,?,?,?)',post)

		#save the changes
		con.commit()
		con.close()
		
		return redirect(url_for('index'))

	con.close()
	
	return render_template('listthreads.html', threads=reversed(threads), posts=posts) 

  def test_oembed_services(self):
    import micawber
    mic = micawber.bootstrap_basic()
    you = mic.request('https://www.youtube.com/watch?v=GGyLP6R4HTE') # youtube
    vim = mic.request('http://vimeo.com/17081933') # vimeo


    print self.assertEqual(you['provider_name'], u'YouTube')
    print self.assertEqual(vim['provider_name'], u'Vimeo')

    document = Document(corpus=self.corpus, name=u'N-L_FR_20140305_.txt')
    document.mimetype = "text/html"
    document.save()

    t1, created = Tag.objects.get_or_create(type=Tag.OEMBED_PROVIDER_NAME, name=vim['provider_name'])
    t1, created = Tag.objects.get_or_create(type=Tag.OEMBED_TITLE, name=vim['title'])
    t2, created = Tag.objects.get_or_create(type=Tag.OEMBED_THUMBNAIL_URL, name=vim['thumbnail_url'])  

 def get_entry_feature(self):
     """
     If there is a video_url we want to include that as a feature, otherwise
     we fall through to the thumbnail
     """
     if self.video_url:
         """
         Ensure that the video is from a site that supports oembed
         """
         if any(v in self.video_url for v in settings.ALLOWED_OMEMBED_SITES):
             """
             Extracting the oembed data using https://github.com/coleifer/micawber
             """
             providers = micawber.bootstrap_basic()
             entry_oembed = micawber.parse_text(self.video_url, providers)
             """ Ensure https, less site warnings the better """
             return string.replace(entry_oembed, 'http://', 'https://')
     if self.thumbnail:
         return '<img src="%s" alt=""/>' % self.get_image_src()

def viewThread(threadID):
	#import shit
	providers = micawber.bootstrap_basic()
	add_oembed_filters(app, providers)
	#database shit
	db_obj = connect_db('data.db')
	cur = db_obj[0]
	con = db_obj[1]	

	#get information about this thread
	cur.execute('SELECT * FROM thread WHERE id ='+threadID+';')
	threadInfo = cur.fetchall()

	title = threadInfo[0][1]

	#get original post belonging to each thread
	cur.execute('SELECT * FROM post WHERE threadID ='+threadID+';')
	posts = cur.fetchall()

	return render_template('viewthread.html',title=title, posts=posts) 

  def save(self, *args, **kwargs):
    self.slug = helper_uuslug(model=Document, instance=self, value=self.title)
    
    if self.pk is None:
      super(Document, self).save(*args, **kwargs)

    if self.permalink:
      import micawber
      mic = micawber.bootstrap_basic()

      try:
        oem = mic.request(self.permalink)
      except micawber.exceptions.ProviderNotFoundException, e:
        pass
      else: # store as oembed tags
        t1, created = Tag.objects.get_or_create(type=Tag.OEMBED_PROVIDER_NAME, name=oem['provider_name'])
        t1, created = Tag.objects.get_or_create(type=Tag.OEMBED_TITLE, name=oem['title'])
        t2, created = Tag.objects.get_or_create(type=Tag.OEMBED_THUMBNAIL_URL, name=oem['thumbnail_url'])
        
        self.tags.add(t1)
        self.tags.add(t2)

def _build_provider_list():
    """
    Construct the provider registry, using the app settings.
    """
    if appsettings.FLUENT_OEMBED_SOURCE == 'basic':
        registry = micawber.bootstrap_basic()
        # make sure http://youtu.be urls are also valid, see https://github.com/coleifer/micawber/pull/7
        registry.register('https?://(\S*.)?youtu(\.be/|be\.com/watch)\S*', Provider('http://www.youtube.com/oembed'))
        return registry
    elif appsettings.FLUENT_OEMBED_SOURCE == 'embedly':
        params = {}
        if appsettings.MICAWBER_EMBEDLY_KEY:
            params['key'] = appsettings.MICAWBER_EMBEDLY_KEY
        return micawber.bootstrap_embedly(**params)
    elif appsettings.FLUENT_OEMBED_SOURCE == 'list':
        # Fi
        registry = ProviderRegistry()
        for regex, provider in appsettings.FLUENT_OEMBED_PROVIDER_LIST:
            registry.register(regex, Provider(provider))
        return registry
    else:
        raise ImproperlyConfigured("Invalid value of FLUENT_OEMBED_SOURCE, only 'basic', 'list' or 'embedly' is supported.")

def get_random_image(retries=5):
    """ Fetch a random image from flickr.

    XXX: Change the search term to an argument
    """
    import micawber
    import lxml.html
    from random import choice

    # from urllib2 import urlopen
    FLICKR = "http://www.flickr.com"
    SEARCH = "/search/?q=ultimate frisbee %s&l=cc&ct=0&mt=photos&adv=1&page=%d"
    TERMS = ["throw", "catch", "layout", "bid", "sky", "huck"]

    # def get_random_image():
    term = choice(TERMS)
    page = choice(range(1, 6))
    search = SEARCH % (term, page)
    print "Searching for images on Flickr..."
    tree = lxml.html.parse("%s%s" % (FLICKR, search))
    photos = [e for e in tree.findall(".//a[@data-track]") if e.attrib["data-track"] == "photo-click"]

    found = False
    count = 0

    print "Getting thumbnail and url for random image...",
    while not found and count < retries:
        count += 1
        the_photo = choice(photos)
        photo_url = FLICKR + the_photo.attrib["href"]

        # FIXME: Catch any possible errors
        providers = micawber.bootstrap_basic()
        data = providers.request(photo_url)

        found = "thumbnail_url" in data and "url" in data
    print "done"
    return data