def fetch_mf2(self, url):
    """Fetches a URL and extracts its mf2 data.

    Side effects: sets self.entity.html on success, calls self.error() on
    errors.

    Args:
      url: string

    Returns:
      (requests.Response, mf2 data dict) on success, None on failure
    """
    try:
      fetched = util.requests_get(url)
      fetched.raise_for_status()
    except BaseException as e:
      util.interpret_http_exception(e)  # log exception
      return self.error('Could not fetch source URL %s' % url)

    if self.entity:
      self.entity.html = fetched.text

    # .text is decoded unicode string, .content is raw bytes. if the HTTP
    # headers didn't specify a charset, pass raw bytes to BeautifulSoup so it
    # can look for a <meta> tag with a charset and decode.
    text = (fetched.text if 'charset' in fetched.headers.get('content-type', '')
            else fetched.content)
    doc = BeautifulSoup(text)

    # special case tumblr's markup: div#content > div.post > div.copy
    # convert to mf2.
    contents = doc.find_all(id='content')
    if contents:
      post = contents[0].find_next(class_='post')
      if post:
        post['class'] = 'h-entry'
        copy = post.find_next(class_='copy')
        if copy:
          copy['class'] = 'e-content'
        photo = post.find_next(class_='photo-wrapper')
        if photo:
          img = photo.find_next('img')
          if img:
            img['class'] = 'u-photo'
        doc = unicode(post)

    # parse microformats, convert to ActivityStreams
    data = parser.Parser(doc=doc, url=fetched.url).to_dict()
    logging.debug('Parsed microformats2: %s', json.dumps(data, indent=2))
    items = data.get('items', [])
    if not items or not items[0]:
      return self.error('No microformats2 data found in ' + fetched.url,
                        data=data, html="""
No <a href="http://microformats.org/get-started">microformats</a> or
<a href="http://microformats.org/wiki/microformats2">microformats2</a> found in
<a href="%s">%s</a>! See <a href="http://indiewebify.me/">indiewebify.me</a>
for details (skip to level 2, <em>Publishing on the IndieWeb</em>).
""" % (fetched.url, util.pretty_link(fetched.url)))

    return fetched, data

    def get_activities_response(self, **kwargs):
        kwargs.setdefault("fetch_events", True)
        kwargs.setdefault("event_owner_id", self.key.id())

        try:
            return super(FacebookPage, self).get_activities_response(**kwargs)
        except urllib2.HTTPError as e:
            code, body = util.interpret_http_exception(e)
            # use a function so any new exceptions (JSON decoding, missing keys) don't
            # clobber the original exception so we can re-raise it below.
            def dead_token():
                try:
                    err = json.loads(body)["error"]
                    return err["code"] in DEAD_TOKEN_ERROR_CODES or err["error_subcode"] in DEAD_TOKEN_ERROR_SUBCODES
                except:
                    return False

            if code == "401":
                if not dead_token():
                    # ask the user to reauthenticate. if this API call fails, it will raise
                    # urllib2.HTTPError instead of DisableSource, so that we don't disable
                    # the source without notifying.
                    self.gr_source.create_notification(
                        self.key.id(),
                        "Brid.gy's access to your account has expired. Click here to renew it now!",
                        "https://brid.gy/facebook/start",
                    )
                raise models.DisableSource()

            raise

  def get(self, type, source_short_name, string_id, *ids):
    source_cls = models.sources.get(source_short_name)
    if not source_cls:
      self.abort(400, "Source type '%s' not found. Known sources: %s" %
                 (source_short_name, models.sources))

    self.source = source_cls.get_by_id(string_id)
    if not self.source:
      self.abort(400, '%s %s not found' % (source_short_name, string_id))

    format = self.request.get('format', 'html')
    if format not in ('html', 'json'):
      self.abort(400, 'Invalid format %s, expected html or json' % format)

    for id in ids:
      if not self.VALID_ID.match(id):
        self.abort(404, 'Invalid id %s' % id)

    label = '%s:%s %s %s' % (source_short_name, string_id, type, ids)
    logging.info('Fetching %s', label)
    try:
      obj = self.get_item(*ids)
    except Exception, e:
      # pass through all API HTTP errors if we can identify them
      code, body = util.interpret_http_exception(e)
      if code:
        self.response.status_int = int(code)
        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write('%s error:\n%s' % (self.source.GR_CLASS.NAME, body))
        return
      else:
        raise

  def poll(self, source):
    """Actually runs the poll.

    Returns: dict of source property names and values to update (transactionally)
    """
    if source.last_activities_etag or source.last_activity_id:
      logging.debug('Using ETag %s, last activity id %s',
                    source.last_activities_etag, source.last_activity_id)
    source_updates = {}

    #
    # Step 1: fetch activities
    #
    cache = util.CacheDict()
    if source.last_activities_cache_json:
      cache.update(json.loads(source.last_activities_cache_json))

    try:
      response = source.get_activities_response(
        fetch_replies=True, fetch_likes=True, fetch_shares=True, count=50,
        etag=source.last_activities_etag, min_id=source.last_activity_id,
        cache=cache)
    except Exception, e:
      code, body = util.interpret_http_exception(e)
      if code == '401':
        msg = 'Unauthorized error: %s' % e
        logging.warning(msg, exc_info=True)
        raise models.DisableSource(msg)
      elif code in util.HTTP_RATE_LIMIT_CODES:
        logging.warning('Rate limited. Marking as error and finishing. %s', e)
        source_updates.update({'status': 'error', 'rate_limited': True})
        return source_updates
      else:
        raise

def handle_exception(self, e, debug):
  """A webapp2 exception handler that propagates HTTP exceptions into the response.

  Use this as a :meth:`webapp2.RequestHandler.handle_exception()` method by
  adding this line to your handler class definition::

    handle_exception = handlers.handle_exception

  I originally tried to put this in a :class:`webapp2.RequestHandler` subclass,
  but it gave me this exception::

    File ".../webapp2-2.5.1/webapp2_extras/local.py", line 136, in _get_current_object
      raise RuntimeError('no object bound to %s' % self.__name__) RuntimeError: no object bound to app

  These are probably related:

  * http://eemyop.blogspot.com/2013/05/digging-around-in-webapp2-finding-out.html
  * http://code.google.com/p/webapp-improved/source/detail?r=d962ac4625ce3c43a3e59fd7fc07daf8d7b7c46a

  """
  code, body = util.interpret_http_exception(e)
  if code:
    self.response.set_status(int(code))
    self.response.write('HTTP Error %s: %s' % (code, body))
  elif util.is_connection_failure(e):
    self.response.set_status(502)
    self.response.write('Upstream server request failed: %s' % e)
  else:
    raise

  def post(self):
    source = self.load_source(param='key')
    module = self.OAUTH_MODULES[source.key.kind()]
    feature = util.get_required_param(self, 'feature')
    state = util.encode_oauth_state({
      'operation': 'delete',
      'feature': feature,
      'source': source.key.urlsafe(),
      'callback': self.request.get('callback'),
    })

    # Blogger don't support redirect_url() yet
    if module is oauth_blogger_v2:
      return self.redirect('/blogger/delete/start?state=%s' % state)

    path = ('/instagram/callback' if module is indieauth
            else '/wordpress/add' if module is oauth_wordpress_rest
            else '/%s/delete/finish' % source.SHORT_NAME)
    kwargs = {}
    if module is oauth_twitter:
      kwargs['access_type'] = 'read' if feature == 'listen' else 'write'

    handler = module.StartHandler.to(path, **kwargs)(self.request, self.response)
    try:
      self.redirect(handler.redirect_url(state=state))
    except Exception as e:
      code, body = util.interpret_http_exception(e)
      if not code and util.is_connection_failure(e):
        code = '-'
        body = unicode(e)
      if code:
        self.messages.add('%s API error %s: %s' % (source.GR_CLASS.NAME, code, body))
        self.redirect(source.bridgy_url(self))
      else:
        raise

  def get_activities_response(self, **kwargs):
    kwargs.setdefault('fetch_events', True)
    kwargs.setdefault('fetch_news', self.auth_entity.get().type == 'user')
    kwargs.setdefault('event_owner_id', self.key.id())

    try:
      return super(FacebookPage, self).get_activities_response(**kwargs)
    except urllib2.HTTPError as e:
      code, body = util.interpret_http_exception(e)
      # use a function so any new exceptions (JSON decoding, missing keys) don't
      # clobber the original exception so we can re-raise it below.
      def dead_token():
        try:
          err = json.loads(body)['error']
          return (err.get('code') in DEAD_TOKEN_ERROR_CODES or
                  err.get('error_subcode') in DEAD_TOKEN_ERROR_SUBCODES or
                  err.get('message') in DEAD_TOKEN_ERROR_MESSAGES)
        except:
          logging.exception("Couldn't determine whether token is still valid")
          return False

      if code == '401':
        if not dead_token():
          # ask the user to reauthenticate. if this API call fails, it will raise
          # urllib2.HTTPError instead of DisableSource, so that we don't disable
          # the source without notifying.
          self.gr_source.create_notification(
            self.key.id(),
            "Brid.gy's access to your account has expired. Click here to renew it now!",
            'https://brid.gy/facebook/start')
        raise models.DisableSource()

      raise

  def do_post(self, source):
    if source.last_activities_etag or source.last_activity_id:
      logging.debug('Using ETag %s, last activity id %s',
                    source.last_activities_etag, source.last_activity_id)

    #
    # Step 1: fetch activities
    #
    try:
      response = source.get_activities_response(
        fetch_replies=True, fetch_likes=True, fetch_shares=True, count=50,
        etag=source.last_activities_etag, min_id=source.last_activity_id,
        cache=memcache)
    except Exception, e:
      code, body = util.interpret_http_exception(e)
      if code == '401':
        # TODO: also interpret oauth2client.AccessTokenRefreshError with
        # {'error': 'invalid_grant'} as disabled? it can mean the user revoked
        # access. it can also mean the token expired, or they deleted their
        # account, or even other things.
        # http://code.google.com/p/google-api-python-client/issues/detail?id=187#c1
        msg = 'Unauthorized error: %s' % e
        logging.exception(msg)
        raise models.DisableSource(msg)
      elif code in util.HTTP_RATE_LIMIT_CODES:
        logging.warning('Rate limited. Marking as error and finishing. %s', e)
        source.status = 'error'
        return
      else:
        raise

    def get(self, type, source_short_name, string_id, *ids):
        source_cls = SOURCES.get(source_short_name)
        if not source_cls:
            self.abort(400, "Source type '%s' not found. Known sources: %s" % (source_short_name, SOURCES))

        self.source = source_cls.get_by_id(string_id)
        if not self.source:
            self.abort(400, "%s %s not found" % (source_short_name, string_id))

        format = self.request.get("format", "html")
        if format not in ("html", "json"):
            self.abort(400, "Invalid format %s, expected html or json" % format)

        for id in ids:
            if not self.VALID_ID.match(id):
                self.abort(404, "Invalid id %s" % id)

        label = "%s:%s %s %s" % (source_short_name, string_id, type, ids)
        logging.info("Fetching %s", label)
        try:
            obj = self.get_item(*ids)
        except Exception, e:
            # pass through all API HTTP errors if we can identify them
            code, body = util.interpret_http_exception(e)
            if code:
                self.response.status_int = int(code)
                self.response.headers["Content-Type"] = "text/plain"
                self.response.write("%s error:\n%s" % (self.source.AS_CLASS.NAME, body))
                return
            else:
                raise

  def get_post(self, id):
    """Fetch a post.

    Args:
      id: string, site-specific post id
      is_event: bool

    Returns: ActivityStreams object dict
    """
    try:
      posts = self.source.get_activities(
          activity_id=id, user_id=self.source.key.id())
      if posts:
        return posts[0]
      logging.warning('Source post %s not found', id)
    except Exception as e:
      util.interpret_http_exception(e)

  def post(self, *path_args):
    logging.debug('Params: %s', self.request.params)

    key = self.request.params['source_key']
    source = ndb.Key(urlsafe=key).get()
    if not source or source.status == 'disabled' or 'listen' not in source.features:
      logging.error('Source not found or disabled. Dropping task.')
      return
    logging.info('Source: %s %s, %s', source.label(), source.key.string_id(),
                 source.bridgy_url(self))

    last_polled = self.request.params['last_polled']
    if last_polled != source.last_polled.strftime(util.POLL_TASK_DATETIME_FORMAT):
      logging.warning('duplicate poll task! deferring to the other task.')
      return

    logging.info('Last poll: %s', self._last_poll_url(source))

    # mark this source as polling
    source.updates = {
      'poll_status': 'polling',
      'last_poll_attempt': util.now_fn(),
      'rate_limited': False,
    }
    source = models.Source.put_updates(source)

    source.updates = {}
    try:
      self.poll(source)
    except Exception, e:
      source.updates['poll_status'] = 'error'
      code, body = util.interpret_http_exception(e)
      if code == '401' or isinstance(e, models.DisableSource):
        # the user deauthorized the bridgy app, so disable this source.
        # let the task complete successfully so that it's not retried.
        logging.warning('Disabling source due to: %s' % e, exc_info=True)
        source.updates.update({
          'status': 'disabled',
          'poll_status': 'ok',
        })
        body = '%s\nLast poll: %s' % (source.bridgy_url(self),
                                      self._last_poll_url(source))
        if source.is_beta_user():
          util.email_me(subject='Bridgy: disabled %s' % source.label(), body=body)

      elif code in util.HTTP_RATE_LIMIT_CODES:
        logging.info('Rate limited. Marking as error and finishing. %s', e)
        source.updates['rate_limited'] = True
      elif ((code and int(code) / 100 == 5) or
            (code == '400' and isinstance(source, flickr.Flickr)) or
            util.is_connection_failure(e)):
        logging.error('API call failed. Marking as error and finishing. %s: %s\n%s',
                      code, body, e)
        self.abort(util.ERROR_HTTP_RETURN_CODE)
      else:
        raise

  def post(self):
    ia_start = util.oauth_starter(indieauth.StartHandler).to('/instagram/callback')(
      self.request, self.response)

    try:
      self.redirect(ia_start.redirect_url(me=util.get_required_param(self, 'user_url')))
    except Exception as e:
      if util.is_connection_failure(e) or util.interpret_http_exception(e)[0]:
        self.messages.add("Couldn't fetch your web site: %s" % e)
        return self.redirect('/')
      raise

    def create_comment(self, post_url, author_name, author_url, content):
        """Creates a new comment in the source silo.

    If the last part of the post URL is numeric, e.g. http://site/post/123999,
    it's used as the post id. Otherwise, we extract the last part of
    the path as the slug, e.g. http: / / site / post / the-slug,
    and look up the post id via the API.

    Args:
      post_url: string
      author_name: string
      author_url: string
      content: string

    Returns:
      JSON response dict with 'id' and other fields
    """
        auth_entity = self.auth_entity.get()
        logging.info("Determining WordPress.com post id for %s", post_url)

        # extract the post's slug and look up its post id
        path = urlparse.urlparse(post_url).path
        if path.endswith("/"):
            path = path[:-1]
        slug = path.split("/")[-1]
        try:
            post_id = int(slug)
        except ValueError:
            logging.info("Looking up post id for slug %s", slug)
            url = API_POST_SLUG_URL % (auth_entity.blog_id, slug)
            post_id = self.urlopen(auth_entity, url).get("ID")
            if not post_id:
                return self.error("Could not find post id")

        logging.info("Post id is %d", post_id)

        # create the comment
        url = API_CREATE_COMMENT_URL % (auth_entity.blog_id, post_id)
        content = u'<a href="%s">%s</a>: %s' % (author_url, author_name, content)
        data = {"content": content.encode("utf-8")}
        try:
            resp = self.urlopen(auth_entity, url, data=urllib.urlencode(data))
        except urllib2.HTTPError, e:
            code, body = util.interpret_http_exception(e)
            try:
                parsed = json.loads(body) if body else {}
                if (code == "400" and parsed.get("error") == "invalid_input") or (
                    code == "403" and parsed.get("message") == "Comments on this post are closed"
                ):
                    return parsed  # known error: https://github.com/snarfed/bridgy/issues/161
            except ValueError:
                pass  # fall through
            raise e

  def poll(self, source):
    """Actually runs the poll.

    Stores property names and values to update in source.updates.
    """
    if source.last_activities_etag or source.last_activity_id:
      logging.debug('Using ETag %s, last activity id %s',
                    source.last_activities_etag, source.last_activity_id)

    #
    # Step 1: fetch activities:
    # * posts by the user
    # * search all posts for the user's domain URLs to find links
    #
    cache = util.CacheDict()
    if source.last_activities_cache_json:
      cache.update(json.loads(source.last_activities_cache_json))

    try:
      # search for links first so that the user's activities and responses
      # override them if they overlap
      links = source.search_for_links()

      # this user's own activities (and user mentions)
      resp = source.get_activities_response(
        fetch_replies=True, fetch_likes=True, fetch_shares=True,
        fetch_mentions=True, count=50, etag=source.last_activities_etag,
        min_id=source.last_activity_id, cache=cache)
      etag = resp.get('etag')  # used later
      user_activities = resp.get('items', [])

      # these map ids to AS objects
      responses = {a['id']: a for a in links}
      activities = {a['id']: a for a in links + user_activities}

    except Exception, e:
      code, body = util.interpret_http_exception(e)
      if code == '401':
        msg = 'Unauthorized error: %s' % e
        logging.warning(msg, exc_info=True)
        source.updates['poll_status'] = 'ok'
        raise models.DisableSource(msg)
      elif code in util.HTTP_RATE_LIMIT_CODES:
        logging.warning('Rate limited. Marking as error and finishing. %s', e)
        source.updates.update({'poll_status': 'error', 'rate_limited': True})
        return
      elif (code and int(code) / 100 == 5) or util.is_connection_failure(e):
        logging.error('API call failed. Marking as error and finishing. %s: %s\n%s',
                      code, body, e)
        self.abort(ERROR_HTTP_RETURN_CODE)
      else:
        raise

  def get(self):
    # https://cloud.google.com/appengine/docs/standard/python/ndb/admin#Metadata_queries
    kinds = [k for k in metadata.get_kinds() if not k.startswith('_')]
    kinds.remove('Response')
    kinds.remove('SyndicatedPost')
    logging.info('Backing up %s', kinds)

    access_token, _ = app_identity.get_access_token(
      'https://www.googleapis.com/auth/datastore')
    app_id = app_identity.get_application_id()

    request = {
        'project_id': app_id,
        'output_url_prefix': ('gs://brid-gy.appspot.com/weekly/' +
                              datetime.datetime.now().strftime('%Y%m%d')),
        'entity_filter': {
          'kinds': kinds,
          # 'namespace_ids': self.request.get_all('namespace_id'),
        },
    }
    headers = {
        'Content-Type': 'application/json',
        'Authorization': 'Bearer ' + access_token,
    }

    try:
      result = urlfetch.fetch(
          url='https://datastore.googleapis.com/v1/projects/%s:export' % app_id,
          payload=json.dumps(request),
          method=urlfetch.POST,
          headers=headers)
      if result.status_code == httplib.OK:
        logging.info(result.content)
      else:
        logging.error(result.content)
        self.abort(result.status_code)
    except urlfetch.Error as e:
      util.interpret_http_exception(e)
      raise

    def post(self, *path_args):
        logging.debug("Params: %s", self.request.params)

        key = self.request.params["source_key"]
        source = ndb.Key(urlsafe=key).get()
        if not source or source.status == "disabled" or "listen" not in source.features:
            logging.error("Source not found or disabled. Dropping task.")
            return
        logging.info("Source: %s %s, %s", source.label(), source.key.string_id(), source.bridgy_url(self))

        last_polled = self.request.params["last_polled"]
        if last_polled != source.last_polled.strftime(util.POLL_TASK_DATETIME_FORMAT):
            logging.warning("duplicate poll task! deferring to the other task.")
            return

        logging.info(
            "Last poll: %s/log?start_time=%s&key=%s",
            self.request.host_url,
            calendar.timegm(source.last_poll_attempt.utctimetuple()),
            source.key.urlsafe(),
        )

        # mark this source as polling
        source.updates = {"poll_status": "polling", "last_poll_attempt": util.now_fn()}
        source = models.Source.put_updates(source)

        source.updates = {}
        try:
            self.poll(source)
        except Exception, e:
            source.updates["poll_status"] = "error"
            code, body = util.interpret_http_exception(e)
            if code == "401" or isinstance(e, models.DisableSource):
                # the user deauthorized the bridgy app, so disable this source.
                # let the task complete successfully so that it's not retried.
                logging.warning("Disabling source due to: %s" % e, exc_info=True)
                source.updates.update({"status": "disabled", "poll_status": "ok"})
            elif code in util.HTTP_RATE_LIMIT_CODES:
                logging.warning("Rate limited. Marking as error and finishing. %s", e)
                source.updates["rate_limited"] = True
            elif (code and int(code) / 100 == 5) or util.is_connection_failure(e):
                logging.error("API call failed. Marking as error and finishing. %s: %s\n%s", code, body, e)
                self.abort(ERROR_HTTP_RETURN_CODE)
            else:
                raise

  def get(self):
    sources = {source.key.id(): source for source in Twitter.query()}
    if not sources:
      return

    # just auth as me or the first user. TODO: use app-only auth instead.
    auther = sources.get('schnarfed') or sources.values()[0]
    usernames = sources.keys()
    users = []
    for i in range(0, len(usernames), TWITTER_USERS_PER_LOOKUP):
      username_batch = usernames[i:i + TWITTER_USERS_PER_LOOKUP]
      url = TWITTER_API_USER_LOOKUP % ','.join(username_batch)
      try:
        users += auther.gr_source.urlopen(url)
      except Exception, e:
        code, body = util.interpret_http_exception(e)
        if not (code == '404' and len(username_batch) == 1):
          # 404 for a single user means they deleted their account. otherwise...
          raise

  def get_post(self, post_id, source_fn=None):
    """Utility method fetches the original post
    Args:
      post_id: string, site-specific post id
      source_fn: optional reference to a Source method,
        defaults to Source.get_post.

    Returns: ActivityStreams object dict
    """
    try:
      post = (source_fn or self.source.get_post)(post_id)
      if not post:
        logging.warning('Source post %s not found', post_id)
      return post
    except Exception, e:
      # use interpret_http_exception to log HTTP errors
      if not util.interpret_http_exception(e)[0]:
        logging.warning(
          'Error fetching source post %s', post_id, exc_info=True)

  def get(self, type, source_short_name, string_id, *ids):
    source_cls = models.sources.get(source_short_name)
    if not source_cls:
      self.abort(400, "Source type '%s' not found. Known sources: %s" %
                 (source_short_name, filter(None, models.sources.keys())))

    self.source = source_cls.get_by_id(string_id)
    if not self.source:
      self.abort(400, 'Source %s %s not found' % (source_short_name, string_id))

    format = self.request.get('format', 'html')
    if format not in ('html', 'json'):
      self.abort(400, 'Invalid format %s, expected html or json' % format)

    for id in ids:
      if not self.VALID_ID.match(id):
        self.abort(404, 'Invalid id %s' % id)

    label = '%s:%s %s %s' % (source_short_name, string_id, type, ids)
    cache_key = 'H ' + label
    obj = memcache.get(cache_key)
    if obj:
      logging.info('Using cached object for %s', label)
    else:
      logging.info('Fetching %s', label)
      try:
        obj = self.get_item(*ids)
      except Exception, e:
        # pass through all API HTTP errors if we can identify them
        code, body = util.interpret_http_exception(e)
        if not code and util.is_connection_failure(e):
          code = 503
          body = str(e)
        if code:
          self.response.status_int = int(code)
          self.response.headers['Content-Type'] = 'text/plain'
          self.response.write('%s error:\n%s' % (self.source.GR_CLASS.NAME, body))
          return
        else:
          raise
      memcache.set(cache_key, obj, time=CACHE_TIME)

  def get_site_info(cls, handler, auth_entity):
    """Fetches the site info from the API.

    Args:
      handler: the current RequestHandler
      auth_entity: oauth_dropins.wordpress.WordPressAuth

    Returns: site info dict, or None if API calls are disabled for this blog
    """
    try:
      return cls.urlopen(auth_entity, API_SITE_URL % auth_entity.blog_id)
    except urllib2.HTTPError, e:
      code, body = util.interpret_http_exception(e)
      if (code == '403' and '"API calls to this blog have been disabled."' in body):
        handler.messages.add(
          'You need to <a href="http://jetpack.me/support/json-api/">enable '
          'the Jetpack JSON API</a> in %s\'s WordPress admin console.' %
          util.pretty_link(auth_entity.blog_url))
        handler.redirect('/')
        return None
      raise