def _http_get_json(self, url):
        """
        Make an HTTP GET request to the specified URL, check that it returned a
        JSON response, and returned the data parsed from that response.

        Parameters
        ----------
        url
            The URL to GET.

        Returns
        -------
        Dictionary of data parsed from a JSON HTTP response.

        Exceptions
        ----------
        * PythonKCMeetupsBadJson
        * PythonKCMeetupsBadResponse
        * PythonKCMeetupsMeetupDown
        * PythonKCMeetupsNotJson
        * PythonKCMeetupsRateLimitExceeded

        """
        response = self._http_get(url)

        content_type = response.headers['content-type']
        parsed_mimetype = mimeparse.parse_mime_type(content_type)
        if parsed_mimetype[1] not in ('json', 'javascript'):
            raise PythonKCMeetupsNotJson(content_type)

        try:
            return json.loads(response.content)
        except ValueError as e:
            raise PythonKCMeetupsBadJson(e)

    def require_representation(self, req):
        """Require raw representation dictionary from falcon request object.

        This does not perform any field parsing or validation but only uses
        allowed content-encoding handler to decode content body.

        Note:
            Currently only JSON is allowed as content type.

        Args:
            req (falcon.Request): request object

        Returns:
            dict: raw dictionary of representation supplied in request body

        """
        try:
            type_, subtype, _ = parse_mime_type(req.content_type)
            content_type = '/'.join((type_, subtype))
        except:
            raise falcon.HTTPUnsupportedMediaType(
                description="Invalid Content-Type header: {}".format(
                    req.content_type
                )
            )

        if content_type == 'application/json':
            body = req.stream.read()
            return json.loads(body.decode('utf-8'))
        else:
            raise falcon.HTTPUnsupportedMediaType(
                description="only JSON supported, got: {}".format(content_type)
            )

    def for_type(cls, attachment):
        """Return the handler that is the best fit for provided mimetype."""
        if not attachment.mimetype:
            return None

        try:
            mimetype = mimeparse.parse_mime_type(attachment.mimetype)
        except:
            logging.warning('Unable to parse MIME type "%s" for %s',
                            attachment, attachment.mimetype)
            mimetype = ('application', 'octet-stream', {})

        # Override the mimetype if mimeparse is known to misinterpret this
        # type of file as `octet-stream`
        extension = os.path.splitext(attachment.filename)[1]

        if extension in MIMETYPE_EXTENSIONS:
            mimetype = MIMETYPE_EXTENSIONS[extension]

        score, handler = cls.get_best_handler(mimetype)

        if handler:
            try:
                return handler(attachment, mimetype)
            except Exception as e:
                logging.error('Unable to load Mimetype Handler for %s: %s',
                              attachment, e)

        return MimetypeHandler(attachment, mimetype)

def get_link_content(link):
    try:
        response = requests.get(link)
        if response.status_code == 400:
            logging.warn(u"404 {}".format(link))
            return None
        if response.status_code != 200:
            raise Exception(u"Unable to fetch release content: {0}".format(link))
    except requests.exceptions.InvalidURL as e:
        logging.warn(u"Invalid link {0}: {1}".format(link, unicode(e)))
        return None

    content_type = response.headers.get('content-type')
    if not content_type:
        logging.warn(u"Response did not contain a Content-Type header: {0}".format(link))
        return None

    (mime_type, mime_subtype, mt_params) = parse_mime_type(content_type)
    if mime_type != 'text' or mime_subtype not in ('html', 'xhtml'):
        logging.warn(u"Skipping non-HTML link: {0}".format(link))
        return None

    if len(response.content) == 0:
        logging.warn(u"Server returned an empty body: {0}".format(link))
        return None

    (title, body) = readability_extract(response.content)
    return kill_control_characters(body)

    def get_best_handler(cls, mimetype):
        """Return the handler and score that that best fit the mimetype.

        Args:
            mimetype (unicode):
                The mimetype to find the best handler for.

        Returns:
            tuple:
            A tuple of ``(best_score, mimetype_handler)``. If no handler
            was found, this will be ``(0, None)``.
        """
        best_score, best_fit = (0, None)

        for mimetype_handler in _registered_mimetype_handlers:
            for mt in mimetype_handler.supported_mimetypes:
                try:
                    score = score_match(mimeparse.parse_mime_type(mt),
                                        mimetype)

                    if score > best_score:
                        best_score, best_fit = (score, mimetype_handler)
                except ValueError:
                    continue

        return (best_score, best_fit)

    def for_type(cls, attachment):
        """Returns the handler that is the best fit for provided mimetype."""
        if attachment.mimetype:
            try:
                mimetype = mimeparse.parse_mime_type(attachment.mimetype)
            except:
                logging.error('Unable to parse MIME type "%s" for %s',
                              attachment.mimetype, attachment)
                return None

            # Override the mimetype if mimeparse is known to misinterpret this
            # type of file as 'octet-stream'
            extension = os.path.splitext(attachment.filename)[1]

            if extension in MIMETYPE_EXTENSIONS:
                mimetype = MIMETYPE_EXTENSIONS[extension]

            score, handler = cls.get_best_handler(mimetype)

            if handler:
                try:
                    return handler(attachment.get_review_request(), attachment)
                except ObjectDoesNotExist as e:
                    logging.error('Unable to load review UI for %s: %s',
                                  attachment, e)
                except Exception as e:
                    logging.error('Error instantiating '
                                  'FileAttachmentReviewUI %r: %s',
                                  handler, e)

        return None

    def serialize(self, obj, accept=None, **opts):
        '''serialize(obj) -> content, content_type

        Serialize an object to text.
        '''
        accept = accept or self.default_content_type
        content_type, format = self.get_format(accept)
        method = getattr(self, 'to_%s' % format)

        # ugly hack to get the params to the header part out
        try:
            accept = [
                part
                for part in accept.split(',')
                if part.startswith(content_type)
            ][0]
        except IndexError:
            # '*/*' case
            accept = content_type

        params = mimeparse.parse_mime_type(accept)[2]
        for key, value in params.items():
            opts.setdefault(key, value)

        return self.SerializedContainer(method(obj, **opts), content_type)

    def get_best_handler(cls, mimetype):
        """Return the Review UI and score that that best fit the mimetype.

        Args:
            mimetype (unicode):
                The mimetype to find a Review UI for.

        Returns:
            tuple:
            A tuple of ``(best_score, review_ui)``, or ``(0, None)`` if one
            could not be found.
        """
        best_score = 0
        best_fit = None

        for review_ui in _file_attachment_review_uis:
            for mt in review_ui.supported_mimetypes:
                try:
                    score = score_match(mimeparse.parse_mime_type(mt),
                                        mimetype)

                    if score > best_score:
                        best_score = score
                        best_fit = review_ui
                except ValueError:
                    continue

        return best_score, best_fit

    def get_best_handler(cls, mimetype):
        """Return the handler and score that that best fit the mimetype.

        Args:
            mimetype (tuple):
                A parsed mimetype to find the best handler for. This is a
                3-tuple of the type, subtype, and parameters as returned by
                :py:func:`mimeparse.parse_mime_type`.

        Returns:
            tuple:
            A tuple of ``(best_score, mimetype_handler)``. If no handler
            was found, this will be ``(0, None)``.
        """
        best_score, best_fit = (0, None)

        for mimetype_handler in _registered_mimetype_handlers:
            for mt in mimetype_handler.supported_mimetypes:
                try:
                    score = score_match(mimeparse.parse_mime_type(mt),
                                        mimetype)

                    if score > best_score:
                        best_score, best_fit = (score, mimetype_handler)
                except ValueError:
                    continue

        return (best_score, best_fit)

 def _test_parse_mime_type(self, args, expected):
     if expected is None:
         self.assertRaises(mimeparse.MimeTypeParseException, mimeparse.parse_mime_type, args)
     else:
         expected = tuple(expected)
         result = mimeparse.parse_mime_type(args)
         message = "Expected: '%s' but got %s" % (expected, result)
         self.assertEqual(expected, result, message)

  def __init__(self, http, postproc, uri,
               method='GET',
               body=None,
               headers=None,
               methodId=None,
               resumable=None):
    """Constructor for an HttpRequest.

    Args:
      http: httplib2.Http, the transport object to use to make a request
      postproc: callable, called on the HTTP response and content to transform
                it into a data object before returning, or raising an exception
                on an error.
      uri: string, the absolute URI to send the request to
      method: string, the HTTP method to use
      body: string, the request body of the HTTP request,
      headers: dict, the HTTP request headers
      methodId: string, a unique identifier for the API method being called.
      resumable: MediaUpload, None if this is not a resumbale request.
    """
    self.uri = uri
    self.method = method
    self.body = body
    self.headers = headers or {}
    self.methodId = methodId
    self.http = http
    self.postproc = postproc
    self.resumable = resumable

    # Pull the multipart boundary out of the content-type header.
    major, minor, params = mimeparse.parse_mime_type(
        headers.get('content-type', 'application/json'))

    # Terminating multipart boundary get a trailing '--' appended.
    self.multipart_boundary = params.get('boundary', '').strip('"') + '--'

    # If this was a multipart resumable, the size of the non-media part.
    self.multipart_size = 0

    # The resumable URI to send chunks to.
    self.resumable_uri = None

    # The bytes that have been uploaded.
    self.resumable_progress = 0

    self.total_size = 0

    if resumable is not None:
      if self.body is not None:
        self.multipart_size = len(self.body)
      else:
        self.multipart_size = 0
      self.total_size = (
          self.resumable.size() +
          self.multipart_size +
          len(self.multipart_boundary))

    def for_type(cls, attachment):
        """Returns the handler that is the best fit for provided mimetype."""
        mimetype = mimeparse.parse_mime_type(attachment.mimetype)
        score, handler = cls.get_best_handler(mimetype)

        if handler:
            try:
                return handler(attachment.get_review_request(), attachment)
            except Exception, e:
                logging.error('Unable to load review UI for %s: %s',
                              attachment, e, exc_info=1)

    def _http_get_json(self, url):
        response = self._http_get(url)

        content_type = response.headers['content-type']
        parsed_mimetype = mimeparse.parse_mime_type(content_type)
        if parsed_mimetype[1] not in ('json', 'javascript'):
            raise MeetupsNotJson(content_type)

        try:
            return json.loads(response.content)
        except ValueError as e:
            raise MeetupsBadJson(e)

    def import_submission(self, submission: praw.objects.Submission) -> dict:
        """ Import a submission from flickr. Uses their oEmbed API.

        flickr.com was nice enough to provide us with an oEmbed API.
        Apparently these guys also support video, so we should also make sure
        to not try to parse that.

        This function will define the following values in its return data:
        - author: simply "a flickr.com user"
        - source: The url of the submission
        - importer_display/header
        - import_urls

        :param submission: A reddit submission to parse.
        """
        try:
            if not self.regex.match(urlsplit(submission.url).netloc):
                return None
            url = submission.url
            data = {'author': 'a flickr.com user',
                    'source': url,
                    'importer_display':
                        {'header': 'Imported flickr.com image:\n\n'}}
            r = requests.head(url, headers=self.headers)
            if r.status_code == 301:
                return None

            mime_text = r.headers.get('Content-Type')
            mime = mimeparse.parse_mime_type(mime_text)
            # If we're already given an image...
            if mime[0] == 'image':
                # Use the already given URL
                image_url = submission.url
            else:
                # Otherwise, find the image in the html
                 self.log.info("Getting submission.url: " + url)
                 html = urllib.request.urlopen(url).read().decode('utf-8')
                 image_urls = re.findall(r'farm[\d]\.[a-z0-9/.\\/_]*', html)
                 if image_urls:
                     image_url = 'http://' + image_urls[-1].replace('\\', '')
                     self.log.info("Got image url %s", image_url)
                 else:
                     self.log.error('Could not find any flickr URL %s', submission.url)
                     return None
                 
            assert image_url
            data['import_urls'] = [image_url]
            return data
        except Exception:
            self.log.error('Could not import flickr URL %s (%s)',
                           submission.url, traceback.format_exc())
            return None

def parse_content_type(contenttype):
    mime_type = mimeparse.parse_mime_type(contenttype)
    
    if "charset" in mime_type[2]:
        # Remove charset from mime_type, if we have it
        encoding = mime_type[2].pop("charset")
    else:
        encoding = None
    
    if encoding == 'x-ctext':
        encoding = 'latin1'
    
    return mime_type, encoding

    def import_submission(self, submission: praw.objects.Submission) -> dict:
        """Import a submission from drawcrowd. Uses raw HTML scraping.

        As it turns out, drawcrowd likes to provide different data
        (all in <meta> tags) to non-web-browser requests.
        Since it provides enough information anyways, we don't bother getting
        around it and just parse that.

        This function will define the following values in its return data:
        - author: The author of the post
        - source: The url of the submission
        - importer_display/header
        - import_urls

        :param submission: A reddit submission to parse.
        """
        try:
            url = html.unescape(submission.url)
            if not self.regex.match(urlsplit(url).netloc):
                return None
            data = {'source': url}
            r = requests.head(url, headers=self.headers)
            if r.status_code == 301:  # Moved Permanently
                return None
            mime_text = r.headers.get('Content-Type')
            mime = mimeparse.parse_mime_type(mime_text)
            if mime[0] == 'image':
                data['author'] = 'An unknown drawcrowd user'
                image_url = url
            else:
                # Note: Drawcrowd provides different content to non-web-browsers.
                r = requests.get(url, headers=self.headers)
                bs = bs4.BeautifulSoup(r.content.decode('utf-8'))
                matched = bs.find(property='og:image')
                if not matched:
                    self.log.warning('Could not find locate drawcrowd image to scrape.')
                    return None
                image_url = matched['content']
                matched = bs.find(property='og:title')
                if matched:
                    data['author'] = matched['content']
                else:
                    data['author'] = 'an unknown drawcrowd author'
                data['importer_display'] = {'header': 'Mirrored image from {}:\n\n'.format(data['author'])}
            assert image_url
            data['import_urls'] = [image_url]
            return data
        except Exception:
            self.log.error('Could not import drawcrowd URL %s (%s)',
                           submission.url, traceback.format_exc())
            return None

 def status(
     self,
     test_id=None,
     test_status=None,
     test_tags=None,
     runnable=True,
     file_name=None,
     file_bytes=None,
     eof=False,
     mime_type=None,
     route_code=None,
     timestamp=None,
 ):
     super(Starts, self).status(
         test_id,
         test_status,
         test_tags=test_tags,
         runnable=runnable,
         file_name=file_name,
         file_bytes=file_bytes,
         eof=eof,
         mime_type=mime_type,
         route_code=route_code,
         timestamp=timestamp,
     )
     if not test_id:
         if not file_bytes:
             return
         if not mime_type or mime_type == "test/plain;charset=utf8":
             mime_type = "text/plain; charset=utf-8"
         primary, sub, parameters = mimeparse.parse_mime_type(mime_type)
         content_type = testtools.content_type.ContentType(primary, sub, parameters)
         content = testtools.content.Content(content_type, lambda: [file_bytes])
         text = content.as_text()
         if text and text[-1] not in "\r\n":
             self._neednewline = True
         self._output.write(text)
     elif test_status == "inprogress" and test_id not in self._emitted:
         if self._neednewline:
             self._neednewline = False
             self._output.write("\n")
         worker = ""
         for tag in test_tags or ():
             if tag.startswith("worker-"):
                 worker = "(" + tag[7:] + ") "
         if timestamp:
             timestr = timestamp.isoformat()
         else:
             timestr = ""
             self._output.write("%s: %s%s [start]\n" % (timestr, worker, test_id))
         self._emitted.add(test_id)

    def import_submission(self, submission: praw.objects.Submission) -> dict:
        """ Import a submission from gyazo. Uses their oEmbed API.

        gyazo.com was nice enough to provide us with an oEmbed API.
        Apparently these guys also support video, so we should also make sure
        to not try to parse that.

        This function will define the following values in its return data:
        - author: simply "a gyazo.com user"
        - source: The url of the submission
        - importer_display/header
        - import_urls

        :param submission: A reddit submission to parse.
        """
        try:
            if not self.regex.match(urlsplit(submission.url).netloc):
                return None
            data = {'author': 'a gyazo.com user',
                    'source': submission.url,
                    'importer_display':
                        {'header': 'Imported gyazo.com image:\n\n'}}
            r = requests.head(submission.url, headers=self.headers)
            if r.status_code == 301:
                return None

            mime_text = r.headers.get('Content-Type')
            mime = mimeparse.parse_mime_type(mime_text)
            # If we're already given an image...
            if mime[0] == 'image':
                # Use the already given URL
                image_url = submission.url
            else:
                # Otherwise, use the gyazo oEmbed API.
                response = requests.get(
                    'https://api.gyazo.com/api/oembed/',
                    {'url': submission.url},
                    headers=self.headers).json()
                if response.get('type') == 'photo':
                    image_url = response.get('url')
                else:
                    # This is something that is not a photo. Do not scrape.
                    return None

            assert image_url
            data['import_urls'] = [image_url]
            return data
        except Exception:
            self.log.error('Could not import gyazo URL %s (%s)',
                           submission.url, traceback.format_exc())
            return None

    def get_best_handler(cls, mimetype):
        """Returns the handler and score that that best fit the mimetype."""
        best_score, best_fit = (0, None)

        for mimetype_handler in _registered_mimetype_handlers:
            for mt in mimetype_handler.supported_mimetypes:
                try:
                    score = score_match(mimeparse.parse_mime_type(mt), mimetype)

                    if score > best_score:
                        best_score, best_fit = (score, mimetype_handler)
                except ValueError:
                    continue

        return (best_score, best_fit)

    def encoding(self):
        if self._encoding is not None:
            # Encoding has been set manually.
            return self._encoding

        # Get the `Content-Type` header, if available.
        content_type = self.headers.get('Content-Type')
        if content_type:
            # Parse out the primary type and parameters from the media type.
            ptype, _, params = mimeparse.parse_mime_type(content_type)

            # Return the specified charset or the default depending on the
            # primary type.
            default = 'utf-8' if ptype == 'application' else 'iso-8859-1'
            return params.get('charset', default)