def clean_url(self):
     url = self.cleaned_data.get('url')
     if not url: return ''
     filename, headers = urllib.urlretrieve(url)
     if not mimetypes.guess_all_extensions(headers.get('Content-Type')):
         raise forms.ValidationError(_('The file type is invalid: %s' % type))
     return SimpleUploadedFile(filename, open(filename, 'rb').read(), content_type=headers.get('Content-Type'))

 def guess_all_extensions(self, content_type):
   l = mimetypes.guess_all_extensions(content_type)
   try:
     l.append(extensions[content_type])
   except:
     pass
   return l

  def getAllowedConversionFormatList(source_mimetype):
    """Returns a list content_type and their titles which are supported
    by enabled handlers.

    [('application/vnd.oasis.opendocument.text', 'ODF Text Document'),
     ('application/pdf', 'PDF - Portable Document Format'),
     ...
    ]
    """
    # XXX please never guess extension from mimetype
    output_set = set()
    if "/" in source_mimetype:
      parsed_mimetype_type = parseContentType(source_mimetype).gettype()
      # here `guess_all_extensions` never handles mimetype parameters
      #   (even for `text/plain;charset=UTF-8` which is standard)
      extension_list = mimetypes.guess_all_extensions(parsed_mimetype_type)  # XXX never guess
    else:
      extension_list = [source_mimetype]

    for ext in extension_list:
      for ext, title in mimemapper.getAllowedExtensionList(extension=ext.replace(".", "")):
        if ext in ("fodt", ".fodt"):  # BBB
          output_set.add(("application/vnd.oasis.opendocument.text-flat-xml", title))
          continue
        if ext:
          mimetype, _ = mimetypes.guess_type("a." + ext)  # XXX never guess
          if mimetype:
            output_set.add((mimetype, title))
    return list(output_set)

def valid_mime_extensions(request):
    """
    Returns the valid mime-types as well as the file extension for each.
    """
    if 'lmkp.file_mime_extensions' in request.registry.settings:
        fme = request.registry.settings['lmkp.file_mime_extensions']

        # Create a new dict which contains only the entries recognized as valid
        # mime types by python's own mimetypes module.
        vfme = {}
        for mt in fme:
            # Make sure that the mime type defined in the ini is valid.
            try:
                mimetypes.types_map[fme[mt]]
            except KeyError:
                continue

            # Make sure that the extension defined in the ini is valid for its
            # mime type
            if fme[mt] not in mimetypes.guess_all_extensions(mt):
                continue

            # Copy it
            vfme[mt] = fme[mt]

        # Add special types by Internet Explorer
        # http://msdn.microsoft.com/en-us/library/ms775147%28v=vs.85%29.aspx#_replace
        if 'image/jpeg' in vfme:
            vfme['image/pjpeg'] = '.jpg'
        if 'image/png' in vfme:
            vfme['image/x-png'] = '.png'

        return vfme

    return {}

 def getAllowedExtensionList(self, request_dict={}):
   """List types which can be generated from given type
   Type can be given as:
     - filename extension
     - document type ('text', 'spreadsheet', 'presentation' or 'drawing')
   e.g
   self.getAllowedMimetypeList(dict(document_type="text"))
   return extension_list
   """
   mimetype = request_dict.get('mimetype')
   extension = request_dict.get('extension')
   document_type = request_dict.get('document_type')
   if mimetype:
     allowed_extension_list = []
     for ext in guess_all_extensions(mimetype):
       ext = ext.replace('.', '')
       extension_list = mimemapper.getAllowedExtensionList(extension=ext,
                                                document_type=document_type)
       for extension in extension_list:
         if extension not in allowed_extension_list:
           allowed_extension_list.append(extension)
     return allowed_extension_list
   elif extension:
     extension = extension.replace('.', '')
     return mimemapper.getAllowedExtensionList(extension=extension,
                                                document_type=document_type)
   elif document_type:
     return mimemapper.getAllowedExtensionList(document_type=document_type)
   else:
     return [('', '')]

    def url2path(self):
        """
        computes a unique filename based on the url and the consumed tasks list

        It automatically adds the file extension based on mimetype.
        """
        lock_id = get_lock_id(url=self.url, pipeline=self.consumed)
        fn = md5(lock_id).hexdigest() # no longer used
        local_path = self.url
        if len(self.consumed) > 0:
            local_path += u"..%s" % '..'.join(self.consumed)
        # For the final filename, we want the requested extension,
        # so that it gets saved on a predictable location
        if len(self.to_go) == 0:
            if len(self.consumed) == 0:
                pass
            elif self.target_ext:
                ext = self.target_ext
                # Yet we do check if the mimetype of the produced result fits with the
                # requested extension:
                if ext.lower() not in mimetypes.guess_all_extensions(self.mime, strict=False):
                    raise TypeError
                local_path += ext
        else:
            ext = mimetypes.guess_extension(self.mime, strict=False)
            local_path += ext
        print "%d steps to go in pipeline" % len(self.to_go)
        local_folder, local_filename = os.path.split(local_path)
        if not os.path.exists(os.path.join(CACHE_PATH,local_folder)):
            os.makedirs(os.path.join(CACHE_PATH,local_folder))
        return os.path.join(CACHE_PATH, local_path)

    def mirror_entity_image(self, tweet, entity_index, url):
        response = requests.get(url)
        if response.status_code != httplib.OK:
            log.warn("Failed to download image {0}", url)
            return
        content_type = response.headers.get('content-type')

        parsed_url = urlparse.urlparse(url)
        (_base, extension) = os.path.splitext(parsed_url.path)
        extension = None
        if not extension:
            extensions = [ext for ext in mimetypes.guess_all_extensions(content_type)
                          if ext != '.jpe']
            extension = extensions[0] if extensions else ''
            log.debug("Possible mime types: {0}, chose {1}", extensions, extension)
        filename = "{tweet}-{index}{extension}".format(tweet=tweet.get('id'),
                                                       index=entity_index,
                                                       extension=extension)

        with NamedTemporaryFile(mode='wb', prefix='twoops', delete=True) as fil:
            fil.write(response.content)
            fil.flush()
            new_url = self.upload_image(fil.name, filename, content_type)
            if new_url:
                self.record_tweet_image(tweet, new_url)

def create_url_filename(url_str, content_type):
	# See also: http://stackoverflow.com/a/7406369/1391325
	split_url = urlsplit(url_str)
	netloc = split_url[1]
	netloc_dirname = os.path.sep.join(reversed(netloc.split('.')))
	path = split_url[2]
	stripped_url_str = "".join((netloc_dirname, path))
	url_without_ext, existing_ext = os.path.splitext(stripped_url_str)
	filename_without_ext = url_without_ext.translate(URL_FILENAME_TRANSLATION_TABLE)
	if filename_without_ext.endswith(os.path.sep):
		filename_without_ext = filename_without_ext[:-len(os.path.sep)]
	if existing_ext:
		acceptable_filename_exts = mimetypes.guess_all_extensions(content_type)
		if existing_ext in acceptable_filename_exts:
			# Re-concatenate the now-normalized filename base with the original extension
			result = filename_without_ext + existing_ext
		else:
			canonical_ext = mimetypes.guess_extension(content_type)
			if canonical_ext:
				# If a canonical extension was found for the given content type, concatenate it to the now-normalized filename base
				result = filename_without_ext + canonical_ext
			else:
				# If no canonical extension was found, re-concatenate the original extension after normalizing it
				normalized_existing_ext = normalize_url_component(existing_ext, ".")
				result = filename_without_ext + normalized_existing_ext
	else:
		# Concatenate the canonical extension for the given content type to the result filename in order to avoid potential clashes with other URLs
		canonical_ext = mimetypes.guess_extension(content_type)
		if canonical_ext:
			result = filename_without_ext + canonical_ext
		else:
			# Just add some extention
			result = filename_without_ext + DEFAULT_OUTPATH_SUFFIX
	
	return result

def get_type_extension(content_type):
    filetypes = set(mimetypes.guess_all_extensions(content_type)) - AUTOCALCULATED_FILENAME_EXTENSION_BLACKLIST

    if len(filetypes) > 0:
        return sorted(list(filetypes))[0]
    else:
        return None

 def test_as_message_attachments(self):
     msg_in = Message()
     msg_in["From"] = "[email protected]"
     msg_in["Message-ID"] = "<msg>"
     msg_in.attach(MIMEText("Dummy message"))
     msg_in.attach(MIMEText("<html><body>Dummy message</body></html>", _subtype="html"))
     add_to_list("[email protected]", msg_in)
     email = Email.objects.get(message_id="msg")
     msg = email.as_message()
     self.assertEqual(msg["From"], "dummy at example.com")
     self.assertEqual(msg["Message-ID"], "<msg>")
     self.assertTrue(msg.is_multipart())
     payload = msg.get_payload()
     self.assertEqual(len(payload), 2)
     self.assertEqual(
         payload[0].get_payload(decode=True).strip(), "Dummy message")
     # The filename extension detection from content type is a bit random
     # (depends on the PYTHON_HASHSEED), make sure we get the right one
     # here for testing.
     expected_ext = guess_all_extensions("text/html", strict=False)[0]
     self.assertEqual(payload[1].get_content_type(), "text/html")
     self.assertEqual(payload[1]["Content-Disposition"],
         'attachment; filename="attachment%s"' % expected_ext)
     self.assertEqual(
         payload[1].get_payload(decode=True),
         "<html><body>Dummy message</body></html>")

def uploads_endpoint(request):
    """ Endpoint for file uploads """
    username = request.matchdict["username"]
    requested_user = User.query.filter_by(username=username).first()

    if requested_user is None:
        return json_error("No such 'user' with id '{0}'".format(username), 404)

    if request.method == "POST":
        # Ensure that the user is only able to upload to their own
        # upload endpoint.
        if requested_user.id != request.user.id:
            return json_error("Not able to post to another users feed.", status=403)

        # Wrap the data in the werkzeug file wrapper
        if "Content-Type" not in request.headers:
            return json_error("Must supply 'Content-Type' header to upload media.")

        mimetype = request.headers["Content-Type"]
        filename = mimetypes.guess_all_extensions(mimetype)
        filename = "unknown" + filename[0] if filename else filename
        file_data = FileStorage(stream=io.BytesIO(request.data), filename=filename, content_type=mimetype)

        # Find media manager
        entry = new_upload_entry(request.user)
        entry.media_type = IMAGE_MEDIA_TYPE
        return api_upload_request(request, file_data, entry)

    return json_error("Not yet implemented", 501)

    def __init__(self, fname=None):
        super(LogProvider, self)

        self.minimum_occurrences = 250
        self.percentage = 10
        self.logs = list()
        self.filters = collections.defaultdict(list)
        self.negative_filters = collections.defaultdict(list)
        self.filters_regexp = collections.defaultdict(list)
        self.negative_filters_regexp = collections.defaultdict(list)
        self.fname = fname

        try:
            ftype = mimetypes.guess_all_extensions(self.fname)[0]
        except AttributeError:  # `self.fname` is None
            self.__transform_logs(fileinput.input("-"))
        except IndexError:  # `self.fname` has no guessable mimtype
            self.__transform_logs(fileinput.input(self.fname))
        else:
            if ftype == 'application/zip':  # zip file!
                with zipfile.ZipFile(self.fname) as f:
                    for name in f.namelist():
                        self.__transform_logs(f.read(name))
            elif ftype == 'application/tar':  # tar file!
                with tarfile.open(self.fname) as f:
                    for name in f.namelist():
                        self.__transform_logs(f.read(name))

    def generate_filename(self, instance, filename):
        if not self.random_filename:
            return super(WebDAVMixin, self).generate_filename(instance, filename)
        uuid_string = unicode(uuid.uuid4())
        file = getattr(instance, self.attname)
        if hasattr(file._file, 'content_type') and file._file.content_type in self.valid_content_types:
            content_type = file._file.content_type
        else:
            try:
                file._file.seek(0)
                if self.custom_magic_file:
                    content_type = magic.Magic(mime=True,
                                               magic_file=self.custom_magic_file).from_buffer(file._file.read(1024))
                else:
                    content_type = magic.from_buffer(file._file.read(1024), mime=True)
            except TypeError as e:
                content_type = 'application/x-unknown'

        #Receiving all extensions and checking if file extension matches MIME Type
        extensions = mimetypes.guess_all_extensions(content_type)
        try:
            file_ext = re.findall(r'\.[^.]+$', filename)[0]
        except IndexError:
            file_ext = None
        if file_ext in extensions:
            ext = file_ext
        elif extensions:
            ext = extensions[0]
        else:
            ext = '.bin'

        return os.path.join(self.upload_to, uuid_string[:2], uuid_string[2:4], '%s%s' % (uuid_string, ext))

def extension_correct_for_mimetype(extension, mimetype):
    """
    Check if the given filename extension (e.g. ".ogg") is a possible
    extension for a given mimetype (e.g. "application/ogg") and return
    a boolean value (True if it's possible, False if not). Also do

    >>> extension_correct_for_mimetype('.ogg', 'application/ogg')
    True
    >>> extension_correct_for_mimetype('.ogv', 'video/ogg')
    True
    >>> extension_correct_for_mimetype('.ogg', 'audio/mpeg')
    False
    >>> extension_correct_for_mimetype('mp3', 'audio/mpeg')
    Traceback (most recent call last):
      ...
    ValueError: "mp3" is not an extension (missing .)
    >>> extension_correct_for_mimetype('.mp3', 'audio mpeg')
    Traceback (most recent call last):
      ...
    ValueError: "audio mpeg" is not a mimetype (missing /)
    """
    if not '/' in mimetype:
        raise ValueError('"%s" is not a mimetype (missing /)' % mimetype)
    if not extension.startswith('.'):
        raise ValueError('"%s" is not an extension (missing .)' % extension)

    # Create a "default" extension from the mimetype, e.g. "application/ogg"
    # becomes ".ogg", "audio/mpeg" becomes ".mpeg", etc...
    default = ['.'+mimetype.split('/')[-1]]

    return extension in default+mimetypes.guess_all_extensions(mimetype)

def random_extension(*args, **kwargs):
    # mimetypes.guess_extension can return any of the values in
    # mimetypes.guess_all_extensions.  it depends on the system.
    # we're using this to make sure our code is robust enough to handle the
    # different possible extensions
    exts = mimetypes.guess_all_extensions(*args, **kwargs)
    return random.choice(exts)

    def save( self, force_insert=False, force_update=False, using=None ):
        if not self.image and self.image_url:
            filename, headers = urllib.urlretrieve( self.image_url.encode( 'utf-8' ) )
            type = headers.get( 'Content-Type' )
            if not type or not mimetypes.guess_all_extensions( type ):
                raise ValidationError( 'Broken image' )

            self.image = SimpleUploadedFile( filename, open( filename ).read(), content_type = type )
            self.image_url = None

        super( Demot, self ).save( force_insert, force_update, using )
        
        # generate demot
        if self.image and not self.final_image:
            demotivator = Demotivator( self.image, self.line1, self.line2, get_address() )
            demotivator.create( self.demot_path['path'] )
            
            # generate default thumbnail
            self.get_thumbnail()
            
            self.final_image = self.demot_path['url']
            self.save()
        
            # create/update activity
            self.save_activity()

 def post(req):
     mime = req.headers['Content-type']
     candidateext = guess_all_extensions(mime)
     ext = candidateext[0] if len(candidateext) > 0 else ""
     filename = 'file' + ext
     fileobj = BytesIO(req.data)
     return postImgRespondUrl(filename, fileobj)

def parse_render(render):
    """Parse render URL parameter.

    >>> parse_render(None)
    'png'
    >>> parse_render('html')
    'png'
    >>> parse_render('png')
    'png'
    >>> parse_render('jpg')
    'jpeg'
    >>> parse_render('gif')
    'gif'
    """
    formats = {
        "jpeg": guess_all_extensions("image/jpeg"),
        "png": guess_all_extensions("image/png"),
        "gif": guess_all_extensions("image/gif"),
        "bmp": guess_all_extensions("image/x-ms-bmp"),
        "tiff": guess_all_extensions("image/tiff"),
        "xbm": guess_all_extensions("image/x-xbitmap"),
        "pdf": guess_all_extensions("application/pdf"),
    }
    if not render:
        render = "png"
    else:
        render = render.lower()
        for k, v in formats.items():
            if ".%s" % render in v:
                render = k
                break
        else:
            render = "png"
    return render

def parse_render(render):
    """Parse render URL parameter.

    >>> parse_render(None)
    'png'
    >>> parse_render('html')
    'png'
    >>> parse_render('png')
    'png'
    >>> parse_render('jpg')
    'jpeg'
    >>> parse_render('gif')
    'gif'
    """
    formats = {
        'jpeg': guess_all_extensions('image/jpeg'),
        'png': guess_all_extensions('image/png'),
        'gif': guess_all_extensions('image/gif'),
        'bmp': guess_all_extensions('image/x-ms-bmp'),
        'tiff': guess_all_extensions('image/tiff'),
        'xbm': guess_all_extensions('image/x-xbitmap'),
        'pdf': guess_all_extensions('application/pdf')
    }
    if not render:
        render = 'png'
    else:
        render = render.lower()
        for k, v in formats.items():
            if '.%s' % render in v:
                render = k
                break
        else:
            render = 'png'
    return render

 def _extension(self, mime_type=DEFAULT_EXT):
     """ Get the common-law file extension for a given MIME type."""
     exts = mimetypes.guess_all_extensions(mime_type)
     if '.jpe' in exts:
         exts.remove('.jpe') # WHO USES THAT.
     ext = bool(exts) and \
         exts[0] or \
         URLRetrievalStorage.DEFAULT_EXT
     return ext