def create_url_filename(url_str, content_type):
	# See also: http://stackoverflow.com/a/7406369/1391325
	split_url = urlsplit(url_str)
	netloc = split_url[1]
	netloc_dirname = os.path.sep.join(reversed(netloc.split('.')))
	path = split_url[2]
	stripped_url_str = "".join((netloc_dirname, path))
	url_without_ext, existing_ext = os.path.splitext(stripped_url_str)
	filename_without_ext = url_without_ext.translate(URL_FILENAME_TRANSLATION_TABLE)
	if filename_without_ext.endswith(os.path.sep):
		filename_without_ext = filename_without_ext[:-len(os.path.sep)]
	if existing_ext:
		acceptable_filename_exts = mimetypes.guess_all_extensions(content_type)
		if existing_ext in acceptable_filename_exts:
			# Re-concatenate the now-normalized filename base with the original extension
			result = filename_without_ext + existing_ext
		else:
			canonical_ext = mimetypes.guess_extension(content_type)
			if canonical_ext:
				# If a canonical extension was found for the given content type, concatenate it to the now-normalized filename base
				result = filename_without_ext + canonical_ext
			else:
				# If no canonical extension was found, re-concatenate the original extension after normalizing it
				normalized_existing_ext = normalize_url_component(existing_ext, ".")
				result = filename_without_ext + normalized_existing_ext
	else:
		# Concatenate the canonical extension for the given content type to the result filename in order to avoid potential clashes with other URLs
		canonical_ext = mimetypes.guess_extension(content_type)
		if canonical_ext:
			result = filename_without_ext + canonical_ext
		else:
			# Just add some extention
			result = filename_without_ext + DEFAULT_OUTPATH_SUFFIX
	
	return result

	def get_file_name_mime(self, url):
		pgctnt, hName, mime = self.wg.getFileNameMime(url)

		parsed = urllib.parse.urlparse(url)
		pathname = os.path.split(parsed.path)[-1]
		if not hName and not mime and not pathname:
			self.log.error("cannot figure out content type for url: %s", url)
			return pgctnt, "unknown.unknown", "application/octet-stream"

		# empty path with mimetype of text/html generally means it's a directory index (or some horrible dynamic shit).
		if not hName and not pathname and mime == "text/html":
			self.log.info("No path and root location. Assuming index.html")
			return pgctnt, "index.html", "text/html"

		ftype, guessed_mime = mimetypes.guess_type(hName)
		if ftype:
			return pgctnt, hName, guessed_mime if not mime else mime

		ftype, guessed_mime = mimetypes.guess_type(pathname)
		if ftype:
			return pgctnt, pathname, guessed_mime if not mime else mime

		chunks = [hName, pathname]
		chunks = [chunk for chunk in chunks if chunk]

		outname = " - ".join(chunks)
		if mime and mimetypes.guess_extension(mime):
			newext = mimetypes.guess_extension(mime)
		else:
			newext = ".unknown"

		if not outname:
			outname = "unknown"
		return pgctnt, outname+newext, mime if mime else "application/octet-stream"

def extract_data(tag):
    if type(tag) == mutagen.easymp4.EasyMP4:
        if 'cover' in tag:
            picture = tag['cover']
            exts = { picture.FORMAT_JPEG: '.jpeg', picture.FORMAT_PNG: '.png' }
            return ((picture, exts[picture.imageformat]), clean_keys(tag))
        else:
            return (None, clean_keys(tag))
        pass
    if type(tag) == mutagen.easyid3.EasyID3:
        if 'cover' in tag:
            picture = tag['cover']
            ext = mimetypes.guess_extension(picture.mime)
            return ((picture.data, ext), clean_keys(tag))
        else:
            return (None, clean_keys(tag))
        pass
    if type(tag) == mutagen.flac.FLAC:
        if tag.pictures:
            picture = tag.pictures[0]
            ext = mimetypes.guess_extension(picture.mime)
            return ((picture.data, ext), clean_keys(tag))
        else:
            return (None, clean_keys(tag))
    pass

def start_recording(name, queue):
    """ Starts stream recording """
    date_format = CONFIG.get(__SETTINGS_SECTION, "date_format")
    jar = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(jar))
    while True:
        schedule = queue.get()
        log.info("Start recording for %s"%schedule)
        end_time = datetime.strptime(schedule['end'],date_format)
        shutil.move(schedule['file'], INPROGRESS_DIR)
        schedule['file'] = os.path.join(INPROGRESS_DIR, schedule['file_name'])
        response = opener.open(schedule['url'])
        extension = ".mpeg"
        if 'Content-type' in response.info():
            extension = mimetypes.guess_extension(response.info()['Content-type'])
        elif 'mime' in schedule:
            extension = mimetypes.guess_extension(schedule['mime'])
        f = open(os.path.join(DOWNLOADS_DIR,schedule['name']+extension),"a+")
        try:
            while end_time>=datetime.now() and os.path.exists(schedule['file']):
                f.write(response.read(__BUFFER))
		f.flush()
            shutil.move(schedule['file'], COMPLETED_DIR)  
            log.info("Recording of %s is done"%schedule)
        except Exception as e:
            log.error("Failed to record %s. Will try to restart."%schedule,e)
            queue.put(schedule, True, __RETRY_INTERVAL)
            pass
        finally:
            f.close()
            response.close()

    def execute(self, transform_manager):
        filename, headers = retrieve(url=self.url,
                                     user=transform_manager.owner,
                                     username=self.username,
                                     password=self.password,
                                     user_agent=self.user_agent)

        try:
            if headers.get('error'):
                raise TransformException("Failed to download %s" % self.url)
            if not filename:
                raise TransformException(headers.get('message'))

            content_type = headers.get('content-type', 'unknown/unknown')
            content_type = content_type.split(';')[0].strip()
            extension = self.extension \
                     or self.mimetype_overrides.get(content_type) \
                     or (mimetypes.guess_extension(content_type) or '').lstrip('.') \
                     or (mimetypes.guess_extension(content_type, strict=False) or '').lstrip('.') \
                     or 'unknown'

            logger.debug("Response had content-type %r; assigning extension %r" % (content_type, extension))

            with open(transform_manager(extension, self.name), 'w') as output:
                transform_manager.start(self, [input], type='identity')
                with open(filename, 'r') as f:
                    shutil.copyfileobj(f, output)

                logger.info("File from %r saved to %r" % (self.url, output.name))
                return output.name
        finally:
            if headers['delete-after']:
                os.unlink(filename)

def write_html(client,xml_string,filename):
    tree = etree.fromstring(strip_ns(xml_string))
    elements_to_download = tree.xpath('//img[@src]') + tree.xpath('//object[starts-with(@data, "https://")]')
    if not elements_to_download:
        return serialise_html(xml_string,filename)

    serialise_html(xml_string,filename)

    filename_base = os.path.splitext(filename)[0]

    for external in elements_to_download:
        part_id = id_generator()
        if external.tag == 'img':
            if not os.path.exists(filename_base + '_images/'):
                os.makedirs(filename_base + '_images/')
            data = client.do_request(external.get('data-fullres-src'),raw=True)
            outfile =  os.path.join( filename_base + '_images/', part_id + mimetypes.guess_extension(external.get('data-src-type')))
            write_image(data,outfile)
            data = client.do_request(external.get('src'),raw=True)
            encoded = base64.b64encode(data)
            external.set('src', 'data:'+external.get('data-src-type')+';base64,'+encoded)
        if external.tag == 'object':
            if not os.path.exists(filename_base + '_attachments/'):
                os.makedirs(filename_base + '_attachments/')
            extension = mimetypes.guess_extension(external.get('type'))
            if ( external.get('type') == 'application/vnd.ms-excel' ):
                extension = ''
            outfile =  os.path.join( filename_base + '_attachments/', external.get('data-attachment') + extension)
            data = client.do_request(external.get('data'),raw=True)
            write_image(data,outfile)
            external.set('data' , 'file://'+os.path.abspath(outfile))

    ET.ElementTree(tree).write(filename,method="html")

    def _guess_destination(self, torrent_files):
        """
        try to identify the correct category of the finished torrent 
        and return the destination path where the torrent has to be moved
        """
        download_path = self.config["download_path"]

        for file in torrent_files:
            try:
                ext = os.path.splitext(file["path"])[1]
                ext = ext.lower()
                mt.guess_extension(ext)
                res = mt.types_map[ext]
                if res in GREY_LIST:
                    log.debug("skipping GREY_LIST extension %s", res)
                    continue
                if (res.startswith("audio")):
                    return [os.path.join(download_path, self.config["sub_audio"]), "audio"]
                elif (res.startswith("video")):
                    return [os.path.join(download_path, self.config["sub_video"]), "video"]
                elif(ext in DOC_FORMAT):
                    return [os.path.join(download_path, self.config["sub_documents"]), "doc"]
                elif(ext in DATA_FORMAT):
                    return [os.path.join(download_path, self.config["sub_data"]), "data"]

            except KeyError:
                    log.debug("unknown extension %s, trying again", ext)
                    continue
                    
        return [os.path.join(download_path, self.config["sub_uncat"]), "uncategorized"]

	def getImage(self, imageUrl, referrer):

		content, handle = self.wg.getpage(imageUrl, returnMultiple=True, addlHeaders={'Referer': referrer})
		if not content or not handle:
			raise ValueError("Failed to retreive image from page '%s'!" % referrer)

		fileN = urllib.parse.unquote(urllib.parse.urlparse(handle.geturl())[2].split("/")[-1])
		fileN = bs4.UnicodeDammit(fileN).unicode_markup
		self.log.info("retreived image '%s' with a size of %0.3f K", fileN, len(content)/1000.0)

		if not "." in fileN:
			info = handle.info()
			if 'Content-Type' in info:
				tp = info['Content-Type']
				if ";" in tp:
					tp = tp.split(";")[0]
				ext = guess_extension(tp)
				if ext == None:
					ext = "unknown_ftype"
				print(info['Content-Type'], ext)
				fileN += "." + ext
			else:
				fileN += ".jpg"

		# Let magic figure out the files for us (it's probably smarter then kissmanga, anyways.)
		guessed = magic.from_buffer(content, mime=True)
		ext = guess_extension(tp)
		if ext:
			fileN = fileN + ext

		return fileN, content

    def post(self):
        # Check if the image uploaded is a multipart/form-data
        if self.multipart_form_data():
            file_data = self.request.files['media'][0]
            body = file_data['body']

            # Retrieve filename from 'filename' field
            filename = file_data['filename']
        else:
            body = self.request.body

            # Retrieve filename from 'Slug' header
            filename = self.request.headers.get('Slug')

        # Check if the image uploaded is valid
        if self.validate(body):

            # Use the default filename for the uploaded images
            if not filename:
                content_type = self.request.headers.get('Content-Type', BaseEngine.get_mimetype(body))
                extension = mimetypes.guess_extension(content_type.split(';',1)[0], False)
                if extension is None: # Content-Type is unknown, try with body
                    extension = mimetypes.guess_extension(BaseEngine.get_mimetype(body), False)
                if extension == '.jpe':
                     extension = '.jpg'  # Hack because mimetypes return .jpe by default
                if extension is None: # Even body is unknown, return an empty string to be contat
                    extension = ''
                filename = self.context.config.UPLOAD_DEFAULT_FILENAME + extension

            # Build image id based on a random uuid (32 characters)
            id = str(uuid.uuid4().hex)
            self.write_file(id, body)
            self.set_status(201)
            self.set_header('Location', self.location(id, filename))

def unpack_mail(msg, only_headers=False, exclude_headers=True):
    # TODO: headers, msg_text, msg_html, attachments
    msg_text = ""
    msg_html = ""
    if not msg.is_multipart():
        msg_payload = msg.get_payload(decode=True)
        msg_payload = decode_text(msg_payload)
        if msg.get_content_type == 'text/html':
            msg_html = msg_payload
        else:   # text/plain. or other?
            msg_text = msg_payload
        return msg_text, msg_html, []

    attachments = []
    counter = 1
    for part in msg.walk():
        # multipart/* are just containers
        if part.get_content_maintype() == 'multipart':
            continue

        is_multipart = part.is_multipart()
        filename = part.get_filename()
        filename = decode_mail_header(filename)
        content_type = part.get_content_type()

        if is_multipart or filename:    # an attachment
            if not filename:    # maybe not possible
                ext = mimetypes.guess_extension(content_type)
                if not ext:
                    ext = '.bin'
                filename = 'part-%03d%s' % (counter, ext)
            attachments.append({
                "data": part.get_payload(),
                "filename": filename,
                "content_type": content_type,
                "is_multipart": is_multipart,
            })
        else:
            part_payload = part.get_payload(decode=True)
            part_payload = decode_text(part_payload)
            if content_type == 'text/plain':
                msg_text = part_payload
            elif content_type == 'text/html':
                msg_html = part_payload
            else:   # maybe not possible
                ext = mimetypes.guess_extension(content_type)
                if not ext:
                    ext = '.bin'
                filename = 'part-%03d%s' % (counter, ext)
                attachments.append({
                    "data": part.get_payload(),
                    "filename": filename,
                    "content_type": content_type,
                    "is_multipart": is_multipart(),
                })

        counter += 1

    return msg_text, msg_html, attachments

    def fetch(self, fetch_info, target_path, progress):
        """
        Fetch a file.
        """

        response = requests.get(fetch_info['url'], stream=True)
        response.raise_for_status()

        mimetype = fetch_info['mimetype'] or response.headers.get('content-type')
        encoding = response.headers.get('content-encoding')
        archive_type = (mimetype, encoding)

        # If the source has an overriden type, we use that instead.
        extension = None

        if fetch_info['mimetype']:
            extension = mimetypes.guess_extension(fetch_info['mimetype'])

        if not extension:
            extension = mimetypes.guess_extension(mimetype)

        if not extension:
            LOGGER.debug('No extension registered for this mimetype (%s). Guessing one from the URL...', mimetype)

            extension = os.path.splitext(urlparse.urlparse(fetch_info['url']).path)[1]

        if extension and extension.startswith('.'):
            extension = extension[1:]

        content_disposition = parse_requests_response(response)

        filename = content_disposition.filename_sanitized(extension=extension, default_filename='archive')

        content_length = response.headers.get('content-length')

        if content_length is not None:
            content_length = int(content_length)

        archive_path = os.path.join(target_path, filename)

        progress.on_start(target=os.path.basename(archive_path), size=content_length)

        with open(archive_path, 'wb') as target_file:
            current_size = 0

            for buf in response.iter_content(1024):

                if buf:
                    target_file.write(buf)
                    current_size += len(buf)

                    progress.on_update(progress=current_size)

        progress.on_finish()

        return {
            'archive_path': archive_path,
            'archive_type': archive_type,
        }

def _find_store_dir(file_path):
    mime = magic.from_file(file_path, mime=True)
    store_dir = None

    if mime in ['application/octet-stream', 'text/plain'] or mimetypes.guess_extension(mime) == None:
        store_dir = os.path.splitext(file_path)[1]
    else:
        store_dir = mimetypes.guess_extension(mime)
    
    return store_dir.lstrip(".").lower()

	def download(self, resource):
		if resource in Main.completedResources: #check if they're using the global id
			with open(str(resource) + mimetypes.guess_extension(Main.completedResources[resource][2]), "wb") as output:
				output.write(Main.completedResources[resource][1])
		else:
			for key in Main.completedResources: #loop throught to check if they're using the local id
				completedResource = Main.completedResources[key]
				if completedResource[3] == resource:
					with open(str(resource) + mimetypes.guess_extension(completedResource[2]), "wb") as output:
						output.write(completedResource[1])

    def _oooConvertByFormat(self, printout, content_type, extra_context, REQUEST, format, batch_mode):
        """
    Convert the ODF document into the given format.

    Keyword arguments:
    printout -- ODF document
    content_type -- the content type of the printout
    extra_context -- extra_context including a format
    REQUEST -- Request object
    format -- requested output format
    batch_mode -- Disable headers overriding
    """
        if REQUEST is not None and not format:
            format = REQUEST.get("format", None)
        filename = self.getProperty("filename")
        # Call refresh through cloudooo
        # XXX This is a temporary implementation:
        # Calling a webservice must be done through a WebServiceMethod
        # and a WebServiceConnection
        from Products.ERP5OOo.Document.OOoDocument import OOoServerProxy, enc, dec

        server_proxy = OOoServerProxy(self)
        extension = guess_extension(content_type).strip(".")
        printout = dec(
            server_proxy.convertFile(
                enc(printout), extension, extension, False, True  # source_format  # destination_format  # zip
            )
        )  # refresh
        # End of temporary implementation
        if not format:
            if REQUEST is not None and not batch_mode:
                REQUEST.RESPONSE.setHeader("Content-Length", len(printout))
                REQUEST.RESPONSE.setHeader("Content-Type", "%s" % content_type)
                REQUEST.RESPONSE.setHeader(
                    "Content-disposition", 'inline;filename="%s%s"' % (filename, guess_extension(content_type) or "")
                )
            return printout
        from Products.ERP5Type.Document import newTempOOoDocument

        tmp_ooo = newTempOOoDocument(self, self.title_or_id())
        tmp_ooo.edit(
            data=printout,
            base_data=printout,
            filename=self.title_or_id(),
            content_type=content_type,
            base_content_type=content_type,
        )
        mime, data = tmp_ooo.convert(format)
        if REQUEST is not None and not batch_mode:
            REQUEST.RESPONSE.setHeader("Content-Length", len(data))
            REQUEST.RESPONSE.setHeader("Content-type", mime)
            REQUEST.RESPONSE.setHeader("Content-disposition", 'attachment;filename="%s.%s"' % (filename, format))
        return str(data)

def guess_extension(response):
    "Return an extension based on the Content-Type header in the response"
    if not response:
        return None
    ct = response.info().get('content-type')
    if ct:
        mimetype = ct.split(';')[0]
        ext = mimetypes.guess_extension(mimetype)
        if ext:
            return ext
        # otherwise try based on URL
        mimetype, _ = mimetypes.guess_type(response.geturl())
        if mimetype:
            ext = mimetypes.guess_extension(mimetype)
        return ext

    def addPicture(self, filename, mediatype=None, content=None):
        """
        Add a picture
        It uses the same convention as OOo, in that it saves the picture in
        the zipfile in the subdirectory 'Pictures'
        If passed a file ptr, mediatype must be set
        @param filename unicode string: name of a file for Pictures
        @param mediatype unicode string: name of a media, None by default
        @param content bytes: content of media, None by default
        @return a unicode string: the file name of the media, eventually
        created on the fly
        """
        if content is None:
            if mediatype is None:
                mediatype, encoding = mimetypes.guess_type(filename)
            if mediatype is None:
                mediatype = u''
                try: ext = filename[filename.rindex(u'.'):]
                except: ext=u''
            else:
                ext = mimetypes.guess_extension(mediatype)
            manifestfn = u"Pictures/%s%s" % (uuid.uuid4().hex.upper(), ext)
            self.Pictures[manifestfn] = (IS_FILENAME, filename, mediatype)
            content=b""  # this value is only use by the assert further
            filename=u"" # this value is only use by the assert further
        else:
            manifestfn = filename
            self.Pictures[manifestfn] = (IS_IMAGE, content, mediatype)

        assert(type(filename)==type(u""))
        assert(type(content) == type(b""))

        return manifestfn

 def get_result(self, path, original_name=False,
                default_ext='.bin', delete_msg=True, get_file=True):
     q = self.get_queue(self.output_queue_name)
     m = q.read()
     if m:
         if get_file:
             outputs = m['OutputKey'].split(',')
             for output in outputs:
                 key_name, type = output.split(';')
                 mime_type = type.split('=')[1]
                 if original_name:
                     file_name = m.get('OriginalFileName', key_name)
                     file_name, ext = os.path.splitext(file_name)
                     ext = mimetypes.guess_extension(mime_type)
                     if not ext:
                         ext = default_ext
                     file_name = file_name + ext
                 else:
                     file_name = key_name
                 bucket = self.get_bucket(m['Bucket'])
                 key = bucket.lookup(key_name)
                 print 'retrieving file: %s' % file_name
                 key.get_contents_to_filename(os.path.join(path, file_name))
         if delete_msg:
             q.delete_message(m)
     return m

    def __indexCycleProcess(self):
        self.__run = 1
        while self.__run:
            self.__lockUrls.acquire()
            if len(self.__urls) > 0:
                url, search_query, weight = self.__urls.pop()
            else:
                url = None
            self.__lockUrls.release()

            if url:
                tmp_filename = tempfile.mktemp("", maay.globalvars.config.getValue("TemporaryDocumentRoot") + os.path.sep)
                fd = file(tmp_filename, "wb")

                infos = self.__fetchURL(url, fd)

                if infos:
                    mime_type, last_modified, content_size, document_id = infos
                else:
                    continue

                newname = document_id + (mimetypes.guess_extension(mime_type) or ".txt")
                absolute_newname = "%s%s%s" % (maay.globalvars.config.getValue("CachedDocumentRoot"), os.path.sep, newname)
                if os.path.exists(absolute_newname):
                    os.remove(absolute_newname)

                maay.globalvars.logger.debug("rename %s => %s" % (tmp_filename, absolute_newname))
                os.rename(tmp_filename, absolute_newname)
                maay.globalvars.logger.debug("done => %s" % absolute_newname)
                maay.globalvars.indexer.addNewDocumentToIndex(absolute_newname, mime_type, last_modified, url, search_query=search_query, weight=weight)
            else:
                time.sleep(2)

def get_extension(content):
    """A handful of workarounds for getting extensions we can trust."""
    file_str = magic.from_buffer(content)
    if file_str.startswith('Composite Document File V2 Document'):
        # Workaround for issue with libmagic1==5.09-2 in Ubuntu 12.04. Fixed
        # in libmagic 5.11-2.
        mime = 'application/msword'
    elif file_str == '(Corel/WP)':
        mime = 'application/vnd.wordperfect'
    elif file_str == 'C source, ASCII text':
        mime = 'text/plain'
    else:
        # No workaround necessary
        mime = magic.from_buffer(content, mime=True)
    extension = mimetypes.guess_extension(mime)
    if extension == '.obj':
        # It could be a wpd, if it's not a PDF
        if 'PDF' in content[0:40]:
            # Does 'PDF' appear in the beginning of the content?
            extension = '.pdf'
        else:
            extension = '.wpd'
    if extension == '.wsdl':
        # It's probably an HTML file, like those from Resource.org
        extension = '.html'
    if extension == '.ksh':
        extension = '.txt'
    if extension == '.asf':
        extension = '.wma'
    return extension

def upload_image_url(request):
    if request.method != 'POST':
        return HttpResponse(status=403)
    image_url = request.POST.get('image_url', None)
    source_domain = request.POST.get('source_domain', None)

    headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
        # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        # 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
        # 'Accept-Encoding': 'none',
        # 'Accept-Language': 'en-US,en;q=0.8',
        # 'Connection': 'keep-alive',
        'referer': source_domain,
    }

    ext = mimetypes.guess_extension(mimetypes.guess_type(image_url)[0])
    req = urllib2.Request(image_url, headers=headers)
    img_temp = NamedTemporaryFile(delete=True)
    img_temp.write(urllib2.urlopen(req).read())
    img_temp.flush()

    post_photo = Post_photo()
    post_photo.photo.save('%s%s'%(uuid.uuid4(), ext), File(img_temp))
    post_photo.save()

    res = {
        'link': post_photo.photo.url,
    }
    return JsonResponse(res, safe=False)