def _fetch(self, url):
        storage = self.context.modules.storage
        buffer = yield gen.maybe_future(storage.get(url))
        mime = None

        if buffer is not None:
            self.context.statsd_client.incr('storage.hit')
            mime = BaseEngine.get_mimetype(buffer)
            self.context.request.extension = EXTENSION.get(mime, '.jpg')
            if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                self.context.request.engine = self.context.modules.gif_engine
            else:
                self.context.request.engine = self.context.modules.engine

            raise gen.Return([False, buffer, None])
        else:
            self.context.statsd_client.incr('storage.miss')

        buffer = yield self.context.modules.loader.load(self.context, url)

        if buffer is None:
            raise gen.Return([False, None, None])

        if mime is None:
            mime = BaseEngine.get_mimetype(buffer)

        if mime is None:
            raise gen.Return([False, None, None])

        self.context.request.extension = EXTENSION.get(mime, '.jpg')

        original_preserve = self.context.config.PRESERVE_EXIF_INFO
        self.context.config.PRESERVE_EXIF_INFO = True

        try:
            mime = BaseEngine.get_mimetype(buffer)
            self.context.request.extension = extension = EXTENSION.get(mime, None)

            if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                self.context.request.engine = self.context.modules.gif_engine
            else:
                self.context.request.engine = self.context.modules.engine

            self.context.request.engine.load(buffer, extension)
            normalized = self.context.request.engine.normalize()
            is_no_storage = isinstance(storage, NoStorage)
            is_mixed_storage = isinstance(storage, MixedStorage)
            is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage)

            if not (is_no_storage or is_mixed_no_file_storage):
                buffer = self.context.request.engine.read(extension)
                storage.put(url, buffer)

            storage.put_crypto(url)
        finally:
            self.context.config.PRESERVE_EXIF_INFO = original_preserve

        raise gen.Return([normalized, None, self.context.request.engine])

    def load(self, buffer, extension):
        self.extension = extension

        if extension is None:
            mime = self.get_mimetype(buffer)
            self.extension = EXTENSION.get(mime, '.jpg')

        if self.extension == '.svg':
            buffer = self.convert_svg_to_png(buffer)

        image_or_frames = self.create_image(buffer)

        if METADATA_AVAILABLE:
            try:
                self.metadata = ImageMetadata.from_buffer(buffer)
                self.metadata.read()
            except Exception as e:
                logger.error('Error reading image metadata: %s' % e)

        if self.context.config.ALLOW_ANIMATED_GIFS and isinstance(
                image_or_frames, (list, tuple)):
            self.image = image_or_frames[0]
            if len(image_or_frames) > 1:
                self.multiple_engine = MultipleEngine(self)
                for frame in image_or_frames:
                    self.multiple_engine.add_frame(frame)
                self.wrap(self.multiple_engine)
        else:
            self.image = image_or_frames

        if self.source_width is None:
            self.source_width = self.size[0]
        if self.source_height is None:
            self.source_height = self.size[1]

    def define_image_type(self, context, result):
        if result is not None:
            if isinstance(result, ResultStorageResult):
                buffer = result.buffer
            else:
                buffer = result
            image_extension = EXTENSION.get(BaseEngine.get_mimetype(buffer), ".jpg")
        else:
            image_extension = context.request.format
            if image_extension is not None:
                image_extension = ".%s" % image_extension
                logger.debug("Image format specified as %s." % image_extension)
            elif self.is_webp(context):
                image_extension = ".webp"
                logger.debug("Image format set by AUTO_WEBP as %s." % image_extension)
            else:
                image_extension = context.request.engine.extension
                logger.debug("No image format specified. Retrieving from the image extension: %s." % image_extension)

        content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE[".jpg"])

        if context.request.meta:
            context.request.meta_callback = (
                context.config.META_CALLBACK_NAME or self.request.arguments.get("callback", [None])[0]
            )
            content_type = "text/javascript" if context.request.meta_callback else "application/json"
            logger.debug("Metadata requested. Serving content type of %s." % content_type)

        logger.debug("Content Type of %s detected." % content_type)

        return (image_extension, content_type)

    def define_image_type(self, context, result):
        if result is not None:
            if isinstance(result, ResultStorageResult):
                buffer = result.buffer
            else:
                buffer = result
            image_extension = EXTENSION.get(BaseEngine.get_mimetype(buffer), '.jpg')
        else:
            image_extension = context.request.format
            if image_extension is not None:
                image_extension = '.%s' % image_extension
                logger.debug('Image format specified as %s.' % image_extension)
            elif self.is_webp(context):
                image_extension = '.webp'
                logger.debug('Image format set by AUTO_WEBP as %s.' % image_extension)
            elif self.can_auto_convert_png_to_jpg():
                image_extension = '.jpg'
                logger.debug('Image format set by AUTO_PNG_TO_JPG as %s.' % image_extension)
            else:
                image_extension = context.request.engine.extension
                logger.debug('No image format specified. Retrieving from the image extension: %s.' % image_extension)

        content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE['.jpg'])

        if context.request.meta:
            context.request.meta_callback = context.config.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0]
            content_type = 'text/javascript' if context.request.meta_callback else 'application/json'
            logger.debug('Metadata requested. Serving content type of %s.' % content_type)

        logger.debug('Content Type of %s detected.' % content_type)

        return (image_extension, content_type)

    def load(self, buffer, extension):
        self.extension = extension

        if extension is None:
            mime = self.get_mimetype(buffer)
            self.extension = EXTENSION.get(mime, '.jpg')

        if self.extension == '.svg':
            buffer = self.convert_svg_to_png(buffer)

        image_or_frames = self.create_image(buffer)

        if self.context.config.ALLOW_ANIMATED_GIFS and isinstance(
                image_or_frames, (list, tuple)):
            self.image = image_or_frames[0]
            if len(image_or_frames) > 1:
                self.multiple_engine = MultipleEngine(self)
                for frame in image_or_frames:
                    self.multiple_engine.add_frame(frame)
                self.wrap(self.multiple_engine)
        else:
            self.image = image_or_frames

        if self.source_width is None:
            self.source_width = self.size[0]
        if self.source_height is None:
            self.source_height = self.size[1]

 def test_can_get_extension(self):
     expect(EXTENSION.get('image/jpeg')).to_equal('.jpg')
     expect(EXTENSION.get('image/gif')).to_equal('.gif')
     expect(EXTENSION.get('image/png')).to_equal('.png')
     expect(EXTENSION.get('image/webp')).to_equal('.webp')
     expect(EXTENSION.get('video/mp4')).to_equal('.mp4')
     expect(EXTENSION.get('video/webm')).to_equal('.webm')
     expect(EXTENSION.get('image/svg+xml')).to_equal('.svg')
     expect(EXTENSION.get('image/tiff')).to_equal('.tif')

    def convert_svg_to_png(self, buffer):
        if not cairosvg:
            msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
            imported (if you want svg conversion to png please install cairosvg)
            """
            logger.error(msg)
            return buffer

        try:
            buffer = cairosvg.svg2png(bytestring=buffer, dpi=self.context.config.SVG_DPI)
            mime = self.get_mimetype(buffer)
            self.extension = EXTENSION.get(mime, '.jpg')
        except ParseError:
            mime = self.get_mimetype(buffer)
            extension = EXTENSION.get(mime)
            if extension is None or extension == '.svg':
                raise
            self.extension = extension

        return buffer

    def load(self, buffer, extension):
        self.extension = extension

        if extension is None:
            mime = self.get_mimetype(buffer)
            self.extension = EXTENSION.get(mime, '.jpg')

        if self.extension == '.tif':  # Pillow does not support 16bit per channel TIFF images
            buffer = self.convert_tif_to_png(buffer)

        super(Engine, self).load(buffer, self.extension)

    def convert_svg_to_png(self, buffer):
        if not cairosvg:
            msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
            imported (if you want svg conversion to png please install cairosvg)
            """
            logger.error(msg)
            return buffer

        buffer = cairosvg.svg2png(bytestring=buffer, dpi=self.context.config.SVG_DPI)
        mime = self.get_mimetype(buffer)
        self.extension = EXTENSION.get(mime, ".jpg")
        return buffer

    def convert_svg_to_png(self, buffer):
        if not cairosvg:
            msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
            imported (if you want svg conversion to png please install cairosvg)
            """
            logger.error(msg)
            return buffer

        buffer = cairosvg.svg2png(bytestring=buffer)
        mime = self.get_mimetype(buffer)
        self.extension = EXTENSION.get(mime, '.jpg')
        self.transformed_body = buffer
        return buffer

    def convert_tif_to_png(self, buffer):
        if not cv2:
            msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
            logger.error(msg)
            return buffer
        if not numpy:
            msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
            logger.error(msg)
            return buffer

        img = cv2.imdecode(numpy.fromstring(buffer, dtype='uint16'), -1)
        buffer = cv2.imencode('.png', img)[1].tostring()

        mime = self.get_mimetype(buffer)
        self.extension = EXTENSION.get(mime, '.jpg')
        return buffer

    def execute_image_operations(self):
        self.context.request.quality = None

        req = self.context.request
        conf = self.context.config

        should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe
        if self.context.modules.result_storage and should_store:
            start = datetime.datetime.now()

            result = yield gen.maybe_future(self.context.modules.result_storage.get())

            finish = datetime.datetime.now()

            self.context.metrics.timing('result_storage.incoming_time', (finish - start).total_seconds() * 1000)

            if result is None:
                self.context.metrics.incr('result_storage.miss')
            else:
                self.context.metrics.incr('result_storage.hit')
                self.context.metrics.incr('result_storage.bytes_read', len(result))

            if result is not None:
                buffer = result.buffer if isinstance(result, ResultStorageResult) else result
                mime = BaseEngine.get_mimetype(buffer)
                if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                    self.context.request.engine = self.context.modules.gif_engine
                else:
                    self.context.request.engine = self.context.modules.engine
                self.context.request.engine.load(buffer, EXTENSION.get(mime, '.jpg'))

                logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url)
                self.finish_request(self.context, result)
                return

        if conf.MAX_WIDTH and (not isinstance(req.width, basestring)) and req.width > conf.MAX_WIDTH:
            req.width = conf.MAX_WIDTH
        if conf.MAX_HEIGHT and (not isinstance(req.height, basestring)) and req.height > conf.MAX_HEIGHT:
            req.height = conf.MAX_HEIGHT

        req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0]

        self.filters_runner = self.context.filters_factory.create_instances(self.context, self.context.request.filters)
        # Apply all the filters from the PRE_LOAD phase and call get_image() afterwards.
        self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)

    def convert_tif_to_png(self, buffer):
        if not cv:
            msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
            logger.error(msg)
            return buffer

        # can not use cv2 here, because ubuntu precise shipped with python-opencv 2.3 which has bug with imencode
        # requires 3rd parameter buf which could not be created in python. Could be replaced with these lines:
        # img = cv2.imdecode(numpy.fromstring(buffer, dtype='uint16'), -1)
        # buffer = cv2.imencode('.png', img)[1].tostring()
        mat_data = cv.CreateMatHeader(1, len(buffer), cv.CV_8UC1)
        cv.SetData(mat_data, buffer, len(buffer))
        img = cv.DecodeImage(mat_data, -1)
        buffer = cv.EncodeImage(".png", img).tostring()

        mime = self.get_mimetype(buffer)
        self.extension = EXTENSION.get(mime, '.jpg')
        return buffer

    def _write_results_to_client(self, context, results, content_type):
        max_age = context.config.MAX_AGE

        if context.request.max_age is not None:
            max_age = context.request.max_age

        if context.request.prevent_result_storage or context.request.detection_error:
            max_age = context.config.MAX_AGE_TEMP_IMAGE

        if max_age:
            self.set_header('Cache-Control', 'max-age=' + str(max_age) + ',public')
            self.set_header('Expires', datetime.datetime.utcnow() + datetime.timedelta(seconds=max_age))

        self.set_header('Server', 'Thumbor/%s' % __version__)
        self.set_header('Content-Type', content_type)

        if isinstance(results, ResultStorageResult):
            buffer = results.buffer
        else:
            buffer = results

        # auto-convert configured?
        should_vary = context.config.AUTO_WEBP
        # we have image (not video)
        should_vary = should_vary and content_type.startswith("image/")
        # output format is not requested via format filter
        should_vary = should_vary and not (
            context.request.format  # format is supported by filter
            and bool(re.search(r"format\([^)]+\)", context.request.filters))  # filter is in request
        )
        # our image is not animated gif
        should_vary = should_vary and not self.is_animated_gif(buffer)

        if should_vary:
            self.set_header('Vary', 'Accept')

        context.headers = self._headers.copy()

        self._response_ext = EXTENSION.get(content_type)
        self._response_length = len(buffer)

        self.write(buffer)
        self.finish()

 def test_should_not_run_for_not_gif(self):
     optimizer = Optimizer(self.get_context())
     for ext in EXTENSION.itervalues():
         if ext != '.gif':
             self.assertFalse(optimizer.should_run(ext, ''))

    def _fetch(self, url):
        """

        :param url:
        :type url:
        :return:
        :rtype:
        """
        fetch_result = FetchResult()

        storage = self.context.modules.storage

        yield self.acquire_url_lock(url)

        try:
            fetch_result.buffer = yield gen.maybe_future(storage.get(url))
            mime = None

            if fetch_result.buffer is not None:
                self.release_url_lock(url)

                fetch_result.successful = True

                self.context.metrics.incr('storage.hit')
                mime = BaseEngine.get_mimetype(fetch_result.buffer)
                self.context.request.extension = EXTENSION.get(mime, '.jpg')
                if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                    self.context.request.engine = self.context.modules.gif_engine
                else:
                    self.context.request.engine = self.context.modules.engine

                raise gen.Return(fetch_result)
            else:
                self.context.metrics.incr('storage.miss')

            loader_result = yield self.context.modules.loader.load(self.context, url)
        finally:
            self.release_url_lock(url)

        if isinstance(loader_result, LoaderResult):
            # TODO _fetch should probably return a result object vs a list to
            # to allow returning metadata
            if not loader_result.successful:
                fetch_result.buffer = None
                fetch_result.loader_error = loader_result.error
                raise gen.Return(fetch_result)

            fetch_result.buffer = loader_result.buffer
        else:
            # Handle old loaders
            fetch_result.buffer = loader_result

        if fetch_result.buffer is None:
            raise gen.Return(fetch_result)

        fetch_result.successful = True

        if mime is None:
            mime = BaseEngine.get_mimetype(fetch_result.buffer)

        self.context.request.extension = extension = EXTENSION.get(mime, '.jpg')

        try:
            if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                self.context.request.engine = self.context.modules.gif_engine
            else:
                self.context.request.engine = self.context.modules.engine

            self.context.request.engine.load(fetch_result.buffer, extension)

            if self.context.request.engine.image is None:
                fetch_result.successful = False
                fetch_result.buffer = None
                fetch_result.engine = self.context.request.engine
                fetch_result.engine_error = EngineResult.COULD_NOT_LOAD_IMAGE
                raise gen.Return(fetch_result)

            fetch_result.normalized = self.context.request.engine.normalize()

            # Allows engine or loader to override storage on the fly for the purpose of
            # marking a specific file as unstoreable
            storage = self.context.modules.storage

            is_no_storage = isinstance(storage, NoStorage)
            is_mixed_storage = isinstance(storage, MixedStorage)
            is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage)

            if not (is_no_storage or is_mixed_no_file_storage):
                storage.put(url, fetch_result.buffer)

            storage.put_crypto(url)
        except Exception:
            fetch_result.successful = False
        finally:
            if not fetch_result.successful:
                raise
            fetch_result.buffer = None
            fetch_result.engine = self.context.request.engine
            raise gen.Return(fetch_result)

    def _fetch(self, url):
        """

        :param url:
        :type url:
        :return:
        :rtype:
        """
        fetch_result = FetchResult()

        storage = self.context.modules.storage

        yield self.acquire_url_lock(url)

        try:
            fetch_result.buffer = yield gen.maybe_future(storage.get(url))
            mime = None

            if fetch_result.buffer is not None:
                self.release_url_lock(url)

                fetch_result.successful = True

                self.context.metrics.incr('storage.hit')
                mime = BaseEngine.get_mimetype(fetch_result.buffer)
                self.context.request.extension = EXTENSION.get(mime, '.jpg')
                if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                    self.context.request.engine = self.context.modules.gif_engine
                else:
                    self.context.request.engine = self.context.modules.engine

                raise gen.Return(fetch_result)
            else:
                self.context.metrics.incr('storage.miss')

            loader_result = yield self.context.modules.loader.load(self.context, url)
        finally:
            self.release_url_lock(url)

        if isinstance(loader_result, LoaderResult):
            # TODO _fetch should probably return a result object vs a list to
            # to allow returning metadata
            if not loader_result.successful:
                fetch_result.buffer = None
                fetch_result.loader_error = loader_result.error
                raise gen.Return(fetch_result)

            fetch_result.buffer = loader_result.buffer
        else:
            # Handle old loaders
            fetch_result.buffer = loader_result

        if fetch_result.buffer is None:
            raise gen.Return(fetch_result)

        fetch_result.successful = True

        if mime is None:
            mime = BaseEngine.get_mimetype(fetch_result.buffer)

        self.context.request.extension = extension = EXTENSION.get(mime, '.jpg')

        original_preserve = self.context.config.PRESERVE_EXIF_INFO

        try:
            if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                self.context.request.engine = self.context.modules.gif_engine
            else:
                self.context.request.engine = self.context.modules.engine

            self.context.request.engine.load(fetch_result.buffer, extension)

            if self.context.request.engine.image is None:
                fetch_result.successful = False
                fetch_result.buffer = None
                fetch_result.engine = self.context.request.engine
                fetch_result.engine_error = EngineResult.COULD_NOT_LOAD_IMAGE
                raise gen.Return(fetch_result)

            fetch_result.normalized = self.context.request.engine.normalize()

            # Allows engine or loader to override storage on the fly for the purpose of
            # marking a specific file as unstoreable
            storage = self.context.modules.storage

            is_no_storage = isinstance(storage, NoStorage)
            is_mixed_storage = isinstance(storage, MixedStorage)
            is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage)

            if not (is_no_storage or is_mixed_no_file_storage):
                # in the case that we have local storage, it's important that we don't try
                # and strip the exif data - so that we can make the choice about stripping
                # it on output, instead of input.
                # unfortunately, the only interface to the engine is the global config
                # referenced by our context, which has the current setting of preserving
                # the exif info or not. this is also being read by concurrent
                # coroutine-threads, so just setting this value outright causes a race.
                # we solve this in order not to influence these other coroutine-threads
                # by doing an effective copy-on-write for the config until the end of this
                # context (ie. this request).
#.........这里部分代码省略.........

    def _fetch(self, url):
        fetch_result = FetchResult()

        storage = self.context.modules.storage
        fetch_result.buffer = yield gen.maybe_future(storage.get(url))
        mime = None

        if fetch_result.buffer is not None:
            fetch_result.successful = True

            self.context.metrics.incr('storage.hit')
            mime = BaseEngine.get_mimetype(fetch_result.buffer)
            self.context.request.extension = EXTENSION.get(mime, '.jpg')
            if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                self.context.request.engine = self.context.modules.gif_engine
            else:
                self.context.request.engine = self.context.modules.engine

            raise gen.Return(fetch_result)
        else:
            self.context.metrics.incr('storage.miss')

        loader_result = yield self.context.modules.loader.load(self.context, url)

        if isinstance(loader_result, LoaderResult):
            # TODO _fetch should probably return a result object vs a list to
            # to allow returning metadata
            if not loader_result.successful:
                fetch_result.buffer = None
                fetch_result.loader_error = loader_result.error
                raise gen.Return(fetch_result)

            fetch_result.buffer = loader_result.buffer
        else:
            # Handle old loaders
            fetch_result.buffer = loader_result

        if fetch_result.buffer is None:
            raise gen.Return(fetch_result)

        fetch_result.successful = True

        if mime is None:
            mime = BaseEngine.get_mimetype(fetch_result.buffer)

        self.context.request.extension = EXTENSION.get(mime, '.jpg')

        original_preserve = self.context.config.PRESERVE_EXIF_INFO
        self.context.config.PRESERVE_EXIF_INFO = True

        try:
            mime = BaseEngine.get_mimetype(fetch_result.buffer)
            self.context.request.extension = extension = EXTENSION.get(mime, None)

            if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
                self.context.request.engine = self.context.modules.gif_engine
            else:
                self.context.request.engine = self.context.modules.engine

            self.context.request.engine.load(fetch_result.buffer, extension)

            fetch_result.normalized = self.context.request.engine.normalize()
            is_no_storage = isinstance(storage, NoStorage)
            is_mixed_storage = isinstance(storage, MixedStorage)
            is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage)

            if not (is_no_storage or is_mixed_no_file_storage):
                fetch_result.buffer = self.context.request.engine.read(extension)
                storage.put(url, fetch_result.buffer)

            storage.put_crypto(url)
        finally:
            self.context.config.PRESERVE_EXIF_INFO = original_preserve

        fetch_result.buffer = None
        fetch_result.engine = self.context.request.engine
        raise gen.Return(fetch_result)

 def convert_svg_to_png(self, buffer):
     buffer = cairosvg.svg2png(bytestring=buffer)
     mime = self.get_mimetype(buffer)
     self.extension = EXTENSION.get(mime, '.jpg')
     self.transformed_body = buffer
     return buffer