def webp_encode(coding, image, supports_transparency, quality, speed, options):
    stride = image.get_rowstride()
    enc_webp = get_codec("enc_webp")
    if enc_webp and stride>0 and stride%4==0 and image.get_pixel_format() in ("BGRA", "BGRX"):
        #prefer Cython module:
        alpha = supports_transparency and image.get_pixel_format().find("A")>=0
        w = image.get_width()
        h = image.get_height()
        if quality==100:
            #webp lossless is unbearibly slow for only marginal compression improvements,
            #so force max speed:
            speed = 100
        else:
            #normalize speed for webp: avoid low speeds!
            speed = int(sqrt(speed) * 10)
        speed = max(0, min(100, speed))
        cdata = enc_webp.compress(image.get_pixels(), w, h, stride=stride/4, quality=quality, speed=speed, has_alpha=alpha)
        client_options = {"speed" : speed}
        if quality>=0 and quality<=100:
            client_options["quality"] = quality
        if alpha:
            client_options["has_alpha"] = True
        return "webp", Compressed("webp", cdata), client_options, image.get_width(), image.get_height(), 0, 24
    enc_webm = get_codec("enc_webm")
    webp_handlers = get_codec("webm_bitmap_handlers")
    if enc_webm and webp_handlers:
        return webm_encode(image, quality)
    #fallback to PIL
    return PIL_encode(coding, image, quality, speed, supports_transparency)

def load_csc_options():
    global CSC_OPTIONS
    if CSC_OPTIONS is None:
        CSC_OPTIONS = {}
        opts = [x.strip() for x in XPRA_CLIENT_CSC.split(",")]
        log("load_csc_options() module options=%s", opts)
        for opt in opts:
            csc_module = get_codec("csc_%s" % opt)
            if not csc_module:
                log.warn("csc module %s not found", opt)
                continue
            log("csc_module(%s)=%s", opt, csc_module)
            try:
                in_cscs = csc_module.get_input_colorspaces()
                log("input colorspaces(%s)=%s", csc_module, in_cscs)
                for in_csc in in_cscs:
                    in_opts = CSC_OPTIONS.setdefault(in_csc, {})
                    out_cscs = csc_module.get_output_colorspaces(in_csc)
                    log("output colorspaces(%s, %s)=%s", csc_module, in_csc, out_cscs)
                    for out_csc in out_cscs:
                        spec = csc_module.get_spec(in_csc, out_csc)
                        specs = in_opts.setdefault(out_csc, [])
                        specs.append(spec)
                        log("specs(%s, %s)=%s", in_csc, out_csc, specs)
            except:
                log.warn("failed to load csc module %s", csc_module, exc_info=True)

 def test_all_codecs_found(self):
     from xpra.codecs import loader
     #the self tests would swallow the exceptions and produce a warning:
     loader.RUN_SELF_TESTS = False
     loader.load_codecs()
     #test them all:
     for codec_name in loader.ALL_CODECS:
         codec = loader.get_codec(codec_name)
         if not codec:
             continue
         try:
             #try to suspend error logging for full tests,
             #as those may cause errors
             log = getattr(codec, "log", None)
             if SUSPEND_CODEC_ERROR_LOGGING and log:
                 import logging
                 log.logger.setLevel(logging.CRITICAL)
             init_module = getattr(codec, "init_module", None)
             #print("%s.init_module=%s" % (codec, init_module))
             if init_module:
                 try:
                     init_module()
                 except Exception as e:
                     print("cannot initialize %s: %s" % (codec, e))
                     print(" test skipped")
                     continue
             #print("found %s: %s" % (codec_name, codec))
             selftest = getattr(codec, "selftest", None)
             #print("selftest(%s)=%s" % (codec_name, selftest))
             if selftest:
                 selftest(True)
         finally:
             if log:
                 log.logger.setLevel(logging.DEBUG)

 def init_video_decoder_option(self, decoder_name):
     decoder_module = get_codec(decoder_name)
     log("init_video_decoder_option(%s)", decoder_name)
     log(" module=%s", decoder_module)
     if not decoder_module:
         log(" video decoder %s could not be loaded:", decoder_name)
         log(" %s", get_codec_error(decoder_name))
         return
     decoder_type = decoder_module.get_type()
     try:
         decoder_module.init_module()
         self._cleanup_modules.append(decoder_module)
     except Exception as e:
         log("exception in %s module initialization %s: %s", decoder_type, decoder_module.init_module, e, exc_info=True)
         log.warn("Warning: cannot use %s module %s: %s", decoder_type, decoder_module, e, exc_info=True)
         return
     encodings = decoder_module.get_encodings()
     log(" %s encodings=%s", decoder_type, csv(encodings))
     for encoding in encodings:
         colorspaces = decoder_module.get_input_colorspaces(encoding)
         log(" %s input colorspaces for %s: %s", decoder_type, encoding, csv(colorspaces))
         for colorspace in colorspaces:
             output_colorspace = decoder_module.get_output_colorspace(encoding, colorspace)
             log(" %s output colorspace for %s/%s: %s", decoder_type, encoding, colorspace, output_colorspace)
             try:
                 assert decoder_module.Decoder
                 self.add_decoder(encoding, colorspace, decoder_name, decoder_module)
             except Exception as e:
                 log.warn("failed to add decoder %s: %s", decoder_module, e)

 def nasty_rgb_via_png_paint(self, cairo_format, has_alpha, img_data, x, y, width, height, rowstride, rgb_format):
     log("nasty_rgb_via_png_paint%s", (cairo_format, has_alpha, len(img_data), x, y, width, height, rowstride, rgb_format))
     #PIL fallback
     PIL = get_codec("PIL")
     if has_alpha:
         oformat = "RGBA"
     else:
         oformat = "RGB"
     #use frombytes rather than frombuffer to be compatible with python3 new-style buffers
     #this is slower, but since this codepath is already dreadfully slow, we don't care
     bdata = strtobytes(memoryview_to_bytes(img_data))
     try:
         img = PIL.Image.frombytes(oformat, (width,height), bdata, "raw", rgb_format.replace("X", "A"), rowstride, 1)
     except ValueError as e:
         raise Exception("failed to parse raw %s data to %s: %s" % (rgb_format, oformat, e))
     #This is insane, the code below should work, but it doesn't:
     # img_data = bytearray(img.tostring('raw', oformat, 0, 1))
     # pixbuf = pixbuf_new_from_data(img_data, COLORSPACE_RGB, True, 8, width, height, rowstride)
     # success = self.cairo_paint_pixbuf(pixbuf, x, y)
     #So we still rountrip via PNG:
     png = BytesIOClass()
     img.save(png, format="PNG")
     reader = BytesIOClass(png.getvalue())
     png.close()
     img = cairo.ImageSurface.create_from_png(reader)
     self.cairo_paint_surface(img, x, y)
     return True

 def paint_image(self, coding, img_data, x, y, width, height, options, callbacks):
     """ can be called from any thread """
     #log("paint_image(%s, %s bytes, %s, %s, %s, %s, %s, %s)", coding, len(img_data), x, y, width, height, options, callbacks)
     PIL = get_codec("PIL")
     assert PIL, "PIL not found"
     buf = BytesIOClass(img_data)
     img = PIL.Image.open(buf)
     assert img.mode in ("L", "P", "RGB", "RGBA"), "invalid image mode: %s" % img.mode
     if img.mode in ("P", "L"):
         transparency = options.get("transparency", -1)
         if transparency>=0:
             img = img.convert("RGBA")
         else:
             img = img.convert("RGB")
     raw_data = img.tostring("raw", img.mode)
     if img.mode=="RGB":
         #PIL flattens the data to a continuous straightforward RGB format:
         rowstride = width*3
         img_data = self.process_delta(raw_data, width, height, rowstride, options)
         self.idle_add(self.do_paint_rgb24, img_data, x, y, width, height, rowstride, options, callbacks)
     elif img.mode=="RGBA":
         rowstride = width*4
         img_data = self.process_delta(raw_data, width, height, rowstride, options)
         self.idle_add(self.do_paint_rgb32, img_data, x, y, width, height, rowstride, options, callbacks)
     return False

def get_DEFAULT_CSC_MODULES():
    """ returns all the csc modules installed """
    csc = []
    for x in list(ALL_CSC_MODULE_OPTIONS):
        mod = get_csc_module_name(x)
        c = get_codec(mod)
        if c:
            csc.append(x)
    return csc

def get_DEFAULT_VIDEO_ENCODERS():
    """ returns all the video encoders installed """
    encoders = []
    for x in list(ALL_VIDEO_ENCODER_OPTIONS):
        mod = get_encoder_module_name(x)
        c = get_codec(mod)
        if c:
            encoders.append(x)
    return encoders

def rgb_reformat(image, rgb_formats, supports_transparency):
    """ convert the RGB pixel data into a format supported by the client """
    # need to convert to a supported format!
    PIL = get_codec("PIL")
    pixel_format = image.get_pixel_format()
    pixels = image.get_pixels()
    assert pixels, "failed to get pixels from %s" % image
    if not PIL or pixel_format == "r210":
        # try to fallback to argb module
        # (required for r210 which is not handled by PIL directly)
        log("rgb_reformat: using argb_swap for %s", image)
        return argb_swap(image, rgb_formats, supports_transparency)
    if supports_transparency:
        modes = PIL_conv.get(pixel_format)
    else:
        modes = PIL_conv_noalpha.get(pixel_format)
    assert modes, "no PIL conversion from %s" % (pixel_format)
    target_rgb = [(im, om) for (im, om) in modes if om in rgb_formats]
    if len(target_rgb) == 0:
        log("rgb_reformat: no matching target modes for converting %s to %s", image, rgb_formats)
        # try argb module:
        if argb_swap(image, rgb_formats, supports_transparency):
            return True
        warning_key = "rgb_reformat(%s, %s, %s)" % (pixel_format, rgb_formats, supports_transparency)
        warn_encoding_once(warning_key, "cannot convert %s to one of: %s" % (pixel_format, rgb_formats))
        return False
    input_format, target_format = target_rgb[0]
    start = time.time()
    w = image.get_width()
    h = image.get_height()
    # PIL cannot use the memoryview directly:
    if isinstance(pixels, memoryview):
        pixels = pixels.tobytes()
    log("rgb_reformat: converting %s from %s to %s using PIL", image, input_format, target_format)
    img = PIL.Image.frombuffer(target_format, (w, h), pixels, "raw", input_format, image.get_rowstride())
    rowstride = w * len(target_format)  # number of characters is number of bytes per pixel!
    data = img.tobytes("raw", target_format)
    assert len(data) == rowstride * h, "expected %s bytes in %s format but got %s" % (rowstride * h, len(data))
    image.set_pixels(data)
    image.set_rowstride(rowstride)
    image.set_pixel_format(target_format)
    end = time.time()
    log(
        "rgb_reformat(%s, %s, %s) converted from %s (%s bytes) to %s (%s bytes) in %.1fms, rowstride=%s",
        image,
        rgb_formats,
        supports_transparency,
        pixel_format,
        len(pixels),
        target_format,
        len(data),
        (end - start) * 1000.0,
        rowstride,
    )
    return True

def webp_encode(coding, image, rgb_formats, supports_transparency, quality, speed, options):
    pixel_format = image.get_pixel_format()
    #log("rgb_encode%s pixel_format=%s, rgb_formats=%s", (coding, image, rgb_formats, supports_transparency, speed, rgb_zlib, rgb_lz4), pixel_format, rgb_formats)
    if pixel_format not in rgb_formats:
        if not rgb_reformat(image, rgb_formats, supports_transparency):
            raise Exception("cannot find compatible rgb format to use for %s! (supported: %s)" % (pixel_format, rgb_formats))
        #get the new format:
        pixel_format = image.get_pixel_format()
    stride = image.get_rowstride()
    enc_webp = get_codec("enc_webp")
    #log("webp_encode%s stride=%s, enc_webp=%s", (coding, image, rgb_formats, supports_transparency, quality, speed, options), stride, enc_webp)
    if enc_webp and stride>0 and stride%4==0 and image.get_pixel_format() in ("BGRA", "BGRX", "RGBA", "RGBX"):
        #prefer Cython module:
        alpha = supports_transparency and image.get_pixel_format().find("A")>=0
        w = image.get_width()
        h = image.get_height()
        if quality==100:
            #webp lossless is unbearibly slow for only marginal compression improvements,
            #so force max speed:
            speed = 100
        else:
            #normalize speed for webp: avoid low speeds!
            speed = int(sqrt(speed) * 10)
        speed = max(0, min(100, speed))
        pixels = image.get_pixels()
        assert pixels, "failed to get pixels from %s" % image
        cdata = enc_webp.compress(pixels, w, h, stride=stride/4, quality=quality, speed=speed, has_alpha=alpha)
        client_options = {"speed"       : speed,
                          "rgb_format"  : pixel_format}
        if quality>=0 and quality<=100:
            client_options["quality"] = quality
        if alpha:
            client_options["has_alpha"] = True
        return "webp", compression.Compressed("webp", cdata), client_options, image.get_width(), image.get_height(), 0, 24
    #fallback to PIL
    enc_pillow = get_codec("enc_pillow")
    if enc_pillow:
        log("using PIL fallback for webp: enc_webp=%s, stride=%s, pixel format=%s", enc_webp, stride, image.get_pixel_format())
        for x in ("webp", "png"):
            if x in enc_pillow.get_encodings():
                return enc_pillow.encode(x, image, quality, speed, supports_transparency)
    raise Exception("BUG: cannot use 'webp' encoding and none of the PIL fallbacks are available!")

 def init_video_encoder_option(self, encoder_name):
     encoder_module = get_codec(encoder_name)
     debug("init_video_encoder_option(%s) module=%s", encoder_name, encoder_module)
     if not encoder_module:
         return
     encoder_type = encoder_module.get_type()
     try:
         encoder_module.init_module()
     except Exception, e:
         log.warn("cannot use %s module %s: %s", encoder_type, encoder_module, e, exc_info=True)
         return

    def paint_image(self, coding, img_data, x, y, width, height, options, callbacks):
        """ can be called from any thread """
        # log("paint_image(%s, %s bytes, %s, %s, %s, %s, %s, %s)", coding, len(img_data), x, y, width, height, options, callbacks)
        PIL = get_codec("PIL")
        assert PIL.Image, "PIL.Image not found"
        buf = BytesIOClass(img_data)
        img = PIL.Image.open(buf)
        assert img.mode in ("L", "P", "RGB", "RGBA"), "invalid image mode: %s" % img.mode
        transparency = options.get("transparency", -1)
        if img.mode == "P":
            if transparency >= 0:
                # this deals with alpha without any extra work
                img = img.convert("RGBA")
            else:
                img = img.convert("RGB")
        elif img.mode == "L":
            if transparency >= 0:
                # why do we have to deal with alpha ourselves??
                def mask_value(a):
                    if a != transparency:
                        return 255
                    return 0

                mask = PIL.Image.eval(img, mask_value)
                mask = mask.convert("L")

                def nomask_value(a):
                    if a != transparency:
                        return a
                    return 0

                img = PIL.Image.eval(img, nomask_value)
                img = img.convert("RGBA")
                img.putalpha(mask)
            else:
                img = img.convert("RGB")

        # use tobytes() if present, fallback to tostring():
        data_fn = getattr(img, "tobytes", getattr(img, "tostring", None))
        raw_data = data_fn("raw", img.mode)
        paint_options = typedict(options)
        rgb_format = img.mode
        if rgb_format == "RGB":
            # PIL flattens the data to a continuous straightforward RGB format:
            rowstride = width * 3
            img_data = self.process_delta(raw_data, width, height, rowstride, options)
        elif rgb_format == "RGBA":
            rowstride = width * 4
            img_data = self.process_delta(raw_data, width, height, rowstride, options)
        else:
            raise Exception("invalid image mode: %s" % img.mode)
        paint_options["rgb_format"] = rgb_format
        self.idle_add(self.do_paint_rgb, rgb_format, img_data, x, y, width, height, rowstride, paint_options, callbacks)
        return False

 def init_csc_option(self, csc_name):
     csc_module = get_codec(csc_name)
     debug("init_csc_option(%s) module=%s", csc_name, csc_module)
     if csc_module is None:
         return
     csc_type = csc_module.get_type()
     try:
         csc_module.init_module()
     except Exception, e:
         log.warn("cannot use %s module %s: %s", csc_type, csc_module, e, exc_info=True)
         return

 def bgr_to_rgb(self, img_data, width, height, rowstride, rgb_format, target_format):
     if not rgb_format.startswith("BGR"):
         return img_data, rowstride
     from xpra.codecs.loader import get_codec
     #use an rgb format name that PIL will recognize:
     in_format = rgb_format.replace("X", "A")
     PIL = get_codec("PIL")
     img = PIL.Image.frombuffer(target_format, (width, height), img_data, "raw", in_format, rowstride)
     img_data = img.tobytes("raw", target_format)
     log.warn("%s converted to %s", rgb_format, target_format)
     return img_data, width*len(target_format)

def load_video_decoders():
    global VIDEO_DECODERS
    if VIDEO_DECODERS is None:
        VIDEO_DECODERS = {}
        for codec in ("vp8", "vp9", "h264"):
            #prefer native vpx ahead of avcodec:
            for module in ("dec_vpx", "dec_avcodec", "dec_avcodec2"):
                decoder = get_codec(module)
                if decoder and (codec in decoder.get_encodings()):
                    VIDEO_DECODERS[codec] = module
                    break
    log("video decoders: %s", VIDEO_DECODERS)

 def init_video_decoder_option(self, decoder_name):
     decoder_module = get_codec(decoder_name)
     log("init_video_decoder_option(%s) module=%s", decoder_name, decoder_module)
     if not decoder_module:
         log.warn("video decoder %s could not be loaded: %s", decoder_name, get_codec_error(decoder_name))
         return
     encoder_type = decoder_module.get_type()
     try:
         decoder_module.init_module()
         self._cleanup_modules.append(decoder_module)
     except Exception, e:
         log.warn("cannot use %s module %s: %s", encoder_type, decoder_module, e, exc_info=True)
         return

 def paint_webp_using_cwebp(self, img_data, x, y, width, height, options, callbacks):
     dec_webp = get_codec("dec_webp")
     has_alpha = options.get("has_alpha", False)
     buffer_wrapper, width, height, stride, has_alpha, rgb_format = dec_webp.decompress(img_data, has_alpha)
     options["rgb_format"] = rgb_format
     def free_buffer(*args):
         buffer_wrapper.free()
     callbacks.append(free_buffer)
     data = buffer_wrapper.get_pixels()
     if has_alpha:
         return self.paint_rgb32(data, x, y, width, height, stride, options, callbacks)
     else:
         return self.paint_rgb24(data, x, y, width, height, stride, options, callbacks)

 def init_csc_option(self, csc_name):
     csc_module = get_codec(csc_name)
     log("init_csc_option(%s) module=%s", csc_name, csc_module)
     if csc_module is None:
         log.warn("csc module %s could not be loaded: %s", csc_name, get_codec_error(csc_name))
         return
     csc_type = csc_module.get_type()
     try:
         csc_module.init_module()
         self._cleanup_modules.append(csc_module)
     except Exception, e:
         log.warn("cannot use %s module %s: %s", csc_type, csc_module, e)
         return

    def _do_paint_rgb(self, cairo_format, has_alpha, img_data, x, y, width, height, rowstride, options):
        """ must be called from UI thread """
        log("cairo._do_paint_rgb(%s, %s, %s bytes,%s,%s,%s,%s,%s,%s)", cairo_format, has_alpha, len(img_data), x, y, width, height, rowstride, options)
        rgb_format = options.strget("rgb_format", "RGB")
        if _memoryview and isinstance(img_data, _memoryview):
            #Pixbuf cannot use the memoryview directly:
            img_data = img_data.tobytes()
        #"cairo.ImageSurface.create_for_data" is not implemented in GTK3! ARGH!
        # http://cairographics.org/documentation/pycairo/3/reference/surfaces.html#cairo.ImageSurface.create_for_data
        # "Not yet available in Python 3"
        #
        #It is available in the cffi cairo bindings, which can be used instead of pycairo
        # but then we can't use it from the draw callbacks:
        # https://mail.gnome.org/archives/python-hackers-list/2011-December/msg00004.html
        # "PyGObject just lacks the glue code that allows it to pass the statically-wrapped
        # cairo.Pattern to introspected methods"

        if not is_gtk3() and rgb_format in ("ARGB", "XRGB"):
            #the pixel format is also what cairo expects
            #maybe we should also check that the stride is acceptable for cairo?
            #cairo_stride = cairo.ImageSurface.format_stride_for_width(cairo_format, width)
            #log("cairo_stride=%s, stride=%s", cairo_stride, rowstride)
            img_surface = cairo.ImageSurface.create_for_data(img_data, cairo_format, width, height, rowstride)
            return self.cairo_paint_surface(img_surface, x, y)

        if not is_gtk3() and rgb_format in ("RGBA", "RGBX"):
            #with GTK2, we can use a pixbuf from RGB(A) pixels
            if rgb_format=="RGBA":
                #we have to unpremultiply for pixbuf!
                img_data = self.unpremultiply(img_data)
            pixbuf = pixbuf_new_from_data(img_data, COLORSPACE_RGB, has_alpha, 8, width, height, rowstride)
            return self.cairo_paint_pixbuf(pixbuf, x, y)

        #PIL fallback
        PIL = get_codec("PIL")
        if has_alpha:
            oformat = "RGBA"
        else:
            oformat = "RGB"
        img = PIL.Image.frombuffer(oformat, (width,height), img_data, "raw", rgb_format, rowstride, 1)
        #This is insane, the code below should work, but it doesn't:
        # img_data = bytearray(img.tostring('raw', oformat, 0, 1))
        # pixbuf = pixbuf_new_from_data(img_data, COLORSPACE_RGB, True, 8, width, height, rowstride)
        # success = self.cairo_paint_pixbuf(pixbuf, x, y)
        #So we still rountrip via PNG:
        png = BytesIOClass()
        img.save(png, format="PNG")
        reader = BytesIOClass(png.getvalue())
        png.close()
        img = cairo.ImageSurface.create_from_png(reader)
        return self.cairo_paint_surface(img, x, y)

 def paint_webp(self, img_data, x, y, width, height, options, callbacks):
     dec_webp = get_codec("dec_webp")
     if not dec_webp or WEBP_PILLOW:
         #if webp is enabled, then Pillow should be able to take care of it:
         return self.paint_image("webp", img_data, x, y, width, height, options, callbacks)
     has_alpha = options.get("has_alpha", False)
     buffer_wrapper, width, height, stride, has_alpha, rgb_format = dec_webp.decompress(img_data, has_alpha, options.get("rgb_format"))
     #replace with the actual rgb format we get from the decoder:
     options["rgb_format"] = rgb_format
     def free_buffer(*args):
         buffer_wrapper.free()
     callbacks.append(free_buffer)
     data = buffer_wrapper.get_pixels()
     if len(rgb_format)==4:
         return self.paint_rgb32(data, x, y, width, height, stride, options, callbacks)
     else:
         return self.paint_rgb24(data, x, y, width, height, stride, options, callbacks)