def cook_layout(layout, ajax):
    """Return main_template compatible layout"""
    # Fix XHTML layouts with CR[+LF] line endings
    layout = re.sub('\r', '\n', re.sub('\r\n', '\n', layout))

    # Parse layout
    if isinstance(layout, unicode):
        result = getHTMLSerializer([layout.encode('utf-8')], encoding='utf-8')
    else:
        result = getHTMLSerializer([layout], encoding='utf-8')

    # Fix XHTML layouts with inline js (etree.tostring breaks all <![CDATA[)
    if '<![CDATA[' in layout:
        result.serializer = html.tostring

    # Wrap all panels with a metal:fill-slot -tag:
    all_slots = []
    for layoutPanelNode in slotsXPath(result.tree):
        data_slots = layoutPanelNode.attrib['data-slots']
        all_slots += wrap_append_prepend_slots(layoutPanelNode, data_slots)
        del layoutPanelNode.attrib['data-slots']

    # When no slots are explicitly defined, try to inject the very default
    # slots
    if len(all_slots) == 0:
        for node in result.tree.xpath('//*[@data-panel="content"]'):
            wrap_append_prepend_slots(
                node, 'content > body header main * content-core')

    # Append implicit slots
    head = result.tree.getroot().find('head')
    if not ajax and head is not None:
        for name in ['top_slot', 'head_slot',
                     'style_slot', 'javascript_head_slot']:
            slot = etree.Element('{%s}%s' % (NSMAP['metal'], name),
                                 nsmap=NSMAP)
            slot.attrib['define-slot'] = name
            head.append(slot)

    template = """\
<metal:page define-macro="master"
            tal:define="portal_state context/@@plone_portal_state;
                        context_state context/@@plone_context_state;
                        plone_view context/@@plone;
                        plone_layout context/@@plone_layout | nothing;
                        lang portal_state/language;
                        view nocall: view | nocall: plone_view;
                        dummy python:plone_view.mark_view(view);
                        portal_url portal_state/portal_url;
                        checkPermission nocall: context/portal_membership/checkPermission;
                        site_properties nocall: context/portal_properties/site_properties;
                        ajax_include_head request/ajax_include_head | nothing;
                        ajax_load request/ajax_load | python: False;
                        toolbar_class python:request.cookies.get('plone-toolbar', 'plone-toolbar-left pat-toolbar');
                        dummy python:request.RESPONSE.setHeader('X-UA-Compatible', 'IE=edge,chrome=1');">
%s
</metal:page>"""
    metal = 'xmlns:metal="http://namespaces.zope.org/metal"'
    return (template % ''.join(result)).replace(metal, '')

    def transformIterable(self, result, encoding):

        try:
            parser = SnippetParser()
        except AttributeError:
            return result

        if self.request['PATH_INFO'].endswith('edit'):
            return result

        contentType = self.request.response.getHeader('Content-Type')
        if contentType is None or not contentType.startswith('text/html'):
            return None

        ce = self.request.response.getHeader('Content-Encoding')
        if ce and ce in ('zip', 'deflate', 'compress'):
            return None
        try:
            if result == ['']:
                return None

            result = getHTMLSerializer(result, pretty_print=False)
        except (TypeError):
            return None

        return [ parser.parsePage(r) for r in result ]

def cook_layout(layout):
    """Return main_template compatible layout"""
    result = getHTMLSerializer(layout, encoding='utf-8')
    nsmap = {'metal': 'http://namespaces.zope.org/metal'}

    # wrap all panels with a metal:fill-slot -tag
    for layoutPanelNode in panelXPath(result.tree):
        panelId = layoutPanelNode.attrib['data-panel']
        slot = etree.Element('{%s}%s' % (nsmap['metal'], panelId), nsmap=nsmap)
        slot.attrib['define-slot'] = panelId
        slot_parent = layoutPanelNode.getparent()
        slot_parent_index = slot_parent.index(layoutPanelNode)
        slot.append(layoutPanelNode)
        slot_parent.insert(slot_parent_index, slot)

    root = result.tree.getroot()
    root.attrib['tal:define'] = """\
portal_state python:context.restrictedTraverse('@@plone_portal_state');
context_state python:context.restrictedTraverse('@@plone_context_state');
plone_view python:context.restrictedTraverse('@@plone');
lang portal_state/language;
view nocall:view | nocall: plone_view;
dummy python: plone_view.mark_view(view);
portal_url portal_state/portal_url;
checkPermission nocall: context/portal_membership/checkPermission;
site_properties nocall:context/portal_properties/site_properties;
ajax_load request/ajax_load | nothing;
ajax_include_head request/ajax_include_head | nothing;
dummy python:request.RESPONSE.setHeader('X-UA-Compatible', 'IE=edge,chrome=1');
dummy python:options.update({'state': options.get('state', request.get('controller_state'))});
"""

    template = '<metal:page define-macro="master">\n%s\n</metal:page>'
    metal = 'xmlns:metal="http://namespaces.zope.org/metal"'
    return (template % ''.join(result)).replace(metal, '')

    def parseTree(self, result, encoding):
        # if it's a redirect, the result shall not be transformed
        request = getRequest()
        if request.response.status in (301, 302):
            return None

        if isinstance(result, XMLSerializer):
            return result

        # hhmmm, this is kind of taken right out of plone.app.theming
        # maybe this logic(parsing dom) should be someone central?
        contentType = self.request.response.getHeader('Content-Type')
        if contentType is None or not contentType.startswith('text/html'):
            return None

        contentEncoding = self.request.response.getHeader('Content-Encoding')
        if contentEncoding and contentEncoding in ('zip', 'deflate',
                                                   'compress',):
            return None

        try:
            result = getHTMLSerializer(
                result, pretty_print=False, encoding=encoding)
            # We are going to force html output here always as XHTML
            # output does odd character encodings
            result.serializer = html.tostring
            return result
        except (TypeError, etree.ParseError):
            # XXX handle something special?
            LOGGER.warn('error parsing dom, failure to add csrf '
                        'token to response for url %s' % self.request.URL)
            return None

    def render(self):
        # Render fields by iterating over the form field tiles, rendering
        # each one, and replacing the tile with the result.
        layout = ILayoutAware(self.context).content
        layoutTree = getHTMLSerializer(layout,
            pretty_print=True, encoding='utf8')
        
        for tileId, tile in self.context.field_tiles(layoutTree.tree):
            # XXX need to include the full Plone view of the field
            widget = self.widgets[tile.id]
            widgetRenderer = getMultiAdapter((widget, self.request), name=u'ploneform-render-widget')
            widgetHtml = widgetRenderer()
            tileTree = html.fromstring(widgetHtml).getroottree()
            tileTarget = xpath1("//*[@id='%s']" % tileId, layoutTree.tree)
            
            if tileTarget is None:
                continue
            
            # clear children, but keep attributes
            oldAttrib = dict(tileTarget.attrib)
            tileTarget.clear()
            tileTarget.attrib.update(oldAttrib)

            # insert tile target with tile body
            tileBody = tileTree.find('body')
            if tileBody is not None:
                tileTarget.text = tileBody.text
                for tileBodyChild in tileBody:
                    tileTarget.append(tileBodyChild)
        
        # TODO:
        # - create form tag
        # - fill in status message
        return str(layoutTree)

    def transformIterable(self, result, encoding):
        if self.request.get('plone.app.blocks.disabled', False):
            return None

        content_type = self.request.response.getHeader('Content-Type')
        if content_type is None or not content_type.startswith('text/html'):
            return None

        contentEncoding = self.request.response.getHeader('Content-Encoding')
        if contentEncoding and contentEncoding in ('zip', 'deflate',
                                                   'compress',):
            return None

        try:
            # Fix layouts with CR[+LF] line endings not to lose their heads
            # (this has been seen with downloaded themes with CR[+LF] endings)
            iterable = [re.sub('
', '\n', re.sub('
\n', '\n', item))
                        for item in result if item]
            result = getHTMLSerializer(
                iterable, pretty_print=self.pretty_print, encoding=encoding)
            # Fix XHTML layouts with where etree.tostring breaks <![CDATA[
            if any(['<![CDATA[' in item for item in iterable]):
                result.serializer = html.tostring
            self.request['plone.app.blocks.enabled'] = True
            return result
        except (AttributeError, TypeError, etree.ParseError):
            return None

    def __init__(self, context, request, tile=None):
        self.context = context
        self.request = request
        self.tile = tile

        # Parse layout
        data_layout = (ILayoutAware(self.context).content or DATA_LAYOUT)
        self.storage = getHTMLSerializer([data_layout.encode('utf-8')],
                                         encoding='utf-8')

    def _parse(self, result):
        """Create an XMLSerializer from an HTML string, if needed."""
        content_type = self.request.response.getHeader('Content-Type')
        if not content_type or not content_type.startswith('text/html'):
            return

        try:
            return getHTMLSerializer(result)
        except (AttributeError, TypeError, etree.ParseError):
            return

 def test_getHTMLSerializer_doctype_xhtml_serializes_to_xhtml(self):
     t = utils.getHTMLSerializer(self.create_iterable(preamble='<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n', body='<img src="foo.png" />'), pretty_print=True)
     self.failUnless(isinstance(t, serializer.XMLSerializer))
     
     t2 = utils.getXMLSerializer(t)
     self.failUnless(t2 is t)
     
     self.assertEqual(
         '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n<html xmlns="http://www.w3.org/1999/xhtml">\n  <head>\n    <meta http-equiv="Content-Type" content="text/html; charset=ASCII" />\n    <title>My homepage</title>\n  </head>\n  <body>Hello, world!<img src="foo.png" /></body>\n</html>\n',
         "".join(t2))

 def test_getHTMLSerializer(self):
     t = utils.getHTMLSerializer(self.create_iterable(body='<img src="foo.png" />'), pretty_print=True)
     self.failUnless(isinstance(t, serializer.XMLSerializer))
     
     t2 = utils.getXMLSerializer(t)
     self.failUnless(t2 is t)
     
     self.assertEqual(
         '<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">\n<html>\n<head><title>My homepage</title></head>\n<body>Hello, world!<img src="foo.png">\n</body>\n</html>\n',
         "".join(t2))

    def parseTree(self, result):
        contentType = self.request.response.getHeader("Content-Type")
        if contentType is None or not contentType.startswith("text/html"):
            return None

        contentEncoding = self.request.response.getHeader("Content-Encoding")
        if contentEncoding and contentEncoding in ("zip", "deflate", "compress"):
            return None

        try:
            return getHTMLSerializer(result, pretty_print=False)
        except (TypeError, etree.ParseError):
            return None

    def parseTree(self, result):
        contentType = self.request.response.getHeader('Content-Type')
        if contentType is None or not contentType.startswith('text/html'):
            return None

        contentEncoding = self.request.response.getHeader('Content-Encoding')
        if contentEncoding and contentEncoding in ('zip', 'deflate', 'compress',):
            return None

        try:
            return getHTMLSerializer(result, pretty_print=False)
        except (TypeError, etree.ParseError):
            return None

def transform(doc, rules, extras={}, throw_errors=False):
    if isinstance(doc, basestring):
        doc = getHTMLSerializer([doc])

    root = doc.tree.getroot()

    for rule in rules:
        try:
            rule(root, extras)
        except:
            if throw_errors:
                raise

    return doc

    def test_replace_doctype_blank(self):
        t = utils.getHTMLSerializer(self.create_iterable(preamble='<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n', body='<img src="foo.png" />'), pretty_print=True, doctype="")
        self.failUnless(isinstance(t, serializer.XMLSerializer))

        t2 = utils.getXMLSerializer(t)
        self.failUnless(t2 is t)
        
        self.assertEqual(
            b'<html xmlns="http://www.w3.org/1999/xhtml">\n  <head>\n    <meta http-equiv="Content-Type" content="text/html; charset=ASCII" />\n    <title>My homepage</title>\n  </head>\n  <body>Hello, w&#246;rld!<img src="foo.png" /></body>\n</html>\n',
            b"".join(t2))

        self.assertEqual(
            u'<html xmlns="http://www.w3.org/1999/xhtml">\n  <head>\n    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />\n    <title>My homepage</title>\n  </head>\n  <body>Hello, wörld!<img src="foo.png" /></body>\n</html>\n',
            u"".join(t2.serialize(encoding=unicode)))

 def test_xsl(self):
     t = utils.getHTMLSerializer(self.create_iterable(body='<br>'))
     transform = lxml.etree.XSLT(lxml.etree.XML(b'''
         <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
             <xsl:output method="xml" indent="no" omit-xml-declaration="yes"
                 media-type="text/html" encoding="UTF-8"
                 doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN"
                 doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"
                 />
             <xsl:template match="@*|node()">
               <xsl:copy>
                 <xsl:apply-templates select="@*|node()"/>
               </xsl:copy>
             </xsl:template>
         </xsl:stylesheet>
         '''))
     t.tree = transform(t.tree)
     self.failUnless('<br />' in str(t))

    def transformIterable(self, result, encoding):
        if self.request.get("plone.app.blocks.disabled", False):
            return None

        content_type = self.request.response.getHeader("Content-Type")
        if content_type is None or not content_type.startswith("text/html"):
            return None

        contentEncoding = self.request.response.getHeader("Content-Encoding")
        if contentEncoding and contentEncoding in ("zip", "deflate", "compress"):
            return None

        try:
            result = getHTMLSerializer(result, pretty_print=self.pretty_print, encoding=encoding)
            self.request["plone.app.blocks.enabled"] = True
            return result
        except (TypeError, etree.ParseError):
            return None

def protect(raw):
    if isinstance(raw, six.text_type):
        raw = raw.encode('utf-8')
    parser = getHTMLSerializer(
        [raw], pretty_print=False, encoding='utf-8')
    parser.serializer = html.tostring

    root = parser.tree.getroot()
    token = createToken()

    for form in root.cssselect('form'):
        authenticator = form.cssselect('[name="_authenticator"]')
        if len(authenticator) == 0:
            authenticator = etree.Element("input")
            authenticator.attrib['name'] = '_authenticator'
            authenticator.attrib['type'] = 'hidden'
            authenticator.attrib['value'] = token
            form.append(authenticator)
    return parser.serialize().decode('utf-8')

    def parseTree(self, result):
        # hhmmm, this is kind of taken right out of plone.app.theming
        # maybe this logic(parsing dom) should be someone central?
        contentType = self.request.response.getHeader('Content-Type')
        if contentType is None or not contentType.startswith('text/html'):
            return None

        contentEncoding = self.request.response.getHeader('Content-Encoding')
        if contentEncoding and contentEncoding in ('zip', 'deflate',
                                                   'compress',):
            return None

        try:
            return getHTMLSerializer(result, pretty_print=False)
        except (TypeError, etree.ParseError):
            # XXX handle something special?
            LOGGER.warn('error parsing dom, failure to add csrf '
                        'token to response for url %s' % self.request.URL)
            return None

    def transformIterable(self, result, encoding):
        if self.request.get('plone.app.blocks.disabled', False):
            return None

        content_type = self.request.response.getHeader('Content-Type')
        if content_type is None or not content_type.startswith('text/html'):
            return None

        contentEncoding = self.request.response.getHeader('Content-Encoding')
        if contentEncoding and contentEncoding in ('zip', 'deflate',
                                                   'compress',):
            return None

        try:
            result = getHTMLSerializer(result, pretty_print=True,
                                       encoding=encoding)
            self.request['plone.app.blocks.enabled'] = True
            return result
        except (TypeError, etree.ParseError):
            return None

 def __call__(self, environ, start_response):
     request = Request(environ)
     response = request.get_response(self.app)
     
     app_iter = response(environ, start_response)
     
     if self.should_ignore(request) or not self.should_transform(response):
         return app_iter
     
     # Set up parameters
     
     params = {}
     for key, value in self.environ_param_map.items():
         if key in environ:
             params[value] = quote_param(environ[key])
     for key, value in self.params.items():
         params[key] = quote_param(value)
     
     # Apply the transformation
     app_iter = getHTMLSerializer(app_iter)
     tree = self.transform(app_iter.tree, **params)
     
     # Set content type and choose XHTML or HTML serializer
     serializer = html.tostring
     response.headers['Content-Type'] = 'text/html'
     
     if tree.docinfo.doctype and 'XHTML' in tree.docinfo.doctype:
         serializer = etree.tostring
         response.headers['Content-Type'] = 'application/xhtml+xml'
     
     app_iter = XMLSerializer(tree, serializer=serializer)
     
     # Calculate the content length - we still return the parsed tree
     # so that other middleware could avoid having to re-parse, even if
     # we take a hit on serialising here
     if self.update_content_length and 'Content-Length' in response.headers:
         response.headers['Content-Length'] = str(len(str(app_iter)))
     
     # Return a repoze.xmliter XMLSerializer, which helps avoid re-parsing
     # the content tree in later middleware stages
     return app_iter