def munge(document, template, linkrel, d, fullpath, ext, url, config):
    fixRelativeLinks(template, linkrel)
    addMtime(template, fullpath)
    removeH1(document)
    fixAPI(document, url)
    fontifyPython(document)
    addPyListings(document, d)
    addHTMLListings(document, d)
    fixLinks(document, ext)
    putInToC(template, generateToC(document))
    footnotes(document)
    notes(document)

    # Insert the document into the template
    title = domhelpers.findNodesNamed(document, 'title')[0].childNodes
    setTitle(template, title)

    authors = domhelpers.findNodesNamed(document, 'link')
    authors = [(n.getAttribute('title',''), n.getAttribute('href', ''))
               for n in authors if n.getAttribute('rel', '') == 'author']
    setAuthors(template, authors)

    body = domhelpers.findNodesNamed(document, "body")[0]
    tmplbody = domhelpers.findElementsWithAttribute(template, "class",
                                                              "body")[0]
    tmplbody.childNodes = body.childNodes
    tmplbody.setAttribute("class", "content")

 def check_lists(self, dom, filename):
     for node in domhelpers.findNodesNamed(dom, "ul") + domhelpers.findNodesNamed(dom, "ol"):
         if not node.childNodes:
             self._reportError(filename, node, "empty list")
         for child in node.childNodes:
             if child.nodeName != "li":
                 self._reportError(filename, node, "only list items allowed in lists")

def makeBook(dom, d):
    body = microdom.Element('body')
    body.appendChild(domhelpers.findNodesNamed(dom, 'h1')[0])
    toc = domhelpers.findElementsWithAttribute(dom, 'class', 'toc')[0]
    toc = domhelpers.findNodesNamed(toc, 'li')
    for node in toc:
        if (node.hasAttribute('class') and
            node.getAttribute('class')=='tocignore'):
             continue
        parents = domhelpers.getParents(node)
        nodeLevel = len([1 for parent in parents if hasattr(parent, 'tagName')
                                           and parent.tagName in ('ol', 'ul')])
        data = node.childNodes[0].data != ''
        if not data:
            node = node.childNodes[1]
            newNode = lowerDocument(node.getAttribute('href'), d, nodeLevel)
            for child in newNode.childNodes:
                body.appendChild(child)
        else:
            text = microdom.Text(node.childNodes[0].data)
            newNode = microdom.Element('h'+str(nodeLevel))
            newNode.appendChild(text)
            body.appendChild(newNode)
    origBody = domhelpers.findNodesNamed(dom, 'body')[0]
    origBody.parentNode.replaceChild(body, origBody)

    def check_80_columns(self, dom, filename):
        for node in domhelpers.findNodesNamed(dom, 'pre'):
            # the ps/pdf output is in a font that cuts off at 80 characters,
            # so this is enforced to make sure the interesting parts (which
            # are likely to be on the right-hand edge) stay on the printed
            # page.
            for line in domhelpers.gatherTextNodes(node, 1).split('\n'):
                if len(line.rstrip()) > 80:
                    self._reportError(filename, node, 
                                      'text wider than 80 columns in pre')
        for node in domhelpers.findNodesNamed(dom, 'a'):
            if node.getAttribute('class', '').endswith('listing'):
                try:
                    fn = os.path.dirname(filename) 
                    fn = os.path.join(fn, node.getAttribute('href'))
                    lines = open(fn,'r').readlines()
                except:
                    self._reportError(filename, node,
                                      'bad listing href: %r' %
                                      node.getAttribute('href'))
                    continue

                for line in lines:
                    if len(line.rstrip()) > 80:
                        self._reportError(filename, node, 
                                          'listing wider than 80 columns')

def getTitleLink(url):
    d = client.getPage("http://moshez.org/discuss/rss")
    d.addCallback(microdom.parseString)
    d.addCallback(lambda d: domhelpers.findNodesNamed(d, 'item')[0])
    d.addCallback(lambda d: (
               domhelpers.getNodeText(domhelpers.findNodesNamed(d, 'link')[0]),
               domhelpers.getNodeText(domhelpers.findNodesNamed(d, 'title')[0]),
              ))
    return d

 def check_lists(self, dom, filename):
     for node in (domhelpers.findNodesNamed(dom, 'ul')+
                  domhelpers.findNodesNamed(dom, 'ol')):
         if not node.childNodes:
             self._reportError(filename, node, 'empty list')
         for child in node.childNodes:
             if child.nodeName != 'li':
                 self._reportError(filename, node,
                                   'only list items allowed in lists')

 def check_title(self, dom, filename):
     doc = dom.documentElement
     title = domhelpers.findNodesNamed(dom, 'title')
     if len(title)!=1:
         return self._reportError(filename, doc, 'not exactly one title')
     h1 = domhelpers.findNodesNamed(dom, 'h1')
     if len(h1)!=1:
         return self._reportError(filename, doc, 'not exactly one h1')
     if domhelpers.getNodeText(h1[0]) != domhelpers.getNodeText(title[0]):
         self._reportError(filename, h1[0], 'title and h1 text differ')

def extractRSSFeeds(dom):
    for bookmark in domhelpers.findNodesNamed(dom, "bookmark"):
        titleNodes = domhelpers.findNodesNamed(bookmark, "title")
        if titleNodes:
            title = domhelpers.getNodeText(titleNodes[0])
        else:
            title = None

        url = bookmark.getAttribute("href")
        url = urllib.unquote(url)
        yield title, url

def findRSSFolder(dom, folderName=None):
    if folderName is None:
        folderName = "RSS"
    if folderName == "":
        return dom
    for folder in domhelpers.findNodesNamed(dom, "folder"):
        for title in domhelpers.findNodesNamed(folder, "title"):
            text = domhelpers.getNodeText(title)
            if text == folderName:
                return folder
    return None

def numberDocument(document, chapterNumber):
    """
    Number the sections of the given document.

    A dot-separated chapter, section number is added to the beginning of each
    section, as defined by C{h2} nodes.

    This is probably intended to interact in a rather specific way with
    L{getSectionNumber}.

    @type document: A DOM Node or Document
    @param document: The input document which contains all of the content to be
    presented.

    @type chapterNumber: C{int}
    @param chapterNumber: The chapter number of this content in an overall
    document.

    @return: C{None}
    """
    i = 1
    for node in domhelpers.findNodesNamed(document, "h2"):
        label = dom.Text()
        label.data = "%s.%d " % (chapterNumber, i)
        node.insertBefore(label, node.firstChild)
        i += 1

 def getDescription(self):
     # http://purl.org/dc/elements/1.1/ description
     l = domhelpers.findNodesNamed(self.dom, 'description')
     if l:
         return domhelpers.getNodeText(l[0])
     else:
         return None

    def testAwfulTagSoup(self):
        s = """
        <html>
        <head><title> I send you this message to have your advice!!!!</titl e
        </headd>

        <body bgcolor alink hlink vlink>

        <h1><BLINK>SALE</blINK> TWENTY MILLION EMAILS & FUR COAT NOW
        FREE WITH `ENLARGER'</h1>

        YES THIS WONDERFUL AWFER IS NOW HERER!!!

        <script LANGUAGE="javascript">
function give_answers() {
if (score < 70) {
alert("I hate you");
}}
        </script><a href=/foo.com/lalal name=foo>lalal</a>
        </body>
        </HTML>
        """
        d = microdom.parseString(s, beExtremelyLenient=1)
        l = domhelpers.findNodesNamed(d.documentElement, 'blink')
        self.assertEquals(len(l), 1)

def setAuthors(template, authors):
    # First, similarly to setTitle, insert text into an <div class="authors">
    text = ''
    for name, href in authors:
        # FIXME: Do proper quoting/escaping (is it ok to use
        # xml.sax.saxutils.{escape,quoteattr}?)
        anchor = '<a href="%s">%s</a>' % (href, name)
        if (name, href) == authors[-1]:
            if len(authors) == 1:
                text = anchor
            else:
                text += 'and ' + anchor
        else:
            text += anchor + ','

    childNodes = microdom.parseString('<span>' + text +'</span>').childNodes

    for node in domhelpers.findElementsWithAttribute(template, 
                                                     "class", 'authors'):
        node.childNodes.extend(childNodes) 

    # Second, add appropriate <link rel="author" ...> tags to the <head>.
    head = domhelpers.findNodesNamed(template, 'head')[0]
    authors = [microdom.parseString('<link rel="author" href="%s" title="%s"/>'
                                    % (href, name)).childNodes[0]
               for name, href in authors]
    head.childNodes.extend(authors)

def getFirstAncestorWithSectionHeader(entry):
    """Go up ancestors until one with at least one <h2> is found, then return the <h2> nodes"""
    for a in domhelpers.getParents(entry)[1:]:
        headers = domhelpers.findNodesNamed(a, "h2")
        if len(headers) > 0:
            return headers
    return []

def setTitle(template, title, chapterNumber):
    """
    Add title and chapter number information to the template document.

    The title is added to the end of the first C{title} tag and the end of the
    first tag with a C{class} attribute set to C{title}.  If specified, the
    chapter is inserted before the title.

    @type template: A DOM Node or Document
    @param template: The output template which defines the presentation of the
    version information.

    @type title: C{list} of DOM Nodes
    @param title: Nodes from the input document defining its title.

    @type chapterNumber: C{int}
    @param chapterNumber: The chapter number of this content in an overall
    document.  If not applicable, any C{False} value will result in this
    information being omitted.

    @return: C{None}
    """
    for nodeList in (
        domhelpers.findNodesNamed(template, "title"),
        domhelpers.findElementsWithAttribute(template, "class", "title"),
    ):
        if nodeList:
            if numberer.getNumberSections() and chapterNumber:
                nodeList[0].childNodes.append(microdom.Text("%s. " % chapterNumber))
            nodeList[0].childNodes.extend(title)

def footnotes(document):
    """
    Find footnotes in the given document, move them to the end of the body, and
    generate links to them.

    A footnote is any node with a C{class} attribute set to C{footnote}.
    Footnote links are generated as superscript.  Footnotes are collected in a
    C{ol} node at the end of the document.

    @type document: A DOM Node or Document
    @param document: The input document which contains all of the content to be
    presented.

    @return: C{None}
    """
    footnotes = domhelpers.findElementsWithAttribute(document, "class", "footnote")
    if not footnotes:
        return
    footnoteElement = microdom.Element("ol")
    id = 1
    for footnote in footnotes:
        href = microdom.parseString('<a href="#footnote-%(id)d">' "<super>%(id)d</super></a>" % vars()).documentElement
        text = " ".join(domhelpers.getNodeText(footnote).split())
        href.setAttribute("title", text)
        target = microdom.Element("a", attributes={"name": "footnote-%d" % id})
        target.childNodes = [footnote]
        footnoteContent = microdom.Element("li")
        footnoteContent.childNodes = [target]
        footnoteElement.childNodes.append(footnoteContent)
        footnote.parentNode.replaceChild(href, footnote)
        id += 1
    body = domhelpers.findNodesNamed(document, "body")[0]
    header = microdom.parseString("<h2>Footnotes</h2>").documentElement
    body.childNodes.append(header)
    body.childNodes.append(footnoteElement)

def setTitle(template, title, chapterNumber):
    for nodeList in (domhelpers.findNodesNamed(template, "title"),
                     domhelpers.findElementsWithAttribute(template, "class",
                                                          'title')):
        if nodeList:
            if numberer.getNumberSections() and chapterNumber:
                nodeList[0].childNodes.append(microdom.Text('%s. ' % chapterNumber))
            nodeList[0].childNodes.extend(title)

 def render_GET(self, request):
     url = urlTemplate % request.args['user'][0]
     client.getPage(url).addCallback(
     microdom.parseString).addCallback(
     lambda t: domhelpers.findNodesNamed(t, 'item')).addCallback(
     lambda itms: zip([domhelpers.findNodesNamed(x, 'title')[0]
                                                            for x in itms],
                      [domhelpers.findNodesNamed(x, 'link')[0]
                                                            for x in itms]
                     )).addCallback(
     lambda itms: '<html><head></head><body><ul>%s</ul></body></html>' %
                       '\n'.join(
            ['<li><a href="%s">%s</a></li>' % (
               domhelpers.getNodeText(link), domhelpers.getNodeText(title))
                    for (title, link) in itms])
     ).addCallback(lambda s: (request.write(s),request.finish())).addErrback(
     lambda e: (request.write('Error: %s' % e),request.finish()))
     return server.NOT_DONE_YET

def lowerDocument(href, d, nodeLevel):
    newNode = microdom.parse(open(os.path.join(d, href)))
    newNode = domhelpers.findNodesNamed(newNode, 'body')[0]
    headers = domhelpers.findElements(newNode,
              lambda x: len(x.tagName)==2 and x.tagName[0]=='h' and
                        x.tagName[1] in '123456')
    for header in headers:
        header.tagName = 'h'+str(int(header.tagName[1])+nodeLevel)
    return newNode

 def check_texturl_matches_href(self, dom, filename):
     for node in domhelpers.findNodesNamed(dom, "a"):
         if not node.hasAttribute("href"):
             continue
         text = domhelpers.getNodeText(node)
         proto = urlparse.urlparse(text)[0]
         if proto and " " not in text:
             if text != node.getAttribute("href"):
                 self._reportError(filename, node, "link text does not match href")