def html_to_etree(in_str, remove_blank_text=True):
    """
    Parses a tree of possibly malformed HTML5, according to WHATWG HTML5 rules.

    Result is either:
     - parsed input, or;
     - if multiple fragments (> 1 top-level tags) are given: parsed input wrapped in either a `div' or `span', or;
     - None for empty input.
    """
    if in_str is None:
        return None

    if not isinstance(in_str, basestring):
        raise ValueError('input must be a string')

    in_str = _nfc(in_str).strip()

    if not in_str:
        return None

    # Double-parse to remove (hopefully irrelevant) whitespace - some not-so-irrelevant whitespace will most likely be removed too
    etree = fromstring(in_str, parser=_html5Parser) # ATTENTION: tag/attributes namespace-info mangled here due to html5lib bugs.
    _etree_mutate_fix_localname(etree)
    if remove_blank_text:
        s = lxmltostring(etree)
        etree = parse(StringIO(s), parser=_xmlParser)
        etree = fromstring(lxmltostring(etree), parser=_html5Parser)
        _etree_mutate_fix_localname(etree)  # and they spawn again after fromstring, so remove them again.

    return etree.getroot() if hasattr(etree, 'getroot') else etree

    def testQueryTimeInExtraResponse(self):
        handler = SruHandler(includeQueryTimes=True)
        observer = CallTrace('observer', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData'])

        times = [1, 2.5, 3.5]
        def timeNow():
            return times.pop(0)
        handler._timeNow = timeNow

        def executeQuery(**kwargs):
            response = Response(total=0, hits=[])
            response.queryTime=5
            raise StopIteration(response)
            yield
        observer.methods['executeQuery'] = executeQuery
        handler.addObserver(observer)
        arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema')
        result = "".join(compose(handler.searchRetrieve(sruArguments=arguments, **arguments)))
        sruResponse = parse(StringIO(result))
        extraResponseData = sruResponse.xpath('/srw:searchRetrieveResponse/srw:extraResponseData', namespaces={'srw':"http://www.loc.gov/zing/srw/"})[0]
        self.assertEqualsWS("""<srw:extraResponseData %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s>
        <querytimes xmlns="http://meresco.org/namespace/timing">
            <sruHandling>PT2.500S</sruHandling>
            <sruQueryTime>PT1.500S</sruQueryTime>
            <index>PT0.005S</index>
        </querytimes>
</srw:extraResponseData>""" % namespaces, lxmltostring(extraResponseData))
        queryTimes = lxmltostring(extraResponseData.xpath('//ti:querytimes', namespaces={'ti':"http://meresco.org/namespace/timing"})[0])
        assertValid(queryTimes, join(schemasPath, 'timing-20120827.xsd'))
        self.assertEquals(['executeQuery', 'echoedExtraRequestData', 'extraResponseData', 'handleQueryTimes'], observer.calledMethodNames())
        self.assertEquals({'sru': Decimal("2.500"), 'queryTime': Decimal("1.500"), 'index': Decimal("0.005")}, observer.calledMethods[3].kwargs)

    def testListMetadataFormatsForIdentifier(self):
        header, body = self._request(verb=['ListMetadataFormats'], identifier=[self.prefix + 'record:id:01'])

        self.assertEquals(0, len(xpath(body, '/oai:OAI-PMH/oai:error')), lxmltostring(body, pretty_print=True))
        formats = xpath(body, '/oai:OAI-PMH/oai:ListMetadataFormats/oai:metadataFormat')
        self.assertEquals(1, len(formats), lxmltostring(body, pretty_print=True))
        self.assertEquals(['oai_dc'], xpath(formats[0], 'oai:metadataPrefix/text()'))

    def testTailTakenCareOfWithoutAffectingOriginal(self):
        observer = CallTrace('observer', methods={'test': lambda *args, **kwargs: (x for x in [])})
        observable = be(
            (Observable(),
                (XmlXPath(
                        ['/myns:root/myns:path'],
                        fromKwarg='lxmlNode',
                        namespaces={'myns': 'http://myns.org/'}
                    ),
                    (observer, ),
                )
            )
        )

        XML = """\
<root xmlns:myns="http://myns.org/" xmlns="http://myns.org/">
    <myns:path>
        <to>me</to>
    </myns:path>\n
</root>"""

        lxmlNode = parse(StringIO(XML))
        self.assertEquals(XML, lxmltostring(lxmlNode))
        list(compose(observable.all.test('een tekst', lxmlNode=lxmlNode)))

        self.assertEquals(1, len(observer.calledMethods))
        method = observer.calledMethods[0]
        self.assertEquals('test', method.name)
        self.assertEqualsWS('<myns:path xmlns:myns="http://myns.org/" xmlns="http://myns.org/"><to>me</to></myns:path>', lxmltostring(method.kwargs['lxmlNode']))
        self.assertEquals("""\
<myns:path xmlns:myns="http://myns.org/" xmlns="http://myns.org/">
        <to>me</to>
    </myns:path>""", lxmltostring(method.kwargs['lxmlNode']))

        self.assertEquals(XML, lxmltostring(lxmlNode))

 def assertWaterMarked(**oaiArgs):
     header, body = self._request(**oaiArgs)
     try:
         comment = xpath(body, "/oai:OAI-PMH/comment()")[0]
     except:
         print lxmltostring(body, pretty_print=True)
         raise
     self.assertEquals(" Watermarked by Seecr ", comment.text)

    def testNoLxmlTailOnPart(self):
        inputEvent = fromstring("""<document><part name="partone">&lt;some&gt;message&lt;/some&gt;\n\n\n\n</part><part name="parttwo"><second>message</second>\n\n\n\n</part></document>""")
        interceptor = CallTrace('Interceptor', methods={'add': yieldNothing})
        v = createVenturiHelix([{'partname': 'partone', 'xpath': '/document/part[@name="partone"]/text()'}, {'partname': 'parttwo', 'xpath': '/document/part/second'}], [], interceptor)
        list(compose(v.all.add('identifier', 'document', inputEvent)))

        self.assertEquals('<some>message</some>', lxmltostring(interceptor.calledMethods[1].kwargs['lxmlNode']))
        secondXml = interceptor.calledMethods[2].kwargs['lxmlNode']
        self.assertEquals('<second>message</second>', lxmltostring(secondXml))

    def testFindUsingMultipleXPaths(self):
        self.createXmlXPath(['/does/not/exist', '/a/b', '/a/b/c'], {})

        self.observable.do.test(data='<a><b><c>one</c></b><b><d>two</d></b></a>')

        self.assertEquals(3, len(self.observer.calledMethods))
        allResults = []
        for method in self.observer.calledMethods:
            allResults.append(method.kwargs['lxmlNode'])
        self.assertEqualsWS('<b><c>one</c></b>', lxmltostring(allResults[0]))
        self.assertEqualsWS('<b><d>two</d></b>', lxmltostring(allResults[1]))
        self.assertEqualsWS('<c>one</c>', lxmltostring(allResults[2]))

    def testPartsWithUnicodeChars(self):
        inputEvent = fromstring("""<document><part name="partone">&lt;some&gt;t€xt&lt;/some&gt;\n\n\n\n</part><part name="parttwo"><second>t€xt</second>\n\n\n\n</part></document>""")
        interceptor = CallTrace('Interceptor', methods={'add': yieldNothing})
        v = createVenturiHelix([{'partname': 'partone', 'xpath': '/document/part[@name="partone"]/text()'}, {'partname': 'parttwo', 'xpath': '/document/part/second'}], [], interceptor)
        list(compose(v.all.add('identifier', 'document', inputEvent)))

        firstXml = interceptor.calledMethods[1].kwargs['lxmlNode']
        self.assertEquals('<some>t€xt</some>', lxmltostring(firstXml))
        self.assertEquals('t€xt', firstXml.getroot().text)
        secondXml = interceptor.calledMethods[2].kwargs['lxmlNode']
        self.assertEquals('<second>t€xt</second>', lxmltostring(secondXml))
        self.assertEquals('t€xt', secondXml.getroot().text)

    def testSimpleXPath(self):
        self.createXmlXPath(['/root/path'], {})

        xml = '<root><path><to>me</to></path>\n</root>'
        self.observable.do.test('een tekst', data=xml)

        self.assertEquals(1, len(self.observer.calledMethods))
        method = self.observer.calledMethods[0]
        self.assertEquals('test', method.name)
        self.assertEquals(1, len(method.args))
        self.assertEquals('een tekst', method.args[0])
        self.assertEqualsWS('<path><to>me</to></path>', lxmltostring(method.kwargs['lxmlNode']))
        self.assertEquals('<path><to>me</to></path>', lxmltostring(method.kwargs['lxmlNode']))

    def testXPathWithMultipleResults(self):
        self.createXmlXPath(['/root/element/data'], {})

        self.observable.do.aMethod(data="""<root>
    <element>
        <data>one</data>
    </element>
    <element>
        <data>two</data>
    </element>
</root>""")
        self.assertEquals(2, len(self.observer.calledMethods))
        self.assertEqualsWS('<data>one</data>', lxmltostring(self.observer.calledMethods[0].kwargs['lxmlNode']))
        self.assertEqualsWS('<data>two</data>', lxmltostring(self.observer.calledMethods[1].kwargs['lxmlNode']))

    def testTestWithCondition(self):
        self.createXmlXPath(['/a/*[not(self::b) and not(self::c)]'], {})

        self.observable.do.test(data='<a><b>zero</b><c>one</c><d>two</d></a>')

        self.assertEquals(1, len(self.observer.calledMethods))
        self.assertEqualsWS('<d>two</d>', lxmltostring(self.observer.calledMethods[0].kwargs['lxmlNode']))

 def testListRecords(self):
     header, body = self._request(
         verb=['ListRecords'], metadataPrefix=['prefix2'])
     records = xpath(body, '/oai:OAI-PMH/oai:ListRecords/oai:record')
     self.assertEquals(10, len(records))
     self.assertEquals([self.prefix + 'record:id:11'],
                       xpath(records[1],
                             'oai:header/oai:identifier/text()'))
     self.assertEquals(['record:id:11'],
                       xpath(records[1],
                             'oai:metadata/oai_dc:dc/dc:subject/text()'),
                       lxmltostring(records[1]))
     self.assertEquals(['hierarchical', 'setSpec10'],
                       sorted(
                           xpath(records[1],
                                 'oai:header/oai:setSpec/text()')))
     deletedRecords = xpath(
         body,
         '/oai:OAI-PMH/oai:ListRecords/oai:record[oai:header/@status="deleted"]'
     )
     self.assertEquals(2, len(deletedRecords))
     self.assertEquals(
         [0, 0], [len(xpath(r, 'oai:metadata')) for r in deletedRecords])
     self.assertEquals(['hierarchical', 'setSpec10'],
                       sorted(
                           xpath(deletedRecords[0],
                                 'oai:header/oai:setSpec/text()')))

 def testOnlyPassPartsSpecified(self):
     inputEvent = fromstring("""<document><part name="partone">&lt;some&gt;message&lt;/some&gt;</part><part name="parttwo"><second/></part></document>""")
     interceptor = CallTrace('Interceptor', methods={'add': yieldNothing})
     v = createVenturiHelix([{'partname': 'partone', 'xpath': '/document/part[@name="partone"]/text()'}], [], interceptor)
     list(compose(v.all.add('identifier', 'document', inputEvent)))
     self.assertEquals(['begin', 'add'], [m.name for m in interceptor.calledMethods])
     self.assertEquals('<some>message</some>', lxmltostring(interceptor.calledMethods[1].kwargs['lxmlNode']))

 def testCouldHave(self):
     inputEvent = fromstring('<document><one/></document>')
     interceptor = CallTrace('Interceptor', ignoredAttributes=['getData', 'all_unknown', 'any_unknown', 'call_unknown'], methods={'add': yieldNothing})
     v = createVenturiHelix([], [{'partname': 'one', 'xpath': '/document/one'}], interceptor)
     list(compose(v.all.add('identifier', 'document', inputEvent)))
     self.assertEquals(['begin', 'add'], [m.name for m in interceptor.calledMethods])
     self.assertEquals('<one/>', lxmltostring(interceptor.calledMethods[1].kwargs['lxmlNode']))

    def testListMetadataFormatsForWrongIdentifier(self):
        header, body = self._request(
            verb=['ListMetadataFormats'], identifier=['does:not:exist'])

        self.assertEquals(['idDoesNotExist'],
                          xpath(body, '/oai:OAI-PMH/oai:error/@code'),
                          lxmltostring(body, pretty_print=True))

 def testTwoTags(self):
     target = Target('aap')
     p = XMLParser(target = target)
     p.feed("<aap>")
     p.feed("noot")
     p.feed("</aap>")
     self.assertEquals("<aap>noot</aap>", lxmltostring(target.root))

 def _combine(self, erfgeoEnrichment, summary):
     yield '<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">\n'
     for data in [summary, erfgeoEnrichment]:
         rdfLxml = XML(data)
         for child in xpath(rdfLxml, '/rdf:RDF/*'):
             yield lxmltostring(child)
     yield '</rdf:RDF>'

 def handleRequest(self, Body='', **kwargs):
     yield '\r\n'.join(['HTTP/1.0 200 Ok', 'Content-Type: text/xml; charset=utf-8\r\n', ''])
     try:
         updateRequest = XML(Body)
         recordId = xpathFirst(updateRequest, 'ucp:recordIdentifier/text()')
         action = xpathFirst(updateRequest, 'ucp:action/text()')
         if self._allInvalid and action == "info:srw/action/1/replace":
             if 'oai:record:02' in recordId:
                 raise InvalidDataException()
             raise InvalidDataException('Invalid data')
         if recordId in self._raiseExceptionOnIds:
             raise Exception("ERROR")
         self._number +=1
         filename = '%05d_%s.updateRequest' %(self._number, action.rsplit('/')[-1])
         with open(join(self._dumpdir, filename), 'w') as f:
             stdout.flush()
             f.write(lxmltostring(updateRequest, pretty_print=True))
         answer = RESPONSE_XML % {
             "operationStatus": "success",
             "diagnostics": ""}
     except InvalidDataException, e:
         answer = RESPONSE_XML % {
             "operationStatus": "fail",
             "diagnostics": DIAGNOSTIC_XML % {
                 'uri': 'info:srw/diagnostic/12/12',
                 'details': escapeXml(str(e)),
                 'message': 'Invalid data:  record rejected'}}

    def testAddInitialRecord(self):
        uri = "some:uri"

        rdfDescription = """<rdf:Description rdf:about="%s" xmlns:dcterms="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://www.openarchives.org/OAI/2.0/">
    <dc:title xmlns:dc="http://purl.org/dc/elements/1.1/" xml:lang="en">title</dc:title>
    <prov:wasDerivedFrom xmlns:prov="http://www.w3.org/ns/prov#">
        <prov:Entity>
            <dcterms:source rdf:resource="http://first.example.org"/>
        </prov:Entity>
    </prov:wasDerivedFrom>
</rdf:Description>""" % uri

        lxmlNode = parse(StringIO("""<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
        %s
</rdf:RDF>""" % rdfDescription))

        consume(self.dna.all.add(identifier="identifier", lxmlNode=lxmlNode))

        record = self.oaiJazz.getRecord(identifier=uri)
        expected = XML(lxmltostring(xpathFirst(lxmlNode, '//rdf:RDF')))
        cleanup_namespaces(expected)
        self.assertXmlEquals(expected, self.storage.getData(identifier=record.identifier, name='rdf'))

        self.assertEquals(set(['rdf']), record.prefixes)
        self.assertEquals(set(), record.sets)

        self.plein.close()
        plein2 = self._newPlein()
        self.assertEquals(['some:uri'], [fragment.uri for fragment in plein2._fragmentsForRecord('identifier')])

 def _log(self, message, *args, **kwargs):
     printKwargs = dict(kwargs)
     for key, value in kwargs.items():
         if type(value) == ElementTreeType:
             printKwargs[key] = "%s(%s)" % (value.__class__.__name__, lxmltostring(value))
     sys.stdout.write("[%s] %s(*%s, **%s)\n" % (self.observable_name(), message, args, printKwargs))
     sys.stdout.flush()