def parseXML(stream, parser=None):
    if isinstance(stream, six.string_types):
        events = pulldom.parseString(stream, parser)
    else:
        events = pulldom.parse(stream, parser)

    document = None
    chain = []
    for event, node in events:
        if event == "START_DOCUMENT":
            chain.append(XMLNode("DOCUMENT", {}))

        elif event == "START_ELEMENT":
            node = XMLNode.fromDOMNode(node)
            if chain:
                chain[-1].children.append(node)
            chain.append(node)

        elif event == "END_ELEMENT":
            chain.pop(-1)

        elif event == "CHARACTERS":
            chain[-1].data += node.data

        elif event == "END_DOCUMENT":
            document = chain.pop(-1)
    return document or chain[0]

 def test_expandItem(self):
     """Ensure expandItem works as expected."""
     items = pulldom.parseString(SMALL_SAMPLE)
     # Loop through the nodes until we get to a "title" start tag:
     for evt, item in items:
         if evt == pulldom.START_ELEMENT and item.tagName == "title":
             items.expandNode(item)
             self.assertEqual(1, len(item.childNodes))
             break
     else:
         self.fail("No \"title\" element detected in SMALL_SAMPLE!")
     # Loop until we get to the next start-element:
     for evt, node in items:
         if evt == pulldom.START_ELEMENT:
             break
     self.assertEqual("hr", node.tagName,
         "expandNode did not leave DOMEventStream in the correct state.")
     # Attempt to expand a standalone element:
     items.expandNode(node)
     self.assertEqual(next(items)[0], pulldom.CHARACTERS)
     evt, node = next(items)
     self.assertEqual(node.tagName, "p")
     items.expandNode(node)
     next(items) # Skip character data
     evt, node = next(items)
     self.assertEqual(node.tagName, "html")
     with self.assertRaises(StopIteration):
         next(items)
     items.clear()
     self.assertIsNone(items.parser)
     self.assertIsNone(items.stream)

 def search (self, terms):
     """
     Search for a set of terms, returns a list of IDs to parse, which
     is then fed to self.fetch for data retrieval.
     """
     
     import types, urllib
     from xml.dom import pulldom
     
     id_list = []
     
     try:
         if isinstance(terms, types.ListType):
             url = self.esearch_url.replace('[[TERMS]]',
                 urllib.quote_plus((' '.join([str[term] for term in terms]))))
         else:
             url = self.esearch_url.replace('[[TERMS]]', 
                 urllib.quote_plus(str(terms)))
         xmls = urllib.urlopen(url).read()
         events = pulldom.parseString(xmls)
         for event, node in events:
             if event == 'START_ELEMENT' \
                 and node.tagName == 'Id':
                 events.expandNode(node)
                 id = self._get_text(node)
                 id_list.append(id)
     except Exception, e:
         self.logger.error('Unable to search Pubmed:', e)
         self.logger.error(traceback.format_stack())
         return []

 def test_comment(self):
     """PullDOM does not receive "comment" events."""
     items = pulldom.parseString(SMALL_SAMPLE)
     for evt, _ in items:
         if evt == pulldom.COMMENT:
             break
     else:
         self.fail("No comment was encountered")

    def test_parse_semantics(self):
        """Test DOMEventStream parsing semantics."""

        items = pulldom.parseString(SMALL_SAMPLE)
        evt, node = next(items)
        # Just check the node is a Document:
        self.assertTrue(hasattr(node, "createElement"))
        self.assertEqual(pulldom.START_DOCUMENT, evt)
        evt, node = next(items)
        self.assertEqual(pulldom.START_ELEMENT, evt)
        self.assertEqual("html", node.tagName)
        self.assertEqual(2, len(node.attributes))
        self.assertEqual(node.attributes.getNamedItem("xmlns:xdc").value,
              "http://www.xml.com/books")
        evt, node = next(items)
        self.assertEqual(pulldom.CHARACTERS, evt) # Line break
        evt, node = next(items)
        # XXX - A comment should be reported here!
        # self.assertEqual(pulldom.COMMENT, evt)
        # Line break after swallowed comment:
        self.assertEqual(pulldom.CHARACTERS, evt)
        evt, node = next(items)
        self.assertEqual("title", node.tagName)
        title_node = node
        evt, node = next(items)
        self.assertEqual(pulldom.CHARACTERS, evt)
        self.assertEqual("Introduction to XSL", node.data)
        evt, node = next(items)
        self.assertEqual(pulldom.END_ELEMENT, evt)
        self.assertEqual("title", node.tagName)
        self.assertTrue(title_node is node)
        evt, node = next(items)
        self.assertEqual(pulldom.CHARACTERS, evt)
        evt, node = next(items)
        self.assertEqual(pulldom.START_ELEMENT, evt)
        self.assertEqual("hr", node.tagName)
        evt, node = next(items)
        self.assertEqual(pulldom.END_ELEMENT, evt)
        self.assertEqual("hr", node.tagName)
        evt, node = next(items)
        self.assertEqual(pulldom.CHARACTERS, evt)
        evt, node = next(items)
        self.assertEqual(pulldom.START_ELEMENT, evt)
        self.assertEqual("p", node.tagName)
        evt, node = next(items)
        self.assertEqual(pulldom.START_ELEMENT, evt)
        self.assertEqual("xdc:author", node.tagName)
        evt, node = next(items)
        self.assertEqual(pulldom.CHARACTERS, evt)
        evt, node = next(items)
        self.assertEqual(pulldom.END_ELEMENT, evt)
        self.assertEqual("xdc:author", node.tagName)
        evt, node = next(items)
        self.assertEqual(pulldom.END_ELEMENT, evt)
        evt, node = next(items)
        self.assertEqual(pulldom.CHARACTERS, evt)
        evt, node = next(items)
        self.assertEqual(pulldom.END_ELEMENT, evt)

def fetchQuadrangle(dataset,yearMonth,resolution,sequence):

   # Format a URI
   strYearMonth = "{}-{:02d}".format(yearMonth.year,yearMonth.month)
   url = serviceURI+dataset+"/"+strYearMonth+"/"+str(resolution)+"/"+str(sequence);
   print url
   
   # Open an HTTP Request
   response = None
   try:
      response = urllib2.urlopen(url)
   except urllib2.HTTPError as e:
      return None
      
   html = None
   
   # Unpack the response
   if response.headers.get('content-encoding', '') == 'gzip':
      data = response.read()
      compressedstream = StringIO.StringIO(data)
      gzipper = gzip.GzipFile(fileobj=compressedstream)
      html = gzipper.read()
   else:
      html = response.read()
      
   # Parse the markup
   parser = sax.make_parser()
   parser.setFeature(sax.handler.feature_namespaces, 1)
   doc = pulldom.parseString(html,parser)
   
   inTable = False
   
   def textContent(parent):
      s = "";
      for n in parent.childNodes:
         if n.data != None:
            s += n.data
      return s
   
   # Process the markup as a stream and detect the table of data
   data = []
   for event, node in doc:
       if event == pulldom.START_ELEMENT and node.tagName == 'table':
          if node.getAttribute("typeof") == "IndexedTable":
             inTable = True
       if event == pulldom.END_ELEMENT and node.tagName == 'table':
          inTable = False
       if inTable and event == pulldom.START_ELEMENT and node.tagName == 'td':
          doc.expandNode(node)
          if len(node.childNodes) > 0:
             data.append(float(textContent(node)))
             
   if len(data) == 0:
      return None
   
   # Return the sequence number data object
   return {"dataset": dataset, "yearMonth": strYearMonth, "resolution" : resolution, "sequence": sequence, "data": data }

def make_parser(stream_or_string):
    """Create a xml.dom.pulldom parser."""

    if isinstance(stream_or_string, six.string_types):

        # XXX: the pulldom.parseString() function doesn't seem to
        # like operating on unicode strings!

        return pulldom.parseString(str(stream_or_string))

    else:

        return pulldom.parse(stream_or_string)

def get_nodes_from_xml(src):
	if type(src)==str:
		events = pulldom.parseString(src)
	else:
		# file like object
		events = pulldom.parse(src)
	try:
		for (event, node) in events:
			if event == pulldom.START_ELEMENT and node.tagName == "node":			
				events.expandNode(node)
				yield node
	except Exception as e:
		print(e, file=sys.stderr)

 def test_end_document(self):
     """PullDOM does not receive "end-document" events."""
     items = pulldom.parseString(SMALL_SAMPLE)
     # Read all of the nodes up to and including </html>:
     for evt, node in items:
         if evt == pulldom.END_ELEMENT and node.tagName == "html":
             break
     try:
         # Assert that the next node is END_DOCUMENT:
         evt, node = next(items)
         self.assertEqual(pulldom.END_DOCUMENT, evt)
     except StopIteration:
         self.fail(
             "Ran out of events, but should have received END_DOCUMENT")

    def _parse_response(self, content):
        bugs = {}
        stream = pulldom.parseString(content)
        for (event, node) in stream:
            if event == "START_ELEMENT" and node.tagName == "bug":
                stream.expandNode(node)
                error = node.getAttribute("error")
                if error:
                    raise IssueError(error)

                bugs['alias'] = node.getElementsByTagName("bug_id")[0].firstChild.data
                bugs['name'] = node.getElementsByTagName("short_desc")[0].firstChild.data
                bugs['status'] = node.getElementsByTagName("bug_status")[0].firstChild.data
                bugs['resolution'] = node.getElementsByTagName("resolution") or ""
                if bugs['resolution']:
                    bugs['resolution'] = bugs['resolution'][0].firstChild.data

        return bugs

    def __process_event(self, eventdata):
        """
        Private method called while nmap process is running. It enables the
        library to handle specific data/events produced by nmap process.
        So far, the following events are supported:

        1. task progress: updates estimated time to completion and percentage
           done while scan is running. Could be used in combination with a
           callback function which could then handle this data while scan is
           running.
        2. nmap run: header of the scan. Usually displayed when nmap is started
        3. finished: when nmap scan ends.

        :return: True is event is known.

        :todo: handle parsing directly via NmapParser.parse()
        """
        rval = False
        try:
            edomdoc = pulldom.parseString(eventdata)
            for xlmnt, xmlnode in edomdoc:
                if xlmnt is not None and xlmnt == pulldom.START_ELEMENT:
                    if (xmlnode.nodeName == 'taskprogress' and
                            xmlnode.attributes.keys()):
                        percent_done = xmlnode.attributes['percent'].value
                        etc_done = xmlnode.attributes['etc'].value
                        self.__progress = percent_done
                        self.__etc = etc_done
                        rval = True
                    elif (xmlnode.nodeName == 'nmaprun' and
                            xmlnode.attributes.keys()):
                        self.__starttime = xmlnode.attributes['start'].value
                        self.__version = xmlnode.attributes['version'].value
                        rval = True
                    elif (xmlnode.nodeName == 'finished' and
                            xmlnode.attributes.keys()):
                        self.__endtime = xmlnode.attributes['time'].value
                        self.__elapsed = xmlnode.attributes['elapsed'].value
                        self.__summary = xmlnode.attributes['summary'].value
                        rval = True
        except:
            pass
        return rval

    def _fetchhead(self):
        """
        Fetches the head information. If there are no variables in the
        <head>, then we also fetch the boolean result.
        """
        self.events = pulldom.parseString(self.__xml)

        for (event, node) in self.events:
            if event == pulldom.START_ELEMENT:
                if node.tagName == 'variable':
                    self.variables.append(node.attributes['name'].value)
                elif node.tagName == 'boolean':
                    self.events.expandNode(node)
                    self._hasResult = (node.firstChild.data == 'true')
                elif node.tagName == 'result':
                    return # We should not arrive here
            elif event == pulldom.END_ELEMENT:
                if node.tagName == 'head' and self.variables:
                    return
                elif node.tagName == 'sparql':
                    return

    def __call__(self):

        self.readXML = self.aggregateServiceDataToXML()
        doc = parseString(self.readXML)

        finalXML = StringIO()
        finalXML.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
        finalXML.write("<Cloud xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:noNamespaceSchemaLocation=\""+XSD+"\">")

        headNodeXML = StringIO()
        workerNodeXML = StringIO()
        # Here the pulldom API is used to extract the XML nodes under any "HeadNode" tags and write them to the finalXML for XSLT processing
        for event, node in doc:
            if event == xml.dom.pulldom.START_ELEMENT:
                
                if node.localName == "HeadNode":
                    doc.expandNode(node)
                    tempString = node.toxml()
                    # The fancy string index [10:-11] is used to eliminate the <HeadeNode></HeadNode> tags from the output
                    headNodeXML.write(tempString[10:-11])
                if node.localName =="Node":
                    doc.expandNode(node)
                    tempString = node.toxml()
                    workerNodeXML.write(tempString)

        finalXML.write("<HeadNode>") 
        # This tag is added for the "Optional Cloud Name" of the public XML schema. An 'id' attribute MUST be specified or the XSLs will remove this CloudName tag from the final XML. The 'id' is arbritrary
        finalXML.write("<CloudName id='arbitrary11235813'>"+ConfigMapping[CLOUD_NAME]+"</CloudName>")
        finalXML.write(headNodeXML.getvalue())
        finalXML.write("</HeadNode>")
        finalXML.write("<WorkerNodes>")
        finalXML.write(workerNodeXML.getvalue())
        finalXML.write("</WorkerNodes>")
        finalXML.write("</Cloud>")

        # The various stylesheets are applied "serially" to the final XML to pepare it for publishing 
        return self.applyStyleSheet(ConfigMapping[NAGIOS_LOCATION]+ATTRIBUTE_STRIP_XSL,self.applyStyleSheet(ConfigMapping[NAGIOS_LOCATION]+MERGE_NODES_XSL,self.applyStyleSheet(ConfigMapping[NAGIOS_LOCATION]+REMOVE_DUP_XSL,finalXML.getvalue())))

 def parseString(self, st):
    self.log.debug("Setting up parser...")
    evt_stream = pulldom.parseString(st)
    self.log.debug("{}Parsing started with Parsable set {}".format('Sub-' if self.active else '', dict((p,self.__parsables[p].tag()) for p in self.__parsables)))
    for x in self._parse_evt_stream(evt_stream):
       yield x

from xml.dom.pulldom import CHARACTERS, START_ELEMENT, parseString, END_ELEMENT

# Use djb development version of collatex (https://github.com/djbpitt/collatex, "experimental" branch)
sys.path.append('/Users/djb/collatex/collatex-pythonport/')
from collatex import *

class Stack(list):
    def push(self, item):
        self.append(item)

    def peek(self):
        return self[-1]

# Initialize input and output
source = open('pizarnik.xml','r').read()
doc = parseString(source)
witnesses = {}

# Only process content inside witnesses
inWitness = False
inLine = False

# Tokenize, keeping leading whitespace (whitespace after last token is processed separately)
def tokenize(contents):
    return re.findall(r'\s*\S+', contents)

# Regex
startWhite = re.compile(r'\s+') # strip leading whitespace; match() is automatically anchored at the start
endWhite = re.compile(r'\S\s+$') # test for trailing whitespace to include in output

for event, node in doc:

def buscarcep(cep):
    """
    Localiza o CEP informado no argumento utilizando o serviço
    disponibilizado pelo site www.buscarcep.com.br. Retorna um
    dicionário contendo as informações obtidas. As chaves retornadas
    são: 'cep', 'uf', 'cidade', 'bairro', 'tipo_logradouro', e
    'logradouro'.
 
    Para avaliar o resultado do retorno, verifique as chaves
    'resultado' e 'resultado_txt'. Para maiores detalhes consulte o
    site do serviço em www.buscarcep.com.br.
    """

    url = urllib.urlopen("http://www.buscarcep.com.br/?cep=" + cep + "&formato=xml")

    cepinfo = {
        "cep": "",
        "uf": "",
        "cidade": "",
        "bairro": "",
        "tipo_logradouro": "",
        "logradouro": "",
        "resultado": 0,
        "resultado_txt": "",
    }

    if url:
        texto = url.read()
        url.close()

        events = pulldom.parseString(texto)
        xpath = ""

        for event, node in events:
            if event == pulldom.START_ELEMENT:
                xpath += "/" + node.nodeName

            elif event == pulldom.END_ELEMENT:
                pos = xpath.rfind("/")
                xpath = xpath[0:pos]

            elif event == pulldom.CHARACTERS:
                if xpath == "/webservicecep/retorno/cep":
                    cepinfo["cep"] = node.nodeValue

                elif xpath == "/webservicecep/retorno/uf":
                    cepinfo["uf"] = node.nodeValue

                elif xpath == "/webservicecep/retorno/cidade":
                    cepinfo["cidade"] = node.nodeValue

                elif xpath == "/webservicecep/retorno/bairro":
                    cepinfo["bairro"] = node.nodeValue

                elif xpath == "/webservicecep/retorno/tipo_logradouro":
                    cepinfo["tipo_logradouro"] = node.nodeValue

                elif xpath == "/webservicecep/retorno/logradouro":
                    cepinfo["logradouro"] = node.nodeValue

                elif xpath == "/webservicecep/retorno/resultado":
                    cepinfo["resultado"] = int(node.nodeValue)

                elif xpath == "/webservicecep/retorno/resultado_txt":
                    cepinfo["resultado_txt"] = node.nodeValue

    else:
        # erro na conexão
        cepinfo["resultado"] = 0
        cepinfo["resultado_txt"] = "Erro na conexão"

    return [cepinfo]

 def loadString(self,xml_string):
     events = pulldom.parseString(xml_string)
     self.parse(events)

 def __init__(self, xml):
     self._events = pulldom.parseString(xml)

    def __process_event(self, eventdata):
        """
        Private method called while nmap process is running. It enables the
        library to handle specific data/events produced by nmap process.
        So far, the following events are supported:

        1. task progress: updates estimated time to completion and percentage
           done while scan is running. Could be used in combination with a
           callback function which could then handle this data while scan is
           running.
        2. nmap run: header of the scan. Usually displayed when nmap is started
        3. finished: when nmap scan ends.

        :return: True is event is known.

        :todo: handle parsing directly via NmapParser.parse()
        """
        rval = False
        try:
            edomdoc = pulldom.parseString(eventdata)
            for xlmnt, xmlnode in edomdoc:
                if xlmnt is not None and xlmnt == pulldom.START_ELEMENT:
                    if (xmlnode.nodeName == 'taskbegin' and
                            xmlnode.attributes.keys()):
                        xt = xmlnode.attributes
                        taskname = xt['task'].value
                        starttime = xt['time'].value
                        xinfo = ''
                        if 'extrainfo' in xt.keys():
                            xinfo = xt['extrainfo'].value
                        newtask = NmapTask(taskname, starttime, xinfo)
                        self.__nmap_tasks[newtask.name] = newtask
                        self.__current_task = newtask.name
                        rval = True
                    elif (xmlnode.nodeName == 'taskend' and
                            xmlnode.attributes.keys()):
                        xt = xmlnode.attributes
                        tname = xt['task'].value
                        xinfo = ''
                        self.__nmap_tasks[tname].endtime = xt['time'].value
                        if 'extrainfo' in xt.keys():
                            xinfo = xt['extrainfo'].value
                        self.__nmap_tasks[tname].extrainfo = xinfo
                        self.__nmap_tasks[tname].status = "ended"
                        rval = True
                    elif (xmlnode.nodeName == 'taskprogress' and
                            xmlnode.attributes.keys()):
                        xt = xmlnode.attributes
                        tname = xt['task'].value
                        percent = xt['percent'].value
                        etc = xt['etc'].value
                        remaining = xt['remaining'].value
                        updated = xt['time'].value
                        self.__nmap_tasks[tname].percent = percent
                        self.__nmap_tasks[tname].progress = percent
                        self.__nmap_tasks[tname].etc = etc
                        self.__nmap_tasks[tname].remaining = remaining
                        self.__nmap_tasks[tname].updated = updated
                        rval = True
                    elif (xmlnode.nodeName == 'nmaprun' and
                            xmlnode.attributes.keys()):
                        self.__starttime = xmlnode.attributes['start'].value
                        self.__version = xmlnode.attributes['version'].value
                        rval = True
                    elif (xmlnode.nodeName == 'finished' and
                            xmlnode.attributes.keys()):
                        self.__endtime = xmlnode.attributes['time'].value
                        self.__elapsed = xmlnode.attributes['elapsed'].value
                        self.__summary = xmlnode.attributes['summary'].value
                        rval = True
        except:
            pass
        return rval

 def test_external_ges_default(self):
     parser = pulldom.parseString(SMALL_SAMPLE)
     saxparser = parser.parser
     ges = saxparser.getFeature(feature_external_ges)
     self.assertEqual(ges, False)