def parse_captions(soup):
	"""	Converts custom iView captions into SRT format, usable in most
		decent media players.
	"""
	
	# Horrible hack to escape literal ampersands, which have been seen in
	# some captions XML. Inspired by
	# http://stackoverflow.com/questions/6088760/fix-invalid-xml-with-ampersands-in-python
	if b"<![CDATA[" not in soup:  # Not seen, but be future proof
		soup = re.sub(b"&(?![#\w]+;)", b"&amp;", soup)
	
	xml = XML(soup)

	output = ''

	i = 1
	for title in xml.getiterator('title'):
		start = title.get('start')
		ids = start.rfind(':')
		end = title.get('end')
		ide = end.rfind(':')
		output = output + str(i) + '\n'
		output = output + start[:ids] + ',' + start[ids+1:] + ' --> ' + end[:ide] + ',' + end[ide+1:] + '\n'
		output = output + title.text.replace('|','\n') + '\n\n'
		i += 1

	return output

def fix_etree():
    try:
        from xml.etree.cElementTree import XML
        e = XML('<test><t a="1"/></test>')
        e.find('t[@a="1"]')
    except SyntaxError:
        import canari.xmltools.fixetree

def get_status_code(output):
    try:
        tree = XML(output)
        child = tree.getchildren()[0]
        return int(child.get('status_code'))
    except:
        return None

 def parseOpenSearch(self,xmldata,format="atom"):
     """
     OpenSearchの出力XMLをパースします。
     xmldata=OpenSearch("keyword")[1]
     format=atomまたはrss
     """
     root=XML(xmldata)#xmldata
     if format == "rss":
         entrys=[elm for elm in root.getchildren()[0].getchildren() if "item" == elm.tag]
     elif format == "atom":
         entrys=[elm for elm in root if elm.tag.endswith("entry")]
     else:
         raise Exception("Unknown format : %s" % format)#xmlのフォーマットはatom/rssのみ対応
     hits=len(entrys)
     entrys=[entry.getiterator for entry in entrys]
     entrys=[j for j in [i() for i in entrys]]
     ret=[]
     h="content"
     for entry in entrys:
         buf={}
         for val in entry:
             dual=1
             if val != None and val.text != None:
                 if not val.text.startswith("\n"):
                     if buf.get(val.tag[val.tag.find("}")+1:]) == None:
                         buf[val.tag[val.tag.find("}")+1:]] = val.text
                     else:
                         dual= dual+1
                         buf[val.tag[val.tag.find("}")+1:] + str(dual)] = val.text
         if h in buf.keys():buf[h]=self.stripHTMLTags(buf[h])
         ret.append(buf)
     return hits,ret#data[0]["name"] で著者

def get_property(output, key):
    try:
        tree = XML(output)
        child = tree.getchildren()[0]   # first tag
        return child.get(key)
    except IndexError:
        return None

def get_raw_text(pthFile):
	"""
	gets a path to a file as an argument and returns a list containing
	the paragraphs of the word document file
	"""
	
	"""
	Constants used to iterate over the XML tree
	"""
	WORD_NAMESPACE = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
	PARA = WORD_NAMESPACE + 'p'
	TEXT = WORD_NAMESPACE + 't'

	docWordDoc = zipfile.ZipFile(pthFile) #gets the documents of the word
	xmlContent = docWordDoc.read('word/document.xml') #access the xml file
	docWordDoc.close()
	treeXML = XML(xmlContent) #parses the xml content into a tree that will be further used to access the text

	lstParagraphs = [] #output list with the paragraphs of the text
	#now we proceed to extract the text from the tree
	#the idea is to iterate over the tree and 
	#for each node that contains text, substract it and add it to
	#the output
	for parParagraph in treeXML.getiterator(PARA):
		lstTexts = [nodElement.text
			    for nodElement in parParagraph.getiterator(TEXT)
			    if nodElement.text]
		if lstTexts:
			print lstTexts
			lstParagraphs.append(''.join(lstTexts))
		
	return lstParagraphs

def parse_config(soup):
    """There are lots of goodies in the config we get back from the ABC.
    In particular, it gives us the URLs of all the other XML data we
    need.
    """

    xml = XML(soup)
    params = dict()
    for param in xml.iter('param'):
        params.setdefault(param.get('name'), param.get('value'))

    # should look like "rtmp://cp53909.edgefcs.net/ondemand"
    # Looks like the ABC don't always include this field.
    # If not included, that's okay -- ABC usually gives us the server in the auth result as well.
    rtmp_url = params['server_streaming']
    categories_url = params['categories']

    params.update({
        'rtmp_url'  : rtmp_url,
        'auth_url'  : params['auth'],
        'api_url' : params['api'],
        'categories_url' : categories_url,
        'captions_url' : params['captions'],
    })
    return params

 def listChildrenViaPropfind():
     data = yield self.simpleSend(
         "PROPFIND", "/", resultcode=responsecode.MULTI_STATUS,
         headers=[('Depth', '1')]
     )
     tree = XML(data)
     seq = [e.text for e in tree.findall("{DAV:}response/{DAV:}href")]
     shortest = min(seq, key=len)
     seq.remove(shortest)
     filtered = [elem[len(shortest):].rstrip("/") for elem in seq]
     returnValue(filtered)

 def get_docx_text(self, path):
     document = zipfile.ZipFile(path)
     xml_content = document.read('word/document.xml')
     tree = XML(xml_content)
     document.close()
     paragraphs = []
     for paragraph in tree.getiterator(para_tag):
         texts = [node.text for node in paragraph.getiterator(text_tag) if node.text]
         if texts:
             paragraphs.append(''.join(texts))
     return paragraphs

def queryUniprot(id, expand=[], regex=True):
    """Query Uniprot with *id* and return a `dictionary` containing the results
    
    :arg expand: entries through which you want to loop dictElements
        until there aren't any elements left
    :type expand: list
    """

    if not isinstance(id, str):
        raise TypeError('id should be a string')

    try:
        record_file = openURL('http://www.uniprot.org/uniprot/{0}.xml'.format(id))
    except:
        raise ValueError('No Uniprot record found with that id')
    
    data = record_file.read()
    record_file.close()
    data = XML(data)

    data = dictElement(data.getchildren()[0], '{http://uniprot.org/uniprot}', number_multiples=True)

    for key in data:
        value = data[key]
        if not key.startswith('dbReference'):
            continue
        
        try:
            if value.get('type') != 'PDB':
                continue
        except AttributeError:
            continue

        pdbid = value.get('id')
        refdata = {'PDB': pdbid}
        for prop in value:
            prop_key = prop.get('type')
            prop_val = prop.get('value')
            refdata[prop_key] = prop_val
        data[key] = refdata
            
    if expand:
        keys = []
        if regex:
            for lt in expand:
                lt_re = re.compile(lt)
                for key in data.keys():
                    if lt_re.match(key):
                        keys.append(key)
        else:
            keys = expand
        data = dictElementLoop(data, keys, '{http://uniprot.org/uniprot}')
    
    return data

def get_docx_text(path):
    document = zipfile.ZipFile(path)
    xml_content = document.read("word/document.xml")
    document.close()
    tree = XML(xml_content)

    paragraphs = []
    for paragraph in tree.getiterator(PARA):
        texts = [node.text for node in paragraph.getiterator(TEXT) if node.text]
        if texts:
            paragraphs.append("".join(texts))
    return paragraphs

def parse_highlights(xml):

    soup = XML(xml)

    highlightList = []

    for series in soup.iterfind('series'):
        tempSeries = dict(series.items())
        tempSeries.update(xml_text_elements(series))
        highlightList.append(tempSeries)

    return highlightList

def get_docx_text(path):
	"""
	Take the path of a docx file as argument, return the text in unicode
	in the form of a list.
	"""
	document = zipfile.ZipFile(path)
	xml_content = document.read('word/document.xml')
	document.close()
	tree = XML(xml_content)
 
	paragraphs = []
	for paragraph in tree.getiterator(PARA):
		texts = [node.text.encode('utf-8')
				 for node in paragraph.getiterator(TEXT)
				 if node.text]
		if texts:
			paragraphs.append(''.join(texts))
 
	return paragraphs



 
# def get_docx_text(path):
# 	"""
# 	Take the path of a docx file as argument, return the text in unicode
# 	in the form of a list.
# 	"""
# 	document = zipfile.ZipFile(path)
# 	xml_content = document.read('word/document.xml')
# 	document.close()
# 	tree = XML(xml_content)
 
# 	sections = []
# 	for section in tree.getiterator(SECT):

# 		paragraphs = []
# 		for paragraph in section.getiterator(PARA):
# 			print 'para'
# 			texts = [node.text.encode('utf-8')
# 					 for node in paragraph.getiterator(TEXT)
# 					 if node.text]
# 			if texts:
# 				paragraphs.append(''.join(texts))

# 		print str(paragraphs)

# 		if paragraphs:
# 			sections.append(''.join(paragraphs))

 
# 	return sections    

        def listChildrenViaPropfind():
            request = SimpleStoreRequest(self, "PROPFIND", "/calendars/__uids__/user01/", authid="user01")
            request.headers.setHeader("depth", "1")
            response = yield self.send(request)
            response = IResponse(response)
            data = yield allDataFromStream(response.stream)

            tree = XML(data)
            seq = [e.text for e in tree.findall("{DAV:}response/{DAV:}href")]
            shortest = min(seq, key=len)
            seq.remove(shortest)
            filtered = [elem[len(shortest):].rstrip("/") for elem in seq]
            returnValue(filtered)

def mtgx2json(graph):
    zipfile = ZipFile(graph)
    graphs = filter(lambda x: x.endswith(".graphml"), zipfile.namelist())
    for f in graphs:
        multikeys = []
        xml = XML(zipfile.open(f).read())
        links = {}
        for edge in xml.findall(
            "{http://graphml.graphdrawing.org/xmlns}graph/" "{http://graphml.graphdrawing.org/xmlns}edge"
        ):
            src = edge.get("source")
            dst = edge.get("target")
            if src not in links:
                links[src] = dict(in_=[], out=[])
            if dst not in links:
                links[dst] = dict(in_=[], out=[])
            links[src]["out"].append(dst)
            links[dst]["in_"].append(src)

        for node in xml.findall(
            "{http://graphml.graphdrawing.org/xmlns}graph/" "{http://graphml.graphdrawing.org/xmlns}node"
        ):

            node_id = node.get("id")
            node = node.find(
                "{http://graphml.graphdrawing.org/xmlns}data/" "{http://maltego.paterva.com/xml/mtgx}MaltegoEntity"
            )

            record = OrderedDict({"NodeID": node_id, "EntityType": node.get("type").strip()})
            props = {"Data": {}}
            for prop in node.findall(
                "{http://maltego.paterva.com/xml/mtgx}Properties/" "{http://maltego.paterva.com/xml/mtgx}Property"
            ):
                value = prop.find("{http://maltego.paterva.com/xml/mtgx}Value").text or ""
                entity_prop = {prop.get("displayName"): value.strip()}
                props["Data"].update(entity_prop)
            record.update(props)
            s = " - ".join(["%s: %s" % (key, value) for (key, value) in record["Data"].items()])
            record.pop("Data")
            data = {"Data": s}
            record.update(data)
            link = {"Links": {}}
            i_link = {"Incoming": links.get(node_id, {}).get("in_", 0)}
            link["Links"].update(i_link)
            o_link = {"Outgoing": links.get(node_id, {}).get("out", 0)}
            link["Links"].update(o_link)
            record.update(link)
            multikeys.append(record)
        return multikeys

def read_docx(file,document,path,trie):
	xml_content = document.read('word/document.xml')
	document.close()
	tree = XML(xml_content)
	paragraphs = ""
	for paragraph in tree.getiterator(PARA):
		texts=""
		for node in paragraph.getiterator(TEXT):
			if node.text:
				texts += node.text.replace('\u7460',"")
		if texts:
			paragraphs+=str(texts)
	#print(paragraphs)
	string_spilt(paragraphs,path,trie)
	trie.insert_doc_len(path,len(file)+len(paragraphs))

def get_doc_text(path):
    """
    Take the path of a docx or a dot file as argument, return the text in unicode.
    """
    if "docx" == path[-4:]:
        document = zipfile.ZipFile(path)
        xml_content = document.read('word/document.xml')
        document.close()
        tree = XML(xml_content)
    #print tree

        paragraphs = []
        for paragraph in tree.getiterator(PARA):
            texts = [node.text for node in paragraph.iter(TEXT) if node.text]
            if texts:
                paragraphs.append(''.join(texts))
                pass
            pass
    #print paragraphs
        return paragraphs
#        
    elif "odt" == path[-3:]:
        document = zipfile.ZipFile(path)
        xml_content = document.read('content.xml')
        document.close()
        doc = xml.dom.minidom.parseString(xml_content)
        print(" doc: ",doc)
        print("doc::end")
        #paras = doc.getElementsByTagName('text:span')
        #paras = doc.getElementsByTagName('text:p')
        #
        # we get here all elements Headers, text and table components: 
        #
        paras = doc.getElementsByTagName("*")
        print("I have ", len(paras), " paragraphs ")
        paragraphs = []
        for p in paras:
            for ch in p.childNodes:
                if ch.nodeType == ch.TEXT_NODE:
                    paragraphs.append(''.join(ch.wholeText))
                    pass
                pass
            pass
        print(paragraphs)
        return paragraphs
    else:
        print() 
        raise Warning("only docx and odt files are handled")    

def docx_do_docx(azip, afile):
    word_namespace = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
    par = word_namespace + "p"
    txt = word_namespace + "t"

    xml_content = azip.read("word/document.xml")
    tree = XML(xml_content)

    paragraphs = []
    for paragraph in tree.getiterator(par):
        texts = [node.text for node in paragraph.getiterator(txt) if node.text]
        if texts:
            paragraphs.append("".join(texts))

    text = "\n\n".join(paragraphs)
    text_do_data(text, afile)

def get_docx_text(path):
	"""	Take the path of a docx file as argument, return the text in unicode."""
	document = zipfile.ZipFile(path)
	xml_content = document.read('word/document.xml')
	document.close()
	tree = XML(xml_content)
 
	paragraphs = []
	for paragraph in tree.getiterator(PARA):
		texts = [node.text
				for node in paragraph.getiterator(TEXT)
				if node.text]
		if texts:
			paragraphs.append(''.join(texts))
 
	return '\n\n'.join(paragraphs) 

def docx_do_docx(azip, afile):
    namespace = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
    par = namespace + 'p'
    txt = namespace + 't'

    xml_content = azip.read('word/document.xml')
    tree = XML(xml_content)

    paragraphs = []
    for paragraph in tree.getiterator(par):
        texts = [node.text for node in paragraph.getiterator(txt)
                 if node.text]
        if texts:
            paragraphs.append(''.join(texts))

    text = '\n\n'.join(paragraphs)
    text_do_data(text, afile)