def annotate_head_and_pos(entry):
    # delete head annotations
    head_annotations = [a for a in entry.annotations if a.value == 'head' or
                        a.value == 'pos' or
                        a.value == 'iso-639-3' or a.value == 'doculect']

    for a in head_annotations:
        Session.delete(a)

    heads = []
    head_start = 0

    first_italic = functions.get_first_italic_range(entry)
    if first_italic != -1:
        head_end = first_italic[0]
    else:
        head_end = functions.get_last_bold_pos_at_start(entry)

    for h_start, h_end in functions.split_entry_at(entry, r'/|$', head_start, head_end):
        match = head_end_pattern.search(entry.fullentry, h_start, h_end)
        if match:
            h_end = match.start()
        h_start, h_end  = functions.strip(entry, h_start, h_end, u'\'-–!? ')
        head = entry.fullentry[h_start:h_end].translate(head_translate)
        if head:
            functions.insert_head(entry, h_start, h_end, head)
            heads.append(head)

    # add pos
    if first_italic != -1:
        entry.append_annotation(first_italic[0], first_italic[1], u'pos', u'dictinterpretation')
        head_end = first_italic[1]

    return heads, head_end

def annotate_everything(entry):
    # delete head annotations
    head_annotations = [ a for a in entry.annotations if a.value=='head' or a.value=="iso-639-3" or a.value=="doculect" or a.value=='translation']
    for a in head_annotations:
        Session.delete(a)
        
    # Delete this code and insert your code
    heads = []

    sorted_annotations = [ a for a in entry.annotations if a.value=='tab']
    sorted_annotations = sorted(sorted_annotations, key=attrgetter('start'))

    if len(sorted_annotations) < 1:
        functions.print_error_in_entry(entry, "number of tabs is lower 1")
        return heads

    head_end = sorted_annotations[0].start
    head = functions.insert_head(entry, 0, head_end)
    heads.append(head)

    trans_end = len(entry.fullentry)
    if len(sorted_annotations) >= 2:
        trans_end = sorted_annotations[1].start

    for s, e in functions.split_entry_at(entry, r"(?:, |$)", sorted_annotations[0].end, trans_end):
        match_bracket = re.search("\(K[^\)]*\) ?$", entry.fullentry[s:e])
        if not match_bracket:
            functions.insert_translation(entry, s, e)
    
    return heads

def main(argv):
    if len(argv) < 2:
        print "call: updatecomponents.py ini_file"
        exit(1)

    ini_file = argv[1]
    
    dictdata_path = 'quanthistling/dictdata'
    
    conf = appconfig('config:' + ini_file, relative_to='.')
    if not pylons.test.pylonsapp:
        load_environment(conf.global_conf, conf.local_conf)
    
    # Create the tables if they don't already exist
    metadata.create_all(bind=Session.bind)
    
    # update languages
    for c in quanthistling.dictdata.components.list:
        db_lang = Session.query(model.Component).filter_by(name=c['name']).first()
        if db_lang == None:
            component = model.Component()
            component.name = c['name']
            component.description = c['description']
            Session.add(component)
            Session.commit()
            print("Inserted component " + c['name'] + ".")

def annotate_head(entry):
    # delete head annotations
    head_annotations = [a for a in entry.annotations if a.value == 'head' or a.value == "iso-639-3" or a.value == "doculect"]
    for a in head_annotations:
        Session.delete(a)

    heads = []

    newlines = [a for a in entry.annotations if a.value == 'newline']
    newlines = sorted(newlines, key=attrgetter('start'))
    if len(newlines) > 0:
        head_end = functions.get_last_bold_pos_at_start(entry)

        for h_start, h_end in functions.split_entry_at(entry, ',|$', 0, head_end):
            for i in ( index for index in xrange(h_start, h_end) if entry.fullentry[index] == '-' ):
                entry.append_annotation(i, i+1, u'boundary', u'dictinterpretation', u"morpheme boundary")
            h_start, h_end, head = functions.remove_parts(entry, h_start, h_end)
            if not head.strip():
                functions.print_error_in_entry(entry, "head is None")
            else:
                head = head.replace('-', '')
                head = functions.insert_head(entry, h_start, h_end, head)
                heads.append(head)

    return heads

def annotate_translation(entry):
    trans_annotations = [a for a in entry.annotations if a.value == 'translation']
    for a in trans_annotations:
        Session.delete(a)

    trans_start = max(functions.get_pos_or_head_end(entry) + 1, #skip ')' at end of POS
                      functions.get_first_bold_in_range(entry, 0, len(entry.fullentry))[1])

    newlines = functions.get_list_ranges_for_annotation(entry, 'newline')
    in_brackets = functions.get_in_brackets_func(entry)
    if len(newlines) == 0 or entry.fullentry[trans_start:newlines[0][0]].strip() != '':
        if len(newlines) == 0:
            trans_end = len(entry.fullentry)
        else:
            trans_end = newlines[0][0]
        for t_start, t_end in functions.split_entry_at(entry, r', |$', trans_start, trans_end, in_brackets=in_brackets):
            functions.insert_translation(entry, t_start, t_end)
    else:
        newlines.pop(0)
        for i, n in enumerate(newlines):
            trans_end = n[0]
            digit = re.compile('\d ').search(entry.fullentry, trans_start, trans_end)
            if digit is not None:
                trans_start = digit.end()
                for t_start, t_end in functions.split_entry_at(entry, r', |$', trans_start, trans_end, in_brackets):
                    functions.insert_translation(entry, t_start, t_end)
            trans_start = n[1]

def annotate_everything(entry):
    # delete head annotations
    annotations = [ a for a in entry.annotations if a.value=='head' or a.value=='pos' or a.value=='translation' or a.value=='doculect' or a.value=='iso-639-3']
    for a in annotations:
        Session.delete(a)

    tab_annotations = [ a for a in entry.annotations if a.value=='tab' ]
    newline_annotations = [ a for a in entry.annotations if a.value=='newline' ]
    #translation_end = ""
    #if len(newline_annotations) == 1:
     #   translation_end = " " + entry.fullentry[newline_annotations[0].start:]

    translation_end = len(entry.fullentry)

    heads = []
    
    if len(tab_annotations) != 2:
        print "not 2 tabs in entry " + entry.fullentry.encode("utf-8")
    else:
        head = entry.fullentry[0:tab_annotations[0].start]
        inserted_head = functions.insert_head(entry, 0, tab_annotations[0].start, head)
        # waar komt de waarde voor pos vandaan?
        entry.append_annotation(tab_annotations[0].start + 1, tab_annotations[1].start, u'pos', u'dictinterpretation')
        translation = entry.fullentry[tab_annotations[1].end + 1:]
        functions.insert_translation(entry, tab_annotations[1].end + 1, translation_end, translation)
        
        
        #entry.append_annotation(start, end, u'head', u'dictinterpretation')
        heads.append(inserted_head)
        
    return heads

def annotate_translations(entry):
    # delete translation annotations
    trans_annotations = [ a for a in entry.annotations if a.value=='translation']
    for a in trans_annotations:
        Session.delete(a)

    translation_start = functions.get_pos_or_head_end(entry)
    translation_end = len(entry.fullentry)
    italic = functions.get_first_italic_in_range(entry, translation_start, translation_end)
    if italic != -1:
        translation_end = italic[0] - 1

    for numbered_start, numbered_end in functions.split_entry_at(entry, r'[0-9]\) |$',
                                                                 translation_start, translation_end):
        bold = functions.get_first_bold_in_range(entry, numbered_start, numbered_end)
        if bold != -1:
            numbered_end = bold[0] - 1

        #search for start of examples and end translation there
        ex_match = re.compile(",\s*(e\.\s*g\.|i\.\s*e\.)").search(entry.fullentry, numbered_start, numbered_end)
        if ex_match:
            numbered_end = ex_match.start()

        for start, end in functions.split_entry_at(entry, r'[;,] |$', numbered_start, numbered_end):
            functions.insert_translation(entry, start, end)

def annotate_head(entry):
    # delete head annotations
    head_annotations = [ a for a in entry.annotations if a.value=='head' or a.value=='doculect' or a.value=='iso-639-3' or a.value=='boundary']
    for a in head_annotations:
        Session.delete(a)
        
    heads = []

    head_end_pos = functions.get_last_bold_pos_at_start(entry)
    head_start_pos = 0

    if entry.fullentry[0] == "*":
        head_start_pos = 1
        
    if entry.fullentry[head_end_pos-1] == ":":
        head_end_pos = head_end_pos - 1

    start = head_start_pos
    for match in re.finditer(u"(?:, |$)", entry.fullentry[head_start_pos:head_end_pos]):
        end = head_start_pos + match.start(0)
        inserted_head = insert_head(entry, start, end)
        heads.append(inserted_head)
        start = head_start_pos + match.end(0)
    
    return heads

def annotate_examples_and_crossrefs(entry): 
    # delete pos annotations
    ex_annotations = [ a for a in entry.annotations if a.value=='example-src' or a.value=='example-tgt']
    for a in ex_annotations:
        Session.delete(a)

    trans_end = functions.get_translation_end(entry)

    #crossrefs are sometimes mixed with examples. Crossrefs in italic will be known already.
    known_crossrefs = [a for a in entry.annotations if a.value == 'crossref']
    
    #process examples and crossrefs
    re_ex = re.compile(r'[.!?] ?(.*?) ?\= ?(.*?)(?=[.?!])')
    re_crossref = re.compile(r'\s*[.!?]?\s*[(]?\s*[Ii]s[.]?\s*([^.]*)\s*[)]?(?=[.])')
    oldpos = -1
    pos = trans_end
    while oldpos != pos:
        oldpos = pos
        cr_match = re_crossref.match(entry.fullentry, pos)
        if cr_match:
            if not _already_known(known_crossrefs, cr_match.start(1), cr_match.end(1)):
                entry.append_annotation(cr_match.start(1), cr_match.end(1), u'crossref', u'dictinterpretation')
            pos = cr_match.end()
            continue
        ex_match = re_ex.match(entry.fullentry, pos)
        if ex_match:
            match_suspect = re.search(u"[Ii]s\.", entry.fullentry[ex_match.start():ex_match.end()])
            if match_suspect:
                functions.print_error_in_entry(entry, "WARN: '[Ii]s' in examples.")
            entry.append_annotation(ex_match.start(1), ex_match.end(1), u'example-src', u'dictinterpretation',
                                    entry.fullentry[ex_match.start(1):ex_match.end(1)].lower())
            entry.append_annotation(ex_match.start(2), ex_match.end(2), u'example-tgt', u'dictinterpretation',
                                    entry.fullentry[ex_match.start(2):ex_match.end(2)].lower())
            pos = ex_match.end()

def annotate_translations(entry):
    # delete translation annotations
    trans_annotations = [ a for a in entry.annotations if a.value=='translation']
    for a in trans_annotations:
        Session.delete(a)

    #head_end_pos = functions.get_last_bold_pos_at_start(entry)
    head_starts, head_ends = get_bold_annotations(entry)

    for i in range(len(head_starts)):
        trans_start_pos = head_ends[i]
        if len(head_starts) > i+1:
            trans_end_pos = head_starts[i+1]
        else:
            trans_end_pos = len(entry.fullentry)

        if trans_start_pos > -1:
            substr = entry.fullentry[trans_start_pos:trans_end_pos]
            start = trans_start_pos
            for match in re.finditer(r'(?:, ?|; ?|\d\) )', substr):
                mybreak = False
                # are we in a bracket?
                for m in re.finditer(r'\(.*?\)', substr):
                    if match.start(0) >= m.start(0) and match.end(0) <= m.end(0):
                        mybreak = True
                    
                if not mybreak:
                    end = match.start(0) + trans_start_pos
                    if end > start and not re.match(r' +$', entry.fullentry[start:end]):
                        functions.insert_translation(entry, start, end)
                    start = match.end(0) + trans_start_pos
                    
            end = trans_end_pos
            if end > start and not re.match(r'^ +$', entry.fullentry[start:end]):
                functions.insert_translation(entry, start, end)

def annotate_head_without_comma(entry):
    # delete head annotations
    head_annotations = [ a for a in entry.annotations if a.value=='head' or a.value=="iso-639-3" or a.value=="doculect"]
    for a in head_annotations:
        Session.delete(a)

    head = None
    heads = []
    
    sorted_annotations = [ a for a in entry.annotations if a.value=='bold']
    sorted_annotations = sorted(sorted_annotations, key=attrgetter('start'))

    head_starts, head_ends = get_bold_annotations(entry)    

    heads = []

    for i in range(len(head_starts)):
        head_start_pos = head_starts[i]
        head_end_pos = head_ends[i]
        #head_end_pos = functions.get_last_bold_pos_at_start(entry)
        #head_start_pos = 0
    
    
        if head_end_pos > -1:
            inserted_head = functions.insert_head(entry, head_start_pos, head_end_pos)
            heads.append(inserted_head)
        else:
            print "no head"
            print entry.fullentry.encode('utf-8')
        
    return heads

def annotate_head(entry):
    # delete head annotations
    head_annotations = [a for a in entry.annotations if a.value == 'head' or
                        a.value == "iso-639-3" or a.value == "doculect" or
                        a.value == 'translation']
    for a in head_annotations:
        Session.delete(a)

    # Delete this code and insert your code
    head = None
    heads = []

    first_italic = functions.get_first_italic_range(entry)
    if first_italic != -1:
        head_end = first_italic[0]
        head = functions.insert_head(entry, 0, head_end)
        heads.append(head)
        trans_start = head_end
        trans_end = len(entry.fullentry)
        trans = entry.fullentry[trans_start:]
        if trans.startswith(u'“') and trans.endswith(u'”'):
            trans_start += 1
            trans_end -= 1

        functions.insert_translation(entry, trans_start, trans_end)

    return heads

def annotate_everything(entry):
    # delete annotations
    for a in entry.annotations:
        if a.value in ['head', 'pos', 'translation', 'crossreference', 'iso-639-3', 'doculect']:
            Session.delete(a)

    heads = annotate_heads(entry)
    
    h_end = functions.get_last_bold_pos_at_start(entry)
    t_start = annotate_pos(entry, h_end)
    
    substr = entry.fullentry[t_start:]
    if re.search(u'\(Vea .*?\)', substr): # cross-references
        for match_vea in re.finditer(u'\(Vea (.*?)\)', substr):
            cref_start = match_vea.start(1) + t_start
            cref_end = match_vea.end(1) + t_start
            substr = entry.fullentry[cref_start:cref_end]
            for match in re.finditer(u', ?', substr):
                end = match.start(0) + cref_start
                entry.append_annotation(cref_start, end, u'crossreference', u'dictinterpretation')
                cref_start = match.end(0) + cref_start
            entry.append_annotation(cref_start, cref_end, u'crossreference', u'dictinterpretation')
    else: # translations
        annotate_translation(entry, t_start)
    return heads

def annotate_dialect(entry):
    # delete head annotations
    dialect_annotations = [ a for a in entry.annotations if a.value=='dialectidentification']
    for a in dialect_annotations:
        Session.delete(a)
    
    match = re.finditer(u'\((b|p|s|b, p|b, s|p, s|B, P, S)\)', entry.fullentry)
    if match:
        for x in match:
            if x.group(1) == u'b':
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los bʉgʉyeri porã')
            elif x.group(1) == u'p':
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los payatérã porã')
            elif x.group(1) == u's':
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los sʉ̃meperu porã')
            elif x.group(1) == u'b, p':
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los bʉgʉyeri porã')
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los payatérã porã')
            elif x.group(1) == u'b, s':
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los bʉgʉyeri porã')
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los sʉ̃meperu porã')
            elif x.group(1) == u'p, s':
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los payatérã porã')
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los sʉ̃meperu porã')
            elif x.group(1) == u'B, P, S':
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los bʉgʉyeri porã')
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los payatérã porã')
                entry.append_annotation(x.start(1), x.end(1), u'dialectidentification', u'dictinterpretation', u'dialecto de los sʉ̃meperu porã')

def annotate_head(entry):
    # delete head annotations
    head_annotations = [ a for a in entry.annotations if a.value=='head' or a.value=="iso-639-3" or a.value=="doculect"]
    for a in head_annotations:
        Session.delete(a)

    head = None
    heads = []
    
    head_end = functions.get_last_bold_pos_at_start(entry)
    head_all = entry.fullentry[:head_end]
    head_all = head_all.rstrip()
    
    re_slash = re.compile("/") #search "/"
    match_slash = re_slash.search(head_all) #search in all heads
    
    if match_slash:
        # erster Head
        head1 = head_all[:match_slash.start()]
        head = functions.insert_head(entry, 0, head_end, head1)        
        heads.append(head)
        
        # zweiter Head
        head_suffix = head_all[match_slash.end():]
        head2 = head1[:-len(head_suffix)] + head_suffix #count length of suffix recursive and add the first part of head1 
        head = functions.insert_head(entry, 0, head_end, head2)        
        heads.append(head)
    else:
        head = functions.insert_head(entry, 0, head_end)
        heads.append(head)

    return heads

def annotate_translations(entry):
    # delete translation annotations
    trans_annotations = [a for a in entry.annotations
                         if a.value == 'translation']

    for a in trans_annotations:
        Session.delete(a)

    trans_start = functions.get_pos_or_head_end(entry)
    if entry.fullentry[trans_start:].startswith(')'):
        trans_start = trans_start + 1

    #crossrefs
    crossref_match = re.compile('\s*V\. ').match(entry.fullentry, trans_start)
    if crossref_match:
        return trans_start

    in_brackets = functions.get_in_brackets_func(entry)
    trans_end = min(functions.find_first_point(entry, trans_start, len(entry.fullentry), in_brackets),
                    functions.find_first(entry, u' V. ', trans_start, len(entry.fullentry), in_brackets))

    for t_start, t_end in functions.split_entry_at(entry, r'[,;]|$', trans_start, trans_end, False, in_brackets):
        t_start, t_end, translation = functions.remove_parts(entry, t_start, t_end)
        translation = clean_out_pagebreaks(translation)
        functions.insert_translation(entry, t_start, t_end, translation)
    return trans_end

def annotate_pos_and_translation(entry):
    # delete pos annotations
    pos_annotations = [ a for a in entry.annotations if a.value=='pos' or a.value=="translation" ]
    for a in pos_annotations:
        Session.delete(a)

    newline_annotations = [ a for a in entry.annotations if a.value=='newline']
    newline_annotations = sorted(newline_annotations, key=attrgetter('start'))
    
    if len(newline_annotations) < 2:
        functions.print_error_in_entry(entry)
        return
    else:
        first_newline = newline_annotations[1]
    
    first_line = entry.fullentry[:first_newline.start]
    
    match_pos = re.search("\(([^)]{1,8})\)", first_line)
    
    translation_end = newline_annotations[0].start
    if match_pos.end(0) > newline_annotations[0].start:
        translation_end =  newline_annotations[1].start
    
    if match_pos:
        entry.append_annotation(match_pos.start(1), match_pos.end(1), u"pos", u"dictinterpretation")
        functions.insert_translation(entry, match_pos.end(0), translation_end)
    else:
        functions.print_error_in_entry(entry)

def annotate_head(entry):
    # delete head annotations
    head_annotations = [ a for a in entry.annotations if a.value in ['head', "iso-639-3", "doculect", 'boundary']]
    for a in head_annotations:
        Session.delete(a)
        
    # Delete this code and insert your code
    head = None
    heads = []
    
    head_end = functions.get_last_bold_pos_at_start(entry)

    for start, end in functions.split_entry_at(entry, '[/,]|$', 0, head_end):
        start, end, head = functions.remove_parts(entry, start, end)
        if head[0] == '-':
            entry.append_annotation(start, start+1, u'boundary', u'dictinterpretation', u"morpheme boundary")
            head = head[1:]
            start = start + 1
        if head[-1] == '-':
            entry.append_annotation(end-1, end, u'boundary', u'dictinterpretation', u"morpheme boundary")
            head = head[:-1]
            end = end-1
        head = functions.insert_head(entry, start, end, head)
        if head:
            heads.append(head)
    return heads

def annotate_translations(entry):
    # delete translation annotations
    trans_annotations = [ a for a in entry.annotations if a.value=='translation']
    for a in trans_annotations:
        Session.delete(a)

    translation_start = functions.get_last_bold_pos_at_start(entry) + 1
    translation_end = functions.get_first_bold_start_in_range(entry, translation_start, len(entry.fullentry))
    if translation_end == -1:
        translation_end = len(entry.fullentry)

    match_bracket = re.search("^ ?\([^\)]*\) ?", entry.fullentry[translation_start:translation_end])
    if match_bracket:
        translation_start += len(match_bracket.group(0))

    match_capitals = re.search("^ ?(?:PI|PE) ?", entry.fullentry[translation_start:translation_end])  
    if match_capitals:
        translation_start += len(match_capitals.group(0))

    start = translation_start
    for t_start, t_end in functions.split_entry_at(entry, r'[,;] |/|$', translation_start, translation_end):
        t_start, t_end, translation = functions.remove_parts(entry, t_start, t_end)
        match = re.match(r'\(vr-[it]\.( irr\.)?\)|\(vt\.\)', translation)
        if match:
            translation = translation[len(match.group()):]
        functions.insert_translation(entry, t_start, t_end, translation)

def annotate_translations(entry):
    # delete translation annotations
    trans_annotations = [ a for a in entry.annotations if a.value=='translation']
    for a in trans_annotations:
        Session.delete(a)

    translation_start = max(functions.get_pos_or_head_end(entry),functions.get_last_bold_pos_at_start(entry)) 

    if re.match(" ?\(vea ", entry.fullentry[translation_start:]):
        return

    in_bracket = functions.get_in_brackets_func(entry)

    if re.search("\d\. ", entry.fullentry[translation_start:]):
        for match_number in re.finditer("\d\. ", entry.fullentry[translation_start:]):
            start = translation_start + match_number.end(0)
            match_translation = re.match("([^\.]*)\.", entry.fullentry[start:])
            end = get_first_point(entry, start, len(entry.fullentry), in_bracket)
            for s, e in functions.split_entry_at(entry, r"(?:[;,] |/|$)", start, end):
                insert_translation(entry, s, e)
    else:
        match_translation = re.match("([^\.]*)\.", entry.fullentry[translation_start:])
        end = get_first_point(entry, translation_start, len(entry.fullentry), in_bracket)
        for s, e in functions.split_entry_at(entry, r"(?:[;,] |/|$)", translation_start, end):
            insert_translation(entry, s, e)