def get_prev_curr_production_rule(arg_clauses, clause_index, parse_dict):
    DocID = arg_clauses.DocID
    sent_index = arg_clauses.sent_index
    curr_clause_indices = arg_clauses.clauses[clause_index][0]# ([1,2,3],yes)
    if clause_index > 0:
        prev_clause_index = clause_index - 1
        curr_clause_indices = arg_clauses.clauses[prev_clause_index][0] + curr_clause_indices

    subtrees = []
    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)
    if syntax_tree.tree != None:
        clause_leaves = set([syntax_tree.get_leaf_node_by_token_index(index) for index in curr_clause_indices])
        no_need = []
        for node in syntax_tree.tree.traverse(strategy="levelorder"):
            if node not in no_need:
                if set(node.get_leaves()) <= clause_leaves:
                    subtrees.append(node)
                    no_need.extend(node.get_descendants())

    production_rule = []
    for tree in subtrees:
        for node in tree.traverse(strategy="levelorder"):
            if not node.is_leaf():
                rule = node.name + "-->" + " ".join([child.name for child in node.get_children()])
                production_rule.append(rule)

    return production_rule

def _get_constituents(parse_dict, connective):
    DocID = connective.DocID
    sent_index = connective.sent_index
    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)
    if syntax_tree.tree == None:
        return []

    conn_indices = connective.token_indices
    constituent_nodes = []
    if len(conn_indices) == 1:# like and or so...
        conn_node = syntax_tree.get_leaf_node_by_token_index(conn_indices[0]).up
    else:
        conn_node = syntax_tree.get_common_ancestor_by_token_indices(conn_indices)
        conn_leaves = set([syntax_tree.get_leaf_node_by_token_index(conn_index) for conn_index in conn_indices])
        children = conn_node.get_children()
        for child in children:
            leaves = set(child.get_leaves())
            if conn_leaves & leaves == set([]):
                constituent_nodes.append(child)

    curr = conn_node
    while not curr.is_root():
        constituent_nodes.extend(syntax_tree.get_siblings(curr))
        curr = curr.up

    # obtain the Constituent object according to the node.
    constituents = []
    for node in constituent_nodes:
        cons = Constituent(syntax_tree, node)
        cons.connective = connective
        constituents.append(cons)
    return constituents

def get_Arg_production_rules(relation, Arg, doc):
    #1.  dict[sent_index] = [token_list]
    dict = {}
    Arg_TokenList = get_Arg_TokenList(relation, Arg)
    for sent_index, word_index in Arg_TokenList:
        if sent_index not in dict:
            dict[sent_index] = [word_index]
        else:
            dict[sent_index].append(word_index)

    #2. production_rules
    Arg_subtrees = []
    for sent_index in dict.keys():
        parse_tree = doc["sentences"][sent_index]["parsetree"].strip()
        syntax_tree = Syntax_tree(parse_tree)
        if syntax_tree.tree != None:
            Arg_indices = dict[sent_index]
            Arg_leaves = set([syntax_tree.get_leaf_node_by_token_index(index) for index in Arg_indices])
            Arg_leaves_labels = set([leaf.label() for leaf in Arg_leaves])
            for nodeposition in syntax_tree.tree.treepositions():
                node = syntax_tree.tree[nodeposition]
                if set(node.leaves()) <= Arg_leaves_labels:
                    Arg_subtrees.append(node)

    production_rules = []
    for node in Arg_subtrees:
        if not isinstance(node, str):
            rule = node.label() + '-->' + ' '.join([child.label() for child in node])
            production_rules.append(rule)

    production_rules = list(set(production_rules))
    return production_rules

def get_self_category(parse_dict, DocID, sent_index, conn_indices):
    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    if syntax_tree.tree == None:
        self_category = "NONE_TREE"
    else:
        self_category = syntax_tree.get_self_category_node_by_token_indices(conn_indices).name

    return self_category

def get_conn_parent_categoryCtx(parse_dict, DocID, sent_index, conn_indices):
    conn_name = get_conn_name(parse_dict, DocID, sent_index, conn_indices)

    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)
    if syntax_tree.tree == None:
        parent_categoryCtx = "NONE_TREE"
    else:
        parent_category_node = syntax_tree.get_parent_category_node_by_token_indices(conn_indices)
        parent_categoryCtx = get_node_linked_Ctx(parent_category_node)

    conn_parent_categoryCtx = "%s|%s" % (conn_name, parent_categoryCtx)

    return conn_parent_categoryCtx

def get_conn_connCtx(parse_dict,docID,sentID,conn_indices,conn_words):
    parse_tree = parse_dict[docID]["sentences"][sentID]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    # conn + connCtx
    if syntax_tree.tree == None:
        connCtx = "NONE_TREE"
    else:
        conn_node = syntax_tree.get_self_category_node_by_token_indices(conn_indices)
        connCtx = get_node_Ctx(conn_node, syntax_tree)

    #conn_connCtx = "%s|%s" % (conn_name, connCtx)
    conn_connCtx ='_'.join(conn_words)+'-'+connCtx
    return conn_connCtx

def get_conn_leftSibling_ctx(parse_dict, DocID, sent_index, conn_indices):
    conn_name = get_C_String(parse_dict, DocID, sent_index, conn_indices)

    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    if syntax_tree.tree == None:
        leftSiblingCtx = "NONE_TREE"
    else:
        leftSibling_node = syntax_tree.get_left_sibling_category_node_by_token_indices(conn_indices)
        leftSiblingCtx = get_node_linked_Ctx(leftSibling_node, syntax_tree)

    conn_leftSiblingCtx = "%s|%s" % (conn_name, leftSiblingCtx)

    return conn_leftSiblingCtx

def get_CParent_to_root_path_node_names(parse_dict,docID,sentID,conn_indices):
    parse_tree = parse_dict[docID]["sentences"][sentID]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    if syntax_tree.tree == None:
        path = "NONE_TREE"
    else:
        path = ""
        for conn_index in conn_indices:
            conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
            conn_parent_node = conn_node.up
            path += syntax_tree.get_node_path_to_root(conn_parent_node) + "-->"
        if path[-3:] == "-->":
            path = path[:-3]
    return path.split("-->")

def get_conn_to_root_path(parse_dict,docID,sentID,conn_indices):
    parse_tree = parse_dict[docID]["sentences"][sentID]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)
    if syntax_tree.tree == None:
        path = "NONE_TREE"
    else:
        path = ""
        for conn_index in conn_indices:
            conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
            t = syntax_tree.get_node_path_to_root(conn_node)
            path += t + "&"
        if path[-1] == "&":
            path = path[:-1]

    return path

def ssArgumentExt(inputFilenamePath):
 parse_file = codecs.open(inputFilenamePath+'/parses.json', encoding='utf8');
 en_parse_dict = json.load(parse_file);
 i = 0;
 for prediction in observedArray:
  filename = bigDiction[i][2];
  sentenceNumber = int(bigDiction[i+1][3]) + 1;
  connWordID = int(bigDiction[i][4]);
  print "ConnWordID: " + str(connWordID);
  parse_tree = en_parse_dict[filename]["sentences"][sentenceNumber]["parsetree"].strip();
  syntax_tree = Syntax_tree(parse_tree)
  if syntax_tree.tree == None:
   return []
  #Get Connective Indices
  conn_indices = [connWordID];
  constituent_nodes = [];
  if len(conn_indices) == 1:# like and or so...
        conn_node = syntax_tree.get_leaf_node_by_token_index(conn_indices[0]).up
  else:
        conn_node = syntax_tree.get_common_ancestor_by_token_indices(conn_indices)
        conn_leaves = set([syntax_tree.get_leaf_node_by_token_index(conn_index) for conn_index in conn_indices])
        children = conn_node.get_children()
        for child in children:
            leaves = set(child.get_leaves())
            if conn_leaves & leaves == set([]):
                constituent_nodes.append(child)
  
  curr = conn_node
  while not curr.is_root():
   constituent_nodes.extend(syntax_tree.get_siblings(curr))
   curr = curr.up

  # obtain the Constituent object according to the node.
  constituents = []
  for node in constituent_nodes:
   cons = Constituent(syntax_tree, node)
   #print "Object Type: " + str(cons.type());
   #print "Object Dir: " + str(cons.dir());
   #print "Object id: " + str(cons.id());
   #print "cons: " + str(cons.connective);
   connective = Connective(filename, sentenceNumber, conn_indices, "text");
   cons.connective = connective
   constituents.append(cons)
  i = i + 1;
  print "Connective ID:" + str(connWordID); 
  print "Size of Observed Array: " + str(len(observedArray));
  print "Size of Constituents Array: " + str(len(constituents)); 

def get_conn_to_root_compressed_path(parse_dict,docID,sentID,conn_indices):
    parse_tree = parse_dict[docID]["sentences"][sentID]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)
    if syntax_tree.tree == None:
        compressed_path = "NONE_TREE"
    else:
        compressed_path = ""
        for conn_index in conn_indices:
            conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
	    conn_parent_node = conn_node.up
	    path = syntax_tree.get_node_path_to_root(conn_parent_node)
	    compressed_path += util.get_compressed_path(path) + "&"
            #t = syntax_tree.get_node_path_to_root(conn_node)
            #path += t + "&"
        if compressed_path[-1] == "&":
            compressed_path = compressed_path[:-1]
    return compressed_path

def get_CParent_to_root_path(parse_dict, DocID, sent_index, conn_indices):
    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    ''' c parent to root '''
    if syntax_tree.tree == None:
        cparent_to_root_path = "NONE_TREE"
    else:
        cparent_to_root_path = ""
        for conn_index in conn_indices:
            conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
            conn_parent_node = conn_node.up
            cparent_to_root_path += syntax_tree.get_node_path_to_root(conn_parent_node) + "&"
        if cparent_to_root_path[-1] == "&":
            cparent_to_root_path = cparent_to_root_path[:-1]

    return cparent_to_root_path

def get_curr_first_prev_last_parse_path(arg_clauses, clause_index, parse_dict):
    DocID = arg_clauses.DocID
    sent_index = arg_clauses.sent_index

    if clause_index - 1 < 0:
        return "NONE"

    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    curr_first_index = arg_clauses.clauses[clause_index][0][0]
    prev_last_index = arg_clauses.clauses[clause_index - 1][0][-1]

    curr_first_node = syntax_tree.get_leaf_node_by_token_index(curr_first_index)
    prev_last_node = syntax_tree.get_leaf_node_by_token_index(prev_last_index)

    return syntax_tree.get_node_to_node_path(curr_first_node, prev_last_node)

def get_conn_to_root_path(arg_clauses, clause_index, parse_dict):
    conn_indices = arg_clauses.conn_indices
    DocID = arg_clauses.DocID
    sent_index = arg_clauses.sent_index
    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)
    if syntax_tree.tree == None:
        path = "NONE_TREE"
    else:
        path = ""
        for conn_index in conn_indices:
            conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
            t = syntax_tree.get_node_path_to_root(conn_node)
            path += t + "&"
        if path[-1] == "&":
            path = path[:-1]

    return path

def get_conn_parent_category_Ctx(arg_clauses, clause_index, parse_dict):
    DocID = arg_clauses.DocID
    sent_index = arg_clauses.sent_index
    conn_indices = arg_clauses.conn_indices

    conn_name = get_con_str(arg_clauses, clause_index, parse_dict)

    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    if syntax_tree.tree == None:
        parent_categoryCtx = "NONE_TREE"
    else:
        parent_category_node = syntax_tree.get_parent_category_node_by_token_indices(conn_indices)
        parent_categoryCtx = get_node_linked_Ctx(parent_category_node, syntax_tree)

    conn_parent_categoryCtx = "%s|%s" % (conn_name, parent_categoryCtx)

    return conn_parent_categoryCtx

def get_CParent_to_root_path_node_names(arg_clauses, clause_index, parse_dict):
    DocID = arg_clauses.DocID
    sent_index = arg_clauses.sent_index
    conn_indices = arg_clauses.conn_indices

    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    if syntax_tree.tree == None:
        path = "NONE_TREE"
    else:
        path = ""
        for conn_index in conn_indices:
            conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
            conn_parent_node = conn_node.up
            path += syntax_tree.get_node_path_to_root(conn_parent_node) + "-->"
        if path[-3:] == "-->":
            path = path[:-3]

    return path.split("-->")

def get_curr_first_to_prev_last_path(arg_clauses, clause_index, parse_dict):
    if clause_index == 0:
        return "NULL"

    DocID = arg_clauses.DocID
    sent_index = arg_clauses.sent_index
    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    if syntax_tree.tree == None:
        return "NOTREE"

    curr_first_index = arg_clauses.clauses[clause_index][0][0]
    prev_last_index = arg_clauses.clauses[clause_index - 1][0][-1]

    curr_first_node = syntax_tree.get_leaf_node_by_token_index(curr_first_index).up
    prev_last_node = syntax_tree.get_leaf_node_by_token_index(prev_last_index).up

    path = syntax_tree.get_node_to_node_path(curr_first_node, prev_last_node)

    return path

def get_conn_to_root_compressed_path(arg_clauses, clause_index, parse_dict):
    DocID = arg_clauses.DocID
    sent_index = arg_clauses.sent_index
    conn_indices = arg_clauses.conn_indices
    parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
    syntax_tree = Syntax_tree(parse_tree)

    if syntax_tree.tree == None:
        compressed_path = "NONE_TREE"
    else:
        compressed_path = ""
        for conn_index in conn_indices:
            conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
            conn_parent_node = conn_node.up

            path = syntax_tree.get_node_path_to_root(conn_parent_node)

            compressed_path += util.get_compressed_path(path) + "&"

        if compressed_path[-1] == "&":
            compressed_path = compressed_path[:-1]
    return compressed_path

    def getProductionRules(self, clause):
        curr_clause_indices = clause# ([1,2,3],yes)

        subtrees = []
        parse_tree = self.parseTree
        syntax_tree = Syntax_tree(parse_tree)
        if syntax_tree.tree != None:
            clause_leaves = set([syntax_tree.get_leaf_node_by_token_index(index) for index in curr_clause_indices])
            no_need = []
            for node in syntax_tree.tree.traverse(strategy="levelorder"):
                if node not in no_need:
                    if set(node.get_leaves()) <= clause_leaves:
                        subtrees.append(node)
                        no_need.extend(node.get_descendants())

        production_rule = []
        for tree in subtrees:
            for node in tree.traverse(strategy="levelorder"):
                if not node.is_leaf():
                    rule = node.name + "-->" + " ".join([child.name for child in node.get_children()])
                    production_rule.append(rule)

        return production_rule

def get_Arg_production_rules(relation, Arg, parse_dict):
    #1.  dict[(DocID, sent_index)] = [token_list]
    dict = {}
    DocID = relation["DocID"]
    Arg_TokenList = get_Arg_TokenList(relation, Arg)
    for sent_index, word_index in Arg_TokenList:
        if (DocID, sent_index) not in dict:
            dict[(DocID, sent_index)] = [word_index]
        else:
            dict[(DocID, sent_index)].append(word_index)

    #2.
    Arg_subtrees = []
    for (DocID, sent_index) in dict.keys():
        parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip()
        syntax_tree = Syntax_tree(parse_tree)
        if syntax_tree.tree != None:
            Arg_indices = dict[(DocID, sent_index) ]
            Arg_leaves = set([syntax_tree.get_leaf_node_by_token_index(index) for index in Arg_indices])

            no_need = []
            for node in syntax_tree.tree.traverse(strategy="levelorder"):
                if node not in no_need:
                    if set(node.get_leaves()) <= Arg_leaves:
                        Arg_subtrees.append(node)
                        no_need.extend(node.get_descendants())


    production_rule = []
    for tree in Arg_subtrees:
        for node in tree.traverse(strategy="levelorder"):
            if not node.is_leaf():
                rule = node.name + "-->" + " ".join([child.name for child in node.get_children()])
                production_rule.append(rule)

    return production_rule