def func_tokens(dictionary, parse_action):
    func_name = Word(alphas+'_', alphanums+'_')

    func_ident = Combine('$' + func_name.copy()('funcname'))
    func_tok = func_ident + originalTextFor(nestedExpr())('args')
    func_tok.leaveWhitespace()
    func_tok.setParseAction(parse_action)
    func_tok.enablePackrat()

    rx_tok = Combine(Literal('$').suppress() + Word(nums)('num'))

    def replace_token(tokens):
        index = int(tokens.num)
        return dictionary.get(index, u'')

    rx_tok.setParseAction(replace_token)

    strip = lambda s, l, tok: tok[0].strip()
    text_tok = CharsNotIn(u',').setParseAction(strip)
    quote_tok = QuotedString('"')

    if dictionary:
        arglist = Optional(delimitedList(quote_tok | rx_tok | text_tok))
    else:
        arglist = Optional(delimitedList(quote_tok | text_tok))

    return func_tok, arglist, rx_tok

def parse_commanders(x):
    # Grammar
    LBRAK, RBRAK, COMMA = (pp.Literal(x).suppress() for x in "[],")
    AND = pp.Literal("and").suppress()
    and_ = pp.Optional(COMMA) + AND
    RANK = pp.Or(pp.Literal(x) for x in RANKS_LOOKUP.keys())("rank")
    NAME = pp.Word(pp.srange("[A-Z]"), pp.alphas + pp.alphas8bit, min=2)
    ABBR = pp.Regex(r"([A-Z]\.)+")
    SUFFIX = pp.Literal("Jr.")
    NOCMDR = (pp.Literal("None"))("none")
    COMBATANT = pp.oneOf(' '.join(('US', 'CS', 'I')))("combatant")
    
    nametoken = ABBR | pp.quotedString | NAME
    name = (pp.OneOrMore(nametoken) + pp.Optional(pp.Literal(",") + SUFFIX))("fullname")
    name.addParseAction(lambda s,l,t: ' '.join(t))
    
    cmdrname = RANK + name 
    commander = pp.Group(cmdrname | NOCMDR)
    commander_list = pp.Group(pp.delimitedList(commander, ",") +
                              pp.Optional(and_ + commander))("commanders")
    milforce = pp.Group(commander_list + LBRAK + COMBATANT + RBRAK)
    grammar = pp.delimitedList(milforce, ";")
    toks = grammar.parseString(x)

    # A smarter grammar could probably have avoided this
    res = {}
    for _force in toks:
        k = _force['combatant']
        res[k] = [x.asDict() for x in _force['commanders']
                  if "none" not in x.asDict()]
    return res

def parse_connection_str(connstr):
    ## Grammar for connection syntax
    digits="0123456789"
    othervalid="[email protected]"
    identifier= Word(alphas+digits+othervalid)
    nodename=identifier.setResultsName('nodename')

    outputnames = delimitedList( identifier ).setResultsName('outputnames')
    inputnames  = delimitedList( identifier ).setResultsName('inputnames')

    # middle nodes have both inputs and outputs
    middlenode= Group( nodename + Suppress('(') + inputnames
                       + Optional( "|" + outputnames)
                       + Suppress(")") ).setResultsName('middlenode')
    # first node has only outputs
    headnode = (nodename + Suppress("(") + outputnames
                + Suppress(")")).setResultsName('headnode')
    # last node has only inputs
    tailnode = (nodename + Suppress("(") + inputnames
                + Suppress(")")).setResultsName('tailnode')

    # connect head -> [middle ->] tail
    connect= Group( headnode
                    + Group(ZeroOrMore(Suppress("->") \
                        + middlenode + FollowedBy("->") )).setResultsName('middlenodes')
                    + Suppress("->")+tailnode).setResultsName('nodes')

    connectlist = Group( connect + ZeroOrMore( Suppress(";")\
                        + connect )).setResultsName('connects')

    parsed=connectlist.parseString(connstr)
    check_numconnections(parsed)
    return parsed

def _create_update_expression():
    """ Create the grammar for an update expression """
    ine = (Word('if_not_exists') + Suppress('(') + var +
           Suppress(',') + var_val + Suppress(')'))
    list_append = (Word('list_append') + Suppress('(') + var_val +
                   Suppress(',') + var_val + Suppress(')'))
    fxn = Group(ine | list_append).setResultsName('set_function')
    # value has to come before var to prevent parsing TRUE/FALSE as variables
    path = (value | fxn | var)
    set_val = ((path + oneOf('+ -') + path) | path)
    set_cmd = Group(var + Suppress('=') + set_val)
    set_expr = (Suppress(upkey('set')) +
                delimitedList(set_cmd)).setResultsName('set_expr')
    add_expr = (Suppress(upkey('add')) +
                delimitedList(Group(var + value)))\
        .setResultsName('add_expr')
    delete_expr = (Suppress(upkey('delete')) +
                   delimitedList(Group(var + value)))\
        .setResultsName('delete_expr')
    remove_expr = (
        Suppress(
            upkey('remove')) +
        delimitedList(var)).setResultsName('remove_expr')
    return OneOrMore(set_expr | add_expr | delete_expr | remove_expr)\
        .setResultsName('update')

def parse_select_columns(string):
    """Parse a select query and return the columns

    Args:
        string(str): Input string to be parsed

    Returns:
        result(list of str): List of columns
    """

    if string == "":
        return list()

    if string.upper().startswith("WITH"):
        suppressor = _with + delimitedList(_db_name + _as + subquery)
        string = suppressor.suppress().transformString(string)

    # Supress everything after the first from
    suppressor = MatchFirst(_from) + restOfLine
    string = suppressor.suppress().transformString(string)

    parser = _select + delimitedList(field_parser).setResultsName("columns")
    output = parser.parseString(string).columns.asList()

    # Strip extra whitespace from the string
    return [column.strip() for column in output]

    def __init__(self):
        # literals
        star = Literal('*')
        comma = Suppress(',')

        # indentifiers
        identifier = Word(alphas, alphanums+'_')
        alias      = identifier.copy()

        # select clause
        column_name = Combine(Optional(alias + '.') +
                              identifier +
                              Optional(' as ' + identifier))\
                              .setResultsName('column_name')
        select = Keyword('select', caseless=1)
        select_clause = (star | Group(delimitedList(column_name, comma)))\
            .setResultsName('select_clause')

        # from clause
        from_  = Keyword('from', caseless=1)
        table_name =  delimitedList(identifier + Optional(alias), comma)
        from_clause = table_name.setResultsName('from_clause')

        # select statment
        self.select_stmt = select + select_clause + from_ + from_clause

def parse_file(file_name):

    number = pp.Word(pp.nums)
    identifier = pp.Word(pp.alphas + "_", pp.alphanums + "_")

    lbrace = pp.Literal('{').suppress()
    rbrace = pp.Literal('}').suppress()
    cls = pp.Keyword('class')
    colon = pp.Literal(":")
    semi = pp.Literal(";").suppress()
    langle = pp.Literal("<")
    rangle = pp.Literal(">")
    equals = pp.Literal("=")
    comma = pp.Literal(",")
    lparen = pp.Literal("(")
    rparen = pp.Literal(")")
    lbrack = pp.Literal("[")
    rbrack = pp.Literal("]")
    mins = pp.Literal("-")
    struct = pp.Keyword('struct')
    template = pp.Keyword('template')
    final = pp.Keyword('final')("final")
    stub = pp.Keyword('stub')("stub")
    with_colon = pp.Word(pp.alphanums + "_" + ":")
    btype = with_colon
    type = pp.Forward()
    nestedParens = pp.nestedExpr('<', '>')

    tmpl = pp.Group(btype("template_name") + langle.suppress() + pp.Group(pp.delimitedList(type)) + rangle.suppress())
    type << (tmpl | btype)
    enum_lit = pp.Keyword('enum')
    enum_class = pp.Group(enum_lit + cls)
    ns = pp.Keyword("namespace")

    enum_init = equals.suppress() + pp.Optional(mins) + number
    enum_value = pp.Group(identifier + pp.Optional(enum_init))
    enum_values = pp.Group(lbrace + pp.delimitedList(enum_value) + pp.Optional(comma) + rbrace)
    content = pp.Forward()

    member_name = pp.Combine(pp.Group(identifier + pp.Optional(lparen + rparen)))
    attrib = pp.Group(lbrack.suppress() + lbrack.suppress() + pp.SkipTo(']') + rbrack.suppress() + rbrack.suppress())
    opt_attribute = pp.Optional(attrib)("attribute")
    namespace = pp.Group(ns("type") + identifier("name") + lbrace + pp.Group(pp.OneOrMore(content))("content") + rbrace)
    enum = pp.Group(enum_class("type") + identifier("name") + colon.suppress() + identifier("underline_type") + enum_values("enum_values") + pp.Optional(semi).suppress())
    default_value = equals.suppress() + pp.SkipTo(';')
    class_member = pp.Group(type("type") + member_name("name") + opt_attribute + pp.Optional(default_value)("default") + semi.suppress())("member")
    template_param = pp.Group(identifier("type") + identifier("name"))
    template_def = pp.Group(template + langle + pp.Group(pp.delimitedList(template_param))("params") + rangle)
    class_content = pp.Forward()
    class_def = pp.Group(pp.Optional(template_def)("template") + (cls | struct)("type") + with_colon("name") + pp.Optional(final) + pp.Optional(stub) + opt_attribute + lbrace + pp.Group(pp.ZeroOrMore(class_content))("members") + rbrace + pp.Optional(semi))
    content << (enum | class_def | namespace)
    class_content << (enum | class_def | class_member)
    for varname in "enum class_def class_member content namespace template_def".split():
        locals()[varname].setName(varname)
    rt = pp.OneOrMore(content)
    singleLineComment = "//" + pp.restOfLine
    rt.ignore(singleLineComment)
    rt.ignore(pp.cStyleComment)
    return rt.parseFile(file_name, parseAll=True)

def _tdb_grammar(): #pylint: disable=R0914
    """
    Convenience function for getting the pyparsing grammar of a TDB file.
    """
    int_number = Word(nums).setParseAction(lambda t: [int(t[0])])
    # matching float w/ regex is ugly but is recommended by pyparsing
    float_number = Regex(r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?') \
        .setParseAction(lambda t: [float(t[0])])
    # symbol name, e.g., phase name, function name
    symbol_name = Word(alphanums+'_', min=1)
    # species name, e.g., CO2, AL, FE3+
    species_name = Word(alphanums+'+-', min=1)
    constituent_array = Group(
        delimitedList(Group(delimitedList(species_name, ',')), ':')
        )
    param_types = CaselessKeyword('G') | CaselessKeyword('L') | \
                  CaselessKeyword('TC') | CaselessKeyword('BMAGN')
    # Let sympy do heavy arithmetic / algebra parsing for us
    # a convenience function will handle the piecewise details
    func_expr = float_number + OneOrMore(SkipTo(';') \
        + Suppress(';') + float_number + Suppress(Word('YNyn', exact=1)))
    # ELEMENT
    cmd_element = CaselessKeyword('ELEMENT') + Word(alphas+'/-', min=1, max=2)
    # TYPE_DEFINITION
    cmd_typedef = CaselessKeyword('TYPE_DEFINITION') + \
        Suppress(White()) + CharsNotIn(' !', exact=1) + SkipTo(LineEnd())
    # FUNCTION
    cmd_function = CaselessKeyword('FUNCTION') + symbol_name + \
        func_expr.setParseAction(_make_piecewise_ast)
    # DEFINE_SYSTEM_DEFAULT
    cmd_defsysdef = CaselessKeyword('DEFINE_SYSTEM_DEFAULT')
    # DEFAULT_COMMAND
    cmd_defcmd = CaselessKeyword('DEFAULT_COMMAND')
    # PHASE
    cmd_phase = CaselessKeyword('PHASE') + symbol_name + \
        Suppress(White()) + CharsNotIn(' !', min=1) + Suppress(White()) + \
        Suppress(int_number) + Group(OneOrMore(float_number)) + LineEnd()
    # CONSTITUENT
    cmd_constituent = CaselessKeyword('CONSTITUENT') + symbol_name + \
        Suppress(White()) + Suppress(':') + constituent_array + \
        Suppress(':') + LineEnd()
    # PARAMETER
    cmd_parameter = CaselessKeyword('PARAMETER') + param_types + \
        Suppress('(') + symbol_name + Suppress(',') + constituent_array + \
        Suppress(';') + int_number + Suppress(')') + \
        func_expr.setParseAction(_make_piecewise_ast)
    # Now combine the grammar together
    all_commands = cmd_element | \
                    cmd_typedef | \
                    cmd_function | \
                    cmd_defsysdef | \
                    cmd_defcmd | \
                    cmd_phase | \
                    cmd_constituent | \
                    cmd_parameter | \
                    Empty()
    return all_commands

def generate_grammar():
    the_range = generate_range()
    visual = generate_fullvisual()
    visual = visual ^ generate_sub_cmd()
    vim_cmd = generate_vim_cmd()

    complex_cmd = p.Group(the_range)('range') + p.delimitedList(p.Group(visual), delim=";")('cmd')

    return p.Group(vim_cmd)('vim_cmd') ^ p.Group(the_range)('range') ^ p.Group(p.delimitedList(p.Group(visual), delim=";"))('cmd') ^ p.Group(complex_cmd)('complex_cmd')

    def parse(self, request):
        input = request._rest_context.get('filter')
        if not input:
            return None

        condition_positions = []

        operator = pp.Regex('|'.join(self.ALLOWED_OPERATORS))
        number = pp.Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")

        AND = pp.Literal(LOGICAL_OPERATORS.AND)
        OR = pp.Literal(LOGICAL_OPERATORS.OR)
        NOT = pp.Literal(LOGICAL_OPERATORS.NOT)

        identifier = pp.Regex(r"[a-zA-Z]+[a-zA-Z0-9]*(_[a-zA-Z0-9]+)*")
        identifiers = pp.Group(pp.delimitedList(identifier, delim="__", combine=False))

        comparison_term = pp.Forward()
        list_term = (
            pp.Group(
                pp.Suppress('[') + pp.delimitedList(comparison_term, delim=",", combine=False) + pp.Suppress(']')
            ) |
            pp.Group(
                pp.Suppress('(') + pp.delimitedList(comparison_term, delim=",", combine=False) + pp.Suppress(')')
            ) |
            pp.Group(
                pp.Suppress('{') + pp.delimitedList(comparison_term, delim=",", combine=False) + pp.Suppress('}')
            )
        )
        string = (
            pp.QuotedString("'", escChar='\\', unquoteResults=True) | pp.QuotedString('"', escChar='\\',
                                                                                      unquoteResults=True)
        )
        null = pp.Literal('null').setParseAction(lambda s,l,t: None)
        boolean = pp.Regex('|'.join(('true', 'false'))).setParseAction(lambda s, l, t: t[0] == 'true')

        comparison_term << (string | number | list_term | null | boolean)

        condition = pp.Group(identifiers + operator + comparison_term).setResultsName('condition')
        condition.setParseAction(lambda s, loc, tocs: condition_positions.append(loc))

        expr = pp.operatorPrecedence(
            condition, [
                (NOT, 1, pp.opAssoc.RIGHT,),
                (AND, 2, pp.opAssoc.LEFT,),
                (OR, 2, pp.opAssoc.LEFT,),
            ]
        )

        try:
            return self._parse_to_conditions(
                expr.parseString(input, parseAll=True).asList()[0], list(condition_positions), condition, input
            )
        except pp.ParseException as ex:
            raise FilterParserError(
                mark_safe(ugettext('Invalid filter value "{}"').format(input))
            )

def parse(str):
    tokens = ''
    # define SQL tokens
    selectStmt = Forward()
    selectToken = Keyword("select", caseless=True)
    fromToken   = Keyword("from", caseless=True)
    
    ident          = Word( alphas, alphanums + "_$" ).setName("identifier")
    columnName     = Upcase( delimitedList( ident, ".", combine=True ) )
    columnNameList = Group( delimitedList( columnName ) )
    tableName      = Upcase( delimitedList( ident, ".", combine=True ) )
    tableNameList  = Group( delimitedList( tableName ) )
    
    whereExpression = Forward()
    and_ = Keyword("and", caseless=True)
    or_ = Keyword("or", caseless=True)
    in_ = Keyword("in", caseless=True)
    
    E = CaselessLiteral("E")
    binop = oneOf("= != < > >= <= eq ne lt le gt ge", caseless=True)
    arithSign = Word("+-",exact=1)
    realNum = Combine( Optional(arithSign) + ( Word( nums ) + "." + Optional( Word(nums) )  |
                                                             ( "." + Word(nums) ) ) + 
                Optional( E + Optional(arithSign) + Word(nums) ) )
    intNum = Combine( Optional(arithSign) + Word( nums ) + 
                Optional( E + Optional("+") + Word(nums) ) )
    
    columnRval = realNum | intNum | quotedString | columnName # need to add support for alg expressions
    whereCondition = Group(
        ( columnName + binop + columnRval ) |
        ( columnName + in_ + "(" + delimitedList( columnRval ) + ")" ) |
        ( columnName + in_ + "(" + selectStmt + ")" ) |
        ( "(" + whereExpression + ")" )
        )
    whereExpression << whereCondition + ZeroOrMore( ( and_ | or_ ) + whereExpression ) 
    
    # define the grammar
    selectStmt      << ( selectToken + 
                       ( '*' | columnNameList ).setResultsName( "columns" ) + 
                       fromToken + 
                       tableNameList.setResultsName( "tables" ) + 
                       Optional( Group( CaselessLiteral("where") + whereExpression ), "" ).setResultsName("where") )
    
    simpleSQL = selectStmt
    
    # define Oracle comment format, and ignore them
    oracleSqlComment = "--" + restOfLine
    simpleSQL.ignore( oracleSqlComment )
    
    try:
        tokens = simpleSQL.parseString( str )
    except ParseException, err:
        print " "*err.loc + "^\n" + err.msg
        print err

def create_insert():
    """ Create the grammar for the 'insert' statement """
    insert = upkey('insert').setResultsName('action')
    attrs = Group(delimitedList(var)).setResultsName('attrs')

    # VALUES
    values_key = upkey('values')
    value_group = Group(Suppress('(') + delimitedList(value) + Suppress(')'))
    values = Group(delimitedList(value_group)).setResultsName('data')

    return (insert + into + table + Suppress('(') + attrs + Suppress(')') +
            values_key + values)

def syntax():
    """https://docs.python.org/2/reference/expressions.html
    """

    identifier = Word(alphas+'_', alphanums+'_')
    string_ = (
        QuotedString('"') | QuotedString("'")
    ).setParseAction(composition(str, ''.join))
    # FIXME unicode string (how is this handled in python 3?) important to be
    # able to destinguice between unicode and not
    unicode_string = (
        Literal('u').suppress() + string_
    ).setParseAction(composition(unicode, u''.join))
    integer = (
        Optional(oneOf(['+', '-']), default='+') + Word(nums)
    ).setParseAction(composition(int, ''.join))
    long_ = (
        integer + Literal('L').suppress()
    ).setParseAction(composition(long, ''.join, partial(map, str)))
    float_ = (
        Optional(integer, default=0) + Literal('.') + Optional(Word(nums), default='0')
    ).setParseAction(composition(float, ''.join, partial(map, str)))
    # TODO complex
    number = float_ | long_ | integer
    atom = unicode_string | string_ | number

    expression = Forward()  # FIXME name of this?
    dictionary = Group(
        Literal('{').suppress() + delimitedList(
            expression + Literal(':').suppress() + expression
        )
    ).setParseAction(list)
    set_ = Group(
        Literal('{').suppress() + delimitedList(
            expression
        ) + Literal('}').suppress()
    ).setParseAction(list)
    list_ = Group(
        Literal('[').suppress() + delimitedList(
            expression
        ) + Literal(']').suppress()
    ).setParseAction(list)
    tuple_ = Group(
        Literal('(').suppress() + delimitedList(
            expression
        ) + Literal(')').suppress()
    ).setParseAction(tuple)

    instance = identifier + tuple_

    expression << (atom | tuple_ | instance | dictionary | set_ | list_)

    return expression

def CORBA_IDL_BNF():
    global bnf
    
    if not bnf:

        # punctuation
        (colon,lbrace,rbrace,lbrack,rbrack,lparen,rparen,
        equals,comma,dot,slash,bslash,star,semi,langle,rangle) = map(Literal, r":{}[]()=,./\*;<>")
        
        # keywords
        (any_, attribute_, boolean_, case_, char_, const_, context_, default_, double_, enum_, exception_, 
        FALSE_, fixed_, float_, inout_, interface_, in_, long_, module_, Object_, octet_, oneway_, out_, raises_, 
        readonly_, sequence_, short_, string_, struct_, switch_, TRUE_, typedef_, unsigned_, union_, void_, 
        wchar_, wstring_) = map(Keyword, """any attribute boolean case char const context 
            default double enum exception FALSE fixed float inout interface in long module 
            Object octet oneway out raises readonly sequence short string struct switch
            TRUE typedef unsigned union void wchar wstring""".split())
        
        identifier = Word( alphas, alphanums + "_" ).setName("identifier")
        
        real = Regex(r"[+-]?\d+\.\d*([Ee][+-]?\d+)?").setName("real")
        integer = Regex(r"0x[0-9a-fA-F]+|[+-]?\d+").setName("int")

        udTypeName = delimitedList( identifier, "::", combine=True ).setName("udType")
        typeName = ( any_ | boolean_ | char_ | double_ | fixed_ | 
                    float_ | long_ | octet_ | short_ | string_ | 
                    wchar_ | wstring_ | udTypeName ).setName("type")
        sequenceDef = Forward().setName("seq")
        sequenceDef << Group( sequence_ + langle + ( sequenceDef | typeName ) + rangle )
        typeDef = sequenceDef | ( typeName + Optional( lbrack + integer + rbrack ) )
        typedefDef = Group( typedef_ + typeDef + identifier + semi ).setName("typedef")

        moduleDef = Forward()
        constDef = Group( const_ + typeDef + identifier + equals + ( real | integer | quotedString ) + semi ) #| quotedString )
        exceptionItem = Group( typeDef + identifier + semi )
        exceptionDef = ( exception_ + identifier + lbrace + ZeroOrMore( exceptionItem ) + rbrace + semi )
        attributeDef = Optional( readonly_ ) + attribute_ + typeDef + identifier + semi
        paramlist = delimitedList( Group( ( inout_ | in_ | out_ ) + typeName + identifier ) ).setName( "paramlist" )
        operationDef = ( ( void_ ^ typeDef ) + identifier + lparen + Optional( paramlist ) + rparen + \
                        Optional( raises_ + lparen + Group( delimitedList( typeName ) ) + rparen ) + semi )
        interfaceItem = ( constDef | exceptionDef | attributeDef | operationDef )
        interfaceDef = Group( interface_ + identifier  + Optional( colon + delimitedList( typeName ) ) + lbrace + \
                        ZeroOrMore( interfaceItem ) + rbrace + semi ).setName("opnDef")
        moduleItem = ( interfaceDef | exceptionDef | constDef | typedefDef | moduleDef )
        moduleDef << module_ + identifier + lbrace + ZeroOrMore( moduleItem ) + rbrace + semi

        bnf = ( moduleDef | OneOrMore( moduleItem ) )
        
        singleLineComment = "//" + restOfLine
        bnf.ignore( singleLineComment )
        bnf.ignore( cStyleComment )
        
    return bnf

def build_foreign_key_parser():
    left = pp.Literal('(').suppress()
    right = pp.Literal(')').suppress()
    attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
    new_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('new_attrs')
    arrow = pp.Literal('->').suppress()
    lbracket = pp.Literal('[').suppress()
    rbracket = pp.Literal(']').suppress()
    option = pp.Word(pp.srange('[a-zA-Z]'))
    options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket)
    ref_table = pp.Word(pp.alphas, pp.alphanums + '._').setResultsName('ref_table')
    ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('ref_attrs')
    return new_attrs + arrow + options + ref_table + ref_attrs

 def buildMapSyntax(self):
     '''Subset of grammar for DMM files.
        
        "aai" = (/obj/structure/sign/securearea{desc = "A warning sign which reads 'HIGH VOLTAGE'"; icon_state = "shock"; name = "HIGH VOLTAGE"; pixel_y = -32},/turf/space,/area)
     '''
     dreamList = pyp.Forward()
     
     # Literals
     singlelineString = pyp.QuotedString('"', '\\').setResultsName('string').setParseAction(self.makeListString)
     fileRef = pyp.QuotedString("'", '\\').setResultsName('fileRef').setParseAction(self.makeFileRef)
     multilineString = pyp.QuotedString(quoteChar='{"', endQuoteChar='"}', multiline=True).setResultsName('string').setParseAction(self.makeListString)
     number = pyp.Regex(r'\-?\d+(\.\d*)?([eE]\d+)?').setResultsName('number').setParseAction(self.makeListNumber)
     
     #  Paths
     relpath = pyp.ident | relpath + SLASH + pyp.ident
     abspath = SLASH + relpath
     path = (abspath | relpath).setParseAction(self.handlePath)
     pathslash = path + SLASH
     
     # Other symbols
     listStart = pyp.Suppress('list(')
     openParen = pyp.Suppress("(")
     closeParen = pyp.Suppress(')')
     
     # Grammar
     listConstant = singlelineString | fileRef | multilineString | number | dreamList | abspath
     listElement = listConstant | (listConstant + '=' + listConstant)
     listElement = pyp.operatorPrecedence(listElement, [
                             ("=", 2, pyp.opAssoc.LEFT,),
                             ])
     listContents = pyp.delimitedList(listElement)
     dreamList << pyp.Group(listStart + listContents + closeParen)
     dreamList.setParseAction(self.makeList)
     
     
     # DMM Atom definition
     atomDefProperty = pyp.ident + "=" + listConstant 
     atomDefProperty = pyp.operatorPrecedence(atomDefProperty, [
                             ("=", 2, pyp.opAssoc.LEFT,),
                             ])
     atomDefPropertyListContents = pyp.delimitedList(listElement, delim=';')
     atomDefProperties = pyp.Suppress("{") + atomDefPropertyListContents + pyp.Suppress("}")
     atomDef = abspath | abspath + atomDefProperties
     
     # DMM Tile Definition
     tileDefListContents = pyp.delimitedList(atomDef)
     tileDefAtomList = openParen + tileDefListContents + closeParen
     tileDef = singlelineString + '=' + tileDefAtomList
     tileDef.setParseAction(self.makeTileDef)
     return tileDef

def build_insert_grammar():
    insert_grammar = Forward()

    insert_into_keyword_token = Keyword("insert into", caseless=True)
    values_token = Keyword("values", caseless=True)

    columns = Optional(Group(delimitedList(identifier_token, ",")))
    values_list_token = Group(delimitedList(Word(alphanums + " "), ","))

    insert_grammar << insert_into_keyword_token + table_name_token.setResultsName("table_name") \
                      + Literal("(") + columns.setResultsName("columns") + Literal(")") + \
                      values_token + Literal("(") + values_list_token.setResultsName("values_list") + Literal(")")

    return insert_grammar

def grammar():
    # terminals ----------------------------------
    colon = Literal(':')
    equal = Suppress('=')
    slash = Suppress('/')
    open_paren = Suppress('(')
    close_paren = Suppress(')')
    open_brace = Suppress('{')
    close_brace = Suppress('}')

    # np:tagName ---------------------------------
    nspfx = Word(alphas)
    local_name = Word(alphas)
    tagname = Combine(nspfx + colon + local_name)

    # np:attr_name=attr_val ----------------------
    attr_name = Word(alphas + ':')
    attr_val = Word(alphanums + ' -.%')
    attr_def = Group(attr_name + equal + attr_val)
    attr_list = open_brace + delimitedList(attr_def) + close_brace

    text = dblQuotedString.setParseAction(removeQuotes)

    # w:jc{val=right} ----------------------------
    element = (
        tagname('tagname')
        + Group(Optional(attr_list))('attr_list')
        + Optional(text, default='')('text')
    ).setParseAction(Element.from_token)

    child_node_list = Forward()

    node = Group(
        element('element')
        + Group(Optional(slash + child_node_list))('child_node_list')
    ).setParseAction(connect_node_children)

    child_node_list << (
        open_paren + delimitedList(node) + close_paren
        | node
    )

    root_node = (
        element('element')
        + Group(Optional(slash + child_node_list))('child_node_list')
        + stringEnd
    ).setParseAction(connect_root_node_children)

    return root_node

 def define_operand(self):
     """
     Return the syntax definition for an operand.
     
     An operand can be a variable, a string, a number or a set. A set
     is made of other operands, including other sets.
     
     **This method shouldn't be overridden**. Instead, override the syntax
     definitions for variables, strings and/or numbers.
     
     If you want to customize the sets, check :meth:`T_SET_START`,
     :meth:`T_SET_END` and :meth:`T_ELEMENT_SEPARATOR`.
     
     """
     identifier = self.define_identifier()
     operand = Forward()
     
     # Defining the sets:
     set_start = Suppress(self._grammar.get_token("set_start"))
     set_end = Suppress(self._grammar.get_token("set_end"))
     element_separator = self._grammar.get_token("element_separator")
     elements = delimitedList(operand, delim=element_separator)
     set_ = Group(set_start + Optional(elements) + set_end)
     set_.setParseAction(self.make_set)
     set_.setName("set")
     
     # Defining the variables:
     variable = identifier.copy()
     variable.setName("variable")
     variable.addParseAction(self.make_variable)
     
     # Defining the functions:
     function_name = identifier.setResultsName("function_name")
     function_name.setName("function_name")
     args_start = Suppress(self._grammar.get_token("arguments_start"))
     args_end = Suppress(self._grammar.get_token("arguments_end"))
     args_sep = self._grammar.get_token("arguments_separator")
     arguments = Optional(Group(delimitedList(operand, delim=args_sep)),
                          default=())
     arguments = arguments.setResultsName("arguments")
     arguments.setParseAction(lambda tokens: tokens[0])
     function = function_name + args_start + arguments + args_end
     function.setName("function")
     function.setParseAction(self.make_function)
     
     operand << (function | variable | self.define_number() | \
                 self.define_string() | set_)
     
     return operand

    def __init__(self):
        self.realNum = Combine( Optional(tokens.arithSign) + ( Word( nums ) + "." + Optional( Word(nums) )  |
                                                                 ( "." + Word(nums) ) ) +
                    Optional( tokens.E + Optional(tokens.arithSign) + Word(nums) ) )
        self.intNum = Combine( Optional(tokens.arithSign) + Word( nums ) +
                    Optional( tokens.E + Optional("+") + Word(nums) ) )

        self.columnRval = self.realNum | self.intNum | quotedString | tokens.columnName # need to add support for alg expressions
        self.whereCondition = Group(
            ( tokens.columnName + tokens.binop + self.columnRval ) |
            ( tokens.columnName + tokens.in_ + tokens.LPAREN + delimitedList( self.columnRval ) + tokens.RPAREN ) |
            ( tokens.columnName + tokens.in_ + tokens.LPAREN + tokens.selectStmt + tokens.RPAREN ) |
            ( tokens.LPAREN + tokens.whereExpression + tokens.RPAREN )
            )
        tokens.whereExpression << (self.whereCondition + Optional( ZeroOrMore(   tokens.and_ | tokens.or_ ) + tokens.whereExpression ))


        self.joinCondition = Group(
            ( tokens.tableName + tokens.on_ + tokens.whereExpression )
            )
        tokens.joinExpression << (self.joinCondition  )


        # define the grammar
        tokens.selectStmt      << ( tokens.selectToken.setResultsName("command") +
                           ( '*' | tokens.columnNameList ).setResultsName( "columns" ) +

                           tokens.fromToken +
                           tokens.tableNameList.setResultsName( "tables" ) +
                Optional( Group( CaselessLiteral("join") + tokens.joinExpression ), "" ).setResultsName("join") +
                           Optional( Group( Suppress(CaselessLiteral("where")) + tokens.whereExpression ), "" ).setResultsName("where") )

        #self.valuesIter = ( self.columnRval | "," + self.columnRval)

        tokens.insertStmt << (tokens.insertToken.setResultsName("command") +
                                tokens.intoToken.setResultsName("middle") +
                                tokens.columnNameList.setResultsName( "tables" ) +
                                tokens.valuesToken.setResultsName("val") +
                                tokens.LPAREN + Group(delimitedList(self.columnRval, delim=r', ')).setResultsName("insValues") + tokens.RPAREN



                                )

        self.simpleSQL =  tokens.selectStmt | tokens.insertStmt

        # define Oracle comment format, and ignore them
        self.oracleSqlComment = "--" + restOfLine
        self.simpleSQL.ignore( self.oracleSqlComment )