def build_index_parser():
    left = pp.Literal('(').suppress()
    right = pp.Literal(')').suppress()
    unique = pp.Optional(pp.CaselessKeyword('unique')).setResultsName('unique')
    index = pp.CaselessKeyword('index').suppress()
    attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
    return unique + index + left + pp.delimitedList(attribute_name).setResultsName('attr_list') + right

def build_attribute_parser():
    quoted = pp.Or(pp.QuotedString('"'), pp.QuotedString("'"))
    colon = pp.Literal(':').suppress()
    attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]')).setResultsName('name')
    data_type = pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)).setResultsName('type')
    default = pp.Literal('=').suppress() + pp.SkipTo(colon, ignore=quoted).setResultsName('default')
    comment = pp.Literal('#').suppress() + pp.restOfLine.setResultsName('comment')
    return attribute_name + pp.Optional(default) + colon + data_type + comment

 def __init__(self, logger):
   self.salt_client = salt.client.LocalClient()
   self.etcd = Etcd(logger)
   self.logger = logger
   # Parse out the username and formation name 
   # from the ETCD directory string
   self.formation_parser = Literal('/formations/') + \
     Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
     Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')

def handleMacro(toks):
    macroChar = toks[0][1]
    if macroChar == "d":
        return CharSetRandomizer(srange("[0-9]"))
    elif macroChar == "w":
        return CharSetRandomizer(srange("[A-Za-z0-9_]"))
    elif macroChar == "s":
        return CharSetRandomizer(" ")
    else:
        raise ParseFatalException("", 0, "unsupported macro character (" + macroChar + ")")

def grammer():
    lparen = pp.Suppress("(")
    rparen = pp.Suppress(")")
    equal = pp.Suppress("=")
    nl = pp.Suppress(pp.LineEnd())
    reg = pp.Combine("$" + pp.Optional("cr") + pp.Word(pp.srange("[0-7]"),
                                                       max=1))
    num = pp.Word(pp.srange("[0-9]")).setParseAction(
        lambda s, l, t: int(t[0]))
    val = pp.Word(pp.srange("[0-9a-fA-F]")).setParseAction(
        lambda s, l, t: int(t[0], 16))
    values = pp.Dict(pp.OneOrMore(pp.Group(reg + equal + val)))
    return num + lparen + values + rparen + nl

def build_foreign_key_parser():
    left = pp.Literal('(').suppress()
    right = pp.Literal(')').suppress()
    attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
    new_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('new_attrs')
    arrow = pp.Literal('->').suppress()
    lbracket = pp.Literal('[').suppress()
    rbracket = pp.Literal(']').suppress()
    option = pp.Word(pp.srange('[a-zA-Z]'))
    options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket)
    ref_table = pp.Word(pp.alphas, pp.alphanums + '._').setResultsName('ref_table')
    ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('ref_attrs')
    return new_attrs + arrow + options + ref_table + ref_attrs

def build_foreign_key_parser_old():
    # old-style foreign key parser. Superceded by expression-based syntax. See issue #436
    # This will be deprecated in a future release.
    left = pp.Literal('(').suppress()
    right = pp.Literal(')').suppress()
    attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
    new_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('new_attrs')
    arrow = pp.Literal('->').suppress()
    lbracket = pp.Literal('[').suppress()
    rbracket = pp.Literal(']').suppress()
    option = pp.Word(pp.srange('[a-zA-Z]'))
    options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket).setResultsName('options')
    ref_table = pp.Word(pp.alphas, pp.alphanums + '._').setResultsName('ref_table')
    ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('ref_attrs')
    return new_attrs + arrow + options + ref_table + ref_attrs

def _construct_grammar():
    logical_operator = get_logical_operator()
    logical_expression = get_logical_expression()

    facets_expression = get_facet_expression()
    highlight_expression = get_highlight_expression()
    sort_expression = get_sort_expression()
    aggs_expression = get_aggregations_expression()
    nested_expression = get_nested_expression()

    # The below line describes how the type expression should be.
    type_expression = Word('type')\
        + Word(':').suppress()\
        + Word(srange("[a-zA-Z0-9_]"))\
        + Optional(CaselessLiteral('AND')).suppress()
    type_expression.setParseAction(parse_type_expression)

    base_expression = Optional(highlight_expression)\
        + Optional(sort_expression)\
        + Optional(type_expression)\
        + ZeroOrMore(
            (facets_expression
             | aggs_expression
             | nested_expression
             | logical_expression)
            + Optional(logical_operator)
        ).setParseAction(parse_one_or_more_logical_expressions)
    base_expression.setParseAction(parse_type_logical_facets_expression)

    return base_expression

def parse_commanders(x):
    # Grammar
    LBRAK, RBRAK, COMMA = (pp.Literal(x).suppress() for x in "[],")
    AND = pp.Literal("and").suppress()
    and_ = pp.Optional(COMMA) + AND
    RANK = pp.Or(pp.Literal(x) for x in RANKS_LOOKUP.keys())("rank")
    NAME = pp.Word(pp.srange("[A-Z]"), pp.alphas + pp.alphas8bit, min=2)
    ABBR = pp.Regex(r"([A-Z]\.)+")
    SUFFIX = pp.Literal("Jr.")
    NOCMDR = (pp.Literal("None"))("none")
    COMBATANT = pp.oneOf(' '.join(('US', 'CS', 'I')))("combatant")
    
    nametoken = ABBR | pp.quotedString | NAME
    name = (pp.OneOrMore(nametoken) + pp.Optional(pp.Literal(",") + SUFFIX))("fullname")
    name.addParseAction(lambda s,l,t: ' '.join(t))
    
    cmdrname = RANK + name 
    commander = pp.Group(cmdrname | NOCMDR)
    commander_list = pp.Group(pp.delimitedList(commander, ",") +
                              pp.Optional(and_ + commander))("commanders")
    milforce = pp.Group(commander_list + LBRAK + COMBATANT + RBRAK)
    grammar = pp.delimitedList(milforce, ";")
    toks = grammar.parseString(x)

    # A smarter grammar could probably have avoided this
    res = {}
    for _force in toks:
        k = _force['combatant']
        res[k] = [x.asDict() for x in _force['commanders']
                  if "none" not in x.asDict()]
    return res

def build_foreign_key_parser():
    arrow = pp.Literal('->').suppress()
    lbracket = pp.Literal('[').suppress()
    rbracket = pp.Literal(']').suppress()
    option = pp.Word(pp.srange('[a-zA-Z]'))
    options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket).setResultsName('options')
    ref_table = pp.restOfLine.setResultsName('ref_table')
    return arrow + options + ref_table

def Syntax():
    op = oneOf( '+ - / *')
    lpar  = Literal('(') .suppress()
    rpar  = Literal( ')' ).suppress()
    num = Word(srange('[1-9]'),nums)
    expr = Forward()
    atom = num | Group( lpar + expr + rpar )
    expr << atom + ZeroOrMore( op + expr )
    return expr

def get_highlight_expression():
    field_expression = Word(srange("[a-zA-Z0-9_.*]"))
    field_expression.setParseAction(parse_highlight_field_expression)
    fields_expression = OneOrMore(
        field_expression + Optional(',').suppress())
    fields_expression.setParseAction(parse_highlight_expression)
    highlight_expression = Word('highlight:').suppress() \
        + Word('[').suppress() \
        + fields_expression + Word(']').suppress()
    return highlight_expression

def handle_macro(toks):
    macroChar = toks[0][1]
    if macroChar == "d":
        return CharacterRangeEmitter("0123456789")
    elif macroChar == "w":
        return CharacterRangeEmitter(srange("[A-Za-z0-9_]"))
    elif macroChar == "s":
        return LiteralEmitter(" ")
    else:
        raise ParseFatalException("", 0, "unsupported macro character (" + macroChar + ")")

    def _query(self):
        ## Utility
        ident = pp.Word(pp.srange("[a-zA-Z0-9_.+-]"))

        ## Terms
        ident_term = pp.Word(pp.srange("[a-zA-Z0-9_.+-]")).setParseAction(
                make_IdentTerm)
        quoted_term = ('"' + pp.OneOrMore(ident) + '"').setParseAction(
                make_QuotedTerm)
        not_term = ('-' + (ident_term | quoted_term)).setParseAction(
                make_NotTerm)
        disj_term = pp.Forward()
        disj_term = (ident + pp.OneOrMore(pp.CaselessKeyword('or') + (disj_term | ident))).setParseAction(make_DisjTerm)
        term = quoted_term | disj_term | not_term | ident_term

        ## Selector base
        basic_selector = pp.Forward()
        selector_start = ident + ':'

        ## Selector bodies only
        selector_body = pp.Forward()
        list_selector_body = '[' + pp.delimitedList(selector_body, ',') + ']'
        source_selector_body = ident + pp.Optional('/' + ident)
        generic_selector_body = ident
        selector_body << (
                  list_selector_body
                | source_selector_body
                | generic_selector_body)

        ## Full selectors
        list_selector =  selector_start + list_selector_body
        source_selector = pp.CaselessKeyword('source') + ':' + source_selector_body
        generic_selector = selector_start + generic_selector_body
        basic_selector << (
                   list_selector.setParseAction(make_ListSelector) \
                 | source_selector.setParseAction(make_source) \
                 | generic_selector.setParseAction(make_GenericSelector))
        not_selector = ('!' + basic_selector).setParseAction(make_NotSelector)
        selector = not_selector | basic_selector
        query = pp.ZeroOrMore(selector | term).setParseAction(
                make_Query)
        return query.parseString(self.query)

def check_if_function(code):
    return_type = Word(alphanums + '_[]') # Bad style to have "_" but syntactically valid
    function_name = Word(alphanums + '_:')
    args = Word(alphanums + ',_[]&* ')
    function_open = Literal("{")
    function_close = Literal("}")
    function_declaration = Optional(srange("[a-z]")) + return_type + function_name + "(" + Optional(args) + ")"
    grammar = function_declaration + Optional(function_open)
    if len(grammar.searchString(code)):
        return True
    return False

def _build():
    """Encapsulate so the variables don't leak out."""
    # Basic punctuation
    colon = Literal(':').suppress()
    hashmark = Literal('#').suppress()
    comment = (hashmark + restOfLine).suppress()

    # Enforce Python-style naming conventions
    command_name = Word(srange("[A-Z]"), srange("[a-zA-Z0-9]"))  # StudlyCaps
    field_name = Word(srange("[a-z_]"), srange("[a-z0-9_]"))  # lower_underscore

    # Put it all together
    fields = Dict(OneOrMore(Group(field_name + colon + restOfLine)))
    fields = nestedExpr(opener="{", closer="}", content=fields)
    command = Group(command_name + fields)

    # Configure the parser
    tml_parser = OneOrMore(command) + stringEnd
    tml_parser.ignore(comment)
    return tml_parser

def compile_attribute(line, in_key=False):
    """
    Convert attribute definition from DataJoint format to SQL

    :param line: attribution line
    :param in_key: set to True if attribute is in primary key set
    :returns: (name, sql) -- attribute name and sql code for its declaration
    """
    quoted = pp.Or(pp.QuotedString('"'), pp.QuotedString("'"))
    colon = pp.Literal(':').suppress()
    attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]')).setResultsName('name')

    data_type = pp.Combine(pp.Word(pp.alphas)+pp.SkipTo("#", ignore=quoted)).setResultsName('type')
    default = pp.Literal('=').suppress() + pp.SkipTo(colon, ignore=quoted).setResultsName('default')
    comment = pp.Literal('#').suppress() + pp.restOfLine.setResultsName('comment')

    attribute_parser = attribute_name + pp.Optional(default) + colon + data_type + comment

    match = attribute_parser.parseString(line+'#', parseAll=True)
    match['comment'] = match['comment'].rstrip('#')
    if 'default' not in match:
        match['default'] = ''
    match = {k: v.strip() for k, v in match.items()}
    match['nullable'] = match['default'].lower() == 'null'

    literals = ['CURRENT_TIMESTAMP']   # not to be enclosed in quotes
    if match['nullable']:
        if in_key:
            raise DataJointError('Primary key attributes cannot be nullable in line %s' % line)
        match['default'] = 'DEFAULT NULL'  # nullable attributes default to null
    else:
        if match['default']:
            quote = match['default'].upper() not in literals and match['default'][0] not in '"\''
            match['default'] = ('NOT NULL DEFAULT ' +
                                ('"%s"' if quote else "%s") % match['default'])
        else:
            match['default'] = 'NOT NULL'
    match['comment'] = match['comment'].replace('"', '\\"')   # escape double quotes in comment
    sql = ('`{name}` {type} {default}' + (' COMMENT "{comment}"' if match['comment'] else '')).format(**match)
    return match['name'], sql

	def ImportFromString(self, text):
		#alphabet + number + underbar
		nameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
		stringDoubleG = Literal('"') + SkipTo('"') + Literal('"')
		stringSingleG = Literal("'") + SkipTo("'") + Literal("'")
		stringG = stringDoubleG | stringSingleG
		#comment
		comment = (Literal('/*').suppress() + SkipTo('*/').suppress() + Literal('*/').suppress())
		digraphNameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
		digraphNameG.setParseAction( self.DigraphNameAction )
		nodeNameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
		nodeNameG.setParseAction( self.NodeNameAction )
		startG = Literal('digraph').suppress() + digraphNameG.suppress() + Literal('{').suppress()
		endG = Literal('}').suppress()

		attrBaseG = nameG+Literal('=').suppress()+(nameG ^ stringG)
		attrBaseG.setParseAction( self.AttributeAction )
		attrRepeatG = attrBaseG + ZeroOrMore( Literal(',').suppress() + attrBaseG )
		attrG = ZeroOrMore( Literal('[').suppress() + ZeroOrMore( attrRepeatG ) + Literal(']').suppress() )
		nodeG = nodeNameG + attrG
		edgeNameG = nameG + Literal('->').suppress() + nameG
		edgeNameG.setParseAction( self.EdgeNameAction )
		edgeG = edgeNameG + attrG
		sizeG = Literal('size') + Literal('=') + (quotedString | dblQuotedString)
		baseG = (sizeG | (nodeG ^ edgeG))  + Literal(';').suppress()
		baseG = baseG | comment
		baseG = ZeroOrMore(baseG)
		grammerG = startG + baseG + endG
		grammerG.parseString(text)

def get_sort_expression():
    value_expression = Word(srange("[a-zA-Z0-9_.*]"))
    value_expression.setParseAction(lambda tokens: tokens[0])

    quoted_value_expression = Word('"').suppress() +\
        value_expression + Word('"').suppress()

    option_value = value_expression | quoted_value_expression
    option_value.setParseAction(lambda tokens: tokens[0])

    simple_option = Word(srange("[a-zA-Z0-9_.*]")) +\
        Word(':').suppress() + option_value

    simple_option.setParseAction(lambda tokens: (tokens[0], tokens[1]))

    option = Forward()
    option << (simple_option |
               (Word(srange("[a-zA-Z0-9_.*]")) +
                Word(':').suppress() +
                nestedExpr(content=option)))

    option.setParseAction(
        lambda tokens: parse_sort_field_option(tokens.asList())
    )

    exp = option + ZeroOrMore(Word(',').suppress() + option)

    field_expression = Optional('-') + Word(
        srange("[a-zA-Z0-9_.*]")
    ) + Optional(nestedExpr(content=exp))

    field_expression.setParseAction(parse_sort_field_expression)
    fields_expression = field_expression + ZeroOrMore(
        Word(',').suppress() + field_expression)
    fields_expression.setParseAction(parse_sort_expression)
    sort_expression = Word('sort:').suppress() \
        + Word('[').suppress() \
        + fields_expression + Word(']').suppress()
    return sort_expression

def parse_list_vms(stdout, stderr):
    """
    """
    id_vm_name = dblQuotedString(alphas).setResultsName('name')
    id_vm_uuid = Word(srange("[a-zA-Z0-9_\-]")).setResultsName('uuid')
    left_brace = Suppress("{")
    right_brace = Suppress("}")
    vm_group = Group(id_vm_name + left_brace + id_vm_uuid + right_brace)
    vm_list = OneOrMore(vm_group)

    token_lists = vm_list.parseString(stdout, parseAll=True)
    return [{'name': token_list.name.replace('\"', ''),
             'uuid': token_list.uuid} for token_list in token_lists]