def char_literal(char_stream, location):
    char = consume(char_stream) and consume(char_stream)  # consume initial single quote, consume char
    if char == TOKENS.SINGLE_QUOTE:  # empty char ...
        return CHAR('', location)
    if char == '\\':  # if char is being escaped
        char = escape_characters.get(peek(char_stream), consume(char_stream))
    return error_if_not_value(char_stream, TOKENS.SINGLE_QUOTE) and CHAR(char, location)

def composite_specifier(
        tokens,
        symbol_table,
        obj_type=StructType,
        member_parse_func=parse_struct_members,
        terminal=object()
):
    """
    : 'composite type' IDENTIFIER
    | 'composite type' IDENTIFIER  '{' members '}'
    | 'composite type' '{' members '}'
    """
    location = loc(consume(tokens))
    if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE:  # anonymous composite ...
        return obj_type(None, member_parse_func(tokens, symbol_table), location)

    if isinstance(peek_or_terminal(tokens), IDENTIFIER):
        obj = symbol_table.get(obj_type.get_name(peek(tokens)), obj_type(consume(tokens), None, location))
        # some composites are bit tricky such as Struct/Union ...
        # since any of its members may contain itself as a reference, so we'll add the type to
        # the symbol table before adding the members ...
        # TODO: make types immutable, right now they are being shared.
        if symbol_table.get(obj.name, terminal) is terminal:
            symbol_table[name(obj)] = obj
        if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE:
            obj.members = member_parse_func(tokens, symbol_table)

        return obj

    raise ValueError('{l} Expected IDENTIFIER or LEFT_BRACE got {got}'.format(
        l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')
    ))

def arrow_operator(tokens, symbol_table, primary_exp):
    l = loc(consume(tokens))
    _ = error_if_not_type(c_type(primary_exp), PointerType), \
        error_if_not_type(c_type(c_type(primary_exp)), (StructType, UnionType))
    member_name = error_if_not_type(consume(tokens, EOFLocation), IDENTIFIER)
    return ElementSelectionThroughPointerExpression(
        primary_exp, member_name, c_type(member(c_type(c_type(primary_exp)), member_name))(l), l
    )

def unary_operator(tokens, symbol_table):
    operator = consume(tokens)
    if operator == TOKENS.LOGICAL_AND:
        return AddressOfLabelExpression(
            error_if_not_type(consume(tokens, ''), IDENTIFIER), void_pointer_type(loc(operator)), loc(operator)
        )
    cast_exp = symbol_table['__ cast_expression __'](tokens, symbol_table)
    return rules(unary_operator)[operator](cast_exp, operator)

 def _values(char_stream):
     while peek_or_terminal(char_stream) is not terminal:
         current_value = consume(char_stream)
         yield current_value
         # we have consumed a star check if its adjacent value is a forward slash if it is consume and break
         if current_value == TOKENS.STAR and peek_or_terminal(char_stream) == TOKENS.FORWARD_SLASH:
             yield consume(char_stream)
             break

def pre_processor(char_stream, location):  # returns pre_processing symbol or #identifier ...
    values = consume(char_stream)
    if peek_or_terminal(char_stream) == TOKENS.NUMBER_SIGN:  # token concatenation symbol ...
        values += consume(char_stream)
    else:
        _ = exhaust(takewhile({' ', '\t', '\a'}.__contains__, char_stream))
        values += ''.join(takewhile(letters.__contains__, char_stream))
    return rules(pre_processor).get(values, IDENTIFIER)(values, location)

def number(char_stream, hexadecimal_chars={'x', 'X'}):
    initial_char, _digits = '', digits
    if peek_or_terminal(char_stream) == digit(0):
        initial_char = consume(char_stream)
        if peek_or_terminal(char_stream) in hexadecimal_chars:
            initial_char += consume(char_stream)
            _digits = hexadecimal_digits
    return initial_char + ''.join(takewhile(_digits.__contains__, char_stream))

def DEFINE(token_seq, macros):
    line = get_line(token_seq)
    define_token = consume(line)
    name = consume(line)
    value = consume(line, default=IGNORE())
    if value == TOKENS.LEFT_PARENTHESIS and column_number(name) + len(name) == column_number(value):
        macro = _func_macro_definition(name, line)
    else:  # object macro
        macro = ObjectMacro(name, tuple(filter_out_empty_tokens(chain((value,), line))))

    _ = name in macros and macros.pop(name) and logger.warning('{0} Redefining macro {1}'.format(loc(name), name))

    macros[name] = macro
    yield IGNORE(location=loc(define_token))

def remove_allocation(instrs):
    """
        optimize 1 or more sequence of allocations ...
        take their sum and if zero replace with the next instruction in case this one is referenced.
        other wise do one allocation and remove rest
        replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
    """
    alloc_instrs = tuple(takewhile(lambda i: isinstance(i, Allocate) and isinstance(opern(i), (int, long)), instrs))

    if not alloc_instrs:  # Operand must be non-primitive type (Address) ... must wait for its value.
        yield consume(instrs)
    else:
        total = sum(imap(opern, alloc_instrs))

        if total:  # non-zero allocates changes the state of the stack.
            if total in pop_instrs:
                new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
            elif len(alloc_instrs) != 1:
                new_instr = alloc_instrs[0]
            else:
                new_instr = Allocate(loc(alloc_instrs[-1]), total)
            yield replace_instrs(new_instr, alloc_instrs)
        else:  # stack remains unchanged, get next instruction for referencing, it one exists ...
            if peek_or_terminal(instrs) is terminal:
                yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
            else:
                replace_instrs(peek(instrs), alloc_instrs)

def _func_macro_arguments(line):
    symbol_table = SymbolTable()
    while peek(line, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS:
        if peek(line) == TOKENS.ELLIPSIS:
            arg = FunctionMacroVariadicArgument(IDENTIFIER('__VA_ARGS__', loc(consume(line))))
        else:
            arg = FunctionMacroArgument(error_if_not_type(consume(line, EOFLocation), (IDENTIFIER, KEYWORD)))
            if peek_or_terminal(line) == TOKENS.ELLIPSIS:
                arg = FunctionMacroVariadicArgument(IDENTIFIER(arg, loc(consume(line))))
        symbol_table[arg] = arg     # check for duplicate argument name
        yield arg       # if ok add to the rest ...
        if isinstance(arg, FunctionMacroVariadicArgument):  # if variadic argument break ...
            break
        # consume expected comma if we don't see a right parenthesis ...
        _ = peek(line, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS \
            and error_if_not_value(line, TOKENS.COMMA, loc(arg))

def string_literal(char_stream, location):
    def _values(char_stream):
        while peek(char_stream, TOKENS.DOUBLE_QUOTE) != TOKENS.DOUBLE_QUOTE:
            value = consume(char_stream)
            value = escape_characters.get(peek(char_stream), consume(char_stream)) if value == '\\' else value
            yield value
        _ = error_if_not_value(char_stream, TOKENS.DOUBLE_QUOTE)
    return consume(char_stream) and STRING(''.join(_values(char_stream)), location)

def function_call(tokens, symbol_table, primary_exp):
    l = loc(consume(tokens))
    func_type = error_if_not_type(c_type(c_type(primary_exp)), FunctionType)
    # get expression arguments.
    expression_argument_list = ArgumentExpressionList(tuple(get_args(tokens, symbol_table, func_type)), l)
    return error_if_not_value(tokens, TOKENS.RIGHT_PARENTHESIS) and FunctionCallExpression(
        primary_exp, expression_argument_list, c_type(func_type)(l), l
    )

def get_repositioned_line(char_seq, location):  # get next line ...
    while not isinstance(peek(char_seq), NewLineStr):
        char = consume(char_seq)
        if char == '\\' and isinstance(peek(char_seq), NewLineStr):
            _ = exhaust(takewhile(lambda token: isinstance(token, NewLineStr), char_seq))
            for char in get_repositioned_line(char_seq, location):
                yield char
        else:
            yield Str(char, location)

def argument_expression_list(tokens, symbol_table):  # : assignment_expression (',' assignment_expression)*
    assignment_expression = symbol_table['__ assignment_expression __']
    return chain(
        (assignment_expression(tokens, symbol_table),),
        starmap(
            assignment_expression,
            takewhile(lambda i: peek(i[0]) == TOKENS.COMMA and consume(i[0]), repeat((tokens, symbol_table)))
        )
    )

def parameter_type_list(tokens, symbol_table):  # : parameter_declaration (',' parameter_declaration)*
    return chain(
        (parameter_declaration(tokens, symbol_table),),
        imap(
            parameter_declaration,
            takewhile(lambda tokens: peek(tokens) == TOKENS.COMMA and consume(tokens), repeat(tokens)),
            repeat(symbol_table)
        )
    )

def merge_lines(char_seq):
    while True:
        char = consume(char_seq)
        if char == '\\' and isinstance(peek(char_seq), NewLineStr):  # if current char is \ followed by end of line seq
            _ = exhaust(takewhile(lambda token: isinstance(token, NewLineStr), char_seq))
            for char in get_repositioned_line(char_seq, loc(char)):
                yield char
        else:
            yield char

def symbol(char_stream, location):
    def _values(char_stream):
        value = ''
        while value + peek(char_stream) in TOKENS.non_keyword_symbols:
            current_value = consume(char_stream)
            value += current_value
            yield current_value
    value = ''.join(_values(char_stream))
    next_char = peek_or_terminal(char_stream)
    # if value is a single dot check if the next value is a number for possible float or ellipsis ...
    if value == TOKENS.DOT and next_char is not terminal:
        if next_char in digits:  # check for float ...
            return FLOAT(value + number(char_stream), location)
        if next_char == TOKENS.DOT:  # check for ellipsis ...
            value += consume(char_stream)
            if peek_or_terminal(char_stream) == TOKENS.DOT:
                return SYMBOL(value + consume(char_stream), location)  # TOKENS.ELLIPSIS
            raise_error('{l} Unable to tokenize: `{t}`'.format(l=location, t=TOKENS.DOT + TOKENS.DOT))
    return SYMBOL(value, location)

def error_if_not_value(value_stream, value, location=LocationNotSet):
    try:
        error_if_empty(value_stream, location)
    except ValueError as _:
        raise ValueError('{l} Expected {value} but got nothing'.format(l=location, value=value))

    curr = consume(value_stream)
    return (
        curr != value and raise_error('{l} Expected {value} but got {got}'.format(
            l=loc(curr or location), value=value, got=curr))) or curr

def parse_enum_members(tokens, symbol_table):
    constant_expression = symbol_table['__ constant_expression __']
    location, members, current_value = loc(consume(tokens)), OrderedDict(), 0

    while peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE:
        ident = error_if_not_type(consume(tokens, ''), IDENTIFIER)
        value = ConstantExpression(current_value, IntegerType(location), location)
        if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
            value = constant_expression(tokens, symbol_table)
            _ = error_if_not_type(c_type(value), IntegerType)
        current_value = error_if_not_type(exp(value), (int, long))

        symbol_table[ident] = value  # Add value to symbol_table
        members[ident] = Definition(ident, c_type(value), value, location)

        _ = peek_or_terminal(tokens) == TOKENS.COMMA and consume(tokens)
    _ = error_if_not_value(tokens, TOKENS.RIGHT_BRACE)

    return members

def init_declarator_list(tokens, symbol_table, base_type=CType(''), storage_class=None):
    return chain(   # init_declarator (',' init_declarator)*
        (init_declarator(tokens, symbol_table, base_type=base_type, storage_class=storage_class),),
        starmap(
            init_declarator,
            takewhile(
                lambda i: peek(i[0]) == TOKENS.COMMA and consume(i[0]),
                repeat((tokens, symbol_table, base_type, storage_class))
            )
        )
    )