def test_identifiers(self):
        s = 'select foo.bar from "myscheme"."table" where fail. order'
        parsed = sqlparse.parse(s)[0]
        self.ndiffAssertEqual(s, u(parsed))
        self.assert_(isinstance(parsed.tokens[2], sql.Identifier))
        self.assert_(isinstance(parsed.tokens[6], sql.Identifier))
        self.assert_(isinstance(parsed.tokens[8], sql.Where))
        s = 'select * from foo where foo.id = 1'
        parsed = sqlparse.parse(s)[0]
        self.ndiffAssertEqual(s, u(parsed))
        self.assert_(isinstance(parsed.tokens[-1].tokens[-1].tokens[0],
                                sql.Identifier))
        s = 'select * from (select "foo"."id" from foo)'
        parsed = sqlparse.parse(s)[0]
        self.ndiffAssertEqual(s, u(parsed))
        self.assert_(isinstance(parsed.tokens[-1].tokens[3], sql.Identifier))

        s = "INSERT INTO `test` VALUES('foo', 'bar');"
        parsed = sqlparse.parse(s)[0]
        types = [l.ttype for l in parsed.tokens if not l.is_whitespace()]
        self.assertEquals(types, [T.DML, T.Keyword, None,
                                  T.Keyword, None, T.Punctuation])

        s = "select 1.0*(a+b) as col, sum(c)/sum(d) from myschema.mytable"
        parsed = sqlparse.parse(s)[0]
        self.assertEqual(len(parsed.tokens), 7)
        self.assert_(isinstance(parsed.tokens[2], sql.IdentifierList))
        self.assertEqual(len(parsed.tokens[2].tokens), 4)
        identifiers = list(parsed.tokens[2].get_identifiers())
        self.assertEqual(len(identifiers), 2)
        self.assertEquals(identifiers[0].get_alias(), u"col")

    def get_tokens(text, encoding=None):
        """
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.

        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        if isinstance(text, string_types):
            text = u(text, encoding)
        elif isinstance(text, file_types):
            text = u(text.read(), encoding)

        iterable = enumerate(text)
        for pos, char in iterable:
            for rexmatch, action in SQL_REGEX:
                m = rexmatch(text, pos)

                if not m:
                    continue
                elif isinstance(action, tokens._TokenType):
                    yield action, m.group()
                elif callable(action):
                    yield action(m.group())

                consume(iterable, m.end() - pos - 1)
                break
            else:
                yield tokens.Error, char

 def test_comment_end_of_line(self):
     sql = ('select * from foo; -- foo\n'
            'select * from bar;')
     stmts = sqlparse.parse(sql)
     self.assertEqual(len(stmts), 2)
     self.ndiffAssertEqual(''.join(u(q) for q in stmts), sql)
     # make sure the comment belongs to first query
     self.ndiffAssertEqual(u(stmts[0]), 'select * from foo; -- foo\n')

 def test_where(self):
     s = 'select * from foo where bar = 1 order by id desc'
     p = sqlparse.parse(s)[0]
     self.ndiffAssertEqual(s, u(p))
     self.assertTrue(len(p.tokens), 16)
     s = 'select x from (select y from foo where bar = 1) z'
     p = sqlparse.parse(s)[0]
     self.ndiffAssertEqual(s, u(p))
     self.assertTrue(isinstance(p.tokens[-1].tokens[0].tokens[-2],
                     sql.Where))

 def test_typecast(self):
     s = 'select foo::integer from bar'
     p = sqlparse.parse(s)[0]
     self.ndiffAssertEqual(s, u(p))
     self.assertEqual(p.tokens[2].get_typecast(), 'integer')
     self.assertEqual(p.tokens[2].get_name(), 'foo')
     s = 'select (current_database())::information_schema.sql_identifier'
     p = sqlparse.parse(s)[0]
     self.ndiffAssertEqual(s, u(p))
     self.assertEqual(p.tokens[2].get_typecast(),
                      'information_schema.sql_identifier')

 def test_newlines(self):
     sql = u'select\n*from foo;'
     p = sqlparse.parse(sql)[0]
     self.assertEqual(u(p), sql)
     sql = u'select\r\n*from foo'
     p = sqlparse.parse(sql)[0]
     self.assertEqual(u(p), sql)
     sql = u'select\r*from foo'
     p = sqlparse.parse(sql)[0]
     self.assertEqual(u(p), sql)
     sql = u'select\r\n*from foo\n'
     p = sqlparse.parse(sql)[0]
     self.assertEqual(u(p), sql)

    def ndiffAssertEqual(self, first, second):
        """Like failUnlessEqual except use ndiff for readable output."""
        if first != second:
            # Using the built-in .splitlines() method here will cause incorrect
            # results when splitting statements that have quoted CR/CR+LF
            # characters.
            sfirst = split_unquoted_newlines(u(first))
            ssecond = split_unquoted_newlines(u(second))
            diff = difflib.ndiff(sfirst, ssecond)

            fp = StringIO()
            fp.write('\n')
            fp.write('\n'.join(diff))

            raise self.failureException(fp.getvalue())

def Tokens2Unicode(stream):
    result = ""

    for _, value in stream:
        result += u(value)

    return result

    def _split_kwds(self, tlist):
        split_words = ('FROM', 'STRAIGHT_JOIN$', 'JOIN$', 'AND', 'OR',
                       'GROUP', 'ORDER', 'UNION', 'VALUES',
                       'SET', 'BETWEEN', 'EXCEPT', 'HAVING')

        def _next_token(i):
            t = tlist.token_next_match(i, T.Keyword, split_words,
                                       regex=True)
            if t and t.value.upper() == 'BETWEEN':
                t = _next_token(tlist.token_index(t) + 1)
                if t and t.value.upper() == 'AND':
                    t = _next_token(tlist.token_index(t) + 1)
            return t

        idx = 0
        token = _next_token(idx)
        added = set()
        while token:
            prev = tlist.token_prev(tlist.token_index(token), False)
            offset = 1
            if prev and prev.is_whitespace() and prev not in added:
                tlist.tokens.pop(tlist.token_index(prev))
                offset += 1
            uprev = u(prev)
            if (prev and (uprev.endswith('\n') or uprev.endswith('\r'))):
                nl = tlist.token_next(token)
            else:
                nl = self.nl()
                added.add(nl)
                tlist.insert_before(token, nl)
                offset += 1
            token = _next_token(tlist.token_index(nl) + offset)

 def test_comment_with_umlaut(self):
     sql = (u'select * from foo;\n'
            u'-- Testing an umlaut: ä\n'
            u'select * from bar;')
     stmts = sqlparse.parse(sql)
     self.assertEqual(len(stmts), 2)
     self.ndiffAssertEqual(''.join(u(q) for q in stmts), sql)

 def get_typecast(self):
     """Returns the typecast or ``None`` of this object as a string."""
     marker = self.token_next_match(0, T.Punctuation, '::')
     if marker is None:
         return None
     next_ = self.token_next(self.token_index(marker), False)
     if next_ is None:
         return None
     return u(next_)

def split(sql, encoding=None):
    """Split *sql* into single statements.

    :param sql: A string containing one or more SQL statements.
    :param encoding: The encoding of the statement (optional).
    :returns: A list of strings.
    """
    stack = engine.FilterStack()
    return [u(stmt).strip() for stmt in stack.run(sql, encoding)]

    def process(self, stack, stmt):
        self.count += 1
        if self.count > 1:
            varname = '%s%d' % (self.varname, self.count)
        else:
            varname = self.varname

        has_nl = len(u(stmt).strip().splitlines()) > 1
        stmt.tokens = self._process(stmt.tokens, varname, has_nl)
        return stmt

 def test_alias(self):
     s = 'select foo as bar from mytable'
     p = sqlparse.parse(s)[0]
     self.ndiffAssertEqual(s, u(p))
     self.assertEqual(p.tokens[2].get_real_name(), 'foo')
     self.assertEqual(p.tokens[2].get_alias(), 'bar')
     s = 'select foo from mytable t1'
     p = sqlparse.parse(s)[0]
     self.ndiffAssertEqual(s, u(p))
     self.assertEqual(p.tokens[6].get_real_name(), 'mytable')
     self.assertEqual(p.tokens[6].get_alias(), 't1')
     s = 'select foo::integer as bar from mytable'
     p = sqlparse.parse(s)[0]
     self.ndiffAssertEqual(s, u(p))
     self.assertEqual(p.tokens[2].get_alias(), 'bar')
     s = ('SELECT DISTINCT '
          '(current_database())::information_schema.sql_identifier AS view')
     p = sqlparse.parse(s)[0]
     self.ndiffAssertEqual(s, u(p))
     self.assertEqual(p.tokens[4].get_alias(), 'view')

 def process(self, stack, stmt):
     if isinstance(stmt, sql.Statement):
         self._curr_stmt = stmt
     self._process(stmt)
     if isinstance(stmt, sql.Statement):
         if self._last_stmt is not None:
             if u(self._last_stmt).endswith("\n"):
                 nl = "\n"
             else:
                 nl = "\n\n"
             stmt.tokens.insert(0, sql.Token(T.Whitespace, nl))
         if self._last_stmt != stmt:
             self._last_stmt = stmt

    def _split_kwds(self, tlist):
        split_words = (
            "FROM",
            "STRAIGHT_JOIN$",
            "JOIN$",
            "AND",
            "OR",
            "GROUP",
            "ORDER",
            "UNION",
            "VALUES",
            "SET",
            "BETWEEN",
            "EXCEPT",
            "HAVING",
        )

        def _next_token(i):
            t = tlist.token_next_match(i, T.Keyword, split_words, regex=True)
            if t and t.value.upper() == "BETWEEN":
                t = _next_token(tlist.token_index(t) + 1)
                if t and t.value.upper() == "AND":
                    t = _next_token(tlist.token_index(t) + 1)
            return t

        idx = 0
        token = _next_token(idx)
        added = set()
        while token:
            prev = tlist.token_prev(tlist.token_index(token), False)
            offset = 1
            if prev and prev.is_whitespace() and prev not in added:
                tlist.tokens.pop(tlist.token_index(prev))
                offset += 1
            uprev = u(prev)
            if prev and (uprev.endswith("\n") or uprev.endswith("\r")):
                nl = tlist.token_next(token)
            else:
                nl = self.nl()
                added.add(nl)
                tlist.insert_before(token, nl)
                offset += 1
            token = _next_token(tlist.token_index(nl) + offset)

 def _process(self, stack, group, stream):
     for token in stream:
         if token.is_whitespace() and "\n" in token.value:
             if token.value.endswith("\n"):
                 self.line = ""
             else:
                 self.line = token.value.splitlines()[-1]
         elif token.is_group() and token.__class__ not in self.keep_together:
             token.tokens = self._process(stack, token, token.tokens)
         else:
             val = u(token)
             if len(self.line) + len(val) > self.width:
                 match = re.search("^ +", self.line)
                 if match is not None:
                     indent = match.group()
                 else:
                     indent = ""
                 yield sql.Token(T.Whitespace, "\n%s" % indent)
                 self.line = indent
             self.line += val
         yield token

 def _get_repr_value(self):
     raw = u(self)
     if len(raw) > 7:
         raw = raw[:6] + u'...'
     return re.sub('\s+', ' ', raw)

 def test_dropif(self):
     sql = 'DROP TABLE IF EXISTS FOO;\n\nSELECT * FROM BAR;'
     stmts = sqlparse.parse(sql)
     self.assertEqual(len(stmts), 2)
     self.ndiffAssertEqual(''.join(u(q) for q in stmts), sql)

 def test_identifier_list_with_order(self):  # issue101
     p = sqlparse.parse('1, 2 desc, 3')[0]
     self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
     self.assert_(isinstance(p.tokens[0].tokens[3], sql.Identifier))
     self.ndiffAssertEqual(u(p.tokens[0].tokens[3]), '2 desc')