public static AST.Node LoadFile(string fileName, LexerMode mode, FunctionInformation functionInfo)
 {
     using (StreamReader file = new StreamReader(fileName, Parse.Latin1))
     {
         return Parse.String(file.ReadToEnd(), mode, functionInfo);
     }
 }

 internal ResetPoint(int resetCount, LexerMode mode, int position, CSharpSyntaxNode prevTokenTrailingTrivia)
 {
     this.ResetCount = resetCount;
     this.Mode = mode;
     this.Position = position;
     this.PrevTokenTrailingTrivia = prevTokenTrailingTrivia;
 }

 /// <summary>
 /// 重置分析器
 /// </summary>
 public void Reset()
 {
     this.mode = LexerMode.None;
     this.line = 1;
     this.column = 1;
     this.kind = TokenKind.Text;
     this.startColumn = 1;
     this.startLine = 1;
     this.scanner = new CharScanner(this.document);
     this.collection = new List<Token>();
     this.pos = new Stack<String>();
 }

        public void Next(LexerMode mode)
        {
            switch (mode)
            {
                case LexerMode.BLOCK:
                    this.NextBlock ();

                    break;

                case LexerMode.RAW:
                    this.NextRaw ();

                    break;

                default:
                    throw new UnknownException (this, "invalid lexem");
            }
        }

        public void Next(LexerMode mode)
        {
            switch (mode)
            {
                case LexerMode.Block:
                    this.current = this.NextBlock ();

                    break;

                case LexerMode.Raw:
                    this.current = this.NextRaw ();

                    break;

                default:
                    throw new ParseException (this.column, this.line, "<?>", "block or raw text");
            }
        }

        public static AST.Node String(string input, LexerMode mode, FunctionInformation functionInfo)
        {
            switch (mode)
            {
                case LexerMode.ASCII:
                    return ASCIIString(input, functionInfo);

                case LexerMode.APL:
                    return APLString(input, functionInfo);

                case LexerMode.UNI:
                    return UNIString(input, functionInfo);

                default:
                    break;
            }

            throw new ParseException("Invalid Parse Mode");
        }

        /// <summary>
        /// 分析所有Token
        /// </summary>
        /// <returns></returns>
        public Token[] Parse()
        {
            if (this.kind != TokenKind.EOF)
            {
                do
                {
                    if (this.mode == LexerMode.EnterLabel)
                    {
                        Next(this.pos.Peek().Length - 1);
                        AddToken(GetToken(GetTokenKind(this.scanner.Read())));
                        switch (this.kind)
                        {
                            case TokenKind.StringStart:
                                this.pos.Push("\"");
                                break;
                            case TokenKind.LeftParentheses:
                                this.pos.Push("(");
                                break;
                        }
                        ReadToken();
                    }
                    else if (IsTagStart())
                    {
                        AddToken(GetToken(TokenKind.TagStart));
                        this.mode = LexerMode.EnterLabel;

                    }
                    else if (this.scanner.Read() == '\n')
                    {
                        this.line++;
                        this.column = 1;
                    }
                }
                while (Next());

                AddToken(GetToken(TokenKind.EOF));


                if (this.mode == LexerMode.EnterLabel)
                {
                    this.mode = LexerMode.LeaveLabel;
                    AddToken(new Token(TokenKind.TagEnd, String.Empty));
                }

            }

            return this.collection.ToArray();

        }

        private SyntaxNode LexSingleDirective(
            bool isActive,
            bool endIsActive,
            bool afterNonWhitespaceOnLine,
            List<SyntaxNode> triviaList)
        {
            _start = _charReader.Position;

            if (char.IsWhiteSpace(_charReader.Current))
            {
                ReadWhitespace();
                AddTrivia(triviaList, SyntaxKind.WhitespaceTrivia);
            }

            var saveMode = _mode;
            var saveExpandMacros = ExpandMacros;

            _mode = LexerMode.Directive;
            ExpandMacros = false;

            var dp = new DirectiveParser(this, _directives);
            var directive = dp.ParseDirective(isActive, endIsActive, afterNonWhitespaceOnLine);

            if (!isActive || directive.Kind != SyntaxKind.IncludeDirectiveTrivia)
                triviaList.Add(directive);

            _directives = directive.ApplyDirectives(_directives);
            ExpandMacros = saveExpandMacros;
            _mode = saveMode;

            // Directive parser sometimes leaves charReader at start of token *after* the one we want.
            _charReader.Reset(directive.GetLastToken().GetLastSpanIncludingTrivia().End);
            _start = _charReader.Position;

            return directive;
        }

 private BlendedNode ReadNodeOrToken(LexerMode mode, bool asToken)
 {
     var reader = new Reader(this);
     return reader.ReadNodeOrToken(mode, asToken);
 }

 public BlendedNode ReadToken(LexerMode mode)
 {
     return ReadNodeOrToken(mode, asToken: true);
 }

 public BlendedNode ReadNode(LexerMode mode)
 {
     return ReadNodeOrToken(mode, asToken: false);
 }

            internal BlendedNode ReadNodeOrToken(LexerMode mode, bool asToken)
            {
                // This is the core driver of the blender.  It just sits in a loop trying to keep our
                // positions in the old and new text in sync.  When they're out of sync it will try
                // to match them back up, and it will appropriately determine which nodes or tokens
                // from the old tree can be reused as long as they don't overlap and changes or
                // contain any errors.

                while (true)
                {
                    // If the cursor in the old tree is finished, then our choice is easy.  We just
                    // read from the new text.
                    if (this.oldTreeCursor.IsFinished)
                    {
                        return this.ReadNewToken(mode);
                    }

                    // If delta is non-zero then that means our positions in the respective text
                    // streams are not in sync.  This can be because of to reasons.  Either:
                    //
                    //   a) we're further ahead in the new text (i.e. 'changeDelta' is negative). We
                    //      should keep skipping tokens in the old text until we catch up.
                    //      TODO(cyrusn): We could actually be smarter here and skip whole nodes if
                    //      they're shorter than the changeDelta.  We can try doing that in the future.
                    //
                    //   b) we're further ahead in the old text (i.e. 'changeDelta' is positive).
                    //      This can happen when we are skipping over portions of the old tree because
                    //      it overlapped with changed text spans. In this case, we want to read a
                    //      token to try to consume that changed text and ensure that we get synced up.
                    if (this.changeDelta < 0)
                    {
                        // Case '1' above.  We're behind in the old text, so move forward a token.
                        // And try again.
                        this.SkipOldToken();
                    }
                    else if (this.changeDelta > 0)
                    {
                        // Case '2' above.  We're behind in the new text, so read a token to try to
                        // catch up.
                        return this.ReadNewToken(mode);
                    }
                    else
                    {
                        // Attempt to take a node or token from the old tree.  If we can't, then
                        // either break down the current node we're looking at to its first child
                        // and try again, or move to the next token.
                        BlendedNode blendedNode;
                        if (this.TryTakeOldNodeOrToken(asToken, out blendedNode))
                        {
                            return blendedNode;
                        }

                        // Couldn't take the current node or token.  Figure out the next node or
                        // token to reconsider and try again.
                        if (this.oldTreeCursor.CurrentNodeOrToken.IsNode)
                        {
                            // It was a node.  Just move to its first token and try again.
                            this.oldTreeCursor = this.oldTreeCursor.MoveToFirstChild();
                        }
                        else
                        {
                            // It was a token, just move to the next token.
                            this.SkipOldToken();
                        }
                    }
                }
            }

            private SyntaxToken LexNewToken(LexerMode mode)
            {
                if (this.lexer.TextWindow.Position != this.newPosition)
                {
                    this.lexer.Reset(this.newPosition, this.newDirectives);
                }

                if (mode >= LexerMode.XmlDocComment)
                {
                    mode |= this.newLexerDrivenMode;
                }

                var token = this.lexer.Lex(ref mode);
                this.newDirectives = this.lexer.Directives;
                this.newLexerDrivenMode = mode & (LexerMode.MaskXmlDocCommentLocation | LexerMode.MaskXmlDocCommentStyle);
                return token;
            }

            private BlendedNode ReadNewToken(LexerMode mode)
            {
                Debug.Assert(this.changeDelta > 0 || this.oldTreeCursor.IsFinished);

                // The new text is either behind the cursor, or the cursor is done.  In either event,
                // we need to lex a real token from the stream.
                var token = this.LexNewToken(mode);

                // If the oldTreeCursor was finished, then the below code isn't really necessary.
                // We'll just repeat the outer reader loop and call right back into ReadNewToken.
                // That will then call LexNewToken (which doesn't use either of these variables).  If
                // oldTreeCursor wasn't finished then we need to update our state based on the token
                // we just read.
                var width = token.FullWidth;
                this.newPosition += width;
                this.changeDelta -= width;

                // By reading a token we may either have read into, or past, change ranges.  Skip
                // past them.  This will increase changeDelta which will indicate to us that we need
                // to keep on lexing.
                this.SkipPastChanges();

                return this.CreateBlendedNode(node: null, token: token);
            }

        public SyntaxToken Lex(LexerMode mode)
        {
#if DEBUG
            TokensLexed++;
#endif
            this.mode = mode;
            switch (this.mode)
            {
                case LexerMode.Syntax:
                case LexerMode.DebuggerSyntax:
#if true
                    var result = this.QuickScanSyntaxToken();
                    if (result == null)
                    {
                        result = this.LexSyntaxToken();
                    }

                    return result;
#else
                    return this.LexSyntaxToken();
#endif
                case LexerMode.Directive:
                    return this.LexDirectiveToken();
            }

            switch (ModeOf(this.mode))
            {
                case LexerMode.XmlDocComment:
                    return this.LexXmlToken();
                case LexerMode.XmlElementTag:
                    return this.LexXmlElementTagToken();
                case LexerMode.XmlAttributeTextQuote:
                case LexerMode.XmlAttributeTextDoubleQuote:
                    return this.LexXmlAttributeTextToken();
                case LexerMode.XmlCDataSectionText:
                    return this.LexXmlCDataSectionTextToken();
                case LexerMode.XmlCommentText:
                    return this.LexXmlCommentTextToken();
                case LexerMode.XmlProcessingInstructionText:
                    return this.LexXmlProcessingInstructionTextToken();
                case LexerMode.XmlCrefQuote:
                case LexerMode.XmlCrefDoubleQuote:
                    return this.LexXmlCrefOrNameToken();
                case LexerMode.XmlNameQuote:
                case LexerMode.XmlNameDoubleQuote:
                    // Same lexing as a cref attribute, just treat the identifiers a little differently.
                    return this.LexXmlCrefOrNameToken();
                case LexerMode.XmlCharacter:
                    return this.LexXmlCharacter();
            }

            Debug.Assert(false, "Unknown LexMode passed to Lexer.Lex");
            return this.LexSyntaxToken();
        }

        public SyntaxToken Lex(LexerMode mode)
        {
            // First check if we're in the middle of expanding a macro reference token.
            if (_expandedMacroTokens != null)
            {
                var result = _expandedMacroTokens[_expandedMacroIndex++];
                if (_expandedMacroIndex == _expandedMacroTokens.Count)
                    _expandedMacroTokens = null;
                return result;
            }

            _mode = mode;

            SyntaxToken token;
            switch (_mode)
            {
                case LexerMode.Syntax:
                    token = LexSyntaxToken();
                    break;
                case LexerMode.Directive:
                    token = LexDirectiveToken();
                    break;
                default:
                    throw new ArgumentOutOfRangeException();
            }

            // Swallow end-of-file tokens from include files.
            if (token.Kind == SyntaxKind.EndOfFileToken && _includeStack.Count > 1)
            {
                var originalToken = token;

                PopIncludeContext();
                token = Lex(mode);
                token = token.WithLeadingTrivia(originalToken.LeadingTrivia.AddRange(token.LeadingTrivia));

                // this is a bit weird, but we need to also update the leading trivia on the macro reference,
                // because that's what we use when outputting code.
                if (token.MacroReference != null)
                    token = token.WithOriginalMacroReference(token.MacroReference.WithLeadingTrivia(token.LeadingTrivia), token.IsFirstTokenInMacroExpansion);
            }

            // Expand macros and attach as a special kind of trivia.
            if (token.Kind == SyntaxKind.IdentifierToken && ExpandMacros)
            {
                List<SyntaxToken> expandedTokens;
                if (TryExpandMacro(token, new BaseMacroExpansionLexer(this), out expandedTokens))
                {
                    if (expandedTokens.Count == 0) // Can happen for macros with empty body.
                    {
                        // Attach macro call as leading trivia on next token.
                        var originalToken = token;
                        token = Lex(mode);

                        var leadingTrivia = new List<SyntaxNode>();
                        leadingTrivia.AddRange(originalToken.LeadingTrivia);
                        leadingTrivia.Add(new SyntaxTrivia(SyntaxKind.EmptyExpandedMacroTrivia, originalToken.Text, originalToken.SourceRange, originalToken.Span, ImmutableArray<Diagnostic>.Empty));
                        leadingTrivia.AddRange(originalToken.TrailingTrivia);
                        leadingTrivia.AddRange(token.LeadingTrivia);
                        token = token.WithLeadingTrivia(leadingTrivia.ToImmutableArray());
                    }
                    else
                    {
                        if (expandedTokens.Count > 1)
                        {
                            _expandedMacroTokens = expandedTokens;
                            _expandedMacroIndex = 1;
                        }
                        token = expandedTokens[0];
                    }
                }
            }

            return token;
        }

        private void ReadToken()
        {
            while (Next())
            {
                if (this.scanner.Read() == '"')
                {
                    if (this.pos.Count > 1 && this.pos.Peek() == "\"")
                    {
                        if (this.kind == TokenKind.StringStart)
                        {
                            AddToken(GetToken(TokenKind.String));
                        }
                        AddToken(GetToken(TokenKind.StringEnd));
                        this.pos.Pop();
                        continue;
                    }

                    if (this.kind == TokenKind.TagStart
                        || this.kind == TokenKind.LeftBracket
                        || this.kind == TokenKind.LeftParentheses
                        || this.kind == TokenKind.Operator
                        || this.kind == TokenKind.Punctuation
                        || this.kind == TokenKind.Comma
                        || this.kind == TokenKind.Space)
                    {
                        AddToken(GetToken(TokenKind.StringStart));
                        this.pos.Push("\"");
                        continue;
                    }
                }

                if (this.kind == TokenKind.StringStart)
                {
                    AddToken(GetToken(TokenKind.String));
                    continue;
                }

                if (this.kind == TokenKind.String)
                {
                    continue;
                }

                if (this.scanner.Read() == '(')
                {
                    this.pos.Push("(");
                }
                else if (this.scanner.Read() == ')' && this.pos.Peek() == "(")// && this.pos.Count > 2
                {
                    this.pos.Pop();
                    if (this.pos.Count == 1)
                    {

                    }
                }
                else if (IsTagEnd())
                {
                    //Next(1);
                    //this.pos.Pop();
                    AddToken(GetToken(TokenKind.TagEnd));
                    this.mode = LexerMode.LeaveLabel;
                    if (this.pos.Pop().Length == 2)
                    {
                        Next(1);
                    }
                    if (IsTagStart())
                    {
                        AddToken(GetToken(TokenKind.TagStart));
                        this.mode = LexerMode.EnterLabel;
                    }
                    else
                    {
                        AddToken(GetToken(TokenKind.Text));
                    }
                    break;
                }
                TokenKind tk;
                if (this.scanner.Read() == '+' || this.scanner.Read() == '-') //正负数符号识别
                {
                    if (Char.IsNumber(this.scanner.Read(1)) && 
                        (this.kind == TokenKind.Operator || this.kind == TokenKind.LeftParentheses))
                    {
                        tk = TokenKind.Number;
                    }
                    else
                    {
                        tk = TokenKind.Operator;
                    }
                }
                else
                {
                    tk = GetTokenKind(this.scanner.Read());
                }
                //if (this.kind == tk || (tk == TokenKind.Number && this.kind == TokenKind.TextData))
                if ((this.kind != tk || this.kind == TokenKind.LeftParentheses || this.kind == TokenKind.RightParentheses)
                    && (tk != TokenKind.Number || this.kind != TokenKind.TextData)
                    //&& (this.kind == TokenKind.Number && tk != TokenKind.Dot)
                    )
                //|| (this.kind != TokenKind.Number && tk == TokenKind.Dot)
                {
                    if (tk == TokenKind.Dot && this.kind == TokenKind.Number)
//.........这里部分代码省略.........

 public Reader(Blender blender)
 {
     this.lexer = blender.lexer;
     this.oldTreeCursor = blender.oldTreeCursor;
     this.changes = blender.changes;
     this.newPosition = blender.newPosition;
     this.changeDelta = blender.changeDelta;
     this.newDirectives = blender.newDirectives;
     this.oldDirectives = blender.oldDirectives;
     this.newLexerDrivenMode = blender.newLexerDrivenMode;
 }

 public Reader(Blender blender)
 {
     _lexer = blender._lexer;
     _oldTreeCursor = blender._oldTreeCursor;
     _changes = blender._changes;
     _newPosition = blender._newPosition;
     _changeDelta = blender._changeDelta;
     _newDirectives = blender._newDirectives;
     _oldDirectives = blender._oldDirectives;
     _newLexerDrivenMode = blender._newLexerDrivenMode;
 }

        public Blender(Lexer lexer, CSharp.CSharpSyntaxNode oldTree, IEnumerable<TextChangeRange> changes)
        {
            Debug.Assert(lexer != null);
            _lexer = lexer;
            _changes = ImmutableStack.Create<TextChangeRange>();

            if (changes != null)
            {
                // TODO: Consider implementing NormalizedChangeCollection for TextSpan. the real
                // reason why we are collapsing is because we want to extend change ranges and
                // cannot allow them to overlap. This does not seem to be a big deal since multiple
                // changes are infrequent and typically close to each other. However if we have
                // NormalizedChangeCollection for TextSpan we can have both - we can extend ranges
                // and not require collapsing them. NormalizedChangeCollection would also ensure
                // that changes are always normalized.

                // TODO: this is a temporary measure to prevent individual change spans from
                // overlapping after they are widened to effective width (+1 token at the start).
                // once we have normalized collection for TextSpan we will not need to collapse all
                // the change spans.

                var collapsed = TextChangeRange.Collapse(changes);

                // extend the change to its affected range. This will make it easier 
                // to filter out affected nodes since we will be able simply check 
                // if node intersects with a change.
                var affectedRange = ExtendToAffectedRange(oldTree, collapsed);
                _changes = _changes.Push(affectedRange);
            }

            if (oldTree == null)
            {
                // start at lexer current position if no nodes specified
                _oldTreeCursor = new Cursor();
                _newPosition = lexer.TextWindow.Position;
            }
            else
            {
                _oldTreeCursor = Cursor.FromRoot(oldTree).MoveToFirstChild();
                _newPosition = 0;
            }

            _changeDelta = 0;
            _newDirectives = default(DirectiveStack);
            _oldDirectives = default(DirectiveStack);
            _newLexerDrivenMode = 0;
        }