public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            Token result;
            if (context.VsLineScanState.Value != 0)
            {
                byte level = context.VsLineScanState.TokenSubType;
                result = CompleteMatch(context, source, level);
            }
            else
            {
                //we are starting from scratch
                byte level = 0;
                if (!BeginMatch(context, source, ref level))
                    return null;

                result = CompleteMatch(context, source, level);
            }

            if (result != null)
                return result;

            if (context.Mode == ParseMode.VsLineScan)
                return CreateIncompleteToken(context, source);

            return source.CreateErrorToken("Unclosed comment block");
        }

        public override Token TryMatch(ParsingContext context, ISourceStream source) 
        {
            Token result;
            if (context.VsLineScanState.Value != 0)
            {
                byte commentLevel = context.VsLineScanState.TokenSubType;
                result = CompleteMatch(context, source, commentLevel);
            } 
            else 
            {
                byte commentLevel = 0;
                if (!BeginMatch(context, source, ref commentLevel)) 
                    return null;

                result = CompleteMatch(context, source, commentLevel);
            }
            
            if (result != null) 
                return result;

            if (context.Mode == ParseMode.VsLineScan)
                return CreateIncompleteToken(context, source);

            return context.CreateErrorToken("unclosed comment");
        }

        private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
        {
            string text = source.Text.Substring(source.PreviewPosition);
            var matches = Regex.Matches(text, @"\](=*)\]");
            foreach(Match match in matches) 
            {
                if (match.Groups[1].Value.Length == (int)level)
                {
                    source.PreviewPosition += match.Index + match.Length;


                    if (context.VsLineScanState.Value != 0)
                    {
                        SourceLocation tokenStart = new SourceLocation();
                        tokenStart.Position = 0;

                        string lexeme = source.Text.Substring(0, source.PreviewPosition);

                        context.VsLineScanState.Value = 0;
                        return new Token(this, tokenStart, lexeme, null);
                    }
                    else
                    {
                        return source.CreateToken(this.OutputTerminal); 
                    }
                }
            }

            context.VsLineScanState.TerminalIndex = this.MultilineIndex;
            context.VsLineScanState.TokenSubType = level;
            return null;
        }

 private Token CompleteMatch(ParsingContext context, ISourceStream source) {
   //Find end symbol
   while (!source.EOF()) {
     int firstCharPos;
     if (EndSymbols.Count == 1)
       firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition);
     else 
       firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
     if (firstCharPos < 0) {
       source.PreviewPosition = source.Text.Length;
       return null; //indicating error
     }
     //We found a character that might start an end symbol; let's see if it is true.
     source.PreviewPosition = firstCharPos;
     foreach (string endSymbol in EndSymbols) {
       if (source.MatchSymbol(endSymbol)) {
         //We found end symbol; eat end symbol only if it is not line comment.
         // For line comment, leave LF symbol there, it might be important to have a separate LF token
         if (!_isLineComment)
           source.PreviewPosition += endSymbol.Length;
         return source.CreateToken(this.OutputTerminal); 
       }//if
     }//foreach endSymbol
     source.PreviewPosition++; //move to the next char and try again    
   }//while
   return null; //might happen if we found a start char of end symbol, but not the full endSymbol
 }//method

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   string tokenText = string.Empty;
   while (true) {
     //Find next position
     var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
     if(newPos == -1) {
       if(IsSet(FreeTextOptions.AllowEof)) {
         source.PreviewPosition = source.Text.Length;
         return source.CreateToken(this.OutputTerminal);
       }  else
         return null;
     }
     if (newPos == source.PreviewPosition)   // DC
     {
         context.AddParserError("(DC) in TryMatch, newPos == source.PreviewPosition", new object[] {});
         break;                              // DC
     }
     tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition);
     source.PreviewPosition = newPos;
     //if it is escape, add escaped text and continue search
     if (CheckEscape(source, ref tokenText)) 
       continue;
     //check terminators
     if (CheckTerminators(source, ref tokenText))
       break; //from while (true)        
   }
   return source.CreateToken(this.OutputTerminal, tokenText);
 }

        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;
              //Check starting symbol
              if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null;
              //Find end symbol
              source.Position += StartSymbol.Length;

              while(!source.EOF()) {
            int firstCharPos;
            if (EndSymbols.Count == 1)
              firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
            else
              firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
            if (firstCharPos < 0) {
              source.Position = source.Text.Length;
              if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
            }
            //We found a character that might start an end symbol; let's see if it is true.
            source.Position = firstCharPos;
            foreach (string endSymbol in EndSymbols)
              if (source.MatchSymbol(endSymbol, ignoreCase)) {
            //We found end symbol
            source.Position += endSymbol.Length;
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              }//if
            source.Position++; //move to the next char and try again
              }//while
              return null; //never happens
        }

        private Token MatchQuoted(ParsingContext context, ISourceStream source)
        {
            char quoteChar = source.PreviewChar;
            if ((quoteChar != '\'') && (quoteChar != '"'))
            {
                return null;
            }

            source.PreviewPosition++;
            while (!source.EOF())
            {
                if (source.PreviewChar == quoteChar)
                {
                    source.PreviewPosition++;
                    return source.CreateToken(this.OutputTerminal);
                }

                // Escaped?
                if (source.PreviewChar == '\\')
                {
                    // Consume next
                    ++source.PreviewPosition;
                }

                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced quoted string");
        }

 private Token CreateIncompleteToken(ParsingContext context, ISourceStream source) {
   source.PreviewPosition = source.Text.Length;
   Token result = source.CreateToken(this.OutputTerminal);
   result.Flags |= TokenFlags.IsIncomplete;
   context.VsLineScanState.TerminalIndex = this.MultilineIndex;
   return result; 
 }

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   Match m = _expression.Match(source.Text, source.PreviewPosition);
   if (!m.Success || m.Index != source.PreviewPosition) 
     return null;
   source.PreviewPosition += m.Length;
   return source.CreateToken(this.OutputTerminal); 
 }

 private bool BeginMatch(ParsingContext context, ISourceStream source)
 {
     //Check starting symbol
     if (!source.MatchSymbol(StartSymbol)) return false;
     source.PreviewPosition += StartSymbol.Length;
     return true;
 }

    private Token CompleteMatch(ISourceStream source) {
      if (source.EOF())
        return null;

      do {
        // Match NewLine
        var lookAhead = source.PreviewChar;
        if (LineTerminators.IndexOf(lookAhead) >= 0)
        {
          source.PreviewPosition++;
          // Treat \r\n as single NewLine
          if (!source.EOF() && lookAhead == '\r' && source.PreviewChar == '\n')
            source.PreviewPosition++;
          break;
        }

        // Eat up whitespace
        if (GrammarData.Grammar.WhitespaceChars.IndexOf(lookAhead) >= 0)
        {
          source.PreviewPosition++;
          continue;
        }

        // Fail on anything else
        return null;
      }
      while (!source.EOF());

      // Create output token
      return source.CreateToken(this.OutputTerminal);
    }

    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      if (!source.MatchSymbol(_startSymbol, false)) return null;
      source.Position += _startSymbol.Length;

      while (!source.EOF())
      {
        int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

        if (firstCharPos < 0)
        {
          source.Position = source.Text.Length;

          if (_isLineComment)
            return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
          else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
        }
      
        source.Position = firstCharPos;
        if (source.MatchSymbol(_endSymbol, false))
        {
          source.Position += _endSymbol.Length;
          return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
        }
        
        source.Position++; 
      }

      throw new NotSupportedException();
    }

 private static string ReadQuotedBody(ParsingContext context, ISourceStream source)
 {
     const char dQuoute = '"';
       StringBuilder sb = null;
       var from = source.Location.Position + 1; //skip initial double quote
       while(true) {
     var until  = source.Text.IndexOf(dQuoute, from);
     if (until < 0)
       throw new Exception(Resources.ErrDsvNoClosingQuote); // "Could not find a closing quote for quoted value."
     source.PreviewPosition = until; //now points at double-quote
     var piece = source.Text.Substring(from, until - from);
     source.PreviewPosition++; //move after double quote
     if (source.PreviewChar != dQuoute && sb == null)
       return piece; //quick path - if sb (string builder) was not created yet, we are looking at the very first segment;
                 // and if we found a standalone dquote, then we are done - the "piece" is the result.
     if (sb == null)
       sb = new StringBuilder(100);
     sb.Append(piece);
     if (source.PreviewChar != dQuoute)
       return sb.ToString();
     //we have doubled double-quote; add a single double-quoute char to the result and move over both symbols
     sb.Append(dQuoute);
     from = source.PreviewPosition + 1;
       }
 }

 public override Token TryMatch(CompilerContext context, ISourceStream source) {
   Match m = _expression.Match(source.Text, source.Position);
   if (!m.Success || m.Index != source.Position) 
     return null;
   source.Position += m.Length;
   string text = source.GetLexeme();
   return new Token(this, source.TokenStart, text, null);
 }

 public void Prepare(CompilerContext context, ISourceStream source)
 {
   _context = context;
   _source = source;
   _currentToken = null;
   _bufferedTokens.Clear();
   ResetSource();
 }

 protected override string ReadBody(ParsingContext context, ISourceStream source) {
   if (!source.MatchSymbol(StartSymbol)) return null; //this will result in null returned from TryMatch, no token
   var start = source.Location.Position + StartSymbol.Length;
   var end = source.Text.IndexOf(EndSymbol, start);
   if (end < 0) return null;
   var body = source.Text.Substring(start, end - start);
   source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
   return body; 
 }

 public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
 {
   Match result = Expression.Match(source.Text, source.Position);
   if (!result.Success)
     return null;
   source.Position += result.Length;
   
   return CreateToken(context, source);
 }

 public override Token TryMatch(CompilerContext context, ISourceStream source)
 {
     Match m = _expression.Match(source.Text, source.Position);
       if (!m.Success)
     return null;
       source.Position += m.Length + 1;
       string text = source.GetLexeme();
       return Token.Create(this, context, source.TokenStart, text);
 }

    protected override TokenAst CreateToken(CompilerContext context, ISourceStream source)
    {
      TokenAst token = base.CreateToken(context, source);

      if (Keywords.Contains(token.Text))
        token.IsKeyword = true;

      return token;
    }

    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      bool isVerbatim = false;
      int start = source.Position;

      if (source.CurrentChar == '@')
      {
        isVerbatim = true;
        source.Position++;
        start++;
      }

      if (IsCurrentQuote(source))
      {
        source.Position++;
        start++;
      }
      else 
        return null;

      while (!source.EOF())
      {
        if (!isVerbatim)
        {
          if (source.CurrentChar == '\\')
          {
            //TODO: Escape processing
            source.Position += 2;
            continue;
          }
          else
            //Single line string ends incorrectly
            if (ParserData.LineTerminators.IndexOf(source.CurrentChar) >= 0)
              return null;
        }

        if (IsCurrentQuote(source)) break;
        
        source.Position++;
      }

      if (IsCurrentQuote(source))
        source.Position++;
      else
        return null;
      
      string lexeme = source.GetLexeme();
      string body = source.Text.Substring(start, source.Position - start - 1);
      //TODO: handle this in escape processing
      if (!isVerbatim)
        body = body.Replace("\\'", "'").Replace("\\\"", "\"").Replace("\\\\", "\\");

      TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, body);     
      return token;

      //return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value");
    }