• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    迪恩网络公众号

C++ TokenStream类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了C++中TokenStream的典型用法代码示例。如果您正苦于以下问题:C++ TokenStream类的具体用法?C++ TokenStream怎么用?C++ TokenStream使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。



在下文中一共展示了TokenStream类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的C++代码示例。

示例1: Parse_Type_ErasedType

TypeRef Parse_Type_ErasedType(TokenStream& lex, bool allow_trait_list)
{
    Token   tok;

    auto ps = lex.start_span();
    ::std::vector<Type_TraitPath>   traits;
    ::std::vector<AST::LifetimeRef>   lifetimes;
    do {
        if( LOOK_AHEAD(lex) == TOK_LIFETIME ) {
            GET_TOK(tok, lex);
            lifetimes.push_back(AST::LifetimeRef( /*lex.point_span(),*/ lex.get_ident(mv$(tok)) ));
        }
        else
        {
            AST::HigherRankedBounds hrbs;
            if( lex.lookahead(0) == TOK_RWORD_FOR )
            {
                hrbs = Parse_HRB(lex);
            }
            traits.push_back({ mv$(hrbs), Parse_Path(lex, PATH_GENERIC_TYPE) });
        }
    } while( GET_TOK(tok, lex) == TOK_PLUS );
    PUTBACK(tok, lex);

    return TypeRef(lex.end_span(mv$(ps)), TypeData::make_ErasedType({ mv$(traits), mv$(lifetimes) }));
}
开发者ID:thepowersgang,项目名称:mrustc,代码行数:26,代码来源:types.cpp


示例2: parse_ascii_integer

IntegerType
parse_ascii_integer(TokenStream & stream) {
  if (!stream.peek()) throw std::runtime_error("eof");

  if (!ascii_digit_value(*stream.peek())) {
    throw std::runtime_error("not at a number!");
  }

  IntegerType toret = 0;
  while (true) {
    auto inb = stream.peek();
    if (!inb) break;
    auto digit_val = ascii_digit_value(*inb);
    if (!digit_val) break;

    auto oldtoret = toret;
    toret = toret * 10 + *digit_val;
    if (toret < oldtoret) {
      throw std::runtime_error("overflow occured");
    }

    stream.skip();
  }

  return toret;
}
开发者ID:BenjaminSchiborr,项目名称:safe,代码行数:26,代码来源:parse.hpp


示例3: pushInput

// Macro-expand a macro argument 'arg' to create 'expandedArg'.
// Does not replace 'arg'.
// Returns nullptr if no expanded argument is created.
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay)
{
    // expand the argument
    TokenStream* expandedArg = new TokenStream;
    pushInput(new tMarkerInput(this));
    pushTokenStreamInput(arg);
    int token;
    while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) {
        token = tokenPaste(token, *ppToken);
        if (token == tMarkerInput::marker || token == EndOfInput)
            break;
        if (token == PpAtomIdentifier && MacroExpand(ppToken, false, newLineOkay) != 0)
            continue;
        expandedArg->putToken(token, ppToken);
    }

    if (token == EndOfInput) {
        // MacroExpand ate the marker, so had bad input, recover
        delete expandedArg;
        expandedArg = nullptr;
    } else {
        // remove the marker
        popInput();
    }

    return expandedArg;
}
开发者ID:MIPS,项目名称:prebuilts-ndk,代码行数:30,代码来源:Pp.cpp


示例4: calculate

void calculate()
{
    const char quit = 'q';       // t.kind==quit means that t is a quit Token
    const char print = ';';      // t.kind==print means that t is a print Token
    const std::string prompt = "> ";
    const std::string equals = "= ";  // used to indicate that what follows is an evaluation
    TokenStream tokenStream;
    std::string enterToClose = "~~";

    while (std::cin)
        try
        {
            std::cout << prompt;
            Token token = tokenStream.getToken();

            if (token.kind == print)
                token = tokenStream.getToken();   // eat ';'

            if (token.kind == quit)
                break;

            tokenStream.setToken(token);      // reset token into tokenStream
            std::cout << equals << expression(tokenStream) << '\n';
        }

        catch (std::exception& e)
        {
            std::cerr << e.what() << '\n';                      // write error message
            tokenStream.ignore(print);
        }
}
开发者ID:silverbowen,项目名称:Calculator,代码行数:31,代码来源:calculator01.cpp


示例5: singleTerm

        Option<Term> singleTerm(TokenStream& tokens) {
            if (tokens->type == Token::LAMBDA) {
                return lambdaTerm(tokens);

            } else if (tokens->type == Token::OPEN_BRACKET) {
                tokens.advance();
                Option<Term> t = term(tokens);
                if (tokens->type == Token::CLOSE_BRACKET) {
                    tokens.advance();
                    return t;
                } else return errorTerm("expected closing bracket");

            } else if (tokens->type == Token::IDENTIFIER) {
                string id = tokens->identifierValue;
                tokens.advance();
                return Option<Term>(variableTerm(id));

            } else if (tokens->type == Token::INTEGER) {
                int value = tokens->intValue;
                tokens.advance();
                return Option<Term>(integerTerm(value));

            } else if (tokens->type == Token::CLOSE_BRACKET) {
                return errorTerm("unexpected closing bracket");

            } else if (!tokens.good()) {
                return errorTerm("expected a term");
            } else return errorTerm("unknown token");
        }
开发者ID:DouglasOrr,项目名称:LanguagePlayground,代码行数:29,代码来源:core.cpp


示例6: expression

double expression(TokenStream& tokenStream)
{
    double result = term(tokenStream);       // set result to value of term
    Token token = tokenStream.getToken();  // get next token from tokenStream

    while (true)
    {
        if (token.kind == '+')
        {
            result += term(tokenStream);      // evaluate term and add to result
            token = tokenStream.getToken(); // get next token from tokenStream
        }

        if (token.kind == '-')
        {
            result -= term(tokenStream);      // evaluate term and subtract from result
            token = tokenStream.getToken(); // get next token from tokenStream
        }

        if (token.kind != '+' && token.kind != '-')
        {
            tokenStream.setToken(token);    // reset token into token stream
            return result;                    // return result
        }
    }
}
开发者ID:silverbowen,项目名称:Calculator,代码行数:26,代码来源:calculator01.cpp


示例7: SyntaxError

SAWYER_EXPORT std::string
Grammar::evalFunction(TokenStream &tokens, ErrorLocation &eloc) const {
    ASSERT_require(tokens.isa(TOK_FUNCTION));
    std::string funcName = tokens.lexeme();
    ASSERT_require(funcName.size() >= 2 && '@' == funcName[0]);
    funcName = funcName.substr(1);
    tokens.consume();

    // Get the function declaration
    const Function::Ptr func = functions_.getOrDefault(funcName);
    if (!func)
        throw SyntaxError("function \"" + funcName + "\" is not declared");

    // Parse the actual arguments
    std::vector<std::string> actuals;
    while (tokens.isa(TOK_LEFT)) {
        tokens.consume();
        if (func->isMacro()) {
            actuals.push_back(readArgument(tokens, eloc, CONSUME));
        } else {
            actuals.push_back(evalArgument(tokens, eloc, CONSUME));
        }
    }
    func->validateArgs(actuals, tokens);

    ErrorLocation::Trap t(eloc, tokens, "in function \"" + funcName + "\"");
    std::string retval = func->eval(*this, actuals);
    t.passed();
    return retval;
}
开发者ID:billhoffman,项目名称:rose-develop,代码行数:30,代码来源:DocumentMarkup.C


示例8: main

int main(int argc, char *argv[]){

  if (argc != 2){

    cout << "Usage: " << argv[0] << " <filename>" << endl;
    exit (1);
  }
  else {

    try {
      FileReader *reader =  _CLNEW FileReader(argv[1],"UTF-8");
      
      Analyzer *analyzer = new StandardAnalyzer();
      TokenStream *tokenStream = analyzer->tokenStream(_T("iets"), reader);
      Token token;
      while (tokenStream->next(&token)){
	
	char buffer[1000];
	STRCPY_TtoA(buffer, token.termText(), 1000);
	cout << "\t token: " << buffer;
	STRCPY_TtoA(buffer, token.type(), 1000);
	
	cout << " type: " << buffer << endl;
      }
      tokenStream->close();
      delete tokenStream;
      //reader->close();
      delete reader;
      delete analyzer;
    } catch (CLuceneError &e){

      cerr << e.what() << endl;
    }
  }
}
开发者ID:DNPA,项目名称:OcfaModules,代码行数:35,代码来源:TestTokenizer.cpp


示例9: print_remaining_tokens

void print_remaining_tokens(std::ostream& out, TokenStream& tokens)
{
    for (int i=0; i < tokens.remaining(); i++) {
        if (i != 0) out << " ";
        out << get_token_text(tokens.next(i).match);
        out << "(" << tokens.nextStr(i) << ")";
    }
}
开发者ID:levelplane,项目名称:circa,代码行数:8,代码来源:token.cpp


示例10: skip_token

void
skip_token(TokenStream & stream, Token token) {
  while (true) {
    auto inb = stream.peek();
    if (!inb || *inb != token) break;
    stream.skip();
  }
}
开发者ID:BenjaminSchiborr,项目名称:safe,代码行数:8,代码来源:parse.hpp


示例11: expect

void
expect(TokenStream & stream, Token expected) {
  auto maybe_token = stream.peek();
  if (!maybe_token || *maybe_token != expected) {
    throw std::runtime_error("Failed expect");
  }
  stream.skip();
}
开发者ID:BenjaminSchiborr,项目名称:safe,代码行数:8,代码来源:parse.hpp


示例12: SetDisplayURL

static bool
SetDisplayURL(ExclusiveContext *cx, TokenStream &tokenStream, ScriptSource *ss)
{
    if (tokenStream.hasDisplayURL()) {
        if (!ss->setDisplayURL(cx, tokenStream.displayURL()))
            return false;
    }
    return true;
}
开发者ID:eric30,项目名称:gecko-dev,代码行数:9,代码来源:BytecodeCompiler.cpp


示例13: SetSourceMap

static bool
SetSourceMap(JSContext *cx, TokenStream &tokenStream, ScriptSource *ss, JSScript *script)
{
    if (tokenStream.hasSourceMap()) {
        if (!ss->setSourceMap(cx, tokenStream.releaseSourceMap(), script->filename()))
            return false;
    }
    return true;
}
开发者ID:aiked,项目名称:MetaMonkey,代码行数:9,代码来源:BytecodeCompiler.cpp


示例14: SetSourceMap

static bool
SetSourceMap(ExclusiveContext *cx, TokenStream &tokenStream, ScriptSource *ss)
{
    if (tokenStream.hasSourceMap()) {
        if (!ss->setSourceMap(cx, tokenStream.releaseSourceMap()))
            return false;
    }
    return true;
}
开发者ID:SlateScience,项目名称:Spidermonkey,代码行数:9,代码来源:BytecodeCompiler.cpp


示例15: SetSourceMap

static bool
SetSourceMap(ExclusiveContext* cx, TokenStream& tokenStream, ScriptSource* ss)
{
    if (tokenStream.hasSourceMapURL()) {
        MOZ_ASSERT(!ss->hasSourceMapURL());
        if (!ss->setSourceMapURL(cx, tokenStream.sourceMapURL()))
            return false;
    }
    return true;
}
开发者ID:CeperaCPP,项目名称:mongo,代码行数:10,代码来源:BytecodeCompiler.cpp


示例16: dump_remaining_tokens

void dump_remaining_tokens(TokenStream& tokens)
{
    for (int i=0; i < tokens.remaining(); i++) {
        if (i != 0)
            printf(" ");
        Value nextStr;
        tokens.getNextStr(&nextStr, i);
        printf("%s(%s)", get_token_text(tokens.next(i).match), as_cstring(&nextStr));
    }
    printf("\n");
}
开发者ID:andyfischer,项目名称:circa,代码行数:11,代码来源:token.cpp


示例17: MatchToken_

/**
 * A convenience method used for parsing out tokens, this is kind of a repetitive
 * task so it has been written once here.
 *
 * @param ts The token stream to draw from.
 * @param token The token as string to match.
 * @return true if the token was matched, false otherwise.
 */
bool Case_::MatchToken_( TokenStream& ts
                       , std::string const& token) const
{
    bool success = ts.HasTokens();
    if (success)
    {
        Element currentElement = ts.NextToken();
        int const TYPE = currentElement.Type();
        std::string const VALUE(currentElement.ToString());
        success = (TYPE == Types::TOKEN && VALUE == token);
    }
    return success;
}
开发者ID:signatal,项目名称:strine,代码行数:21,代码来源:case_.cpp


示例18: lambdaTerm

        Option<Term> lambdaTerm(TokenStream& tokens) {
            assert(tokens->type == Token::LAMBDA);
            tokens.advance();

            if (tokens->type == Token::IDENTIFIER) {
                string id = tokens->identifierValue;
                tokens.advance();
                Option<Term> body = term(tokens);
                if (body.exists()) {
                    return Option<Term>(lambdaTerm(id, *body));

                } else return body;
            } else return errorTerm("expected identifier");
        }
开发者ID:DouglasOrr,项目名称:LanguagePlayground,代码行数:14,代码来源:core.cpp


示例19: reportNoViableAlternative

void DefaultErrorStrategy::reportNoViableAlternative(Parser *recognizer, const NoViableAltException &e) {
  TokenStream *tokens = recognizer->getTokenStream();
  std::string input;
  if (tokens != nullptr) {
    if (e.getStartToken()->getType() == Token::EOF) {
      input = "<EOF>";
    } else {
      input = tokens->getText(e.getStartToken(), e.getOffendingToken());
    }
  } else {
    input = "<unknown input>";
  }
  std::string msg = "no viable alternative at input " + escapeWSAndQuote(input);
  recognizer->notifyErrorListeners(e.getOffendingToken(), msg, std::make_exception_ptr(e));
}
开发者ID:akosthekiss,项目名称:antlr4,代码行数:15,代码来源:DefaultErrorStrategy.cpp


示例20: ParseEnd_

/**
 * Parse the end of the token stream.
 *
 * @param stream -- Stream that has been prepped.
 */
bool Builtin_::ParseEnd_(TokenStream& stream) const
{
    bool is_end = false;

    if (stream.HasTokens())
    {
        stream.Push();
        Element current_element = stream.NextToken();
        is_end =  (current_element.Type() == Types::TOKEN 
               &&  current_element.ToString() == ")" ); 

        stream.Rollback();
    }

    return is_end;
}
开发者ID:signatal,项目名称:codeography,代码行数:21,代码来源:builtin_.cpp



注:本文中的TokenStream类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
C++ TokenT类代码示例发布时间:2022-05-31
下一篇:
C++ TokenScanner类代码示例发布时间:2022-05-31
热门推荐
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap