C++ TokenizeContext类代码示例

OGeek|极客世界-中国程序员成长平台 › 门户 › 编程› C++›C++教程

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了C++中TokenizeContext类的典型用法代码示例。如果您正苦于以下问题：C++ TokenizeContext类的具体用法？C++ TokenizeContext怎么用？C++ TokenizeContext使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

在下文中一共展示了TokenizeContext类的17个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的C++代码示例。

示例1: consume_string_literal

void consume_string_literal(TokenizeContext &context)
{
    int lookahead = 0;

    // Consume starting quote, this can be ' or "
    char quote_type = context.next();
    lookahead++;

    bool escapedNext = false;
    while (context.withinRange(lookahead)) {

        char c = context.next(lookahead);

        if (c == quote_type && !escapedNext)
            break;

        if (c == '\\' && !escapedNext)
            escapedNext = true;
        else
            escapedNext = false;

        lookahead++;
    }

    // consume ending quote
    lookahead++;

    context.consume(tok_String, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:29，代码来源:token.cpp

示例2: try_to_consume_keyword

bool try_to_consume_keyword(TokenizeContext& context, int keyword)
{
    const char* str = get_token_text(keyword);
    int str_len = (int) strlen(str);

    // Check if every letter matches
    for (int i=0; i < str_len; i++) {
        if (context.next(i) != str[i])
            return false;
    }

    // Check that this is really the end of the word
    if (is_acceptable_inside_identifier(context.next(str_len)))
        return false;

    // Don't match as a keyword if the next character is (. This might be
    // a bad idea.
    if (context.next(str_len) == '(')
        return false;

    // Keyword matches, now consume it
    context.consume(keyword, str_len);

    return true;
}

开发者ID:andyfischer，项目名称:circa，代码行数:25，代码来源:token.cpp

示例3: consume_whitespace

void consume_whitespace(TokenizeContext &context)
{
    int lookahead = 0;
    while (is_whitespace(context.next(lookahead)))
        lookahead++;

    context.consume(tok_Whitespace, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:8，代码来源:token.cpp

示例4: consume_identifier

void consume_identifier(TokenizeContext &context)
{
    int lookahead = 0;
    while (is_acceptable_inside_identifier(context.next(lookahead)))
        lookahead++;

    context.consume(tok_Identifier, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:8，代码来源:token.cpp

示例5: consume_whitespace

void consume_whitespace(TokenizeContext &context)
{
    int lookahead = 0;
    while (is_whitespace(context.next(lookahead)))
        lookahead++;

    context.consume(TK_WHITESPACE, lookahead);
}

开发者ID:levelplane，项目名称:circa，代码行数:8，代码来源:token.cpp

示例6: consume_comment

void consume_comment(TokenizeContext& context)
{
    int lookahead = 0;
    while (context.withinRange(lookahead) && !is_newline(context.next(lookahead)))
        lookahead++;

    context.consume(tok_Comment, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:8，代码来源:token.cpp

示例7: consume_name

void consume_name(TokenizeContext &context)
{
    int lookahead = 0;

    // consume the leading :
    lookahead++;

    while (is_acceptable_inside_identifier(context.next(lookahead)))
        lookahead++;

    context.consume(TK_NAME, lookahead);
}

开发者ID:levelplane，项目名称:circa，代码行数:12，代码来源:token.cpp

示例8: consume_hex_number

void consume_hex_number(TokenizeContext &context)
{
    int lookahead = 0;

    // consume the 0x part
    lookahead += 2;

    while (is_hexadecimal_digit(context.next(lookahead)))
        lookahead++;

    context.consume(TK_HEX_INTEGER, lookahead);
}

开发者ID:levelplane，项目名称:circa，代码行数:12，代码来源:token.cpp

示例9: consume_symbol

void consume_symbol(TokenizeContext &context)
{
    int lookahead = 0;

    // consume the leading :
    lookahead++;

    while (is_acceptable_inside_identifier(context.next(lookahead)))
        lookahead++;

    context.consume(tok_ColonString, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:12，代码来源:token.cpp

示例10: consume_hex_number

void consume_hex_number(TokenizeContext &context)
{
    int lookahead = 0;

    // consume the 0x part
    lookahead += 2;

    while (is_hexadecimal_digit(context.next(lookahead)))
        lookahead++;

    context.consume(tok_HexInteger, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:12，代码来源:token.cpp

示例11: match_number

bool match_number(TokenizeContext &context)
{
    int lookahead = 0;

    if (context.next(lookahead) == '.')
        lookahead++;

    if (is_number(context.next(lookahead)))
        return true;

    return false;
}

开发者ID:andyfischer，项目名称:circa，代码行数:12，代码来源:token.cpp

示例12: consume_triple_quoted_string_literal

void consume_triple_quoted_string_literal(TokenizeContext &context)
{
    int lookahead = 0;

    // Consume initial <<<
    lookahead += 3;

    while (context.withinRange(lookahead) &&
            !(context.next(lookahead) == '>'
                && context.next(lookahead + 1) == '>'
                && context.next(lookahead + 2) == '>'))
        lookahead++;

    // Consume closing >>>
    lookahead += 3;
    context.consume(tok_String, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:17，代码来源:token.cpp

示例13: consume_color_literal

void consume_color_literal(TokenizeContext &context)
{
    int lookahead = 0;

    // consume #
    lookahead++;

    while (is_hexadecimal_digit(context.next(lookahead)))
        lookahead++;

    int hex_digits = lookahead - 1;

    // acceptable lengths are 3, 4, 6 or 8 characters (not including #)
    if (hex_digits == 3 || hex_digits == 4 || hex_digits == 6 || hex_digits == 8)
        context.consume(tok_Color, lookahead);
    else
        context.consume(tok_Unrecognized, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:18，代码来源:token.cpp

示例14: consume_multiline_comment

void consume_multiline_comment(TokenizeContext& context)
{
    int lookahead = 0;
    
    // Keep track of the current depth, for nested blocks.
    int depth = 0;

    while (context.withinRange(lookahead)) {
        if (context.next(lookahead) == '{' && context.next(lookahead + 1) == '-') {

            // Found a comment opener, increase depth. Also advance lookahead so that
            // we don't get confused by this: {-}
            lookahead += 2;
            depth++;
            continue;
        }

        if (context.next(lookahead) == '-' && context.next(lookahead + 1) == '}') {

            // Found a comment ender.
            depth--;
            lookahead += 2;

            if (depth == 0)
                break;

            continue;
        }

        lookahead++;
    }

    context.consume(tok_Comment, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:34，代码来源:token.cpp

示例15: consume_number

void consume_number(TokenizeContext &context)
{
    int lookahead = 0;
    bool dot_encountered = false;

    // Possibly consume minus sign
    if (context.next(lookahead) == '-') {
        lookahead++;
    }

    while (true) {
        if (is_number(context.next(lookahead))) {
            lookahead++;
        } else if (context.next(lookahead) == '.') {
            // If we've already encountered a dot, finish and don't consume
            // this one.
            if (dot_encountered)
                break;

            // Special case: if this dot is followed by another dot, then it should
            // be tokenized as TWO_DOTS, so don't consume it here.
            if (context.next(lookahead+1) == '.')
                break;

            // Another special case, if the dot is followed by an identifier, then
            // don't consume it here. It might be an object call.
            if (is_identifier_first_letter(context.next(lookahead + 1)))
                break;

            // Otherwise, consume the dot
            lookahead++;
            dot_encountered = true;
        }
        else {
            break;
        }
    }

    if (dot_encountered)
        context.consume(tok_Float, lookahead);
    else
        context.consume(tok_Integer, lookahead);
}

开发者ID:andyfischer，项目名称:circa，代码行数:43，代码来源:token.cpp

示例16: top_level_consume_token

void top_level_consume_token(TokenizeContext &context)
{
    if (is_identifier_first_letter(context.next())) {

        if (try_to_consume_keyword(context, TK_DEF)) return;
        if (try_to_consume_keyword(context, TK_TYPE)) return;
        if (try_to_consume_keyword(context, TK_BEGIN)) return;
        if (try_to_consume_keyword(context, TK_END)) return;
        if (try_to_consume_keyword(context, TK_IF)) return;
        if (try_to_consume_keyword(context, TK_ELSE)) return;
        if (try_to_consume_keyword(context, TK_ELIF)) return;
        if (try_to_consume_keyword(context, TK_FOR)) return;
        if (try_to_consume_keyword(context, TK_STATE)) return;
        if (try_to_consume_keyword(context, TK_IN)) return;
        if (try_to_consume_keyword(context, TK_TRUE)) return;
        if (try_to_consume_keyword(context, TK_FALSE)) return;
        // check 'do once' before 'do'
        if (try_to_consume_keyword(context, TK_DO_ONCE)) return;
        if (try_to_consume_keyword(context, TK_DO)) return;
        if (try_to_consume_keyword(context, TK_NAMESPACE)) return;
        if (try_to_consume_keyword(context, TK_INCLUDE)) return;
        if (try_to_consume_keyword(context, TK_IMPORT)) return;
        if (try_to_consume_keyword(context, TK_AND)) return;
        if (try_to_consume_keyword(context, TK_OR)) return;
        if (try_to_consume_keyword(context, TK_DISCARD)) return;
        if (try_to_consume_keyword(context, TK_NULL)) return;
        if (try_to_consume_keyword(context, TK_RETURN)) return;
        if (try_to_consume_keyword(context, TK_BREAK)) return;
        if (try_to_consume_keyword(context, TK_CONTINUE)) return;
        if (try_to_consume_keyword(context, TK_SWITCH)) return;
        if (try_to_consume_keyword(context, TK_CASE)) return;
        if (try_to_consume_keyword(context, TK_WHILE)) return;

        consume_identifier(context);
        return;
    }

    if (is_whitespace(context.next())) {
        consume_whitespace(context);
        return;
    }

    if (context.next() == '0'
            && context.next(1) == 'x') {
        consume_hex_number(context);
        return;
    }

    if (match_number(context)) {
        consume_number(context);
        return;
    }

    // Check for specific characters
    switch(context.next()) {
    case '(':
        context.consume(TK_LPAREN, 1);
        return;
    case ')':
        context.consume(TK_RPAREN, 1);
        return;
    case '{':
        if (context.next(1) == '-') {
            consume_multiline_comment(context);
            return;
        }
        context.consume(TK_LBRACE, 1);
        return;
    case '}':
        context.consume(TK_RBRACE, 1);
        return;
    case '[':
        context.consume(TK_LBRACKET, 1);
        return;
    case ']':
        context.consume(TK_RBRACKET, 1);
        return;
    case ',':
        context.consume(TK_COMMA, 1);
        return;
    case '@':
        if (context.next(1) == '.') {
            context.consume(TK_AT_DOT, 2);
        } else {
            context.consume(TK_AT_SIGN, 1);
        }
        return;
    case '=':
        if (context.next(1) == '=') {
            context.consume(TK_DOUBLE_EQUALS, 2);
            return;
        }

        context.consume(TK_EQUALS, 1);
        return;
    case '"':
    case '\'':
        consume_string_literal(context);
        return;
    case '\n':
//.........这里部分代码省略.........

开发者ID:levelplane，项目名称:circa，代码行数:101，代码来源:token.cpp

示例17: top_level_consume_token

void top_level_consume_token(TokenizeContext &context)
{
    if (is_identifier_first_letter(context.next())) {

        if (context.next() <= 'm') {
            // a through m
            if (try_to_consume_keyword(context, tok_And)) return;
            if (try_to_consume_keyword(context, tok_Break)) return;
            if (try_to_consume_keyword(context, tok_Case)) return;
            if (try_to_consume_keyword(context, tok_Continue)) return;
            if (try_to_consume_keyword(context, tok_Def)) return;
            if (try_to_consume_keyword(context, tok_Discard)) return;
            if (try_to_consume_keyword(context, tok_Else)) return;
            if (try_to_consume_keyword(context, tok_Elif)) return;
            if (try_to_consume_keyword(context, tok_False)) return;
            if (try_to_consume_keyword(context, tok_For)) return;
            if (try_to_consume_keyword(context, tok_If)) return;
            if (try_to_consume_keyword(context, tok_In)) return;
            if (try_to_consume_keyword(context, tok_Import)) return;
            if (try_to_consume_keyword(context, tok_Include)) return;
            if (try_to_consume_keyword(context, tok_Let)) return;
        } else {
            // n through z
            if (try_to_consume_keyword(context, tok_Namespace)) return;
            if (try_to_consume_keyword(context, tok_Not)) return;
            if (try_to_consume_keyword(context, tok_Nil)) return;
            if (try_to_consume_keyword(context, tok_Or)) return;
            if (try_to_consume_keyword(context, tok_Return)) return;
            if (try_to_consume_keyword(context, tok_State)) return;
            if (try_to_consume_keyword(context, tok_Struct)) return;
            if (try_to_consume_keyword(context, tok_Switch)) return;
            if (try_to_consume_keyword(context, tok_True)) return;
            if (try_to_consume_keyword(context, tok_Require)) return;
            if (try_to_consume_keyword(context, tok_RequireLocal)) return;
            if (try_to_consume_keyword(context, tok_Package)) return;
            if (try_to_consume_keyword(context, tok_Section)) return;
            if (try_to_consume_keyword(context, tok_While)) return;
        }

        consume_identifier(context);
        return;
    }

    if (is_whitespace(context.next())) {
        consume_whitespace(context);
        return;
    }

    if (context.next() == '0'
        && context.next(1) == 'x') {
        consume_hex_number(context);
        return;
    }

    if (match_number(context)) {
        consume_number(context);
        return;
    }

    // Check for specific characters
    switch(context.next()) {
        case '(':
            context.consume(tok_LParen, 1);
            return;
        case ')':
            context.consume(tok_RParen, 1);
            return;
        case '{':
            if (context.next(1) == '-') {
                consume_multiline_comment(context);
                return;
            }
            context.consume(tok_LBrace, 1);
            return;
        case '}':
            context.consume(tok_RBrace, 1);
            return;
        case '[':
            context.consume(tok_LSquare, 1);
            return;
        case ']':
            context.consume(tok_RSquare, 1);
            return;
        case ',':
            context.consume(tok_Comma, 1);
            return;
        case '@':
            context.consume(tok_At, 1);
            return;
        case '=':
            if (context.next(1) == '=') {
                context.consume(tok_DoubleEquals, 2);
                return;
            }  else if (context.next(1) == '>') {
                context.consume(tok_FatArrow, 2);
                return;
            }

            context.consume(tok_Equals, 1);
            return;
//.........这里部分代码省略.........

开发者ID:andyfischer，项目名称:circa，代码行数:101，代码来源:token.cpp

注：本文中的TokenizeContext类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。

鲜花

握手

雷人

路过

鸡蛋

该文章已有0人参与评论

请发表评论

全部评论

专题导读

More+

10-27 六六分期app的软件客服如何联系？(六六分期

11-06 可心卡盟:win10系统火狐flash插件崩溃怎么

11-06 亲亲特价:怎么删除回收站图标

11-06 济南大学虚拟社区:鲁大师节能降温的具体办

11-06 xlueops.exe:无线网络安装向导

11-06 女斗合众国:win7系统cf与主机连接不稳定怎

11-06 0xc000022-[cf烟雾头]cf怎么调烟雾头

11-06 qizideyouhuo:应用程序无法正常启动0xc0000

11-06 ipz-185:win7系统vcf文件怎么打开

11-06 傻哥蹦迪:win10系统s4怎么打开usb调试

11-06 八神浩树gtaste:回收站清空了怎么恢复

11-06 妖尾之黑色守护:win10系统电脑没有1440x900

11-06 校园至尊魔王小说:win7系统浏览网页时字体

11-06 女斗合众国:win10系统访问共享文件夹提示请

11-06 tokyo hot n0654:恢复win7系统默认字体一招

11-06 雨酷仙境:设置win7系统转移临时文件夹腾出

11-06 阿穆纳伊之杖:win7系统开始菜单在右边还原

11-06 tunespotting:win10系统火狐flash插件总是

11-06 甘尔葛分析师：计谋网站seo关键词暴涨有什

11-06 蔡贵霖: 计谋网站seo关键词暴涨有什么秘密

11-06 博益网首页:ao3网页版进入不了解决方法

11-06 漏斗子专栏: 网站数据分析小白易懂精华篇

11-06 见证双虹怎么做:win7系统开启telnet命令的

11-06 颾狐蝶蜋:系统资源不足无法完成请求的服务

11-06 国光中学校歌:提交网站到alexa查询详细步骤

11-06 西安有情天:静态网页和动态网页的区别

11-06 红木雅尚斋:外部链接构造对网站的好处

11-06 前官礼遇：防止域名劫持–增强域安全性的10

11-06 密传二转答案: 中文分词算法有哪些

11-06 金泉家园邮编:百度快照劫持的表现及应对方

C++ Tokenizer类代码示例发布时间：2022-05-31

C++ Tokeniser类代码示例发布时间：2022-05-31

剪的笔顺,诠释剪的笔画,认识剪的部首

1 六六分期app的软件客服如何联系？(六六分期

六六分期app的软件客服如何联系？不知道吗？加qq群【895510560】即可！标题：六六分期

阅读：18222|2023-10-27

2 可心卡盟:win10系统火狐flash插件崩溃怎么

今天小编告诉大家如何处理win10系统火狐flash插件总是崩溃的问题，可能很多用户都不知

阅读：9658|2022-11-06

3 亲亲特价:怎么删除回收站图标

今天小编告诉大家如何对win10系统删除桌面回收站图标进行设置，可能很多用户都不知道

阅读：8173|2022-11-06

4 济南大学虚拟社区:鲁大师节能降温的具体办

今天小编告诉大家如何对win10系统电脑设置节能降温的设置方法，想必大家都遇到过需要

阅读：8543|2022-11-06

5 xlueops.exe:无线网络安装向导

我们在使用xp系统的过程中,经常需要对xp系统无线网络安装向导设置进行设置，可能很多

阅读：8450|2022-11-06

6 女斗合众国:win7系统cf与主机连接不稳定怎

今天小编告诉大家如何处理win7系统玩cf老是与主机连接不稳定的问题，可能很多用户都不

阅读：9375|2022-11-06

7 0xc000022-[cf烟雾头]cf怎么调烟雾头

电脑对日常生活的重要性小编就不多说了，可是一旦碰到win7系统设置cf烟雾头的问题，很

阅读：8420|2022-11-06

8 qizideyouhuo:应用程序无法正常启动0xc0000

我们在日常使用电脑的时候，有的小伙伴们可能在打开应用的时候会遇见提示应用程序无法

阅读：7855|2022-11-06

9 ipz-185:win7系统vcf文件怎么打开

今天小编告诉大家如何对win7系统打开vcf文件进行设置，可能很多用户都不知道怎么对win

阅读：8405|2022-11-06

10 傻哥蹦迪:win10系统s4怎么打开usb调试

今天小编告诉大家如何对win10系统s4开启USB调试模式进行设置，可能很多用户都不知道怎

阅读：7392|2022-11-06

客服电话

电子邮件

C++ TokenizeContext类代码示例

示例1: consume_string_literal

示例2: try_to_consume_keyword

示例3: consume_whitespace

示例4: consume_identifier

示例5: consume_whitespace

示例6: consume_comment

示例7: consume_name

示例8: consume_hex_number

示例9: consume_symbol

示例10: consume_hex_number

示例11: match_number

示例12: consume_triple_quoted_string_literal

示例13: consume_color_literal

示例14: consume_multiline_comment

示例15: consume_number

示例16: top_level_consume_token

示例17: top_level_consume_token

请发表评论

全部评论

上一篇：

下一篇：

librespeed/speedtest: Self-hosted Speedt

CVE-2022-34965

avehtari/BDA_m_demos: Bayesian Data Anal

四维彩超怎么看性别？四维看男孩女孩诀窍

个体户条例实施细则，2020 最新个体工商户

剪的笔顺,诠释剪的笔画,认识剪的部首

六六分期app的软件客服如何联系？(六六分期

florent37/ViewAnimator: A fluent Android

florent37/Shrine-MaterialDesign2: implem

CVE-2020-36276

SimpleSoftwareIO/simple-sms: Send and re

关于我们

产品与服务

解决方案

139-2527-9053