////////////////////////////////////////////////////////////////////////////////
	// LexOther:  Lex the token types known to subclass (must be defined by subclass)
	// Return value:
	//	RegularExprToken::Type		The type of the matched token.
	//
	// Note: May throw an integer to indicate error.
	////////////////////////////////////////////////////////////////////////////////
	RegularExprToken::Type RegularExprLexerSymbol::LexOther()
	{
		//the symbol strings are separated by spaces
		if (!ISALNUM(current)) {
			throw 1;
		}
		while (ISALNUM(current) || current == '_') {
			Consume();
		}

		return RegularExprToken::Literal;
	}

  void Encoding::add_constant(STATE, const char* name, Encoding* enc) {
    if(ISDIGIT(*name) || !ISALNUM(*name)) return;

    char* s = const_cast<char*>(name);
    bool has_lower = false;

    if(ISUPPER(*s)) {
      while(*++s && (ISALNUM(*s) || *s == '_')) {
        if(ISLOWER(*s)) has_lower = true;
      }
    }

    if(!*s && !has_lower) {
      if(s - name > ENCODING_NAMELEN_MAX) return;

      G(encoding)->set_const(state, state->symbol(name), enc);
      return;
    }

    char* p = s = strdup(name);
    if(!s) return;

    if(ISUPPER(*s)) {
      while(*++s) {
        if(!ISALNUM(*s)) *s = '_';
        if(ISLOWER(*s)) has_lower = true;
      }

      if(s - p > ENCODING_NAMELEN_MAX) {
        free(s);
        return;
      }
      G(encoding)->set_const(state, state->symbol(p), enc);
    } else {
      has_lower = true;
    }

    if(has_lower) {
      s = p;
      while(*s) {
        if(!ISALNUM(*s)) *s = '_';
        if(ISLOWER(*s)) *s = toupper((int)*s);
        ++s;
      }
      G(encoding)->set_const(state, state->symbol(p), enc);
    }

    free(p);
  }

static void
set_encoding_const(const char *name, rb_encoding *enc)
{
    VALUE encoding = rb_enc_from_encoding(enc);
    char *s = (char *)name;
    int haslower = 0, hasupper = 0, valid = 0;

    if (ISDIGIT(*s)) return;
    if (ISUPPER(*s)) {
	hasupper = 1;
	while (*++s && (ISALNUM(*s) || *s == '_')) {
	    if (ISLOWER(*s)) haslower = 1;
	}
    }
    if (!*s) {
	if (s - name > ENCODING_NAMELEN_MAX) return;
	valid = 1;
	rb_define_const(rb_cEncoding, name, encoding);
    }
    if (!valid || haslower) {
	size_t len = s - name;
	if (len > ENCODING_NAMELEN_MAX) return;
	if (!haslower || !hasupper) {
	    do {
		if (ISLOWER(*s)) haslower = 1;
		if (ISUPPER(*s)) hasupper = 1;
	    } while (*++s && (!haslower || !hasupper));
	    len = s - name;
	}
	len += strlen(s);
	if (len++ > ENCODING_NAMELEN_MAX) return;
	MEMCPY(s = ALLOCA_N(char, len), name, char, len);
	name = s;
	if (!valid) {
	    if (ISLOWER(*s)) *s = ONIGENC_ASCII_CODE_TO_UPPER_CASE((int)*s);
	    for (; *s; ++s) {
		if (!ISALNUM(*s)) *s = '_';
	    }
	    if (hasupper) {
		rb_define_const(rb_cEncoding, name, encoding);
	    }
	}
	if (haslower) {
	    for (s = (char *)name; *s; ++s) {
		if (ISLOWER(*s)) *s = ONIGENC_ASCII_CODE_TO_UPPER_CASE((int)*s);
	    }
	    rb_define_const(rb_cEncoding, name, encoding);
	}
    }

/* Test if a string is a valid C++ identifier (possibly scope qualified).  */
int
cxx_scoped_identifier (const char *s)
{
    if (*s != ':')
        goto no_scope_prefix;

   while (1)
    {
        if (*++s != ':')
            return 0;
        ++s;

      no_scope_prefix:
        if (!ISALPHA ((unsigned char)*s) && *s != '_')
            return 0;

        while (*++s != ':')
        {
            if (*s == '\0')
                return 1;
            else if (!ISALNUM ((unsigned char)*s) && *s != '_')
                return 0;
        }
    }
}

/* Create header and code file.  */
void
output_code (const char *const cfilename, const char *hfilename, const struct parsed_infile *const pf)
{
    if (hfilename != 0)
    {
        char *filename_macro;

        /* Determine filename macro.  */
        {
            size_t i, len;
            const char *read_ptr;
            char *write_ptr;

            if ((read_ptr = strrchr (hfilename, '/')) == 0 || *++read_ptr == '/')
                read_ptr = hfilename;

            len = strlen (read_ptr);
            if (len > 2 && hfilename [len - 1] == 'h' && hfilename [len - 2] == '.')
                len -= 2;

            filename_macro = write_ptr = xmalloc (len + 1);

            for (i = 0; i < len; ++i)
            {
                if (ISALNUM ((unsigned char)*read_ptr))
                    *write_ptr++ = TOUPPER ((unsigned char)*read_ptr);
                else
                    *write_ptr++ = '_';
                ++read_ptr;
            }

            *write_ptr = '\0';
        }

        /* Generate header file.  */
        {
            FILE *const f = fopen (hfilename, "w");

            if (f == 0 || generate_header (f, filename_macro, pf) == -1 || fclose (f) == EOF)
            {
                fprintf (stderr, "%s: %s: %s\n", invocation_name, hfilename, strerror (errno));
                exit (2);
            }
        }

        free (filename_macro);
    }


    /* Generate code file.  */
    {
        FILE *const f = cfilename != 0 ? fopen (cfilename, "w") : stdout;

        if (f == 0 || generate_code (f, pf) == -1 || (cfilename != 0 && fclose (f) == EOF))
        {
            fprintf (stderr, "%s: %s: %s\n", invocation_name, cfilename != 0 ? cfilename : "STDOUT", strerror (errno));
            exit (2);
        }
    }
}

static int
load_encoding(const char *name)
{
    VALUE enclib = rb_sprintf("enc/%s.so", name);
    VALUE verbose = ruby_verbose;
    VALUE debug = ruby_debug;
    VALUE errinfo;
    char *s = RSTRING_PTR(enclib) + 4, *e = RSTRING_END(enclib) - 3;
    int loaded;
    int idx;

    while (s < e) {
	if (!ISALNUM(*s)) *s = '_';
	else if (ISUPPER(*s)) *s = (char)TOLOWER(*s);
	++s;
    }
    FL_UNSET(enclib, FL_TAINT);
    enclib = rb_fstring(enclib);
    ruby_verbose = Qfalse;
    ruby_debug = Qfalse;
    errinfo = rb_errinfo();
    loaded = rb_require_internal(enclib, rb_safe_level());
    ruby_verbose = verbose;
    ruby_debug = debug;
    rb_set_errinfo(errinfo);
    if (loaded < 0 || 1 < loaded) return -1;
    if ((idx = rb_enc_registered(name)) < 0) return -1;
    if (enc_autoload_p(enc_table.list[idx].enc)) return -1;
    return idx;
}

static char	*scan_labelref(char *c)
{
	char	*b = c;
	for ( ; (ISALNUM(*b) || '_' == *b || '^' == *b); b++,ext_source_column++)
		;
	return (b == c) ? 0 : b;
}

static int
load_encoding(const char *name)
{
    VALUE enclib = rb_sprintf("enc/%s.so", name);
    VALUE verbose = ruby_verbose;
    VALUE debug = ruby_debug;
    VALUE loaded;
    char *s = RSTRING_PTR(enclib) + 4, *e = RSTRING_END(enclib) - 3;
    int idx;

    while (s < e) {
	if (!ISALNUM(*s)) *s = '_';
	else if (ISUPPER(*s)) *s = TOLOWER(*s);
	++s;
    }
    OBJ_FREEZE(enclib);
    ruby_verbose = Qfalse;
    ruby_debug = Qfalse;
    loaded = rb_protect(require_enc, enclib, 0);
    ruby_verbose = verbose;
    ruby_debug = debug;
    rb_set_errinfo(Qnil);
    if (NIL_P(loaded)) return -1;
    if ((idx = rb_enc_registered(name)) < 0) return -1;
    if (enc_autoload_p(enc_table.list[idx].enc)) return -1;
    return idx;
}

void
cgen_keyword_add (CGEN_KEYWORD *kt, CGEN_KEYWORD_ENTRY *ke)
{
  unsigned int hash;
  size_t i;

  if (kt->name_hash_table == NULL)
    build_keyword_hash_tables (kt);

  hash = hash_keyword_name (kt, ke->name, 0);
  ke->next_name = kt->name_hash_table[hash];
  kt->name_hash_table[hash] = ke;

  hash = hash_keyword_value (kt, ke->value);
  ke->next_value = kt->value_hash_table[hash];
  kt->value_hash_table[hash] = ke;

  if (ke->name[0] == 0)
    kt->null_entry = ke;

  for (i = 1; i < strlen (ke->name); i++)
    if (! ISALNUM (ke->name[i])
	&& ! strchr (kt->nonalpha_chars, ke->name[i]))
      {
	size_t idx = strlen (kt->nonalpha_chars);
	
	/* If you hit this limit, please don't just
	   increase the size of the field, instead
	   look for a better algorithm.  */
	if (idx >= sizeof (kt->nonalpha_chars) - 1)
	  abort ();
	kt->nonalpha_chars[idx] = ke->name[i];
	kt->nonalpha_chars[idx+1] = 0;
      }
}

static VSTRING *flush_site_to_path(VSTRING *path, const char *site)
{
    const char *ptr;
    int     ch;

    /*
     * Convert the name to ASCII, so that we don't to end up with non-ASCII
     * names in the file system. The IDNA library functions fold case.
     */
#ifndef NO_EAI
    if ((site = midna_domain_to_ascii(site)) == 0)
	return (0);
#endif

    /*
     * Allocate buffer on the fly; caller still needs to clean up.
     */
    if (path == 0)
	path = vstring_alloc(10);

    /*
     * Mask characters that could upset the name-to-queue-file mapping code.
     */
    for (ptr = site; (ch = *(unsigned const char *) ptr) != 0; ptr++)
	if (ISALNUM(ch))
	    VSTRING_ADDCH(path, tolower(ch));
	else
	    VSTRING_ADDCH(path, '_');
    VSTRING_TERMINATE(path);

    if (msg_verbose)
	msg_info("site %s to path %s", site, STR(path));

    return (path);
}

static bool peek_ipv6(const char *str, size_t *skip)
{
  /*
   * Scan for a potential IPv6 literal.
   * - Valid globs contain a hyphen and <= 1 colon.
   * - IPv6 literals contain no hyphens and >= 2 colons.
   */
  size_t i = 0;
  size_t colons = 0;
  if(str[i++] != '[') {
    return FALSE;
  }
  for(;;) {
    const char c = str[i++];
    if(ISALNUM(c) || c == '.' || c == '%') {
      /* ok */
    }
    else if(c == ':') {
      colons++;
    }
    else if(c == ']') {
      *skip = i;
      return colons >= 2 ? TRUE : FALSE;
    }
    else {
      return FALSE;
    }
  }
}

static VSTRING *flush_site_to_path(VSTRING *path, const char *site)
{
    const char *ptr;
    int     ch;

    /*
     * Allocate buffer on the fly; caller still needs to clean up.
     */
    if (path == 0)
	path = vstring_alloc(10);

    /*
     * Mask characters that could upset the name-to-queue-file mapping code.
     */
    for (ptr = site; (ch = *(unsigned const char *) ptr) != 0; ptr++)
	if (ISALNUM(ch))
	    VSTRING_ADDCH(path, ch);
	else
	    VSTRING_ADDCH(path, '_');
    VSTRING_TERMINATE(path);

    if (msg_verbose)
	msg_info("site %s to path %s", site, STR(path));

    return (path);
}

static int
unicode_mangling_length (const char *name, int len)
{
  const unsigned char *ptr;
  const unsigned char *limit = (const unsigned char *)name + len;
  int need_escapes = 0;		/* Whether we need an escape or not */
  int num_chars = 0;		/* Number of characters in the mangled name */
  int uuU = 0;			/* Help us to find __U. 0: '_', 1: '__' */
  for (ptr = (const unsigned char *) name;  ptr < limit;  )
    {
      int ch = UTF8_GET(ptr, limit);

      if (ch < 0)
	error ("internal error - invalid Utf8 name");
      if ((ISALNUM (ch) && ch != 'U') || ch == '$')
	num_chars++;
      /* Everything else needs encoding */
      else
	{
	  int encoding_length = 2;

	  if (ch == '_' || ch == 'U')
	    {
	      /* It's always at least one character. */
	      num_chars++;

	      /* Prepare to recognize __U */
	      if (ch == '_' && (uuU < 3))
		uuU++;

	      /* We recognize __U that we wish to encode __U_, we
	         count one more character. */
	      else if (ch == 'U' && (uuU == 2))
		{
		  num_chars++;
		  need_escapes = 1;
		  uuU = 0;
		}
	      /* Otherwise, just reset uuU */
	      else
		uuU = 0;

	      continue;
	    }
	  
	  if (ch > 0xff)
	    encoding_length++;
	  if (ch > 0xfff)
	    encoding_length++;
	  
	  num_chars += (4 + encoding_length);
	  need_escapes = 1;
	  uuU = 0;
	}
    }
  if (need_escapes)
    return num_chars;
  else
    return 0;
}

static const char *
parse_mem8 (CGEN_CPU_DESC cd,
	    const char **strp,
	    int opindex,
	    unsigned long *valuep)
{
  if (**strp == '(')
    {
      const char *s = *strp;
      
      if (s[1] == '-' && s[2] == '-')
	return _("Bad register in preincrement");

      while (ISALNUM (*++s))
	;
      if (s[0] == '+' && s[1] == '+' && (s[2] == ')' || s[2] == ','))
	return _("Bad register in postincrement");
      if (s[0] == ',' || s[0] == ')')
	return _("Bad register name");
    }
  else if (cgen_parse_keyword (cd, strp, & xstormy16_cgen_opval_gr_names, 
			       (long *) valuep) == NULL)
    return _("Label conflicts with register name");
  else if (strncasecmp (*strp, "rx,", 3) == 0
	   || strncasecmp (*strp, "rxl,", 3) == 0
	   || strncasecmp (*strp, "rxh,", 3) == 0)
    return _("Label conflicts with `Rx'");
  else if (**strp == '#')
    return _("Bad immediate expression");
  
  return cgen_parse_unsigned_integer (cd, strp, opindex, valuep);
}

static bool
parse_feature_tag (const char **pp, const char *end, hb_feature_t *feature)
{
  parse_space (pp, end);

  char quote = 0;

  if (*pp < end && (**pp == '\'' || **pp == '"'))
  {
    quote = **pp;
    (*pp)++;
  }

  const char *p = *pp;
  while (*pp < end && ISALNUM(**pp))
    (*pp)++;

  if (p == *pp || *pp - p > 4)
    return false;

  feature->tag = hb_tag_from_string (p, *pp - p);

  if (quote)
  {
    /* CSS expects exactly four bytes.  And we only allow quotations for
     * CSS compatibility.  So, enforce the length. */
     if (*pp - p != 4)
       return false;
    if (*pp == end || **pp != quote)
      return false;
    (*pp)++;
  }

  return true;
}

static void qmgr_trigger_event(char *buf, int len,
			               char *unused_service, char **argv)
{
    int     incoming_flag = 0;
    int     deferred_flag = 0;
    int     i;

    /*
     * Sanity check. This service takes no command-line arguments.
     */
    if (argv[0])
	msg_fatal("unexpected command-line argument: %s", argv[0]);

    /*
     * Collapse identical requests that have arrived since we looked last
     * time. There is no client feedback so there is no need to process each
     * request in order. And as long as we don't have conflicting requests we
     * are free to sort them into the most suitable order.
     */
#define QMGR_FLUSH_BEFORE	(QMGR_FLUSH_ONCE | QMGR_FLUSH_DFXP)

    for (i = 0; i < len; i++) {
	if (msg_verbose)
	    msg_info("request: %d (%c)",
		     buf[i], ISALNUM(buf[i]) ? buf[i] : '?');
	switch (buf[i]) {
	case TRIGGER_REQ_WAKEUP:
	case QMGR_REQ_SCAN_INCOMING:
	    incoming_flag |= QMGR_SCAN_START;
	    break;
	case QMGR_REQ_SCAN_DEFERRED:
	    deferred_flag |= QMGR_SCAN_START;
	    break;
	case QMGR_REQ_FLUSH_DEAD:
	    deferred_flag |= QMGR_FLUSH_BEFORE;
	    incoming_flag |= QMGR_FLUSH_BEFORE;
	    break;
	case QMGR_REQ_SCAN_ALL:
	    deferred_flag |= QMGR_SCAN_ALL;
	    incoming_flag |= QMGR_SCAN_ALL;
	    break;
	default:
	    if (msg_verbose)
		msg_info("request ignored");
	    break;
	}
    }

    /*
     * Process each request type at most once. Modifiers take effect upon the
     * next queue run. If no queue run is in progress, and a queue scan is
     * requested, the request takes effect immediately.
     */
    if (incoming_flag != 0)
	qmgr_scan_request(qmgr_scans[QMGR_SCAN_IDX_INCOMING], incoming_flag);
    if (deferred_flag != 0)
	qmgr_scan_request(qmgr_scans[QMGR_SCAN_IDX_DEFERRED], deferred_flag);
}

static inline void get_ident ()
{
	CTok.type = IDENT_DUMMY;
	CTok.p = &Cpp [Ci];

	while (ISALNUM (Cpp [Ci]))
		if (++Ci >= Clen) break;

	CTok.len = &Cpp [Ci] - CTok.p;
}

const char *
cgen_parse_keyword (CGEN_CPU_DESC cd ATTRIBUTE_UNUSED,
		    const char **strp,
		    CGEN_KEYWORD *keyword_table,
		    long *valuep)
{
  const CGEN_KEYWORD_ENTRY *ke;
  char buf[256];
  const char *p,*start;

  if (keyword_table->name_hash_table == NULL)
    (void) cgen_keyword_search_init (keyword_table, NULL);

  p = start = *strp;

  /* Allow any first character.  This is to make life easier for
     the fairly common case of suffixes, eg. 'ld.b.w', where the first
     character of the suffix ('.') is special.  */
  if (*p)
    ++p;
  
  /* Allow letters, digits, and any special characters.  */
  while (((p - start) < (int) sizeof (buf))
	 && *p
	 && (ISALNUM (*p)
	     || *p == '_'
	     || strchr (keyword_table->nonalpha_chars, *p)))
    ++p;

  if (p - start >= (int) sizeof (buf))
    {
      /* All non-empty CGEN keywords can fit into BUF.  The only thing
	 we can match here is the empty keyword.  */
      buf[0] = 0;
    }
  else
    {
      memcpy (buf, start, p - start);
      buf[p - start] = 0;
    }

  ke = cgen_keyword_lookup_name (keyword_table, buf);

  if (ke != NULL)
    {
      *valuep = ke->value;
      /* Don't advance pointer if we recognized the null keyword.  */
      if (ke->name[0] != 0)
	*strp = p;
      return NULL;
    }

  return "unrecognized keyword/register name";
}

/* Test if a string is a valid C identifier.  */
int
c_identifier (const char *s)
{
    if (!ISALPHA ((unsigned char)*s) && *s != '_')
        return 0;

    while (*++s != '\0')
        if (!ISALNUM ((unsigned char)*s) && *s != '_')
            return 0;

    return 1;
}

PyObject*
_Py_bytes_isalnum(const char *cptr, Py_ssize_t len)
{
    register const unsigned char *p
        = (unsigned char *) cptr;
    register const unsigned char *e;

    /* Shortcut for single character strings */
    if (len == 1 && ISALNUM(*p))
	Py_RETURN_TRUE;

    /* Special case for empty strings */
    if (len == 0)
	Py_RETURN_FALSE;

    e = p + len;
    for (; p < e; p++) {
	if (!ISALNUM(*p))
	    Py_RETURN_FALSE;
    }
    Py_RETURN_TRUE;
}