func fileTokens(tf *token.File) (toks []string, err error) {
	src, err := ioutil.ReadFile(tf.Name())
	if err != nil {
		return nil, err
	}
	s := &scanner.Scanner{}
	s.Init(tf, src, nil, 0)
	tokmap := make(TokenSet)
	for {
		_, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		if tok == token.STRING {
			// XXX: what if strings are misspelled?
			lit = lit[1 : len(lit)-1]
		}
		tokmap[lit] = struct{}{}
	}

	for k, _ := range tokmap {
		toks = append(toks, k)
	}
	return toks, nil
}

func (d *DIBuilder) getFile(file *token.File) llvm.Metadata {
	if diFile := d.files[file]; diFile.C != nil {
		return diFile
	}
	diFile := d.builder.CreateFile(d.remapFilePath(file.Name()), "")
	d.files[file] = diFile
	return diFile
}

func (d *DIBuilder) getFile(file *token.File) llvm.Value {
	if diFile := d.files[file]; !diFile.IsNil() {
		return diFile
	}
	diFile := d.builder.CreateFile(d.remapFilePath(file.Name()), "")
	d.files[file] = diFile
	return diFile
}

// findQueryPos searches fset for filename and translates the
// specified file-relative byte offsets into token.Pos form.  It
// returns an error if the file was not found or the offsets were out
// of bounds.
//
func findQueryPos(fset *token.FileSet, filename string, startOffset, endOffset int) (start, end token.Pos, err error) {
	var file *token.File
	fset.Iterate(func(f *token.File) bool {
		if sameFile(filename, f.Name()) {
			// (f.Name() is absolute)
			file = f
			return false // done
		}
		return true // continue
	})
	if file == nil {
		err = fmt.Errorf("couldn't find file containing position")
		return
	}

	// Range check [start..end], inclusive of both end-points.

	if 0 <= startOffset && startOffset <= file.Size() {
		start = file.Pos(int(startOffset))
	} else {
		err = fmt.Errorf("start position is beyond end of file")
		return
	}

	if 0 <= endOffset && endOffset <= file.Size() {
		end = file.Pos(int(endOffset))
	} else {
		err = fmt.Errorf("end position is beyond end of file")
		return
	}

	return
}

// parseQueryPos parses a string of the form "file:pos" or
// file:start,end" where pos, start, end match #%d and represent byte
// offsets, and returns the extent to which it refers.
//
// (Numbers without a '#' prefix are reserved for future use,
// e.g. to indicate line/column positions.)
//
func parseQueryPos(fset *token.FileSet, queryPos string) (start, end token.Pos, err error) {
	if queryPos == "" {
		err = fmt.Errorf("no source position specified (-pos flag)")
		return
	}

	colon := strings.LastIndex(queryPos, ":")
	if colon < 0 {
		err = fmt.Errorf("invalid source position -pos=%q", queryPos)
		return
	}
	filename, offset := queryPos[:colon], queryPos[colon+1:]
	startOffset := -1
	endOffset := -1
	if hyphen := strings.Index(offset, ","); hyphen < 0 {
		// e.g. "foo.go:#123"
		startOffset = parseOctothorpDecimal(offset)
		endOffset = startOffset
	} else {
		// e.g. "foo.go:#123,#456"
		startOffset = parseOctothorpDecimal(offset[:hyphen])
		endOffset = parseOctothorpDecimal(offset[hyphen+1:])
	}
	if startOffset < 0 || endOffset < 0 {
		err = fmt.Errorf("invalid -pos offset %q", offset)
		return
	}

	var file *token.File
	fset.Iterate(func(f *token.File) bool {
		if sameFile(filename, f.Name()) {
			// (f.Name() is absolute)
			file = f
			return false // done
		}
		return true // continue
	})
	if file == nil {
		err = fmt.Errorf("couldn't find file containing position -pos=%q", queryPos)
		return
	}

	// Range check [start..end], inclusive of both end-points.

	if 0 <= startOffset && startOffset <= file.Size() {
		start = file.Pos(int(startOffset))
	} else {
		err = fmt.Errorf("start position is beyond end of file -pos=%q", queryPos)
		return
	}

	if 0 <= endOffset && endOffset <= file.Size() {
		end = file.Pos(int(endOffset))
	} else {
		err = fmt.Errorf("end position is beyond end of file -pos=%q", queryPos)
		return
	}

	return
}

// createCompileUnit creates and returns debug metadata for the compile
// unit as a whole, using the first file in the file set as a representative
// (the choice of file is arbitrary).
func (d *DIBuilder) createCompileUnit() llvm.Metadata {
	var file *token.File
	d.fset.Iterate(func(f *token.File) bool {
		file = f
		return false
	})
	dir, err := os.Getwd()
	if err != nil {
		panic("could not get current directory: " + err.Error())
	}
	return d.builder.CreateCompileUnit(llvm.DICompileUnit{
		Language: llvm.DW_LANG_Go,
		File:     d.remapFilePath(file.Name()),
		Dir:      dir,
		Producer: "llgo",
	})
}

// Init prepares the scanner S to tokenize the text src by setting the
// scanner at the beginning of src. The scanner uses the file set file
// for position information and it adds line information for each line.
// It is ok to re-use the same file when re-scanning the same file as
// line information which is already present is ignored. Init causes a
// panic if the file size does not match the src size.
//
// Calls to Scan will use the error handler err if they encounter a
// syntax error and err is not nil. Also, for each error encountered,
// the Scanner field ErrorCount is incremented by one. The mode parameter
// determines how comments, illegal characters, and semicolons are handled.
//
// Note that Init may call err if there is an error in the first character
// of the file.
//
func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint) {
	// Explicitly initialize all fields since a scanner may be reused.
	if file.Size() != len(src) {
		panic("file size does not match src len")
	}
	S.file = file
	S.dir, _ = filepath.Split(file.Name())
	S.src = src
	S.err = err
	S.mode = mode

	S.ch = ' '
	S.offset = 0
	S.rdOffset = 0
	S.lineOffset = 0
	S.insertSemi = false
	S.ErrorCount = 0

	S.next()
}

// Init prepares the scanner s to tokenize the text src by setting the
// scanner at the beginning of src. The scanner uses the file set file
// for position information and it adds line information for each line.
// It is ok to re-use the same file when re-scanning the same file as
// line information which is already present is ignored. Init causes a
// panic if the file size does not match the src size.
//
// Calls to Scan will invoke the error handler err if they encounter a
// syntax error and err is not nil. Also, for each error encountered,
// the Scanner field ErrorCount is incremented by one. The mode parameter
// determines how comments are handled.
//
// Note that Init may call err if there is an error in the first character
// of the file.
//
func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) {
	// Explicitly initialize all fields since a scanner may be reused.
	if file.Size() != len(src) {
		panic("file size does not match src len")
	}
	s.file = file
	s.dir, _ = filepath.Split(file.Name())
	s.src = src
	s.err = err
	s.mode = mode

	s.ch = ' '
	s.offset = 0
	s.rdOffset = 0
	s.lineOffset = 0
	s.insertSemi = false
	s.ErrorCount = 0

	s.next()
}

func (d *debugInfo) getCompileUnit(file *token.File) *debug.CompileUnitDescriptor {
	if d.cu == nil {
		d.cu = make(map[*token.File]*debug.CompileUnitDescriptor)
	}
	cu := d.cu[file]
	if cu == nil {
		var path string
		if file != nil {
			path = d.Fset.File(file.Pos(0)).Name()
		}
		cu = &debug.CompileUnitDescriptor{
			Language: debug.DW_LANG_Go,
			Path:     debug.FileDescriptor(path),
			Producer: "llgo",
			Runtime:  LLGORuntimeVersion,
		}
		d.cu[file] = cu
	}
	return cu
}

// fileOffsetToPos translates the specified file-relative byte offsets
// into token.Pos form.  It returns an error if the file was not found
// or the offsets were out of bounds.
//
func fileOffsetToPos(file *token.File, startOffset, endOffset int) (start, end token.Pos, err error) {
	// Range check [start..end], inclusive of both end-points.

	if 0 <= startOffset && startOffset <= file.Size() {
		start = file.Pos(int(startOffset))
	} else {
		err = fmt.Errorf("start position is beyond end of file")
		return
	}

	if 0 <= endOffset && endOffset <= file.Size() {
		end = file.Pos(int(endOffset))
	} else {
		err = fmt.Errorf("end position is beyond end of file")
		return
	}

	return
}

func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) {
	if file.Size() != len(src) {
		panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
	}

	s.tokScanner = scan.Scanner{Matcher: getTokenMatcher()}
	s.errScanner = scan.Scanner{Matcher: getErrorMatcher()}
	s.src = skipBOM(src)
	s.tokScanner.SetSource(s.src)
	s.errScanner.SetSource(s.src)

	s.file = file
	s.fileBase = s.file.Base()
	s.dir, _ = filepath.Split(file.Name())
	s.err = err
	s.mode = mode

	s.ErrorCount = 0

	s.preSemi = false
	s.semiPos = 0
}

func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) {
	//fmt.Println("Init src", strconv.Quote(string(src)), mode)

	if file.Size() != len(src) {
		panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
	}

	s.gombiScanner = newGombiScanner()
	s.SetSource(skipBOM(src))

	s.file = file
	s.dir, _ = filepath.Split(file.Name())
	s.err = err
	s.mode = mode

	s.ErrorCount = 0

	s.lastIsPreSemi = false
	s.commentAfterPreSemi = false
	s.endOfLinePos = 0
	s.endOfLine = 0
	s.commentQueue.reset()
}

// Init prepares the scanner s to tokenize the text src by setting the
// scanner at the beginning of src. The scanner uses the file set file
// for position information and it adds line information for each line.
// It is ok to re-use the same file when re-scanning the same file as
// line information which is already present is ignored. Init causes a
// panic if the file size does not match the src size.
//
// Calls to Scan will invoke the error handler err if they encounter a
// syntax error and err is not nil. Also, for each error encountered,
// the Scanner field ErrorCount is incremented by one. The mode parameter
// determines how comments are handled.
//
// Note that Init may call err if there is an error in the first character
// of the file.
//
func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) {
	// Explicitly initialize all fields since a scanner may be reused.
	if file.Size() != len(src) {
		panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
	}
	s.file = file
	s.dir, _ = filepath.Split(file.Name())
	s.src = src
	s.err = err
	s.mode = mode

	s.ch = ' '
	s.offset = 0
	s.rdOffset = 0
	s.lineOffset = 0
	s.insertSemi = false
	s.ErrorCount = 0

	s.next()
	if s.ch == bom {
		s.next() // ignore BOM at file beginning
	}
}

func linenum(f *token.File, p token.Pos) int32 {
	return int32(f.Line(p))
}

// TODO(adonovan): make this a method: func (*token.File) Contains(token.Pos)
func tokenFileContainsPos(f *token.File, pos token.Pos) bool {
	p := int(pos)
	base := f.Base()
	return base <= p && p < base+f.Size()
}

//.........这里部分代码省略.........
			if '<' == r {
				skipws()
				if done {
					panic("'<' lacks action")
				}
				x.code = fmt.Sprintf("yylex = yylex.push(%d)\n", familyn)
				nested = true
			}
			x.code += readCode()
			if nested {
				familyn++
				parse(familyn - 1)
			}
		}
		if 0 != family {
			panic("unmatched <")
		}
		x := newRule(family, -1)
		x.code = "// [END]\n"
		declvar()
	}
	parse(0)

	if !usercode {
		return
	}

	skipws()
	buf = buf[:0]
	for !done {
		buf = append(buf, r)
		read()
	}
	fs := token.NewFileSet()
	t, err := parser.ParseFile(fs, "", string(buf), parser.ImportsOnly)
	if err != nil {
		panic(err.Error())
	}
	printer.Fprint(out, fs, t)

	var file *token.File
	fs.Iterate(func(f *token.File) bool {
		file = f
		return true
	})

	for m := file.LineCount(); m > 1; m-- {
		i := 0
		for '\n' != buf[i] {
			i++
		}
		buf = buf[i+1:]
	}

	fmt.Fprintf(out, `import ("bufio";"io";"strings")
type dfa struct {
  acc []bool
  f []func(rune) int
  id int
}
type family struct {
  a []dfa
  endcase int
}
`)
	out.WriteString(decls)

func getRangeLinesAtLeastOne(f *token.File, Pos, End token.Pos, fileSize int) (lines []int, firstLineNum int) {
	lines = []int{}
	firstLineNum = -1

	l := f.Line(Pos)
	for p := Pos; p <= End; p++ {
		if f.Line(p) > l {
			l = f.Line(p)
			if firstLineNum == -1 {
				firstLineNum = l
			}
			lines = append(lines, f.Offset(p))
		}
	}
	if (int(End) == fileSize+f.Base()-1) || f.Line(End+1) > l {
		lines = append(lines, f.Offset(End+1))
		if firstLineNum == -1 {
			firstLineNum = f.Line(End + 1)
		}
	}
	if firstLineNum < 0 {
		for p := End; ; p++ {
			if f.Line(p) > l {
				firstLineNum = l
				lines = append(lines, f.Offset(p))
				break
			}
		}
	}
	return
}

//.........这里部分代码省略.........
			var regex []rune
			for {
				if r == delim && (len(regex) == 0 || regex[len(regex)-1] != '\\') {
					break
				}
				if '\n' == r {
					return ErrUnexpectedNewline
				}
				regex = append(regex, r)
				panicIf(read, ErrUnexpectedEOF)
			}
			if "" == string(regex) {
				break
			}
			panicIf(skipws, ErrUnexpectedEOF)
			x := new(rule)
			x.id = fmt.Sprintf("%d", lineno)
			node.kid = append(node.kid, x)
			x.regex = make([]rune, len(regex))
			copy(x.regex, regex)
			if '<' == r {
				panicIf(skipws, ErrUnexpectedEOF)
				x.startCode = readCode()
				parse(x)
			} else {
				x.code = readCode()
			}
		}
		return nil
	}
	err := parse(&root)
	if err != nil {
		return err
	}

	buf = nil
	for done := skipws(); !done; done = read() {
		buf = append(buf, r)
	}
	fs := token.NewFileSet()
	// Append a blank line to make things easier when there are only package and
	// import declarations.
	t, err := parser.ParseFile(fs, "", string(buf)+"\n", parser.ImportsOnly)
	if err != nil {
		panic(err)
	}
	printer.Fprint(out, fs, t)

	var file *token.File
	fs.Iterate(func(f *token.File) bool {
		file = f
		return true
	})

	// Skip over package and import declarations. This is why we appended a blank
	// line above.
	for m := file.LineCount(); m > 1; m-- {
		i := 0
		for '\n' != buf[i] {
			i++
		}
		buf = buf[i+1:]
	}

	prefixReplacer.WriteString(out, lexertext)

	for _, kid := range root.kid {
		gen(out, kid)
	}
	prefixReplacer.WriteString(out, lexeroutro)
	if !standalone {
		writeLex(out, root)
		out.WriteString(string(buf))
		out.Flush()
		if len(outFilename) > 0 {
			gofmt()
		}
		return nil
	}
	m := 0
	const funmac = "NN_FUN"
	for m < len(buf) {
		m++
		if funmac[:m] != string(buf[:m]) {
			out.WriteString(string(buf[:m]))
			buf = buf[m:]
			m = 0
		} else if funmac == string(buf[:m]) {
			writeNNFun(out, root)
			buf = buf[m:]
			m = 0
		}
	}
	out.WriteString(string(buf))
	out.Flush()
	if len(outFilename) > 0 {
		gofmt()
	}
	return nil
}

//.........这里部分代码省略.........
				if r == delim && (len(regex) == 0 || regex[len(regex)-1] != '\\') {
					break
				}
				if '\n' == r {
					panic(ErrUnexpectedNewline)
				}
				regex = append(regex, r)
				panicIf(read, ErrUnexpectedEOF)
			}
			if "" == string(regex) {
				break
			}
			panicIf(skipws, ErrUnexpectedEOF)
			x := new(rule)
			x.id = fmt.Sprintf("%d", lineno)
			node.kid = append(node.kid, x)
			x.regex = make([]rune, len(regex))
			copy(x.regex, regex)
			if '<' == r {
				panicIf(skipws, ErrUnexpectedEOF)
				x.startCode = readCode()
				parse(x)
			} else {
				x.code = readCode()
			}
		}
	}
	parse(&root)

	buf = nil
	for done := skipws(); !done; done = read() {
		buf = append(buf, r)
	}
	fs := token.NewFileSet()
	// Append a blank line to make things easier when there are only package and
	// import declarations.
	t, err := parser.ParseFile(fs, "", string(buf)+"\n", parser.ImportsOnly)
	if err != nil {
		panic(err)
	}
	printer.Fprint(out, fs, t)

	var file *token.File
	fs.Iterate(func(f *token.File) bool {
		file = f
		return true
	})

	// Skip over package and import declarations. This is why we appended a blank
	// line above.
	for m := file.LineCount(); m > 1; m-- {
		i := 0
		for '\n' != buf[i] {
			i++
		}
		buf = buf[i+1:]
	}

	out.WriteString(`import ("bufio";"io";"strings")
type frame struct {
  i int
  s string
  line, column int
}
type Lexer struct {
  // The lexer runs in its own goroutine, and communicates via channel 'ch'.

func GetRangeLines(f *token.File, Pos, End token.Pos, fileSize int) (lines []int, firstLineNum int) {
	lines = []int{}
	firstLineNum = -1

	l := f.Line(Pos)
	for p := Pos; p <= End; p++ {
		if f.Line(p) > l {
			l = f.Line(p)
			if firstLineNum == -1 {
				firstLineNum = l
			}
			lines = append(lines, f.Offset(p))
		}
	}
	print(End)
	print(" -> ")
	println(fileSize + f.Base() - 1)
	if (int(End) == fileSize+f.Base()-1) || f.Line(End+1) > l {
		lines = append(lines, f.Offset(End+1))
		if firstLineNum == -1 {
			firstLineNum = f.Line(End + 1)
		}
	}
	return
}