func firstSentence(s string) string {
i := -1 // index+1 of first terminator (punctuation ending a sentence)
j := -1 // index+1 of first terminator followed by white space
- prev := 'A'
+ prev := rune('A')
for k, ch := range s {
k1 := k + 1
if ch == '.' || ch == '!' || ch == '?' {
scanner scanner.Scanner
prev int // offset of previous token
pos int // offset of current token
- tok int // one token look-ahead
+ tok rune // one token look-ahead
lit string // token literal
}
p.printf(`<span class="highlight">error: expected %s, found %s</span>`, msg, scanner.TokenString(p.tok))
}
-func (p *ebnfParser) expect(tok int) {
+func (p *ebnfParser) expect(tok rune) {
if p.tok != tok {
p.errorExpected(scanner.TokenString(tok))
}
}
}
-func (v *verifier) verifyChar(x *Token) int {
+func (v *verifier) verifyChar(x *Token) rune {
s := x.String
if utf8.RuneCountInString(s) != 1 {
v.error(x.Pos(), "single char expected, found "+s)
errors errorList
scanner scanner.Scanner
pos scanner.Position // token position
- tok int // one token look-ahead
+ tok rune // one token look-ahead
lit string // token literal
}
p.error(pos, msg)
}
-func (p *parser) expect(tok int) scanner.Position {
+func (p *parser) expect(tok rune) scanner.Position {
pos := p.pos
if p.tok != tok {
p.errorExpected(pos, scanner.TokenString(tok))
// object/archive file and populates its scope with the results.
type gcParser struct {
scanner scanner.Scanner
- tok int // current token
+ tok rune // current token
lit string // literal string; only valid for Ident, Int, String tokens
id string // package id of imported package
imports map[string]*ast.Object // package id -> package object
p.error(fmt.Sprintf(format, args...))
}
-func (p *gcParser) expect(tok int) string {
+func (p *gcParser) expect(tok rune) string {
lit := p.lit
if p.tok != tok {
p.errorf("expected %q, got %q (%q)", scanner.TokenString(tok), scanner.TokenString(p.tok), lit)
}
func (p *gcParser) expectSpecial(tok string) {
- sep := 'x' // not white space
+ sep := rune('x') // not white space
i := 0
- for i < len(tok) && p.tok == int(tok[i]) && sep > ' ' {
+ for i < len(tok) && p.tok == rune(tok[i]) && sep > ' ' {
sep = p.scanner.Peek() // if sep <= ' ', there is white space before the next token
p.next()
i++
func (p *gcParser) parseDotIdent() string {
ident := ""
if p.tok != scanner.Int {
- sep := 'x' // not white space
+ sep := rune('x') // not white space
for (p.tok == scanner.Ident || p.tok == scanner.Int || p.tok == 'ยท') && sep > ' ' {
ident += p.lit
sep = p.scanner.Peek() // if sep <= ' ', there is white space before the next token
mode uint // scanning mode
// scanning state
- ch int // current character
+ ch rune // current character
offset int // character offset
rdOffset int // reading offset (position after current character)
lineOffset int // current line offset
S.lineOffset = S.offset
S.file.AddLine(S.offset)
}
- r, w := int(S.src[S.rdOffset]), 1
+ r, w := rune(S.src[S.rdOffset]), 1
switch {
case r == 0:
S.error(S.offset, "illegal character NUL")
return false
}
-func isLetter(ch int) bool {
+func isLetter(ch rune) bool {
return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch)
}
-func isDigit(ch int) bool {
+func isDigit(ch rune) bool {
return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
}
return token.Lookup(S.src[offs:S.offset])
}
-func digitVal(ch int) int {
+func digitVal(ch rune) int {
switch {
case '0' <= ch && ch <= '9':
- return ch - '0'
+ return int(ch - '0')
case 'a' <= ch && ch <= 'f':
- return ch - 'a' + 10
+ return int(ch - 'a' + 10)
case 'A' <= ch && ch <= 'F':
- return ch - 'A' + 10
+ return int(ch - 'A' + 10)
}
return 16 // larger than any legal digit val
}
return tok
}
-func (S *Scanner) scanEscape(quote int) {
+func (S *Scanner) scanEscape(quote rune) {
offs := S.offset
var i, base, max uint32
return tok0
}
-func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token {
+func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token {
if S.ch == '=' {
S.next()
return tok1
return tok0
}
-func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token {
+func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token {
if S.ch == '=' {
S.next()
return tok1
skipComment
)
-var tokenString = map[int]string{
+var tokenString = map[rune]string{
EOF: "EOF",
Ident: "Ident",
Int: "Int",
}
// TokenString returns a (visible) string for a token or Unicode character.
-func TokenString(tok int) string {
+func TokenString(tok rune) string {
if s, found := tokenString[tok]; found {
return s
}
tokEnd int // token text tail end (srcBuf index)
// One character look-ahead
- ch int // character before current srcPos
+ ch rune // character before current srcPos
// Error is called for each error encountered. If no Error
// function is set, the error is reported to os.Stderr.
// that only a minimal amount of work needs to be done in the common ASCII
// case (one test to check for both ASCII and end-of-buffer, and one test
// to check for newlines).
-func (s *Scanner) next() int {
- ch, width := int(s.srcBuf[s.srcPos]), 1
+func (s *Scanner) next() rune {
+ ch, width := rune(s.srcBuf[s.srcPos]), 1
if ch >= utf8.RuneSelf {
// uncommon case: not ASCII or not enough bytes
}
}
// at least one byte
- ch = int(s.srcBuf[s.srcPos])
+ ch = rune(s.srcBuf[s.srcPos])
if ch >= utf8.RuneSelf {
// uncommon case: not ASCII
ch, width = utf8.DecodeRune(s.srcBuf[s.srcPos:s.srcEnd])
// it prints an error message to os.Stderr. Next does not
// update the Scanner's Position field; use Pos() to
// get the current position.
-func (s *Scanner) Next() int {
+func (s *Scanner) Next() rune {
s.tokPos = -1 // don't collect token text
s.Line = 0 // invalidate token position
ch := s.Peek()
// Peek returns the next Unicode character in the source without advancing
// the scanner. It returns EOF if the scanner's position is at the last
// character of the source.
-func (s *Scanner) Peek() int {
+func (s *Scanner) Peek() rune {
if s.ch < 0 {
s.ch = s.next()
}
fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg)
}
-func (s *Scanner) scanIdentifier() int {
+func (s *Scanner) scanIdentifier() rune {
ch := s.next() // read character after first '_' or letter
for ch == '_' || unicode.IsLetter(ch) || unicode.IsDigit(ch) {
ch = s.next()
return ch
}
-func digitVal(ch int) int {
+func digitVal(ch rune) int {
switch {
case '0' <= ch && ch <= '9':
- return ch - '0'
+ return int(ch - '0')
case 'a' <= ch && ch <= 'f':
- return ch - 'a' + 10
+ return int(ch - 'a' + 10)
case 'A' <= ch && ch <= 'F':
- return ch - 'A' + 10
+ return int(ch - 'A' + 10)
}
return 16 // larger than any legal digit val
}
-func isDecimal(ch int) bool { return '0' <= ch && ch <= '9' }
+func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
-func (s *Scanner) scanMantissa(ch int) int {
+func (s *Scanner) scanMantissa(ch rune) rune {
for isDecimal(ch) {
ch = s.next()
}
return ch
}
-func (s *Scanner) scanFraction(ch int) int {
+func (s *Scanner) scanFraction(ch rune) rune {
if ch == '.' {
ch = s.scanMantissa(s.next())
}
return ch
}
-func (s *Scanner) scanExponent(ch int) int {
+func (s *Scanner) scanExponent(ch rune) rune {
if ch == 'e' || ch == 'E' {
ch = s.next()
if ch == '-' || ch == '+' {
return ch
}
-func (s *Scanner) scanNumber(ch int) (int, int) {
+func (s *Scanner) scanNumber(ch rune) (rune, rune) {
// isDecimal(ch)
if ch == '0' {
// int or float
return Int, ch
}
-func (s *Scanner) scanDigits(ch, base, n int) int {
+func (s *Scanner) scanDigits(ch rune, base, n int) rune {
for n > 0 && digitVal(ch) < base {
ch = s.next()
n--
return ch
}
-func (s *Scanner) scanEscape(quote int) int {
+func (s *Scanner) scanEscape(quote rune) rune {
ch := s.next() // read character after '/'
switch ch {
case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
return ch
}
-func (s *Scanner) scanString(quote int) (n int) {
+func (s *Scanner) scanString(quote rune) (n int) {
ch := s.next() // read character after quote
for ch != quote {
if ch == '\n' || ch < 0 {
}
}
-func (s *Scanner) scanComment(ch int) int {
+func (s *Scanner) scanComment(ch rune) rune {
// ch == '/' || ch == '*'
if ch == '/' {
// line comment
// It returns EOF at the end of the source. It reports scanner errors (read and
// token errors) by calling s.Error, if not nil; otherwise it prints an error
// message to os.Stderr.
-func (s *Scanner) Scan() int {
+func (s *Scanner) Scan() rune {
ch := s.Peek()
// reset token text position
}
type token struct {
- tok int
+ tok rune
text string
}
return &buf
}
-func checkTok(t *testing.T, s *Scanner, line, got, want int, text string) {
+func checkTok(t *testing.T, s *Scanner, line int, got, want rune, text string) {
if got != want {
t.Fatalf("tok = %s, want %s for %q", TokenString(got), TokenString(want), text)
}
}
}
-func testScanSelectedMode(t *testing.T, mode uint, class int) {
+func testScanSelectedMode(t *testing.T, mode uint, class rune) {
src := makeSource("%s\n")
s := new(Scanner).Init(src)
s.Mode = mode
}
}
-func testError(t *testing.T, src, pos, msg string, tok int) {
+func testError(t *testing.T, src, pos, msg string, tok rune) {
s := new(Scanner).Init(bytes.NewBufferString(src))
errorCalled := false
s.Error = func(s *Scanner, m string) {
}
}
-func checkNextPos(t *testing.T, s *Scanner, offset, line, column, char int) {
+func checkNextPos(t *testing.T, s *Scanner, offset, line, column int, char rune) {
if ch := s.Next(); ch != char {
t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))
}
checkPos(t, s.Pos(), want)
}
-func checkScanPos(t *testing.T, s *Scanner, offset, line, column, char int) {
+func checkScanPos(t *testing.T, s *Scanner, offset, line, column int, char rune) {
want := Position{Offset: offset, Line: line, Column: column}
checkPos(t, s.Pos(), want)
if ch := s.Scan(); ch != char {