}
}
-func (S *Scanner) scanComment() {
+func (S *Scanner) scanComment() string {
// initial '/' already consumed; S.ch == '/' || S.ch == '*'
offs := S.offset - 1 // position of initial '/'
// comment starts at the beginning of the current line
S.interpretLineComment(S.src[offs:S.offset])
}
- return
+ goto exit
}
/*-style comment */
S.next()
if ch == '*' && S.ch == '/' {
S.next()
- return
+ goto exit
}
}
S.error(offs, "comment not terminated")
+
+exit:
+ return string(S.src[offs:S.offset])
}
func (S *Scanner) findLineEnd() bool {
return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
}
-func (S *Scanner) scanIdentifier() token.Token {
+func (S *Scanner) scanIdentifier() string {
offs := S.offset
for isLetter(S.ch) || isDigit(S.ch) {
S.next()
}
- return token.Lookup(S.src[offs:S.offset])
+ return string(S.src[offs:S.offset])
}
func digitVal(ch rune) int {
}
}
-func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token {
+func (S *Scanner) scanNumber(seenDecimalPoint bool) (token.Token, string) {
// digitVal(S.ch) < 10
+ offs := S.offset
tok := token.INT
if seenDecimalPoint {
+ offs--
tok = token.FLOAT
S.scanMantissa(10)
goto exponent
}
exit:
- return tok
+ return tok, string(S.src[offs:S.offset])
}
func (S *Scanner) scanEscape(quote rune) {
}
}
-func (S *Scanner) scanChar() {
+func (S *Scanner) scanChar() string {
// '\'' opening already consumed
offs := S.offset - 1
if n != 1 {
S.error(offs, "illegal character literal")
}
+
+ return string(S.src[offs:S.offset])
}
-func (S *Scanner) scanString() {
+func (S *Scanner) scanString() string {
// '"' opening already consumed
offs := S.offset - 1
}
S.next()
+
+ return string(S.src[offs:S.offset])
+}
+
+func stripCR(b []byte) []byte {
+ c := make([]byte, len(b))
+ i := 0
+ for _, ch := range b {
+ if ch != '\r' {
+ c[i] = ch
+ i++
+ }
+ }
+ return c[:i]
}
-func (S *Scanner) scanRawString() (hasCR bool) {
+func (S *Scanner) scanRawString() string {
// '`' opening already consumed
offs := S.offset - 1
+ hasCR := false
for S.ch != '`' {
ch := S.ch
S.next()
}
S.next()
- return
+
+ lit := S.src[offs:S.offset]
+ if hasCR {
+ lit = stripCR(lit)
+ }
+
+ return string(lit)
}
func (S *Scanner) skipWhitespace() {
return tok0
}
-func stripCR(b []byte) []byte {
- c := make([]byte, len(b))
- i := 0
- for _, ch := range b {
- if ch != '\r' {
- c[i] = ch
- i++
- }
- }
- return c[:i]
-}
-
-// Scan scans the next token and returns the token position,
-// the token, and the literal string corresponding to the
-// token. The source end is indicated by token.EOF.
+// Scan scans the next token and returns the token position, the token,
+// and its literal string if applicable. The source end is indicated by
+// token.EOF.
+//
+// If the returned token is a literal (token.IDENT, token.INT, token.FLOAT,
+// token.IMAG, token.CHAR, token.STRING) or token.COMMENT, the literal string
+// has the corresponding value.
//
// If the returned token is token.SEMICOLON, the corresponding
// literal string is ";" if the semicolon was present in the source,
// and "\n" if the semicolon was inserted because of a newline or
// at EOF.
//
+// If the returned token is token.ILLEGAL, the literal string is the
+// offending character.
+//
+// In all other cases, Scan returns an empty literal string.
+//
// For more tolerant parsing, Scan will return a valid token if
// possible even if a syntax error was encountered. Thus, even
// if the resulting token sequence contains no illegal tokens,
// set with Init. Token positions are relative to that file
// and thus relative to the file set.
//
-func (S *Scanner) Scan() (token.Pos, token.Token, string) {
+func (S *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) {
scanAgain:
S.skipWhitespace()
// current token start
- insertSemi := false
- offs := S.offset
- tok := token.ILLEGAL
- hasCR := false
+ pos = S.file.Pos(S.offset)
// determine token value
+ insertSemi := false
switch ch := S.ch; {
case isLetter(ch):
- tok = S.scanIdentifier()
+ lit = S.scanIdentifier()
+ tok = token.Lookup(lit)
switch tok {
case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
insertSemi = true
}
case digitVal(ch) < 10:
insertSemi = true
- tok = S.scanNumber(false)
+ tok, lit = S.scanNumber(false)
default:
S.next() // always make progress
switch ch {
case -1:
if S.insertSemi {
S.insertSemi = false // EOF consumed
- return S.file.Pos(offs), token.SEMICOLON, "\n"
+ return pos, token.SEMICOLON, "\n"
}
tok = token.EOF
case '\n':
// set in the first place and exited early
// from S.skipWhitespace()
S.insertSemi = false // newline consumed
- return S.file.Pos(offs), token.SEMICOLON, "\n"
+ return pos, token.SEMICOLON, "\n"
case '"':
insertSemi = true
tok = token.STRING
- S.scanString()
+ lit = S.scanString()
case '\'':
insertSemi = true
tok = token.CHAR
- S.scanChar()
+ lit = S.scanChar()
case '`':
insertSemi = true
tok = token.STRING
- hasCR = S.scanRawString()
+ lit = S.scanRawString()
case ':':
tok = S.switch2(token.COLON, token.DEFINE)
case '.':
if digitVal(S.ch) < 10 {
insertSemi = true
- tok = S.scanNumber(true)
+ tok, lit = S.scanNumber(true)
} else if S.ch == '.' {
S.next()
if S.ch == '.' {
tok = token.COMMA
case ';':
tok = token.SEMICOLON
+ lit = ";"
case '(':
tok = token.LPAREN
case ')':
if S.insertSemi && S.findLineEnd() {
// reset position to the beginning of the comment
S.ch = '/'
- S.offset = offs
- S.rdOffset = offs + 1
+ S.offset = S.file.Offset(pos)
+ S.rdOffset = S.offset + 1
S.insertSemi = false // newline consumed
- return S.file.Pos(offs), token.SEMICOLON, "\n"
+ return pos, token.SEMICOLON, "\n"
}
- S.scanComment()
+ lit = S.scanComment()
if S.mode&ScanComments == 0 {
// skip comment
S.insertSemi = false // newline consumed
case '|':
tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
default:
- S.error(offs, fmt.Sprintf("illegal character %#U", ch))
+ S.error(S.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch))
insertSemi = S.insertSemi // preserve insertSemi info
+ tok = token.ILLEGAL
+ lit = string(ch)
}
}
-
if S.mode&dontInsertSemis == 0 {
S.insertSemi = insertSemi
}
- // TODO(gri): The scanner API should change such that the literal string
- // is only valid if an actual literal was scanned. This will
- // permit a more efficient implementation.
- lit := S.src[offs:S.offset]
- if hasCR {
- lit = stripCR(lit)
- }
- return S.file.Pos(offs), tok, string(lit)
+ return
}
const whitespace = " \t \n\n\n" // to separate tokens
+var source = func() []byte {
+ var src []byte
+ for _, t := range tokens {
+ src = append(src, t.lit...)
+ src = append(src, whitespace...)
+ }
+ return src
+}()
+
type testErrorHandler struct {
t *testing.T
}
// Verify that calling Scan() provides the correct results.
func TestScan(t *testing.T) {
// make source
- var src string
- for _, e := range tokens {
- src += e.lit + whitespace
- }
- src_linecount := newlineCount(src)
+ src_linecount := newlineCount(string(source))
whitespace_linecount := newlineCount(whitespace)
// verify scan
var s Scanner
- s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments|dontInsertSemis)
+ s.Init(fset.AddFile("", fset.Base(), len(source)), source, &testErrorHandler{t}, ScanComments|dontInsertSemis)
index := 0
epos := token.Position{"", 0, 1, 1} // expected position
for {
pos, tok, lit := s.Scan()
+ if lit == "" {
+ // no literal value for non-literal tokens
+ lit = tok.String()
+ }
e := elt{token.EOF, "", special}
if index < len(tokens) {
e = tokens[index]
checkError(t, e.src, e.tok, e.pos, e.err)
}
}
+
+func BenchmarkScan(b *testing.B) {
+ b.StopTimer()
+ fset := token.NewFileSet()
+ file := fset.AddFile("", fset.Base(), len(source))
+ var s Scanner
+ b.StartTimer()
+ for i := b.N - 1; i >= 0; i-- {
+ s.Init(file, source, nil, ScanComments)
+ for {
+ _, tok, _ := s.Scan()
+ if tok == token.EOF {
+ break
+ }
+ }
+ }
+}