scanner: invalidate scanner.Position when no token is present

scanner.Position is the position of the most recently
scanned token. Make sure it is invalid if there is no
token scanned and update corresponding comment. This
is particularly important when reporting errors.

Fixes #2371.

R=r
CC=golang-dev
https://golang.org/cl/5294041

diff --git a/src/pkg/scanner/scanner.go b/src/pkg/scanner/scanner.go
index 8fbcb9c1155c74ed7d280410249f5eba91c80b90..29e5f8c742845d964f8c8dd307abbcb394708db2 100644 (file)
--- a/src/pkg/scanner/scanner.go
+++ b/src/pkg/scanner/scanner.go
@@ -164,9 +164,12 @@ type Scanner struct {
        // for values ch > ' '). The field may be changed at any time.
        Whitespace uint64
 
-       // Current token position. The Offset, Line, and Column fields
-       // are set by Scan(); the Filename field is left untouched by the
-       // Scanner.
+       // Start position of most recently scanned token; set by Scan.
+       // Calling Init or Next invalidates the position (Line == 0).
+       // The Filename field is always left untouched by the Scanner.
+       // If an error is reported (via Error) and Position is invalid,
+       // the scanner is not inside a token. Call Pos to obtain an error
+       // position in that case.
        Position
 }
 
@@ -201,6 +204,7 @@ func (s *Scanner) Init(src io.Reader) *Scanner {
        s.ErrorCount = 0
        s.Mode = GoTokens
        s.Whitespace = GoWhitespace
+       s.Line = 0 // invalidate token position
 
        return s
 }
@@ -302,6 +306,7 @@ func (s *Scanner) next() int {
 // get the current position.
 func (s *Scanner) Next() int {
        s.tokPos = -1 // don't collect token text
+       s.Line = 0    // invalidate token position
        ch := s.Peek()
        s.ch = s.next()
        return ch
@@ -323,7 +328,11 @@ func (s *Scanner) error(msg string) {
                s.Error(s, msg)
                return
        }
-       fmt.Fprintf(os.Stderr, "%s: %s\n", s.Position, msg)
+       pos := s.Position
+       if !pos.IsValid() {
+               pos = s.Pos()
+       }
+       fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg)
 }
 
 func (s *Scanner) scanIdentifier() int {
@@ -520,6 +529,7 @@ func (s *Scanner) Scan() int {
 
        // reset token text position
        s.tokPos = -1
+       s.Line = 0
 
 redo:
        // skip white space

diff --git a/src/pkg/scanner/scanner_test.go b/src/pkg/scanner/scanner_test.go
index 8403d6153542f086bfbbfa7ee37badfeb0be1672..bbbba128348c2b588fc7cc0c7b785fcf698f338d 100644 (file)
--- a/src/pkg/scanner/scanner_test.go
+++ b/src/pkg/scanner/scanner_test.go
@@ -100,11 +100,8 @@ var tokenList = []token{
        {Ident, "_abc_123_"},
        {Ident, "_äöü"},
        {Ident, "_本"},
-       // TODO for unknown reasons these fail when checking the literals
-       /*
-               token{Ident, "äöü"},
-               token{Ident, "本"},
-       */
+       {Ident, "äöü"},
+       {Ident, "本"},
        {Ident, "a۰۱۸"},
        {Ident, "foo६४"},
        {Ident, "bar９８７６"},
@@ -365,14 +362,14 @@ func TestScanNext(t *testing.T) {
        checkTok(t, s, 1, s.Scan(), Ident, "if")
        checkTok(t, s, 1, s.Scan(), Ident, "a")
        checkTok(t, s, 1, s.Scan(), '=', "=")
-       checkTok(t, s, 1, s.Next(), '=', "")
-       checkTok(t, s, 1, s.Next(), ' ', "")
-       checkTok(t, s, 1, s.Next(), 'b', "")
+       checkTok(t, s, 0, s.Next(), '=', "")
+       checkTok(t, s, 0, s.Next(), ' ', "")
+       checkTok(t, s, 0, s.Next(), 'b', "")
        checkTok(t, s, 1, s.Scan(), Ident, "cd")
        checkTok(t, s, 1, s.Scan(), '{', "{")
        checkTok(t, s, 2, s.Scan(), Ident, "a")
        checkTok(t, s, 2, s.Scan(), '+', "+")
-       checkTok(t, s, 2, s.Next(), '=', "")
+       checkTok(t, s, 0, s.Next(), '=', "")
        checkTok(t, s, 2, s.Scan(), Ident, "c")
        checkTok(t, s, 3, s.Scan(), '}', "}")
        checkTok(t, s, 3, s.Scan(), -1, "")