text/scanner: don't crash when calling TokenText in error handler

Make sure Scanner.tokEnd is set before we call Scanner.Error
and update documentation accordingly.
(Until now tokEnd was only set before returning from Scan,
so a call to TokenText during error handling may have crashed.)

While at it, tighten a check in Scanner.TokenText to ensure
Scanner.tokEnd >= Scanner.tokPos if we have a token.

Also, silence error messages to Stderr in unrelated TestIllegalExponent.

Fixes #29723.

Change-Id: Ia97beeae91eaf9e0ed3dada0a806f1f7122461cc
Reviewed-on: https://go-review.googlesource.com/c/157819
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>

diff --git a/src/text/scanner/scanner.go b/src/text/scanner/scanner.go
index 893a4edbaf92f1bd5b5e070b98075ed16f17740f..62b3231e5e5e41638842eb038e174771e8cf15b5 100644 (file)
--- a/src/text/scanner/scanner.go
+++ b/src/text/scanner/scanner.go
@@ -322,6 +322,7 @@ func (s *Scanner) Peek() rune {
 }
 
 func (s *Scanner) error(msg string) {
+       s.tokEnd = s.srcPos - s.lastCharLen // make sure token text is terminated
        s.ErrorCount++
        if s.Error != nil {
                s.Error(s, msg)
@@ -664,17 +665,18 @@ func (s *Scanner) Pos() (pos Position) {
 }
 
 // TokenText returns the string corresponding to the most recently scanned token.
-// Valid after calling Scan().
+// Valid after calling Scan and in calls of Scanner.Error.
 func (s *Scanner) TokenText() string {
        if s.tokPos < 0 {
                // no token text
                return ""
        }
 
-       if s.tokEnd < 0 {
+       if s.tokEnd < s.tokPos {
                // if EOF was reached, s.tokEnd is set to -1 (s.srcPos == 0)
                s.tokEnd = s.tokPos
        }
+       // s.tokEnd >= s.tokPos
 
        if s.tokBuf.Len() == 0 {
                // common case: the entire token text is still in srcBuf

diff --git a/src/text/scanner/scanner_test.go b/src/text/scanner/scanner_test.go
index e26e816f519894ae0cb6d6d25cca25da2430122a..e7539a058bd5e93a51250fa7c3aa806f7d245953 100644 (file)
--- a/src/text/scanner/scanner_test.go
+++ b/src/text/scanner/scanner_test.go
@@ -293,6 +293,12 @@ func TestScan(t *testing.T) {
 func TestIllegalExponent(t *testing.T) {
        const src = "1.5e 1.5E 1e+ 1e- 1.5z"
        s := new(Scanner).Init(strings.NewReader(src))
+       s.Error = func(s *Scanner, msg string) {
+               const want = "illegal exponent"
+               if msg != want {
+                       t.Errorf("%s: got error %q; want %q", s.TokenText(), msg, want)
+               }
+       }
        checkTokErr(t, s, 1, Float, "1.5e")
        checkTokErr(t, s, 1, Float, "1.5E")
        checkTokErr(t, s, 1, Float, "1e+")
@@ -692,3 +698,16 @@ func TestScanEOFHandling(t *testing.T) {
                t.Errorf("scanner called Read %d times, not once", r)
        }
 }
+
+func TestIssue29723(t *testing.T) {
+       s := new(Scanner).Init(strings.NewReader(`x "`))
+       s.Error = func(s *Scanner, _ string) {
+               got := s.TokenText() // this call shouldn't panic
+               const want = `"`
+               if got != want {
+                       t.Errorf("got %q; want %q", got, want)
+               }
+       }
+       for r := s.Scan(); r != EOF; r = s.Scan() {
+       }
+}