bufio: fix handling of empty tokens at end of line/file

Fixes #8672.

LGTM=rsc
R=golang-codereviews, rsc
CC=golang-codereviews
https://golang.org/cl/145390043

diff --git a/src/bufio/scan.go b/src/bufio/scan.go
index c5714f331ef52b5e25812d1f4b48eade42325cd8..a41451524db0afae49bb5b5dddc5d0a2db5d7f6e 100644 (file)
--- a/src/bufio/scan.go
+++ b/src/bufio/scan.go
@@ -112,7 +112,9 @@ func (s *Scanner) Scan() bool {
        // Loop until we have a token.
        for {
                // See if we can get a token with what we already have.
-               if s.end > s.start {
+               // If we've run out of data but have an error, give the split function
+               // a chance to recover any remaining, possibly empty token.
+               if s.end > s.start || s.err != nil {
                        advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
                        if err != nil {
                                s.setErr(err)

diff --git a/src/bufio/scan_test.go b/src/bufio/scan_test.go
index 3ddb25acf922a26290d8c4b36cda51fcfa13296f..1454a8113c6f1ad20dd9681d713438e53e6d50d7 100644 (file)
--- a/src/bufio/scan_test.go
+++ b/src/bufio/scan_test.go
@@ -419,3 +419,39 @@ func TestScanWordsExcessiveWhiteSpace(t *testing.T) {
                t.Fatalf("unexpected token: %v", token)
        }
 }
+
+// Test that empty tokens, including at end of line or end of file, are found by the scanner.
+// Issue 8672: Could miss final empty token.
+
+func commaSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
+       for i := 0; i < len(data); i++ {
+               if data[i] == ',' {
+                       return i + 1, data[:i], nil
+               }
+       }
+       if !atEOF {
+               return 0, nil, nil
+       }
+       return 0, data, nil
+}
+
+func TestEmptyTokens(t *testing.T) {
+       s := NewScanner(strings.NewReader("1,2,3,"))
+       values := []string{"1", "2", "3", ""}
+       s.Split(commaSplit)
+       var i int
+       for i = 0; i < len(values); i++ {
+               if !s.Scan() {
+                       break
+               }
+               if s.Text() != values[i] {
+                       t.Errorf("%d: expected %q got %q", i, values[i], s.Text())
+               }
+       }
+       if i != len(values) {
+               t.Errorf("got %d fields, expected %d", i, len(values))
+       }
+       if err := s.Err(); err != nil {
+               t.Fatal(err)
+       }
+}