go/scanner: fall back to next() when encountering 0 bytes in parseIdentifier

CL 308611 optimized parseIdentifier for ASCII, but inadvertently skipped
error handling for 0 bytes. Don't take the optimized path when
encountering 0.

Fixes #46855

Change-Id: Ic584e077eb74c012611fefa20eb71ca09c81b3c7
Reviewed-on: https://go-review.googlesource.com/c/go/+/329790
Trust: Robert Findley <rfindley@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

diff --git a/src/go/scanner/scanner.go b/src/go/scanner/scanner.go
index 29cbf39721f6d2746142c5728fe1dfd6c1ea387f..f08e28cdd6b72b35da5b669a0592b88c6e012c53 100644 (file)
--- a/src/go/scanner/scanner.go
+++ b/src/go/scanner/scanner.go
@@ -373,7 +373,7 @@ func (s *Scanner) scanIdentifier() string {
                        continue
                }
                s.rdOffset += rdOffset
-               if b < utf8.RuneSelf {
+               if 0 < b && b < utf8.RuneSelf {
                        // Optimization: we've encountered an ASCII character that's not a letter
                        // or number. Avoid the call into s.next() and corresponding set up.
                        //

diff --git a/src/go/scanner/scanner_test.go b/src/go/scanner/scanner_test.go
index ac8d2577169bc94b41114fe256fa85dafd4db39e..db123c32e01d59e9d46fa6429efcefb9fc60bdff 100644 (file)
--- a/src/go/scanner/scanner_test.go
+++ b/src/go/scanner/scanner_test.go
@@ -812,6 +812,8 @@ var errors = []struct {
        {"//\ufeff", token.COMMENT, 2, "//\ufeff", "illegal byte order mark"},                                // only first BOM is ignored
        {"'\ufeff" + `'`, token.CHAR, 1, "'\ufeff" + `'`, "illegal byte order mark"},                         // only first BOM is ignored
        {`"` + "abc\ufeffdef" + `"`, token.STRING, 4, `"` + "abc\ufeffdef" + `"`, "illegal byte order mark"}, // only first BOM is ignored
+       {"abc\x00def", token.IDENT, 3, "abc", "illegal character NUL"},
+       {"abc\x00", token.IDENT, 3, "abc", "illegal character NUL"},
 }
 
 func TestScanErrors(t *testing.T) {