encoding/json: fix truncated Token error regression in goexperiment.jsonv2

The jsontext.Decoder.ReadToken method reports a non-EOF error,
if the token stream is truncated and does not form a valid JSON value.
In contrast, the v1 json.Decoder.Token method would report EOF
so long as the input was a prefix of some valid JSON value.
Modify json.Decoder.Token to preserve historical behavior.

This only modifies code that is compiled in under goexperiment.jsonv2.

Updates #69449
Fixes #74750

Change-Id: Ifd281c46f118f0e748076013fefc7659f77c56ed
Reviewed-on: https://go-review.googlesource.com/c/go/+/689516
Reviewed-by: Damien Neil <dneil@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Joseph Tsai <joetsai@digital-static.net>
Reviewed-by: Michael Knyszek <mknyszek@google.com>

diff --git a/src/encoding/json/stream_test.go b/src/encoding/json/stream_test.go
index 478ee1829195d68ce5ab0d8bd5eee01e3cecb757..9e5d48d39d2fcfe51862e871bff4ad4f05e9217a 100644 (file)
--- a/src/encoding/json/stream_test.go
+++ b/src/encoding/json/stream_test.go
@@ -522,3 +522,38 @@ func TestHTTPDecoding(t *testing.T) {
                t.Errorf("Decode error:\n\tgot:  %v\n\twant: io.EOF", err)
        }
 }
+
+func TestTokenTruncation(t *testing.T) {
+       tests := []struct {
+               in  string
+               err error
+       }{
+               {in: ``, err: io.EOF},
+               {in: `{`, err: io.EOF},
+               {in: `{"`, err: io.ErrUnexpectedEOF},
+               {in: `{"k"`, err: io.EOF},
+               {in: `{"k":`, err: io.EOF},
+               {in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}},
+               {in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}},
+               {in: ` [0`, err: io.EOF},
+               {in: `[0.`, err: io.ErrUnexpectedEOF},
+               {in: `[0. `, err: &SyntaxError{"invalid character ' ' after decimal point in numeric literal", int64(len(`[0.`))}},
+               {in: `[0,`, err: io.EOF},
+               {in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}},
+               {in: `n`, err: io.ErrUnexpectedEOF},
+               {in: `nul`, err: io.ErrUnexpectedEOF},
+               {in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal `))}},
+               {in: `false`, err: io.EOF},
+       }
+       for _, tt := range tests {
+               d := NewDecoder(strings.NewReader(tt.in))
+               for i := 0; true; i++ {
+                       if _, err := d.Token(); err != nil {
+                               if !reflect.DeepEqual(err, tt.err) {
+                                       t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err)
+                               }
+                               break
+                       }
+               }
+       }
+}

diff --git a/src/encoding/json/v2_stream.go b/src/encoding/json/v2_stream.go
index ccbef6077bbbfb2f89d2d14b504ac3e243413639..28e72c0a529e150bb089ebe45d37076d00f72c8b 100644 (file)
--- a/src/encoding/json/v2_stream.go
+++ b/src/encoding/json/v2_stream.go
@@ -8,6 +8,7 @@ package json
 
 import (
        "bytes"
+       "errors"
        "io"
 
        "encoding/json/jsontext"
@@ -193,6 +194,16 @@ func (d Delim) String() string {
 func (dec *Decoder) Token() (Token, error) {
        tok, err := dec.dec.ReadToken()
        if err != nil {
+               // Historically, v1 would report just [io.EOF] if
+               // the stream is a prefix of a valid JSON value.
+               // It reports an unwrapped [io.ErrUnexpectedEOF] if
+               // truncated within a JSON token such as a literal, number, or string.
+               if errors.Is(err, io.ErrUnexpectedEOF) {
+                       if len(bytes.Trim(dec.dec.UnreadBuffer(), " \r\n\t,:")) == 0 {
+                               return nil, io.EOF
+                       }
+                       return nil, io.ErrUnexpectedEOF
+               }
                return nil, transformSyntacticError(err)
        }
        switch k := tok.Kind(); k {

diff --git a/src/encoding/json/v2_stream_test.go b/src/encoding/json/v2_stream_test.go
index 38eb6660a282b6e86488032ed437fa049fe24406..b8951f82054e96fad569b2661940b0f5ea8bd1b3 100644 (file)
--- a/src/encoding/json/v2_stream_test.go
+++ b/src/encoding/json/v2_stream_test.go
@@ -502,3 +502,38 @@ func TestHTTPDecoding(t *testing.T) {
                t.Errorf("Decode error:\n\tgot:  %v\n\twant: io.EOF", err)
        }
 }
+
+func TestTokenTruncation(t *testing.T) {
+       tests := []struct {
+               in  string
+               err error
+       }{
+               {in: ``, err: io.EOF},
+               {in: `{`, err: io.EOF},
+               {in: `{"`, err: io.ErrUnexpectedEOF},
+               {in: `{"k"`, err: io.EOF},
+               {in: `{"k":`, err: io.EOF},
+               {in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}},
+               {in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}},
+               {in: ` [0`, err: io.EOF},
+               {in: `[0.`, err: io.ErrUnexpectedEOF},
+               {in: `[0. `, err: &SyntaxError{"invalid character ' ' in numeric literal", int64(len(`[0.`))}},
+               {in: `[0,`, err: io.EOF},
+               {in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}},
+               {in: `n`, err: io.ErrUnexpectedEOF},
+               {in: `nul`, err: io.ErrUnexpectedEOF},
+               {in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal`))}},
+               {in: `false`, err: io.EOF},
+       }
+       for _, tt := range tests {
+               d := NewDecoder(strings.NewReader(tt.in))
+               for i := 0; true; i++ {
+                       if _, err := d.Token(); err != nil {
+                               if !reflect.DeepEqual(err, tt.err) {
+                                       t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err)
+                               }
+                               break
+                       }
+               }
+       }
+}