From: Joe Tsai Date: Fri, 25 Jul 2025 00:10:54 +0000 (-0700) Subject: encoding/json: fix truncated Token error regression in goexperiment.jsonv2 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=c76c3abc5426ab3d183514c834bcd7d6a653ae89;p=gostls13.git encoding/json: fix truncated Token error regression in goexperiment.jsonv2 The jsontext.Decoder.ReadToken method reports a non-EOF error, if the token stream is truncated and does not form a valid JSON value. In contrast, the v1 json.Decoder.Token method would report EOF so long as the input was a prefix of some valid JSON value. Modify json.Decoder.Token to preserve historical behavior. This only modifies code that is compiled in under goexperiment.jsonv2. Updates #69449 Fixes #74750 Change-Id: Ifd281c46f118f0e748076013fefc7659f77c56ed Reviewed-on: https://go-review.googlesource.com/c/go/+/689516 Reviewed-by: Damien Neil LUCI-TryBot-Result: Go LUCI Auto-Submit: Joseph Tsai Reviewed-by: Michael Knyszek --- diff --git a/src/encoding/json/stream_test.go b/src/encoding/json/stream_test.go index 478ee18291..9e5d48d39d 100644 --- a/src/encoding/json/stream_test.go +++ b/src/encoding/json/stream_test.go @@ -522,3 +522,38 @@ func TestHTTPDecoding(t *testing.T) { t.Errorf("Decode error:\n\tgot: %v\n\twant: io.EOF", err) } } + +func TestTokenTruncation(t *testing.T) { + tests := []struct { + in string + err error + }{ + {in: ``, err: io.EOF}, + {in: `{`, err: io.EOF}, + {in: `{"`, err: io.ErrUnexpectedEOF}, + {in: `{"k"`, err: io.EOF}, + {in: `{"k":`, err: io.EOF}, + {in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}}, + {in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}}, + {in: ` [0`, err: io.EOF}, + {in: `[0.`, err: io.ErrUnexpectedEOF}, + {in: `[0. `, err: &SyntaxError{"invalid character ' ' after decimal point in numeric literal", int64(len(`[0.`))}}, + {in: `[0,`, err: io.EOF}, + {in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}}, + {in: `n`, err: io.ErrUnexpectedEOF}, + {in: `nul`, err: io.ErrUnexpectedEOF}, + {in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal `))}}, + {in: `false`, err: io.EOF}, + } + for _, tt := range tests { + d := NewDecoder(strings.NewReader(tt.in)) + for i := 0; true; i++ { + if _, err := d.Token(); err != nil { + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err) + } + break + } + } + } +} diff --git a/src/encoding/json/v2_stream.go b/src/encoding/json/v2_stream.go index ccbef6077b..28e72c0a52 100644 --- a/src/encoding/json/v2_stream.go +++ b/src/encoding/json/v2_stream.go @@ -8,6 +8,7 @@ package json import ( "bytes" + "errors" "io" "encoding/json/jsontext" @@ -193,6 +194,16 @@ func (d Delim) String() string { func (dec *Decoder) Token() (Token, error) { tok, err := dec.dec.ReadToken() if err != nil { + // Historically, v1 would report just [io.EOF] if + // the stream is a prefix of a valid JSON value. + // It reports an unwrapped [io.ErrUnexpectedEOF] if + // truncated within a JSON token such as a literal, number, or string. + if errors.Is(err, io.ErrUnexpectedEOF) { + if len(bytes.Trim(dec.dec.UnreadBuffer(), " \r\n\t,:")) == 0 { + return nil, io.EOF + } + return nil, io.ErrUnexpectedEOF + } return nil, transformSyntacticError(err) } switch k := tok.Kind(); k { diff --git a/src/encoding/json/v2_stream_test.go b/src/encoding/json/v2_stream_test.go index 38eb6660a2..b8951f8205 100644 --- a/src/encoding/json/v2_stream_test.go +++ b/src/encoding/json/v2_stream_test.go @@ -502,3 +502,38 @@ func TestHTTPDecoding(t *testing.T) { t.Errorf("Decode error:\n\tgot: %v\n\twant: io.EOF", err) } } + +func TestTokenTruncation(t *testing.T) { + tests := []struct { + in string + err error + }{ + {in: ``, err: io.EOF}, + {in: `{`, err: io.EOF}, + {in: `{"`, err: io.ErrUnexpectedEOF}, + {in: `{"k"`, err: io.EOF}, + {in: `{"k":`, err: io.EOF}, + {in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}}, + {in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}}, + {in: ` [0`, err: io.EOF}, + {in: `[0.`, err: io.ErrUnexpectedEOF}, + {in: `[0. `, err: &SyntaxError{"invalid character ' ' in numeric literal", int64(len(`[0.`))}}, + {in: `[0,`, err: io.EOF}, + {in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}}, + {in: `n`, err: io.ErrUnexpectedEOF}, + {in: `nul`, err: io.ErrUnexpectedEOF}, + {in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal`))}}, + {in: `false`, err: io.EOF}, + } + for _, tt := range tests { + d := NewDecoder(strings.NewReader(tt.in)) + for i := 0; true; i++ { + if _, err := d.Token(); err != nil { + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err) + } + break + } + } + } +}