encoding/json/v2: report EOF for top-level values in UnmarshalDecode

The fully streaming UnmarshalJSONFrom method and UnmarshalFromFunc
introduce an edge case where they can encounter EOF in the stream,
where it should be reported upstream as EOF rather than
ErrUnexpectedEOF or be wrapped within a SemanticError.

This is not possible with other unmarshal methods since the
"json" package would read the appropriate JSON value
before calling the custom method or function.

To avoid custom unmarshal methods from encountering EOF,
check whether the stream is already at EOF for top-level values
before calling the custom method.

Also, when wrapping EOF within a SemanticError, convert it
to ErrUnexpectedEOF to better indicate that this is unexpected.

Fixes #75802

Change-Id: I001396734b7e95b5337f77b71326284974ee730a
Reviewed-on: https://go-review.googlesource.com/c/go/+/710877
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Johan Brandhorst-Satzkorn <johan.brandhorst@gmail.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
Reviewed-by: Carlos Amedee <carlos@golang.org>

diff --git a/src/encoding/json/jsontext/decode.go b/src/encoding/json/jsontext/decode.go
index f505de44684a51c06a25fbe935c66fb7169eea59..511832f2ae087970600a4d30d1e5d3e408ad50aa 100644 (file)
--- a/src/encoding/json/jsontext/decode.go
+++ b/src/encoding/json/jsontext/decode.go
@@ -792,6 +792,12 @@ func (d *decoderState) CheckNextValue(last bool) error {
        return nil
 }
 
+// AtEOF reports whether the decoder is at EOF.
+func (d *decoderState) AtEOF() bool {
+       _, err := d.consumeWhitespace(d.prevEnd)
+       return err == io.ErrUnexpectedEOF
+}
+
 // CheckEOF verifies that the input has no more data.
 func (d *decoderState) CheckEOF() error {
        return d.checkEOF(d.prevEnd)

diff --git a/src/encoding/json/v2/arshal.go b/src/encoding/json/v2/arshal.go
index 6b4bcb0c74cf7c9c56099ccc63156c0c66f8f9a6..573d26567f3368edad8e8c9dfbda31f214db5d54 100644 (file)
--- a/src/encoding/json/v2/arshal.go
+++ b/src/encoding/json/v2/arshal.go
@@ -440,8 +440,9 @@ func UnmarshalRead(in io.Reader, out any, opts ...Options) (err error) {
 // Unlike [Unmarshal] and [UnmarshalRead], decode options are ignored because
 // they must have already been specified on the provided [jsontext.Decoder].
 //
-// The input may be a stream of one or more JSON values,
+// The input may be a stream of zero or more JSON values,
 // where this only unmarshals the next JSON value in the stream.
+// If there are no more top-level JSON values, it reports [io.EOF].
 // The output must be a non-nil pointer.
 // See [Unmarshal] for details about the conversion of JSON into a Go value.
 func UnmarshalDecode(in *jsontext.Decoder, out any, opts ...Options) (err error) {

diff --git a/src/encoding/json/v2/arshal_funcs.go b/src/encoding/json/v2/arshal_funcs.go
index 673caf3c37693d8d571d809481300fbe571960dd..28916af948db6e8190e74b066fa09411a0c671b9 100644 (file)
--- a/src/encoding/json/v2/arshal_funcs.go
+++ b/src/encoding/json/v2/arshal_funcs.go
@@ -9,6 +9,7 @@ package json
 import (
        "errors"
        "fmt"
+       "io"
        "reflect"
        "sync"
 
@@ -306,6 +307,9 @@ func UnmarshalFromFunc[T any](fn func(*jsontext.Decoder, T) error) *Unmarshalers
                fnc: func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error {
                        xd := export.Decoder(dec)
                        prevDepth, prevLength := xd.Tokens.DepthLength()
+                       if prevDepth == 1 && xd.AtEOF() {
+                               return io.EOF // check EOF early to avoid fn reporting an EOF
+                       }
                        xd.Flags.Set(jsonflags.WithinArshalCall | 1)
                        v, _ := reflect.TypeAssert[T](va.castTo(t))
                        err := fn(dec, v)

diff --git a/src/encoding/json/v2/arshal_methods.go b/src/encoding/json/v2/arshal_methods.go
index 2decd144dbeaf9651dd02afaa30e677768ea7cb4..1621eadc0807631750e261fefc71d91fca972b01 100644 (file)
--- a/src/encoding/json/v2/arshal_methods.go
+++ b/src/encoding/json/v2/arshal_methods.go
@@ -9,6 +9,7 @@ package json
 import (
        "encoding"
        "errors"
+       "io"
        "reflect"
 
        "encoding/json/internal"
@@ -302,6 +303,9 @@ func makeMethodArshaler(fncs *arshaler, t reflect.Type) *arshaler {
                        }
                        xd := export.Decoder(dec)
                        prevDepth, prevLength := xd.Tokens.DepthLength()
+                       if prevDepth == 1 && xd.AtEOF() {
+                               return io.EOF // check EOF early to avoid fn reporting an EOF
+                       }
                        xd.Flags.Set(jsonflags.WithinArshalCall | 1)
                        unmarshaler, _ := reflect.TypeAssert[UnmarshalerFrom](va.Addr())
                        err := unmarshaler.UnmarshalJSONFrom(dec)

diff --git a/src/encoding/json/v2/arshal_test.go b/src/encoding/json/v2/arshal_test.go
index 75093345a3b93eb6f6a41cfec64750bf78d3c74d..dc15c5a5f531b53b468dedb992d75f1a01769aaa 100644 (file)
--- a/src/encoding/json/v2/arshal_test.go
+++ b/src/encoding/json/v2/arshal_test.go
@@ -7833,7 +7833,8 @@ func TestUnmarshal(t *testing.T) {
                })),
                wantErr: EU(errSomeError).withType(0, T[unmarshalJSONv2Func]()),
        }, {
-               name: jsontest.Name("Methods/Invalid/JSONv2/TooFew"),
+               name:  jsontest.Name("Methods/Invalid/JSONv2/TooFew"),
+               inBuf: `{}`,
                inVal: addr(unmarshalJSONv2Func(func(*jsontext.Decoder) error {
                        return nil // do nothing
                })),
@@ -9234,6 +9235,43 @@ func TestUnmarshalReuse(t *testing.T) {
        })
 }
 
+type unmarshalerEOF struct{}
+
+func (unmarshalerEOF) UnmarshalJSONFrom(dec *jsontext.Decoder) error {
+       return io.EOF // should be wrapped and converted by Unmarshal to io.ErrUnexpectedEOF
+}
+
+// TestUnmarshalEOF verifies that io.EOF is only ever returned by
+// UnmarshalDecode for a top-level value.
+func TestUnmarshalEOF(t *testing.T) {
+       opts := WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, _ *struct{}) error {
+               return io.EOF // should be wrapped and converted by Unmarshal to io.ErrUnexpectedEOF
+       }))
+
+       for _, in := range []string{"", "[", "[null", "[null]"} {
+               for _, newOut := range []func() any{
+                       func() any { return new(unmarshalerEOF) },
+                       func() any { return new([]unmarshalerEOF) },
+                       func() any { return new(struct{}) },
+                       func() any { return new([]struct{}) },
+               } {
+                       wantErr := io.ErrUnexpectedEOF
+                       if gotErr := Unmarshal([]byte(in), newOut(), opts); !errors.Is(gotErr, wantErr) {
+                               t.Errorf("Unmarshal = %v, want %v", gotErr, wantErr)
+                       }
+                       if gotErr := UnmarshalRead(strings.NewReader(in), newOut(), opts); !errors.Is(gotErr, wantErr) {
+                               t.Errorf("Unmarshal = %v, want %v", gotErr, wantErr)
+                       }
+                       switch gotErr := UnmarshalDecode(jsontext.NewDecoder(strings.NewReader(in)), newOut(), opts); {
+                       case in != "" && !errors.Is(gotErr, wantErr):
+                               t.Errorf("Unmarshal = %v, want %v", gotErr, wantErr)
+                       case in == "" && gotErr != io.EOF:
+                               t.Errorf("Unmarshal = %v, want %v", gotErr, io.EOF)
+                       }
+               }
+       }
+}
+
 type ReaderFunc func([]byte) (int, error)
 
 func (f ReaderFunc) Read(b []byte) (int, error) { return f(b) }

diff --git a/src/encoding/json/v2/errors.go b/src/encoding/json/v2/errors.go
index 0f50d608c9ed27f348f32162b13b5ab294a1b756..4421f8b821cb5cf95c3a0a10e09c828793f847f8 100644 (file)
--- a/src/encoding/json/v2/errors.go
+++ b/src/encoding/json/v2/errors.go
@@ -10,6 +10,7 @@ import (
        "cmp"
        "errors"
        "fmt"
+       "io"
        "reflect"
        "strconv"
        "strings"
@@ -118,7 +119,7 @@ func newInvalidFormatError(c coder, t reflect.Type) error {
 // newMarshalErrorBefore wraps err in a SemanticError assuming that e
 // is positioned right before the next token or value, which causes an error.
 func newMarshalErrorBefore(e *jsontext.Encoder, t reflect.Type, err error) error {
-       return &SemanticError{action: "marshal", GoType: t, Err: err,
+       return &SemanticError{action: "marshal", GoType: t, Err: toUnexpectedEOF(err),
                ByteOffset:  e.OutputOffset() + int64(export.Encoder(e).CountNextDelimWhitespace()),
                JSONPointer: jsontext.Pointer(export.Encoder(e).AppendStackPointer(nil, +1))}
 }
@@ -134,7 +135,7 @@ func newUnmarshalErrorBefore(d *jsontext.Decoder, t reflect.Type, err error) err
        if export.Decoder(d).Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
                k = d.PeekKind()
        }
-       return &SemanticError{action: "unmarshal", GoType: t, Err: err,
+       return &SemanticError{action: "unmarshal", GoType: t, Err: toUnexpectedEOF(err),
                ByteOffset:  d.InputOffset() + int64(export.Decoder(d).CountNextDelimWhitespace()),
                JSONPointer: jsontext.Pointer(export.Decoder(d).AppendStackPointer(nil, +1)),
                JSONKind:    k}
@@ -157,7 +158,7 @@ func newUnmarshalErrorBeforeWithSkipping(d *jsontext.Decoder, t reflect.Type, er
 // is positioned right after the previous token or value, which caused an error.
 func newUnmarshalErrorAfter(d *jsontext.Decoder, t reflect.Type, err error) error {
        tokOrVal := export.Decoder(d).PreviousTokenOrValue()
-       return &SemanticError{action: "unmarshal", GoType: t, Err: err,
+       return &SemanticError{action: "unmarshal", GoType: t, Err: toUnexpectedEOF(err),
                ByteOffset:  d.InputOffset() - int64(len(tokOrVal)),
                JSONPointer: jsontext.Pointer(export.Decoder(d).AppendStackPointer(nil, -1)),
                JSONKind:    jsontext.Value(tokOrVal).Kind()}
@@ -206,6 +207,7 @@ func newSemanticErrorWithPosition(c coder, t reflect.Type, prevDepth int, prevLe
        if serr == nil {
                serr = &SemanticError{Err: err}
        }
+       serr.Err = toUnexpectedEOF(serr.Err)
        var currDepth int
        var currLength int64
        var coderState interface{ AppendStackPointer([]byte, int) []byte }
@@ -432,3 +434,11 @@ func newDuplicateNameError(ptr jsontext.Pointer, quotedName []byte, offset int64
                Err:         jsontext.ErrDuplicateName,
        }
 }
+
+// toUnexpectedEOF converts [io.EOF] to [io.ErrUnexpectedEOF].
+func toUnexpectedEOF(err error) error {
+       if err == io.EOF {
+               return io.ErrUnexpectedEOF
+       }
+       return err
+}