From 1e48ca7020adf13e0fa4a6c2053f210b869bfa24 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Wed, 9 Jul 2025 17:08:49 -0700 Subject: [PATCH] encoding/json: remove legacy option to EscapeInvalidUTF8 In the presence of invalid UTF-8, the AllowInvalidUTF8 option allows such bytes to be present, but silently mangles them using the Unicode replacement character. The v2 default is to emit the replacement character verbatim (which is valid UTF-8 and exactly what it is for). However, the v1 behavior has historically been to emit the escaped form of the replacement character. This behavior was introduced in https://go.dev/cl/11211045 where the documentation says that it is: replacing invalid bytes with the Unicode replacement rune U+FFFD but the implementation actually replaces it with the escaped form of the Unicode replacement rune. Given that the documentation differs from the implementation, the actual behavior is likely an oversight. Given how esoteric of behavior this is, we change the v1in2 behavior to avoid the unnecesary escaping and drop support for EscapeInvalidUTF8. This does not violate the Go compatibility agreement since we do not document what the exact syntactic output is. Also, there has already been prior precedence for changing the output: * [encoding/json: encode \b and \f as '\b' and '\f' in JSON strings](https://go.dev/cl/521675) * [encoding/json: encode \n in strings as "\n", not "\u000A"](https://go.dev/cl/4678046) * [encoding/json: encode \t as \t instead of \u0009](https://go.dev/cl/162340043) * [encoding/json: use standard ES6 formatting for numbers during marshal](https://go.dev/cl/30371) Fixes #74551 Change-Id: Ib59a873c44713d302f1f6ab103ffba2520d63276 Reviewed-on: https://go-review.googlesource.com/c/go/+/687116 Auto-Submit: Joseph Tsai LUCI-TryBot-Result: Go LUCI Reviewed-by: Damien Neil Reviewed-by: Johan Brandhorst-Satzkorn Reviewed-by: Carlos Amedee --- src/encoding/json/internal/jsonflags/flags.go | 4 +--- src/encoding/json/internal/jsonwire/encode.go | 6 +----- src/encoding/json/v2_decode_test.go | 10 +++++----- src/encoding/json/v2_diff_test.go | 4 ++-- src/encoding/json/v2_options.go | 18 ------------------ 5 files changed, 9 insertions(+), 33 deletions(-) diff --git a/src/encoding/json/internal/jsonflags/flags.go b/src/encoding/json/internal/jsonflags/flags.go index 4496359c89..1e8c2842d4 100644 --- a/src/encoding/json/internal/jsonflags/flags.go +++ b/src/encoding/json/internal/jsonflags/flags.go @@ -52,7 +52,6 @@ const ( AllowInvalidUTF8 | EscapeForHTML | EscapeForJS | - EscapeInvalidUTF8 | PreserveRawStrings | Deterministic | FormatNilMapAsNull | @@ -77,7 +76,7 @@ const ( WhitespaceFlags = AnyWhitespace | Indent | IndentPrefix // AnyEscape is the set of flags related to escaping in a JSON string. - AnyEscape = EscapeForHTML | EscapeForJS | EscapeInvalidUTF8 + AnyEscape = EscapeForHTML | EscapeForJS // CanonicalizeNumbers is the set of flags related to raw number canonicalization. CanonicalizeNumbers = CanonicalizeRawInts | CanonicalizeRawFloats @@ -97,7 +96,6 @@ const ( ReorderRawObjects // encode only EscapeForHTML // encode only EscapeForJS // encode only - EscapeInvalidUTF8 // encode only; only exposed in v1 Multiline // encode only SpaceAfterColon // encode only SpaceAfterComma // encode only diff --git a/src/encoding/json/internal/jsonwire/encode.go b/src/encoding/json/internal/jsonwire/encode.go index 3901ff8bed..8f9b8ab09e 100644 --- a/src/encoding/json/internal/jsonwire/encode.go +++ b/src/encoding/json/internal/jsonwire/encode.go @@ -92,11 +92,7 @@ func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes, flags *jsonflag case isInvalidUTF8(r, rn): hasInvalidUTF8 = true dst = append(dst, src[i:n-rn]...) - if flags.Get(jsonflags.EscapeInvalidUTF8) { - dst = append(dst, `\ufffd`...) - } else { - dst = append(dst, "\ufffd"...) - } + dst = append(dst, "\ufffd"...) i = n case (r == '\u2028' || r == '\u2029') && flags.Get(jsonflags.EscapeForJS): dst = append(dst, src[i:n-rn]...) diff --git a/src/encoding/json/v2_decode_test.go b/src/encoding/json/v2_decode_test.go index cfcefbfcdc..1e4914efc4 100644 --- a/src/encoding/json/v2_decode_test.go +++ b/src/encoding/json/v2_decode_test.go @@ -1249,12 +1249,12 @@ func TestMarshalInvalidUTF8(t *testing.T) { in string want string }{ - {Name(""), "hello\xffworld", `"hello\ufffdworld"`}, + {Name(""), "hello\xffworld", "\"hello\ufffdworld\""}, {Name(""), "", `""`}, - {Name(""), "\xff", `"\ufffd"`}, - {Name(""), "\xff\xff", `"\ufffd\ufffd"`}, - {Name(""), "a\xffb", `"a\ufffdb"`}, - {Name(""), "\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", `"日本\ufffd\ufffd\ufffd"`}, + {Name(""), "\xff", "\"\ufffd\""}, + {Name(""), "\xff\xff", "\"\ufffd\ufffd\""}, + {Name(""), "a\xffb", "\"a\ufffdb\""}, + {Name(""), "\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", "\"日本\ufffd\ufffd\ufffd\""}, } for _, tt := range tests { t.Run(tt.Name, func(t *testing.T) { diff --git a/src/encoding/json/v2_diff_test.go b/src/encoding/json/v2_diff_test.go index 7a561732f4..9d0798ed1d 100644 --- a/src/encoding/json/v2_diff_test.go +++ b/src/encoding/json/v2_diff_test.go @@ -786,8 +786,8 @@ func TestInvalidUTF8(t *testing.T) { switch { case json.Version == "v1" && err != nil: t.Fatalf("json.Marshal error: %v", err) - case json.Version == "v1" && string(got) != `"\ufffd"`: - t.Fatalf(`json.Marshal = %s, want "\ufffd"`, got) + case json.Version == "v1" && string(got) != "\"\ufffd\"": + t.Fatalf(`json.Marshal = %s, want %q`, got, "\ufffd") case json.Version == "v2" && err == nil: t.Fatal("json.Marshal error is nil, want non-nil") } diff --git a/src/encoding/json/v2_options.go b/src/encoding/json/v2_options.go index 4006d764cc..66bd01eb3c 100644 --- a/src/encoding/json/v2_options.go +++ b/src/encoding/json/v2_options.go @@ -204,7 +204,6 @@ type Options = jsonopts.Options // It is equivalent to the following boolean options being set to true: // // - [CallMethodsWithLegacySemantics] -// - [EscapeInvalidUTF8] // - [FormatBytesWithLegacySemantics] // - [FormatTimeWithLegacySemantics] // - [MatchCaseSensitiveDelimiter] @@ -279,23 +278,6 @@ func CallMethodsWithLegacySemantics(v bool) Options { } } -// EscapeInvalidUTF8 specifies that when encoding a [jsontext.String] -// with bytes of invalid UTF-8, such bytes are escaped as -// a hexadecimal Unicode codepoint (i.e., \ufffd). -// In contrast, the v2 default is to use the minimal representation, -// which is to encode invalid UTF-8 as the Unicode replacement rune itself -// (without any form of escaping). -// -// This only affects encoding and is ignored when decoding. -// The v1 default is true. -func EscapeInvalidUTF8(v bool) Options { - if v { - return jsonflags.EscapeInvalidUTF8 | 1 - } else { - return jsonflags.EscapeInvalidUTF8 | 0 - } -} - // FormatBytesWithLegacySemantics specifies that handling of // []~byte and [N]~byte types follow legacy semantics: // -- 2.50.0