From: Joe Tsai Date: Mon, 20 Feb 2023 01:11:46 +0000 (-0800) Subject: encoding/json: use append for Compact and Indent X-Git-Tag: go1.21rc1~1485 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=21ff6704bc8efa72abe191263aae938f3c867480;p=gostls13.git encoding/json: use append for Compact and Indent This is part of the effort to reduce direct reliance on bytes.Buffer so that we can use a buffer with better pooling characteristics. Avoid direct use of bytes.Buffer in Compact and Indent and instead modify the logic to rely only on append. This avoids reliance on the bytes.Buffer.Truncate method, which makes switching to a custom buffer implementation easier. Performance: name old time/op new time/op delta EncodeMarshaler 25.5ns ± 8% 25.7ns ± 9% ~ (p=0.724 n=10+10) name old alloc/op new alloc/op delta EncodeMarshaler 4.00B ± 0% 4.00B ± 0% ~ (all equal) name old allocs/op new allocs/op delta EncodeMarshaler 1.00 ± 0% 1.00 ± 0% ~ (all equal) Updates #27735 Change-Id: I8cded03fab7651d43b5a238ee721f3472530868e Reviewed-on: https://go-review.googlesource.com/c/go/+/469555 Run-TryBot: Joseph Tsai Reviewed-by: Daniel Martí TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Auto-Submit: Joseph Tsai Reviewed-by: Bryan Mills --- diff --git a/src/encoding/json/encode.go b/src/encoding/json/encode.go index 9d59b0ff2b..d2f752a4f8 100644 --- a/src/encoding/json/encode.go +++ b/src/encoding/json/encode.go @@ -175,12 +175,12 @@ func MarshalIndent(v any, prefix, indent string) ([]byte, error) { if err != nil { return nil, err } - var buf bytes.Buffer - err = Indent(&buf, b, prefix, indent) + b2 := make([]byte, 0, indentGrowthFactor*len(b)) + b2, err = appendIndent(b2, b, prefix, indent) if err != nil { return nil, err } - return buf.Bytes(), nil + return b2, nil } // HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 @@ -476,8 +476,10 @@ func marshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { } b, err := m.MarshalJSON() if err == nil { - // copy JSON into buffer, checking validity. - err = compact(&e.Buffer, b, opts.escapeHTML) + e.Grow(len(b)) + out := availableBuffer(&e.Buffer) + out, err = appendCompact(out, b, opts.escapeHTML) + e.Buffer.Write(out) } if err != nil { e.error(&MarshalerError{v.Type(), err, "MarshalJSON"}) @@ -493,8 +495,10 @@ func addrMarshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { m := va.Interface().(Marshaler) b, err := m.MarshalJSON() if err == nil { - // copy JSON into buffer, checking validity. - err = compact(&e.Buffer, b, opts.escapeHTML) + e.Grow(len(b)) + out := availableBuffer(&e.Buffer) + out, err = appendCompact(out, b, opts.escapeHTML) + e.Buffer.Write(out) } if err != nil { e.error(&MarshalerError{v.Type(), err, "MarshalJSON"}) diff --git a/src/encoding/json/indent.go b/src/encoding/json/indent.go index 2924d3b49b..375f71605a 100644 --- a/src/encoding/json/indent.go +++ b/src/encoding/json/indent.go @@ -4,69 +4,73 @@ package json -import ( - "bytes" -) +import "bytes" + +// TODO(https://go.dev/issue/53685): Use bytes.Buffer.AvailableBuffer instead. +func availableBuffer(b *bytes.Buffer) []byte { + return b.Bytes()[b.Len():] +} // Compact appends to dst the JSON-encoded src with // insignificant space characters elided. func Compact(dst *bytes.Buffer, src []byte) error { - return compact(dst, src, false) + dst.Grow(len(src)) + b := availableBuffer(dst) + b, err := appendCompact(b, src, false) + dst.Write(b) + return err } -func compact(dst *bytes.Buffer, src []byte, escape bool) error { - origLen := dst.Len() +func appendCompact(dst, src []byte, escape bool) ([]byte, error) { + origLen := len(dst) scan := newScanner() defer freeScanner(scan) start := 0 for i, c := range src { if escape && (c == '<' || c == '>' || c == '&') { - if start < i { - dst.Write(src[start:i]) - } - dst.WriteString(`\u00`) - dst.WriteByte(hex[c>>4]) - dst.WriteByte(hex[c&0xF]) + dst = append(dst, src[start:i]...) + dst = append(dst, '\\', 'u', '0', '0', hex[c>>4], hex[c&0xF]) start = i + 1 } // Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9). if escape && c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 { - if start < i { - dst.Write(src[start:i]) - } - dst.WriteString(`\u202`) - dst.WriteByte(hex[src[i+2]&0xF]) - start = i + 3 + dst = append(dst, src[start:i]...) + dst = append(dst, '\\', 'u', '2', '0', '2', hex[src[i+2]&0xF]) + start = i + len("\u2029") } v := scan.step(scan, c) if v >= scanSkipSpace { if v == scanError { break } - if start < i { - dst.Write(src[start:i]) - } + dst = append(dst, src[start:i]...) start = i + 1 } } if scan.eof() == scanError { - dst.Truncate(origLen) - return scan.err - } - if start < len(src) { - dst.Write(src[start:]) + return dst[:origLen], scan.err } - return nil + dst = append(dst, src[start:]...) + return dst, nil } -func newline(dst *bytes.Buffer, prefix, indent string, depth int) { - dst.WriteByte('\n') - dst.WriteString(prefix) +func appendNewline(dst []byte, prefix, indent string, depth int) []byte { + dst = append(dst, '\n') + dst = append(dst, prefix...) for i := 0; i < depth; i++ { - dst.WriteString(indent) + dst = append(dst, indent...) } + return dst } +// indentGrowthFactor specifies the growth factor of indenting JSON input. +// Empirically, the growth factor was measured to be between 1.4x to 1.8x +// for some set of compacted JSON with the indent being a single tab. +// Specify a growth factor slightly larger than what is observed +// to reduce probability of allocation in appendIndent. +// A factor no higher than 2 ensures that wasted space never exceeds 50%. +const indentGrowthFactor = 2 + // Indent appends to dst an indented form of the JSON-encoded src. // Each element in a JSON object or array begins on a new, // indented line beginning with prefix followed by one or more @@ -79,7 +83,15 @@ func newline(dst *bytes.Buffer, prefix, indent string, depth int) { // For example, if src has no trailing spaces, neither will dst; // if src ends in a trailing newline, so will dst. func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { - origLen := dst.Len() + dst.Grow(indentGrowthFactor * len(src)) + b := availableBuffer(dst) + b, err := appendIndent(b, src, prefix, indent) + dst.Write(b) + return err +} + +func appendIndent(dst, src []byte, prefix, indent string) ([]byte, error) { + origLen := len(dst) scan := newScanner() defer freeScanner(scan) needIndent := false @@ -96,13 +108,13 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { if needIndent && v != scanEndObject && v != scanEndArray { needIndent = false depth++ - newline(dst, prefix, indent, depth) + dst = appendNewline(dst, prefix, indent, depth) } // Emit semantically uninteresting bytes // (in particular, punctuation in strings) unmodified. if v == scanContinue { - dst.WriteByte(c) + dst = append(dst, c) continue } @@ -111,33 +123,27 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { case '{', '[': // delay indent so that empty object and array are formatted as {} and []. needIndent = true - dst.WriteByte(c) - + dst = append(dst, c) case ',': - dst.WriteByte(c) - newline(dst, prefix, indent, depth) - + dst = append(dst, c) + dst = appendNewline(dst, prefix, indent, depth) case ':': - dst.WriteByte(c) - dst.WriteByte(' ') - + dst = append(dst, c, ' ') case '}', ']': if needIndent { // suppress indent in empty object/array needIndent = false } else { depth-- - newline(dst, prefix, indent, depth) + dst = appendNewline(dst, prefix, indent, depth) } - dst.WriteByte(c) - + dst = append(dst, c) default: - dst.WriteByte(c) + dst = append(dst, c) } } if scan.eof() == scanError { - dst.Truncate(origLen) - return scan.err + return dst[:origLen], scan.err } - return nil + return dst, nil } diff --git a/src/encoding/json/stream.go b/src/encoding/json/stream.go index 1442ef29ef..b4146a359e 100644 --- a/src/encoding/json/stream.go +++ b/src/encoding/json/stream.go @@ -183,7 +183,7 @@ type Encoder struct { err error escapeHTML bool - indentBuf *bytes.Buffer + indentBuf []byte indentPrefix string indentValue string } @@ -221,15 +221,11 @@ func (enc *Encoder) Encode(v any) error { b := e.Bytes() if enc.indentPrefix != "" || enc.indentValue != "" { - if enc.indentBuf == nil { - enc.indentBuf = new(bytes.Buffer) - } - enc.indentBuf.Reset() - err = Indent(enc.indentBuf, b, enc.indentPrefix, enc.indentValue) + enc.indentBuf, err = appendIndent(enc.indentBuf[:0], b, enc.indentPrefix, enc.indentValue) if err != nil { return err } - b = enc.indentBuf.Bytes() + b = enc.indentBuf } if _, err = enc.w.Write(b); err != nil { enc.err = err