bytes: rely on runtime.growslice for growing

Rather than naively making a slice of capacity 2*c+n,
rely on the append(..., make(...)) pattern to allocate a
slice that aligns up to the closest size class.

Performance:
	name                          old time/op    new time/op    delta
	BufferWriteBlock/N4096       3.03µs ± 6%    2.04µs ± 6%  -32.60%  (p=0.000 n=10+10)
	BufferWriteBlock/N65536      47.8µs ± 6%    28.1µs ± 2%  -41.32%  (p=0.000 n=9+8)
	BufferWriteBlock/N1048576     844µs ± 7%     510µs ± 5%  -39.59%  (p=0.000 n=8+9)

	name                          old alloc/op   new alloc/op   delta
	BufferWriteBlock/N4096       12.3kB ± 0%     7.2kB ± 0%  -41.67%  (p=0.000 n=10+10)
	BufferWriteBlock/N65536       258kB ± 0%     130kB ± 0%  -49.60%  (p=0.000 n=10+10)
	BufferWriteBlock/N1048576    4.19MB ± 0%    2.10MB ± 0%  -49.98%  (p=0.000 n=10+8)

	name                          old allocs/op  new allocs/op  delta
	BufferWriteBlock/N4096         3.00 ± 0%      3.00 ± 0%     ~     (all equal)
	BufferWriteBlock/N65536        7.00 ± 0%      7.00 ± 0%     ~     (all equal)
	BufferWriteBlock/N1048576      11.0 ± 0%      11.0 ± 0%     ~     (all equal)

The performance is faster since the growth rate is capped at 2x,
while previously it could grow by amounts potentially much greater than 2x,
leading to significant amounts of memory waste and extra copying.

Credit goes to Martin Möhrmann for suggesting the
append(b, make([]T, n)...) pattern.

Fixes #42984
Updates #51462

Change-Id: I7b23f75dddbf53f8b8b93485bb1a1fff9649b96b
Reviewed-on: https://go-review.googlesource.com/c/go/+/349994
Trust: Joseph Tsai <joetsai@digital-static.net>
Trust: Josh Bleecher Snyder <josharian@gmail.com>
Reviewed-by: Bryan Mills <bcmills@google.com>
Reviewed-by: Ian Lance Taylor <iant@golang.org>
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>

diff --git a/src/bytes/buffer.go b/src/bytes/buffer.go
index 549b077708f80e3742e79b0e93e29d3432d3c6c7..0bacbda16490ed355c3420ced97cf39a222eecc8 100644 (file)
--- a/src/bytes/buffer.go
+++ b/src/bytes/buffer.go
@@ -138,10 +138,8 @@ func (b *Buffer) grow(n int) int {
        } else if c > maxInt-c-n {
                panic(ErrTooLarge)
        } else {
-               // Not enough space anywhere, we need to allocate.
-               buf := makeSlice(2*c + n)
-               copy(buf, b.buf[b.off:])
-               b.buf = buf
+               // Add b.off to account for b.buf[:b.off] being sliced off the front.
+               b.buf = growSlice(b.buf[b.off:], b.off+n)
        }
        // Restore b.off and len(b.buf).
        b.off = 0
@@ -217,16 +215,31 @@ func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
        }
 }
 
-// makeSlice allocates a slice of size n. If the allocation fails, it panics
-// with ErrTooLarge.
-func makeSlice(n int) []byte {
-       // If the make fails, give a known error.
+// growSlice grows b by n, preserving the original content of b.
+// If the allocation fails, it panics with ErrTooLarge.
+func growSlice(b []byte, n int) []byte {
        defer func() {
                if recover() != nil {
                        panic(ErrTooLarge)
                }
        }()
-       return make([]byte, n)
+       // TODO(http://golang.org/issue/51462): We should rely on the append-make
+       // pattern so that the compiler can call runtime.growslice. For example:
+       //      return append(b, make([]byte, n)...)
+       // This avoids unnecessary zero-ing of the first len(b) bytes of the
+       // allocated slice, but this pattern causes b to escape onto the heap.
+       //
+       // Instead use the append-make pattern with a nil slice to ensure that
+       // we allocate buffers rounded up to the closest size class.
+       c := len(b) + n // ensure enough space for n elements
+       if c < 2*cap(b) {
+               // The growth rate has historically always been 2x. In the future,
+               // we could rely purely on append to determine the growth rate.
+               c = 2 * cap(b)
+       }
+       b2 := append([]byte(nil), make([]byte, c)...)
+       copy(b2, b)
+       return b2[:len(b)]
 }
 
 // WriteTo writes data to w until the buffer is drained or an error occurs.

diff --git a/src/bytes/buffer_test.go b/src/bytes/buffer_test.go
index 9c9b7440ffaa753625a365534a2c7ec30baf4ff2..c0855007c188ea08c418a39c55c952037aa772bd 100644 (file)
--- a/src/bytes/buffer_test.go
+++ b/src/bytes/buffer_test.go
@@ -672,3 +672,18 @@ func BenchmarkBufferFullSmallReads(b *testing.B) {
                }
        }
 }
+
+func BenchmarkBufferWriteBlock(b *testing.B) {
+       block := make([]byte, 1024)
+       for _, n := range []int{1 << 12, 1 << 16, 1 << 20} {
+               b.Run(fmt.Sprintf("N%d", n), func(b *testing.B) {
+                       b.ReportAllocs()
+                       for i := 0; i < b.N; i++ {
+                               var bb Buffer
+                               for bb.Len() < n {
+                                       bb.Write(block)
+                               }
+                       }
+               })
+       }
+}