runtime: do not execute write barrier on newly allocated slice in growslice

The new slice created in growslice is cleared during malloc for
element types containing pointers and therefore can only contain
nil pointers. This change avoids executing write barriers for these
nil pointers by adding and using a special bulkBarrierPreWriteSrcOnly
function that does not enqueue pointers to slots in dst to the write
barrier buffer.

Change-Id: If9b18248bfeeb6a874b0132d19520adea593bfc4
Reviewed-on: https://go-review.googlesource.com/115996
Run-TryBot: Martin Möhrmann <moehrmann@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>

diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
index 75f23a16b413d256debe5e7e8d5bd71771399486..e217e7695fdd41159b3831cabd02cf4c3b90f52d 100644 (file)
--- a/src/runtime/mbitmap.go
+++ b/src/runtime/mbitmap.go
@@ -647,6 +647,35 @@ func bulkBarrierPreWrite(dst, src, size uintptr) {
        }
 }
 
+// bulkBarrierPreWriteSrcOnly is like bulkBarrierPreWrite but
+// does not execute write barriers for [dst, dst+size).
+//
+// In addition to the requirements of bulkBarrierPreWrite
+// callers need to ensure [dst, dst+size) is zeroed.
+//
+// This is used for special cases where e.g. dst was just
+// created and zeroed with malloc.
+//go:nosplit
+func bulkBarrierPreWriteSrcOnly(dst, src, size uintptr) {
+       if (dst|src|size)&(sys.PtrSize-1) != 0 {
+               throw("bulkBarrierPreWrite: unaligned arguments")
+       }
+       if !writeBarrier.needed {
+               return
+       }
+       buf := &getg().m.p.ptr().wbBuf
+       h := heapBitsForAddr(dst)
+       for i := uintptr(0); i < size; i += sys.PtrSize {
+               if h.isPointer() {
+                       srcx := (*uintptr)(unsafe.Pointer(src + i))
+                       if !buf.putFast(0, *srcx) {
+                               wbBufFlush(nil, 0)
+                       }
+               }
+               h = h.next()
+       }
+}
+
 // bulkBarrierBitmap executes write barriers for copying from [src,
 // src+size) to [dst, dst+size) using a 1-bit pointer bitmap. src is
 // assumed to start maskOffset bytes into the data covered by the

diff --git a/src/runtime/slice.go b/src/runtime/slice.go
index 737aab5704060bb53420f7f7615951051d0abb65..4206f4384af62b0b06e7f5cdad58a8b78e948d79 100644 (file)
--- a/src/runtime/slice.go
+++ b/src/runtime/slice.go
@@ -202,7 +202,9 @@ func growslice(et *_type, old slice, cap int) slice {
                // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan uninitialized memory.
                p = mallocgc(capmem, et, true)
                if writeBarrier.enabled {
-                       bulkBarrierPreWrite(uintptr(p), uintptr(old.array), lenmem)
+                       // Only shade the pointers in old.array since we know the destination slice p
+                       // only contains nil pointers because it has been cleared during alloc.
+                       bulkBarrierPreWriteSrcOnly(uintptr(p), uintptr(old.array), lenmem)
                }
        }
        memmove(p, old.array, lenmem)