runtime: make zeroing of large objects containing pointers preemptible

This change makes it possible for the runtime to preempt the zeroing of
large objects that contain pointers. It turns out this is fairly
straightforward with allocation headers, since we can just temporarily
tell the GC that there's nothing to scan for a large object with a
single pointer write (as opposed to trying to zero a whole bunch of
bits, as we would've had to do once upon a time).

Fixes #31222.

Change-Id: I10d0dcfa3938c383282a3eb485a6f00070d07bd2
Reviewed-on: https://go-review.googlesource.com/c/go/+/577495
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 3879bfa9d792649e1390de2915200afdb462cab9..48cace9171b1155719845b6ea79064c20e0c2e27 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -1165,17 +1165,15 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
                size = span.elemsize
                x = unsafe.Pointer(span.base())
                if needzero && span.needzero != 0 {
-                       if noscan {
-                               delayedZeroing = true
-                       } else {
-                               memclrNoHeapPointers(x, size)
-                       }
+                       delayedZeroing = true
                }
                if !noscan {
+                       // Tell the GC not to look at this yet.
+                       span.largeType = nil
                        header = &span.largeType
                }
        }
-       if !noscan {
+       if !noscan && !delayedZeroing {
                c.scanAlloc += heapSetType(uintptr(x), dataSize, typ, header, span)
        }
 
@@ -1243,17 +1241,23 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
        mp.mallocing = 0
        releasem(mp)
 
-       // Pointerfree data can be zeroed late in a context where preemption can occur.
+       // Objects can be zeroed late in a context where preemption can occur.
+       // If the object contains pointers, its pointer data must be cleared
+       // or otherwise indicate that the GC shouldn't scan it.
        // x will keep the memory alive.
        if delayedZeroing {
-               if !noscan {
-                       throw("delayed zeroing on data that may contain pointers")
-               }
-               if header != nil {
-                       throw("unexpected malloc header in delayed zeroing of large object")
-               }
                // N.B. size == fullSize always in this case.
                memclrNoHeapPointersChunked(size, x) // This is a possible preemption point: see #47302
+
+               // Finish storing the type information for this case.
+               if !noscan {
+                       mp := acquirem()
+                       getMCache(mp).scanAlloc += heapSetType(uintptr(x), dataSize, typ, header, span)
+
+                       // Publish the type information with the zeroed memory.
+                       publicationBarrier()
+                       releasem(mp)
+               }
        }
 
        if debug.malloc {

diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
index d2ab89edb4e308586ab2613b1948b1f6b8969750..e7a712377ba8168374010fdfe2993e8b412af931 100644 (file)
--- a/src/runtime/mbitmap.go
+++ b/src/runtime/mbitmap.go
@@ -192,6 +192,10 @@ func (span *mspan) typePointersOfUnchecked(addr uintptr) typePointers {
                addr += mallocHeaderSize
        } else {
                typ = span.largeType
+               if typ == nil {
+                       // Allow a nil type here for delayed zeroing. See mallocgc.
+                       return typePointers{}
+               }
        }
        gcdata := typ.GCData
        return typePointers{elem: addr, addr: addr, mask: readUintptr(gcdata), typ: typ}