runtime: eliminate remaining recordspan write barriers

recordspan has two remaining write barriers from writing to the
pointer to the backing store of h.allspans. However, h.allspans is
always backed by off-heap memory, so let the compiler know this.
Unfortunately, this isn't quite as clean as most go:notinheap uses
because we can't directly name the backing store of a slice, but we
can get it done with some judicious casting.

For #22460.

Change-Id: I296f92fa41cf2cb6ae572b35749af23967533877
Reviewed-on: https://go-review.googlesource.com/73414
Reviewed-by: Rick Hudson <rlh@golang.org>

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index 698dbd7479c2880f7621a91b50cb0ba45893420b..12cf29a01d48a241365c89207288d10b88e14fa3 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -319,6 +319,17 @@ func (s *mspan) layout() (size, n, total uintptr) {
        return
 }
 
+// recordspan adds a newly allocated span to h.allspans.
+//
+// This only happens the first time a span is allocated from
+// mheap.spanalloc (it is not called when a span is reused).
+//
+// Write barriers are disallowed here because it can be called from
+// gcWork when allocating new workbufs. However, because it's an
+// indirect call from the fixalloc initializer, the compiler can't see
+// this.
+//
+//go:nowritebarrierrec
 func recordspan(vh unsafe.Pointer, p unsafe.Pointer) {
        h := (*mheap)(vh)
        s := (*mspan)(p)
@@ -339,12 +350,13 @@ func recordspan(vh unsafe.Pointer, p unsafe.Pointer) {
                        copy(new, h.allspans)
                }
                oldAllspans := h.allspans
-               h.allspans = new
+               *(*notInHeapSlice)(unsafe.Pointer(&h.allspans)) = *(*notInHeapSlice)(unsafe.Pointer(&new))
                if len(oldAllspans) != 0 {
                        sysFree(unsafe.Pointer(&oldAllspans[0]), uintptr(cap(oldAllspans))*unsafe.Sizeof(oldAllspans[0]), &memstats.other_sys)
                }
        }
-       h.allspans = append(h.allspans, s)
+       h.allspans = h.allspans[:len(h.allspans)+1]
+       h.allspans[len(h.allspans)-1] = s
 }
 
 // A spanClass represents the size class and noscan-ness of a span.

diff --git a/src/runtime/slice.go b/src/runtime/slice.go
index 937d15a51ba9b1d54c523ebda817d20cde1ca41f..f79aa02c3b38a6df6b916f2d84f36e0ab8d02ffc 100644 (file)
--- a/src/runtime/slice.go
+++ b/src/runtime/slice.go
@@ -14,6 +14,13 @@ type slice struct {
        cap   int
 }
 
+// An notInHeapSlice is a slice backed by go:notinheap memory.
+type notInHeapSlice struct {
+       array *notInHeap
+       len   int
+       cap   int
+}
+
 // maxElems is a lookup table containing the maximum capacity for a slice.
 // The index is the size of the slice element.
 var maxElems = [...]uintptr{