runtime: mark tiny blocks at GC start

The hybrid barrier requires allocate-black, but there's one case where
we don't currently allocate black: the tiny allocator. If we allocate
a *new* tiny alloc block during GC, it will be allocated black, but if
we allocated the current block before GC, it won't be black, and the
further allocations from it won't mark it, which means we may free a
reachable tiny block during sweeping.

Fix this by passing over all mcaches at the beginning of mark, while
the world is still stopped, and greying their tiny blocks.

Updates #17503.

Change-Id: I04d4df7cc2f553f8f7b1e4cb0b52e2946588111a
Reviewed-on: https://go-review.googlesource.com/31456
Reviewed-by: Rick Hudson <rlh@golang.org>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index aa7aa8bd0d9256775171590ca6c494ad448f9359..29bb8cde191447182ee213d4b31ea7a5dd74e83d 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1025,6 +1025,13 @@ func gcStart(mode gcMode, forceTrigger bool) {
                gcBgMarkPrepare() // Must happen before assist enable.
                gcMarkRootPrepare()
 
+               // Mark all active tinyalloc blocks. Since we're
+               // allocating from these, they need to be black like
+               // other allocations. The alternative is to blacken
+               // the tiny block on every allocation from it, which
+               // would slow down the tiny allocator.
+               gcMarkTinyAllocs()
+
                // At this point all Ps have enabled the write
                // barrier, thus maintaining the no white to
                // black invariant. Enable mutator assists to

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index edb8af25f5cea0efc2236e50b7d15fad79f8a482..13bbd071b159c2a8706a459407170b2feba1a7e4 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -1396,6 +1396,27 @@ func gcmarknewobject(obj, size, scanSize uintptr) {
        }
 }
 
+// gcMarkTinyAllocs greys all active tiny alloc blocks.
+//
+// The world must be stopped.
+func gcMarkTinyAllocs() {
+       for _, p := range &allp {
+               if p == nil || p.status == _Pdead {
+                       break
+               }
+               c := p.mcache
+               if c == nil || c.tiny == 0 {
+                       continue
+               }
+               _, hbits, span, objIndex := heapBitsForObject(c.tiny, 0, 0)
+               gcw := &p.gcw
+               greyobject(c.tiny, 0, 0, hbits, span, gcw, objIndex)
+               if gcBlackenPromptly {
+                       gcw.dispose()
+               }
+       }
+}
+
 // Checkmarking
 
 // To help debug the concurrent GC we remark with the world