runtime: count black allocations toward scan work

Currently we count black allocations toward the scannable heap size,
but not toward the scan work we've done so far. This is clearly
inconsistent (we have, in effect, scanned these allocations and since
they're already black, we're not going to scan them again). Worse, it
means we don't count black allocations toward the scannable heap size
as of the *next* GC because this is based on the amount of scan work
we did in this cycle.

Fix this by counting black allocations as scan work. Currently the GC
spends very little time in allocate-black mode, so this probably
hasn't been a problem, but this will become important when we switch
to always allocating black.

Change-Id: If6ff693b070c385b65b6ecbbbbf76283a0f9d990
Reviewed-on: https://go-review.googlesource.com/22119
Reviewed-by: Rick Hudson <rlh@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 30f2a4fca54415dd5b1c224d228a22d86e742954..3f437bc02f35eda1e17c7f43d8ac60ebe9e0ef77 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -655,6 +655,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
                size = s.elemsize
        }
 
+       var scanSize uintptr
        if noscan {
                // All objects are pre-marked as noscan. Nothing to do.
        } else {
@@ -673,11 +674,12 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
                        // pointers, GC has to scan to the last
                        // element.
                        if typ.ptrdata != 0 {
-                               c.local_scan += dataSize - typ.size + typ.ptrdata
+                               scanSize = dataSize - typ.size + typ.ptrdata
                        }
                } else {
-                       c.local_scan += typ.ptrdata
+                       scanSize = typ.ptrdata
                }
+               c.local_scan += scanSize
 
                // Ensure that the stores above that initialize x to
                // type-safe memory and set the heap bits occur before
@@ -694,7 +696,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
        // a race marking the bit.
        if gcphase == _GCmarktermination || gcBlackenPromptly {
                systemstack(func() {
-                       gcmarknewobject_m(uintptr(x), size)
+                       gcmarknewobject_m(uintptr(x), size, scanSize)
                })
        }
 

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 1c184db10b44f58486fe63c13d27d04571994ca8..d120dae05a2f109bab885974699b350d6c5fc06c 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -304,7 +304,8 @@ type gcControllerState struct {
        // scanWork is the total scan work performed this cycle. This
        // is updated atomically during the cycle. Updates occur in
        // bounded batches, since it is both written and read
-       // throughout the cycle.
+       // throughout the cycle. At the end of the cycle, this is how
+       // much of the retained heap is scannable.
        //
        // Currently this is the bytes of heap scanned. For most uses,
        // this is an opaque unit of work, but for estimation the
@@ -1578,9 +1579,13 @@ func gcMark(start_time int64) {
        work.markrootDone = true
 
        for i := 0; i < int(gomaxprocs); i++ {
-               if !allp[i].gcw.empty() {
+               gcw := &allp[i].gcw
+               if !gcw.empty() {
                        throw("P has cached GC work at end of mark termination")
                }
+               if gcw.scanWork != 0 || gcw.bytesMarked != 0 {
+                       throw("P has unflushed stats at end of mark termination")
+               }
        }
 
        if trace.enabled {

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 1ab8315a29fa0e7d57acb6d5d8d79a443dd34f2f..0d05838987d7c076a299caf88aa64bdda5ba5717 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -1134,12 +1134,14 @@ func gcDumpObject(label string, obj, off uintptr) {
 
 // If gcBlackenPromptly is true we are in the second mark phase phase so we allocate black.
 //go:nowritebarrier
-func gcmarknewobject_m(obj, size uintptr) {
+func gcmarknewobject_m(obj, size, scanSize uintptr) {
        if useCheckmark && !gcBlackenPromptly { // The world should be stopped so this should not happen.
                throw("gcmarknewobject called while doing checkmark")
        }
        heapBitsForAddr(obj).setMarked()
        atomic.Xadd64(&work.bytesMarked, int64(size))
+       gcw := &getg().m.p.ptr().gcw
+       gcw.scanWork += int64(scanSize)
 }
 
 // Checkmarking