runtime: track "scannable" bytes of heap

This tracks the number of scannable bytes in the allocated heap. That
is, bytes that the garbage collector must scan before reaching the
last pointer field in each object.

This will be used to compute a more robust estimate of the GC scan
work.

Change-Id: I1eecd45ef9cdd65b69d2afb5db5da885c80086bb
Reviewed-on: https://go-review.googlesource.com/9695
Reviewed-by: Russ Cox <rsc@golang.org>

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 22ad6b581f52437d1784d63b6861313ad20164b3..1619ccb9f4fd2871aff2b52aca821fff63c1d20d 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -647,6 +647,16 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
                        dataSize = unsafe.Sizeof(_defer{})
                }
                heapBitsSetType(uintptr(x), size, dataSize, typ)
+               if dataSize > typ.size {
+                       // Array allocation. If there are any
+                       // pointers, GC has to scan to the last
+                       // element.
+                       if typ.ptrdata != 0 {
+                               c.local_scan += dataSize - typ.size + typ.ptrdata
+                       }
+               } else {
+                       c.local_scan += typ.ptrdata
+               }
        }
 
        // GCmarkterminate allocates black

diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go
index f01215379aef018290872ac2552eb44d202bc8c6..8c2a6b00cef4614c90fc240003adbd033d2ca7ef 100644 (file)
--- a/src/runtime/mcache.go
+++ b/src/runtime/mcache.go
@@ -13,6 +13,7 @@ type mcache struct {
        // so they are grouped here for better caching.
        next_sample      int32   // trigger heap sample after allocating this many bytes
        local_cachealloc uintptr // bytes allocated from cache since last lock of heap
+       local_scan       uintptr // bytes of scannable heap allocated
        // Allocator cache for tiny objects w/o pointers.
        // See "Tiny allocator" comment in malloc.go.
        tiny             unsafe.Pointer

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index fa3573df56487fe2a95db3cd0073c2a1b2f72d8e..678fe8f3225fb15ee138206428868575eb067deb 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1279,6 +1279,7 @@ func gcMark(start_time int64) {
        // Update other GC heap size stats.
        memstats.heap_live = work.bytesMarked
        memstats.heap_marked = work.bytesMarked
+       memstats.heap_scan = uint64(gcController.scanWork)
 
        if trace.enabled {
                traceHeapAlloc()

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 5b207679f83b17b9e9b98923037e7f2799ee11fe..f69166ee2237ec871fe0d45ae2c7d89b0017fca0 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -530,6 +530,10 @@ func gcDrainN(gcw *gcWork, scanWork int64) {
 
 // scanblock scans b as scanobject would, but using an explicit
 // pointer bitmap instead of the heap bitmap.
+//
+// This is used to scan non-heap roots, so it does not update
+// gcw.bytesMarked or gcw.scanWork.
+//
 //go:nowritebarrier
 func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork) {
        // Use local copies of original parameters, so that a stack trace
@@ -565,8 +569,6 @@ func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork) {
                        i += ptrSize
                }
        }
-
-       gcw.scanWork += int64(n)
 }
 
 // scanobject scans the object starting at b, adding pointers to gcw.

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index 653448363cd31ada97db93d90acda3494c7626d3..10878ee5cf013b285f134804aefaa9f5036fbb42 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -398,6 +398,8 @@ func mHeap_Alloc_m(h *mheap, npage uintptr, sizeclass int32, large bool) *mspan
        // transfer stats from cache to global
        memstats.heap_live += uint64(_g_.m.mcache.local_cachealloc)
        _g_.m.mcache.local_cachealloc = 0
+       memstats.heap_scan += uint64(_g_.m.mcache.local_scan)
+       _g_.m.mcache.local_scan = 0
        memstats.tinyallocs += uint64(_g_.m.mcache.local_tinyallocs)
        _g_.m.mcache.local_tinyallocs = 0
 
@@ -656,6 +658,8 @@ func mHeap_Free(h *mheap, s *mspan, acct int32) {
                lock(&h.lock)
                memstats.heap_live += uint64(mp.mcache.local_cachealloc)
                mp.mcache.local_cachealloc = 0
+               memstats.heap_scan += uint64(mp.mcache.local_scan)
+               mp.mcache.local_scan = 0
                memstats.tinyallocs += uint64(mp.mcache.local_tinyallocs)
                mp.mcache.local_tinyallocs = 0
                if acct != 0 {

diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go
index 098f5da8dcaaf9d7f927213d15eaf08e5da3b6ea..c8e524915624777741cbf22948bcfe7fa1ad6a1b 100644 (file)
--- a/src/runtime/mstats.go
+++ b/src/runtime/mstats.go
@@ -69,6 +69,11 @@ type mstats struct {
        // excludes unmarked objects that have not yet been swept.
        heap_live uint64
 
+       // heap_scan is the number of bytes of "scannable" heap. This
+       // is the live heap (as counted by heap_live), but omitting
+       // no-scan objects and no-scan tails of objects.
+       heap_scan uint64
+
        // heap_marked is the number of bytes marked by the previous
        // GC. After mark termination, heap_live == heap_marked, but
        // unlike heap_live, heap_marked does not change until the
@@ -340,6 +345,8 @@ func purgecachedstats(c *mcache) {
        if trace.enabled {
                traceHeapAlloc()
        }
+       memstats.heap_scan += uint64(c.local_scan)
+       c.local_scan = 0
        memstats.tinyallocs += uint64(c.local_tinyallocs)
        c.local_tinyallocs = 0
        memstats.nlookup += uint64(c.local_nlookup)