runtime: ensure minimum heap distance via heap goal

Currently we ensure a minimum heap distance of 1MB when computing the
assist ratio. Rather than enforcing this minimum on the heap distance,
it makes more sense to enforce that the heap goal itself is at least
1MB over the live heap size at the beginning of GC. Currently the two
approaches are semantically equivalent, but this will let us switch to
basing the assist ratio on current heap distance rather than the
initial heap distance, since we can't enforce this minimum on the
current heap distance (the GC may never finish because the goal posts
will always be 1MB away).

Change-Id: I0027b1c26a41a0152b01e5b67bdb1140d43ee903
Reviewed-on: https://go-review.googlesource.com/15604
Reviewed-by: Rick Hudson <rlh@golang.org>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index e78233111915810dc843a9ecb991ef3c79b059b0..b69168a069e23f6fd2eb2434f9c30f2602f8eccf 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -408,6 +408,17 @@ func (c *gcControllerState) startCycle() {
        // Compute the heap goal for this cycle
        c.heapGoal = memstats.heap_reachable + memstats.heap_reachable*uint64(gcpercent)/100
 
+       // Ensure that the heap goal is at least a little larger than
+       // the current live heap size. This may not be the case if GC
+       // start is delayed or if the allocation that pushed heap_live
+       // over next_gc is large or if the trigger is really close to
+       // GOGC. Assist is proportional to this distance, so enforce a
+       // minimum distance, even if it means going over the GOGC goal
+       // by a tiny bit.
+       if c.heapGoal < memstats.heap_live+1024*1024 {
+               c.heapGoal = memstats.heap_live + 1024*1024
+       }
+
        // Compute the total mark utilization goal and divide it among
        // dedicated and fractional workers.
        totalUtilizationGoal := float64(gomaxprocs) * gcGoalUtilization
@@ -444,6 +455,10 @@ func (c *gcControllerState) startCycle() {
 // revise updates the assist ratio during the GC cycle to account for
 // improved estimates. This should be called either under STW or
 // whenever memstats.heap_scan is updated (with mheap_.lock held).
+//
+// It should only be called when gcBlackenEnabled != 0 (because this
+// is when assists are enabled and the necessary statistics are
+// available).
 func (c *gcControllerState) revise() {
        // Compute the expected scan work.
        //
@@ -467,14 +482,9 @@ func (c *gcControllerState) revise() {
        // allocates the remaining heap bytes up to next_gc, it will
        // have done (or stolen) the estimated amount of scan work.
        heapDistance := int64(c.heapGoal) - int64(work.initialHeapLive)
-       if heapDistance <= 1024*1024 {
-               // heapDistance can be negative if GC start is delayed
-               // or if the allocation that pushed heap_live over
-               // next_gc is large or if the trigger is really close
-               // to GOGC. We don't want to set the assist negative
-               // (or divide by zero, or set it really high), so
-               // enforce a minimum on the distance.
-               heapDistance = 1024 * 1024
+       if heapDistance <= 0 {
+               print("runtime: heap goal=", heapDistance, " initial heap live=", work.initialHeapLive, "\n")
+               throw("negative heap distance")
        }
        c.assistRatio = float64(scanWorkExpected) / float64(heapDistance)
 }

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index 6740470827eb06a8d972f950663f61e16bb39469..54cefb4e0be1f7972e1b61362d2f0cb502ebb6e8 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -423,7 +423,9 @@ func mHeap_Alloc_m(h *mheap, npage uintptr, sizeclass int32, large bool) *mspan
        memstats.tinyallocs += uint64(_g_.m.mcache.local_tinyallocs)
        _g_.m.mcache.local_tinyallocs = 0
 
-       gcController.revise()
+       if gcBlackenEnabled != 0 {
+               gcController.revise()
+       }
 
        s := mHeap_AllocSpanLocked(h, npage)
        if s != nil {
@@ -703,7 +705,9 @@ func mHeap_Free(h *mheap, s *mspan, acct int32) {
                if acct != 0 {
                        memstats.heap_objects--
                }
-               gcController.revise()
+               if gcBlackenEnabled != 0 {
+                       gcController.revise()
+               }
                mHeap_FreeSpanLocked(h, s, true, true, 0)
                if trace.enabled {
                        traceHeapAlloc()