runtime: detangle sweeper pacing from GC pacing

The sweeper's pacing state is global, so detangle it from the GC pacer's
state updates so that the GC pacer can be tested.

For #44167.

Change-Id: Ibcea989cd435b73c5891f777d9f95f9604e03bd1
Reviewed-on: https://go-review.googlesource.com/c/go/+/309273
Trust: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index e7c023919c92e03e004347b71460b9dee9a2b272..b2ed18fe6ac0fd533430ba20969da9d1f11ba317 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -971,6 +971,7 @@ func gcMarkTermination(nextTriggerRatio float64) {
 
        // Update GC trigger and pacing for the next cycle.
        gcController.commit(nextTriggerRatio)
+       gcPaceSweeper(gcController.trigger)
        gcPaceScavenger(gcController.heapGoal, gcController.lastHeapGoal)
 
        // Update timing memstats

diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go
index 73fe6e15e44a995b9596df5a071fe28090ff7056..980cb2f086452ffe7f3babb629f4c54fe88d58a9 100644 (file)
--- a/src/runtime/mgcpacer.go
+++ b/src/runtime/mgcpacer.go
@@ -735,40 +735,6 @@ func (c *gcControllerState) commit(triggerRatio float64) {
        if gcphase != _GCoff {
                c.revise()
        }
-
-       // Update sweep pacing.
-       if isSweepDone() {
-               mheap_.sweepPagesPerByte = 0
-       } else {
-               // Concurrent sweep needs to sweep all of the in-use
-               // pages by the time the allocated heap reaches the GC
-               // trigger. Compute the ratio of in-use pages to sweep
-               // per byte allocated, accounting for the fact that
-               // some might already be swept.
-               heapLiveBasis := atomic.Load64(&c.heapLive)
-               heapDistance := int64(trigger) - int64(heapLiveBasis)
-               // Add a little margin so rounding errors and
-               // concurrent sweep are less likely to leave pages
-               // unswept when GC starts.
-               heapDistance -= 1024 * 1024
-               if heapDistance < _PageSize {
-                       // Avoid setting the sweep ratio extremely high
-                       heapDistance = _PageSize
-               }
-               pagesSwept := mheap_.pagesSwept.Load()
-               pagesInUse := mheap_.pagesInUse.Load()
-               sweepDistancePages := int64(pagesInUse) - int64(pagesSwept)
-               if sweepDistancePages <= 0 {
-                       mheap_.sweepPagesPerByte = 0
-               } else {
-                       mheap_.sweepPagesPerByte = float64(sweepDistancePages) / float64(heapDistance)
-                       mheap_.sweepHeapLiveBasis = heapLiveBasis
-                       // Write pagesSweptBasis last, since this
-                       // signals concurrent sweeps to recompute
-                       // their debt.
-                       mheap_.pagesSweptBasis.Store(pagesSwept)
-               }
-       }
 }
 
 // effectiveGrowthRatio returns the current effective heap growth
@@ -819,6 +785,7 @@ func setGCPercent(in int32) (out int32) {
        systemstack(func() {
                lock(&mheap_.lock)
                out = gcController.setGCPercent(in)
+               gcPaceSweeper(gcController.trigger)
                gcPaceScavenger(gcController.heapGoal, gcController.lastHeapGoal)
                unlock(&mheap_.lock)
        })

diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go
index a431d8a2af7f0edaef3cead714162826c5e46fc4..b06df32b202b7019788c646ea6a162a05d8ea932 100644 (file)
--- a/src/runtime/mgcsweep.go
+++ b/src/runtime/mgcsweep.go
@@ -830,3 +830,46 @@ func clobberfree(x unsafe.Pointer, size uintptr) {
                *(*uint32)(add(x, i)) = 0xdeadbeef
        }
 }
+
+// gcPaceSweeper updates the sweeper's pacing parameters.
+//
+// Must be called whenever the GC's pacing is updated.
+//
+// The world must be stopped, or mheap_.lock must be held.
+func gcPaceSweeper(trigger uint64) {
+       assertWorldStoppedOrLockHeld(&mheap_.lock)
+
+       // Update sweep pacing.
+       if isSweepDone() {
+               mheap_.sweepPagesPerByte = 0
+       } else {
+               // Concurrent sweep needs to sweep all of the in-use
+               // pages by the time the allocated heap reaches the GC
+               // trigger. Compute the ratio of in-use pages to sweep
+               // per byte allocated, accounting for the fact that
+               // some might already be swept.
+               heapLiveBasis := atomic.Load64(&gcController.heapLive)
+               heapDistance := int64(trigger) - int64(heapLiveBasis)
+               // Add a little margin so rounding errors and
+               // concurrent sweep are less likely to leave pages
+               // unswept when GC starts.
+               heapDistance -= 1024 * 1024
+               if heapDistance < _PageSize {
+                       // Avoid setting the sweep ratio extremely high
+                       heapDistance = _PageSize
+               }
+               pagesSwept := mheap_.pagesSwept.Load()
+               pagesInUse := mheap_.pagesInUse.Load()
+               sweepDistancePages := int64(pagesInUse) - int64(pagesSwept)
+               if sweepDistancePages <= 0 {
+                       mheap_.sweepPagesPerByte = 0
+               } else {
+                       mheap_.sweepPagesPerByte = float64(sweepDistancePages) / float64(heapDistance)
+                       mheap_.sweepHeapLiveBasis = heapLiveBasis
+                       // Write pagesSweptBasis last, since this
+                       // signals concurrent sweeps to recompute
+                       // their debt.
+                       mheap_.pagesSweptBasis.Store(pagesSwept)
+               }
+       }
+}