runtime: make the memory limit heap goal headroom proportional

Currently if GOGC=off and GOMEMLIMIT is set, then the synchronous
scavenger is likely to work fairly often to maintain the limit, since
the heap goal goes right up to the edge of the memory limit (minus a
fixed 1 MiB of headroom).

If the application's allocation rate is high, and page-level
fragmentation is high, then most allocations will scavenge.

This change mitigates this problem by adding a proportional component
to constant headroom added to the memory-limit-based heap goal. This
means the runtime will have much more headroom before fragmentation
forces memory to be eagerly scavenged.

The proportional headroom in this case is 3%, or ~30 MiB for a 1 GiB
heap. This technically will increase GC frequency in the GOGC=off case
by a tiny amount, but will likely have a positive impact on both
allocation throughput and latency that outweighs this difference.

I wrote a small program to reproduce this issue and confirmed that the
issue is resolved by this patch:

https://github.com/golang/go/issues/57069#issuecomment-1551746565

This value of 3% is chosen as it seems to be a inflection point in this
particular small program. 2% still resulted in quite a bit of eager
scavenging work. I confirmed this results in a GC frequency increase of
about 3%.

This choice is still somewhat arbitrary because the program is
arbitrary, so perhaps worth revisiting in the future. Still, it should
help a good number of programs.

Fixes #57069.

Change-Id: Icb9829db0dfefb4fe42a0cabc5aa8d35970dd7d5
Reviewed-on: https://go-review.googlesource.com/c/go/+/460375
Reviewed-by: Michael Pratt <mpratt@google.com>
Auto-Submit: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go
index ebac7fa9978723ea607a94549ef38f16f37a713b..d5a3258a06ca31df0eca57e813b6fe6843860c20 100644 (file)
--- a/src/runtime/export_test.go
+++ b/src/runtime/export_test.go
@@ -1345,10 +1345,11 @@ func GCTestPointerClass(p unsafe.Pointer) string {
 const Raceenabled = raceenabled
 
 const (
-       GCBackgroundUtilization     = gcBackgroundUtilization
-       GCGoalUtilization           = gcGoalUtilization
-       DefaultHeapMinimum          = defaultHeapMinimum
-       MemoryLimitHeapGoalHeadroom = memoryLimitHeapGoalHeadroom
+       GCBackgroundUtilization            = gcBackgroundUtilization
+       GCGoalUtilization                  = gcGoalUtilization
+       DefaultHeapMinimum                 = defaultHeapMinimum
+       MemoryLimitHeapGoalHeadroomPercent = memoryLimitHeapGoalHeadroomPercent
+       MemoryLimitMinHeapGoalHeadroom     = memoryLimitMinHeapGoalHeadroom
 )
 
 type GCController struct {

diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go
index 3a35c2c594b8f75099cb89aa9640bace0064ebd7..32e19f96e101b430c731439cdcdeeccb389705be 100644 (file)
--- a/src/runtime/mgcpacer.go
+++ b/src/runtime/mgcpacer.go
@@ -61,11 +61,16 @@ const (
        // that can accumulate on a P before updating gcController.stackSize.
        maxStackScanSlack = 8 << 10
 
-       // memoryLimitHeapGoalHeadroom is the amount of headroom the pacer gives to
-       // the heap goal when operating in the memory-limited regime. That is,
-       // it'll reduce the heap goal by this many extra bytes off of the base
-       // calculation.
-       memoryLimitHeapGoalHeadroom = 1 << 20
+       // memoryLimitMinHeapGoalHeadroom is the minimum amount of headroom the
+       // pacer gives to the heap goal when operating in the memory-limited regime.
+       // That is, it'll reduce the heap goal by this many extra bytes off of the
+       // base calculation, at minimum.
+       memoryLimitMinHeapGoalHeadroom = 1 << 20
+
+       // memoryLimitHeapGoalHeadroomPercent is how headroom the memory-limit-based
+       // heap goal should have as a percent of the maximum possible heap goal allowed
+       // to maintain the memory limit.
+       memoryLimitHeapGoalHeadroomPercent = 3
 )
 
 // gcController implements the GC pacing controller that determines
@@ -968,8 +973,10 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
        //
        // In practice this computation looks like the following:
        //
-       //    memoryLimit - ((mappedReady - heapFree - heapAlloc) + max(mappedReady - memoryLimit, 0)) - memoryLimitHeapGoalHeadroom
-       //                    ^1                                    ^2                                   ^3
+       //    goal := memoryLimit - ((mappedReady - heapFree - heapAlloc) + max(mappedReady - memoryLimit, 0))
+       //                    ^1                                    ^2
+       //    goal -= goal / 100 * memoryLimitHeapGoalHeadroomPercent
+       //    ^3
        //
        // Let's break this down.
        //
@@ -1001,11 +1008,14 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
        // terms of heap objects, but it takes more than X bytes (e.g. due to fragmentation) to store
        // X bytes worth of objects.
        //
-       // The third term (marker 3) subtracts an additional memoryLimitHeapGoalHeadroom bytes from the
-       // heap goal. As the name implies, this is to provide additional headroom in the face of pacing
-       // inaccuracies. This is a fixed number of bytes because these inaccuracies disproportionately
-       // affect small heaps: as heaps get smaller, the pacer's inputs get fuzzier. Shorter GC cycles
-       // and less GC work means noisy external factors like the OS scheduler have a greater impact.
+       // The final adjustment (marker 3) reduces the maximum possible memory limit heap goal by
+       // memoryLimitHeapGoalPercent. As the name implies, this is to provide additional headroom in
+       // the face of pacing inaccuracies, and also to leave a buffer of unscavenged memory so the
+       // allocator isn't constantly scavenging. The reduction amount also has a fixed minimum
+       // (memoryLimitMinHeapGoalHeadroom, not pictured) because the aforementioned pacing inaccuracies
+       // disproportionately affect small heaps: as heaps get smaller, the pacer's inputs get fuzzier.
+       // Shorter GC cycles and less GC work means noisy external factors like the OS scheduler have a
+       // greater impact.
 
        memoryLimit := uint64(c.memoryLimit.Load())
 
@@ -1029,12 +1039,19 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
        // Compute the goal.
        goal := memoryLimit - (nonHeapMemory + overage)
 
-       // Apply some headroom to the goal to account for pacing inaccuracies.
-       // Be careful about small limits.
-       if goal < memoryLimitHeapGoalHeadroom || goal-memoryLimitHeapGoalHeadroom < memoryLimitHeapGoalHeadroom {
-               goal = memoryLimitHeapGoalHeadroom
+       // Apply some headroom to the goal to account for pacing inaccuracies and to reduce
+       // the impact of scavenging at allocation time in response to a high allocation rate
+       // when GOGC=off. See issue #57069. Also, be careful about small limits.
+       headroom := goal / 100 * memoryLimitHeapGoalHeadroomPercent
+       if headroom < memoryLimitMinHeapGoalHeadroom {
+               // Set a fixed minimum to deal with the particularly large effect pacing inaccuracies
+               // have for smaller heaps.
+               headroom = memoryLimitMinHeapGoalHeadroom
+       }
+       if goal < headroom || goal-headroom < headroom {
+               goal = headroom
        } else {
-               goal = goal - memoryLimitHeapGoalHeadroom
+               goal = goal - headroom
        }
        // Don't let us go below the live heap. A heap goal below the live heap doesn't make sense.
        if goal < c.heapMarked {

diff --git a/src/runtime/mgcpacer_test.go b/src/runtime/mgcpacer_test.go
index ac2a3fa56cc0022135c16f7a2d4d55bf32b198af..ef1483d629ea13345422f5a91524b456b65446d1 100644 (file)
--- a/src/runtime/mgcpacer_test.go
+++ b/src/runtime/mgcpacer_test.go
@@ -417,7 +417,7 @@ func TestGcPacer(t *testing.T) {
                        length:        50,
                        checker: func(t *testing.T, c []gcCycleResult) {
                                n := len(c)
-                               if peak := c[n-1].heapPeak; peak >= (512<<20)-MemoryLimitHeapGoalHeadroom {
+                               if peak := c[n-1].heapPeak; peak >= applyMemoryLimitHeapGoalHeadroom(512<<20) {
                                        t.Errorf("peak heap size reaches heap limit: %d", peak)
                                }
                                if n >= 25 {
@@ -446,7 +446,7 @@ func TestGcPacer(t *testing.T) {
                        length:        50,
                        checker: func(t *testing.T, c []gcCycleResult) {
                                n := len(c)
-                               if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom {
+                               if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) {
                                        t.Errorf("heap goal is not the heap limit: %d", goal)
                                }
                                if n >= 25 {
@@ -510,7 +510,7 @@ func TestGcPacer(t *testing.T) {
                        checker: func(t *testing.T, c []gcCycleResult) {
                                n := len(c)
                                if n < 10 {
-                                       if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom {
+                                       if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) {
                                                t.Errorf("heap goal is not the heap limit: %d", goal)
                                        }
                                }
@@ -550,7 +550,7 @@ func TestGcPacer(t *testing.T) {
                                n := len(c)
                                if n > 12 {
                                        // We're trying to saturate the memory limit.
-                                       if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom {
+                                       if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) {
                                                t.Errorf("heap goal is not the heap limit: %d", goal)
                                        }
                                }
@@ -581,7 +581,7 @@ func TestGcPacer(t *testing.T) {
                        length:        50,
                        checker: func(t *testing.T, c []gcCycleResult) {
                                n := len(c)
-                               if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom {
+                               if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) {
                                        t.Errorf("heap goal is not the heap limit: %d", goal)
                                }
                                if n >= 25 {
@@ -1019,6 +1019,19 @@ func (f float64Stream) limit(min, max float64) float64Stream {
        }
 }
 
+func applyMemoryLimitHeapGoalHeadroom(goal uint64) uint64 {
+       headroom := goal / 100 * MemoryLimitHeapGoalHeadroomPercent
+       if headroom < MemoryLimitMinHeapGoalHeadroom {
+               headroom = MemoryLimitMinHeapGoalHeadroom
+       }
+       if goal < headroom || goal-headroom < headroom {
+               goal = headroom
+       } else {
+               goal -= headroom
+       }
+       return goal
+}
+
 func TestIdleMarkWorkerCount(t *testing.T) {
        const workers = 10
        c := NewGCController(100, math.MaxInt64)