From: Michael Anthony Knyszek Date: Tue, 3 Jan 2023 20:39:23 +0000 (+0000) Subject: runtime: make the memory limit heap goal headroom proportional X-Git-Tag: go1.21rc1~426 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=e97bd776f9cb9c1ab781262a4a0827351fc04775;p=gostls13.git runtime: make the memory limit heap goal headroom proportional Currently if GOGC=off and GOMEMLIMIT is set, then the synchronous scavenger is likely to work fairly often to maintain the limit, since the heap goal goes right up to the edge of the memory limit (minus a fixed 1 MiB of headroom). If the application's allocation rate is high, and page-level fragmentation is high, then most allocations will scavenge. This change mitigates this problem by adding a proportional component to constant headroom added to the memory-limit-based heap goal. This means the runtime will have much more headroom before fragmentation forces memory to be eagerly scavenged. The proportional headroom in this case is 3%, or ~30 MiB for a 1 GiB heap. This technically will increase GC frequency in the GOGC=off case by a tiny amount, but will likely have a positive impact on both allocation throughput and latency that outweighs this difference. I wrote a small program to reproduce this issue and confirmed that the issue is resolved by this patch: https://github.com/golang/go/issues/57069#issuecomment-1551746565 This value of 3% is chosen as it seems to be a inflection point in this particular small program. 2% still resulted in quite a bit of eager scavenging work. I confirmed this results in a GC frequency increase of about 3%. This choice is still somewhat arbitrary because the program is arbitrary, so perhaps worth revisiting in the future. Still, it should help a good number of programs. Fixes #57069. Change-Id: Icb9829db0dfefb4fe42a0cabc5aa8d35970dd7d5 Reviewed-on: https://go-review.googlesource.com/c/go/+/460375 Reviewed-by: Michael Pratt Auto-Submit: Michael Knyszek Run-TryBot: Michael Knyszek TryBot-Result: Gopher Robot --- diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index ebac7fa997..d5a3258a06 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -1345,10 +1345,11 @@ func GCTestPointerClass(p unsafe.Pointer) string { const Raceenabled = raceenabled const ( - GCBackgroundUtilization = gcBackgroundUtilization - GCGoalUtilization = gcGoalUtilization - DefaultHeapMinimum = defaultHeapMinimum - MemoryLimitHeapGoalHeadroom = memoryLimitHeapGoalHeadroom + GCBackgroundUtilization = gcBackgroundUtilization + GCGoalUtilization = gcGoalUtilization + DefaultHeapMinimum = defaultHeapMinimum + MemoryLimitHeapGoalHeadroomPercent = memoryLimitHeapGoalHeadroomPercent + MemoryLimitMinHeapGoalHeadroom = memoryLimitMinHeapGoalHeadroom ) type GCController struct { diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go index 3a35c2c594..32e19f96e1 100644 --- a/src/runtime/mgcpacer.go +++ b/src/runtime/mgcpacer.go @@ -61,11 +61,16 @@ const ( // that can accumulate on a P before updating gcController.stackSize. maxStackScanSlack = 8 << 10 - // memoryLimitHeapGoalHeadroom is the amount of headroom the pacer gives to - // the heap goal when operating in the memory-limited regime. That is, - // it'll reduce the heap goal by this many extra bytes off of the base - // calculation. - memoryLimitHeapGoalHeadroom = 1 << 20 + // memoryLimitMinHeapGoalHeadroom is the minimum amount of headroom the + // pacer gives to the heap goal when operating in the memory-limited regime. + // That is, it'll reduce the heap goal by this many extra bytes off of the + // base calculation, at minimum. + memoryLimitMinHeapGoalHeadroom = 1 << 20 + + // memoryLimitHeapGoalHeadroomPercent is how headroom the memory-limit-based + // heap goal should have as a percent of the maximum possible heap goal allowed + // to maintain the memory limit. + memoryLimitHeapGoalHeadroomPercent = 3 ) // gcController implements the GC pacing controller that determines @@ -968,8 +973,10 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 { // // In practice this computation looks like the following: // - // memoryLimit - ((mappedReady - heapFree - heapAlloc) + max(mappedReady - memoryLimit, 0)) - memoryLimitHeapGoalHeadroom - // ^1 ^2 ^3 + // goal := memoryLimit - ((mappedReady - heapFree - heapAlloc) + max(mappedReady - memoryLimit, 0)) + // ^1 ^2 + // goal -= goal / 100 * memoryLimitHeapGoalHeadroomPercent + // ^3 // // Let's break this down. // @@ -1001,11 +1008,14 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 { // terms of heap objects, but it takes more than X bytes (e.g. due to fragmentation) to store // X bytes worth of objects. // - // The third term (marker 3) subtracts an additional memoryLimitHeapGoalHeadroom bytes from the - // heap goal. As the name implies, this is to provide additional headroom in the face of pacing - // inaccuracies. This is a fixed number of bytes because these inaccuracies disproportionately - // affect small heaps: as heaps get smaller, the pacer's inputs get fuzzier. Shorter GC cycles - // and less GC work means noisy external factors like the OS scheduler have a greater impact. + // The final adjustment (marker 3) reduces the maximum possible memory limit heap goal by + // memoryLimitHeapGoalPercent. As the name implies, this is to provide additional headroom in + // the face of pacing inaccuracies, and also to leave a buffer of unscavenged memory so the + // allocator isn't constantly scavenging. The reduction amount also has a fixed minimum + // (memoryLimitMinHeapGoalHeadroom, not pictured) because the aforementioned pacing inaccuracies + // disproportionately affect small heaps: as heaps get smaller, the pacer's inputs get fuzzier. + // Shorter GC cycles and less GC work means noisy external factors like the OS scheduler have a + // greater impact. memoryLimit := uint64(c.memoryLimit.Load()) @@ -1029,12 +1039,19 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 { // Compute the goal. goal := memoryLimit - (nonHeapMemory + overage) - // Apply some headroom to the goal to account for pacing inaccuracies. - // Be careful about small limits. - if goal < memoryLimitHeapGoalHeadroom || goal-memoryLimitHeapGoalHeadroom < memoryLimitHeapGoalHeadroom { - goal = memoryLimitHeapGoalHeadroom + // Apply some headroom to the goal to account for pacing inaccuracies and to reduce + // the impact of scavenging at allocation time in response to a high allocation rate + // when GOGC=off. See issue #57069. Also, be careful about small limits. + headroom := goal / 100 * memoryLimitHeapGoalHeadroomPercent + if headroom < memoryLimitMinHeapGoalHeadroom { + // Set a fixed minimum to deal with the particularly large effect pacing inaccuracies + // have for smaller heaps. + headroom = memoryLimitMinHeapGoalHeadroom + } + if goal < headroom || goal-headroom < headroom { + goal = headroom } else { - goal = goal - memoryLimitHeapGoalHeadroom + goal = goal - headroom } // Don't let us go below the live heap. A heap goal below the live heap doesn't make sense. if goal < c.heapMarked { diff --git a/src/runtime/mgcpacer_test.go b/src/runtime/mgcpacer_test.go index ac2a3fa56c..ef1483d629 100644 --- a/src/runtime/mgcpacer_test.go +++ b/src/runtime/mgcpacer_test.go @@ -417,7 +417,7 @@ func TestGcPacer(t *testing.T) { length: 50, checker: func(t *testing.T, c []gcCycleResult) { n := len(c) - if peak := c[n-1].heapPeak; peak >= (512<<20)-MemoryLimitHeapGoalHeadroom { + if peak := c[n-1].heapPeak; peak >= applyMemoryLimitHeapGoalHeadroom(512<<20) { t.Errorf("peak heap size reaches heap limit: %d", peak) } if n >= 25 { @@ -446,7 +446,7 @@ func TestGcPacer(t *testing.T) { length: 50, checker: func(t *testing.T, c []gcCycleResult) { n := len(c) - if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom { + if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) { t.Errorf("heap goal is not the heap limit: %d", goal) } if n >= 25 { @@ -510,7 +510,7 @@ func TestGcPacer(t *testing.T) { checker: func(t *testing.T, c []gcCycleResult) { n := len(c) if n < 10 { - if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom { + if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) { t.Errorf("heap goal is not the heap limit: %d", goal) } } @@ -550,7 +550,7 @@ func TestGcPacer(t *testing.T) { n := len(c) if n > 12 { // We're trying to saturate the memory limit. - if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom { + if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) { t.Errorf("heap goal is not the heap limit: %d", goal) } } @@ -581,7 +581,7 @@ func TestGcPacer(t *testing.T) { length: 50, checker: func(t *testing.T, c []gcCycleResult) { n := len(c) - if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom { + if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) { t.Errorf("heap goal is not the heap limit: %d", goal) } if n >= 25 { @@ -1019,6 +1019,19 @@ func (f float64Stream) limit(min, max float64) float64Stream { } } +func applyMemoryLimitHeapGoalHeadroom(goal uint64) uint64 { + headroom := goal / 100 * MemoryLimitHeapGoalHeadroomPercent + if headroom < MemoryLimitMinHeapGoalHeadroom { + headroom = MemoryLimitMinHeapGoalHeadroom + } + if goal < headroom || goal-headroom < headroom { + goal = headroom + } else { + goal -= headroom + } + return goal +} + func TestIdleMarkWorkerCount(t *testing.T) { const workers = 10 c := NewGCController(100, math.MaxInt64)