From d419a80bc76cf18d321f9bf37c2deb2b8c0a1415 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Fri, 15 Oct 2021 19:22:10 +0000 Subject: [PATCH] runtime: retype mheap.pagesInUse as atomic.Uint64 [git-generate] cd src/runtime mv export_test.go export.go GOROOT=$(dirname $(dirname $PWD)) rf ' add mheap.pagesInUse \ // Proportional sweep \ // \ // These parameters represent a linear function from gcController.heapLive \ // to page sweep count. The proportional sweep system works to \ // stay in the black by keeping the current page sweep count \ // above this line at the current gcController.heapLive. \ // \ // The line has slope sweepPagesPerByte and passes through a \ // basis point at (sweepHeapLiveBasis, pagesSweptBasis). At \ // any given time, the system is at (gcController.heapLive, \ // pagesSwept) in this space. \ // \ // It is important that the line pass through a point we \ // control rather than simply starting at a 0,0 origin \ // because that lets us adjust sweep pacing at any time while \ // accounting for current progress. If we could only adjust \ // the slope, it would create a discontinuity in debt if any \ // progress has already been made. \ pagesInUse_ atomic.Uint64 // pages of spans in stats mSpanInUse ex { import "runtime/internal/atomic" var t mheap var v, w uint64 var d int64 t.pagesInUse -> t.pagesInUse_.Load() t.pagesInUse = v -> t.pagesInUse_.Store(v) atomic.Load64(&t.pagesInUse) -> t.pagesInUse_.Load() atomic.LoadAcq64(&t.pagesInUse) -> t.pagesInUse_.LoadAcquire() atomic.Store64(&t.pagesInUse, v) -> t.pagesInUse_.Store(v) atomic.StoreRel64(&t.pagesInUse, v) -> t.pagesInUse_.StoreRelease(v) atomic.Cas64(&t.pagesInUse, v, w) -> t.pagesInUse_.CompareAndSwap(v, w) atomic.Xchg64(&t.pagesInUse, v) -> t.pagesInUse_.Swap(v) atomic.Xadd64(&t.pagesInUse, d) -> t.pagesInUse_.Add(d) } rm mheap.pagesInUse mv mheap.pagesInUse_ mheap.pagesInUse ' mv export.go export_test.go Change-Id: I495d188683dba0778518563c46755b5ad43be298 Reviewed-on: https://go-review.googlesource.com/c/go/+/356549 Trust: Michael Knyszek Reviewed-by: Austin Clements --- src/runtime/export_test.go | 2 +- src/runtime/mgcpacer.go | 2 +- src/runtime/mheap.go | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 01fd1dbd97..01f917bded 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -263,7 +263,7 @@ var ReadUnaligned64 = readUnaligned64 func CountPagesInUse() (pagesInUse, counted uintptr) { stopTheWorld("CountPagesInUse") - pagesInUse = uintptr(mheap_.pagesInUse) + pagesInUse = uintptr(mheap_.pagesInUse.Load()) for _, s := range mheap_.allspans { if s.state.get() == mSpanInUse { diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go index 342ea419fe..3cd4ea2fde 100644 --- a/src/runtime/mgcpacer.go +++ b/src/runtime/mgcpacer.go @@ -752,7 +752,7 @@ func (c *gcControllerState) commit(triggerRatio float64) { heapDistance = _PageSize } pagesSwept := atomic.Load64(&mheap_.pagesSwept) - pagesInUse := atomic.Load64(&mheap_.pagesInUse) + pagesInUse := mheap_.pagesInUse.Load() sweepDistancePages := int64(pagesInUse) - int64(pagesSwept) if sweepDistancePages <= 0 { mheap_.sweepPagesPerByte = 0 diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index b78f752ded..a787f67460 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -96,17 +96,17 @@ type mheap struct { // any given time, the system is at (gcController.heapLive, // pagesSwept) in this space. // - // It's important that the line pass through a point we - // control rather than simply starting at a (0,0) origin + // It is important that the line pass through a point we + // control rather than simply starting at a 0,0 origin // because that lets us adjust sweep pacing at any time while // accounting for current progress. If we could only adjust // the slope, it would create a discontinuity in debt if any // progress has already been made. - pagesInUse uint64 // pages of spans in stats mSpanInUse; updated atomically - pagesSwept uint64 // pages swept this cycle; updated atomically - pagesSweptBasis uint64 // pagesSwept to use as the origin of the sweep ratio; updated atomically - sweepHeapLiveBasis uint64 // value of gcController.heapLive to use as the origin of sweep ratio; written with lock, read without - sweepPagesPerByte float64 // proportional sweep ratio; written with lock, read without + pagesInUse atomic.Uint64 // pages of spans in stats mSpanInUse + pagesSwept uint64 // pages swept this cycle; updated atomically + pagesSweptBasis uint64 // pagesSwept to use as the origin of the sweep ratio; updated atomically + sweepHeapLiveBasis uint64 // value of gcController.heapLive to use as the origin of sweep ratio; written with lock, read without + sweepPagesPerByte float64 // proportional sweep ratio; written with lock, read without // TODO(austin): pagesInUse should be a uintptr, but the 386 // compiler can't 8-byte align fields. @@ -1311,7 +1311,7 @@ HaveSpan: atomic.Or8(&arena.pageInUse[pageIdx], pageMask) // Update related page sweeper stats. - atomic.Xadd64(&h.pagesInUse, int64(npages)) + h.pagesInUse.Add(int64(npages)) } // Make sure the newly allocated span will be observed @@ -1468,7 +1468,7 @@ func (h *mheap) freeSpanLocked(s *mspan, typ spanAllocType) { print("mheap.freeSpanLocked - span ", s, " ptr ", hex(s.base()), " allocCount ", s.allocCount, " sweepgen ", s.sweepgen, "/", h.sweepgen, "\n") throw("mheap.freeSpanLocked - invalid free") } - atomic.Xadd64(&h.pagesInUse, -int64(s.npages)) + h.pagesInUse.Add(-int64(s.npages)) // Clear in-use bit in arena page bitmap. arena, pageIdx, pageMask := pageIndexOf(s.base()) -- 2.50.0