From: Michael Anthony Knyszek <mknyszek@google.com>
Date: Mon, 18 Oct 2021 23:12:16 +0000 (+0000)
Subject: runtime: retype mheap.reclaimIndex as atomic.Uint64
X-Git-Tag: go1.18beta1~818
X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=a91e976fd2cc6f1c156877eae40262d7e47c745c;p=gostls13.git

runtime: retype mheap.reclaimIndex as atomic.Uint64

[git-generate]
cd src/runtime
mv export_test.go export.go
GOROOT=$(dirname $(dirname $PWD)) rf '
  add mheap.reclaimIndex \
// reclaimIndex is the page index in allArenas of next page to \
// reclaim. Specifically, it refers to page (i % \
// pagesPerArena) of arena allArenas[i / pagesPerArena]. \
// \
// If this is >= 1<<63, the page reclaimer is done scanning \
// the page marks. \
reclaimIndex_ atomic.Uint64
  ex {
    import "runtime/internal/atomic"

    var t mheap
    var v, w uint64
    var d int64

    t.reclaimIndex -> t.reclaimIndex_.Load()
    t.reclaimIndex = v -> t.reclaimIndex_.Store(v)
    atomic.Load64(&t.reclaimIndex) -> t.reclaimIndex_.Load()
    atomic.LoadAcq64(&t.reclaimIndex) -> t.reclaimIndex_.LoadAcquire()
    atomic.Store64(&t.reclaimIndex, v) -> t.reclaimIndex_.Store(v)
    atomic.StoreRel64(&t.reclaimIndex, v) -> t.reclaimIndex_.StoreRelease(v)
    atomic.Cas64(&t.reclaimIndex, v, w) -> t.reclaimIndex_.CompareAndSwap(v, w)
    atomic.Xchg64(&t.reclaimIndex, v) -> t.reclaimIndex_.Swap(v)
    atomic.Xadd64(&t.reclaimIndex, d) -> t.reclaimIndex_.Add(d)
  }
  rm mheap.reclaimIndex
  mv mheap.reclaimIndex_ mheap.reclaimIndex
'
mv export.go export_test.go

Change-Id: I1d619e3ac032285b5f7eb6c563a5188c8e36d089
Reviewed-on: https://go-review.googlesource.com/c/go/+/356711
Reviewed-by: Austin Clements <austin@google.com>
Trust: Michael Knyszek <mknyszek@google.com>
---

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 654fa4118a..56ef1c4e38 100644
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1459,7 +1459,7 @@ func gcSweep(mode gcMode) {
 	mheap_.sweepDrained = 0
 	mheap_.pagesSwept.Store(0)
 	mheap_.sweepArenas = mheap_.allArenas
-	mheap_.reclaimIndex = 0
+	mheap_.reclaimIndex.Store(0)
 	mheap_.reclaimCredit = 0
 	unlock(&mheap_.lock)
 
diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index 90e55315a6..fc86023f4d 100644
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -123,9 +123,7 @@ type mheap struct {
 	//
 	// If this is >= 1<<63, the page reclaimer is done scanning
 	// the page marks.
-	//
-	// This is accessed atomically.
-	reclaimIndex uint64
+	reclaimIndex atomic.Uint64
 	// reclaimCredit is spare credit for extra pages swept. Since
 	// the page reclaimer works in large chunks, it may reclaim
 	// more than requested. Any spare pages released go to this
@@ -739,7 +737,7 @@ func (h *mheap) reclaim(npage uintptr) {
 	// batching heap frees.
 
 	// Bail early if there's no more reclaim work.
-	if atomic.Load64(&h.reclaimIndex) >= 1<<63 {
+	if h.reclaimIndex.Load() >= 1<<63 {
 		return
 	}
 
@@ -769,10 +767,10 @@ func (h *mheap) reclaim(npage uintptr) {
 		}
 
 		// Claim a chunk of work.
-		idx := uintptr(atomic.Xadd64(&h.reclaimIndex, pagesPerReclaimerChunk) - pagesPerReclaimerChunk)
+		idx := uintptr(h.reclaimIndex.Add(pagesPerReclaimerChunk) - pagesPerReclaimerChunk)
 		if idx/pagesPerArena >= uintptr(len(arenas)) {
 			// Page reclaiming is done.
-			atomic.Store64(&h.reclaimIndex, 1<<63)
+			h.reclaimIndex.Store(1 << 63)
 			break
 		}