runtime: retype mheap.reclaimIndex as atomic.Uint64

[git-generate]
cd src/runtime
mv export_test.go export.go
GOROOT=$(dirname $(dirname $PWD)) rf '
  add mheap.reclaimIndex \
// reclaimIndex is the page index in allArenas of next page to \
// reclaim. Specifically, it refers to page (i % \
// pagesPerArena) of arena allArenas[i / pagesPerArena]. \
// \
// If this is >= 1<<63, the page reclaimer is done scanning \
// the page marks. \
reclaimIndex_ atomic.Uint64
  ex {
    import "runtime/internal/atomic"

    var t mheap
    var v, w uint64
    var d int64

    t.reclaimIndex -> t.reclaimIndex_.Load()
    t.reclaimIndex = v -> t.reclaimIndex_.Store(v)
    atomic.Load64(&t.reclaimIndex) -> t.reclaimIndex_.Load()
    atomic.LoadAcq64(&t.reclaimIndex) -> t.reclaimIndex_.LoadAcquire()
    atomic.Store64(&t.reclaimIndex, v) -> t.reclaimIndex_.Store(v)
    atomic.StoreRel64(&t.reclaimIndex, v) -> t.reclaimIndex_.StoreRelease(v)
    atomic.Cas64(&t.reclaimIndex, v, w) -> t.reclaimIndex_.CompareAndSwap(v, w)
    atomic.Xchg64(&t.reclaimIndex, v) -> t.reclaimIndex_.Swap(v)
    atomic.Xadd64(&t.reclaimIndex, d) -> t.reclaimIndex_.Add(d)
  }
  rm mheap.reclaimIndex
  mv mheap.reclaimIndex_ mheap.reclaimIndex
'
mv export.go export_test.go

Change-Id: I1d619e3ac032285b5f7eb6c563a5188c8e36d089
Reviewed-on: https://go-review.googlesource.com/c/go/+/356711
Reviewed-by: Austin Clements <austin@google.com>
Trust: Michael Knyszek <mknyszek@google.com>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 654fa4118a068896e89b48ff251f669ad1a19f2c..56ef1c4e384b1a0c96864d08a58933328ae5916b 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1459,7 +1459,7 @@ func gcSweep(mode gcMode) {
        mheap_.sweepDrained = 0
        mheap_.pagesSwept.Store(0)
        mheap_.sweepArenas = mheap_.allArenas
-       mheap_.reclaimIndex = 0
+       mheap_.reclaimIndex.Store(0)
        mheap_.reclaimCredit = 0
        unlock(&mheap_.lock)
 

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index 90e55315a609c27fe4babf0533347b48817a3cae..fc86023f4db71a970710d21916a29983e065cfb1 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -123,9 +123,7 @@ type mheap struct {
        //
        // If this is >= 1<<63, the page reclaimer is done scanning
        // the page marks.
-       //
-       // This is accessed atomically.
-       reclaimIndex uint64
+       reclaimIndex atomic.Uint64
        // reclaimCredit is spare credit for extra pages swept. Since
        // the page reclaimer works in large chunks, it may reclaim
        // more than requested. Any spare pages released go to this
@@ -739,7 +737,7 @@ func (h *mheap) reclaim(npage uintptr) {
        // batching heap frees.
 
        // Bail early if there's no more reclaim work.
-       if atomic.Load64(&h.reclaimIndex) >= 1<<63 {
+       if h.reclaimIndex.Load() >= 1<<63 {
                return
        }
 
@@ -769,10 +767,10 @@ func (h *mheap) reclaim(npage uintptr) {
                }
 
                // Claim a chunk of work.
-               idx := uintptr(atomic.Xadd64(&h.reclaimIndex, pagesPerReclaimerChunk) - pagesPerReclaimerChunk)
+               idx := uintptr(h.reclaimIndex.Add(pagesPerReclaimerChunk) - pagesPerReclaimerChunk)
                if idx/pagesPerArena >= uintptr(len(arenas)) {
                        // Page reclaiming is done.
-                       atomic.Store64(&h.reclaimIndex, 1<<63)
+                       h.reclaimIndex.Store(1 << 63)
                        break
                }