runtime: retype mheap.reclaimCredit as atomic.Uintptr

[git-generate]
cd src/runtime
mv export_test.go export.go
GOROOT=$(dirname $(dirname $PWD)) rf '
  add mheap.reclaimCredit \
// reclaimCredit is spare credit for extra pages swept. Since \
// the page reclaimer works in large chunks, it may reclaim \
// more than requested. Any spare pages released go to this \
// credit pool. \
reclaimCredit_ atomic.Uintptr
  ex {
    import "runtime/internal/atomic"

    var t mheap
    var v, w uintptr
    var d uintptr

    t.reclaimCredit -> t.reclaimCredit_.Load()
    t.reclaimCredit = v -> t.reclaimCredit_.Store(v)
    atomic.Loaduintptr(&t.reclaimCredit) -> t.reclaimCredit_.Load()
    atomic.LoadAcquintptr(&t.reclaimCredit) -> t.reclaimCredit_.LoadAcquire()
    atomic.Storeuintptr(&t.reclaimCredit, v) -> t.reclaimCredit_.Store(v)
    atomic.StoreReluintptr(&t.reclaimCredit, v) -> t.reclaimCredit_.StoreRelease(v)
    atomic.Casuintptr(&t.reclaimCredit, v, w) -> t.reclaimCredit_.CompareAndSwap(v, w)
    atomic.Xchguintptr(&t.reclaimCredit, v) -> t.reclaimCredit_.Swap(v)
    atomic.Xadduintptr(&t.reclaimCredit, d) -> t.reclaimCredit_.Add(d)
  }
  rm mheap.reclaimCredit
  mv mheap.reclaimCredit_ mheap.reclaimCredit
'
mv export.go export_test.go

Change-Id: I2c567781a28f5d8c2275ff18f2cf605b82f22d09
Reviewed-on: https://go-review.googlesource.com/c/go/+/356712
Trust: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 56ef1c4e384b1a0c96864d08a58933328ae5916b..6f8463c25369552d606f5edab5eab4feed5bde15 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1460,7 +1460,7 @@ func gcSweep(mode gcMode) {
        mheap_.pagesSwept.Store(0)
        mheap_.sweepArenas = mheap_.allArenas
        mheap_.reclaimIndex.Store(0)
-       mheap_.reclaimCredit = 0
+       mheap_.reclaimCredit.Store(0)
        unlock(&mheap_.lock)
 
        sweep.centralIndex.clear()

diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go
index aedd6c316e2e32c66a328db240530c3fd32f2acc..9c7f9d340d2fdab266e5c30ae6d6d7a3212fe1e9 100644 (file)
--- a/src/runtime/mgcsweep.go
+++ b/src/runtime/mgcsweep.go
@@ -291,7 +291,7 @@ func sweepone() uintptr {
                                // Whole span was freed. Count it toward the
                                // page reclaimer credit since these pages can
                                // now be used for span allocation.
-                               atomic.Xadduintptr(&mheap_.reclaimCredit, npages)
+                               mheap_.reclaimCredit.Add(npages)
                        } else {
                                // Span is still in-use, so this returned no
                                // pages to the heap and the span needs to

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index fc86023f4db71a970710d21916a29983e065cfb1..0e7694aab77a60d1fa05227a4f3508acd937dafd 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -124,13 +124,12 @@ type mheap struct {
        // If this is >= 1<<63, the page reclaimer is done scanning
        // the page marks.
        reclaimIndex atomic.Uint64
+
        // reclaimCredit is spare credit for extra pages swept. Since
        // the page reclaimer works in large chunks, it may reclaim
        // more than requested. Any spare pages released go to this
        // credit pool.
-       //
-       // This is accessed atomically.
-       reclaimCredit uintptr
+       reclaimCredit atomic.Uintptr
 
        // arenas is the heap arena map. It points to the metadata for
        // the heap for every arena frame of the entire usable virtual
@@ -754,13 +753,13 @@ func (h *mheap) reclaim(npage uintptr) {
        locked := false
        for npage > 0 {
                // Pull from accumulated credit first.
-               if credit := atomic.Loaduintptr(&h.reclaimCredit); credit > 0 {
+               if credit := h.reclaimCredit.Load(); credit > 0 {
                        take := credit
                        if take > npage {
                                // Take only what we need.
                                take = npage
                        }
-                       if atomic.Casuintptr(&h.reclaimCredit, credit, credit-take) {
+                       if h.reclaimCredit.CompareAndSwap(credit, credit-take) {
                                npage -= take
                        }
                        continue
@@ -786,7 +785,7 @@ func (h *mheap) reclaim(npage uintptr) {
                        npage -= nfound
                } else {
                        // Put spare pages toward global credit.
-                       atomic.Xadduintptr(&h.reclaimCredit, nfound-npage)
+                       h.reclaimCredit.Add(nfound - npage)
                        npage = 0
                }
        }