runtime: use new list type for spanSPMCs

Now that we have a reusable list type, use it to replace the custom
linked list code for spanSPMCs.

Change-Id: I6a6a636ca54f2ba4b5c7dddba607c94ebf3c3ac8
Reviewed-on: https://go-review.googlesource.com/c/go/+/714021
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Auto-Submit: Michael Pratt <mpratt@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index bf9d9fd8883b65b33429c7f739707b07547221d0..22150dfd170390daf102c22cae0d77b4791a71d0 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -195,6 +195,7 @@ func gcinit() {
 
        work.startSema = 1
        work.markDoneSema = 1
+       work.spanSPMCs.list.init(unsafe.Offsetof(spanSPMC{}.allnode))
        lockInit(&work.sweepWaiters.lock, lockRankSweepWaiters)
        lockInit(&work.assistQueue.lock, lockRankAssistQueue)
        lockInit(&work.strongFromWeak.lock, lockRankStrongFromWeakQueue)
@@ -354,7 +355,7 @@ type workType struct {
        // Only used if goexperiment.GreenTeaGC.
        spanSPMCs struct {
                lock mutex
-               all  *spanSPMC
+               list listHeadManual // *spanSPMC
        }
 
        // Restore 64-bit alignment on 32-bit.

diff --git a/src/runtime/mgcmark_greenteagc.go b/src/runtime/mgcmark_greenteagc.go
index 7b78611cf7b6feed5b126c3af07aa1d48832b2fe..3594b33cfd4c56c2472ef35a884e05b791797f92 100644 (file)
--- a/src/runtime/mgcmark_greenteagc.go
+++ b/src/runtime/mgcmark_greenteagc.go
@@ -635,20 +635,9 @@ func (q *spanQueue) destroy() {
 
        lock(&work.spanSPMCs.lock)
 
-       // Remove and free each ring.
+       // Remove, deinitialize, and free each ring.
        for r := (*spanSPMC)(q.chain.tail.Load()); r != nil; r = (*spanSPMC)(r.prev.Load()) {
-               prev := r.allprev
-               next := r.allnext
-               if prev != nil {
-                       prev.allnext = next
-               }
-               if next != nil {
-                       next.allprev = prev
-               }
-               if work.spanSPMCs.all == r {
-                       work.spanSPMCs.all = next
-               }
-
+               work.spanSPMCs.list.remove(unsafe.Pointer(r))
                r.deinit()
                mheap_.spanSPMCAlloc.free(unsafe.Pointer(r))
        }
@@ -688,15 +677,10 @@ func (q *spanQueue) destroy() {
 type spanSPMC struct {
        _ sys.NotInHeap
 
-       // allnext is the link to the next spanSPMC on the work.spanSPMCs list.
-       // This is used to find and free dead spanSPMCs. Protected by
-       // work.spanSPMCs.lock.
-       allnext *spanSPMC
-
-       // allprev is the link to the previous spanSPMC on the work.spanSPMCs
-       // list. This is used to find and free dead spanSPMCs. Protected by
+       // allnode is the linked list node for work.spanSPMCs list. This is
+       // used to find and free dead spanSPMCs. Protected by
        // work.spanSPMCs.lock.
-       allprev *spanSPMC
+       allnode listNodeManual
 
        // dead indicates whether the spanSPMC is no longer in use.
        // Protected by the CAS to the prev field of the spanSPMC pointing
@@ -724,12 +708,7 @@ type spanSPMC struct {
 func newSpanSPMC(cap uint32) *spanSPMC {
        lock(&work.spanSPMCs.lock)
        r := (*spanSPMC)(mheap_.spanSPMCAlloc.alloc())
-       next := work.spanSPMCs.all
-       r.allnext = next
-       if next != nil {
-               next.allprev = r
-       }
-       work.spanSPMCs.all = r
+       work.spanSPMCs.list.push(unsafe.Pointer(r))
        unlock(&work.spanSPMCs.lock)
 
        // If cap < the capacity of a single physical page, round up.
@@ -765,8 +744,7 @@ func (r *spanSPMC) deinit() {
        r.head.Store(0)
        r.tail.Store(0)
        r.cap = 0
-       r.allnext = nil
-       r.allprev = nil
+       r.allnode = listNodeManual{}
 }
 
 // slot returns a pointer to slot i%r.cap.
@@ -795,26 +773,16 @@ func freeDeadSpanSPMCs() {
        // GOMAXPROCS, or if this list otherwise gets long, it would be nice to
        // have a way to batch work that allows preemption during processing.
        lock(&work.spanSPMCs.lock)
-       if gcphase != _GCoff || work.spanSPMCs.all == nil {
+       if gcphase != _GCoff || work.spanSPMCs.list.empty() {
                unlock(&work.spanSPMCs.lock)
                return
        }
-       r := work.spanSPMCs.all
+       r := (*spanSPMC)(work.spanSPMCs.list.head())
        for r != nil {
-               next := r.allnext
+               next := (*spanSPMC)(unsafe.Pointer(r.allnode.next))
                if r.dead.Load() {
-                       // It's dead. Deinitialize and free it.
-                       prev := r.allprev
-                       if prev != nil {
-                               prev.allnext = next
-                       }
-                       if next != nil {
-                               next.allprev = prev
-                       }
-                       if work.spanSPMCs.all == r {
-                               work.spanSPMCs.all = next
-                       }
-
+                       // It's dead. Remove, deinitialize and free it.
+                       work.spanSPMCs.list.remove(unsafe.Pointer(r))
                        r.deinit()
                        mheap_.spanSPMCAlloc.free(unsafe.Pointer(r))
                }

diff --git a/src/runtime/mgcmark_nogreenteagc.go b/src/runtime/mgcmark_nogreenteagc.go
index 024565ef3e250d221ee7ed7250be3f5534aa4dd9..a0470c6e32298a62d9ce6a8e8fc643008872bf52 100644 (file)
--- a/src/runtime/mgcmark_nogreenteagc.go
+++ b/src/runtime/mgcmark_nogreenteagc.go
@@ -68,6 +68,7 @@ func (q *spanQueue) destroy() {
 
 type spanSPMC struct {
        _ sys.NotInHeap
+       allnode listNodeManual
 }
 
 func freeDeadSpanSPMCs() {