runtime: make work.spanSPMCs.all doubly-linked

Making this a doubly-linked list allows spanQueue.destroy to immediately
remove and free rings rather than simply marking them as dead and
waiting for the sweeper to deal with them.

For #75771.

Change-Id: I6a6a636c0fb6be08ee967cb6d8f0577511a33c13
Reviewed-on: https://go-review.googlesource.com/c/go/+/709657
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Knyszek <mknyszek@google.com>

diff --git a/src/runtime/mgcmark_greenteagc.go b/src/runtime/mgcmark_greenteagc.go
index 6ebd7ced81b9018885ae4df5681309463fbaefa4..7b78611cf7b6feed5b126c3af07aa1d48832b2fe 100644 (file)
--- a/src/runtime/mgcmark_greenteagc.go
+++ b/src/runtime/mgcmark_greenteagc.go
@@ -635,14 +635,22 @@ func (q *spanQueue) destroy() {
 
        lock(&work.spanSPMCs.lock)
 
-       // Mark each ring as dead. The sweeper will actually free them.
-       //
-       // N.B., we could free directly here, but work.spanSPMCs.all is a
-       // singly-linked list, so we'd need to walk the entire list to find the
-       // previous node. If the list becomes doubly-linked, we can free
-       // directly.
+       // Remove and free each ring.
        for r := (*spanSPMC)(q.chain.tail.Load()); r != nil; r = (*spanSPMC)(r.prev.Load()) {
-               r.dead.Store(true)
+               prev := r.allprev
+               next := r.allnext
+               if prev != nil {
+                       prev.allnext = next
+               }
+               if next != nil {
+                       next.allprev = prev
+               }
+               if work.spanSPMCs.all == r {
+                       work.spanSPMCs.all = next
+               }
+
+               r.deinit()
+               mheap_.spanSPMCAlloc.free(unsafe.Pointer(r))
        }
 
        q.chain.head = nil
@@ -685,6 +693,11 @@ type spanSPMC struct {
        // work.spanSPMCs.lock.
        allnext *spanSPMC
 
+       // allprev is the link to the previous spanSPMC on the work.spanSPMCs
+       // list. This is used to find and free dead spanSPMCs. Protected by
+       // work.spanSPMCs.lock.
+       allprev *spanSPMC
+
        // dead indicates whether the spanSPMC is no longer in use.
        // Protected by the CAS to the prev field of the spanSPMC pointing
        // to this spanSPMC. That is, whoever wins that CAS takes ownership
@@ -711,7 +724,11 @@ type spanSPMC struct {
 func newSpanSPMC(cap uint32) *spanSPMC {
        lock(&work.spanSPMCs.lock)
        r := (*spanSPMC)(mheap_.spanSPMCAlloc.alloc())
-       r.allnext = work.spanSPMCs.all
+       next := work.spanSPMCs.all
+       r.allnext = next
+       if next != nil {
+               next.allprev = r
+       }
        work.spanSPMCs.all = r
        unlock(&work.spanSPMCs.lock)
 
@@ -748,6 +765,8 @@ func (r *spanSPMC) deinit() {
        r.head.Store(0)
        r.tail.Store(0)
        r.cap = 0
+       r.allnext = nil
+       r.allprev = nil
 }
 
 // slot returns a pointer to slot i%r.cap.
@@ -780,22 +799,26 @@ func freeDeadSpanSPMCs() {
                unlock(&work.spanSPMCs.lock)
                return
        }
-       rp := &work.spanSPMCs.all
-       for {
-               r := *rp
-               if r == nil {
-                       break
-               }
+       r := work.spanSPMCs.all
+       for r != nil {
+               next := r.allnext
                if r.dead.Load() {
                        // It's dead. Deinitialize and free it.
-                       *rp = r.allnext
+                       prev := r.allprev
+                       if prev != nil {
+                               prev.allnext = next
+                       }
+                       if next != nil {
+                               next.allprev = prev
+                       }
+                       if work.spanSPMCs.all == r {
+                               work.spanSPMCs.all = next
+                       }
+
                        r.deinit()
                        mheap_.spanSPMCAlloc.free(unsafe.Pointer(r))
-               } else {
-                       // Still alive, likely in some P's chain.
-                       // Skip it.
-                       rp = &r.allnext
                }
+               r = next
        }
        unlock(&work.spanSPMCs.lock)
 }