runtime: doubly-linked sched.midle list

This will be used by CL 714801 to remove Ms from the middle of the list.

We could simply convert schedlink to the doubly-linked list, bringing
along all other uses of schedlink.

However, CL 714801 removes Ms from the middle of the midle list. It
would be an easy mistake to make to accidentally remove an M from
schedlink, assuming that it is on the midle list when it is actually on
a completely different list. Using separate a list node makes this
impossible.

For #65694.

Change-Id: I6a6a636c223d925fdc30c0c648460cbf5c2af4d6
Reviewed-on: https://go-review.googlesource.com/c/go/+/714800
Reviewed-by: Michael Knyszek <mknyszek@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Michael Pratt <mpratt@google.com>

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 91740d1fa6d58faa3de598635998fd7c8b5cc0e2..30d2a6862603f88897d1feb89f47c3e0958d8d0b 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -849,6 +849,8 @@ func schedinit() {
 
        lockVerifyMSize()
 
+       sched.midle.init(unsafe.Offsetof(m{}.idleNode))
+
        // raceinit must be the first call to race detector.
        // In particular, it must be done before mallocinit below calls racemapshadow.
        gp := getg()
@@ -6976,8 +6978,7 @@ func schedEnabled(gp *g) bool {
 func mput(mp *m) {
        assertLockHeld(&sched.lock)
 
-       mp.schedlink = sched.midle
-       sched.midle.set(mp)
+       sched.midle.push(unsafe.Pointer(mp))
        sched.nmidle++
        checkdead()
 }
@@ -6990,9 +6991,8 @@ func mput(mp *m) {
 func mget() *m {
        assertLockHeld(&sched.lock)
 
-       mp := sched.midle.ptr()
+       mp := (*m)(sched.midle.pop())
        if mp != nil {
-               sched.midle = mp.schedlink
                sched.nmidle--
        }
        return mp

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index 6d1f9b13a28563b7ae57820963dd8aea19c8b0b0..2b79717703167fdfc05250b4ebe749c6eacdb167 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -665,6 +665,7 @@ type m struct {
        park            note
        alllink         *m // on allm
        schedlink       muintptr
+       idleNode        listNodeManual
        lockedg         guintptr
        createstack     [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it.
        lockedExt       uint32      // tracking for external LockOSThread
@@ -875,7 +876,7 @@ type schedt struct {
        // When increasing nmidle, nmidlelocked, nmsys, or nmfreed, be
        // sure to call checkdead().
 
-       midle        muintptr // idle m's waiting for work
+       midle        listHeadManual // idle m's waiting for work
        nmidle       int32    // number of idle m's waiting for work
        nmidlelocked int32    // number of locked m's waiting for work
        mnext        int64    // number of m's that have been created and next M ID