runtime: change p.runq from []*g to []guintptr

This eliminates many write barriers in the scheduler code that are
unnecessary and will interfere with upcoming changes where the garbage
collector will have to invoke run queue functions in contexts that
must not have write barriers.

Change-Id: I702d0ac99cfd00ffff406e7362917db6a43e7e55
Reviewed-on: https://go-review.googlesource.com/16556
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 714421f5f327ecaa56cd819f813a8c3c8ea1fbeb..41e5ea9751c450604321689fe5e81583d67b362c 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -3064,7 +3064,7 @@ func procresize(nprocs int32) *p {
                for p.runqhead != p.runqtail {
                        // pop from tail of local queue
                        p.runqtail--
-                       gp := p.runq[p.runqtail%uint32(len(p.runq))]
+                       gp := p.runq[p.runqtail%uint32(len(p.runq))].ptr()
                        // push onto head of global queue
                        globrunqputhead(gp)
                }
@@ -3753,7 +3753,7 @@ retry:
        h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers
        t := _p_.runqtail
        if t-h < uint32(len(_p_.runq)) {
-               _p_.runq[t%uint32(len(_p_.runq))] = gp
+               _p_.runq[t%uint32(len(_p_.runq))].set(gp)
                atomicstore(&_p_.runqtail, t+1) // store-release, makes the item available for consumption
                return
        }
@@ -3776,7 +3776,7 @@ func runqputslow(_p_ *p, gp *g, h, t uint32) bool {
                throw("runqputslow: queue is not full")
        }
        for i := uint32(0); i < n; i++ {
-               batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))]
+               batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))].ptr()
        }
        if !cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
                return false
@@ -3824,7 +3824,7 @@ func runqget(_p_ *p) (gp *g, inheritTime bool) {
                if t == h {
                        return nil, false
                }
-               gp := _p_.runq[h%uint32(len(_p_.runq))]
+               gp := _p_.runq[h%uint32(len(_p_.runq))].ptr()
                if cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume
                        return gp, false
                }
@@ -3835,7 +3835,7 @@ func runqget(_p_ *p) (gp *g, inheritTime bool) {
 // Batch is a ring buffer starting at batchHead.
 // Returns number of grabbed goroutines.
 // Can be executed by any P.
-func runqgrab(_p_ *p, batch *[256]*g, batchHead uint32, stealRunNextG bool) uint32 {
+func runqgrab(_p_ *p, batch *[256]guintptr, batchHead uint32, stealRunNextG bool) uint32 {
        for {
                h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers
                t := atomicload(&_p_.runqtail) // load-acquire, synchronize with the producer
@@ -3856,7 +3856,7 @@ func runqgrab(_p_ *p, batch *[256]*g, batchHead uint32, stealRunNextG bool) uint
                                        if !_p_.runnext.cas(next, 0) {
                                                continue
                                        }
-                                       batch[batchHead%uint32(len(batch))] = next.ptr()
+                                       batch[batchHead%uint32(len(batch))] = next
                                        return 1
                                }
                        }
@@ -3885,7 +3885,7 @@ func runqsteal(_p_, p2 *p, stealRunNextG bool) *g {
                return nil
        }
        n--
-       gp := _p_.runq[(t+n)%uint32(len(_p_.runq))]
+       gp := _p_.runq[(t+n)%uint32(len(_p_.runq))].ptr()
        if n == 0 {
                return gp
        }

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index f1337e570eca5b1bab9e4d3b8fae75be2dc5017f..a1ea3e78f0d8fbb4a1621c22f705f27b6a020a7c 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -358,7 +358,7 @@ type p struct {
        // Queue of runnable goroutines. Accessed without lock.
        runqhead uint32
        runqtail uint32
-       runq     [256]*g
+       runq     [256]guintptr
        // runnext, if non-nil, is a runnable G that was ready'd by
        // the current G and should be run next instead of what's in
        // runq if there's time remaining in the running G's time