runtime: split gfree list into with-stacks and without-stacks

Currently all free Gs are added to one list. Split this into two
lists: one for free Gs with cached stacks and one for Gs without
cached stacks.

This lets us preferentially allocate Gs that already have a stack, but
more importantly, it sets us up to free cached G stacks concurrently.

Change-Id: Idbe486f708997e1c9d166662995283f02d1eeb3c
Reviewed-on: https://go-review.googlesource.com/20664
Reviewed-by: Rick Hudson <rlh@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index a847823da4b476b6b596b228113c48e1e007dbb5..9c840882b6cd7db4243308880a85d43aaa809907 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -2798,8 +2798,13 @@ func gfput(_p_ *p, gp *g) {
                        _p_.gfreecnt--
                        gp = _p_.gfree
                        _p_.gfree = gp.schedlink.ptr()
-                       gp.schedlink.set(sched.gfree)
-                       sched.gfree = gp
+                       if gp.stack.lo == 0 {
+                               gp.schedlink.set(sched.gfreeNoStack)
+                               sched.gfreeNoStack = gp
+                       } else {
+                               gp.schedlink.set(sched.gfreeStack)
+                               sched.gfreeStack = gp
+                       }
                        sched.ngfree++
                }
                unlock(&sched.gflock)
@@ -2811,12 +2816,20 @@ func gfput(_p_ *p, gp *g) {
 func gfget(_p_ *p) *g {
 retry:
        gp := _p_.gfree
-       if gp == nil && sched.gfree != nil {
+       if gp == nil && (sched.gfreeStack != nil || sched.gfreeNoStack != nil) {
                lock(&sched.gflock)
-               for _p_.gfreecnt < 32 && sched.gfree != nil {
+               for _p_.gfreecnt < 32 {
+                       if sched.gfreeStack != nil {
+                               // Prefer Gs with stacks.
+                               gp = sched.gfreeStack
+                               sched.gfreeStack = gp.schedlink.ptr()
+                       } else if sched.gfreeNoStack != nil {
+                               gp = sched.gfreeNoStack
+                               sched.gfreeNoStack = gp.schedlink.ptr()
+                       } else {
+                               break
+                       }
                        _p_.gfreecnt++
-                       gp = sched.gfree
-                       sched.gfree = gp.schedlink.ptr()
                        sched.ngfree--
                        gp.schedlink.set(_p_.gfree)
                        _p_.gfree = gp
@@ -2853,8 +2866,13 @@ func gfpurge(_p_ *p) {
                _p_.gfreecnt--
                gp := _p_.gfree
                _p_.gfree = gp.schedlink.ptr()
-               gp.schedlink.set(sched.gfree)
-               sched.gfree = gp
+               if gp.stack.lo == 0 {
+                       gp.schedlink.set(sched.gfreeNoStack)
+                       sched.gfreeNoStack = gp
+               } else {
+                       gp.schedlink.set(sched.gfreeStack)
+                       sched.gfreeStack = gp
+               }
                sched.ngfree++
        }
        unlock(&sched.gflock)

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index 8cfe6b06e613a1b4bf25c2a6cc4ea59c99c6fdfa..0a988ce46947e8bfd455bbac241d44bf3c9688ea 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -523,9 +523,10 @@ type schedt struct {
        runqsize int32
 
        // Global cache of dead G's.
-       gflock mutex
-       gfree  *g
-       ngfree int32
+       gflock       mutex
+       gfreeStack   *g
+       gfreeNoStack *g
+       ngfree       int32
 
        // Central cache of sudog structs.
        sudoglock  mutex