runtime: more changes in preparation to the new scheduler

add per-P cache of dead G's
add global runnable queue (not used for now)
add list of idle P's (not used for now)

R=golang-dev, rsc
CC=golang-dev
https://golang.org/cl/7397061

diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c
index 3a5652c91559c3dbc3d7809634ff8371577e6356..5ba8a4caa3ee639cdd8f4b68a65b22be0b1e27e3 100644 (file)
--- a/src/pkg/runtime/proc.c
+++ b/src/pkg/runtime/proc.c
@@ -60,7 +60,6 @@ int32 runtime·ncpu;
 struct Sched {
        Lock;
 
-       G *gfree;       // available g's (status == Gdead)
        int64 goidgen;
 
        G *ghead;       // g's waiting to run
@@ -73,6 +72,20 @@ struct Sched {
        int32 mwait;    // number of m's waiting for work
        int32 mcount;   // number of m's that have been created
 
+       P       p;  // temporary
+
+       P*      pidle;  // idle P's
+       uint32  npidle;
+
+       // Global runnable queue.
+       G*      runqhead;
+       G*      runqtail;
+       int32   runqsize;
+
+       // Global cache of dead G's.
+       Lock    gflock;
+       G*      gfree;
+
        volatile uint32 atomic; // atomic scheduling word (see below)
 
        int32 profilehz;        // cpu profiling rate
@@ -148,8 +161,9 @@ static void gput(G*);       // put/get on ghead/gtail
 static G* gget(void);
 static void mput(M*);  // put/get on mhead
 static M* mget(G*);
-static void gfput(G*); // put/get on gfree
-static G* gfget(void);
+static void gfput(P*, G*);
+static G* gfget(P*);
+static void gfpurge(P*);
 static void matchmg(void);     // match m's to g's
 static void readylocked(G*);   // ready, but sched is locked
 static void mnextg(M*, G*);
@@ -158,6 +172,10 @@ static void runqput(P*, G*);
 static G* runqget(P*);
 static void runqgrow(P*);
 static G* runqsteal(P*, P*);
+static void globrunqput(G*);
+static G* globrunqget(P*);
+static P* pidleget(void);
+static void pidleput(P*);
 
 void
 setmcpumax(uint32 n)
@@ -1153,7 +1171,7 @@ schedule(G *gp)
                        }
                        gp->idlem = nil;
                        runtime·unwindstack(gp, nil);
-                       gfput(gp);
+                       gfput(&runtime·sched.p, gp);
                        if(--runtime·sched.gcount == 0)
                                runtime·exit(0);
                        break;
@@ -1477,7 +1495,7 @@ runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerp
 
        schedlock();
 
-       if((newg = gfget()) != nil) {
+       if((newg = gfget(&runtime·sched.p)) != nil) {
                if(newg->stackguard - StackGuard != newg->stack0)
                        runtime·throw("invalid stack in newg");
        } else {
@@ -1518,28 +1536,74 @@ runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerp
 //printf(" goid=%d\n", newg->goid);
 }
 
-// Put on gfree list.  Sched must be locked.
+// Put on gfree list.
+// If local list is too long, transfer a batch to the global list.
 static void
-gfput(G *gp)
+gfput(P *p, G *gp)
 {
        if(gp->stackguard - StackGuard != gp->stack0)
                runtime·throw("invalid stack in gfput");
-       gp->schedlink = runtime·sched.gfree;
-       runtime·sched.gfree = gp;
+       gp->schedlink = p->gfree;
+       p->gfree = gp;
+       p->gfreecnt++;
+       if(p->gfreecnt >= 64) {
+               runtime·lock(&runtime·sched.gflock);
+               while(p->gfreecnt >= 32) {
+                       p->gfreecnt--;
+                       gp = p->gfree;
+                       p->gfree = gp->schedlink;
+                       gp->schedlink = runtime·sched.gfree;
+                       runtime·sched.gfree = gp;
+               }
+               runtime·unlock(&runtime·sched.gflock);
+       }
 }
 
-// Get from gfree list.  Sched must be locked.
+// Get from gfree list.
+// If local list is empty, grab a batch from global list.
 static G*
-gfget(void)
+gfget(P *p)
 {
        G *gp;
 
-       gp = runtime·sched.gfree;
-       if(gp)
-               runtime·sched.gfree = gp->schedlink;
+retry:
+       gp = p->gfree;
+       if(gp == nil && runtime·sched.gfree) {
+               runtime·lock(&runtime·sched.gflock);
+               while(p->gfreecnt < 32 && runtime·sched.gfree) {
+                       p->gfreecnt++;
+                       gp = runtime·sched.gfree;
+                       runtime·sched.gfree = gp->schedlink;
+                       gp->schedlink = p->gfree;
+                       p->gfree = gp;
+               }
+               runtime·unlock(&runtime·sched.gflock);
+               goto retry;
+       }
+       if(gp) {
+               p->gfree = gp->schedlink;
+               p->gfreecnt--;
+       }
        return gp;
 }
 
+// Purge all cached G's from gfree list to the global list.
+static void
+gfpurge(P *p)
+{
+       G *gp;
+
+       runtime·lock(&runtime·sched.gflock);
+       while(p->gfreecnt) {
+               p->gfreecnt--;
+               gp = p->gfree;
+               p->gfree = gp->schedlink;
+               gp->schedlink = runtime·sched.gfree;
+               runtime·sched.gfree = gp;
+       }
+       runtime·unlock(&runtime·sched.gflock);
+}
+
 void
 runtime·Breakpoint(void)
 {
@@ -1761,6 +1825,72 @@ runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz)
                runtime·resetcpuprofiler(hz);
 }
 
+// Put gp on the global runnable queue.
+// Sched must be locked.
+static void
+globrunqput(G *gp)
+{
+       gp->schedlink = nil;
+       if(runtime·sched.runqtail)
+               runtime·sched.runqtail->schedlink = gp;
+       else
+               runtime·sched.runqhead = gp;
+       runtime·sched.runqtail = gp;
+       runtime·sched.runqsize++;
+}
+
+// Try get a batch of G's from the global runnable queue.
+// Sched must be locked.
+static G*
+globrunqget(P *p)
+{
+       G *gp, *gp1;
+       int32 n;
+
+       if(runtime·sched.runqsize == 0)
+               return nil;
+       n = runtime·sched.runqsize/runtime·gomaxprocs+1;
+       if(n > runtime·sched.runqsize)
+               n = runtime·sched.runqsize;
+       runtime·sched.runqsize -= n;
+       if(runtime·sched.runqsize == 0)
+               runtime·sched.runqtail = nil;
+       gp = runtime·sched.runqhead;
+       runtime·sched.runqhead = gp->schedlink;
+       n--;
+       while(n--) {
+               gp1 = runtime·sched.runqhead;
+               runtime·sched.runqhead = gp1->schedlink;
+               runqput(p, gp1);
+       }
+       return gp;
+}
+
+// Put p to on pidle list.
+// Sched must be locked.
+static void
+pidleput(P *p)
+{
+       p->link = runtime·sched.pidle;
+       runtime·sched.pidle = p;
+       runtime·sched.npidle++;
+}
+
+// Try get a p from pidle list.
+// Sched must be locked.
+static P*
+pidleget(void)
+{
+       P *p;
+
+       p = runtime·sched.pidle;
+       if(p) {
+               runtime·sched.pidle = p->link;
+               runtime·sched.npidle--;
+       }
+       return p;
+}
+
 // Put g on local runnable queue.
 // TODO(dvyukov): consider using lock-free queue.
 static void

diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h
index 6d7a3152ff56ed4faa54e6d1787619564d128e1b..11f455780221826f70c39526d1a1eaf0bf9c72c8 100644 (file)
--- a/src/pkg/runtime/runtime.h
+++ b/src/pkg/runtime/runtime.h
@@ -317,11 +317,19 @@ struct P
 {
        Lock;
 
+       P*      link;
+
        // Queue of runnable goroutines.
        G**     runq;
        int32   runqhead;
        int32   runqtail;
        int32   runqsize;
+
+       // Available G's (status == Gdead)
+       G*      gfree;
+       int32   gfreecnt;
+
+       byte    pad[64];
 };
 
 // The m->locked word holds a single bit saying whether