From 5e963a826c4406963048a42ce7ea28f7487b0a36 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 11 Mar 2011 18:14:45 -0800 Subject: [PATCH] runtime: reduce lock contention via wakeup on scheduler unlock. R=rsc CC=golang-dev https://golang.org/cl/4275043 --- src/pkg/runtime/proc.c | 79 +++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c index 3d076632fb..8dc467e6b2 100644 --- a/src/pkg/runtime/proc.c +++ b/src/pkg/runtime/proc.c @@ -78,6 +78,13 @@ struct Sched { Sched runtime·sched; int32 gomaxprocs; +// An m which is waiting for notewakeup(&m->havenextg). This may be +// only be accessed while the scheduler lock is held. This is used to +// minimize the number of times we call notewakeup while the scheduler +// lock is held, since the m will normally move quickly to lock the +// scheduler itself, producing lock contention. +static M* mwakeup; + // Scheduling helpers. Sched must be locked. static void gput(G*); // put/get on ghead/gtail static G* gget(void); @@ -133,6 +140,26 @@ runtime·schedinit(void) m->nomemprof--; } +// Lock the scheduler. +static void +schedlock(void) +{ + runtime·lock(&runtime·sched); +} + +// Unlock the scheduler. +static void +schedunlock(void) +{ + M *m; + + m = mwakeup; + mwakeup = nil; + runtime·unlock(&runtime·sched); + if(m != nil) + runtime·notewakeup(&m->havenextg); +} + // Called after main·init_function; main·main will be called on return. void runtime·initdone(void) @@ -144,9 +171,9 @@ runtime·initdone(void) // If main·init_function started other goroutines, // kick off new ms to handle them, like ready // would have, had it not been pre-dawn. - runtime·lock(&runtime·sched); + schedlock(); matchmg(); - runtime·unlock(&runtime·sched); + schedunlock(); } void @@ -264,9 +291,9 @@ mget(G *g) void runtime·ready(G *g) { - runtime·lock(&runtime·sched); + schedlock(); readylocked(g); - runtime·unlock(&runtime·sched); + schedunlock(); } // Mark g ready to run. Sched is already locked. @@ -317,7 +344,9 @@ mnextg(M *m, G *g) m->nextg = g; if(m->waitnextg) { m->waitnextg = 0; - runtime·notewakeup(&m->havenextg); + if(mwakeup != nil) + runtime·notewakeup(&mwakeup->havenextg); + mwakeup = m; } } @@ -338,7 +367,7 @@ nextgandunlock(void) if(m->nextg != nil) { gp = m->nextg; m->nextg = nil; - runtime·unlock(&runtime·sched); + schedunlock(); return gp; } @@ -356,7 +385,7 @@ nextgandunlock(void) continue; } runtime·sched.mcpu++; // this m will run gp - runtime·unlock(&runtime·sched); + schedunlock(); return gp; } // Otherwise, wait on global m queue. @@ -371,7 +400,7 @@ nextgandunlock(void) runtime·sched.waitstop = 0; runtime·notewakeup(&runtime·sched.stopped); } - runtime·unlock(&runtime·sched); + schedunlock(); runtime·notesleep(&m->havenextg); if((gp = m->nextg) == nil) @@ -385,7 +414,7 @@ nextgandunlock(void) void runtime·stoptheworld(void) { - runtime·lock(&runtime·sched); + schedlock(); runtime·gcwaiting = 1; runtime·sched.mcpumax = 1; while(runtime·sched.mcpu > 1) { @@ -395,11 +424,11 @@ runtime·stoptheworld(void) // so this is okay. runtime·noteclear(&runtime·sched.stopped); runtime·sched.waitstop = 1; - runtime·unlock(&runtime·sched); + schedunlock(); runtime·notesleep(&runtime·sched.stopped); - runtime·lock(&runtime·sched); + schedlock(); } - runtime·unlock(&runtime·sched); + schedunlock(); } // TODO(rsc): Remove. This is only temporary, @@ -407,11 +436,11 @@ runtime·stoptheworld(void) void runtime·starttheworld(void) { - runtime·lock(&runtime·sched); + schedlock(); runtime·gcwaiting = 0; runtime·sched.mcpumax = runtime·gomaxprocs; matchmg(); - runtime·unlock(&runtime·sched); + schedunlock(); } // Called to start an M. @@ -500,7 +529,7 @@ matchmg(void) static void schedule(G *gp) { - runtime·lock(&runtime·sched); + schedlock(); if(gp != nil) { if(runtime·sched.predawn) runtime·throw("init rescheduling"); @@ -584,7 +613,7 @@ runtime·entersyscall(void) runtime·gosave(&g->sched); if(runtime·sched.predawn) return; - runtime·lock(&runtime·sched); + schedlock(); g->status = Gsyscall; runtime·sched.mcpu--; runtime·sched.msyscall++; @@ -594,7 +623,7 @@ runtime·entersyscall(void) runtime·sched.waitstop = 0; runtime·notewakeup(&runtime·sched.stopped); } - runtime·unlock(&runtime·sched); + schedunlock(); } // The goroutine g exited its system call. @@ -607,13 +636,13 @@ runtime·exitsyscall(void) if(runtime·sched.predawn) return; - runtime·lock(&runtime·sched); + schedlock(); runtime·sched.msyscall--; runtime·sched.mcpu++; // Fast path - if there's room for this m, we're done. if(runtime·sched.mcpu <= runtime·sched.mcpumax) { g->status = Grunning; - runtime·unlock(&runtime·sched); + schedunlock(); return; } // Tell scheduler to put g back on the run queue: @@ -621,7 +650,7 @@ runtime·exitsyscall(void) // but keeps the garbage collector from thinking // that g is running right now, which it's not. g->readyonstop = 1; - runtime·unlock(&runtime·sched); + schedunlock(); // Slow path - all the cpus are taken. // The scheduler will ready g and put this m to sleep. @@ -815,7 +844,7 @@ runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) if(siz > 1024) runtime·throw("runtime.newproc: too many args"); - runtime·lock(&runtime·sched); + schedlock(); if((newg = gfget()) != nil){ newg->status = Gwaiting; @@ -848,7 +877,7 @@ runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) newg->goid = runtime·goidgen; newprocreadylocked(newg); - runtime·unlock(&runtime·sched); + schedunlock(); return newg; //printf(" goid=%d\n", newg->goid); @@ -1156,7 +1185,7 @@ runtime·gomaxprocsfunc(int32 n) { int32 ret; - runtime·lock(&runtime·sched); + schedlock(); ret = runtime·gomaxprocs; if (n <= 0) n = ret; @@ -1164,7 +1193,7 @@ runtime·gomaxprocsfunc(int32 n) runtime·sched.mcpumax = n; // handle fewer procs? if(runtime·sched.mcpu > runtime·sched.mcpumax) { - runtime·unlock(&runtime·sched); + schedunlock(); // just give up the cpu. // we'll only get rescheduled once the // number has come down. @@ -1173,7 +1202,7 @@ runtime·gomaxprocsfunc(int32 n) } // handle more procs matchmg(); - runtime·unlock(&runtime·sched); + schedunlock(); return ret; } -- 2.50.0