{
String status;
int64 waitfor;
+ uint32 gpstatus;
- switch(gp->status) {
+ gpstatus = runtime·readgstatus(gp);
+ switch(gpstatus) {
case Gidle:
status = runtime·gostringnocopy((byte*)"idle");
break;
else
status = runtime·gostringnocopy((byte*)"waiting");
break;
+ case Gscan:
+ status = runtime·gostringnocopy((byte*)"scan");
+ break;
+ case Gscanrunnable:
+ status = runtime·gostringnocopy((byte*)"scanrunnable");
+ break;
+ case Gscanrunning:
+ status = runtime·gostringnocopy((byte*)"scanrunning");
+ break;
+ case Gscansyscall:
+ status = runtime·gostringnocopy((byte*)"scansyscall");
+ break;
+ case Gscanenqueue:
+ status = runtime·gostringnocopy((byte*)"scanenqueue");
+ break;
+ case Gscanwaiting:
+ if(gp->waitreason.str != nil)
+ status = gp->waitreason;
+ else
+ status = runtime·gostringnocopy((byte*)"scanwaiting");
+ break;
+ case Gcopystack:
+ status = runtime·gostringnocopy((byte*)"copystack");
+ break;
default:
status = runtime·gostringnocopy((byte*)"???");
break;
// approx time the G is blocked, in minutes
waitfor = 0;
- if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince != 0)
+ gpstatus = gpstatus&~Gscan; // drop the scan bit
+ if((gpstatus == Gwaiting || gpstatus == Gsyscall) && gp->waitsince != 0)
waitfor = (runtime·nanotime() - gp->waitsince) / (60LL*1000*1000*1000);
runtime·printf("goroutine %D [%S", gp->goid, status);
runtime·printf("]:\n");
}
+static void
+dumpgstatus(G* gp)
+{
+ runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp));
+}
+
void
runtime·tracebackothers(G *me)
{
G *gp;
int32 traceback;
uintptr i;
+ uint32 status;
traceback = runtime·gotraceback(nil);
runtime·lock(&allglock);
for(i = 0; i < runtime·allglen; i++) {
gp = runtime·allg[i];
- if(gp == me || gp == g->m->curg || gp->status == Gdead)
+ if(gp == me || gp == g->m->curg || runtime·readgstatus(gp) == Gdead)
continue;
if(gp->issystem && traceback < 2)
continue;
runtime·printf("\n");
runtime·goroutineheader(gp);
- if(gp->status == Grunning) {
+ status = runtime·readgstatus(gp);
+ if((status&~Gscan) == Grunning){
runtime·printf("\tgoroutine running on other thread; stack unavailable\n");
runtime·printcreatedby(gp);
} else
checkmcount(void)
{
// sched lock is held
- if(runtime·sched.mcount > runtime·sched.maxmcount) {
+ if(runtime·sched.mcount > runtime·sched.maxmcount){
runtime·printf("runtime: program exceeds %d-thread limit\n", runtime·sched.maxmcount);
runtime·throw("thread exhaustion");
}
void
runtime·ready(G *gp)
{
+ uint32 status;
+
+ status = runtime·readgstatus(gp);
// Mark runnable.
g->m->locks++; // disable preemption because it can be holding p in a local var
- if(gp->status != Gwaiting) {
- runtime·printf("goroutine %D has status %d\n", gp->goid, gp->status);
+ if((status&~Gscan) != Gwaiting){
+ dumpgstatus(gp);
runtime·throw("bad g->status in ready");
}
- gp->status = Grunnable;
+ // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
+ runtime·casgstatus(gp, Gwaiting, Grunnable);
runqput(g->m->p, gp);
if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(&runtime·sched.nmspinning) == 0) // TODO: fast atomic
wakep();
runtime·usleep(1000);
}
+static bool
+isscanstatus(uint32 status)
+{
+ if(status == Gscan)
+ runtime·throw("isscanstatus: Bad status Gscan");
+ return (status&Gscan) == Gscan;
+}
+
+// All reads and writes of g's status go through readgstatus, casgstatus
+// castogscanstatus, casfromgscanstatus.
+uint32
+runtime·readgstatus(G *gp)
+{
+ return runtime·atomicload(&gp->atomicstatus);
+}
+
+// The Gscanstatuses are acting like locks and this releases them.
+// If it proves to be a performance hit we should be able to make these
+// simple atomic stores but for now we are going to throw if
+// we see an inconsistent state.
+void
+runtime·casfromgscanstatus(G *gp, uint32 oldval, uint32 newval)
+{
+ bool success = false;
+
+ // Check that transition is valid.
+ switch(oldval) {
+ case Gscanrunnable:
+ case Gscanwaiting:
+ case Gscanrunning:
+ case Gscansyscall:
+ if(newval == (oldval&~Gscan))
+ success = runtime·cas(&gp->atomicstatus, oldval, newval);
+ break;
+ case Gscanenqueue:
+ if(newval == Gwaiting)
+ success = runtime·cas(&gp->atomicstatus, oldval, newval);
+ break;
+ }
+ if(!success){
+ runtime·printf("runtime: casfromgscanstatus failed gp=%p, oldval=%d, newval=%d\n",
+ gp, oldval, newval);
+ dumpgstatus(gp);
+ runtime·throw("casfromgscanstatus: gp->status is not in scan state");
+ }
+}
+
+// This will return false if the gp is not in the expected status and the cas fails.
+// This acts like a lock acquire while the casfromgstatus acts like a lock release.
+bool
+runtime·castogscanstatus(G *gp, uint32 oldval, uint32 newval)
+{
+ switch(oldval) {
+ case Grunnable:
+ case Gwaiting:
+ case Gsyscall:
+ if(newval == (oldval|Gscan))
+ return runtime·cas(&gp->atomicstatus, oldval, newval);
+ break;
+ case Grunning:
+ if(newval == Gscanrunning || newval == Gscanenqueue)
+ return runtime·cas(&gp->atomicstatus, oldval, newval);
+ break;
+ }
+
+ runtime·printf("runtime: castogscanstatus oldval=%d newval=%d\n", oldval, newval);
+ runtime·throw("castogscanstatus");
+ return false; // not reached
+}
+
+// If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus
+// and casfromgscanstatus instead.
+// casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that
+// put it in the Gscan state is finished.
+void
+runtime·casgstatus(G *gp, uint32 oldval, uint32 newval)
+{
+ if(isscanstatus(oldval) || isscanstatus(newval) || oldval == newval) {
+ runtime·printf("casgstatus: oldval=%d, newval=%d\n", oldval, newval);
+ runtime·throw("casgstatus: bad incoming values");
+ }
+
+ while(!runtime·cas(&gp->atomicstatus, oldval, newval)) {
+ // loop if gp->atomicstatus is in a scan state giving
+ // GC time to finish and change the state to oldval.
+ }
+}
+
+// This is used by the GC as well as the routines that do stack dumps. In the case
+// of GC all the routines can be reliably stopped. This is not always the case
+// when the system is in panic or being exited.
void
runtime·stoptheworld(void)
{
runtime·atomicstore((uint32*)&runtime·sched.gcwaiting, 1);
preemptall();
// stop current P
- g->m->p->status = Pgcstop;
+ g->m->p->status = Pgcstop; // Pgcstop is only diagnostic.
runtime·sched.stopwait--;
// try to retake all P's in Psyscall status
for(i = 0; i < runtime·gomaxprocs; i++) {
gp->syscallsp = gp->sched.sp;
gp->syscallstack = gp->stackbase;
gp->syscallguard = gp->stackguard;
- gp->status = Gsyscall;
+ // malg returns status as Gidle, change to Gsyscall before adding to allg
+ // where GC will see it.
+ runtime·casgstatus(gp, Gidle, Gsyscall);
gp->m = mp;
mp->curg = gp;
mp->locked = LockInternal;
// no local work, check that there are no spinning/idle M's,
// otherwise our help is not required
if(runtime·atomicload(&runtime·sched.nmspinning) + runtime·atomicload(&runtime·sched.npidle) == 0 && // TODO: fast atomic
- runtime·cas(&runtime·sched.nmspinning, 0, 1)) {
+ runtime·cas(&runtime·sched.nmspinning, 0, 1)){
startm(p, true);
return;
}
stoplockedm(void)
{
P *p;
+ uint32 status;
if(g->m->lockedg == nil || g->m->lockedg->lockedm != g->m)
runtime·throw("stoplockedm: inconsistent locking");
// Wait until another thread schedules lockedg again.
runtime·notesleep(&g->m->park);
runtime·noteclear(&g->m->park);
- if(g->m->lockedg->status != Grunnable)
+ status = runtime·readgstatus(g->m->lockedg);
+ if((status&~Gscan) != Grunnable){
+ runtime·printf("runtime:stoplockedm: g is not Grunnable or Gscanrunnable");
+ dumpgstatus(g);
runtime·throw("stoplockedm: not runnable");
+ }
acquirep(g->m->nextp);
g->m->nextp = nil;
}
execute(G *gp)
{
int32 hz;
-
- if(gp->status != Grunnable) {
- runtime·printf("execute: bad g status %d\n", gp->status);
- runtime·throw("execute: bad g status");
- }
- gp->status = Grunning;
+
+ runtime·casgstatus(gp, Grunnable, Grunning);
gp->waitsince = 0;
gp->preempt = false;
gp->stackguard0 = gp->stackguard;
gp = runtime·netpoll(false); // non-blocking
if(gp) {
injectglist(gp->schedlink);
- gp->status = Grunnable;
+ runtime·casgstatus(gp, Gwaiting, Grunnable);
return gp;
}
// If number of spinning M's >= number of busy P's, block.
if(p) {
acquirep(p);
injectglist(gp->schedlink);
- gp->status = Grunnable;
+ runtime·casgstatus(gp, Gwaiting, Grunnable);
return gp;
}
injectglist(gp);
for(n = 0; glist; n++) {
gp = glist;
glist = gp->schedlink;
- gp->status = Grunnable;
+ runtime·casgstatus(gp, Gwaiting, Grunnable);
globrunqput(gp);
}
runtime·unlock(&runtime·sched.lock);
void
runtime·park(bool(*unlockf)(G*, void*), void *lock, String reason)
{
- if(g->status != Grunning)
- runtime·throw("bad g status");
g->m->waitlock = lock;
g->m->waitunlockf = unlockf;
g->waitreason = reason;
{
bool ok;
- gp->status = Gwaiting;
+ runtime·casgstatus(gp, Grunning, Gwaiting);
dropg();
if(g->m->waitunlockf) {
g->m->waitunlockf = nil;
g->m->waitlock = nil;
if(!ok) {
- gp->status = Grunnable;
+ runtime·casgstatus(gp, Gwaiting, Grunnable);
execute(gp); // Schedule it back, never returns.
}
}
void
runtime·gosched_m(G *gp)
{
- if(gp->status != Grunning)
+ uint32 status;
+
+ status = runtime·readgstatus(gp);
+ if ((status&~Gscan) != Grunning){
+ dumpgstatus(gp);
runtime·throw("bad g status");
- gp->status = Grunnable;
+ }
+ runtime·casgstatus(gp, Grunning, Grunnable);
dropg();
runtime·lock(&runtime·sched.lock);
globrunqput(gp);
void
runtime·goexit(void)
{
- if(g->status != Grunning)
- runtime·throw("bad g status");
if(raceenabled)
runtime·racegoend();
runtime·mcall(goexit0);
static void
goexit0(G *gp)
{
- gp->status = Gdead;
+ runtime·casgstatus(gp, Grunning, Gdead);
gp->m = nil;
gp->lockedm = nil;
g->m->lockedg = nil;
gp->waitreason.str = nil;
gp->waitreason.len = 0;
gp->param = nil;
-
+
dropg();
if(g->m->locked & ~LockExternal) {
g->syscallpc = g->sched.pc;
g->syscallstack = g->stackbase;
g->syscallguard = g->stackguard;
- g->status = Gsyscall;
+ runtime·casgstatus(g, Grunning, Gsyscall);
if(g->syscallsp < g->syscallguard-StackGuard || g->syscallstack < g->syscallsp) {
// runtime·printf("entersyscall inconsistent %p [%p,%p]\n",
// g->syscallsp, g->syscallguard-StackGuard, g->syscallstack);
g->syscallpc = g->sched.pc;
g->syscallstack = g->stackbase;
g->syscallguard = g->stackguard;
- g->status = Gsyscall;
+ runtime·casgstatus(g, Grunning, Gsyscall);
if(g->syscallsp < g->syscallguard-StackGuard || g->syscallstack < g->syscallsp) {
// runtime·printf("entersyscall inconsistent %p [%p,%p]\n",
// g->syscallsp, g->syscallguard-StackGuard, g->syscallstack);
gp->syscallpc = gp->sched.pc;
gp->syscallstack = gp->stackbase;
gp->syscallguard = gp->stackguard;
- gp->status = Gsyscall;
+ runtime·casgstatus(gp, Grunning, Gsyscall);
if(gp->syscallsp < gp->syscallguard-StackGuard || gp->syscallstack < gp->syscallsp) {
// runtime·printf("entersyscall inconsistent %p [%p,%p]\n",
// gp->syscallsp, gp->syscallguard-StackGuard, gp->syscallstack);
if(exitsyscallfast()) {
// There's a cpu for us, so we can run.
g->m->p->syscalltick++;
- g->status = Grunning;
+ // We need to cas the status and scan before resuming...
+ runtime·casgstatus(g, Gsyscall, Grunning);
+
// Garbage collector isn't running (since we are),
// so okay to clear gcstack and gcsp.
g->syscallstack = (uintptr)nil;
{
P *p;
- gp->status = Grunnable;
+ runtime·casgstatus(gp, Gsyscall, Grunnable);
dropg();
runtime·lock(&runtime·sched.lock);
p = pidleget();
P *p;
int32 siz;
-//runtime·printf("newproc1 %p %p narg=%d nret=%d\n", fn->fn, argp, narg, nret);
if(fn == nil) {
g->m->throwing = -1; // do not dump full stacks
runtime·throw("go of nil func value");
runtime·throw("invalid stack in newg");
} else {
newg = runtime·malg(StackMin);
- allgadd(newg);
+ runtime·casgstatus(newg, Gidle, Gdead);
+ allgadd(newg); // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack.
}
+ if(runtime·readgstatus(newg) != Gdead)
+ runtime·throw("newproc1: new g is not Gdead");
+
sp = (byte*)newg->stackbase;
sp -= siz;
runtime·memmove(sp, argp, narg);
newg->sched.g = newg;
runtime·gostartcallfn(&newg->sched, fn);
newg->gopc = (uintptr)callerpc;
- newg->status = Grunnable;
+ runtime·casgstatus(newg, Gdead, Grunnable);
+
if(p->goidcache == p->goidcacheend) {
// Sched.goidgen is the last allocated id,
// this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch].
G **new;
uintptr cap;
+ if (runtime·readgstatus(gp) == Gidle)
+ runtime·throw("allgadd: bad status Gidle");
+
runtime·lock(&allglock);
if(runtime·allglen >= allgcap) {
cap = 4096/sizeof(new[0]);
uintptr stksize;
Stktop *top;
+ if (runtime·readgstatus(gp) != Gdead)
+ runtime·throw("gfput: bad status (not Gdead)");
+
if(gp->stackguard - StackGuard != gp->stack0)
runtime·throw("invalid stack in gfput");
stksize = gp->stackbase + sizeof(Stktop) - gp->stack0;
gp = runtime·allg[i];
if(gp->issystem)
continue;
- s = gp->status;
- if(s == Gwaiting)
+ s = runtime·readgstatus(gp);
+ switch(s&~Gscan) {
+ case Gwaiting:
grunning++;
- else if(s == Grunnable || s == Grunning || s == Gsyscall) {
+ break;
+ case Grunnable:
+ case Grunning:
+ case Gsyscall:
runtime·unlock(&allglock);
runtime·printf("runtime: checkdead: find g %D in status %d\n", gp->goid, s);
runtime·throw("checkdead: runnable g");
+ break;
}
}
runtime·unlock(&allglock);
// simultaneously executing runtime·newstack.
// No lock needs to be held.
// Returns true if preemption request was issued.
+// The actual preemption will happen at some point in the future
+// and will be indicated by the gp->status no longer being
+// Grunning
static bool
preemptone(P *p)
{
if(gp == nil || gp == mp->g0)
return false;
gp->preempt = true;
+ // Every call in a go routine checks for stack overflow by
+ // comparing the current stack pointer to gp->stackguard0.
+ // Setting gp->stackguard0 to StackPreempt folds
+ // preemption into the normal stack overflow check.
gp->stackguard0 = StackPreempt;
return true;
}
mp = gp->m;
lockedm = gp->lockedm;
runtime·printf(" G%D: status=%d(%S) m=%d lockedm=%d\n",
- gp->goid, gp->status, gp->waitreason, mp ? mp->id : -1,
+ gp->goid, runtime·readgstatus(gp), gp->waitreason, mp ? mp->id : -1,
lockedm ? lockedm->id : -1);
}
runtime·unlock(&allglock);
top->gobuf.pc, top->gobuf.sp, top->gobuf.lr, (uintptr)g->m->cret, (uintptr)argsize);
}
- // gp->status is usually Grunning, but it could be Gsyscall if a stack overflow
- // happens during a function call inside entersyscall.
- oldstatus = gp->status;
-
gp->sched = top->gobuf;
gp->sched.ret = g->m->cret;
g->m->cret = 0; // drop reference
- gp->status = Gwaiting;
- gp->waitreason = runtime·gostringnocopy((byte*)"stack unsplit");
+ // gp->status is usually Grunning, but it could be Gsyscall if a stack overflow
+ // happens during a function call inside entersyscall.
+
+ oldstatus = runtime·readgstatus(gp);
+ oldstatus &= ~Gscan;
+ if(oldstatus != Grunning && oldstatus != Gsyscall) {
+ runtime·printf("runtime: oldstack status=%d\n", oldstatus);
+ runtime·throw("oldstack");
+ }
+ runtime·casgstatus(gp, oldstatus, Gcopystack);
+ gp->waitreason = runtime·gostringnocopy((byte*)"stack unsplit");
if(argsize > 0) {
sp -= argsize;
gp->stackguard0 = gp->stackguard;
gp->panicwrap = top->panicwrap;
runtime·stackfree(gp, old, top);
-
- gp->status = oldstatus;
+ runtime·casgstatus(gp, Gcopystack, oldstatus); // oldstatus is Grunning or Gsyscall
runtime·gogo(&gp->sched);
}
uintptr oldsize, used;
AdjustInfo adjinfo;
Stktop *oldtop, *newtop;
+ uint32 oldstatus;
if(gp->syscallstack != 0)
runtime·throw("can't handle stack copy in syscall yet");
// copy the stack (including Stktop) to the new location
runtime·memmove(newbase - used, oldbase - used, used);
-
+ oldstatus = runtime·readgstatus(gp);
+ oldstatus &= ~Gscan;
+ if (oldstatus == Gwaiting || oldstatus == Grunnable)
+ runtime·casgstatus(gp, oldstatus, Gcopystack); // oldstatus is Gwaiting or Grunnable
+ else
+ runtime·throw("copystack: bad status, not Gwaiting or Grunnable");
// Swap out old stack for new one
gp->stackbase = (uintptr)newtop;
gp->stackguard = (uintptr)newstk + StackGuard;
gp->stack0 = (uintptr)newstk;
gp->sched.sp = (uintptr)(newbase - used);
+ runtime·casgstatus(gp, Gcopystack, oldstatus); // oldstatus is Gwaiting or Grunnable
+
// free old stack
runtime·stackfree(gp, oldstk, oldtop);
}
// m->moreframesize bytes, copy m->moreargsize bytes to the new frame,
// and then act as though runtime·lessstack called the function at
// m->morepc.
+//
+// g->atomicstatus will be Grunning, Gsyscall or Gscanrunning, Gscansyscall upon entry.
+// If the GC is trying to stop this g then it will set preemptscan to true.
void
runtime·newstack(void)
{
runtime·throw("runtime: wrong goroutine in newstack");
}
+ // The goroutine must be executing in order to call newstack, so the possible states are
+ // Grunning and Gsyscall (and, due to GC, also Gscanrunning and Gscansyscall).
+
// gp->status is usually Grunning, but it could be Gsyscall if a stack overflow
// happens during a function call inside entersyscall.
gp = g->m->curg;
- oldstatus = gp->status;
-
+ oldstatus = runtime·readgstatus(gp) & ~Gscan;
framesize = g->m->moreframesize;
argsize = g->m->moreargsize;
moreargp = g->m->moreargp;
g->m->morebuf.pc = (uintptr)nil;
g->m->morebuf.lr = (uintptr)nil;
g->m->morebuf.sp = (uintptr)nil;
- gp->status = Gwaiting;
+
+ runtime·casgstatus(gp, oldstatus, Gwaiting); // oldstatus is not in a Gscan status
gp->waitreason = runtime·gostringnocopy((byte*)"stack growth");
newstackcall = framesize==1;
if(newstackcall)
gp->sched.pc, gp->sched.sp, gp->sched.lr, gp->sched.ctxt);
}
if(sp < gp->stackguard - StackGuard) {
+ runtime·printf("runtime: gp=%p, gp->status=%d, oldstatus=%d\n ", (void*)gp, runtime·readgstatus(gp), oldstatus);
runtime·printf("runtime: split stack overflow: %p < %p\n", sp, gp->stackguard - StackGuard);
runtime·throw("runtime: split stack overflow");
}
runtime·throw("runtime: g is running but p is not");
if(oldstatus == Gsyscall && g->m->locks == 0)
runtime·throw("runtime: stack growth during syscall");
+
// Be conservative about where we preempt.
// We are interested in preempting user Go code, not runtime code.
if(oldstatus != Grunning || g->m->locks || g->m->mallocing || g->m->gcing || g->m->p->status != Prunning) {
// Let the goroutine keep running for now.
// gp->preempt is set, so it will be preempted next time.
gp->stackguard0 = gp->stackguard;
- gp->status = oldstatus;
+ runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
runtime·gogo(&gp->sched); // never return
}
// Act like goroutine called runtime.Gosched.
- gp->status = oldstatus;
+ runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
runtime·gosched_m(gp); // never return
}
oldbase = (byte*)gp->stackbase + sizeof(Stktop);
oldsize = oldbase - oldstk;
newsize = oldsize * 2;
+ // Note that the concurrent GC might be scanning the stack as we try to replace it.
+ // copystack takes care of the appropriate coordination with the stack scanner.
copystack(gp, nframes, newsize);
if(StackDebug >= 1)
runtime·printf("stack grow done\n");
runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize);
runtime·throw("stack overflow");
}
- gp->status = oldstatus;
+ runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
runtime·gogo(&gp->sched);
}
// TODO: if stack is uncopyable because we're in C code, patch return value at
runtime·gostartcall(&label, (void(*)(void))gp->sched.pc, gp->sched.ctxt);
gp->sched.ctxt = nil;
}
- gp->status = oldstatus;
+ runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Grunning or Gsyscall
runtime·gogo(&label);
*(int32*)345 = 123; // never return
if(!runtime·copystack)
return;
- if(gp->status == Gdead)
+ if(runtime·readgstatus(gp) == Gdead)
return;
if(gp->stackbase == 0)
runtime·throw("stackbase == 0");