return ret
}
- semacquire(&worldsema, false)
- gp := getg()
- gp.m.preemptoff = "GOMAXPROCS"
- systemstack(stoptheworld)
+ stopTheWorld("GOMAXPROCS")
- // newprocs will be processed by starttheworld
+ // newprocs will be processed by startTheWorld
newprocs = int32(n)
- gp.m.preemptoff = ""
- semrelease(&worldsema)
- systemstack(starttheworld)
+ startTheWorld()
return ret
}
//go:linkname runtime_debug_WriteHeapDump runtime/debug.WriteHeapDump
func runtime_debug_WriteHeapDump(fd uintptr) {
- semacquire(&worldsema, false)
- gp := getg()
- gp.m.preemptoff = "write heap dump"
- systemstack(stoptheworld)
+ stopTheWorld("write heap dump")
systemstack(func() {
writeheapdump_m(fd)
})
- gp.m.preemptoff = ""
- gp.m.locks++
- semrelease(&worldsema)
- systemstack(starttheworld)
- gp.m.locks--
+ getg().m.locks++ // TODO: Is this necessary?
+ startTheWorld()
+ getg().m.locks--
}
const (
func startGC(mode int) {
// The gc is turned off (via enablegc) until the bootstrap has completed.
// Also, malloc gets called in the guts of a number of libraries that might be
- // holding locks. To avoid deadlocks during stoptheworld, don't bother
+ // holding locks. To avoid deadlocks during stop-the-world, don't bother
// trying to run gc while holding a lock. The next mallocgc without a lock
// will do the gc instead.
mp := acquirem()
traceGCStart()
}
- systemstack(stoptheworld)
+ systemstack(stopTheWorldWithSema)
systemstack(finishsweep_m) // finish sweep before we start concurrent scan.
// clearpools before we start the GC. If we wait they memory will not be
// reclaimed until the next GC cycle.
setGCPhase(_GCscan)
// Concurrent scan.
- starttheworld()
+ startTheWorldWithSema()
if debug.gctrace > 0 {
tScan = nanotime()
}
if debug.gctrace > 0 {
tMarkTerm = nanotime()
}
- systemstack(stoptheworld)
+ systemstack(stopTheWorldWithSema)
// The gcphase is _GCmark, it will transition to _GCmarktermination
// below. The important thing is that the wb remains active until
// all marking is complete. This includes writes made by the GC.
throw("gc done but gcphase != _GCoff")
}
- systemstack(starttheworld)
+ systemstack(startTheWorldWithSema)
releasem(mp)
mp = nil
n = NumGoroutine()
if n <= len(p) {
gp := getg()
- semacquire(&worldsema, false)
- gp.m.preemptoff = "profile"
- systemstack(stoptheworld)
+ stopTheWorld("profile")
n = NumGoroutine()
if n <= len(p) {
}
}
- gp.m.preemptoff = ""
- semrelease(&worldsema)
- systemstack(starttheworld)
+ startTheWorld()
}
return n, ok
// into buf after the trace for the current goroutine.
func Stack(buf []byte, all bool) int {
if all {
- semacquire(&worldsema, false)
- gp := getg()
- gp.m.preemptoff = "stack trace"
- systemstack(stoptheworld)
+ stopTheWorld("stack trace")
}
n := 0
}
if all {
- gp := getg()
- gp.m.preemptoff = ""
- semrelease(&worldsema)
- systemstack(starttheworld)
+ startTheWorld()
}
return n
}
// ReadMemStats populates m with memory allocator statistics.
func ReadMemStats(m *MemStats) {
- // Have to acquire worldsema to stop the world,
- // because stoptheworld can only be used by
- // one goroutine at a time, and there might be
- // a pending garbage collection already calling it.
- semacquire(&worldsema, false)
- gp := getg()
- gp.m.preemptoff = "read mem stats"
- systemstack(stoptheworld)
+ stopTheWorld("read mem stats")
systemstack(func() {
readmemstats_m(m)
})
- gp.m.preemptoff = ""
- gp.m.locks++
- semrelease(&worldsema)
- systemstack(starttheworld)
- gp.m.locks--
+ getg().m.locks++ // TODO: Is this necessary?
+ startTheWorld()
+ getg().m.locks--
}
func readmemstats_m(stats *MemStats) {
// acquireSudog, acquireSudog calls new(sudog),
// new calls malloc, malloc can call the garbage collector,
// and the garbage collector calls the semaphore implementation
- // in stoptheworld.
+ // in stopTheWorld.
// Break the cycle by doing acquirem/releasem around new(sudog).
// The acquirem/releasem increments m.locks during new(sudog),
// which keeps the garbage collector from being invoked.
// sched.stopwait to in order to request that all Gs permanently stop.
const freezeStopWait = 0x7fffffff
-// Similar to stoptheworld but best-effort and can be called several times.
+// Similar to stopTheWorld but best-effort and can be called several times.
// There is no reverse operation, used during crashing.
// This function must not lock any mutexes.
func freezetheworld() {
mcall(mquiesce)
}
+// stopTheWorld stops all P's from executing goroutines, interrupting
+// all goroutines at GC safe points and records reason as the reason
+// for the stop. On return, only the current goroutine's P is running.
+// stopTheWorld must not be called from a system stack and the caller
+// must not hold worldsema. The caller must call startTheWorld when
+// other P's should resume execution.
+//
+// stopTheWorld is safe for multiple goroutines to call at the
+// same time. Each will execute its own stop, and the stops will
+// be serialized.
+//
+// This is also used by routines that do stack dumps. If the system is
+// in panic or being exited, this may not reliably stop all
+// goroutines.
+func stopTheWorld(reason string) {
+ semacquire(&worldsema, false)
+ getg().m.preemptoff = reason
+ systemstack(stopTheWorldWithSema)
+}
+
+// startTheWorld undoes the effects of stopTheWorld.
+func startTheWorld() {
+ getg().m.preemptoff = ""
+ semrelease(&worldsema)
+ systemstack(startTheWorldWithSema)
+}
+
// Holding worldsema grants an M the right to try to stop the world.
-// The procedure is:
+var worldsema uint32 = 1
+
+// stopTheWorldWithSema is the core implementation of stopTheWorld.
+// The caller is responsible for acquiring worldsema and disabling
+// preemption first and then should stopTheWorldWithSema on the system
+// stack:
//
-// semacquire(&worldsema);
-// m.preemptoff = "reason";
-// stoptheworld();
+// semacquire(&worldsema, false)
+// m.preemptoff = "reason"
+// systemstack(stopTheWorldWithSema)
//
-// ... do stuff ...
+// When finished, the caller must either call startTheWorld or undo
+// these three operations separately:
//
-// m.preemptoff = "";
-// semrelease(&worldsema);
-// starttheworld();
+// m.preemptoff = ""
+// semrelease(&worldsema)
+// systemstack(startTheWorldWithSema)
//
-var worldsema uint32 = 1
-
-// This is used by the GC as well as the routines that do stack dumps. In the case
-// of GC all the routines can be reliably stopped. This is not always the case
-// when the system is in panic or being exited.
-func stoptheworld() {
+// It is allowed to acquire worldsema once and then execute multiple
+// startTheWorldWithSema/stopTheWorldWithSema pairs.
+// Other P's are able to execute between successive calls to
+// startTheWorldWithSema and stopTheWorldWithSema.
+// Holding worldsema causes any other goroutines invoking
+// stopTheWorld to block.
+func stopTheWorldWithSema() {
_g_ := getg()
// If we hold a lock, then we won't be able to stop another M
// that is blocked trying to acquire the lock.
if _g_.m.locks > 0 {
- throw("stoptheworld: holding locks")
+ throw("stopTheWorld: holding locks")
}
lock(&sched.lock)
}
}
if sched.stopwait != 0 {
- throw("stoptheworld: not stopped")
+ throw("stopTheWorld: not stopped")
}
for i := 0; i < int(gomaxprocs); i++ {
p := allp[i]
if p.status != _Pgcstop {
- throw("stoptheworld: not stopped")
+ throw("stopTheWorld: not stopped")
}
}
}
_g_.m.helpgc = -1
}
-func starttheworld() {
+func startTheWorldWithSema() {
_g_ := getg()
_g_.m.locks++ // disable preemption because it can be holding p in a local var
mp := p.m.ptr()
p.m = 0
if mp.nextp != 0 {
- throw("starttheworld: inconsistent mp->nextp")
+ throw("startTheWorld: inconsistent mp->nextp")
}
mp.nextp.set(p)
notewakeup(&mp.park)
stopm()
}
-// Stops the current m for stoptheworld.
+// Stops the current m for stopTheWorld.
// Returns when the world is restarted.
func gcstopm() {
_g_ := getg()
func StartTrace() error {
// Stop the world, so that we can take a consistent snapshot
// of all goroutines at the beginning of the trace.
- semacquire(&worldsema, false)
- _g_ := getg()
- _g_.m.preemptoff = "start tracing"
- systemstack(stoptheworld)
+ stopTheWorld("start tracing")
// We are in stop-the-world, but syscalls can finish and write to trace concurrently.
// Exitsyscall could check trace.enabled long before and then suddenly wake up
if trace.enabled || trace.shutdown {
unlock(&trace.bufLock)
- _g_.m.preemptoff = ""
- semrelease(&worldsema)
- systemstack(starttheworld)
+ startTheWorld()
return errorString("tracing is already enabled")
}
unlock(&trace.bufLock)
- _g_.m.preemptoff = ""
- semrelease(&worldsema)
- systemstack(starttheworld)
+ startTheWorld()
return nil
}
func StopTrace() {
// Stop the world so that we can collect the trace buffers from all p's below,
// and also to avoid races with traceEvent.
- semacquire(&worldsema, false)
- _g_ := getg()
- _g_.m.preemptoff = "stop tracing"
- systemstack(stoptheworld)
+ stopTheWorld("stop tracing")
// See the comment in StartTrace.
lock(&trace.bufLock)
if !trace.enabled {
unlock(&trace.bufLock)
- _g_.m.preemptoff = ""
- semrelease(&worldsema)
- systemstack(starttheworld)
+ startTheWorld()
return
}
unlock(&trace.bufLock)
- _g_.m.preemptoff = ""
- semrelease(&worldsema)
- systemstack(starttheworld)
+ startTheWorld()
// The world is started but we've set trace.shutdown, so new tracing can't start.
// Wait for the trace reader to flush pending buffers and stop.
// The caller checked that trace.enabled == true, but trace.enabled might have been
// turned off between the check and now. Check again. traceLockBuffer did mp.locks++,
- // StopTrace does stoptheworld, and stoptheworld waits for mp.locks to go back to zero,
+ // StopTrace does stopTheWorld, and stopTheWorld waits for mp.locks to go back to zero,
// so if we see trace.enabled == true now, we know it's true for the rest of the function.
- // Exitsyscall can run even during stoptheworld. The race with StartTrace/StopTrace
+ // Exitsyscall can run even during stopTheWorld. The race with StartTrace/StopTrace
// during tracing in exitsyscall is resolved by locking trace.bufLock in traceLockBuffer.
if !trace.enabled {
traceReleaseBuffer(pid)
}
func traceProcStop(pp *p) {
- // Sysmon and stoptheworld can stop Ps blocked in syscalls,
+ // Sysmon and stopTheWorld can stop Ps blocked in syscalls,
// to handle this we temporary employ the P.
mp := acquirem()
oldp := mp.p
}
func traceGoSysBlock(pp *p) {
- // Sysmon and stoptheworld can declare syscalls running on remote Ps as blocked,
+ // Sysmon and stopTheWorld can declare syscalls running on remote Ps as blocked,
// to handle this we temporary employ the P.
mp := acquirem()
oldp := mp.p