// status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
casgstatus(gp, _Gwaiting, _Grunnable)
- runqput(_g_.m.p, gp)
+ runqput(_g_.m.p.ptr(), gp)
if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { // TODO: fast atomic
wakep()
}
// Preempt the current g
casgstatus(_g_, _Grunning, _Grunnable)
- runqput(_g_.m.p, _g_)
+ runqput(_g_.m.p.ptr(), _g_)
dropg()
// Ready gp and switch to it
throw("gcprocs inconsistency")
}
mp.helpgc = n
- mp.p = allp[pos]
+ mp.p.set(allp[pos])
mp.mcache = allp[pos].mcache
pos++
notewakeup(&mp.park)
atomicstore(&sched.gcwaiting, 1)
preemptall()
// stop current P
- _g_.m.p.status = _Pgcstop // Pgcstop is only diagnostic.
+ _g_.m.p.ptr().status = _Pgcstop // Pgcstop is only diagnostic.
sched.stopwait--
// try to retake all P's in Psyscall status
for i := 0; i < int(gomaxprocs); i++ {
for p1 != nil {
p := p1
- p1 = p1.link
- if p.m != nil {
- mp := p.m
- p.m = nil
- if mp.nextp != nil {
+ p1 = p1.link.ptr()
+ if p.m != 0 {
+ mp := p.m.ptr()
+ p.m = 0
+ if mp.nextp != 0 {
throw("starttheworld: inconsistent mp->nextp")
}
- mp.nextp = p
+ mp.nextp.set(p)
notewakeup(&mp.park)
} else {
// Start M to run P. Do not start another M below.
initsig()
}
- if _g_.m.mstartfn != 0 {
- fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn))
+ if fn := _g_.m.mstartfn; fn != nil {
fn()
}
_g_.m.helpgc = 0
stopm()
} else if _g_.m != &m0 {
- acquirep(_g_.m.nextp)
- _g_.m.nextp = nil
+ acquirep(_g_.m.nextp.ptr())
+ _g_.m.nextp = 0
}
schedule()
}
var cgoThreadStart unsafe.Pointer
type cgothreadstart struct {
- g *g
+ g guintptr
tls *uint64
fn unsafe.Pointer
}
// Allocate a new m unassociated with any thread.
// Can use p for allocation context if needed.
-func allocm(_p_ *p) *m {
+// fn is recorded as the new m's m.mstartfn.
+func allocm(_p_ *p, fn func()) *m {
_g_ := getg()
_g_.m.locks++ // disable GC because it can be called from sysmon
- if _g_.m.p == nil {
+ if _g_.m.p == 0 {
acquirep(_p_) // temporarily borrow p for mallocs in this function
}
mp := new(m)
+ mp.mstartfn = fn
mcommoninit(mp)
// In case of cgo or Solaris, pthread_create will make us a stack.
}
mp.g0.m = mp
- if _p_ == _g_.m.p {
+ if _p_ == _g_.m.p.ptr() {
releasep()
}
_g_.m.locks--
// after exitsyscall makes sure it is okay to be
// running at all (that is, there's no garbage collection
// running right now).
- mp.needextram = mp.schedlink == nil
- unlockextra(mp.schedlink)
+ mp.needextram = mp.schedlink == 0
+ unlockextra(mp.schedlink.ptr())
// Install g (= m->g0) and set the stack bounds
// to match the current stack. We don't actually know
// The sched.pc will never be returned to, but setting it to
// goexit makes clear to the traceback routines where
// the goroutine stack ends.
- mp := allocm(nil)
+ mp := allocm(nil, nil)
gp := malg(4096)
gp.sched.pc = funcPC(goexit) + _PCQuantum
gp.sched.sp = gp.stack.hi
// Add m to the extra list.
mnext := lockextra(true)
- mp.schedlink = mnext
+ mp.schedlink.set(mnext)
unlockextra(mp)
}
// with no pointer manipulation.
mp := getg().m
mnext := lockextra(true)
- mp.schedlink = mnext
+ mp.schedlink.set(mnext)
setg(nil)
unlockextra(mp)
// May run with m.p==nil, so write barriers are not allowed.
//go:nowritebarrier
func newm(fn func(), _p_ *p) {
- mp := allocm(_p_)
- // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated
- setPNoWriteBarrier(&mp.nextp, _p_)
- // Store &fn as a uintptr since it is not heap allocated so the WB can be eliminated
- mp.mstartfn = *(*uintptr)(unsafe.Pointer(&fn))
+ mp := allocm(_p_, fn)
+ mp.nextp.set(_p_)
if iscgo {
var ts cgothreadstart
if _cgo_thread_start == nil {
throw("_cgo_thread_start missing")
}
- // mp is reachable via allm and mp.g0 never changes, so WB can be eliminated.
- setGNoWriteBarrier(&ts.g, mp.g0)
+ ts.g.set(mp.g0)
ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0]))
ts.fn = unsafe.Pointer(funcPC(mstart))
asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts))
if _g_.m.locks != 0 {
throw("stopm holding locks")
}
- if _g_.m.p != nil {
+ if _g_.m.p != 0 {
throw("stopm holding p")
}
if _g_.m.spinning {
gchelper()
_g_.m.helpgc = 0
_g_.m.mcache = nil
- _g_.m.p = nil
+ _g_.m.p = 0
goto retry
}
- acquirep(_g_.m.nextp)
- _g_.m.nextp = nil
+ acquirep(_g_.m.nextp.ptr())
+ _g_.m.nextp = 0
}
func mspinning() {
if mp.spinning {
throw("startm: m is spinning")
}
- if mp.nextp != nil {
+ if mp.nextp != 0 {
throw("startm: m has p")
}
mp.spinning = spinning
- // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated
- setPNoWriteBarrier(&mp.nextp, _p_)
+ mp.nextp.set(_p_)
notewakeup(&mp.park)
}
if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m {
throw("stoplockedm: inconsistent locking")
}
- if _g_.m.p != nil {
+ if _g_.m.p != 0 {
// Schedule another M to run this p.
_p_ := releasep()
handoffp(_p_)
dumpgstatus(_g_)
throw("stoplockedm: not runnable")
}
- acquirep(_g_.m.nextp)
- _g_.m.nextp = nil
+ acquirep(_g_.m.nextp.ptr())
+ _g_.m.nextp = 0
}
// Schedules the locked m to run the locked gp.
if mp == _g_.m {
throw("startlockedm: locked to me")
}
- if mp.nextp != nil {
+ if mp.nextp != 0 {
throw("startlockedm: m has p")
}
// directly handoff current P to the locked m
incidlelocked(-1)
_p_ := releasep()
- // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated
- setPNoWriteBarrier(&mp.nextp, _p_)
+ mp.nextp.set(_p_)
notewakeup(&mp.park)
stopm()
}
gp.waitsince = 0
gp.preempt = false
gp.stackguard0 = gp.stack.lo + _StackGuard
- _g_.m.p.schedtick++
+ _g_.m.p.ptr().schedtick++
_g_.m.curg = gp
gp.m = _g_.m
}
// local runq
- if gp := runqget(_g_.m.p); gp != nil {
+ if gp := runqget(_g_.m.p.ptr()); gp != nil {
return gp
}
// global runq
if sched.runqsize != 0 {
lock(&sched.lock)
- gp := globrunqget(_g_.m.p, 0)
+ gp := globrunqget(_g_.m.p.ptr(), 0)
unlock(&sched.lock)
if gp != nil {
return gp
if netpollinited() && sched.lastpoll != 0 {
if gp := netpoll(false); gp != nil { // non-blocking
// netpoll returns list of goroutines linked by schedlink.
- injectglist(gp.schedlink)
+ injectglist(gp.schedlink.ptr())
casgstatus(gp, _Gwaiting, _Grunnable)
if trace.enabled {
traceGoUnpark(gp, 0)
}
_p_ := allp[fastrand1()%uint32(gomaxprocs)]
var gp *g
- if _p_ == _g_.m.p {
+ if _p_ == _g_.m.p.ptr() {
gp = runqget(_p_)
} else {
- gp = runqsteal(_g_.m.p, _p_)
+ gp = runqsteal(_g_.m.p.ptr(), _p_)
}
if gp != nil {
return gp
goto top
}
if sched.runqsize != 0 {
- gp := globrunqget(_g_.m.p, 0)
+ gp := globrunqget(_g_.m.p.ptr(), 0)
unlock(&sched.lock)
return gp
}
// poll network
if netpollinited() && xchg64(&sched.lastpoll, 0) != 0 {
- if _g_.m.p != nil {
+ if _g_.m.p != 0 {
throw("findrunnable: netpoll with p")
}
if _g_.m.spinning {
unlock(&sched.lock)
if _p_ != nil {
acquirep(_p_)
- injectglist(gp.schedlink)
+ injectglist(gp.schedlink.ptr())
casgstatus(gp, _Gwaiting, _Grunnable)
if trace.enabled {
traceGoUnpark(gp, 0)
return
}
if trace.enabled {
- for gp := glist; gp != nil; gp = gp.schedlink {
+ for gp := glist; gp != nil; gp = gp.schedlink.ptr() {
traceGoUnpark(gp, 0)
}
}
var n int
for n = 0; glist != nil; n++ {
gp := glist
- glist = gp.schedlink
+ glist = gp.schedlink.ptr()
casgstatus(gp, _Gwaiting, _Grunnable)
globrunqput(gp)
}
// Check the global runnable queue once in a while to ensure fairness.
// Otherwise two goroutines can completely occupy the local runqueue
// by constantly respawning each other.
- if _g_.m.p.schedtick%61 == 0 && sched.runqsize > 0 {
+ if _g_.m.p.ptr().schedtick%61 == 0 && sched.runqsize > 0 {
lock(&sched.lock)
- gp = globrunqget(_g_.m.p, 1)
+ gp = globrunqget(_g_.m.p.ptr(), 1)
unlock(&sched.lock)
if gp != nil {
resetspinning()
}
}
if gp == nil {
- gp = runqget(_g_.m.p)
+ gp = runqget(_g_.m.p.ptr())
if gp != nil && _g_.m.spinning {
throw("schedule: spinning with local work")
}
throw("internal lockOSThread error")
}
_g_.m.locked = 0
- gfput(_g_.m.p, gp)
+ gfput(_g_.m.p.ptr(), gp)
schedule()
}
// when syscall returns we emit traceGoSysExit and when the goroutine starts running
// (potentially instantly, if exitsyscallfast returns true) we emit traceGoStart.
// To ensure that traceGoSysExit is emitted strictly after traceGoSysBlock,
-// we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.syscalltick),
+// we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.ptr().syscalltick),
// whoever emits traceGoSysBlock increments p.syscalltick afterwards;
// and we wait for the increment before emitting traceGoSysExit.
// Note that the increment is done even if tracing is not enabled,
save(pc, sp)
}
- _g_.m.syscalltick = _g_.m.p.syscalltick
+ _g_.m.syscalltick = _g_.m.p.ptr().syscalltick
_g_.m.mcache = nil
- _g_.m.p.m = nil
- atomicstore(&_g_.m.p.status, _Psyscall)
+ _g_.m.p.ptr().m = 0
+ atomicstore(&_g_.m.p.ptr().status, _Psyscall)
if sched.gcwaiting != 0 {
systemstack(entersyscall_gcwait)
save(pc, sp)
func entersyscall_gcwait() {
_g_ := getg()
- _p_ := _g_.m.p
+ _p_ := _g_.m.p.ptr()
lock(&sched.lock)
if sched.stopwait > 0 && cas(&_p_.status, _Psyscall, _Pgcstop) {
_g_.m.locks++ // see comment in entersyscall
_g_.throwsplit = true
_g_.stackguard0 = stackPreempt // see comment in entersyscall
- _g_.m.syscalltick = _g_.m.p.syscalltick
- _g_.m.p.syscalltick++
+ _g_.m.syscalltick = _g_.m.p.ptr().syscalltick
+ _g_.m.p.ptr().syscalltick++
// Leave SP around for GC and traceback.
pc := getcallerpc(unsafe.Pointer(&dummy))
func entersyscallblock_handoff() {
if trace.enabled {
traceGoSysCall()
- traceGoSysBlock(getg().m.p)
+ traceGoSysBlock(getg().m.p.ptr())
}
handoffp(releasep())
}
}
_g_.waitsince = 0
- oldp := _g_.m.p
+ oldp := _g_.m.p.ptr()
if exitsyscallfast() {
if _g_.m.mcache == nil {
throw("lost mcache")
}
if trace.enabled {
- if oldp != _g_.m.p || _g_.m.syscalltick != _g_.m.p.syscalltick {
+ if oldp != _g_.m.p.ptr() || _g_.m.syscalltick != _g_.m.p.ptr().syscalltick {
systemstack(traceGoStart)
}
}
// There's a cpu for us, so we can run.
- _g_.m.p.syscalltick++
+ _g_.m.p.ptr().syscalltick++
// We need to cas the status and scan before resuming...
casgstatus(_g_, _Gsyscall, _Grunning)
// we don't know for sure that the garbage collector
// is not running.
_g_.syscallsp = 0
- _g_.m.p.syscalltick++
+ _g_.m.p.ptr().syscalltick++
_g_.throwsplit = false
if exitTicks != 0 {
// Freezetheworld sets stopwait but does not retake P's.
if sched.stopwait == freezeStopWait {
_g_.m.mcache = nil
- _g_.m.p = nil
+ _g_.m.p = 0
return false
}
// Try to re-acquire the last P.
- if _g_.m.p != nil && _g_.m.p.status == _Psyscall && cas(&_g_.m.p.status, _Psyscall, _Prunning) {
+ if _g_.m.p != 0 && _g_.m.p.ptr().status == _Psyscall && cas(&_g_.m.p.ptr().status, _Psyscall, _Prunning) {
// There's a cpu for us, so we can run.
- _g_.m.mcache = _g_.m.p.mcache
- _g_.m.p.m = _g_.m
- if _g_.m.syscalltick != _g_.m.p.syscalltick {
+ _g_.m.mcache = _g_.m.p.ptr().mcache
+ _g_.m.p.ptr().m.set(_g_.m)
+ if _g_.m.syscalltick != _g_.m.p.ptr().syscalltick {
if trace.enabled {
// The p was retaken and then enter into syscall again (since _g_.m.syscalltick has changed).
// traceGoSysBlock for this syscall was already emitted,
// but here we effectively retake the p from the new syscall running on the same p.
systemstack(func() {
// Denote blocking of the new syscall.
- traceGoSysBlock(_g_.m.p)
+ traceGoSysBlock(_g_.m.p.ptr())
// Denote completion of the current syscall.
traceGoSysExit(0)
})
}
- _g_.m.p.syscalltick++
+ _g_.m.p.ptr().syscalltick++
}
return true
}
// Try to get any other idle P.
- oldp := _g_.m.p
+ oldp := _g_.m.p.ptr()
_g_.m.mcache = nil
- _g_.m.p = nil
- if sched.pidle != nil {
+ _g_.m.p = 0
+ if sched.pidle != 0 {
var ok bool
systemstack(func() {
ok = exitsyscallfast_pidle()
throw("newproc: function arguments too large for new goroutine")
}
- _p_ := _g_.m.p
+ _p_ := _g_.m.p.ptr()
newg := gfget(_p_)
if newg == nil {
newg = malg(_StackMin)
gp.stackguard0 = 0
}
- gp.schedlink = _p_.gfree
+ gp.schedlink.set(_p_.gfree)
_p_.gfree = gp
_p_.gfreecnt++
if _p_.gfreecnt >= 64 {
for _p_.gfreecnt >= 32 {
_p_.gfreecnt--
gp = _p_.gfree
- _p_.gfree = gp.schedlink
- gp.schedlink = sched.gfree
+ _p_.gfree = gp.schedlink.ptr()
+ gp.schedlink.set(sched.gfree)
sched.gfree = gp
sched.ngfree++
}
for _p_.gfreecnt < 32 && sched.gfree != nil {
_p_.gfreecnt++
gp = sched.gfree
- sched.gfree = gp.schedlink
+ sched.gfree = gp.schedlink.ptr()
sched.ngfree--
- gp.schedlink = _p_.gfree
+ gp.schedlink.set(_p_.gfree)
_p_.gfree = gp
}
unlock(&sched.gflock)
goto retry
}
if gp != nil {
- _p_.gfree = gp.schedlink
+ _p_.gfree = gp.schedlink.ptr()
_p_.gfreecnt--
if gp.stack.lo == 0 {
// Stack was deallocated in gfput. Allocate a new one.
for _p_.gfreecnt != 0 {
_p_.gfreecnt--
gp := _p_.gfree
- _p_.gfree = gp.schedlink
- gp.schedlink = sched.gfree
+ _p_.gfree = gp.schedlink.ptr()
+ gp.schedlink.set(sched.gfree)
sched.gfree = gp
sched.ngfree++
}
// This is especially important on windows, since all syscalls are cgo calls.
n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0)
}
- if GOOS == "windows" && n == 0 && mp.libcallg != nil && mp.libcallpc != 0 && mp.libcallsp != 0 {
+ if GOOS == "windows" && n == 0 && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 {
// Libcall, i.e. runtime syscall on windows.
// Collect Go stack that leads to the call.
- n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg, 0, &stk[0], len(stk), nil, nil, 0)
+ n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0)
}
if n == 0 {
// If all of the above has failed, account it against abstract "System" or "GC".
for i := nprocs; i < old; i++ {
p := allp[i]
if trace.enabled {
- if p == getg().m.p {
+ if p == getg().m.p.ptr() {
// moving to p[0], pretend that we were descheduled
// and then scheduled again to keep the trace sane.
traceGoSched()
gp := p.runq[p.runqtail%uint32(len(p.runq))]
// push onto head of global queue
gp.schedlink = sched.runqhead
- sched.runqhead = gp
- if sched.runqtail == nil {
- sched.runqtail = gp
+ sched.runqhead.set(gp)
+ if sched.runqtail == 0 {
+ sched.runqtail.set(gp)
}
sched.runqsize++
}
}
_g_ := getg()
- if _g_.m.p != nil && _g_.m.p.id < nprocs {
+ if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs {
// continue to use the current P
- _g_.m.p.status = _Prunning
+ _g_.m.p.ptr().status = _Prunning
} else {
// release the current P and acquire allp[0]
- if _g_.m.p != nil {
- _g_.m.p.m = nil
+ if _g_.m.p != 0 {
+ _g_.m.p.ptr().m = 0
}
- _g_.m.p = nil
+ _g_.m.p = 0
_g_.m.mcache = nil
p := allp[0]
- p.m = nil
+ p.m = 0
p.status = _Pidle
acquirep(p)
if trace.enabled {
var runnablePs *p
for i := nprocs - 1; i >= 0; i-- {
p := allp[i]
- if _g_.m.p == p {
+ if _g_.m.p.ptr() == p {
continue
}
p.status = _Pidle
if p.runqhead == p.runqtail {
pidleput(p)
} else {
- p.m = mget()
- p.link = runnablePs
+ p.m.set(mget())
+ p.link.set(runnablePs)
runnablePs = p
}
}
}
// Associate p and the current m.
+func acquirep(_p_ *p) {
+ acquirep1(_p_)
+
+ // have p; write barriers now allowed
+ _g_ := getg()
+ _g_.m.mcache = _p_.mcache
+
+ if trace.enabled {
+ traceProcStart()
+ }
+}
+
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
-func acquirep(_p_ *p) {
+func acquirep1(_p_ *p) {
_g_ := getg()
- if _g_.m.p != nil || _g_.m.mcache != nil {
+ if _g_.m.p != 0 || _g_.m.mcache != nil {
throw("acquirep: already in go")
}
- if _p_.m != nil || _p_.status != _Pidle {
+ if _p_.m != 0 || _p_.status != _Pidle {
id := int32(0)
- if _p_.m != nil {
- id = _p_.m.id
+ if _p_.m != 0 {
+ id = _p_.m.ptr().id
}
print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n")
throw("acquirep: invalid p state")
}
- // _p_.mcache holds the mcache and _p_ is in allp, so WB can be eliminated
- setMcacheNoWriteBarrier(&_g_.m.mcache, _p_.mcache)
- // _p_ is in allp so WB can be eliminated
- setPNoWriteBarrier(&_g_.m.p, _p_)
- // m is in _g_.m and is reachable through allg, so WB can be eliminated
- setMNoWriteBarrier(&_p_.m, _g_.m)
+ _g_.m.p.set(_p_)
+ _p_.m.set(_g_.m)
_p_.status = _Prunning
-
- if trace.enabled {
- traceProcStart()
- }
}
// Disassociate p and the current m.
func releasep() *p {
_g_ := getg()
- if _g_.m.p == nil || _g_.m.mcache == nil {
+ if _g_.m.p == 0 || _g_.m.mcache == nil {
throw("releasep: invalid arg")
}
- _p_ := _g_.m.p
- if _p_.m != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning {
- print("releasep: m=", _g_.m, " m->p=", _g_.m.p, " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n")
+ _p_ := _g_.m.p.ptr()
+ if _p_.m.ptr() != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning {
+ print("releasep: m=", _g_.m, " m->p=", _g_.m.p.ptr(), " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n")
throw("releasep: invalid p state")
}
if trace.enabled {
- traceProcStop(_g_.m.p)
+ traceProcStop(_g_.m.p.ptr())
}
- _g_.m.p = nil
+ _g_.m.p = 0
_g_.m.mcache = nil
- _p_.m = nil
+ _p_.m = 0
_p_.status = _Pidle
return _p_
}
if mp == nil {
newm(nil, _p_)
} else {
- mp.nextp = _p_
+ mp.nextp.set(_p_)
notewakeup(&mp.park)
}
return
if lastgc != 0 && unixnow-lastgc > forcegcperiod && atomicload(&forcegc.idle) != 0 {
lock(&forcegc.lock)
forcegc.idle = 0
- forcegc.g.schedlink = nil
+ forcegc.g.schedlink = 0
injectglist(forcegc.g)
unlock(&forcegc.lock)
}
// and will be indicated by the gp->status no longer being
// Grunning
func preemptone(_p_ *p) bool {
- mp := _p_.m
+ mp := _p_.m.ptr()
if mp == nil || mp == getg().m {
return false
}
if _p_ == nil {
continue
}
- mp := _p_.m
+ mp := _p_.m.ptr()
h := atomicload(&_p_.runqhead)
t := atomicload(&_p_.runqtail)
if detailed {
}
for mp := allm; mp != nil; mp = mp.alllink {
- _p_ := mp.p
+ _p_ := mp.p.ptr()
gp := mp.curg
lockedg := mp.lockedg
id1 := int32(-1)
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func mput(mp *m) {
- // sched.midle is reachable via allm, so WB can be eliminated.
- setMNoWriteBarrier(&mp.schedlink, sched.midle)
- // mp is reachable via allm, so WB can be eliminated.
- setMNoWriteBarrier(&sched.midle, mp)
+ mp.schedlink = sched.midle
+ sched.midle.set(mp)
sched.nmidle++
checkdead()
}
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func mget() *m {
- mp := sched.midle
+ mp := sched.midle.ptr()
if mp != nil {
- // mp.schedlink is reachable via mp, which is on allm, so WB can be eliminated.
- setMNoWriteBarrier(&sched.midle, mp.schedlink)
+ sched.midle = mp.schedlink
sched.nmidle--
}
return mp
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func globrunqput(gp *g) {
- gp.schedlink = nil
- if sched.runqtail != nil {
- // gp is on allg, so these three WBs can be eliminated.
- setGNoWriteBarrier(&sched.runqtail.schedlink, gp)
+ gp.schedlink = 0
+ if sched.runqtail != 0 {
+ sched.runqtail.ptr().schedlink.set(gp)
} else {
- setGNoWriteBarrier(&sched.runqhead, gp)
+ sched.runqhead.set(gp)
}
- setGNoWriteBarrier(&sched.runqtail, gp)
+ sched.runqtail.set(gp)
sched.runqsize++
}
// Put a batch of runnable goroutines on the global runnable queue.
// Sched must be locked.
func globrunqputbatch(ghead *g, gtail *g, n int32) {
- gtail.schedlink = nil
- if sched.runqtail != nil {
- sched.runqtail.schedlink = ghead
+ gtail.schedlink = 0
+ if sched.runqtail != 0 {
+ sched.runqtail.ptr().schedlink.set(ghead)
} else {
- sched.runqhead = ghead
+ sched.runqhead.set(ghead)
}
- sched.runqtail = gtail
+ sched.runqtail.set(gtail)
sched.runqsize += n
}
sched.runqsize -= n
if sched.runqsize == 0 {
- sched.runqtail = nil
+ sched.runqtail = 0
}
- gp := sched.runqhead
+ gp := sched.runqhead.ptr()
sched.runqhead = gp.schedlink
n--
for ; n > 0; n-- {
- gp1 := sched.runqhead
+ gp1 := sched.runqhead.ptr()
sched.runqhead = gp1.schedlink
runqput(_p_, gp1)
}
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func pidleput(_p_ *p) {
- // sched.pidle, _p_.link and _p_ are reachable via allp, so WB can be eliminated.
- setPNoWriteBarrier(&_p_.link, sched.pidle)
- setPNoWriteBarrier(&sched.pidle, _p_)
+ _p_.link = sched.pidle
+ sched.pidle.set(_p_)
xadd(&sched.npidle, 1) // TODO: fast atomic
}
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func pidleget() *p {
- _p_ := sched.pidle
+ _p_ := sched.pidle.ptr()
if _p_ != nil {
- // _p_.link is reachable via a _p_ in allp, so WB can be eliminated.
- setPNoWriteBarrier(&sched.pidle, _p_.link)
+ sched.pidle = _p_.link
xadd(&sched.npidle, -1) // TODO: fast atomic
}
return _p_
// Link the goroutines.
for i := uint32(0); i < n; i++ {
- batch[i].schedlink = batch[i+1]
+ batch[i].schedlink.set(batch[i+1])
}
// Now put the batch on global queue.
mp := _g_.m
mp.locks++
- return int(mp.p.id)
+ return int(mp.p.ptr().id)
}
//go:nosplit
if i >= active_spin || ncpu <= 1 || gomaxprocs <= int32(sched.npidle+sched.nmspinning)+1 {
return false
}
- if p := getg().m.p; p.runqhead != p.runqtail {
+ if p := getg().m.p.ptr(); p.runqhead != p.runqtail {
return false
}
return true
data unsafe.Pointer
}
+// The guintptr, muintptr, and puintptr are all used to bypass write barriers.
+// It is particularly important to avoid write barriers when the current P has
+// been released, because the GC thinks the world is stopped, and an
+// unexpected write barrier would not be synchronized with the GC,
+// which can lead to a half-executed write barrier that has marked the object
+// but not queued it. If the GC skips the object and completes before the
+// queuing can occur, it will incorrectly free the object.
+//
+// We tried using special assignment functions invoked only when not
+// holding a running P, but then some updates to a particular memory
+// word went through write barriers and some did not. This breaks the
+// write barrier shadow checking mode, and it is also scary: better to have
+// a word that is completely ignored by the GC than to have one for which
+// only a few updates are ignored.
+//
+// Gs, Ms, and Ps are always reachable via true pointers in the
+// allgs, allm, and allp lists or (during allocation before they reach those lists)
+// from stack variables.
+
// A guintptr holds a goroutine pointer, but typed as a uintptr
-// to bypass write barriers. It is used in the Gobuf goroutine state.
+// to bypass write barriers. It is used in the Gobuf goroutine state
+// and in scheduling lists that are manipulated without a P.
//
// The Gobuf.g goroutine pointer is almost always updated by assembly code.
// In one of the few places it is updated by Go code - func save - it must be
// alternate arena. Using guintptr doesn't make that problem any worse.
type guintptr uintptr
-func (gp guintptr) ptr() *g {
- return (*g)(unsafe.Pointer(gp))
-}
+func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) }
+func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) }
-// ps, ms, gs, and mcache are structures that must be manipulated at a level
-// lower than that of the normal Go language. For example the routine that
-// stops the world removes the p from the m structure informing the GC that
-// this P is stopped and then it moves the g to the global runnable queue.
-// If write barriers were allowed to happen at this point not only does
-// the GC think the thread is stopped but the underlying structures
-// like a p or m are not in a state that is not coherent enough to
-// support the write barrier actions.
-// This is particularly painful since a partially executed write barrier
-// may mark the object but be delinquent in informing the GC that the
-// object needs to be scanned.
-
-// setGNoWriteBarriers does *gdst = gval without a write barrier.
-func setGNoWriteBarrier(gdst **g, gval *g) {
- *(*uintptr)(unsafe.Pointer(gdst)) = uintptr(unsafe.Pointer(gval))
-}
+type puintptr uintptr
-// setMNoWriteBarriers does *mdst = mval without a write barrier.
-func setMNoWriteBarrier(mdst **m, mval *m) {
- *(*uintptr)(unsafe.Pointer(mdst)) = uintptr(unsafe.Pointer(mval))
-}
+func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) }
+func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) }
-// setPNoWriteBarriers does *pdst = pval without a write barrier.
-func setPNoWriteBarrier(pdst **p, pval *p) {
- *(*uintptr)(unsafe.Pointer(pdst)) = uintptr(unsafe.Pointer(pval))
-}
+type muintptr uintptr
-// setMcacheNoWriteBarriers does *mcachedst = mcacheval without a write barrier.
-func setMcacheNoWriteBarrier(mcachedst **mcache, mcacheval *mcache) {
- *(*uintptr)(unsafe.Pointer(mcachedst)) = uintptr(unsafe.Pointer(mcacheval))
-}
+func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) }
+func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) }
type gobuf struct {
// The offsets of sp, pc, and g are known to (hard-coded in) libmach.
goid int64
waitsince int64 // approx time when the g become blocked
waitreason string // if status==gwaiting
- schedlink *g
+ schedlink guintptr
preempt bool // preemption signal, duplicates stackguard0 = stackpreempt
paniconfault bool // panic (instead of crash) on unexpected fault address
preemptscan bool // preempted g does scan for gc
procid uint64 // for debuggers, but offset not hard-coded
gsignal *g // signal-handling g
tls [4]uintptr // thread-local storage (for x86 extern register)
- mstartfn uintptr // TODO: type as func(); note: this is a non-heap allocated func()
- curg *g // current running goroutine
- caughtsig *g // goroutine running during fatal signal
- p *p // attached p for executing go code (nil if not executing go code)
- nextp *p
+ mstartfn func()
+ curg *g // current running goroutine
+ caughtsig guintptr // goroutine running during fatal signal
+ p puintptr // attached p for executing go code (nil if not executing go code)
+ nextp puintptr
id int32
mallocing int32
throwing int32
ncgo int32 // number of cgo calls currently in progress
park note
alllink *m // on allm
- schedlink *m
+ schedlink muintptr
machport uint32 // return address for mach ipc (os x)
mcache *mcache
lockedg *g
libcall libcall
libcallpc uintptr // for cpu profiler
libcallsp uintptr
- libcallg *g
+ libcallg guintptr
//#endif
//#ifdef GOOS_solaris
perrno *int32 // pointer to tls errno
id int32
status uint32 // one of pidle/prunning/...
- link *p
- schedtick uint32 // incremented on every scheduler call
- syscalltick uint32 // incremented on every system call
- m *m // back-link to associated m (nil if idle)
+ link puintptr
+ schedtick uint32 // incremented on every scheduler call
+ syscalltick uint32 // incremented on every system call
+ m muintptr // back-link to associated m (nil if idle)
mcache *mcache
deferpool [5][]*_defer // pool of available defer structs of different sizes (see panic.go)
goidgen uint64
- midle *m // idle m's waiting for work
- nmidle int32 // number of idle m's waiting for work
- nmidlelocked int32 // number of locked m's waiting for work
- mcount int32 // number of m's that have been created
- maxmcount int32 // maximum number of m's allowed (or die)
+ midle muintptr // idle m's waiting for work
+ nmidle int32 // number of idle m's waiting for work
+ nmidlelocked int32 // number of locked m's waiting for work
+ mcount int32 // number of m's that have been created
+ maxmcount int32 // maximum number of m's allowed (or die)
- pidle *p // idle p's
+ pidle puintptr // idle p's
npidle uint32
nmspinning uint32
// Global runnable queue.
- runqhead *g
- runqtail *g
+ runqhead guintptr
+ runqtail guintptr
runqsize int32
// Global cache of dead G's.