From 181e26b9fa3c85ca5a512a51278b3f22f6e64dc9 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 17 Apr 2015 00:21:30 -0400 Subject: [PATCH] runtime: replace func-based write barrier skipping with type-based This CL revises CL 7504 to use explicitly uintptr types for the struct fields that are going to be updated sometimes without write barriers. The result is that the fields are now updated *always* without write barriers. This approach has two important properties: 1) Now the GC never looks at the field, so if the missing reference could cause a problem, it will do so all the time, not just when the write barrier is missed at just the right moment. 2) Now a write barrier never happens for the field, avoiding the (correct) detection of inconsistent write barriers when GODEBUG=wbshadow=1. Change-Id: Iebd3962c727c0046495cc08914a8dc0808460e0e Reviewed-on: https://go-review.googlesource.com/9019 Reviewed-by: Austin Clements Run-TryBot: Russ Cox TryBot-Result: Gobot Gobot --- src/runtime/malloc.go | 4 +- src/runtime/mbarrier.go | 6 +- src/runtime/netpoll.go | 22 ++- src/runtime/netpoll_epoll.go | 12 +- src/runtime/netpoll_kqueue.go | 11 +- src/runtime/netpoll_solaris.go | 12 +- src/runtime/netpoll_windows.go | 11 +- src/runtime/os1_windows.go | 3 +- src/runtime/os3_plan9.go | 2 +- src/runtime/panic.go | 4 +- src/runtime/proc.go | 4 +- src/runtime/proc1.go | 315 ++++++++++++++++----------------- src/runtime/runtime2.go | 101 +++++------ src/runtime/signal_386.go | 2 +- src/runtime/signal_amd64x.go | 2 +- src/runtime/signal_arm.go | 2 +- src/runtime/signal_arm64.go | 2 +- src/runtime/signal_ppc64x.go | 2 +- src/runtime/stack1.go | 4 +- src/runtime/trace.go | 6 +- src/runtime/traceback.go | 2 +- 21 files changed, 257 insertions(+), 272 deletions(-) diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index e41c2736b1..5fe0b160e6 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -817,8 +817,8 @@ func persistentalloc(size, align uintptr, stat *uint64) unsafe.Pointer { mp := acquirem() var persistent *persistentAlloc - if mp != nil && mp.p != nil { - persistent = &mp.p.palloc + if mp != nil && mp.p != 0 { + persistent = &mp.p.ptr().palloc } else { lock(&globalAlloc.mutex) persistent = &globalAlloc.persistentAlloc diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go index 2e3e4d8041..9bf0e6b809 100644 --- a/src/runtime/mbarrier.go +++ b/src/runtime/mbarrier.go @@ -92,10 +92,6 @@ func needwb() bool { // the p associated with an m. We use the fact that m.p == nil to indicate // that we are in one these critical section and throw if the write is of // a pointer to a heap object. -// The p, m, and g pointers are the pointers that are used by the scheduler -// and need to be operated on without write barriers. We use -// the setPNoWriteBarrier, setMNoWriteBarrier and setGNowriteBarrier to -// avoid having to do the write barrier. //go:nosplit func writebarrierptr_nostore1(dst *uintptr, src uintptr) { mp := acquirem() @@ -104,7 +100,7 @@ func writebarrierptr_nostore1(dst *uintptr, src uintptr) { return } systemstack(func() { - if mp.p == nil && memstats.enablegc && !mp.inwb && inheap(src) { + if mp.p == 0 && memstats.enablegc && !mp.inwb && inheap(src) { throw("writebarrierptr_nostore1 called with mp.p == nil") } mp.inwb = true diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index 4791e5eebe..c06722fb98 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -275,24 +275,22 @@ func net_runtime_pollUnblock(pd *pollDesc) { // make pd ready, newly runnable goroutines (if any) are returned in rg/wg // May run during STW, so write barriers are not allowed. -// Eliminating WB calls using setGNoWriteBarrier are safe since the gs are -// reachable through allg. //go:nowritebarrier -func netpollready(gpp **g, pd *pollDesc, mode int32) { - var rg, wg *g +func netpollready(gpp *guintptr, pd *pollDesc, mode int32) { + var rg, wg guintptr if mode == 'r' || mode == 'r'+'w' { - setGNoWriteBarrier(&rg, netpollunblock(pd, 'r', true)) + rg.set(netpollunblock(pd, 'r', true)) } if mode == 'w' || mode == 'r'+'w' { - setGNoWriteBarrier(&wg, netpollunblock(pd, 'w', true)) + wg.set(netpollunblock(pd, 'w', true)) } - if rg != nil { - setGNoWriteBarrier(&rg.schedlink, *gpp) - setGNoWriteBarrier(gpp, rg) + if rg != 0 { + rg.ptr().schedlink = *gpp + *gpp = rg } - if wg != nil { - setGNoWriteBarrier(&wg.schedlink, *gpp) - setGNoWriteBarrier(gpp, wg) + if wg != 0 { + wg.ptr().schedlink = *gpp + *gpp = wg } } diff --git a/src/runtime/netpoll_epoll.go b/src/runtime/netpoll_epoll.go index 4f9ccaf477..7b4052a262 100644 --- a/src/runtime/netpoll_epoll.go +++ b/src/runtime/netpoll_epoll.go @@ -55,9 +55,9 @@ func netpollarm(pd *pollDesc, mode int) { // polls for ready network connections // returns list of goroutines that become runnable -func netpoll(block bool) (gp *g) { +func netpoll(block bool) *g { if epfd == -1 { - return + return nil } waitms := int32(-1) if !block { @@ -73,6 +73,7 @@ retry: } goto retry } + var gp guintptr for i := int32(0); i < n; i++ { ev := &events[i] if ev.events == 0 { @@ -87,11 +88,12 @@ retry: } if mode != 0 { pd := *(**pollDesc)(unsafe.Pointer(&ev.data)) - netpollready((**g)(noescape(unsafe.Pointer(&gp))), pd, mode) + + netpollready(&gp, pd, mode) } } - if block && gp == nil { + if block && gp == 0 { goto retry } - return gp + return gp.ptr() } diff --git a/src/runtime/netpoll_kqueue.go b/src/runtime/netpoll_kqueue.go index 080964bb72..01445dc231 100644 --- a/src/runtime/netpoll_kqueue.go +++ b/src/runtime/netpoll_kqueue.go @@ -62,9 +62,9 @@ func netpollarm(pd *pollDesc, mode int) { // Polls for ready network connections. // Returns list of goroutines that become runnable. -func netpoll(block bool) (gp *g) { +func netpoll(block bool) *g { if kq == -1 { - return + return nil } var tp *timespec var ts timespec @@ -81,6 +81,7 @@ retry: } goto retry } + var gp guintptr for i := 0; i < int(n); i++ { ev := &events[i] var mode int32 @@ -91,11 +92,11 @@ retry: mode += 'w' } if mode != 0 { - netpollready((**g)(noescape(unsafe.Pointer(&gp))), (*pollDesc)(unsafe.Pointer(ev.udata)), mode) + netpollready(&gp, (*pollDesc)(unsafe.Pointer(ev.udata)), mode) } } - if block && gp == nil { + if block && gp == 0 { goto retry } - return gp + return gp.ptr() } diff --git a/src/runtime/netpoll_solaris.go b/src/runtime/netpoll_solaris.go index d8f050e1e2..abfe56d801 100644 --- a/src/runtime/netpoll_solaris.go +++ b/src/runtime/netpoll_solaris.go @@ -179,9 +179,9 @@ var netpolllasterr int32 // polls for ready network connections // returns list of goroutines that become runnable -func netpoll(block bool) (gp *g) { +func netpoll(block bool) *g { if portfd == -1 { - return + return nil } var wait *timespec @@ -201,7 +201,7 @@ retry: goto retry } - gp = nil + var gp guintptr for i := 0; i < int(n); i++ { ev := &events[i] @@ -232,12 +232,12 @@ retry: } if mode != 0 { - netpollready((**g)(noescape(unsafe.Pointer(&gp))), pd, mode) + netpollready(&gp, pd, mode) } } - if block && gp == nil { + if block && gp == 0 { goto retry } - return gp + return gp.ptr() } diff --git a/src/runtime/netpoll_windows.go b/src/runtime/netpoll_windows.go index 0861e20f0c..7e15cd231e 100644 --- a/src/runtime/netpoll_windows.go +++ b/src/runtime/netpoll_windows.go @@ -63,14 +63,13 @@ func netpoll(block bool) *g { var wait, qty, key, flags, n, i uint32 var errno int32 var op *net_op - var gp *g + var gp guintptr mp := getg().m if iocphandle == _INVALID_HANDLE_VALUE { return nil } - gp = nil wait = 0 if block { wait = _INFINITE @@ -125,13 +124,13 @@ retry: mp.blocked = false handlecompletion(&gp, op, errno, qty) } - if block && gp == nil { + if block && gp == 0 { goto retry } - return gp + return gp.ptr() } -func handlecompletion(gpp **g, op *net_op, errno int32, qty uint32) { +func handlecompletion(gpp *guintptr, op *net_op, errno int32, qty uint32) { if op == nil { throw("netpoll: GetQueuedCompletionStatus returned op == nil") } @@ -142,5 +141,5 @@ func handlecompletion(gpp **g, op *net_op, errno int32, qty uint32) { } op.errno = errno op.qty = qty - netpollready((**g)(noescape(unsafe.Pointer(gpp))), op.pd, mode) + netpollready(gpp, op.pd, mode) } diff --git a/src/runtime/os1_windows.go b/src/runtime/os1_windows.go index 8df2968196..2b9739fca3 100644 --- a/src/runtime/os1_windows.go +++ b/src/runtime/os1_windows.go @@ -363,8 +363,7 @@ func stdcall(fn stdFunction) uintptr { if mp.profilehz != 0 { // leave pc/sp for cpu profiler - // gp is on allg, so this WB can be eliminated. - setGNoWriteBarrier(&mp.libcallg, gp) + mp.libcallg.set(gp) mp.libcallpc = getcallerpc(unsafe.Pointer(&fn)) // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them diff --git a/src/runtime/os3_plan9.go b/src/runtime/os3_plan9.go index fa49ad681f..248576a102 100644 --- a/src/runtime/os3_plan9.go +++ b/src/runtime/os3_plan9.go @@ -81,7 +81,7 @@ func sighandler(_ureg *ureg, note *byte, gp *g) int { } Throw: _g_.m.throwing = 1 - setGNoWriteBarrier(&_g_.m.caughtsig, gp) + _g_.m.caughtsig.set(gp) startpanic() print(notestr, "\n") print("PC=", hex(c.pc()), "\n") diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 9b937f5ad7..0e4086c7ef 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -165,7 +165,7 @@ func newdefer(siz int32) *_defer { sc := deferclass(uintptr(siz)) mp := acquirem() if sc < uintptr(len(p{}.deferpool)) { - pp := mp.p + pp := mp.p.ptr() if len(pp.deferpool[sc]) == 0 && sched.deferpool[sc] != nil { lock(&sched.deferlock) for len(pp.deferpool[sc]) < cap(pp.deferpool[sc])/2 && sched.deferpool[sc] != nil { @@ -223,7 +223,7 @@ func freedefer(d *_defer) { sc := deferclass(uintptr(d.siz)) if sc < uintptr(len(p{}.deferpool)) { mp := acquirem() - pp := mp.p + pp := mp.p.ptr() if len(pp.deferpool[sc]) == cap(pp.deferpool[sc]) { // Transfer half of local cache to the central cache. var first, last *_defer diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 88b2004c83..f725fc890b 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -208,7 +208,7 @@ func acquireSudog() *sudog { // The acquirem/releasem increments m.locks during new(sudog), // which keeps the garbage collector from being invoked. mp := acquirem() - pp := mp.p + pp := mp.p.ptr() if len(pp.sudogcache) == 0 { lock(&sched.sudoglock) // First, try to grab a batch from central cache. @@ -257,7 +257,7 @@ func releaseSudog(s *sudog) { throw("runtime: releaseSudog with non-nil gp.param") } mp := acquirem() // avoid rescheduling to another P - pp := mp.p + pp := mp.p.ptr() if len(pp.sudogcache) == cap(pp.sudogcache) { // Transfer half of local cache to the central cache. var first, last *sudog diff --git a/src/runtime/proc1.go b/src/runtime/proc1.go index a46ebc8cec..cf6e604ed0 100644 --- a/src/runtime/proc1.go +++ b/src/runtime/proc1.go @@ -145,7 +145,7 @@ func ready(gp *g, traceskip int) { // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq casgstatus(gp, _Gwaiting, _Grunnable) - runqput(_g_.m.p, gp) + runqput(_g_.m.p.ptr(), gp) if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { // TODO: fast atomic wakep() } @@ -185,7 +185,7 @@ func readyExecute(gp *g, traceskip int) { // Preempt the current g casgstatus(_g_, _Grunning, _Grunnable) - runqput(_g_.m.p, _g_) + runqput(_g_.m.p.ptr(), _g_) dropg() // Ready gp and switch to it @@ -239,7 +239,7 @@ func helpgc(nproc int32) { throw("gcprocs inconsistency") } mp.helpgc = n - mp.p = allp[pos] + mp.p.set(allp[pos]) mp.mcache = allp[pos].mcache pos++ notewakeup(&mp.park) @@ -603,7 +603,7 @@ func stoptheworld() { atomicstore(&sched.gcwaiting, 1) preemptall() // stop current P - _g_.m.p.status = _Pgcstop // Pgcstop is only diagnostic. + _g_.m.p.ptr().status = _Pgcstop // Pgcstop is only diagnostic. sched.stopwait-- // try to retake all P's in Psyscall status for i := 0; i < int(gomaxprocs); i++ { @@ -681,14 +681,14 @@ func starttheworld() { for p1 != nil { p := p1 - p1 = p1.link - if p.m != nil { - mp := p.m - p.m = nil - if mp.nextp != nil { + p1 = p1.link.ptr() + if p.m != 0 { + mp := p.m.ptr() + p.m = 0 + if mp.nextp != 0 { throw("starttheworld: inconsistent mp->nextp") } - mp.nextp = p + mp.nextp.set(p) notewakeup(&mp.park) } else { // Start M to run P. Do not start another M below. @@ -768,8 +768,7 @@ func mstart1() { initsig() } - if _g_.m.mstartfn != 0 { - fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn)) + if fn := _g_.m.mstartfn; fn != nil { fn() } @@ -777,8 +776,8 @@ func mstart1() { _g_.m.helpgc = 0 stopm() } else if _g_.m != &m0 { - acquirep(_g_.m.nextp) - _g_.m.nextp = nil + acquirep(_g_.m.nextp.ptr()) + _g_.m.nextp = 0 } schedule() } @@ -789,20 +788,22 @@ func mstart1() { var cgoThreadStart unsafe.Pointer type cgothreadstart struct { - g *g + g guintptr tls *uint64 fn unsafe.Pointer } // Allocate a new m unassociated with any thread. // Can use p for allocation context if needed. -func allocm(_p_ *p) *m { +// fn is recorded as the new m's m.mstartfn. +func allocm(_p_ *p, fn func()) *m { _g_ := getg() _g_.m.locks++ // disable GC because it can be called from sysmon - if _g_.m.p == nil { + if _g_.m.p == 0 { acquirep(_p_) // temporarily borrow p for mallocs in this function } mp := new(m) + mp.mstartfn = fn mcommoninit(mp) // In case of cgo or Solaris, pthread_create will make us a stack. @@ -814,7 +815,7 @@ func allocm(_p_ *p) *m { } mp.g0.m = mp - if _p_ == _g_.m.p { + if _p_ == _g_.m.p.ptr() { releasep() } _g_.m.locks-- @@ -880,8 +881,8 @@ func needm(x byte) { // after exitsyscall makes sure it is okay to be // running at all (that is, there's no garbage collection // running right now). - mp.needextram = mp.schedlink == nil - unlockextra(mp.schedlink) + mp.needextram = mp.schedlink == 0 + unlockextra(mp.schedlink.ptr()) // Install g (= m->g0) and set the stack bounds // to match the current stack. We don't actually know @@ -910,7 +911,7 @@ func newextram() { // The sched.pc will never be returned to, but setting it to // goexit makes clear to the traceback routines where // the goroutine stack ends. - mp := allocm(nil) + mp := allocm(nil, nil) gp := malg(4096) gp.sched.pc = funcPC(goexit) + _PCQuantum gp.sched.sp = gp.stack.hi @@ -936,7 +937,7 @@ func newextram() { // Add m to the extra list. mnext := lockextra(true) - mp.schedlink = mnext + mp.schedlink.set(mnext) unlockextra(mp) } @@ -972,7 +973,7 @@ func dropm() { // with no pointer manipulation. mp := getg().m mnext := lockextra(true) - mp.schedlink = mnext + mp.schedlink.set(mnext) setg(nil) unlockextra(mp) @@ -1019,18 +1020,14 @@ func unlockextra(mp *m) { // May run with m.p==nil, so write barriers are not allowed. //go:nowritebarrier func newm(fn func(), _p_ *p) { - mp := allocm(_p_) - // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated - setPNoWriteBarrier(&mp.nextp, _p_) - // Store &fn as a uintptr since it is not heap allocated so the WB can be eliminated - mp.mstartfn = *(*uintptr)(unsafe.Pointer(&fn)) + mp := allocm(_p_, fn) + mp.nextp.set(_p_) if iscgo { var ts cgothreadstart if _cgo_thread_start == nil { throw("_cgo_thread_start missing") } - // mp is reachable via allm and mp.g0 never changes, so WB can be eliminated. - setGNoWriteBarrier(&ts.g, mp.g0) + ts.g.set(mp.g0) ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0])) ts.fn = unsafe.Pointer(funcPC(mstart)) asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts)) @@ -1047,7 +1044,7 @@ func stopm() { if _g_.m.locks != 0 { throw("stopm holding locks") } - if _g_.m.p != nil { + if _g_.m.p != 0 { throw("stopm holding p") } if _g_.m.spinning { @@ -1065,11 +1062,11 @@ retry: gchelper() _g_.m.helpgc = 0 _g_.m.mcache = nil - _g_.m.p = nil + _g_.m.p = 0 goto retry } - acquirep(_g_.m.nextp) - _g_.m.nextp = nil + acquirep(_g_.m.nextp.ptr()) + _g_.m.nextp = 0 } func mspinning() { @@ -1105,12 +1102,11 @@ func startm(_p_ *p, spinning bool) { if mp.spinning { throw("startm: m is spinning") } - if mp.nextp != nil { + if mp.nextp != 0 { throw("startm: m has p") } mp.spinning = spinning - // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated - setPNoWriteBarrier(&mp.nextp, _p_) + mp.nextp.set(_p_) notewakeup(&mp.park) } @@ -1173,7 +1169,7 @@ func stoplockedm() { if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m { throw("stoplockedm: inconsistent locking") } - if _g_.m.p != nil { + if _g_.m.p != 0 { // Schedule another M to run this p. _p_ := releasep() handoffp(_p_) @@ -1188,8 +1184,8 @@ func stoplockedm() { dumpgstatus(_g_) throw("stoplockedm: not runnable") } - acquirep(_g_.m.nextp) - _g_.m.nextp = nil + acquirep(_g_.m.nextp.ptr()) + _g_.m.nextp = 0 } // Schedules the locked m to run the locked gp. @@ -1202,14 +1198,13 @@ func startlockedm(gp *g) { if mp == _g_.m { throw("startlockedm: locked to me") } - if mp.nextp != nil { + if mp.nextp != 0 { throw("startlockedm: m has p") } // directly handoff current P to the locked m incidlelocked(-1) _p_ := releasep() - // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated - setPNoWriteBarrier(&mp.nextp, _p_) + mp.nextp.set(_p_) notewakeup(&mp.park) stopm() } @@ -1246,7 +1241,7 @@ func execute(gp *g) { gp.waitsince = 0 gp.preempt = false gp.stackguard0 = gp.stack.lo + _StackGuard - _g_.m.p.schedtick++ + _g_.m.p.ptr().schedtick++ _g_.m.curg = gp gp.m = _g_.m @@ -1280,14 +1275,14 @@ top: } // local runq - if gp := runqget(_g_.m.p); gp != nil { + if gp := runqget(_g_.m.p.ptr()); gp != nil { return gp } // global runq if sched.runqsize != 0 { lock(&sched.lock) - gp := globrunqget(_g_.m.p, 0) + gp := globrunqget(_g_.m.p.ptr(), 0) unlock(&sched.lock) if gp != nil { return gp @@ -1303,7 +1298,7 @@ top: if netpollinited() && sched.lastpoll != 0 { if gp := netpoll(false); gp != nil { // non-blocking // netpoll returns list of goroutines linked by schedlink. - injectglist(gp.schedlink) + injectglist(gp.schedlink.ptr()) casgstatus(gp, _Gwaiting, _Grunnable) if trace.enabled { traceGoUnpark(gp, 0) @@ -1329,10 +1324,10 @@ top: } _p_ := allp[fastrand1()%uint32(gomaxprocs)] var gp *g - if _p_ == _g_.m.p { + if _p_ == _g_.m.p.ptr() { gp = runqget(_p_) } else { - gp = runqsteal(_g_.m.p, _p_) + gp = runqsteal(_g_.m.p.ptr(), _p_) } if gp != nil { return gp @@ -1347,7 +1342,7 @@ stop: goto top } if sched.runqsize != 0 { - gp := globrunqget(_g_.m.p, 0) + gp := globrunqget(_g_.m.p.ptr(), 0) unlock(&sched.lock) return gp } @@ -1376,7 +1371,7 @@ stop: // poll network if netpollinited() && xchg64(&sched.lastpoll, 0) != 0 { - if _g_.m.p != nil { + if _g_.m.p != 0 { throw("findrunnable: netpoll with p") } if _g_.m.spinning { @@ -1390,7 +1385,7 @@ stop: unlock(&sched.lock) if _p_ != nil { acquirep(_p_) - injectglist(gp.schedlink) + injectglist(gp.schedlink.ptr()) casgstatus(gp, _Gwaiting, _Grunnable) if trace.enabled { traceGoUnpark(gp, 0) @@ -1432,7 +1427,7 @@ func injectglist(glist *g) { return } if trace.enabled { - for gp := glist; gp != nil; gp = gp.schedlink { + for gp := glist; gp != nil; gp = gp.schedlink.ptr() { traceGoUnpark(gp, 0) } } @@ -1440,7 +1435,7 @@ func injectglist(glist *g) { var n int for n = 0; glist != nil; n++ { gp := glist - glist = gp.schedlink + glist = gp.schedlink.ptr() casgstatus(gp, _Gwaiting, _Grunnable) globrunqput(gp) } @@ -1483,9 +1478,9 @@ top: // Check the global runnable queue once in a while to ensure fairness. // Otherwise two goroutines can completely occupy the local runqueue // by constantly respawning each other. - if _g_.m.p.schedtick%61 == 0 && sched.runqsize > 0 { + if _g_.m.p.ptr().schedtick%61 == 0 && sched.runqsize > 0 { lock(&sched.lock) - gp = globrunqget(_g_.m.p, 1) + gp = globrunqget(_g_.m.p.ptr(), 1) unlock(&sched.lock) if gp != nil { resetspinning() @@ -1493,7 +1488,7 @@ top: } } if gp == nil { - gp = runqget(_g_.m.p) + gp = runqget(_g_.m.p.ptr()) if gp != nil && _g_.m.spinning { throw("schedule: spinning with local work") } @@ -1624,7 +1619,7 @@ func goexit0(gp *g) { throw("internal lockOSThread error") } _g_.m.locked = 0 - gfput(_g_.m.p, gp) + gfput(_g_.m.p.ptr(), gp) schedule() } @@ -1671,7 +1666,7 @@ func save(pc, sp uintptr) { // when syscall returns we emit traceGoSysExit and when the goroutine starts running // (potentially instantly, if exitsyscallfast returns true) we emit traceGoStart. // To ensure that traceGoSysExit is emitted strictly after traceGoSysBlock, -// we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.syscalltick), +// we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.ptr().syscalltick), // whoever emits traceGoSysBlock increments p.syscalltick afterwards; // and we wait for the increment before emitting traceGoSysExit. // Note that the increment is done even if tracing is not enabled, @@ -1713,10 +1708,10 @@ func reentersyscall(pc, sp uintptr) { save(pc, sp) } - _g_.m.syscalltick = _g_.m.p.syscalltick + _g_.m.syscalltick = _g_.m.p.ptr().syscalltick _g_.m.mcache = nil - _g_.m.p.m = nil - atomicstore(&_g_.m.p.status, _Psyscall) + _g_.m.p.ptr().m = 0 + atomicstore(&_g_.m.p.ptr().status, _Psyscall) if sched.gcwaiting != 0 { systemstack(entersyscall_gcwait) save(pc, sp) @@ -1746,7 +1741,7 @@ func entersyscall_sysmon() { func entersyscall_gcwait() { _g_ := getg() - _p_ := _g_.m.p + _p_ := _g_.m.p.ptr() lock(&sched.lock) if sched.stopwait > 0 && cas(&_p_.status, _Psyscall, _Pgcstop) { @@ -1770,8 +1765,8 @@ func entersyscallblock(dummy int32) { _g_.m.locks++ // see comment in entersyscall _g_.throwsplit = true _g_.stackguard0 = stackPreempt // see comment in entersyscall - _g_.m.syscalltick = _g_.m.p.syscalltick - _g_.m.p.syscalltick++ + _g_.m.syscalltick = _g_.m.p.ptr().syscalltick + _g_.m.p.ptr().syscalltick++ // Leave SP around for GC and traceback. pc := getcallerpc(unsafe.Pointer(&dummy)) @@ -1807,7 +1802,7 @@ func entersyscallblock(dummy int32) { func entersyscallblock_handoff() { if trace.enabled { traceGoSysCall() - traceGoSysBlock(getg().m.p) + traceGoSysBlock(getg().m.p.ptr()) } handoffp(releasep()) } @@ -1826,18 +1821,18 @@ func exitsyscall(dummy int32) { } _g_.waitsince = 0 - oldp := _g_.m.p + oldp := _g_.m.p.ptr() if exitsyscallfast() { if _g_.m.mcache == nil { throw("lost mcache") } if trace.enabled { - if oldp != _g_.m.p || _g_.m.syscalltick != _g_.m.p.syscalltick { + if oldp != _g_.m.p.ptr() || _g_.m.syscalltick != _g_.m.p.ptr().syscalltick { systemstack(traceGoStart) } } // There's a cpu for us, so we can run. - _g_.m.p.syscalltick++ + _g_.m.p.ptr().syscalltick++ // We need to cas the status and scan before resuming... casgstatus(_g_, _Gsyscall, _Grunning) @@ -1891,7 +1886,7 @@ func exitsyscall(dummy int32) { // we don't know for sure that the garbage collector // is not running. _g_.syscallsp = 0 - _g_.m.p.syscalltick++ + _g_.m.p.ptr().syscalltick++ _g_.throwsplit = false if exitTicks != 0 { @@ -1909,37 +1904,37 @@ func exitsyscallfast() bool { // Freezetheworld sets stopwait but does not retake P's. if sched.stopwait == freezeStopWait { _g_.m.mcache = nil - _g_.m.p = nil + _g_.m.p = 0 return false } // Try to re-acquire the last P. - if _g_.m.p != nil && _g_.m.p.status == _Psyscall && cas(&_g_.m.p.status, _Psyscall, _Prunning) { + if _g_.m.p != 0 && _g_.m.p.ptr().status == _Psyscall && cas(&_g_.m.p.ptr().status, _Psyscall, _Prunning) { // There's a cpu for us, so we can run. - _g_.m.mcache = _g_.m.p.mcache - _g_.m.p.m = _g_.m - if _g_.m.syscalltick != _g_.m.p.syscalltick { + _g_.m.mcache = _g_.m.p.ptr().mcache + _g_.m.p.ptr().m.set(_g_.m) + if _g_.m.syscalltick != _g_.m.p.ptr().syscalltick { if trace.enabled { // The p was retaken and then enter into syscall again (since _g_.m.syscalltick has changed). // traceGoSysBlock for this syscall was already emitted, // but here we effectively retake the p from the new syscall running on the same p. systemstack(func() { // Denote blocking of the new syscall. - traceGoSysBlock(_g_.m.p) + traceGoSysBlock(_g_.m.p.ptr()) // Denote completion of the current syscall. traceGoSysExit(0) }) } - _g_.m.p.syscalltick++ + _g_.m.p.ptr().syscalltick++ } return true } // Try to get any other idle P. - oldp := _g_.m.p + oldp := _g_.m.p.ptr() _g_.m.mcache = nil - _g_.m.p = nil - if sched.pidle != nil { + _g_.m.p = 0 + if sched.pidle != 0 { var ok bool systemstack(func() { ok = exitsyscallfast_pidle() @@ -2101,7 +2096,7 @@ func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr throw("newproc: function arguments too large for new goroutine") } - _p_ := _g_.m.p + _p_ := _g_.m.p.ptr() newg := gfget(_p_) if newg == nil { newg = malg(_StackMin) @@ -2184,7 +2179,7 @@ func gfput(_p_ *p, gp *g) { gp.stackguard0 = 0 } - gp.schedlink = _p_.gfree + gp.schedlink.set(_p_.gfree) _p_.gfree = gp _p_.gfreecnt++ if _p_.gfreecnt >= 64 { @@ -2192,8 +2187,8 @@ func gfput(_p_ *p, gp *g) { for _p_.gfreecnt >= 32 { _p_.gfreecnt-- gp = _p_.gfree - _p_.gfree = gp.schedlink - gp.schedlink = sched.gfree + _p_.gfree = gp.schedlink.ptr() + gp.schedlink.set(sched.gfree) sched.gfree = gp sched.ngfree++ } @@ -2211,16 +2206,16 @@ retry: for _p_.gfreecnt < 32 && sched.gfree != nil { _p_.gfreecnt++ gp = sched.gfree - sched.gfree = gp.schedlink + sched.gfree = gp.schedlink.ptr() sched.ngfree-- - gp.schedlink = _p_.gfree + gp.schedlink.set(_p_.gfree) _p_.gfree = gp } unlock(&sched.gflock) goto retry } if gp != nil { - _p_.gfree = gp.schedlink + _p_.gfree = gp.schedlink.ptr() _p_.gfreecnt-- if gp.stack.lo == 0 { // Stack was deallocated in gfput. Allocate a new one. @@ -2243,8 +2238,8 @@ func gfpurge(_p_ *p) { for _p_.gfreecnt != 0 { _p_.gfreecnt-- gp := _p_.gfree - _p_.gfree = gp.schedlink - gp.schedlink = sched.gfree + _p_.gfree = gp.schedlink.ptr() + gp.schedlink.set(sched.gfree) sched.gfree = gp sched.ngfree++ } @@ -2453,10 +2448,10 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { // This is especially important on windows, since all syscalls are cgo calls. n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0) } - if GOOS == "windows" && n == 0 && mp.libcallg != nil && mp.libcallpc != 0 && mp.libcallsp != 0 { + if GOOS == "windows" && n == 0 && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 { // Libcall, i.e. runtime syscall on windows. // Collect Go stack that leads to the call. - n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg, 0, &stk[0], len(stk), nil, nil, 0) + n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0) } if n == 0 { // If all of the above has failed, account it against abstract "System" or "GC". @@ -2570,7 +2565,7 @@ func procresize(nprocs int32) *p { for i := nprocs; i < old; i++ { p := allp[i] if trace.enabled { - if p == getg().m.p { + if p == getg().m.p.ptr() { // moving to p[0], pretend that we were descheduled // and then scheduled again to keep the trace sane. traceGoSched() @@ -2584,9 +2579,9 @@ func procresize(nprocs int32) *p { gp := p.runq[p.runqtail%uint32(len(p.runq))] // push onto head of global queue gp.schedlink = sched.runqhead - sched.runqhead = gp - if sched.runqtail == nil { - sched.runqtail = gp + sched.runqhead.set(gp) + if sched.runqtail == 0 { + sched.runqtail.set(gp) } sched.runqsize++ } @@ -2609,18 +2604,18 @@ func procresize(nprocs int32) *p { } _g_ := getg() - if _g_.m.p != nil && _g_.m.p.id < nprocs { + if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs { // continue to use the current P - _g_.m.p.status = _Prunning + _g_.m.p.ptr().status = _Prunning } else { // release the current P and acquire allp[0] - if _g_.m.p != nil { - _g_.m.p.m = nil + if _g_.m.p != 0 { + _g_.m.p.ptr().m = 0 } - _g_.m.p = nil + _g_.m.p = 0 _g_.m.mcache = nil p := allp[0] - p.m = nil + p.m = 0 p.status = _Pidle acquirep(p) if trace.enabled { @@ -2630,15 +2625,15 @@ func procresize(nprocs int32) *p { var runnablePs *p for i := nprocs - 1; i >= 0; i-- { p := allp[i] - if _g_.m.p == p { + if _g_.m.p.ptr() == p { continue } p.status = _Pidle if p.runqhead == p.runqtail { pidleput(p) } else { - p.m = mget() - p.link = runnablePs + p.m.set(mget()) + p.link.set(runnablePs) runnablePs = p } } @@ -2648,53 +2643,57 @@ func procresize(nprocs int32) *p { } // Associate p and the current m. +func acquirep(_p_ *p) { + acquirep1(_p_) + + // have p; write barriers now allowed + _g_ := getg() + _g_.m.mcache = _p_.mcache + + if trace.enabled { + traceProcStart() + } +} + // May run during STW, so write barriers are not allowed. //go:nowritebarrier -func acquirep(_p_ *p) { +func acquirep1(_p_ *p) { _g_ := getg() - if _g_.m.p != nil || _g_.m.mcache != nil { + if _g_.m.p != 0 || _g_.m.mcache != nil { throw("acquirep: already in go") } - if _p_.m != nil || _p_.status != _Pidle { + if _p_.m != 0 || _p_.status != _Pidle { id := int32(0) - if _p_.m != nil { - id = _p_.m.id + if _p_.m != 0 { + id = _p_.m.ptr().id } print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n") throw("acquirep: invalid p state") } - // _p_.mcache holds the mcache and _p_ is in allp, so WB can be eliminated - setMcacheNoWriteBarrier(&_g_.m.mcache, _p_.mcache) - // _p_ is in allp so WB can be eliminated - setPNoWriteBarrier(&_g_.m.p, _p_) - // m is in _g_.m and is reachable through allg, so WB can be eliminated - setMNoWriteBarrier(&_p_.m, _g_.m) + _g_.m.p.set(_p_) + _p_.m.set(_g_.m) _p_.status = _Prunning - - if trace.enabled { - traceProcStart() - } } // Disassociate p and the current m. func releasep() *p { _g_ := getg() - if _g_.m.p == nil || _g_.m.mcache == nil { + if _g_.m.p == 0 || _g_.m.mcache == nil { throw("releasep: invalid arg") } - _p_ := _g_.m.p - if _p_.m != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning { - print("releasep: m=", _g_.m, " m->p=", _g_.m.p, " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n") + _p_ := _g_.m.p.ptr() + if _p_.m.ptr() != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning { + print("releasep: m=", _g_.m, " m->p=", _g_.m.p.ptr(), " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n") throw("releasep: invalid p state") } if trace.enabled { - traceProcStop(_g_.m.p) + traceProcStop(_g_.m.p.ptr()) } - _g_.m.p = nil + _g_.m.p = 0 _g_.m.mcache = nil - _p_.m = nil + _p_.m = 0 _p_.status = _Pidle return _p_ } @@ -2773,7 +2772,7 @@ func checkdead() { if mp == nil { newm(nil, _p_) } else { - mp.nextp = _p_ + mp.nextp.set(_p_) notewakeup(&mp.park) } return @@ -2865,7 +2864,7 @@ func sysmon() { if lastgc != 0 && unixnow-lastgc > forcegcperiod && atomicload(&forcegc.idle) != 0 { lock(&forcegc.lock) forcegc.idle = 0 - forcegc.g.schedlink = nil + forcegc.g.schedlink = 0 injectglist(forcegc.g) unlock(&forcegc.lock) } @@ -2978,7 +2977,7 @@ func preemptall() bool { // and will be indicated by the gp->status no longer being // Grunning func preemptone(_p_ *p) bool { - mp := _p_.m + mp := _p_.m.ptr() if mp == nil || mp == getg().m { return false } @@ -3018,7 +3017,7 @@ func schedtrace(detailed bool) { if _p_ == nil { continue } - mp := _p_.m + mp := _p_.m.ptr() h := atomicload(&_p_.runqhead) t := atomicload(&_p_.runqtail) if detailed { @@ -3047,7 +3046,7 @@ func schedtrace(detailed bool) { } for mp := allm; mp != nil; mp = mp.alllink { - _p_ := mp.p + _p_ := mp.p.ptr() gp := mp.curg lockedg := mp.lockedg id1 := int32(-1) @@ -3089,10 +3088,8 @@ func schedtrace(detailed bool) { // May run during STW, so write barriers are not allowed. //go:nowritebarrier func mput(mp *m) { - // sched.midle is reachable via allm, so WB can be eliminated. - setMNoWriteBarrier(&mp.schedlink, sched.midle) - // mp is reachable via allm, so WB can be eliminated. - setMNoWriteBarrier(&sched.midle, mp) + mp.schedlink = sched.midle + sched.midle.set(mp) sched.nmidle++ checkdead() } @@ -3102,10 +3099,9 @@ func mput(mp *m) { // May run during STW, so write barriers are not allowed. //go:nowritebarrier func mget() *m { - mp := sched.midle + mp := sched.midle.ptr() if mp != nil { - // mp.schedlink is reachable via mp, which is on allm, so WB can be eliminated. - setMNoWriteBarrier(&sched.midle, mp.schedlink) + sched.midle = mp.schedlink sched.nmidle-- } return mp @@ -3116,27 +3112,26 @@ func mget() *m { // May run during STW, so write barriers are not allowed. //go:nowritebarrier func globrunqput(gp *g) { - gp.schedlink = nil - if sched.runqtail != nil { - // gp is on allg, so these three WBs can be eliminated. - setGNoWriteBarrier(&sched.runqtail.schedlink, gp) + gp.schedlink = 0 + if sched.runqtail != 0 { + sched.runqtail.ptr().schedlink.set(gp) } else { - setGNoWriteBarrier(&sched.runqhead, gp) + sched.runqhead.set(gp) } - setGNoWriteBarrier(&sched.runqtail, gp) + sched.runqtail.set(gp) sched.runqsize++ } // Put a batch of runnable goroutines on the global runnable queue. // Sched must be locked. func globrunqputbatch(ghead *g, gtail *g, n int32) { - gtail.schedlink = nil - if sched.runqtail != nil { - sched.runqtail.schedlink = ghead + gtail.schedlink = 0 + if sched.runqtail != 0 { + sched.runqtail.ptr().schedlink.set(ghead) } else { - sched.runqhead = ghead + sched.runqhead.set(ghead) } - sched.runqtail = gtail + sched.runqtail.set(gtail) sched.runqsize += n } @@ -3160,14 +3155,14 @@ func globrunqget(_p_ *p, max int32) *g { sched.runqsize -= n if sched.runqsize == 0 { - sched.runqtail = nil + sched.runqtail = 0 } - gp := sched.runqhead + gp := sched.runqhead.ptr() sched.runqhead = gp.schedlink n-- for ; n > 0; n-- { - gp1 := sched.runqhead + gp1 := sched.runqhead.ptr() sched.runqhead = gp1.schedlink runqput(_p_, gp1) } @@ -3179,9 +3174,8 @@ func globrunqget(_p_ *p, max int32) *g { // May run during STW, so write barriers are not allowed. //go:nowritebarrier func pidleput(_p_ *p) { - // sched.pidle, _p_.link and _p_ are reachable via allp, so WB can be eliminated. - setPNoWriteBarrier(&_p_.link, sched.pidle) - setPNoWriteBarrier(&sched.pidle, _p_) + _p_.link = sched.pidle + sched.pidle.set(_p_) xadd(&sched.npidle, 1) // TODO: fast atomic } @@ -3190,10 +3184,9 @@ func pidleput(_p_ *p) { // May run during STW, so write barriers are not allowed. //go:nowritebarrier func pidleget() *p { - _p_ := sched.pidle + _p_ := sched.pidle.ptr() if _p_ != nil { - // _p_.link is reachable via a _p_ in allp, so WB can be eliminated. - setPNoWriteBarrier(&sched.pidle, _p_.link) + sched.pidle = _p_.link xadd(&sched.npidle, -1) // TODO: fast atomic } return _p_ @@ -3239,7 +3232,7 @@ func runqputslow(_p_ *p, gp *g, h, t uint32) bool { // Link the goroutines. for i := uint32(0); i < n; i++ { - batch[i].schedlink = batch[i+1] + batch[i].schedlink.set(batch[i+1]) } // Now put the batch on global queue. @@ -3413,7 +3406,7 @@ func procPin() int { mp := _g_.m mp.locks++ - return int(mp.p.id) + return int(mp.p.ptr().id) } //go:nosplit @@ -3458,7 +3451,7 @@ func sync_runtime_canSpin(i int) bool { if i >= active_spin || ncpu <= 1 || gomaxprocs <= int32(sched.npidle+sched.nmspinning)+1 { return false } - if p := getg().m.p; p.runqhead != p.runqtail { + if p := getg().m.p.ptr(); p.runqhead != p.runqtail { return false } return true diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 3230d4e1a8..a59d77df85 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -87,8 +87,28 @@ type eface struct { data unsafe.Pointer } +// The guintptr, muintptr, and puintptr are all used to bypass write barriers. +// It is particularly important to avoid write barriers when the current P has +// been released, because the GC thinks the world is stopped, and an +// unexpected write barrier would not be synchronized with the GC, +// which can lead to a half-executed write barrier that has marked the object +// but not queued it. If the GC skips the object and completes before the +// queuing can occur, it will incorrectly free the object. +// +// We tried using special assignment functions invoked only when not +// holding a running P, but then some updates to a particular memory +// word went through write barriers and some did not. This breaks the +// write barrier shadow checking mode, and it is also scary: better to have +// a word that is completely ignored by the GC than to have one for which +// only a few updates are ignored. +// +// Gs, Ms, and Ps are always reachable via true pointers in the +// allgs, allm, and allp lists or (during allocation before they reach those lists) +// from stack variables. + // A guintptr holds a goroutine pointer, but typed as a uintptr -// to bypass write barriers. It is used in the Gobuf goroutine state. +// to bypass write barriers. It is used in the Gobuf goroutine state +// and in scheduling lists that are manipulated without a P. // // The Gobuf.g goroutine pointer is almost always updated by assembly code. // In one of the few places it is updated by Go code - func save - it must be @@ -107,41 +127,18 @@ type eface struct { // alternate arena. Using guintptr doesn't make that problem any worse. type guintptr uintptr -func (gp guintptr) ptr() *g { - return (*g)(unsafe.Pointer(gp)) -} +func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) } +func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) } -// ps, ms, gs, and mcache are structures that must be manipulated at a level -// lower than that of the normal Go language. For example the routine that -// stops the world removes the p from the m structure informing the GC that -// this P is stopped and then it moves the g to the global runnable queue. -// If write barriers were allowed to happen at this point not only does -// the GC think the thread is stopped but the underlying structures -// like a p or m are not in a state that is not coherent enough to -// support the write barrier actions. -// This is particularly painful since a partially executed write barrier -// may mark the object but be delinquent in informing the GC that the -// object needs to be scanned. - -// setGNoWriteBarriers does *gdst = gval without a write barrier. -func setGNoWriteBarrier(gdst **g, gval *g) { - *(*uintptr)(unsafe.Pointer(gdst)) = uintptr(unsafe.Pointer(gval)) -} +type puintptr uintptr -// setMNoWriteBarriers does *mdst = mval without a write barrier. -func setMNoWriteBarrier(mdst **m, mval *m) { - *(*uintptr)(unsafe.Pointer(mdst)) = uintptr(unsafe.Pointer(mval)) -} +func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) } +func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) } -// setPNoWriteBarriers does *pdst = pval without a write barrier. -func setPNoWriteBarrier(pdst **p, pval *p) { - *(*uintptr)(unsafe.Pointer(pdst)) = uintptr(unsafe.Pointer(pval)) -} +type muintptr uintptr -// setMcacheNoWriteBarriers does *mcachedst = mcacheval without a write barrier. -func setMcacheNoWriteBarrier(mcachedst **mcache, mcacheval *mcache) { - *(*uintptr)(unsafe.Pointer(mcachedst)) = uintptr(unsafe.Pointer(mcacheval)) -} +func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) } +func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) } type gobuf struct { // The offsets of sp, pc, and g are known to (hard-coded in) libmach. @@ -224,7 +221,7 @@ type g struct { goid int64 waitsince int64 // approx time when the g become blocked waitreason string // if status==gwaiting - schedlink *g + schedlink guintptr preempt bool // preemption signal, duplicates stackguard0 = stackpreempt paniconfault bool // panic (instead of crash) on unexpected fault address preemptscan bool // preempted g does scan for gc @@ -263,11 +260,11 @@ type m struct { procid uint64 // for debuggers, but offset not hard-coded gsignal *g // signal-handling g tls [4]uintptr // thread-local storage (for x86 extern register) - mstartfn uintptr // TODO: type as func(); note: this is a non-heap allocated func() - curg *g // current running goroutine - caughtsig *g // goroutine running during fatal signal - p *p // attached p for executing go code (nil if not executing go code) - nextp *p + mstartfn func() + curg *g // current running goroutine + caughtsig guintptr // goroutine running during fatal signal + p puintptr // attached p for executing go code (nil if not executing go code) + nextp puintptr id int32 mallocing int32 throwing int32 @@ -286,7 +283,7 @@ type m struct { ncgo int32 // number of cgo calls currently in progress park note alllink *m // on allm - schedlink *m + schedlink muintptr machport uint32 // return address for mach ipc (os x) mcache *mcache lockedg *g @@ -315,7 +312,7 @@ type m struct { libcall libcall libcallpc uintptr // for cpu profiler libcallsp uintptr - libcallg *g + libcallg guintptr //#endif //#ifdef GOOS_solaris perrno *int32 // pointer to tls errno @@ -336,10 +333,10 @@ type p struct { id int32 status uint32 // one of pidle/prunning/... - link *p - schedtick uint32 // incremented on every scheduler call - syscalltick uint32 // incremented on every system call - m *m // back-link to associated m (nil if idle) + link puintptr + schedtick uint32 // incremented on every scheduler call + syscalltick uint32 // incremented on every system call + m muintptr // back-link to associated m (nil if idle) mcache *mcache deferpool [5][]*_defer // pool of available defer structs of different sizes (see panic.go) @@ -379,19 +376,19 @@ type schedt struct { goidgen uint64 - midle *m // idle m's waiting for work - nmidle int32 // number of idle m's waiting for work - nmidlelocked int32 // number of locked m's waiting for work - mcount int32 // number of m's that have been created - maxmcount int32 // maximum number of m's allowed (or die) + midle muintptr // idle m's waiting for work + nmidle int32 // number of idle m's waiting for work + nmidlelocked int32 // number of locked m's waiting for work + mcount int32 // number of m's that have been created + maxmcount int32 // maximum number of m's allowed (or die) - pidle *p // idle p's + pidle puintptr // idle p's npidle uint32 nmspinning uint32 // Global runnable queue. - runqhead *g - runqtail *g + runqhead guintptr + runqtail guintptr runqsize int32 // Global cache of dead G's. diff --git a/src/runtime/signal_386.go b/src/runtime/signal_386.go index b63299511f..f3c36cb07a 100644 --- a/src/runtime/signal_386.go +++ b/src/runtime/signal_386.go @@ -100,7 +100,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { } _g_.m.throwing = 1 - setGNoWriteBarrier(&_g_.m.caughtsig, gp) + _g_.m.caughtsig.set(gp) startpanic() if sig < uint32(len(sigtable)) { diff --git a/src/runtime/signal_amd64x.go b/src/runtime/signal_amd64x.go index 5dc9d8070a..182b16e5ec 100644 --- a/src/runtime/signal_amd64x.go +++ b/src/runtime/signal_amd64x.go @@ -136,7 +136,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { } _g_.m.throwing = 1 - setGNoWriteBarrier(&_g_.m.caughtsig, gp) + _g_.m.caughtsig.set(gp) if crashing == 0 { startpanic() diff --git a/src/runtime/signal_arm.go b/src/runtime/signal_arm.go index 7d417fa2b3..38d7181b2a 100644 --- a/src/runtime/signal_arm.go +++ b/src/runtime/signal_arm.go @@ -95,7 +95,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { } _g_.m.throwing = 1 - setGNoWriteBarrier(&_g_.m.caughtsig, gp) + _g_.m.caughtsig.set(gp) startpanic() if sig < uint32(len(sigtable)) { diff --git a/src/runtime/signal_arm64.go b/src/runtime/signal_arm64.go index d1904de215..dde3c7c43f 100644 --- a/src/runtime/signal_arm64.go +++ b/src/runtime/signal_arm64.go @@ -108,7 +108,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { } _g_.m.throwing = 1 - setGNoWriteBarrier(&_g_.m.caughtsig, gp) + _g_.m.caughtsig.set(gp) startpanic() if sig < uint32(len(sigtable)) { diff --git a/src/runtime/signal_ppc64x.go b/src/runtime/signal_ppc64x.go index 018d7d62ef..04d8cfcec1 100644 --- a/src/runtime/signal_ppc64x.go +++ b/src/runtime/signal_ppc64x.go @@ -113,7 +113,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { } _g_.m.throwing = 1 - setGNoWriteBarrier(&_g_.m.caughtsig, gp) + _g_.m.caughtsig.set(gp) startpanic() if sig < uint32(len(sigtable)) { diff --git a/src/runtime/stack1.go b/src/runtime/stack1.go index 5f28d28757..db7e3cbeca 100644 --- a/src/runtime/stack1.go +++ b/src/runtime/stack1.go @@ -680,7 +680,7 @@ func newstack() { // it needs a lock held by the goroutine), that small preemption turns // into a real deadlock. if preempt { - if thisg.m.locks != 0 || thisg.m.mallocing != 0 || thisg.m.preemptoff != "" || thisg.m.p.status != _Prunning { + if thisg.m.locks != 0 || thisg.m.mallocing != 0 || thisg.m.preemptoff != "" || thisg.m.p.ptr().status != _Prunning { // Let the goroutine keep running for now. // gp->preempt is set, so it will be preempted next time. gp.stackguard0 = gp.stack.lo + _StackGuard @@ -724,7 +724,7 @@ func newstack() { if gp == thisg.m.g0 { throw("runtime: preempt g0") } - if thisg.m.p == nil && thisg.m.locks == 0 { + if thisg.m.p == 0 && thisg.m.locks == 0 { throw("runtime: g is running but p is not") } if gp.preemptscan { diff --git a/src/runtime/trace.go b/src/runtime/trace.go index a149799527..e0eb7d82ce 100644 --- a/src/runtime/trace.go +++ b/src/runtime/trace.go @@ -506,7 +506,7 @@ func traceEvent(ev byte, skip int, args ...uint64) { // traceAcquireBuffer returns trace buffer to use and, if necessary, locks it. func traceAcquireBuffer() (mp *m, pid int32, bufp **traceBuf) { mp = acquirem() - if p := mp.p; p != nil { + if p := mp.p.ptr(); p != nil { return mp, p.id, &p.tracebuf } lock(&trace.bufLock) @@ -732,7 +732,7 @@ func traceProcStop(pp *p) { // to handle this we temporary employ the P. mp := acquirem() oldp := mp.p - mp.p = pp + mp.p.set(pp) traceEvent(traceEvProcStop, -1) mp.p = oldp releasem(mp) @@ -806,7 +806,7 @@ func traceGoSysBlock(pp *p) { // to handle this we temporary employ the P. mp := acquirem() oldp := mp.p - mp.p = pp + mp.p.set(pp) traceEvent(traceEvGoSysBlock, -1) mp.p = oldp releasem(mp) diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index b6f43747d6..91541fabce 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -528,7 +528,7 @@ func gcallers(gp *g, skip int, pcbuf []uintptr) int { func showframe(f *_func, gp *g) bool { g := getg() - if g.m.throwing > 0 && gp != nil && (gp == g.m.curg || gp == g.m.caughtsig) { + if g.m.throwing > 0 && gp != nil && (gp == g.m.curg || gp == g.m.caughtsig.ptr()) { return true } traceback := gotraceback(nil) -- 2.48.1