markrootNext uint32 // next markroot job
markrootJobs uint32 // number of markroot jobs
- nproc uint32
- tstart int64
- nwait uint32
- ndone uint32
- alldone note
+ nproc uint32
+ tstart int64
+ nwait uint32
+ ndone uint32
// Number of roots of various root types. Set by gcMarkRootPrepare.
nFlushCacheRoots int
}
work.tstart = start_time
- work.nwait = 0
- work.ndone = 0
- work.nproc = uint32(gcprocs())
-
// Check that there's no marking work remaining.
if work.full != 0 || work.markrootNext < work.markrootJobs {
print("runtime: full=", hex(work.full), " next=", work.markrootNext, " jobs=", work.markrootJobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n")
panic("non-empty mark queue after concurrent mark")
}
- // Clear root marking queue.
- work.markrootNext = 0
- work.markrootJobs = 0
-
- if work.nproc > 1 {
- noteclear(&work.alldone)
- helpgc(int32(work.nproc))
- }
-
- gchelperstart()
-
- gcw := &getg().m.p.ptr().gcw
- gcDrain(gcw, 0)
-
if debug.gccheckmark > 0 {
// This is expensive when there's a large number of
// Gs, so only do it if checkmark is also enabled.
throw("work.full != 0")
}
- if work.nproc > 1 {
- notesleep(&work.alldone)
- }
-
// Clear out buffers and double-check that all gcWork caches
// are empty. This should be ensured by gcMarkDone before we
// enter mark termination.
unlock(&sched.deferlock)
}
-// gchelper runs mark termination tasks on Ps other than the P
-// coordinating mark termination.
-//
-// The caller is responsible for ensuring that this has a P to run on,
-// even though it's running during STW. Because of this, it's allowed
-// to have write barriers.
-//
-//go:yeswritebarrierrec
-func gchelper() {
- _g_ := getg()
- _g_.m.traceback = 2
- gchelperstart()
-
- // Parallel mark over GC roots and heap
- if gcphase == _GCmarktermination {
- gcw := &_g_.m.p.ptr().gcw
- gcDrain(gcw, 0)
- }
-
- nproc := atomic.Load(&work.nproc) // work.nproc can change right after we increment work.ndone
- if atomic.Xadd(&work.ndone, +1) == nproc-1 {
- notewakeup(&work.alldone)
- }
- _g_.m.traceback = 0
-}
-
-func gchelperstart() {
- _g_ := getg()
-
- if _g_.m.helpgc < 0 || _g_.m.helpgc >= _MaxGcproc {
- throw("gchelperstart: bad m->helpgc")
- }
- if _g_ != _g_.m.g0 {
- throw("gchelper not running on g0 stack")
- }
-}
-
// Timing
// itoaDiv formats val/(10**dec) into buf.
}
}
-func gcprocs() int32 {
- // Figure out how many CPUs to use during GC.
- // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
- lock(&sched.lock)
- n := gomaxprocs
- if n > ncpu {
- n = ncpu
- }
- if n > _MaxGcproc {
- n = _MaxGcproc
- }
- if n > sched.nmidle+1 { // one M is currently running
- n = sched.nmidle + 1
- }
- unlock(&sched.lock)
- return n
-}
-
-func needaddgcproc() bool {
- lock(&sched.lock)
- n := gomaxprocs
- if n > ncpu {
- n = ncpu
- }
- if n > _MaxGcproc {
- n = _MaxGcproc
- }
- n -= sched.nmidle + 1 // one M is currently running
- unlock(&sched.lock)
- return n > 0
-}
-
-func helpgc(nproc int32) {
- _g_ := getg()
- lock(&sched.lock)
- pos := 0
- for n := int32(1); n < nproc; n++ { // one M is currently running
- if allp[pos].mcache == _g_.m.mcache {
- pos++
- }
- mp := mget()
- if mp == nil {
- throw("gcprocs inconsistency")
- }
- mp.helpgc = n
- mp.p.set(allp[pos])
- mp.mcache = allp[pos].mcache
- pos++
- notewakeup(&mp.park)
- }
- unlock(&sched.lock)
-}
-
// freezeStopWait is a large value that freezetheworld sets
// sched.stopwait to in order to request that all Gs permanently stop.
const freezeStopWait = 0x7fffffff
}
}
-func mhelpgc() {
- _g_ := getg()
- _g_.m.helpgc = -1
-}
-
func startTheWorldWithSema(emitTraceEvent bool) int64 {
_g_ := getg()
list := netpoll(false) // non-blocking
injectglist(&list)
}
- add := needaddgcproc()
lock(&sched.lock)
procs := gomaxprocs
} else {
// Start M to run P. Do not start another M below.
newm(nil, p)
- add = false
}
}
wakep()
}
- if add {
- // If GC could have used another helper proc, start one now,
- // in the hope that it will be available next time.
- // It would have been even better to start it before the collection,
- // but doing so requires allocating memory, so it's tricky to
- // coordinate. This lazy approach works out in practice:
- // we don't mind if the first couple gc rounds don't have quite
- // the maximum number of procs.
- newm(mhelpgc, nil)
- }
_g_.m.locks--
if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
_g_.stackguard0 = stackPreempt
fn()
}
- if _g_.m.helpgc != 0 {
- _g_.m.helpgc = 0
- stopm()
- } else if _g_.m != &m0 {
+ if _g_.m != &m0 {
acquirep(_g_.m.nextp.ptr())
_g_.m.nextp = 0
}
throw("stopm spinning")
}
-retry:
lock(&sched.lock)
mput(_g_.m)
unlock(&sched.lock)
notesleep(&_g_.m.park)
noteclear(&_g_.m.park)
- if _g_.m.helpgc != 0 {
- // helpgc() set _g_.m.p and _g_.m.mcache, so we have a P.
- gchelper()
- // Undo the effects of helpgc().
- _g_.m.helpgc = 0
- _g_.m.mcache = nil
- _g_.m.p = 0
- goto retry
- }
acquirep(_g_.m.nextp.ptr())
_g_.m.nextp = 0
}
pc = funcPC(_ExternalCode) + sys.PCQuantum
}
stk[0] = pc
- if mp.preemptoff != "" || mp.helpgc != 0 {
+ if mp.preemptoff != "" {
stk[1] = funcPC(_GC) + sys.PCQuantum
} else {
stk[1] = funcPC(_System) + sys.PCQuantum
if lockedg != nil {
id3 = lockedg.goid
}
- print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " preemptoff=", mp.preemptoff, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", mp.blocked, " lockedg=", id3, "\n")
+ print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " preemptoff=", mp.preemptoff, ""+" locks=", mp.locks, " dying=", mp.dying, " spinning=", mp.spinning, " blocked=", mp.blocked, " lockedg=", id3, "\n")
}
lock(&allglock)