alldone note
// Number of roots of various root types. Set by gcMarkRootPrepare.
- nDataRoots, nBSSRoots, nSpanRoots, nStackRoots int
+ nDataRoots, nBSSRoots, nSpanRoots, nStackRoots, nRescanRoots int
// markrootDone indicates that roots have been marked at least
// once during the current GC cycle. This is checked by root
head, tail guintptr
}
+ // rescan is a list of G's that need to be rescanned during
+ // mark termination. A G adds itself to this list when it
+ // first invalidates its stack scan.
+ rescan struct {
+ lock mutex
+ list []guintptr
+ }
+
// Timing/utilization stats for this cycle.
stwprocs, maxprocs int32
tSweepTerm, tMark, tMarkTerm, tEnd int64 // nanotime() of phase start
func gcResetMarkState() {
// This may be called during a concurrent phase, so make sure
// allgs doesn't change.
+ if !(gcphase == _GCoff || gcphase == _GCmarktermination) {
+ // Accessing gcRescan is unsafe.
+ throw("bad GC phase")
+ }
lock(&allglock)
for _, gp := range allgs {
gp.gcscandone = false // set to true in gcphasework
gp.gcscanvalid = false // stack has not been scanned
+ gp.gcRescan = -1
gp.gcAssistBytes = 0
}
unlock(&allglock)
+ // Clear rescan list.
+ work.rescan.list = work.rescan.list[:0]
+
work.bytesMarked = 0
work.initialHeapLive = memstats.heap_live
work.markrootDone = false
//
// The caller must have call gcCopySpans().
//
+// The world must be stopped.
+//
//go:nowritebarrier
func gcMarkRootPrepare() {
// Compute how many data and BSS root blocks there are.
// after concurrent mark. In STW GC, this will happen
// during mark termination.
work.nSpanRoots = (len(work.spans) + rootBlockSpans - 1) / rootBlockSpans
+
+ // On the first markroot, we need to scan all Gs. Gs
+ // may be created after this point, but it's okay that
+ // we ignore them because they begin life without any
+ // roots, so there's nothing to scan, and any roots
+ // they create during the concurrent phase will be
+ // scanned during mark termination. During mark
+ // termination, allglen isn't changing, so we'll scan
+ // all Gs.
+ work.nStackRoots = int(atomic.Loaduintptr(&allglen))
+ work.nRescanRoots = 0
} else {
// We've already scanned span roots and kept the scan
// up-to-date during concurrent mark.
work.nSpanRoots = 0
- }
- // Snapshot of allglen. During concurrent scan, we just need
- // to be consistent about how many markroot jobs we create and
- // how many Gs we check. Gs may be created after this point,
- // but it's okay that we ignore them because they begin life
- // without any roots, so there's nothing to scan, and any
- // roots they create during the concurrent phase will be
- // scanned during mark termination. During mark termination,
- // allglen isn't changing, so we'll scan all Gs.
- work.nStackRoots = int(atomic.Loaduintptr(&allglen))
+ // On the second pass of markroot, we're just scanning
+ // dirty stacks. It's safe to access rescan since the
+ // world is stopped.
+ work.nStackRoots = 0
+ work.nRescanRoots = len(work.rescan.list)
+ }
work.markrootNext = 0
- work.markrootJobs = uint32(fixedRootCount + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots)
+ work.markrootJobs = uint32(fixedRootCount + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots + work.nRescanRoots)
}
// gcMarkRootCheck checks that all roots have been scanned. It is
}
lock(&allglock)
- // Check that gc work is done.
- for i := 0; i < work.nStackRoots; i++ {
- gp := allgs[i]
- if !gp.gcscandone {
- throw("scan missed a g")
+ // Check that stacks have been scanned.
+ if gcphase == _GCmarktermination {
+ for i := 0; i < len(allgs); i++ {
+ gp := allgs[i]
+ if !(gp.gcscandone && gp.gcscanvalid) && readgstatus(gp) != _Gdead {
+ println("gp", gp, "goid", gp.goid,
+ "status", readgstatus(gp),
+ "gcscandone", gp.gcscandone,
+ "gcscanvalid", gp.gcscanvalid)
+ throw("scan missed a g")
+ }
+ }
+ } else {
+ for i := 0; i < work.nStackRoots; i++ {
+ gp := allgs[i]
+ if !gp.gcscandone {
+ throw("scan missed a g")
+ }
}
}
unlock(&allglock)
//
// Preemption must be disabled (because this uses a gcWork).
//
+// nowritebarrier is only advisory here.
+//
//go:nowritebarrier
func markroot(gcw *gcWork, i uint32) {
+ // TODO(austin): This is a bit ridiculous. Compute and store
+ // the bases in gcMarkRootPrepare instead of the counts.
baseData := uint32(fixedRootCount)
baseBSS := baseData + uint32(work.nDataRoots)
baseSpans := baseBSS + uint32(work.nBSSRoots)
baseStacks := baseSpans + uint32(work.nSpanRoots)
+ baseRescan := baseStacks + uint32(work.nStackRoots)
+ end := baseRescan + uint32(work.nRescanRoots)
// Note: if you add a case here, please also update heapdump.go:dumproots.
switch {
default:
// the rest is scanning goroutine stacks
- if uintptr(i-baseStacks) >= allglen {
+ var gp *g
+ if baseStacks <= i && i < baseRescan {
+ gp = allgs[i-baseStacks]
+ } else if baseRescan <= i && i < end {
+ gp = work.rescan.list[i-baseRescan].ptr()
+ } else {
throw("markroot: bad index")
}
- gp := allgs[i-baseStacks]
// remember when we've first observed the G blocked
// needed only to output in traceback
gp.waitsince = work.tstart
}
- if gcphase != _GCmarktermination && gp.startpc == gcBgMarkWorkerPC {
+ if gcphase != _GCmarktermination && gp.startpc == gcBgMarkWorkerPC && readgstatus(gp) != _Gdead {
// GC background workers may be
// non-preemptible, so we may deadlock if we
// try to scan them during a concurrent phase.
// They also have tiny stacks, so just ignore
// them until mark termination.
gp.gcscandone = true
+ queueRescan(gp)
break
}
gcw.dispose()
}
gcUnlockStackBarriers(gp)
+ if gcphase == _GCmark {
+ // gp may have added itself to the rescan list between
+ // when GC started and now. It's clean now, so remove
+ // it. This isn't safe during mark termination because
+ // mark termination is consuming this list, but it's
+ // also not necessary.
+ dequeueRescan(gp)
+ }
gp.gcscanvalid = true
}
}
}
+// queueRescan adds gp to the stack rescan list and clears
+// gp.gcscanvalid. The caller must own gp and ensure that gp isn't
+// already on the rescan list.
+func queueRescan(gp *g) {
+ if gcphase == _GCoff {
+ gp.gcscanvalid = false
+ return
+ }
+ if gp.gcRescan != -1 {
+ throw("g already on rescan list")
+ }
+
+ lock(&work.rescan.lock)
+ gp.gcscanvalid = false
+
+ // Recheck gcphase under the lock in case there was a phase change.
+ if gcphase == _GCoff {
+ unlock(&work.rescan.lock)
+ return
+ }
+ if len(work.rescan.list) == cap(work.rescan.list) {
+ throw("rescan list overflow")
+ }
+ n := len(work.rescan.list)
+ gp.gcRescan = int32(n)
+ work.rescan.list = work.rescan.list[:n+1]
+ work.rescan.list[n].set(gp)
+ unlock(&work.rescan.lock)
+}
+
+// dequeueRescan removes gp from the stack rescan list, if gp is on
+// the rescan list. The caller must own gp.
+func dequeueRescan(gp *g) {
+ if gp.gcRescan == -1 {
+ return
+ }
+ if gcphase == _GCoff {
+ gp.gcRescan = -1
+ return
+ }
+
+ lock(&work.rescan.lock)
+ if work.rescan.list[gp.gcRescan].ptr() != gp {
+ throw("bad dequeueRescan")
+ }
+ // Careful: gp may itself be the last G on the list.
+ last := work.rescan.list[len(work.rescan.list)-1]
+ work.rescan.list[gp.gcRescan] = last
+ last.ptr().gcRescan = gp.gcRescan
+ gp.gcRescan = -1
+ work.rescan.list = work.rescan.list[:len(work.rescan.list)-1]
+ unlock(&work.rescan.lock)
+}
+
type gcDrainFlags int
const (
lock(&allglock)
allgs = append(allgs, gp)
allglen = uintptr(len(allgs))
+
+ // Grow GC rescan list if necessary.
+ if len(allgs) > cap(work.rescan.list) {
+ lock(&work.rescan.lock)
+ l := work.rescan.list
+ // Let append do the heavy lifting, but keep the
+ // length the same.
+ work.rescan.list = append(l[:cap(l)], 0)[:len(l)]
+ unlock(&work.rescan.lock)
+ }
unlock(&allglock)
}
nextYield = nanotime() + yieldDelay/2
}
}
- if newval == _Grunning {
- gp.gcscanvalid = false
+ if newval == _Grunning && gp.gcscanvalid {
+ // Run queueRescan on the system stack so it has more space.
+ systemstack(func() { queueRescan(gp) })
}
}
gp.syscallpc = gp.sched.pc
gp.syscallsp = gp.sched.sp
gp.stktopsp = gp.sched.sp
+ gp.gcscanvalid = true // fresh G, so no dequeueRescan necessary
+ gp.gcRescan = -1
// malg returns status as Gidle, change to Gsyscall before adding to allg
// where GC will see it.
casgstatus(gp, _Gidle, _Gsyscall)
gp.waitreason = ""
gp.param = nil
+ // Note that gp's stack scan is now "valid" because it has no
+ // stack. We could dequeueRescan, but that takes a lock and
+ // isn't really necessary.
+ gp.gcscanvalid = true
dropg()
if _g_.m.locked&^_LockExternal != 0 {
if newg == nil {
newg = malg(_StackMin)
casgstatus(newg, _Gidle, _Gdead)
+ newg.gcRescan = -1
allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack.
}
if newg.stack.hi == 0 {
if isSystemGoroutine(newg) {
atomic.Xadd(&sched.ngsys, +1)
}
+ // The stack is dirty from the argument frame, so queue it for
+ // scanning. Do this before setting it to runnable so we still
+ // own the G. If we're recycling a G, it may already be on the
+ // rescan list.
+ if newg.gcRescan == -1 {
+ queueRescan(newg)
+ } else {
+ // The recycled G is already on the rescan list. Just
+ // mark the stack dirty.
+ newg.gcscanvalid = false
+ }
casgstatus(newg, _Gdead, _Grunnable)
if _p_.goidcache == _p_.goidcacheend {