nFlushCacheRoots int
nDataRoots, nBSSRoots, nSpanRoots, nStackRoots int
- // markrootDone indicates that roots have been marked at least
- // once during the current GC cycle. This is checked by root
- // marking operations that have to happen only during the
- // first root marking pass, whether that's during the
- // concurrent mark phase in current GC or mark termination in
- // STW GC.
- markrootDone bool
-
// Each type of GC state transition is protected by a lock.
// Since multiple threads can simultaneously detect the state
// transition condition, any thread that detects a transition
// below. The important thing is that the wb remains active until
// all marking is complete. This includes writes made by the GC.
- // Record that one root marking pass has completed.
- work.markrootDone = true
-
// Disable assists and background workers. We must do
// this before waking blocked assists.
atomic.Store(&gcBlackenEnabled, 0)
}
work.tstart = start_time
- // Queue root marking jobs.
- gcMarkRootPrepare()
-
work.nwait = 0
work.ndone = 0
work.nproc = uint32(gcprocs())
// Check that there's no marking work remaining.
- if work.full != 0 || work.nDataRoots+work.nBSSRoots+work.nSpanRoots+work.nStackRoots != 0 {
- print("runtime: full=", hex(work.full), " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n")
+ if work.full != 0 || work.markrootNext < work.markrootJobs {
+ print("runtime: full=", hex(work.full), " next=", work.markrootNext, " jobs=", work.markrootJobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n")
panic("non-empty mark queue after concurrent mark")
}
+ // Clear root marking queue.
+ work.markrootNext = 0
+ work.markrootJobs = 0
+
if work.nproc > 1 {
noteclear(&work.alldone)
helpgc(int32(work.nproc))
notesleep(&work.alldone)
}
- // Record that at least one root marking pass has completed.
- work.markrootDone = true
-
// Clear out buffers and double-check that all gcWork caches
// are empty. This should be ensured by gcMarkDone before we
// enter mark termination.
work.bytesMarked = 0
work.initialHeapLive = atomic.Load64(&memstats.heap_live)
- work.markrootDone = false
}
// Hooks for other packages
work.nDataRoots = 0
work.nBSSRoots = 0
- // Only scan globals once per cycle; preferably concurrently.
- if !work.markrootDone {
- for _, datap := range activeModules() {
- nDataRoots := nBlocks(datap.edata - datap.data)
- if nDataRoots > work.nDataRoots {
- work.nDataRoots = nDataRoots
- }
+ // Scan globals.
+ for _, datap := range activeModules() {
+ nDataRoots := nBlocks(datap.edata - datap.data)
+ if nDataRoots > work.nDataRoots {
+ work.nDataRoots = nDataRoots
}
+ }
- for _, datap := range activeModules() {
- nBSSRoots := nBlocks(datap.ebss - datap.bss)
- if nBSSRoots > work.nBSSRoots {
- work.nBSSRoots = nBSSRoots
- }
+ for _, datap := range activeModules() {
+ nBSSRoots := nBlocks(datap.ebss - datap.bss)
+ if nBSSRoots > work.nBSSRoots {
+ work.nBSSRoots = nBSSRoots
}
}
- if !work.markrootDone {
- // On the first markroot, we need to scan span roots.
- // In concurrent GC, this happens during concurrent
- // mark and we depend on addfinalizer to ensure the
- // above invariants for objects that get finalizers
- // after concurrent mark. In STW GC, this will happen
- // during mark termination.
- //
- // We're only interested in scanning the in-use spans,
- // which will all be swept at this point. More spans
- // may be added to this list during concurrent GC, but
- // we only care about spans that were allocated before
- // this mark phase.
- work.nSpanRoots = mheap_.sweepSpans[mheap_.sweepgen/2%2].numBlocks()
-
- // On the first markroot, we need to scan all Gs. Gs
- // may be created after this point, but it's okay that
- // we ignore them because they begin life without any
- // roots, so there's nothing to scan, and any roots
- // they create during the concurrent phase will be
- // scanned during mark termination. During mark
- // termination, allglen isn't changing, so we'll scan
- // all Gs.
- work.nStackRoots = int(atomic.Loaduintptr(&allglen))
- } else {
- // We've already scanned span roots and kept the scan
- // up-to-date during concurrent mark.
- work.nSpanRoots = 0
-
- // The hybrid barrier ensures that stacks can't
- // contain pointers to unmarked objects, so on the
- // second markroot, there's no need to scan stacks.
- work.nStackRoots = 0
- }
+ // Scan span roots for finalizer specials.
+ //
+ // We depend on addfinalizer to mark objects that get
+ // finalizers after root marking.
+ //
+ // We're only interested in scanning the in-use spans,
+ // which will all be swept at this point. More spans
+ // may be added to this list during concurrent GC, but
+ // we only care about spans that were allocated before
+ // this mark phase.
+ work.nSpanRoots = mheap_.sweepSpans[mheap_.sweepgen/2%2].numBlocks()
+
+ // Scan stacks.
+ //
+ // Gs may be created after this point, but it's okay that we
+ // ignore them because they begin life without any roots, so
+ // there's nothing to scan, and any roots they create during
+ // the concurrent phase will be scanned during mark
+ // termination.
+ work.nStackRoots = int(atomic.Loaduintptr(&allglen))
work.markrootNext = 0
work.markrootJobs = uint32(fixedRootCount + work.nFlushCacheRoots + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots)
}
case i == fixedRootFinalizers:
- // Only do this once per GC cycle since we don't call
- // queuefinalizer during marking.
- if work.markrootDone {
- break
- }
for fb := allfin; fb != nil; fb = fb.alllink {
cnt := uintptr(atomic.Load(&fb.cnt))
scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw)
}
case i == fixedRootFreeGStacks:
- // Only do this once per GC cycle; preferably
- // concurrently.
- if !work.markrootDone {
- // Switch to the system stack so we can call
- // stackfree.
- systemstack(markrootFreeGStacks)
- }
+ // Switch to the system stack so we can call
+ // stackfree.
+ systemstack(markrootFreeGStacks)
case baseSpans <= i && i < baseStacks:
// mark MSpan.specials
// TODO(austin): There are several ideas for making this more
// efficient in issue #11485.
- if work.markrootDone {
- throw("markrootSpans during second markroot")
- }
-
sg := mheap_.sweepgen
spans := mheap_.sweepSpans[mheap_.sweepgen/2%2].block(shard)
// Note that work.spans may not include spans that were
throw("can't scan gchelper stack")
}
- // Shrink the stack if not much of it is being used. During
- // concurrent GC, we can do this during concurrent mark.
- if !work.markrootDone {
- shrinkstack(gp)
- }
+ // Shrink the stack if not much of it is being used.
+ shrinkstack(gp)
// Scan the saved context register. This is effectively a live
// register that gets moved back and forth between the