return triggerratio*(int64(memstats.next_gc)-int64(memstats.heap_alloc)) <= int64(memstats.next_gc) && atomicloaduint(&bggc.working) == 0
}
-var work workdata
-
-type workdata struct {
+var work struct {
full uint64 // lock-free list of full blocks workbuf
empty uint64 // lock-free list of empty blocks workbuf
partial uint64 // lock-free list of partially filled blocks workbuf
// GC runs a garbage collection.
func GC() {
- gogc(2)
+ startGC(gcForceBlockMode)
}
-// force = 0 - start concurrent GC
-// force = 1 - do STW GC regardless of current heap usage
-// force = 2 - go STW GC and eager sweep
-func gogc(force int32) {
+const (
+ gcBackgroundMode = iota // concurrent GC
+ gcForceMode // stop-the-world GC now
+ gcForceBlockMode // stop-the-world GC now and wait for sweep
+)
+
+func startGC(mode int) {
// The gc is turned off (via enablegc) until the bootstrap has completed.
// Also, malloc gets called in the guts of a number of libraries that might be
// holding locks. To avoid deadlocks during stoptheworld, don't bother
// trying to run gc while holding a lock. The next mallocgc without a lock
// will do the gc instead.
-
mp := acquirem()
if gp := getg(); gp == mp.g0 || mp.locks > 1 || !memstats.enablegc || panicking != 0 || gcpercent < 0 {
releasem(mp)
releasem(mp)
mp = nil
- if force == 0 {
- lock(&bggc.lock)
- if !bggc.started {
- bggc.working = 1
- bggc.started = true
- go backgroundgc()
- } else if bggc.working == 0 {
- bggc.working = 1
- ready(bggc.g)
- }
- unlock(&bggc.lock)
- } else {
- gcwork(force)
+ if mode != gcBackgroundMode {
+ // special synchronous cases
+ gc(mode)
+ return
}
+
+ // trigger concurrent GC
+ lock(&bggc.lock)
+ if !bggc.started {
+ bggc.working = 1
+ bggc.started = true
+ go backgroundgc()
+ } else if bggc.working == 0 {
+ bggc.working = 1
+ ready(bggc.g)
+ }
+ unlock(&bggc.lock)
}
// State of the background concurrent GC goroutine.
func backgroundgc() {
bggc.g = getg()
for {
- gcwork(0)
+ gc(gcBackgroundMode)
lock(&bggc.lock)
bggc.working = 0
goparkunlock(&bggc.lock, "Concurrent GC wait", traceEvGoBlock)
}
}
-func gcwork(force int32) {
-
+func gc(mode int) {
+ // Ok, we're doing it! Stop everybody else
semacquire(&worldsema, false)
// Pick up the remaining unswept/not being swept spans concurrently
sweep.nbgsweep++
}
- // Ok, we're doing it! Stop everybody else
-
mp := acquirem()
mp.preemptoff = "gcing"
releasem(mp)
gctimer.count++
- if force == 0 {
+ if mode == gcBackgroundMode {
gctimer.cycle.sweepterm = nanotime()
}
traceGCStart()
}
- // Pick up the remaining unswept/not being swept spans before we STW
- for gosweepone() != ^uintptr(0) {
- sweep.nbgsweep++
- }
systemstack(stoptheworld)
systemstack(finishsweep_m) // finish sweep before we start concurrent scan.
- if force == 0 { // Do as much work concurrently as possible
- gcphase = _GCscan
- systemstack(starttheworld)
- gctimer.cycle.scan = nanotime()
- // Do a concurrent heap scan before we stop the world.
- systemstack(gcscan_m)
- gctimer.cycle.installmarkwb = nanotime()
- systemstack(stoptheworld)
- systemstack(gcinstallmarkwb)
- systemstack(harvestwbufs)
- systemstack(starttheworld)
- gctimer.cycle.mark = nanotime()
- systemstack(gcmark_m)
- gctimer.cycle.markterm = nanotime()
- systemstack(stoptheworld)
- systemstack(gcinstalloffwb_m)
+
+ if mode == gcBackgroundMode { // Do as much work concurrently as possible
+ systemstack(func() {
+ gcphase = _GCscan
+
+ // Concurrent scan.
+ starttheworld()
+ gctimer.cycle.scan = nanotime()
+ gcscan_m()
+ gctimer.cycle.installmarkwb = nanotime()
+
+ // Sync.
+ stoptheworld()
+ gcphase = _GCmark
+ harvestwbufs()
+
+ // Concurrent mark.
+ starttheworld()
+ gctimer.cycle.mark = nanotime()
+ var gcw gcWork
+ gcDrain(&gcw)
+ gcw.dispose()
+
+ // Begin mark termination.
+ gctimer.cycle.markterm = nanotime()
+ stoptheworld()
+ gcphase = _GCoff
+ })
} else {
- // For non-concurrent GC (force != 0) g stack have not been scanned so
- // set gcscanvalid such that mark termination scans all stacks.
+ // For non-concurrent GC (mode != gcBackgroundMode)
+ // g stack have not been scanned so set gcscanvalid
+ // such that mark termination scans all stacks.
// No races here since we are in a STW phase.
for _, gp := range allgs {
gp.gcworkdone = false // set to true in gcphasework
startTime := nanotime()
if mp != acquirem() {
- throw("gogc: rescheduled")
+ throw("gcwork: rescheduled")
}
+ // TODO(rsc): Should the concurrent GC clear pools earlier?
clearpools()
// Run gc on the g0 stack. We do this so that the g stack
if debug.gctrace > 1 {
n = 2
}
- eagersweep := force >= 2
for i := 0; i < n; i++ {
if i > 0 {
// refresh start time if doing a second GC
}
// switch to g0, call gc, then switch back
systemstack(func() {
- gc_m(startTime, eagersweep)
+ gc_m(startTime, mode == gcForceBlockMode)
})
}
systemstack(func() {
- gccheckmark_m(startTime, eagersweep)
+ // Called from malloc.go using systemstack.
+ // The world is stopped. Rerun the scan and mark phases
+ // using the bitMarkedCheck bit instead of the
+ // bitMarked bit. If the marking encounters an
+ // bitMarked bit that is not set then we throw.
+ //go:nowritebarrier
+ if debug.gccheckmark == 0 {
+ return
+ }
+
+ if checkmarkphase {
+ throw("gccheckmark_m, entered with checkmarkphase already true")
+ }
+
+ checkmarkphase = true
+ initCheckmarks()
+ gc_m(startTime, mode == gcForceBlockMode) // turns off checkmarkphase + calls clearcheckmarkbits
})
if trace.enabled {
// all done
mp.preemptoff = ""
- if force == 0 {
+ if mode == gcBackgroundMode {
gctimer.cycle.sweep = nanotime()
}
semrelease(&worldsema)
- if force == 0 {
+ if mode == gcBackgroundMode {
if gctimer.verbose > 1 {
GCprinttimes()
} else if gctimer.verbose > 0 {
}
}
-// For now this must be bracketed with a stoptheworld and a starttheworld to ensure
-// all go routines see the new barrier.
-//go:nowritebarrier
-func gcinstalloffwb_m() {
- gcphase = _GCoff
-}
-
-// For now this must be bracketed with a stoptheworld and a starttheworld to ensure
-// all go routines see the new barrier.
-//go:nowritebarrier
-func gcinstallmarkwb() {
- gcphase = _GCmark
-}
-
-// Mark all objects that are known about.
-// This is the concurrent mark phase.
-//go:nowritebarrier
-func gcmark_m() {
- var gcw gcWork
- gcDrain(&gcw)
- gcw.dispose()
- // TODO add another harvestwbuf and reset work.nwait=0, work.ndone=0, and work.nproc=1
- // and repeat the above gcDrain.
-}
-
-// Called from malloc.go using systemstack.
-// The world is stopped. Rerun the scan and mark phases
-// using the bitMarkedCheck bit instead of the
-// bitMarked bit. If the marking encounters an
-// bitMarked bit that is not set then we throw.
-//go:nowritebarrier
-func gccheckmark_m(startTime int64, eagersweep bool) {
- if debug.gccheckmark == 0 {
- return
- }
-
- if checkmarkphase {
- throw("gccheckmark_m, entered with checkmarkphase already true")
+// STW is in effect at this point.
+//TODO go:nowritebarrier
+func gc_m(start_time int64, eagersweep bool) {
+ if _DebugGCPtrs {
+ print("GC start\n")
}
- checkmarkphase = true
- initCheckmarks()
- gc_m(startTime, eagersweep) // turns off checkmarkphase + calls clearcheckmarkbits
-}
-
-// Called from malloc.go using systemstack, stopping and starting the world handled in caller.
-//go:nowritebarrier
-func gc_m(start_time int64, eagersweep bool) {
_g_ := getg()
gp := _g_.m.curg
casgstatus(gp, _Grunning, _Gwaiting)
gp.waitreason = "garbage collection"
- gc(start_time, eagersweep)
- casgstatus(gp, _Gwaiting, _Grunning)
-}
-
-// STW is in effect at this point.
-//TODO go:nowritebarrier
-func gc(start_time int64, eagersweep bool) {
- if _DebugGCPtrs {
- print("GC start\n")
- }
-
gcphase = _GCmarktermination
if debug.allocfreetrace > 0 {
tracegc()
}
- _g_ := getg()
_g_.m.traceback = 2
t0 := start_time
work.tstart = start_time
if debug.gccheckmark > 0 {
if !checkmarkphase {
// first half of two-pass; don't set up sweep
+ casgstatus(gp, _Gwaiting, _Grunning)
return
}
checkmarkphase = false // done checking marks
if _DebugGCPtrs {
print("GC end\n")
}
+
+ casgstatus(gp, _Gwaiting, _Grunning)
}
// Hooks for other packages
-//go:linkname runtime_debug_freeOSMemory runtime/debug.freeOSMemory
-func runtime_debug_freeOSMemory() {
- gogc(2) // force GC and do eager sweep
- systemstack(scavenge_m)
-}
-
var poolcleanup func()
//go:linkname sync_runtime_registerPoolCleanup sync.runtime_registerPoolCleanup