work.cycles++
- gcController.startCycle()
+ // Assists and workers can start the moment we start
+ // the world.
+ gcController.startCycle(now)
work.heapGoal = gcController.heapGoal
// In STW mode, disable scheduling of user Gs. This may also
// mutators.
atomic.Store(&gcBlackenEnabled, 1)
- // Assists and workers can start the moment we start
- // the world.
- gcController.markStartTime = now
-
// In STW mode, we could block the instant systemstack
// returns, so make sure we're not preemptible.
mp = acquirem()
throw("gc done but gcphase != _GCoff")
}
- // Record heapGoal and heap_inuse for scavenger.
- gcController.lastHeapGoal = gcController.heapGoal
+ // Record heap_inuse for scavenger.
memstats.last_heap_inuse = memstats.heap_inuse
// Update GC trigger and pacing for the next cycle.
// Account for time.
duration := nanotime() - startTime
- switch pp.gcMarkWorkerMode {
- case gcMarkWorkerDedicatedMode:
- atomic.Xaddint64(&gcController.dedicatedMarkTime, duration)
- atomic.Xaddint64(&gcController.dedicatedMarkWorkersNeeded, 1)
- case gcMarkWorkerFractionalMode:
- atomic.Xaddint64(&gcController.fractionalMarkTime, duration)
+ gcController.logWorkTime(pp.gcMarkWorkerMode, duration)
+ if pp.gcMarkWorkerMode == gcMarkWorkerFractionalMode {
atomic.Xaddint64(&pp.gcFractionalMarkTime, duration)
- case gcMarkWorkerIdleMode:
- atomic.Xaddint64(&gcController.idleMarkTime, duration)
}
// Was this the last worker and did we run out
gcw.dispose()
}
- // Update the marked heap stat.
- gcController.heapMarked = work.bytesMarked
-
// Flush scanAlloc from each mcache since we're about to modify
// heapScan directly. If we were to flush this later, then scanAlloc
// might have incorrect information.
+ //
+ // Note that it's not important to retain this information; we know
+ // exactly what heapScan is at this point via scanWork.
for _, p := range allp {
c := p.mcache
if c == nil {
continue
}
- gcController.heapScan += uint64(c.scanAlloc)
c.scanAlloc = 0
}
- // Update other GC heap size stats. This must happen after
- // cachestats (which flushes local statistics to these) and
- // flushallmcaches (which modifies gcController.heapLive).
- gcController.heapLive = work.bytesMarked
- gcController.heapScan = uint64(gcController.scanWork)
-
- if trace.enabled {
- traceHeapAlloc()
- }
+ // Reset controller state.
+ gcController.resetLive(work.bytesMarked)
}
// gcSweep must be called on the system stack because it acquires the heap
// startCycle resets the GC controller's state and computes estimates
// for a new GC cycle. The caller must hold worldsema and the world
// must be stopped.
-func (c *gcControllerState) startCycle() {
+func (c *gcControllerState) startCycle(markStartTime int64) {
c.scanWork = 0
c.bgScanCredit = 0
c.assistTime = 0
c.dedicatedMarkTime = 0
c.fractionalMarkTime = 0
c.idleMarkTime = 0
+ c.markStartTime = markStartTime
// Ensure that the heap goal is at least a little larger than
// the current live heap size. This may not be the case if GC
// userForced indicates whether the current GC cycle was forced
// by the application.
func (c *gcControllerState) endCycle(userForced bool) float64 {
+ // Record last heap goal for the scavenger.
+ // We'll be updating the heap goal soon.
+ gcController.lastHeapGoal = gcController.heapGoal
+
if userForced {
// Forced GC means this cycle didn't start at the
// trigger, so where it finished isn't good
return gp
}
+// resetLive sets up the controller state for the next mark phase after the end
+// of the previous one. Must be called after endCycle and before commit, before
+// the world is started.
+//
+// The world must be stopped.
+func (c *gcControllerState) resetLive(bytesMarked uint64) {
+ c.heapMarked = bytesMarked
+ c.heapLive = bytesMarked
+ c.heapScan = uint64(c.scanWork)
+
+ // heapLive was updated, so emit a trace event.
+ if trace.enabled {
+ traceHeapAlloc()
+ }
+}
+
+// logWorkTime updates mark work accounting in the controller by a duration of
+// work in nanoseconds.
+//
+// Safe to execute at any time.
+func (c *gcControllerState) logWorkTime(mode gcMarkWorkerMode, duration int64) {
+ switch mode {
+ case gcMarkWorkerDedicatedMode:
+ atomic.Xaddint64(&c.dedicatedMarkTime, duration)
+ atomic.Xaddint64(&c.dedicatedMarkWorkersNeeded, 1)
+ case gcMarkWorkerFractionalMode:
+ atomic.Xaddint64(&c.fractionalMarkTime, duration)
+ case gcMarkWorkerIdleMode:
+ atomic.Xaddint64(&c.idleMarkTime, duration)
+ default:
+ throw("logWorkTime: unknown mark worker mode")
+ }
+}
+
// commit sets the trigger ratio and updates everything
// derived from it: the absolute trigger, the heap goal, mark pacing,
// and sweep pacing.