// dedicated mark workers get started.
dedicatedMarkWorkersNeeded int64
- // assistRatio is the ratio of scan work to allocated bytes
- // that should be performed by mutator assists. This is
+ // assistWorkPerByte is the ratio of scan work to allocated
+ // bytes that should be performed by mutator assists. This is
// computed at the beginning of each cycle and updated every
// time heap_scan is updated.
- assistRatio float64
+ assistWorkPerByte float64
+
+ // assistBytesPerWork is 1/assistWorkPerByte.
+ assistBytesPerWork float64
// fractionalUtilizationGoal is the fraction of wall clock
// time that should be spent in the fractional mark worker.
c.revise()
if debug.gcpacertrace > 0 {
- print("pacer: assist ratio=", c.assistRatio,
+ print("pacer: assist ratio=", c.assistWorkPerByte,
" (scan ", memstats.heap_scan>>20, " MB in ",
work.initialHeapLive>>20, "->",
c.heapGoal>>20, " MB)",
// revise updates the assist ratio during the GC cycle to account for
// improved estimates. This should be called either under STW or
-// whenever memstats.heap_scan is updated (with mheap_.lock held).
+// whenever memstats.heap_scan or memstats.heap_live is updated (with
+// mheap_.lock held).
//
// It should only be called when gcBlackenEnabled != 0 (because this
// is when assists are enabled and the necessary statistics are
// available).
func (c *gcControllerState) revise() {
- // Compute the expected scan work.
+ // Compute the expected scan work remaining.
//
// Note that the scannable heap size is likely to increase
// during the GC cycle. This is why it's important to revise
// scannable heap size may target too little scan work.
//
// This particular estimate is a strict upper bound on the
- // possible scan work in the current heap.
+ // possible remaining scan work for the current heap.
// You might consider dividing this by 2 (or by
// (100+GOGC)/100) to counter this over-estimation, but
// benchmarks show that this has almost no effect on mean
// mutator utilization, heap size, or assist time and it
// introduces the danger of under-estimating and letting the
// mutator outpace the garbage collector.
- scanWorkExpected := memstats.heap_scan
+ scanWorkExpected := int64(memstats.heap_scan) - c.scanWork
+ if scanWorkExpected < 1000 {
+ // We set a somewhat arbitrary lower bound on
+ // remaining scan work since if we aim a little high,
+ // we can miss by a little.
+ //
+ // We *do* need to enforce that this is at least 1,
+ // since marking is racy and double-scanning objects
+ // may legitimately make the expected scan work
+ // negative.
+ scanWorkExpected = 1000
+ }
- // Compute the mutator assist ratio so by the time the mutator
- // allocates the remaining heap bytes up to next_gc, it will
- // have done (or stolen) the estimated amount of scan work.
- heapDistance := int64(c.heapGoal) - int64(work.initialHeapLive)
+ // Compute the heap distance remaining.
+ heapDistance := int64(c.heapGoal) - int64(memstats.heap_live)
if heapDistance <= 0 {
- print("runtime: heap goal=", heapDistance, " initial heap live=", work.initialHeapLive, "\n")
- throw("negative heap distance")
+ // This shouldn't happen, but if it does, avoid
+ // dividing by zero or setting the assist negative.
+ heapDistance = 1
}
- c.assistRatio = float64(scanWorkExpected) / float64(heapDistance)
+
+ // Compute the mutator assist ratio so by the time the mutator
+ // allocates the remaining heap bytes up to next_gc, it will
+ // have done (or stolen) the remaining amount of scan work.
+ c.assistWorkPerByte = float64(scanWorkExpected) / float64(heapDistance)
+ c.assistBytesPerWork = float64(heapDistance) / float64(scanWorkExpected)
}
// endCycle updates the GC controller state at the end of the
for _, gp := range allgs {
gp.gcscandone = false // set to true in gcphasework
gp.gcscanvalid = false // stack has not been scanned
- gp.gcalloc = 0
- gp.gcscanwork = 0
+ gp.gcAssistBytes = 0
}
numgs = len(allgs)
unlock(&allglock)
}
// Record allocation.
- gp.gcalloc += size
+ gp.gcAssistBytes -= int64(size)
- if !allowAssist {
+ if !allowAssist || gp.gcAssistBytes >= 0 {
return
}
return
}
- // Compute the amount of assist scan work we need to do.
- scanWork := int64(gcController.assistRatio*float64(gp.gcalloc)) - gp.gcscanwork
- // scanWork can be negative if the last assist scanned a large
- // object and we're still ahead of our assist goal.
- if scanWork <= 0 {
- return
- }
+ // Compute the amount of scan work we need to do to make the
+ // balance positive.
+ debtBytes := -gp.gcAssistBytes
+ scanWork := int64(gcController.assistWorkPerByte * float64(debtBytes))
retry:
// Steal as much credit as we can from the background GC's
if bgScanCredit > 0 {
if bgScanCredit < scanWork {
stolen = bgScanCredit
+ gp.gcAssistBytes += 1 + int64(gcController.assistBytesPerWork*float64(stolen))
} else {
stolen = scanWork
+ gp.gcAssistBytes += debtBytes
}
xaddint64(&gcController.bgScanCredit, -stolen)
scanWork -= stolen
- gp.gcscanwork += stolen
if scanWork == 0 {
+ // We were able to steal all of the credit we
+ // needed.
return
}
}
// will be more cache friendly.
gcw := &getg().m.p.ptr().gcw
workDone := gcDrainN(gcw, scanWork)
- // Record that we did this much scan work.
- gp.gcscanwork += workDone
- scanWork -= workDone
// If we are near the end of the mark phase
// dispose of the gcw.
if gcBlackenPromptly {
gcw.dispose()
}
+
+ // Record that we did this much scan work.
+ scanWork -= workDone
+ // Back out the number of bytes of assist credit that
+ // this scan work counts for. The "1+" is a poor man's
+ // round-up, to ensure this adds credit even if
+ // assistBytesPerWork is very low.
+ gp.gcAssistBytes += 1 + int64(gcController.assistBytesPerWork*float64(workDone))
+
// If this is the last worker and we ran out of work,
// signal a completion point.
incnwait := xadd(&work.nwait, +1)