From ee3d20129a89047ccb4a4e157688d2f24db8f343 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 26 Oct 2016 17:05:41 -0400 Subject: [PATCH] runtime: avoid getfull() barrier most of the time With the hybrid barrier, unless we're doing a STW GC or hit a very rare race (~once per all.bash) that can start mark termination before all of the work is drained, we don't need to drain the work queue at all. Even draining an empty work queue is rather expensive since we have to enter the getfull() barrier, so it's worth avoiding this. Conveniently, it's quite easy to detect whether or not we actually need the getufull() barrier: since the world is stopped when we enter mark termination, everything must have flushed its work to the work queue, so we can just check the queue. If the queue is empty and we haven't queued up any jobs that may create more work (which should always be the case with the hybrid barrier), we can simply have all GC workers perform non-blocking drains. Also conveniently, this solution is quite safe. If we do somehow screw something up and there's work on the work queue, some worker will still process it, it just may not happen in parallel. This is not the "right" solution, but it's simple, expedient, low-risk, and maintains compatibility with debug.gcrescanstacks. When we remove the gcrescanstacks fallback in Go 1.9, we should also fix the race that starts mark termination early, and then we can eliminate work draining from mark termination. Updates #17503. Change-Id: I7b3cd5de6a248ab29d78c2b42aed8b7443641361 Reviewed-on: https://go-review.googlesource.com/32186 Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 47 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 29bb8cde19..d27204176a 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -786,6 +786,16 @@ var work struct { ndone uint32 alldone note + // helperDrainBlock indicates that GC mark termination helpers + // should pass gcDrainBlock to gcDrain to block in the + // getfull() barrier. Otherwise, they should pass gcDrainNoBlock. + // + // TODO: This is a temporary fallback to support + // debug.gcrescanstacks > 0 and to work around some known + // races. Remove this when we remove the debug option and fix + // the races. + helperDrainBlock bool + // Number of roots of various root types. Set by gcMarkRootPrepare. nFlushCacheRoots int nDataRoots, nBSSRoots, nSpanRoots, nStackRoots, nRescanRoots int @@ -1590,6 +1600,31 @@ func gcMark(start_time int64) { work.ndone = 0 work.nproc = uint32(gcprocs()) + if work.full == 0 && work.nDataRoots+work.nBSSRoots+work.nSpanRoots+work.nStackRoots+work.nRescanRoots == 0 { + // There's no work on the work queue and no root jobs + // that can produce work, so don't bother entering the + // getfull() barrier. + // + // With the hybrid barrier enabled, this will be the + // situation the vast majority of the time after + // concurrent mark. However, we still need a fallback + // for STW GC and because there are some known races + // that occasionally leave work around for mark + // termination. + // + // We're still hedging our bets here: if we do + // accidentally produce some work, we'll still process + // it, just not necessarily in parallel. + // + // TODO(austin): When we eliminate + // debug.gcrescanstacks: fix the races, and remove + // work draining from mark termination so we don't + // need the fallback path. + work.helperDrainBlock = false + } else { + work.helperDrainBlock = true + } + if trace.enabled { traceGCScanStart() } @@ -1602,7 +1637,11 @@ func gcMark(start_time int64) { gchelperstart() gcw := &getg().m.p.ptr().gcw - gcDrain(gcw, gcDrainBlock) + if work.helperDrainBlock { + gcDrain(gcw, gcDrainBlock) + } else { + gcDrain(gcw, gcDrainNoBlock) + } gcw.dispose() if debug.gccheckmark > 0 { @@ -1838,7 +1877,11 @@ func gchelper() { // Parallel mark over GC roots and heap if gcphase == _GCmarktermination { gcw := &_g_.m.p.ptr().gcw - gcDrain(gcw, gcDrainBlock) // blocks in getfull + if work.helperDrainBlock { + gcDrain(gcw, gcDrainBlock) // blocks in getfull + } else { + gcDrain(gcw, gcDrainNoBlock) + } gcw.dispose() } -- 2.48.1