runtime: decentralize sweep termination and mark transition

This moves all of GC initialization, sweep termination, and the
transition to concurrent marking in to the off->mark transition
function. This means it's now handled on the goroutine that detected
the state exit condition.

As a result, malloc no longer needs to Gosched() at the beginning of
the GC cycle to prevent over-allocation while the GC is starting up
because it will now *help* the GC to start up. The Gosched hack is
still necessary during GC shutdown (this is easy to test by enabling
gctrace and hitting Ctrl-S to block the gctrace output).

At this point, the GC coordinator still handles later phases. This
requires a small tweak to how we start the GC coordinator. Currently,
starting the GC coordinator is best-effort and may fail if the
coordinator is about to park from the previous cycle but hasn't yet.
We fix this by replacing the park/ready to wake up the coordinator
with a semaphore. This is temporary since the coordinator will be
going away in a few commits.

Updates #11970.

Change-Id: I2c6a11c91e72dfbc59c2d8e7c66146dee9a444fe
Reviewed-on: https://go-review.googlesource.com/16357
Reviewed-by: Rick Hudson <rlh@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 230849609f8f19b0232e3ec1d5885aa7861f9880..45ebe712baafe4e587f280e2426e7adfd6b82a28 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -737,7 +737,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
        if shouldhelpgc && gcShouldStart(false) {
                gcStart(gcBackgroundMode, false)
        } else if shouldhelpgc && bggc.working != 0 && gcBlackenEnabled == 0 {
-               // The GC is starting up or shutting down, so we can't
+               // The GC shutting down, so we can't
                // assist, but we also can't allocate unabated. Slow
                // down this G's allocation and help the GC stay
                // scheduled by yielding.

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 4bbd2d02ba825e7561e92a7889092b7b867ba721..7a4f6f53eab8032e76fdfa5b6c334363fdadaf5b 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -915,31 +915,23 @@ const (
 )
 
 // startGCCoordinator starts and readies the GC coordinator goroutine.
-// If mode is gcBackgroundMode, this will
-// start GC in the background and return. Otherwise, this will block
-// until the new GC cycle is started and finishes.
 //
 // TODO(austin): This function is temporary and will go away when we
 // finish the transition to the decentralized state machine.
-func startGCCoordinator(mode gcMode) {
-       if mode != gcBackgroundMode {
-               // special synchronous cases
-               gc(mode)
-               return
-       }
-
+func startGCCoordinator() {
        // trigger concurrent GC
        readied := false
        lock(&bggc.lock)
        if !bggc.started {
                bggc.working = 1
                bggc.started = true
+               bggc.wakeSema = 1
                readied = true
                go backgroundgc()
-       } else if bggc.working == 0 {
-               bggc.working = 1
+       } else {
+               bggc.working++
                readied = true
-               ready(bggc.g, 0)
+               semrelease(&bggc.wakeSema)
        }
        unlock(&bggc.lock)
        if readied {
@@ -952,20 +944,21 @@ func startGCCoordinator(mode gcMode) {
 // State of the background concurrent GC goroutine.
 var bggc struct {
        lock    mutex
-       g       *g
        working uint
        started bool
+
+       wakeSema uint32
 }
 
 // backgroundgc is running in a goroutine and does the concurrent GC work.
 // bggc holds the state of the backgroundgc.
 func backgroundgc() {
-       bggc.g = getg()
        for {
                gc(gcBackgroundMode)
                lock(&bggc.lock)
-               bggc.working = 0
-               goparkunlock(&bggc.lock, "Concurrent GC wait", traceEvGoBlock, 1)
+               bggc.working--
+               unlock(&bggc.lock)
+               semacquire(&bggc.wakeSema, false)
        }
 }
 
@@ -1042,16 +1035,6 @@ func gcStart(mode gcMode, forceTrigger bool) {
                }
        }
 
-       // TODO: Move sweep termination and initialization from the
-       // coordinator to here.
-       startGCCoordinator(mode)
-
-       if useStartSema {
-               semrelease(&work.startSema)
-       }
-}
-
-func gc(mode gcMode) {
        // Ok, we're doing it!  Stop everybody else
        semacquire(&worldsema, false)
 
@@ -1135,10 +1118,33 @@ func gc(mode gcMode) {
                gcController.assistStartTime = now
                work.tMark = now
 
-               // Enable background mark workers and wait for
-               // background mark completion.
+               // Enable background mark workers.
                gcController.bgMarkStartTime = now
                work.bgMark1.clear()
+
+               // TODO: Make mark 1 completion handle the transition.
+               startGCCoordinator()
+       } else {
+               t := nanotime()
+               work.tMark, work.tMarkTerm = t, t
+               work.heapGoal = work.heap0
+
+               // Perform mark termination. This will restart the world.
+               gc(mode)
+       }
+
+       if useStartSema {
+               semrelease(&work.startSema)
+       }
+}
+
+func gc(mode gcMode) {
+       // If mode == gcBackgroundMode, world is not stopped.
+       // If mode != gcBackgroundMode, world is stopped.
+       // TODO(austin): This is temporary.
+
+       if mode == gcBackgroundMode {
+               // Wait for background mark completion.
                work.bgMark1.wait()
 
                gcMarkRootCheck()
@@ -1170,7 +1176,7 @@ func gc(mode gcMode) {
                work.bgMark2.wait()
 
                // Begin mark termination.
-               now = nanotime()
+               now := nanotime()
                work.tMarkTerm = now
                work.pauseStart = now
                systemstack(stopTheWorldWithSema)
@@ -1192,10 +1198,6 @@ func gc(mode gcMode) {
                gcWakeAllAssists()
 
                gcController.endCycle()
-       } else {
-               t := nanotime()
-               work.tMark, work.tMarkTerm = t, t
-               work.heapGoal = work.heap0
        }
 
        // World is stopped.

diff --git a/src/runtime/trace.go b/src/runtime/trace.go
index 06bdf970ecffcb35f9b9fb8188898b66e892c2c5..7ea4e8a61f1bef191656895858964a02ea3760f6 100644 (file)
--- a/src/runtime/trace.go
+++ b/src/runtime/trace.go
@@ -773,7 +773,7 @@ func traceProcStop(pp *p) {
 }
 
 func traceGCStart() {
-       traceEvent(traceEvGCStart, 5)
+       traceEvent(traceEvGCStart, 3)
 }
 
 func traceGCDone() {