runtime: preempt dedicated background mark workers for STW

Currently, dedicated background mark workers are essentially always
non-preemptible.

This change makes it so that dedicated background mark workers park if
their preemption flag is set and someone is trying to STW, allowing them
to do so.

This change prepares us for allowing a STW to happen (and happen
promptly) during GC marking in a follow-up change.

Updates #19812.

Change-Id: I67fb6085bf0f0aebd18ca500172767818a1f15e3
Reviewed-on: https://go-review.googlesource.com/c/go/+/215157
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 54f988a9023abd4a789d3161491ed2b7e5d9a394..2c17d8befa18fe1a69f08ebc6f3272626fdd7a30 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -963,6 +963,8 @@ const (
 // credit to gcController.bgScanCredit every gcCreditSlack units of
 // scan work.
 //
+// gcDrain will always return if there is a pending STW.
+//
 //go:nowritebarrier
 func gcDrain(gcw *gcWork, flags gcDrainFlags) {
        if !writeBarrier.needed {
@@ -991,7 +993,8 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) {
 
        // Drain root marking jobs.
        if work.markrootNext < work.markrootJobs {
-               for !(preemptible && gp.preempt) {
+               // Stop if we're preemptible or if someone wants to STW.
+               for !(gp.preempt && (preemptible || atomic.Load(&sched.gcwaiting) != 0)) {
                        job := atomic.Xadd(&work.markrootNext, +1) - 1
                        if job >= work.markrootJobs {
                                break
@@ -1004,7 +1007,8 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) {
        }
 
        // Drain heap marking jobs.
-       for !(preemptible && gp.preempt) {
+       // Stop if we're preemptible or if someone wants to STW.
+       for !(gp.preempt && (preemptible || atomic.Load(&sched.gcwaiting) != 0)) {
                // Try to keep work available on the global queue. We used to
                // check if there were waiting workers, but it's better to
                // just keep work available than to make workers wait. In the