runtime: control background scan credit flushing with flag

Currently callers of gcDrain control whether it flushes scan work
credit to gcController.bgScanCredit by passing a value other than -1
for the flush threshold. Shortly we're going to make this always flush
scan work to gcController.scanWork and optionally also flush scan work
to gcController.bgScanCredit. This will be much easier if the flush
threshold is simply a constant (which it is in practice) and callers
merely control whether or not the flush includes the background
credit. Hence, replace the flush threshold argument with a flag.

Change-Id: Ia27db17de8a3f1e462a5d7137d4b5dc72f99a04e
Reviewed-on: https://go-review.googlesource.com/15406
Reviewed-by: Rick Hudson <rlh@golang.org>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 0727391775fcdf7b50ea632223d6aa0f431dbaf2..3cf642d9b256c9203cd1322097c75f678fb1768a 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1334,7 +1334,7 @@ func gcBgMarkWorker(p *p) {
                default:
                        throw("gcBgMarkWorker: unexpected gcMarkWorkerMode")
                case gcMarkWorkerDedicatedMode:
-                       gcDrain(&p.gcw, gcBgCreditSlack, gcDrainBlock)
+                       gcDrain(&p.gcw, gcDrainBlock|gcDrainFlushBgCredit)
                        // gcDrain did the xadd(&work.nwait +1) to
                        // match the decrement above. It only returns
                        // at a mark completion point.
@@ -1343,7 +1343,7 @@ func gcBgMarkWorker(p *p) {
                                throw("gcDrain returned with buffer")
                        }
                case gcMarkWorkerFractionalMode, gcMarkWorkerIdleMode:
-                       gcDrain(&p.gcw, gcBgCreditSlack, gcDrainUntilPreempt)
+                       gcDrain(&p.gcw, gcDrainUntilPreempt|gcDrainFlushBgCredit)
 
                        // If we are nearing the end of mark, dispose
                        // of the cache promptly. We must do this
@@ -1454,7 +1454,7 @@ func gcMark(start_time int64) {
        parfordo(work.markfor)
 
        var gcw gcWork
-       gcDrain(&gcw, -1, gcDrainBlock)
+       gcDrain(&gcw, gcDrainBlock)
        gcw.dispose()
 
        if work.full != 0 {
@@ -1717,7 +1717,7 @@ func gchelper() {
        parfordo(work.markfor)
        if gcphase != _GCscan {
                var gcw gcWork
-               gcDrain(&gcw, -1, gcDrainBlock) // blocks in getfull
+               gcDrain(&gcw, gcDrainBlock) // blocks in getfull
                gcw.dispose()
        }
 

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 4337d71f37a19515b16520aafa90c592bc4f09b2..c3134bda29fc96ef988fcbef4115b4233cd7cd2f 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -553,6 +553,7 @@ type gcDrainFlags int
 
 const (
        gcDrainUntilPreempt gcDrainFlags = 1 << iota
+       gcDrainFlushBgCredit
 
        // gcDrainBlock is the opposite of gcDrainUntilPreempt. This
        // is the default, but callers should use the constant for
@@ -567,21 +568,22 @@ const (
 // g.preempt is set. Otherwise, this will block until all dedicated
 // workers are blocked in gcDrain.
 //
-// If flushScanCredit != -1, gcDrain flushes accumulated scan work
-// credit to gcController.bgScanCredit whenever gcw's local scan work
-// credit exceeds flushScanCredit.
+// If flags&gcDrainFlushBgCredit != 0, gcDrain flushes scan work
+// credit to gcController.bgScanCredit every gcBgCreditSlack units of
+// scan work.
 //go:nowritebarrier
-func gcDrain(gcw *gcWork, flushScanCredit int64, flags gcDrainFlags) {
+func gcDrain(gcw *gcWork, flags gcDrainFlags) {
        if !writeBarrierEnabled {
                throw("gcDrain phase incorrect")
        }
 
        blocking := flags&gcDrainUntilPreempt == 0
+       flushBgCredit := flags&gcDrainFlushBgCredit != 0
 
        var lastScanFlush, nextScanFlush int64
-       if flushScanCredit != -1 {
+       if flushBgCredit {
                lastScanFlush = gcw.scanWork
-               nextScanFlush = lastScanFlush + flushScanCredit
+               nextScanFlush = lastScanFlush + gcBgCreditSlack
        } else {
                nextScanFlush = int64(^uint64(0) >> 1)
        }
@@ -618,10 +620,10 @@ func gcDrain(gcw *gcWork, flushScanCredit int64, flags gcDrainFlags) {
                        credit := gcw.scanWork - lastScanFlush
                        xaddint64(&gcController.bgScanCredit, credit)
                        lastScanFlush = gcw.scanWork
-                       nextScanFlush = lastScanFlush + flushScanCredit
+                       nextScanFlush = lastScanFlush + gcBgCreditSlack
                }
        }
-       if flushScanCredit != -1 {
+       if flushBgCredit {
                credit := gcw.scanWork - lastScanFlush
                xaddint64(&gcController.bgScanCredit, credit)
        }