runtime: consolidate gcDrain and gcDrainUntilPreempt

These functions were nearly identical. Consolidate them by adding a
flags argument. In addition to cleaning up this code, this makes
further changes that affect both functions easier.

Change-Id: I6ec5c947603bbbd3ff4040113b2fbc240e99745f
Reviewed-on: https://go-review.googlesource.com/15405
Reviewed-by: Rick Hudson <rlh@golang.org>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 38c3b93850979e3d283916d52d55dbd1203fb68c..0727391775fcdf7b50ea632223d6aa0f431dbaf2 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1334,7 +1334,7 @@ func gcBgMarkWorker(p *p) {
                default:
                        throw("gcBgMarkWorker: unexpected gcMarkWorkerMode")
                case gcMarkWorkerDedicatedMode:
-                       gcDrain(&p.gcw, gcBgCreditSlack)
+                       gcDrain(&p.gcw, gcBgCreditSlack, gcDrainBlock)
                        // gcDrain did the xadd(&work.nwait +1) to
                        // match the decrement above. It only returns
                        // at a mark completion point.
@@ -1343,7 +1343,7 @@ func gcBgMarkWorker(p *p) {
                                throw("gcDrain returned with buffer")
                        }
                case gcMarkWorkerFractionalMode, gcMarkWorkerIdleMode:
-                       gcDrainUntilPreempt(&p.gcw, gcBgCreditSlack)
+                       gcDrain(&p.gcw, gcBgCreditSlack, gcDrainUntilPreempt)
 
                        // If we are nearing the end of mark, dispose
                        // of the cache promptly. We must do this
@@ -1454,7 +1454,7 @@ func gcMark(start_time int64) {
        parfordo(work.markfor)
 
        var gcw gcWork
-       gcDrain(&gcw, -1)
+       gcDrain(&gcw, -1, gcDrainBlock)
        gcw.dispose()
 
        if work.full != 0 {
@@ -1717,7 +1717,7 @@ func gchelper() {
        parfordo(work.markfor)
        if gcphase != _GCscan {
                var gcw gcWork
-               gcDrain(&gcw, -1) // blocks in getfull
+               gcDrain(&gcw, -1, gcDrainBlock) // blocks in getfull
                gcw.dispose()
        }
 

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 261788c9d8934834ba7398779c2681a0027d9882..4337d71f37a19515b16520aafa90c592bc4f09b2 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -549,19 +549,35 @@ func scanframeworker(frame *stkframe, unused unsafe.Pointer, gcw *gcWork) {
        }
 }
 
-// TODO(austin): Can we consolidate the gcDrain* functions?
+type gcDrainFlags int
 
-// gcDrain scans objects in work buffers, blackening grey
-// objects until all work buffers have been drained.
+const (
+       gcDrainUntilPreempt gcDrainFlags = 1 << iota
+
+       // gcDrainBlock is the opposite of gcDrainUntilPreempt. This
+       // is the default, but callers should use the constant for
+       // documentation purposes.
+       gcDrainBlock gcDrainFlags = 0
+)
+
+// gcDrain scans objects in work buffers, blackening grey objects
+// until all work buffers have been drained.
+//
+// If flags&gcDrainUntilPreempt != 0, gcDrain also returns if
+// g.preempt is set. Otherwise, this will block until all dedicated
+// workers are blocked in gcDrain.
+//
 // If flushScanCredit != -1, gcDrain flushes accumulated scan work
 // credit to gcController.bgScanCredit whenever gcw's local scan work
 // credit exceeds flushScanCredit.
 //go:nowritebarrier
-func gcDrain(gcw *gcWork, flushScanCredit int64) {
+func gcDrain(gcw *gcWork, flushScanCredit int64, flags gcDrainFlags) {
        if !writeBarrierEnabled {
                throw("gcDrain phase incorrect")
        }
 
+       blocking := flags&gcDrainUntilPreempt == 0
+
        var lastScanFlush, nextScanFlush int64
        if flushScanCredit != -1 {
                lastScanFlush = gcw.scanWork
@@ -570,15 +586,21 @@ func gcDrain(gcw *gcWork, flushScanCredit int64) {
                nextScanFlush = int64(^uint64(0) >> 1)
        }
 
-       for {
+       gp := getg()
+       for blocking || !gp.preempt {
                // If another proc wants a pointer, give it some.
                if work.nwait > 0 && work.full == 0 {
                        gcw.balance()
                }
 
-               b := gcw.get()
+               var b uintptr
+               if blocking {
+                       b = gcw.get()
+               } else {
+                       b = gcw.tryGet()
+               }
                if b == 0 {
-                       // work barrier reached
+                       // work barrier reached or tryGet failed.
                        break
                }
                // If the current wbuf is filled by the scan a new wbuf might be
@@ -605,58 +627,6 @@ func gcDrain(gcw *gcWork, flushScanCredit int64) {
        }
 }
 
-// gcDrainUntilPreempt blackens grey objects until g.preempt is set.
-// This is best-effort, so it will return as soon as it is unable to
-// get work, even though there may be more work in the system.
-//go:nowritebarrier
-func gcDrainUntilPreempt(gcw *gcWork, flushScanCredit int64) {
-       if !writeBarrierEnabled {
-               println("gcphase =", gcphase)
-               throw("gcDrainUntilPreempt phase incorrect")
-       }
-
-       var lastScanFlush, nextScanFlush int64
-       if flushScanCredit != -1 {
-               lastScanFlush = gcw.scanWork
-               nextScanFlush = lastScanFlush + flushScanCredit
-       } else {
-               nextScanFlush = int64(^uint64(0) >> 1)
-       }
-
-       gp := getg()
-       for !gp.preempt {
-               // If the work queue is empty, balance. During
-               // concurrent mark we don't really know if anyone else
-               // can make use of this work, but even if we're the
-               // only worker, the total cost of this per cycle is
-               // only O(_WorkbufSize) pointer copies.
-               if work.full == 0 && work.partial == 0 {
-                       gcw.balance()
-               }
-
-               b := gcw.tryGet()
-               if b == 0 {
-                       // No more work
-                       break
-               }
-               scanobject(b, gcw)
-
-               // Flush background scan work credit to the global
-               // account if we've accumulated enough locally so
-               // mutator assists can draw on it.
-               if gcw.scanWork >= nextScanFlush {
-                       credit := gcw.scanWork - lastScanFlush
-                       xaddint64(&gcController.bgScanCredit, credit)
-                       lastScanFlush = gcw.scanWork
-                       nextScanFlush = lastScanFlush + flushScanCredit
-               }
-       }
-       if flushScanCredit != -1 {
-               credit := gcw.scanWork - lastScanFlush
-               xaddint64(&gcController.bgScanCredit, credit)
-       }
-}
-
 // gcDrainN blackens grey objects until it has performed roughly
 // scanWork units of scan work. This is best-effort, so it may perform
 // less work if it fails to get a work buffer. Otherwise, it will