runtime: simplify/optimize allocate-black a bit

Currently allocating black switches to the system stack (which is
probably a historical accident) and atomically updates the global
bytes marked stat. Since we're about to depend on this much more,
optimize it a bit by putting it back on the regular stack and updating
the per-P bytes marked stat, which gets lazily folded into the global
bytes marked stat.

Change-Id: Ibbe16e5382d3fd2256e4381f88af342bf7020b04
Reviewed-on: https://go-review.googlesource.com/22170
Reviewed-by: Rick Hudson <rlh@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 3f437bc02f35eda1e17c7f43d8ac60ebe9e0ef77..9e1f47e1e6517ffc92234014d0bdbb2f5fa327ca 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -695,9 +695,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
        // This may be racing with GC so do it atomically if there can be
        // a race marking the bit.
        if gcphase == _GCmarktermination || gcBlackenPromptly {
-               systemstack(func() {
-                       gcmarknewobject_m(uintptr(x), size, scanSize)
-               })
+               gcmarknewobject(uintptr(x), size, scanSize)
        }
 
        if raceenabled {

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 0d05838987d7c076a299caf88aa64bdda5ba5717..ad64b735a5f6c318af2f41cd5e54031446d76b8f 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -1132,15 +1132,20 @@ func gcDumpObject(label string, obj, off uintptr) {
        }
 }
 
-// If gcBlackenPromptly is true we are in the second mark phase phase so we allocate black.
+// gcmarknewobject marks a newly allocated object black. obj must
+// not contain any non-nil pointers.
+//
+// This is nosplit so it can manipulate a gcWork without preemption.
+//
 //go:nowritebarrier
-func gcmarknewobject_m(obj, size, scanSize uintptr) {
+//go:nosplit
+func gcmarknewobject(obj, size, scanSize uintptr) {
        if useCheckmark && !gcBlackenPromptly { // The world should be stopped so this should not happen.
                throw("gcmarknewobject called while doing checkmark")
        }
        heapBitsForAddr(obj).setMarked()
-       atomic.Xadd64(&work.bytesMarked, int64(size))
        gcw := &getg().m.p.ptr().gcw
+       gcw.bytesMarked += uint64(size)
        gcw.scanWork += int64(scanSize)
 }