runtime: move debug checks behind constant flag in mallocgc

These debug checks are very occasionally helpful, but they do cost real
time. The biggest issue seems to be the bloat of mallocgc due to the
"throw" paths. Overall, after some follow-ups, this change cuts about
1ns off of the mallocgc fast path.

This is a microoptimization that on its own changes very little, but
together with other optimizations and a breaking up of the various
malloc paths will matter all together ("death by a thousand cuts").

Change-Id: I07c4547ad724b9f94281320846677fb558957721
Reviewed-on: https://go-review.googlesource.com/c/go/+/617878
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Keith Randall <khr@golang.org>

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 71fd47a10c0acafd1b1a107399c97e20f752482f..d1605323772c741701e05d5f32bb30cd5a2b5508 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -972,6 +972,14 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, checkGCTrigger
        return
 }
 
+// doubleCheckMalloc enables a bunch of extra checks to malloc to double-check
+// that various invariants are upheld.
+//
+// We might consider turning these on by default; many of them previously were.
+// They account for a few % of mallocgc's cost though, which does matter somewhat
+// at scale.
+const doubleCheckMalloc = false
+
 // Allocate an object of size bytes.
 // Small objects are allocated from the per-P cache's free lists.
 // Large objects (> 32 kB) are allocated straight from the heap.
@@ -991,8 +999,10 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, checkGCTrigger
 //
 //go:linkname mallocgc
 func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
-       if gcphase == _GCmarktermination {
-               throw("mallocgc called with gcphase == _GCmarktermination")
+       if doubleCheckMalloc {
+               if gcphase == _GCmarktermination {
+                       throw("mallocgc called with gcphase == _GCmarktermination")
+               }
        }
 
        if size == 0 {
@@ -1049,11 +1059,13 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
 
        // Set mp.mallocing to keep from being preempted by GC.
        mp := acquirem()
-       if mp.mallocing != 0 {
-               throw("malloc deadlock")
-       }
-       if mp.gsignal == getg() {
-               throw("malloc during signal")
+       if doubleCheckMalloc {
+               if mp.mallocing != 0 {
+                       throw("malloc deadlock")
+               }
+               if mp.gsignal == getg() {
+                       throw("malloc during signal")
+               }
        }
        mp.mallocing = 1
 

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 9a48d15552f32bdccfd93471181b0077ec1c8a68..e47ac3bb0030af9229e121c8d6de281e6b94de1c 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -1694,6 +1694,9 @@ func gcmarknewobject(span *mspan, obj uintptr) {
        if useCheckmark { // The world should be stopped so this should not happen.
                throw("gcmarknewobject called while doing checkmark")
        }
+       if gcphase == _GCmarktermination {
+               throw("mallocgc called with gcphase == _GCmarktermination")
+       }
 
        // Mark object.
        objIndex := span.objIndex(obj)