From 6686edc0e71c31de2e959058693e95e335a14000 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Thu, 3 Oct 2024 18:30:15 +0000 Subject: [PATCH] runtime: move debug checks behind constant flag in mallocgc These debug checks are very occasionally helpful, but they do cost real time. The biggest issue seems to be the bloat of mallocgc due to the "throw" paths. Overall, after some follow-ups, this change cuts about 1ns off of the mallocgc fast path. This is a microoptimization that on its own changes very little, but together with other optimizations and a breaking up of the various malloc paths will matter all together ("death by a thousand cuts"). Change-Id: I07c4547ad724b9f94281320846677fb558957721 Reviewed-on: https://go-review.googlesource.com/c/go/+/617878 LUCI-TryBot-Result: Go LUCI Auto-Submit: Michael Knyszek Reviewed-by: Keith Randall Reviewed-by: Keith Randall --- src/runtime/malloc.go | 26 +++++++++++++++++++------- src/runtime/mgcmark.go | 3 +++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 71fd47a10c..d160532377 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -972,6 +972,14 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, checkGCTrigger return } +// doubleCheckMalloc enables a bunch of extra checks to malloc to double-check +// that various invariants are upheld. +// +// We might consider turning these on by default; many of them previously were. +// They account for a few % of mallocgc's cost though, which does matter somewhat +// at scale. +const doubleCheckMalloc = false + // Allocate an object of size bytes. // Small objects are allocated from the per-P cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. @@ -991,8 +999,10 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, checkGCTrigger // //go:linkname mallocgc func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } } if size == 0 { @@ -1049,11 +1059,13 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // Set mp.mallocing to keep from being preempted by GC. mp := acquirem() - if mp.mallocing != 0 { - throw("malloc deadlock") - } - if mp.gsignal == getg() { - throw("malloc during signal") + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } } mp.mallocing = 1 diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 9a48d15552..e47ac3bb00 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -1694,6 +1694,9 @@ func gcmarknewobject(span *mspan, obj uintptr) { if useCheckmark { // The world should be stopped so this should not happen. throw("gcmarknewobject called while doing checkmark") } + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } // Mark object. objIndex := span.objIndex(obj) -- 2.48.1