runtime: make consistentHeapStats acquire/release nosplit

consistentHeapStats is updated during a stack allocation, so a stack
growth during an acquire or release could cause another acquire to
happen before the operation completes fully. This may lead to an invalid
sequence number.

Fixes #49395.

Change-Id: I41ce3393dff80201793e053d4d6394d7b211a5b7
Reviewed-on: https://go-review.googlesource.com/c/go/+/361158
Trust: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go
index 341ba9a9365c2eef81ad998d5ab173785f3d2f0d..c2303e5b8ec0617f9777f63248e4a8d5e797af2c 100644 (file)
--- a/src/runtime/mstats.go
+++ b/src/runtime/mstats.go
@@ -790,7 +790,15 @@ type consistentHeapStats struct {
 //
 // The caller's P must not change between acquire and
 // release. This also means that the caller should not
-// acquire a P or release its P in between.
+// acquire a P or release its P in between. A P also must
+// not acquire a given consistentHeapStats if it hasn't
+// yet released it.
+//
+// nosplit because a stack growth in this function could
+// lead to a stack allocation that could reenter the
+// function.
+//
+//go:nosplit
 func (m *consistentHeapStats) acquire() *heapStatsDelta {
        if pp := getg().m.p.ptr(); pp != nil {
                seq := atomic.Xadd(&pp.statsSeq, 1)
@@ -814,6 +822,12 @@ func (m *consistentHeapStats) acquire() *heapStatsDelta {
 // The caller's P must not change between acquire and
 // release. This also means that the caller should not
 // acquire a P or release its P in between.
+//
+// nosplit because a stack growth in this function could
+// lead to a stack allocation that causes another acquire
+// before this operation has completed.
+//
+//go:nosplit
 func (m *consistentHeapStats) release() {
        if pp := getg().m.p.ptr(); pp != nil {
                seq := atomic.Xadd(&pp.statsSeq, 1)