runtime: eliminate write barriers from save

As for dropg, save is writing a nil pointer that will generate a write
barrier with the hybrid barrier. However, in this case, ctxt always
should already be nil, so replace the write with an assertion that
this is the case.

At this point, we're ready to disable the write barrier elision
optimizations that interfere with the hybrid barrier.

Updates #17503.

Change-Id: I83208e65aa33403d442401f355b2e013ab9a50e9
Reviewed-on: https://go-review.googlesource.com/31571
Reviewed-by: Rick Hudson <rlh@golang.org>

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index eb2532f3c3eade70d40df9b98ec6629da2a5a3fa..ed8e6bb00a227e3a345fbae5217a17588dc1f26c 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -399,6 +399,11 @@ func badmorestackgsignal() {
        write(2, sp.str, int32(sp.len))
 }
 
+//go:nosplit
+func badctxt() {
+       throw("ctxt != 0")
+}
+
 func lockedOSThread() bool {
        gp := getg()
        return gp.lockedm != nil && gp.m.lockedg != nil
@@ -2285,6 +2290,12 @@ func goexit0(gp *g) {
        schedule()
 }
 
+// save updates getg().sched to refer to pc and sp so that a following
+// gogo will restore pc and sp.
+//
+// save must not have write barriers because invoking a write barrier
+// can clobber getg().sched.
+//
 //go:nosplit
 //go:nowritebarrierrec
 func save(pc, sp uintptr) {
@@ -2294,8 +2305,13 @@ func save(pc, sp uintptr) {
        _g_.sched.sp = sp
        _g_.sched.lr = 0
        _g_.sched.ret = 0
-       _g_.sched.ctxt = nil
        _g_.sched.g = guintptr(unsafe.Pointer(_g_))
+       // We need to ensure ctxt is zero, but can't have a write
+       // barrier here. However, it should always already be zero.
+       // Assert that.
+       if _g_.sched.ctxt != nil {
+               badctxt()
+       }
 }
 
 // The goroutine g is about to enter a system call.