runtime: avoid relying on the unwinder in deferreturn

This CL changes deferreturn so that it never needs to invoke the
unwinder. Instead, in the unusual case that we recover into a frame
with pending open-coded defers, we now save the extra state needed to
find them in g.param.

Change-Id: Ied35f6c1063fee5b6044cc37b2bccd3f90682fe6
Reviewed-on: https://go-review.googlesource.com/c/go/+/515856
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Keith Randall <khr@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Matthew Dempsky <mdempsky@google.com>

diff --git a/src/runtime/panic.go b/src/runtime/panic.go
index e7483b80b6ad9a49e40f08c6ddafd50bd3dc215e..5b7f35a0a503052ab7c389f19cfe5a533bcff599 100644 (file)
--- a/src/runtime/panic.go
+++ b/src/runtime/panic.go
@@ -635,19 +635,26 @@ func (p *_panic) start(pc uintptr, sp unsafe.Pointer) {
        p.startPC = getcallerpc()
        p.startSP = unsafe.Pointer(getcallersp())
 
-       if !p.deferreturn {
-               p.link = gp._panic
-               gp._panic = (*_panic)(noescape(unsafe.Pointer(p)))
-       } else {
-               // Fast path for deferreturn: if there's a pending linked defer
-               // for this frame, then we know there aren't any open-coded
-               // defers, and we don't need to find the parent frame either.
-               if d := gp._defer; d != nil && d.sp == uintptr(sp) {
-                       p.sp = sp
-                       return
+       if p.deferreturn {
+               p.sp = sp
+
+               if s := (*savedOpenDeferState)(gp.param); s != nil {
+                       // recovery saved some state for us, so that we can resume
+                       // calling open-coded defers without unwinding the stack.
+
+                       gp.param = nil
+
+                       p.retpc = s.retpc
+                       p.deferBitsPtr = (*byte)(add(sp, s.deferBitsOffset))
+                       p.slotsPtr = add(sp, s.slotsOffset)
                }
+
+               return
        }
 
+       p.link = gp._panic
+       gp._panic = (*_panic)(noescape(unsafe.Pointer(p)))
+
        // Initialize state machine, and find the first frame with a defer.
        //
        // Note: We could use startPC and startSP here, but callers will
@@ -684,6 +691,15 @@ func (p *_panic) nextDefer() (func(), bool) {
        for {
                for p.deferBitsPtr != nil {
                        bits := *p.deferBitsPtr
+
+                       // Check whether any open-coded defers are still pending.
+                       //
+                       // Note: We need to check this upfront (rather than after
+                       // clearing the top bit) because it's possible that Goexit
+                       // invokes a deferred call, and there were still more pending
+                       // open-coded defers in the frame; but then the deferred call
+                       // panic and invoked the remaining defers in the frame, before
+                       // recovering and restarting the Goexit loop.
                        if bits == 0 {
                                p.deferBitsPtr = nil
                                break
@@ -730,9 +746,7 @@ func (p *_panic) nextFrame() (ok bool) {
        gp := getg()
        systemstack(func() {
                var limit uintptr
-               if p.deferreturn {
-                       limit = uintptr(p.fp)
-               } else if d := gp._defer; d != nil {
+               if d := gp._defer; d != nil {
                        limit = uintptr(d.sp)
                }
 
@@ -749,22 +763,18 @@ func (p *_panic) nextFrame() (ok bool) {
                        // then we can simply loop until we find the next frame where
                        // it's non-zero.
 
-                       if p.initOpenCodedDefers(u.frame.fn, unsafe.Pointer(u.frame.varp)) {
-                               break // found a frame with open-coded defers
+                       if u.frame.sp == limit {
+                               break // found a frame with linked defers
                        }
 
-                       if u.frame.sp == limit {
-                               break // found a frame with linked defers, or deferreturn with no defers
+                       if p.initOpenCodedDefers(u.frame.fn, unsafe.Pointer(u.frame.varp)) {
+                               break // found a frame with open-coded defers
                        }
 
                        u.next()
                }
 
-               if p.deferreturn {
-                       p.lr = 0 // prevent unwinding past this frame
-               } else {
-                       p.lr = u.frame.lr
-               }
+               p.lr = u.frame.lr
                p.sp = unsafe.Pointer(u.frame.sp)
                p.fp = unsafe.Pointer(u.frame.fp)
 
@@ -889,6 +899,7 @@ var paniclk mutex
 func recovery(gp *g) {
        p := gp._panic
        pc, sp := p.retpc, uintptr(p.sp)
+       p0, saveOpenDeferState := p, p.deferBitsPtr != nil && *p.deferBitsPtr != 0
 
        // Unwind the panic stack.
        for ; p != nil && uintptr(p.startSP) < sp; p = p.link {
@@ -913,6 +924,7 @@ func recovery(gp *g) {
                // worthwhile though.
                if p.goexit {
                        pc, sp = p.startPC, uintptr(p.startSP)
+                       saveOpenDeferState = false // goexit is unwinding the stack anyway
                        break
                }
 
@@ -924,6 +936,24 @@ func recovery(gp *g) {
                gp.sig = 0
        }
 
+       if gp.param != nil {
+               throw("unexpected gp.param")
+       }
+       if saveOpenDeferState {
+               // If we're returning to deferreturn and there are more open-coded
+               // defers for it to call, save enough state for it to be able to
+               // pick up where p0 left off.
+               gp.param = unsafe.Pointer(&savedOpenDeferState{
+                       retpc: p0.retpc,
+
+                       // We need to save deferBitsPtr and slotsPtr too, but those are
+                       // stack pointers. To avoid issues around heap objects pointing
+                       // to the stack, save them as offsets from SP.
+                       deferBitsOffset: uintptr(unsafe.Pointer(p0.deferBitsPtr)) - uintptr(p0.sp),
+                       slotsOffset:     uintptr(p0.slotsPtr) - uintptr(p0.sp),
+               })
+       }
+
        // TODO(mdempsky): Currently, we rely on frames containing "defer"
        // to end with "CALL deferreturn; RET". This allows deferreturn to
        // finish running any pending defers in the frame.

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index cdd8c3db7fe1ac85d427dc29f74c15707195c368..54fab050eaa57e44a4f936d15237bf154ed06475 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -433,7 +433,7 @@ type g struct {
        // param is a generic pointer parameter field used to pass
        // values in particular contexts where other storage for the
        // parameter would be difficult to find. It is currently used
-       // in three ways:
+       // in four ways:
        // 1. When a channel operation wakes up a blocked goroutine, it sets param to
        //    point to the sudog of the completed blocking operation.
        // 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed
@@ -441,6 +441,8 @@ type g struct {
        //    stack may have moved in the meantime.
        // 3. By debugCallWrap to pass parameters to a new goroutine because allocating a
        //    closure in the runtime is forbidden.
+       // 4. When a panic is recovered and control returns to the respective frame,
+       //    param may point to a savedOpenDeferState.
        param        unsafe.Pointer
        atomicstatus atomic.Uint32
        stackLock    uint32 // sigprof/scang lock; TODO: fold in to atomicstatus
@@ -1041,6 +1043,15 @@ type _panic struct {
        deferreturn bool
 }
 
+// savedOpenDeferState tracks the extra state from _panic that's
+// necessary for deferreturn to pick up where gopanic left off,
+// without needing to unwind the stack.
+type savedOpenDeferState struct {
+       retpc           uintptr
+       deferBitsOffset uintptr
+       slotsOffset     uintptr
+}
+
 // ancestorInfo records details of where a goroutine was started.
 type ancestorInfo struct {
        pcs  []uintptr // pcs from the stack of this goroutine