return a.Type().Alignment() > b.Type().Alignment()
}
+ // Sort normal variables before open-coded-defer slots, so that the
+ // latter are grouped together and near the top of the frame (to
+ // minimize varint encoding of their varp offset).
+ if a.OpenDeferSlot() != b.OpenDeferSlot() {
+ return a.OpenDeferSlot()
+ }
+
+ // If a and b are both open-coded defer slots, then order them by
+ // index in descending order, so they'll be laid out in the frame in
+ // ascending order.
+ //
+ // Their index was saved in FrameOffset in state.openDeferSave.
+ if a.OpenDeferSlot() {
+ return a.FrameOffset() > b.FrameOffset()
+ }
+
// Tie breaker for stable results.
return a.Sym().Name < b.Sym().Name
}
// top of the local variables) for their starting address. The format is:
//
// - Offset of the deferBits variable
-// - Number of defers in the function
-// - Information about each defer call, in reverse order of appearance in the function:
-// - Offset of the closure value to call
+// - Offset of the first closure slot (the rest are laid out consecutively).
func (s *state) emitOpenDeferInfo() {
+ firstOffset := s.openDefers[0].closureNode.FrameOffset()
+
+ // Verify that cmpstackvarlt laid out the slots in order.
+ for i, r := range s.openDefers {
+ have := r.closureNode.FrameOffset()
+ want := firstOffset + int64(i)*int64(types.PtrSize)
+ if have != want {
+ base.FatalfAt(s.curfn.Pos(), "unexpected frame offset for open-coded defer slot #%v: have %v, want %v", i, have, want)
+ }
+ }
+
x := base.Ctxt.Lookup(s.curfn.LSym.Name + ".opendefer")
x.Set(obj.AttrContentAddressable, true)
s.curfn.LSym.Func().OpenCodedDeferInfo = x
+
off := 0
off = dvarint(x, off, -s.deferBitsTemp.FrameOffset())
- off = dvarint(x, off, int64(len(s.openDefers)))
-
- // Write in reverse-order, for ease of running in that order at runtime
- for i := len(s.openDefers) - 1; i >= 0; i-- {
- r := s.openDefers[i]
- off = dvarint(x, off, -r.closureNode.FrameOffset())
- }
+ off = dvarint(x, off, -firstOffset)
}
func okOffset(offset int64) int64 {
// Main call to ssa package to compile function
ssa.Compile(s.f)
- if s.hasOpenDefers {
+ if len(s.openDefers) != 0 {
s.emitOpenDeferInfo()
}
pos := val.Pos
temp := typecheck.TempAt(pos.WithNotStmt(), s.curfn, t)
temp.SetOpenDeferSlot(true)
+ temp.SetFrameOffset(int64(len(s.openDefers))) // so cmpstackvarlt can order them
var addrTemp *ssa.Value
// Use OpVarLive to make sure stack slot for the closure is not removed by
// dead-store elimination
p.argp = add(p.startSP, sys.MinFrameSize)
for {
- for p.openDefers > 0 {
- p.openDefers--
-
- // Find the closure offset for the next deferred call.
- var closureOffset uint32
- closureOffset, p.closureOffsets = readvarintUnsafe(p.closureOffsets)
-
- bit := uint8(1 << p.openDefers)
- if *p.deferBitsPtr&bit == 0 {
- continue
+ for p.deferBitsPtr != nil {
+ bits := *p.deferBitsPtr
+ if bits == 0 {
+ p.deferBitsPtr = nil
+ break
}
- *p.deferBitsPtr &^= bit
- if *p.deferBitsPtr == 0 {
- p.openDefers = 0 // short circuit: no more active defers
- }
+ // Find index of top bit set.
+ i := 7 - uintptr(sys.LeadingZeros8(bits))
+
+ // Clear bit and store it back.
+ bits &^= 1 << i
+ *p.deferBitsPtr = bits
- return *(*func())(add(p.varp, -uintptr(closureOffset))), true
+ return *(*func())(add(p.slotsPtr, i*goarch.PtrSize)), true
}
if d := gp._defer; d != nil && d.sp == uintptr(p.sp) {
// then we can simply loop until we find the next frame where
// it's non-zero.
- if fd := funcdata(u.frame.fn, abi.FUNCDATA_OpenCodedDeferInfo); fd != nil {
- if u.frame.fn.deferreturn == 0 {
- throw("missing deferreturn")
- }
- p.retpc = u.frame.fn.entry() + uintptr(u.frame.fn.deferreturn)
-
- var deferBitsOffset uint32
- deferBitsOffset, fd = readvarintUnsafe(fd)
- deferBitsPtr := (*uint8)(add(unsafe.Pointer(u.frame.varp), -uintptr(deferBitsOffset)))
-
- if *deferBitsPtr != 0 {
- var openDefers uint32
- openDefers, fd = readvarintUnsafe(fd)
-
- p.openDefers = uint8(openDefers)
- p.deferBitsPtr = deferBitsPtr
- p.closureOffsets = fd
- break // found a frame with open-coded defers
- }
+ if p.initOpenCodedDefers(u.frame.fn, unsafe.Pointer(u.frame.varp)) {
+ break // found a frame with open-coded defers
}
if u.frame.sp == limit {
}
p.sp = unsafe.Pointer(u.frame.sp)
p.fp = unsafe.Pointer(u.frame.fp)
- p.varp = unsafe.Pointer(u.frame.varp)
ok = true
})
return
}
+func (p *_panic) initOpenCodedDefers(fn funcInfo, varp unsafe.Pointer) bool {
+ fd := funcdata(fn, abi.FUNCDATA_OpenCodedDeferInfo)
+ if fd == nil {
+ return false
+ }
+
+ if fn.deferreturn == 0 {
+ throw("missing deferreturn")
+ }
+
+ deferBitsOffset, fd := readvarintUnsafe(fd)
+ deferBitsPtr := (*uint8)(add(varp, -uintptr(deferBitsOffset)))
+ if *deferBitsPtr == 0 {
+ return false // has open-coded defers, but none pending
+ }
+
+ slotsOffset, fd := readvarintUnsafe(fd)
+
+ p.retpc = fn.entry() + uintptr(fn.deferreturn)
+ p.deferBitsPtr = deferBitsPtr
+ p.slotsPtr = add(varp, -uintptr(slotsOffset))
+
+ return true
+}
+
// The implementation of the predeclared function recover.
// Cannot split the stack because it needs to reliably
// find the stack segment of its caller.
startSP unsafe.Pointer
// The current stack frame that we're running deferred calls for.
- sp unsafe.Pointer
- lr uintptr
- fp unsafe.Pointer
- varp unsafe.Pointer
+ sp unsafe.Pointer
+ lr uintptr
+ fp unsafe.Pointer
// retpc stores the PC where the panic should jump back to, if the
// function last returned by _panic.next() recovers the panic.
retpc uintptr
// Extra state for handling open-coded defers.
- deferBitsPtr *uint8
- closureOffsets unsafe.Pointer
- openDefers uint8 // count of pending open-coded defers
+ deferBitsPtr *uint8
+ slotsPtr unsafe.Pointer
recovered bool // whether this panic has been recovered
goexit bool