}
if gp._panic != nil {
// Panics are always stack allocated.
- state.putPtr(uintptr(unsafe.Pointer(gp._panic)))
+ state.putPtr(uintptr(unsafe.Pointer(gp._panic)), false)
}
// Find and scan all reachable stack objects.
+ //
+ // The state's pointer queue prioritizes precise pointers over
+ // conservative pointers so that we'll prefer scanning stack
+ // objects precisely.
state.buildIndex()
for {
- p := state.getPtr()
+ p, conservative := state.getPtr()
if p == 0 {
break
}
}
obj.setType(nil) // Don't scan it again.
if stackTraceDebug {
- println(" live stkobj at", hex(state.stack.lo+uintptr(obj.off)), "of type", t.string())
+ printlock()
+ print(" live stkobj at", hex(state.stack.lo+uintptr(obj.off)), "of type", t.string())
+ if conservative {
+ print(" (conservative)")
+ }
+ println()
+ printunlock()
}
gcdata := t.gcdata
var s *mspan
gcdata = (*byte)(unsafe.Pointer(s.startAddr))
}
- scanblock(state.stack.lo+uintptr(obj.off), t.ptrdata, gcdata, gcw, &state)
+ b := state.stack.lo + uintptr(obj.off)
+ if conservative {
+ scanConservative(b, t.ptrdata, gcdata, gcw, &state)
+ } else {
+ scanblock(b, t.ptrdata, gcdata, gcw, &state)
+ }
if s != nil {
dematerializeGCProg(s)
x.nobj = 0
putempty((*workbuf)(unsafe.Pointer(x)))
}
- if state.buf != nil || state.freeBuf != nil {
+ if state.buf != nil || state.cbuf != nil || state.freeBuf != nil {
throw("remaining pointer buffers")
}
}
print("scanframe ", funcname(frame.fn), "\n")
}
+ isAsyncPreempt := frame.fn.valid() && frame.fn.funcID == funcID_asyncPreempt
+ if state.conservative || isAsyncPreempt {
+ if debugScanConservative {
+ println("conservatively scanning function", funcname(frame.fn), "at PC", hex(frame.continpc))
+ }
+
+ // Conservatively scan the frame. Unlike the precise
+ // case, this includes the outgoing argument space
+ // since we may have stopped while this function was
+ // setting up a call.
+ //
+ // TODO: We could narrow this down if the compiler
+ // produced a single map per function of stack slots
+ // and registers that ever contain a pointer.
+ if frame.varp != 0 {
+ size := frame.varp - frame.sp
+ if size > 0 {
+ scanConservative(frame.sp, size, nil, gcw, state)
+ }
+ }
+
+ // Scan arguments to this frame.
+ if frame.arglen != 0 {
+ // TODO: We could pass the entry argument map
+ // to narrow this down further.
+ scanConservative(frame.argp, frame.arglen, nil, gcw, state)
+ }
+
+ if isAsyncPreempt {
+ // This function's frame contained the
+ // registers for the asynchronously stopped
+ // parent frame. Scan the parent
+ // conservatively.
+ state.conservative = true
+ } else {
+ // We only wanted to scan those two frames
+ // conservatively. Clear the flag for future
+ // frames.
+ state.conservative = false
+ }
+ return
+ }
+
locals, args, objs := getStackMap(frame, &state.cache, false)
// Scan local variables if stack frame has been allocated.
if obj, span, objIndex := findObject(p, b, i); obj != 0 {
greyobject(obj, b, i, span, gcw, objIndex)
} else if stk != nil && p >= stk.stack.lo && p < stk.stack.hi {
- stk.putPtr(p)
+ stk.putPtr(p, false)
}
}
}
gcw.scanWork += int64(i)
}
+// scanConservative scans block [b, b+n) conservatively, treating any
+// pointer-like value in the block as a pointer.
+//
+// If ptrmask != nil, only words that are marked in ptrmask are
+// considered as potential pointers.
+//
+// If state != nil, it's assumed that [b, b+n) is a block in the stack
+// and may contain pointers to stack objects.
+func scanConservative(b, n uintptr, ptrmask *uint8, gcw *gcWork, state *stackScanState) {
+ if debugScanConservative {
+ printlock()
+ print("conservatively scanning [", hex(b), ",", hex(b+n), ")\n")
+ hexdumpWords(b, b+n, func(p uintptr) byte {
+ if ptrmask != nil {
+ word := (p - b) / sys.PtrSize
+ bits := *addb(ptrmask, word/8)
+ if (bits>>(word%8))&1 == 0 {
+ return '$'
+ }
+ }
+
+ val := *(*uintptr)(unsafe.Pointer(p))
+ if state != nil && state.stack.lo <= val && val < state.stack.hi {
+ return '@'
+ }
+
+ span := spanOfHeap(val)
+ if span == nil {
+ return ' '
+ }
+ idx := span.objIndex(val)
+ if span.isFree(idx) {
+ return ' '
+ }
+ return '*'
+ })
+ printunlock()
+ }
+
+ for i := uintptr(0); i < n; i += sys.PtrSize {
+ if ptrmask != nil {
+ word := i / sys.PtrSize
+ bits := *addb(ptrmask, word/8)
+ if bits == 0 {
+ // Skip 8 words (the loop increment will do the 8th)
+ //
+ // This must be the first time we've
+ // seen this word of ptrmask, so i
+ // must be 8-word-aligned, but check
+ // our reasoning just in case.
+ if i%(sys.PtrSize*8) != 0 {
+ throw("misaligned mask")
+ }
+ i += sys.PtrSize*8 - sys.PtrSize
+ continue
+ }
+ if (bits>>(word%8))&1 == 0 {
+ continue
+ }
+ }
+
+ val := *(*uintptr)(unsafe.Pointer(b + i))
+
+ // Check if val points into the stack.
+ if state != nil && state.stack.lo <= val && val < state.stack.hi {
+ // val may point to a stack object. This
+ // object may be dead from last cycle and
+ // hence may contain pointers to unallocated
+ // objects, but unlike heap objects we can't
+ // tell if it's already dead. Hence, if all
+ // pointers to this object are from
+ // conservative scanning, we have to scan it
+ // defensively, too.
+ state.putPtr(val, true)
+ continue
+ }
+
+ // Check if val points to a heap span.
+ span := spanOfHeap(val)
+ if span == nil {
+ continue
+ }
+
+ // Check if val points to an allocated object.
+ idx := span.objIndex(val)
+ if span.isFree(idx) {
+ continue
+ }
+
+ // val points to an allocated object. Mark it.
+ obj := span.base() + idx*span.elemsize
+ greyobject(obj, b, i, span, gcw, idx)
+ }
+}
+
// Shade the object if it isn't already.
// The object is not nil and known to be in the heap.
// Preemption must be disabled.
// stack limits
stack stack
+ // conservative indicates that the next frame must be scanned conservatively.
+ // This applies only to the innermost frame at an async safe-point.
+ conservative bool
+
// buf contains the set of possible pointers to stack objects.
// Organized as a LIFO linked list of buffers.
// All buffers except possibly the head buffer are full.
buf *stackWorkBuf
freeBuf *stackWorkBuf // keep around one free buffer for allocation hysteresis
+ // cbuf contains conservative pointers to stack objects. If
+ // all pointers to a stack object are obtained via
+ // conservative scanning, then the stack object may be dead
+ // and may contain dead pointers, so it must be scanned
+ // defensively.
+ cbuf *stackWorkBuf
+
// list of stack objects
// Objects are in increasing address order.
head *stackObjectBuf
// Add p as a potential pointer to a stack object.
// p must be a stack address.
-func (s *stackScanState) putPtr(p uintptr) {
+func (s *stackScanState) putPtr(p uintptr, conservative bool) {
if p < s.stack.lo || p >= s.stack.hi {
throw("address not a stack address")
}
- buf := s.buf
+ head := &s.buf
+ if conservative {
+ head = &s.cbuf
+ }
+ buf := *head
if buf == nil {
// Initial setup.
buf = (*stackWorkBuf)(unsafe.Pointer(getempty()))
buf.nobj = 0
buf.next = nil
- s.buf = buf
+ *head = buf
} else if buf.nobj == len(buf.obj) {
if s.freeBuf != nil {
buf = s.freeBuf
buf = (*stackWorkBuf)(unsafe.Pointer(getempty()))
}
buf.nobj = 0
- buf.next = s.buf
- s.buf = buf
+ buf.next = *head
+ *head = buf
}
buf.obj[buf.nobj] = p
buf.nobj++
// Remove and return a potential pointer to a stack object.
// Returns 0 if there are no more pointers available.
-func (s *stackScanState) getPtr() uintptr {
- buf := s.buf
- if buf == nil {
- // Never had any data.
- return 0
- }
- if buf.nobj == 0 {
- if s.freeBuf != nil {
- // Free old freeBuf.
- putempty((*workbuf)(unsafe.Pointer(s.freeBuf)))
- }
- // Move buf to the freeBuf.
- s.freeBuf = buf
- buf = buf.next
- s.buf = buf
+//
+// This prefers non-conservative pointers so we scan stack objects
+// precisely if there are any non-conservative pointers to them.
+func (s *stackScanState) getPtr() (p uintptr, conservative bool) {
+ for _, head := range []**stackWorkBuf{&s.buf, &s.cbuf} {
+ buf := *head
if buf == nil {
- // No more data.
- putempty((*workbuf)(unsafe.Pointer(s.freeBuf)))
- s.freeBuf = nil
- return 0
+ // Never had any data.
+ continue
+ }
+ if buf.nobj == 0 {
+ if s.freeBuf != nil {
+ // Free old freeBuf.
+ putempty((*workbuf)(unsafe.Pointer(s.freeBuf)))
+ }
+ // Move buf to the freeBuf.
+ s.freeBuf = buf
+ buf = buf.next
+ *head = buf
+ if buf == nil {
+ // No more data in this list.
+ continue
+ }
}
+ buf.nobj--
+ return buf.obj[buf.nobj], head == &s.cbuf
+ }
+ // No more data in either list.
+ if s.freeBuf != nil {
+ putempty((*workbuf)(unsafe.Pointer(s.freeBuf)))
+ s.freeBuf = nil
}
- buf.nobj--
- return buf.obj[buf.nobj]
+ return 0, false
}
// addObject adds a stack object at addr of type typ to the set of stack objects.