// describe p and the high 4 bits describe p+ptrSize.
//
// The 4 bits for each word are:
-// 0001 - bitBoundary: this is the start of an object
+// 0001 - not used
// 0010 - bitMarked: this object has been marked by GC
// tt00 - word type bits, as in a type bitmap.
//
heapBitsWidth = 4
heapBitmapScale = ptrSize * (8 / heapBitsWidth) // number of data bytes per heap bitmap byte
- bitBoundary = 1
bitMarked = 2
typeShift = 2
)
// heapBitsForObject returns the base address for the heap object
// containing the address p, along with the heapBits for base.
-// If p does not point into a heap object, heapBitsForObject returns base == 0.
-func heapBitsForObject(p uintptr) (base uintptr, hbits heapBits) {
+// If p does not point into a heap object,
+// return base == 0
+// otherwise return the base of the object.
+func heapBitsForObject(p uintptr) (base uintptr, hbits heapBits, s *mspan) {
if p < mheap_.arena_start || p >= mheap_.arena_used {
return
}
- // If heap bits for the pointer-sized word containing p have bitBoundary set,
- // then we know this is the base of the object, and we can stop now.
- // This handles the case where p is the base and, due to rounding
- // when looking up the heap bits, also the case where p points beyond
- // the base but still into the first pointer-sized word of the object.
- hbits = heapBitsForAddr(p)
- if hbits.isBoundary() {
- base = p &^ (ptrSize - 1)
- return
- }
-
- // Otherwise, p points into the middle of an object.
+ // p points into the heap, but possibly to the middle of an object.
// Consult the span table to find the block beginning.
// TODO(rsc): Factor this out.
k := p >> _PageShift
x := k
x -= mheap_.arena_start >> _PageShift
- s := h_spans[x]
+ s = h_spans[x]
if s == nil || pageID(k) < s.start || p >= s.limit || s.state != mSpanInUse {
if s == nil || s.state == _MSpanStack {
// If s is nil, the virtual address has never been part of the heap.
base += n * s.elemsize
}
- if base == p {
- print("runtime: failed to find block beginning for ", hex(p), " s=", hex(s.start*_PageSize), " s.limit=", hex(s.limit), "\n")
- throw("failed to find block beginning")
- }
-
// Now that we know the actual base, compute heapBits to return to caller.
hbits = heapBitsForAddr(base)
- if !hbits.isBoundary() {
- throw("missing boundary at computed object start")
- }
return
}
+// prefetch the bits.
+func (h heapBits) prefetch() {
+ prefetchnta(uintptr(unsafe.Pointer((h.bitp))))
+}
+
// next returns the heapBits describing the next pointer-sized word in memory.
// That is, if h describes address p, h.next() describes p+ptrSize.
// Note that next does not modify h. The caller must record the result.
*h.bitp |= bitMarked << h.shift
}
-// isBoundary reports whether the heap bits have the boundary bit set.
-func (h heapBits) isBoundary() bool {
- return *h.bitp&(bitBoundary<<h.shift) != 0
-}
-
-// Note that there is no setBoundary or setBoundaryNonAtomic.
-// Boundaries are always in bulk, for the entire span.
-
// typeBits returns the heap bits' type bits.
func (h heapBits) typeBits() uint8 {
return (*h.bitp >> (h.shift + typeShift)) & typeMask
// initSpan initializes the heap bitmap for a span.
func (h heapBits) initSpan(size, n, total uintptr) {
- if size == ptrSize {
- // Only possible on 64-bit system, since minimum size is 8.
- // Set all nibbles to bitBoundary using uint64 writes.
- nbyte := n * ptrSize / heapBitmapScale
- nuint64 := nbyte / 8
- bitp := subtractb(h.bitp, nbyte-1)
- for i := uintptr(0); i < nuint64; i++ {
- const boundary64 = bitBoundary |
- bitBoundary<<4 |
- bitBoundary<<8 |
- bitBoundary<<12 |
- bitBoundary<<16 |
- bitBoundary<<20 |
- bitBoundary<<24 |
- bitBoundary<<28 |
- bitBoundary<<32 |
- bitBoundary<<36 |
- bitBoundary<<40 |
- bitBoundary<<44 |
- bitBoundary<<48 |
- bitBoundary<<52 |
- bitBoundary<<56 |
- bitBoundary<<60
-
- *(*uint64)(unsafe.Pointer(bitp)) = boundary64
- bitp = addb(bitp, 8)
- }
- return
- }
-
- if size*n < total {
- // To detect end of object during GC object scan,
- // add boundary just past end of last block.
- // The object scan knows to stop when it reaches
- // the end of the span, but in this case the object
- // ends before the end of the span.
- //
- // TODO(rsc): If the bitmap bits were going to be typeDead
- // otherwise, what's the point of this?
- // Can we delete this logic?
- n++
- }
- step := size / heapBitmapScale
- bitp := h.bitp
- for i := uintptr(0); i < n; i++ {
- *bitp = bitBoundary
- bitp = subtractb(bitp, step)
- }
-}
-
-// clearSpan clears the heap bitmap bytes for the span.
-func (h heapBits) clearSpan(size, n, total uintptr) {
if total%heapBitmapScale != 0 {
- throw("clearSpan: unaligned length")
+ throw("initSpan: unaligned length")
}
nbyte := total / heapBitmapScale
memclr(unsafe.Pointer(subtractb(h.bitp, nbyte-1)), nbyte)
bitp := h.bitp
for i := uintptr(0); i < n; i += 2 {
x := int(*bitp)
- if x&0x11 != 0x11 {
- throw("missing bitBoundary")
- }
+
if (x>>typeShift)&typeMask == typeDead {
x += (typeScalar - typeDead) << typeShift
}
bitp := h.bitp
step := size / heapBitmapScale
for i := uintptr(0); i < n; i++ {
- if *bitp&bitBoundary == 0 {
- throw("missing bitBoundary")
- }
x := *bitp
if (x>>typeShift)&typeMask == typeDead {
x += (typeScalar - typeDead) << typeShift
bitp := h.bitp
for i := uintptr(0); i < n; i += 2 {
x := int(*bitp)
- if x&(bitBoundary|bitBoundary<<4) != (bitBoundary | bitBoundary<<4) {
- throw("missing bitBoundary")
- }
-
switch typ := (x >> typeShift) & typeMask; typ {
case typeScalar:
x += (typeDead - typeScalar) << typeShift
step := size / heapBitmapScale
for i := uintptr(0); i < n; i++ {
x := int(*bitp)
- if x&bitBoundary == 0 {
- throw("missing bitBoundary")
- }
-
switch typ := (x >> typeShift) & typeMask; {
case typ == typeScalarCheckmarked && (x>>(4+typeShift))&typeMask != typeDead:
x += (typeScalar - typeScalarCheckmarked) << typeShift
if x&bitMarked != 0 {
x &^= bitMarked
} else {
- x = bitBoundary // clear marked bit, set type bits to typeDead
+ x = 0
f(base + i*size)
}
*bitp = uint8(x)
// From here till marked label marking the object as allocated
// and storing type info in the GC bitmap.
h := heapBitsForAddr(x)
- if debugMalloc && (*h.bitp>>h.shift)&0x0f != bitBoundary {
- println("runtime: bits =", (*h.bitp>>h.shift)&0x0f)
- throw("bad bits in markallocated")
- }
var ti, te uintptr
var ptrmask *uint8
ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask
}
if size == 2*ptrSize {
- *h.bitp = *ptrmask | bitBoundary
+ // h.shift is 0 for all sizes > ptrSize.
+ *h.bitp = *ptrmask
return
}
te = uintptr(typ.size) / ptrSize
te /= 2
}
// Copy pointer bitmask into the bitmap.
+ // TODO(rlh): add comment addressing the following concerns:
+ // If size > 2*ptrSize, is x guaranteed to be at least 2*ptrSize-aligned?
+ // And if type occupies and odd number of words, why are we only going through half
+ // of ptrmask and why don't we have to shift everything by 4 on odd iterations?
+
for i := uintptr(0); i < dataSize; i += 2 * ptrSize {
v := *(*uint8)(add(unsafe.Pointer(ptrmask), ti))
ti++
if ti == te {
ti = 0
}
- if i == 0 {
- v |= bitBoundary
- }
if i+ptrSize == dataSize {
v &^= typeMask << (4 + typeShift)
}
// Mark first word as bitAllocated.
// Mark word after last as typeDead.
- // TODO(rsc): Explain why we need to set this boundary.
- // Aren't the boundaries always set for the whole span?
- // Did unrollgcproc1 overwrite the boundary bit?
- // Is that okay?
- h := heapBitsForAddr(uintptr(v))
- *h.bitp |= bitBoundary << h.shift
if size0 < size {
h := heapBitsForAddr(uintptr(v) + size0)
*h.bitp &^= typeMask << typeShift
}
}
-// Scan the object b of size n, adding pointers to wbuf.
+// Scan the object b of size n bytes, adding pointers to wbuf.
// Return possibly new wbuf to use.
// If ptrmask != nil, it specifies where pointers are in b.
// If ptrmask == nil, the GC bitmap should be consulted.
// Find bits of the beginning of the object.
var hbits heapBits
+
if ptrmask == nil {
- b, hbits = heapBitsForObject(b)
+ var s *mspan
+ b, hbits, s = heapBitsForObject(b)
if b == 0 {
return
}
+ n = s.elemsize
if n == 0 {
- n = mheap_.arena_used - b
+ throw("scanobject n == 0")
}
}
for i := uintptr(0); i < n; i += ptrSize {
// dense mask (stack or data)
bits = (uintptr(*(*byte)(add(unsafe.Pointer(ptrmask), (i/ptrSize)/4))) >> (((i / ptrSize) % 4) * typeBitsWidth)) & typeMask
} else {
- // Check if we have reached end of span.
- // n is an overestimate of the size of the object.
- if (b+i)%_PageSize == 0 && h_spans[(b-arena_start)>>_PageShift] != h_spans[(b+i-arena_start)>>_PageShift] {
- break
- }
-
bits = uintptr(hbits.typeBits())
- if i > 0 && (hbits.isBoundary() || bits == typeDead) {
- break // reached beginning of the next object
+ if bits == typeDead {
+ break // no more pointers in this object
}
hbits = hbits.next()
}
}
// Mark the object.
- if obj, hbits := heapBitsForObject(obj); obj != 0 {
+ if obj, hbits, _ := heapBitsForObject(obj); obj != 0 {
greyobject(obj, b, i, hbits, gcw)
}
}
if !inheap(b) {
throw("shade: passed an address not in the heap")
}
- if obj, hbits := heapBitsForObject(b); obj != 0 {
+ if obj, hbits, _ := heapBitsForObject(b); obj != 0 {
// TODO: this would be a great place to put a check to see
// if we are harvesting and if we are then we should
// figure out why there is a call to shade when the