};
WorkData runtime·work;
-// Is _cgo_allocate linked into the binary?
+// Is address b in the known heap. If it doesn't have a valid gcmap
+// returns false. For example pointers into stacks will return false.
static bool
-have_cgo_allocate(void)
+inheap(byte *b)
{
- extern byte go·weak·runtime·_cgo_allocate_internal[1];
- return go·weak·runtime·_cgo_allocate_internal != nil;
+ MSpan *s;
+ pageID k;
+ uintptr x;
+
+ if(b == nil || b < runtime·mheap.arena_start || b >= runtime·mheap.arena_used)
+ return false;
+ // Not a beginning of a block, consult span table to find the block beginning.
+ k = (uintptr)b>>PageShift;
+ x = k;
+ x -= (uintptr)runtime·mheap.arena_start>>PageShift;
+ s = runtime·mheap.spans[x];
+ if(s == nil || k < s->start || b >= s->limit || s->state != MSpanInUse)
+ return false;
+ return true;
}
-// scanblock scans a block of n bytes starting at pointer b for references
-// to other objects, scanning any it finds recursively until there are no
-// unscanned objects left. Instead of using an explicit recursion, it keeps
-// a work list in the Workbuf* structures and loops in the main function
-// body. Keeping an explicit work list is easier on the stack allocator and
-// more efficient.
+// Given an address in the heap return the relevant byte from the gcmap. This routine
+// can be used on addresses to the start of an object or to the interior of the an object.
static void
-scanblock(byte *b, uintptr n, byte *ptrmask)
+slottombits(byte *obj, Markbits *mbits)
{
- byte *obj, *obj0, *p, *arena_start, *arena_used, **wp, *scanbuf[8], *ptrbitp, *bitp;
- uintptr i, j, nobj, size, idx, x, off, scanbufpos, bits, xbits, shift;
- Workbuf *wbuf;
- Iface *iface;
- Eface *eface;
- Type *typ;
+ uintptr off;
+
+ off = (uintptr*)((uintptr)obj&~(PtrSize-1)) - (uintptr*)runtime·mheap.arena_start;
+ mbits->bitp = runtime·mheap.arena_start - off/wordsPerBitmapByte - 1;
+ mbits->shift = (off % wordsPerBitmapByte) * gcBits;
+ mbits->xbits = *mbits->bitp;
+ mbits->bits = (mbits->xbits >> mbits->shift) & bitMask;
+}
+
+// b is a pointer into the heap.
+// Find the start of the object refered to by b.
+// Set mbits to the associated bits from the bit map.
+static byte*
+objectstart(byte *b, Markbits *mbits)
+{
+ byte *obj, *p;
MSpan *s;
pageID k;
- bool keepworking;
+ uintptr x, size, idx;
- // Cache memory arena parameters in local vars.
- arena_start = runtime·mheap.arena_start;
- arena_used = runtime·mheap.arena_used;
+ obj = (byte*)((uintptr)b&~(PtrSize-1));
+ for(;;) {
+ slottombits(obj, mbits);
+ if(mbits->bits&bitBoundary == bitBoundary)
+ break;
+
+ // Not a beginning of a block, consult span table to find the block beginning.
+ k = (uintptr)obj>>PageShift;
+ x = k;
+ x -= (uintptr)runtime·mheap.arena_start>>PageShift;
+ s = runtime·mheap.spans[x];
+ if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse){
+ if(s->state == MSpanStack)
+ break; // This is legit.
+
+ // The following is catching some bugs left over from
+ // us not being rigerous about what data structures are
+ // hold valid pointers and different parts of the system
+ // considering different structures as roots. For example
+ // if there is a pointer into a stack that is left in
+ // a global data structure but that part of the runtime knows that
+ // those structures will be reinitialized before they are
+ // reused. Unfortunately the GC believes these roots are valid.
+ // Typically a stack gets moved and only the structures that part of
+ // the system knows are alive are updated. The span is freed
+ // after the stack copy and the pointer is still alive. This
+ // check is catching that bug but for now we will not throw,
+ // instead we will simply break out of this routine and depend
+ // on the caller to recognize that this pointer is not a valid
+ // heap pointer. I leave the code that catches the bug so that once
+ // resolved we can turn this check back on and throw.
+
+ //runtime·printf("Runtime: Span weird: obj=%p, k=%p", obj, k);
+ //if (s == nil)
+ // runtime·printf(" s=nil\n");
+ //else
+ // runtime·printf(" s->start=%p s->limit=%p, s->state=%d\n", s->start*PageSize, s->limit, s->state);
+ //runtime·throw("Blowup on weird span");
+ break; // We are not in a real block throw??
+ }
+ p = (byte*)((uintptr)s->start<<PageShift);
+ if(s->sizeclass != 0) {
+ size = s->elemsize;
+ idx = ((byte*)obj - p)/size;
+ p = p+idx*size;
+ }
+ if(p == obj) {
+ runtime·printf("runtime: failed to find block beginning for %p s=%p s->limit=%p\n",
+ p, s->start*PageSize, s->limit);
+ runtime·throw("failed to find block beginning");
+ }
+ obj = p;
+ }
+ // if size(obj.firstfield) < PtrSize, the &obj.secondfield could map to the boundary bit
+ // Clear any low bits to get to the start of the object.
+ // greyobject depends on this.
+ return obj;
+}
- wbuf = getempty(nil);
- nobj = wbuf->nobj;
- wp = &wbuf->obj[nobj];
- keepworking = b == nil;
- scanbufpos = 0;
- for(i = 0; i < nelem(scanbuf); i++)
- scanbuf[i] = nil;
+// obj is the start of an object with mark mbits.
+// If it isn't already marked, mark it and enqueue into workbuf.
+// Return possibly new workbuf to use.
+static Workbuf*
+greyobject(byte *obj, Markbits *mbits, Workbuf *wbuf)
+{
+ // obj should be start of allocation, and so must be at least pointer-aligned.
+ if(((uintptr)obj & (PtrSize-1)) != 0)
+ runtime·throw("greyobject: obj not pointer-aligned");
+
+ // If marked we have nothing to do.
+ if((mbits->bits&bitMarked) != 0)
+ return wbuf;
+
+ // Each byte of GC bitmap holds info for two words.
+ // If the current object is larger than two words, or if the object is one word
+ // but the object it shares the byte with is already marked,
+ // then all the possible concurrent updates are trying to set the same bit,
+ // so we can use a non-atomic update.
+ if((mbits->xbits&(bitMask|(bitMask<<gcBits))) != (bitBoundary|(bitBoundary<<gcBits)) || runtime·work.nproc == 1)
+ *mbits->bitp = mbits->xbits | (bitMarked<<mbits->shift);
+ else
+ runtime·atomicor8(mbits->bitp, bitMarked<<mbits->shift);
+
+ if(((mbits->xbits>>(mbits->shift+2))&BitsMask) == BitsDead)
+ return wbuf; // noscan object
+
+ // Queue the obj for scanning. The PREFETCH(obj) logic has been removed but
+ // seems like a nice optimization that can be added back in.
+ // There needs to be time between the PREFETCH and the use.
+ // Previously we put the obj in an 8 element buffer that is drained at a rate
+ // to give the PREFETCH time to do its work.
+ // Use of PREFETCHNTA might be more appropriate than PREFETCH
+
+ // If workbuf is full, obtain an empty one.
+ if(wbuf->nobj >= nelem(wbuf->obj)) {
+ wbuf = getempty(wbuf);
+ }
+
+ wbuf->obj[wbuf->nobj] = obj;
+ wbuf->nobj++;
+ return wbuf;
+}
- byte *obj, *arena_start, *arena_used, *ptrbitp, bits, cshift, cached;
- uintptr i;
- intptr ncached;
+// Scan the object b of size n, adding pointers to wbuf.
+// Return possibly new wbuf to use.
+// If ptrmask != nil, it specifies where pointers are in b.
+// If ptrmask == nil, the GC bitmap should be consulted.
+// In this case, n may be an overestimate of the size; the GC bitmap
+// must also be used to make sure the scan stops at the end of b.
+static Workbuf*
+scanobject(byte *b, uintptr n, byte *ptrmask, Workbuf *wbuf)
+{
++ byte *obj, *arena_start, *arena_used, *ptrbitp;
++ uintptr i, j;
++ int32 bits;
+ Markbits mbits;
+
+ arena_start = (byte*)runtime·mheap.arena_start;
+ arena_used = runtime·mheap.arena_used;
ptrbitp = nil;
- cached = 0;
- ncached = 0;
- cshift = mbits.shift; //(off % wordsPerBitmapByte) * gcBits;
- cached = *ptrbitp >> cshift;
- cached &= ~bitBoundary;
- ncached = (8 - cshift)/gcBits;
+ // Find bits of the beginning of the object.
+ if(ptrmask == nil) {
+ b = objectstart(b, &mbits);
+ ptrbitp = mbits.bitp; //arena_start - off/wordsPerBitmapByte - 1;
- if(ncached <= 0) {
- // Refill cache.
- cached = *--ptrbitp;
- ncached = 2;
- }
- bits = cached;
- cached >>= gcBits;
- ncached--;
-
- if((bits&bitBoundary) != 0)
+ }
+ for(i = 0; i < n; i += PtrSize) {
+ // Find bits for this word.
+ if(ptrmask != nil) {
+ // dense mask (stack or data)
+ bits = (ptrmask[(i/PtrSize)/4]>>(((i/PtrSize)%4)*BitsPerPointer))&BitsMask;
+ } else {
+ // Check if we have reached end of span.
+ if((((uintptr)b+i)%PageSize) == 0 &&
+ runtime·mheap.spans[(b-arena_start)>>PageShift] != runtime·mheap.spans[(b+i-arena_start)>>PageShift])
+ break;
+ // Consult GC bitmap.
- if(bits == BitsScalar || bits == BitsDead)
++ bits = *ptrbitp;
++ if(wordsPerBitmapByte != 2)
++ runtime·throw("alg doesn't work for wordsPerBitmapByte != 2");
++ j = ((uintptr)b+i)/PtrSize & 1;
++ bits >>= gcBits*j;
++ if(i == 0)
++ bits &= ~bitBoundary;
++ ptrbitp -= j;
++
++ if((bits&bitBoundary) != 0 && i != 0)
+ break; // reached beginning of the next object
+ bits = (bits>>2)&BitsMask;
+ if(bits == BitsDead)
+ break; // reached no-scan part of the object
+ }
+
- if(bits != BitsPointer)
++ if(bits <= BitsScalar) // Bits Scalar || BitsDead
+ continue;
++ if(bits != BitsPointer) {
++ runtime·printf("gc bits=%x\n", bits);
+ runtime·throw("unexpected garbage collection bits");
++ }
+
+ obj = *(byte**)(b+i);
+ // At this point we have extracted the next potential pointer.
+ // Check if it points into heap.
+ if(obj == nil || obj < arena_start || obj >= arena_used)
+ continue;
+ // Mark the object. return some important bits.
+ // We we combine the following two rotines we don't have to pass mbits or obj around.
+ obj = objectstart(obj, &mbits);
+ wbuf = greyobject(obj, &mbits, wbuf);
+ }
+ return wbuf;
+}
+
+// scanblock starts by scanning b as scanobject would.
+// If the gcphase is GCscan, that's all scanblock does.
+// Otherwise it traverses some fraction of the pointers it found in b, recursively.
+// As a special case, scanblock(nil, 0, nil) means to scan previously queued work,
+// stopping only when no work is left in the system.
+static void
+scanblock(byte *b, uintptr n, byte *ptrmask)
+{
+ Workbuf *wbuf;
+ bool keepworking;
+
+ wbuf = getpartial();
+ if(b != nil) {
+ wbuf = scanobject(b, n, ptrmask, wbuf);
+ if(runtime·gcphase == GCscan) {
+ putpartial(wbuf);
+ return;
+ }
+ }
+
+ keepworking = b == nil;
+
// ptrmask can have 2 possible values:
// 1. nil - obtain pointer mask from GC bitmap.
// 2. pointer to a compact mask (for stacks and data).