From: Jan Ziak <0xe2.0x9a.0x9b@gmail.com> Date: Thu, 10 Jan 2013 20:45:46 +0000 (-0500) Subject: runtime: interpret type information during garbage collection X-Git-Tag: go1.1rc2~1427 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=9204eb4d3ce3ba49cce7d24f4e373d230f865848;p=gostls13.git runtime: interpret type information during garbage collection R=rsc, dvyukov, remyoudompheng, dave, minux.ma, bradfitz CC=golang-dev https://golang.org/cl/6945069 --- diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h index 916b473a00..e6138cfaec 100644 --- a/src/pkg/runtime/malloc.h +++ b/src/pkg/runtime/malloc.h @@ -489,5 +489,6 @@ enum // defined in mgc0.go void runtime·gc_m_ptr(Eface*); +void runtime·gc_itab_ptr(Eface*); void runtime·memorydump(void); diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c index c7c12b49e8..b612e6216a 100644 --- a/src/pkg/runtime/mgc0.c +++ b/src/pkg/runtime/mgc0.c @@ -10,6 +10,8 @@ #include "stack.h" #include "mgc0.h" #include "race.h" +#include "type.h" +#include "typekind.h" enum { Debug = 0, @@ -21,6 +23,11 @@ enum { handoffThreshold = 4, IntermediateBufferCapacity = 64, + + // Bits in type information + PRECISE = 1, + LOOP = 2, + PC_BITS = PRECISE | LOOP, }; // Bits in per-word bitmap. @@ -188,6 +195,9 @@ struct BufferList static BufferList *bufferList; static Lock lock; +static Type *itabtype; + +static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); // flushptrbuf moves data from the PtrTarget buffer to the work buffer. // The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned, @@ -210,10 +220,10 @@ static Lock lock; // flushptrbuf // (2nd part, mark and enqueue) static void -flushptrbuf(PtrTarget *ptrbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, BitTarget *bitbuf) +flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, BitTarget *bitbuf) { byte *p, *arena_start, *obj; - uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti; + uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; MSpan *s; PageID k; Obj *wp; @@ -227,7 +237,9 @@ flushptrbuf(PtrTarget *ptrbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_ wbuf = *_wbuf; nobj = *_nobj; - ptrbuf_end = ptrbuf + n; + ptrbuf_end = *ptrbufpos; + n = ptrbuf_end - ptrbuf; + *ptrbufpos = ptrbuf; // If buffer is nearly full, get a new one. if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) { @@ -326,8 +338,7 @@ flushptrbuf(PtrTarget *ptrbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_ if((bits & (bitAllocated|bitMarked)) != bitAllocated) continue; - *bitbufpos = (BitTarget){obj, ti, bitp, shift}; - bitbufpos++; + *bitbufpos++ = (BitTarget){obj, ti, bitp, shift}; } runtime·lock(&lock); @@ -378,6 +389,13 @@ flushptrbuf(PtrTarget *ptrbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_ // Program that scans the whole block and treats every block element as a potential pointer static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; +// Local variables of a program fragment or loop +typedef struct Frame Frame; +struct Frame { + uintptr count, elemsize, b; + uintptr *loop_or_ret; +}; + // scanblock scans a block of n bytes starting at pointer b for references // to other objects, scanning any it finds recursively until there are no // unscanned objects left. Instead of using an explicit recursion, it keeps @@ -392,22 +410,17 @@ static void scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) { byte *b, *arena_start, *arena_used; - uintptr n, i, end_b; + uintptr n, i, end_b, elemsize, ti, objti, count; + uintptr *pc, precise_type, nominal_size; void *obj; - - // TODO(atom): to be expanded in a next CL - struct Frame {uintptr count, b; uintptr *loop_or_ret;}; - struct Frame stack_top; - - uintptr *pc; - + Type *t; + Slice *sliceptr; + Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; BufferList *scanbuffers; - PtrTarget *ptrbuf, *ptrbuf_end; + PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; BitTarget *bitbuf; - - PtrTarget *ptrbufpos; - - // End of local variable declarations. + Eface *eface; + Iface *iface; if(sizeof(Workbuf) % PageSize != 0) runtime·throw("scanblock: size of Workbuf is suboptimal"); @@ -416,6 +429,11 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) arena_start = runtime·mheap.arena_start; arena_used = runtime·mheap.arena_used; + stack_ptr = stack+nelem(stack)-1; + + precise_type = false; + nominal_size = 0; + // Allocate ptrbuf, bitbuf { runtime·lock(&lock); @@ -445,50 +463,209 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) runtime·printf("scanblock %p %D\n", b, (int64)n); } - // TODO(atom): to be replaced in a next CL - pc = defaultProg; + // TODO(atom): to be expanded in a next CL + if(ti != 0) { + pc = (uintptr*)(ti & ~(uintptr)PC_BITS); + precise_type = (ti & PRECISE); + stack_top.elemsize = pc[0]; + if(!precise_type) + nominal_size = pc[0]; + if(ti & LOOP) { + stack_top.count = 0; // 0 means an infinite number of iterations + stack_top.loop_or_ret = pc+1; + } else { + stack_top.count = 1; + } + } else { + pc = defaultProg; + } pc++; stack_top.b = (uintptr)b; end_b = (uintptr)b + n - PtrSize; - next_instr: - // TODO(atom): to be expanded in a next CL + for(;;) { + obj = nil; + objti = 0; switch(pc[0]) { + case GC_PTR: + obj = *(void**)(stack_top.b + pc[1]); + objti = pc[2]; + pc += 3; + break; + + case GC_SLICE: + sliceptr = (Slice*)(stack_top.b + pc[1]); + if(sliceptr->cap != 0) { + obj = sliceptr->array; + objti = pc[2] | PRECISE | LOOP; + } + pc += 3; + break; + + case GC_APTR: + obj = *(void**)(stack_top.b + pc[1]); + pc += 2; + break; + + case GC_STRING: + obj = *(void**)(stack_top.b + pc[1]); + pc += 2; + break; + + case GC_EFACE: + eface = (Eface*)(stack_top.b + pc[1]); + pc += 2; + if(eface->type != nil && (eface->data >= arena_start && eface->data < arena_used)) { + t = eface->type; + if(t->size <= sizeof(void*)) { + if((t->kind & KindNoPointers)) + break; + + obj = eface->data; + if((t->kind & ~KindNoPointers) == KindPtr) + objti = (uintptr)((PtrType*)t)->elem->gc; + } else { + obj = eface->data; + objti = (uintptr)t->gc; + } + } + break; + + case GC_IFACE: + iface = (Iface*)(stack_top.b + pc[1]); + pc += 2; + if(iface->tab == nil) + break; + + // iface->tab + if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) { + *ptrbufpos++ = (PtrTarget){iface->tab, (uintptr)itabtype->gc}; + if(ptrbufpos == ptrbuf_end) + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + } + + // iface->data + if(iface->data >= arena_start && iface->data < arena_used) { + t = iface->tab->type; + if(t->size <= sizeof(void*)) { + if((t->kind & KindNoPointers)) + break; + + obj = iface->data; + if((t->kind & ~KindNoPointers) == KindPtr) + objti = (uintptr)((PtrType*)t)->elem->gc; + } else { + obj = iface->data; + objti = (uintptr)t->gc; + } + } + break; + case GC_DEFAULT_PTR: - while(true) { - i = stack_top.b; - if(i > end_b) - goto next_block; + while((i = stack_top.b) <= end_b) { stack_top.b += PtrSize; - obj = *(byte**)i; if(obj >= arena_start && obj < arena_used) { - *ptrbufpos = (PtrTarget){obj, 0}; - ptrbufpos++; + *ptrbufpos++ = (PtrTarget){obj, 0}; if(ptrbufpos == ptrbuf_end) - goto flush_buffers; + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + } + } + goto next_block; + + case GC_END: + if(--stack_top.count != 0) { + // Next iteration of a loop if possible. + elemsize = stack_top.elemsize; + stack_top.b += elemsize; + if(stack_top.b + elemsize <= end_b+PtrSize) { + pc = stack_top.loop_or_ret; + continue; + } + i = stack_top.b; + } else { + // Stack pop if possible. + if(stack_ptr+1 < stack+nelem(stack)) { + pc = stack_top.loop_or_ret; + stack_top = *(++stack_ptr); + continue; + } + i = (uintptr)b + nominal_size; + } + if(!precise_type) { + // Quickly scan [b+i,b+n) for possible pointers. + for(; i<=end_b; i+=PtrSize) { + if(*(byte**)i != nil) { + // Found a value that may be a pointer. + // Do a rescan of the entire block. + enqueue((Obj){b, n, 0}, &wbuf, &wp, &nobj); + break; + } } } + goto next_block; + + case GC_ARRAY_START: + i = stack_top.b + pc[1]; + count = pc[2]; + elemsize = pc[3]; + pc += 4; + + // Stack push. + *stack_ptr-- = stack_top; + stack_top = (Frame){count, elemsize, i, pc}; + continue; + + case GC_ARRAY_NEXT: + if(--stack_top.count != 0) { + stack_top.b += stack_top.elemsize; + pc = stack_top.loop_or_ret; + } else { + // Stack pop. + stack_top = *(++stack_ptr); + pc += 1; + } + continue; + + case GC_CALL: + // Stack push. + *stack_ptr-- = stack_top; + stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*return address*/}; + pc = (uintptr*)pc[2]; // target of the CALL instruction + continue; + + case GC_MAP_PTR: + // TODO(atom): to be expanded in a next CL. Same as GC_APTR for now. + obj = *(void**)(stack_top.b + pc[1]); + pc += 3; + break; + + case GC_REGION: + // TODO(atom): to be expanded in a next CL. Same as GC_APTR for now. + obj = (void*)(stack_top.b + pc[1]); + pc += 4; + break; default: runtime·throw("scanblock: invalid GC instruction"); return; } - flush_buffers: - flushptrbuf(ptrbuf, ptrbufpos-ptrbuf, &wp, &wbuf, &nobj, bitbuf); - ptrbufpos = ptrbuf; - goto next_instr; + if(obj >= arena_start && obj < arena_used) { + *ptrbufpos++ = (PtrTarget){obj, objti}; + if(ptrbufpos == ptrbuf_end) + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + } + } next_block: // Done scanning [b, b+n). Prepare for the next iteration of - // the loop by setting b, n to the parameters for the next block. + // the loop by setting b, n, ti to the parameters for the next block. if(nobj == 0) { - flushptrbuf(ptrbuf, ptrbufpos-ptrbuf, &wp, &wbuf, &nobj, bitbuf); - ptrbufpos = ptrbuf; + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); if(nobj == 0) { if(!keepworking) { @@ -509,6 +686,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) --wp; b = wp->p; n = wp->n; + ti = wp->ti; nobj--; } @@ -1271,6 +1449,7 @@ gc(struct gc_args *args) GCStats stats; M *mp; uint32 i; + Eface eface; runtime·semacquire(&runtime·worldsema); if(!args->force && mstats.heap_alloc < mstats.next_gc) { @@ -1301,6 +1480,12 @@ gc(struct gc_args *args) work.sweepfor = runtime·parforalloc(MaxGcproc); m->locks--; + if(itabtype == nil) { + // get C pointer to the Go type "itab" + runtime·gc_itab_ptr(&eface); + itabtype = ((PtrType*)eface.type)->elem; + } + work.nwait = 0; work.ndone = 0; work.debugmarkdone = 0; diff --git a/src/pkg/runtime/mgc0.go b/src/pkg/runtime/mgc0.go index a7ddaf0a7c..b150546622 100644 --- a/src/pkg/runtime/mgc0.go +++ b/src/pkg/runtime/mgc0.go @@ -8,3 +8,8 @@ package runtime func gc_m_ptr(ret *interface{}) { *ret = (*m)(nil) } + +// Called from C. Returns the Go type *itab. +func gc_itab_ptr(ret *interface{}) { + *ret = (*itab)(nil) +}