t = (Type*)((Eface*)typ.data-1);
if(t->kind&KindNoPointers)
- ret = mallocgc(t->size, RefNoPointers, 1);
+ ret = mallocgc(t->size, RefNoPointers, 1, 1);
else
ret = mal(t->size);
FLUSH(&ret);
size = n*t->size;
if(t->kind&KindNoPointers)
- ret = mallocgc(size, RefNoPointers, 1);
+ ret = mallocgc(size, RefNoPointers, 1, 1);
else
ret = mal(size);
FLUSH(&ret);
// Small objects are allocated from the per-thread cache's free lists.
// Large objects (> 32 kB) are allocated straight from the heap.
void*
-mallocgc(uintptr size, uint32 refflag, int32 dogc)
+mallocgc(uintptr size, uint32 refflag, int32 dogc, int32 zeroed)
{
int32 sizeclass;
MCache *c;
sizeclass = SizeToClass(size);
size = class_to_size[sizeclass];
c = m->mcache;
- v = MCache_Alloc(c, sizeclass, size);
+ v = MCache_Alloc(c, sizeclass, size, zeroed);
if(v == nil)
throw("out of memory");
mstats.alloc += size;
void*
malloc(uintptr size)
{
- return mallocgc(size, 0, 0);
+ return mallocgc(size, 0, 0, 1);
}
// Free the object whose base pointer is v.
// Small object.
c = m->mcache;
size = class_to_size[sizeclass];
+ if(size > sizeof(uintptr))
+ ((uintptr*)v)[1] = 1; // mark as "needs to be zeroed"
runtime_memclr(v, size);
mstats.alloc -= size;
mstats.by_size[sizeclass].nfree++;
*base = p + i*n;
if(size)
*size = n;
- nobj = (s->npages << PageShift) / (n + RefcountOverhead);
- if((byte*)s->gcref < p || (byte*)(s->gcref+nobj) > p+(s->npages<<PageShift)) {
- printf("odd span state=%d span=%p base=%p sizeclass=%d n=%D size=%D npages=%D\n",
- s->state, s, p, s->sizeclass, (uint64)nobj, (uint64)n, (uint64)s->npages);
- printf("s->base sizeclass %d v=%p base=%p gcref=%p blocksize=%D nobj=%D size=%D end=%p end=%p\n",
- s->sizeclass, v, p, s->gcref, (uint64)s->npages<<PageShift,
- (uint64)nobj, (uint64)n, s->gcref + nobj, p+(s->npages<<PageShift));
- throw("bad gcref");
+
+ // good for error checking, but expensive
+ if(0) {
+ nobj = (s->npages << PageShift) / (n + RefcountOverhead);
+ if((byte*)s->gcref < p || (byte*)(s->gcref+nobj) > p+(s->npages<<PageShift)) {
+ printf("odd span state=%d span=%p base=%p sizeclass=%d n=%D size=%D npages=%D\n",
+ s->state, s, p, s->sizeclass, (uint64)nobj, (uint64)n, (uint64)s->npages);
+ printf("s->base sizeclass %d v=%p base=%p gcref=%p blocksize=%D nobj=%D size=%D end=%p end=%p\n",
+ s->sizeclass, v, p, s->gcref, (uint64)s->npages<<PageShift,
+ (uint64)nobj, (uint64)n, s->gcref + nobj, p+(s->npages<<PageShift));
+ throw("bad gcref");
+ }
}
if(ref)
*ref = &s->gcref[i];
void*
mal(uint32 n)
{
- return mallocgc(n, 0, 1);
+ return mallocgc(n, 0, 1, 1);
}
// Stack allocator uses malloc/free most of the time,
unlock(&stacks);
return v;
}
- v = malloc(n);
+ v = mallocgc(n, 0, 0, 0);
if(!mlookup(v, nil, nil, &ref))
throw("stackalloc mlookup");
*ref = RefStack;
FuncType *ft;
int32 i, nret;
Type *t;
-
+
if(obj.type == nil) {
printf("runtime.SetFinalizer: first argument is nil interface\n");
throw:
ft = (FuncType*)finalizer.type;
if(ft->dotdotdot || ft->in.len != 1 || *(Type**)ft->in.array != obj.type)
goto badfunc;
-
+
// compute size needed for return parameters
for(i=0; i<ft->out.len; i++) {
t = ((Type**)ft->out.array)[i];
// Allocating and freeing a large object uses the page heap
// directly, bypassing the MCache and MCentral free lists.
//
+// The small objects on the MCache and MCentral free lists
+// may or may not be zeroed. They are zeroed if and only if
+// the second word of the object is zero. The spans in the
+// page heap are always zeroed. When a span full of objects
+// is returned to the page heap, the objects that need to be
+// are zeroed first. There are two main benefits to delaying the
+// zeroing this way:
+//
+// 1. stack frames allocated from the small object lists
+// can avoid zeroing altogether.
+// 2. the cost of zeroing when reusing a small object is
+// charged to the mutator, not the garbage collector.
+//
// This C code was written with an eye toward translating to Go
// in the future. Methods have the form Type_Method(Type *t, ...).
-
typedef struct FixAlloc FixAlloc;
typedef struct MCentral MCentral;
typedef struct MHeap MHeap;
uint64 size;
};
-void* MCache_Alloc(MCache *c, int32 sizeclass, uintptr size);
+void* MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed);
void MCache_Free(MCache *c, void *p, int32 sizeclass, uintptr size);
// span lookup
MHeapMap map;
MHeapMapCache mapcache;
-
+
// range of addresses we might see in the heap
byte *min;
byte *max;
MSpan* MHeap_Lookup(MHeap *h, PageID p);
MSpan* MHeap_LookupMaybe(MHeap *h, PageID p);
-void* mallocgc(uintptr size, uint32 flag, int32 dogc);
+void* mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed);
int32 mlookup(void *v, byte **base, uintptr *size, uint32 **ref);
void gc(int32 force);
RefNone, // no references
RefSome, // some references
RefFinalize, // ready to be finalized
- RefNoPointers = 0x80000000U, // flag - no pointers here
+ RefNoPointers = 0x80000000U, // flag - no pointers here
+ RefHasFinalizer = 0x40000000U, // flag - has finalizer
};
#include "malloc.h"
void*
-MCache_Alloc(MCache *c, int32 sizeclass, uintptr size)
+MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed)
{
MCacheList *l;
MLink *first, *v;
// v is zeroed except for the link pointer
// that we used above; zero that.
v->next = nil;
+ if(zeroed) {
+ // block is zeroed iff second word is zero ...
+ if(size > sizeof(uintptr) && ((uintptr*)v)[1] != 0)
+ runtime_memclr((byte*)v, size);
+ else {
+ // ... except for the link pointer
+ // that we used above; zero that.
+ v->next = nil;
+ }
+ }
return v;
}
MSpan *s;
PageID page;
MLink *p, *next;
+ int32 size;
// Find span for v.
page = (uintptr)v >> PageShift;
// If s is completely freed, return it to the heap.
if(--s->ref == 0) {
+ size = class_to_size[c->sizeclass];
MSpanList_Remove(s);
- // Freed blocks are zeroed except for the link pointer.
- // Zero the link pointers so that the page is all zero.
+ // The second word of each freed block indicates
+ // whether it needs to be zeroed. The first word
+ // is the link pointer and must always be cleared.
for(p=s->freelist; p; p=next) {
next = p->next;
- p->next = nil;
+ if(size > sizeof(uintptr) && ((uintptr*)p)[1] != 0)
+ runtime_memclr((byte*)p, size);
+ else
+ p->next = nil;
}
s->freelist = nil;
- c->nfree -= (s->npages << PageShift) / class_to_size[c->sizeclass];
+ c->nfree -= (s->npages << PageShift) / size;
unlock(c);
MHeap_Free(&mheap, s);
lock(c);
addfintab(Fintab *t, void *k, void *fn, int32 nret)
{
int32 i, j;
-
+
i = (uintptr)k % (uintptr)t->max;
for(j=0; j<t->max; j++) {
if(t->key[i] == nil) {
{
int32 i, j;
void *v;
-
+
if(t->max == 0)
return nil;
i = (uintptr)k % (uintptr)t->max;
{
Fintab newtab;
int32 i;
+ uint32 *ref;
+ byte *base;
+
+ if(!mlookup(p, &base, nil, &ref) || p != base)
+ throw("addfinalizer on invalid pointer");
+ if(f == nil) {
+ if(*ref & RefHasFinalizer) {
+ getfinalizer(p, 1, nil);
+ *ref &= ~RefHasFinalizer;
+ }
+ return;
+ }
+
+ if(*ref & RefHasFinalizer)
+ throw("double finalizer");
+ *ref |= RefHasFinalizer;
if(fintab.nkey >= fintab.max/2+fintab.max/4) {
// keep table at most 3/4 full:
// allocate new table and rehash.
-
+
runtime_memclr((byte*)&newtab, sizeof newtab);
newtab.max = fintab.max;
if(newtab.max == 0)
// otherwise just rehash into table of same size.
newtab.max *= 3;
}
-
- newtab.key = mallocgc(newtab.max*sizeof newtab.key[0], RefNoPointers, 0);
- newtab.val = mallocgc(newtab.max*sizeof newtab.val[0], 0, 0);
-
+
+ newtab.key = mallocgc(newtab.max*sizeof newtab.key[0], RefNoPointers, 0, 1);
+ newtab.val = mallocgc(newtab.max*sizeof newtab.val[0], 0, 0, 1);
+
for(i=0; i<fintab.max; i++) {
void *k;
-
+
k = fintab.key[i];
if(k != nil && k != (void*)-1)
addfintab(&newtab, k, fintab.val[i].fn, fintab.val[i].nret);
free(fintab.val);
fintab = newtab;
}
-
- addfintab(&fintab, p, f, nret);
+
+ addfintab(&fintab, p, f, nret);
}
+// get finalizer; if del, delete finalizer.
+// caller is responsible for updating RefHasFinalizer bit.
void*
getfinalizer(void *p, bool del, int32 *nret)
{
int32 off;
void *obj;
uintptr size;
- uint32 *ref;
+ uint32 *refp, ref;
void **vp;
int64 i;
obj = vp[i];
if(obj == nil || (byte*)obj < mheap.min || (byte*)obj >= mheap.max)
continue;
- if(mlookup(obj, &obj, &size, &ref)) {
- if(*ref == RefFree || *ref == RefStack)
- continue;
-
- // If marked for finalization already, some other finalization-ready
- // object has a pointer: turn off finalization until that object is gone.
- // This means that cyclic finalizer loops never get collected,
- // so don't do that.
-
- if(*ref == (RefNone|RefNoPointers) || *ref == (RefFinalize|RefNoPointers)) {
- *ref = RefSome|RefNoPointers;
- continue;
- }
- if(*ref == RefNone || *ref == RefFinalize) {
+ if(mlookup(obj, &obj, &size, &refp)) {
+ ref = *refp;
+ switch(ref & ~(RefNoPointers|RefHasFinalizer)) {
+ case RefFinalize:
+ // If marked for finalization already, some other finalization-ready
+ // object has a pointer: turn off finalization until that object is gone.
+ // This means that cyclic finalizer loops never get collected,
+ // so don't do that.
+ /* fall through */
+ case RefNone:
if(Debug > 1)
printf("%d found at %p: ", depth, &vp[i]);
- *ref = RefSome;
- scanblock(depth+1, obj, size);
+ *refp = RefSome | (ref & (RefNoPointers|RefHasFinalizer));
+ if(!(ref & RefNoPointers))
+ scanblock(depth+1, obj, size);
+ break;
}
}
}
uint32 gcref;
gcref = *gcrefp;
- switch(gcref) {
+ switch(gcref & ~(RefNoPointers|RefHasFinalizer)) {
default:
throw("bad 'ref count'");
case RefFree:
case RefStack:
break;
case RefNone:
- case RefNone|RefNoPointers:
- if(pass == 0 && getfinalizer(p, 0, nil)) {
+ if(pass == 0 && (gcref & RefHasFinalizer)) {
// Tentatively mark as finalizable.
// Make sure anything it points at will not be collected.
if(Debug > 0)
printf("maybe finalize %p+%D\n", p, n);
- *gcrefp = RefFinalize | (gcref&RefNoPointers);
+ *gcrefp = RefFinalize | RefHasFinalizer | (gcref&RefNoPointers);
scanblock(100, p, n);
} else if(pass == 1) {
if(Debug > 0)
}
break;
case RefFinalize:
- case RefFinalize|RefNoPointers:
if(pass != 1)
throw("sweepspan pass 0 RefFinalize");
if(pfinq < efinq) {
pfinq->p = p;
pfinq->nret = 0;
pfinq->fn = getfinalizer(p, 1, &pfinq->nret);
+ gcref &= ~RefHasFinalizer;
if(pfinq->fn == nil)
throw("getfinalizer inconsistency");
pfinq++;
}
// Reset for next mark+sweep.
- *gcrefp = RefNone | (gcref&RefNoPointers);
+ *gcrefp = RefNone | (gcref&(RefNoPointers|RefHasFinalizer));
break;
case RefSome:
- case RefSome|RefNoPointers:
// Reset for next mark+sweep.
if(pass == 1)
- *gcrefp = RefNone | (gcref&RefNoPointers);
+ *gcrefp = RefNone | (gcref&(RefNoPointers|RefHasFinalizer));
break;
}
}
// Sweep all the spans marking blocks to be finalized.
for(s = mheap.allspans; s != nil; s = s->allnext)
sweepspan(s, 0);
-
+
// Sweep again queueing finalizers and freeing the others.
for(s = mheap.allspans; s != nil; s = s->allnext)
sweepspan(s, 1);
mstats.next_gc = mstats.inuse_pages+mstats.inuse_pages*gcpercent/100;
}
m->gcing = 0;
-
+
// kick off goroutines to run queued finalizers
m->locks++; // disable gc during the mallocs in newproc
for(fp=finq; fp<pfinq; fp++) {
// when it is just in a register (R14 on amd64).
m->alllink = allm;
allm = m;
- m->g0 = malg(8192);
m->id = sched.mcount++;
if(libcgo_thread_start != nil) {
CgoThreadStart ts;
- // pthread_create will make us a stack,
- // so free the one malg made.
- stackfree(m->g0->stack0);
- m->g0->stack0 = nil;
- m->g0->stackguard = nil;
- m->g0->stackbase = nil;
+ // pthread_create will make us a stack.
+ m->g0 = malg(-1);
ts.m = m;
ts.g = m->g0;
ts.fn = mstart;
runcgo(libcgo_thread_start, &ts);
- } else
+ } else {
+ m->g0 = malg(8192);
newosproc(m, m->g0, m->g0->stackbase, mstart);
+ }
}
mnextg(m, g);
}
mcpy(top->fp, sp, args);
}
- stackfree((byte*)g1->stackguard - StackGuard);
+ stackfree(g1->stackguard - StackGuard);
g1->stackbase = old.stackbase;
g1->stackguard = old.stackguard;
frame += 1024; // for more functions, Stktop.
stk = stackalloc(frame);
+
//printf("newstack frame=%d args=%d morepc=%p morefp=%p gobuf=%p, %p newstk=%p\n", frame, args, m->morepc, m->morefp, g->sched.pc, g->sched.sp, stk);
g1 = m->curg;
byte *stk;
g = malloc(sizeof(G));
- stk = stackalloc(stacksize + StackGuard);
- g->stack0 = stk;
- g->stackguard = stk + StackGuard;
- g->stackbase = stk + StackGuard + stacksize;
+ if(stacksize >= 0) {
+ stk = stackalloc(stacksize + StackGuard);
+ g->stack0 = stk;
+ g->stackguard = stk + StackGuard;
+ g->stackbase = stk + StackGuard + stacksize - sizeof(Stktop);
+ runtime_memclr(g->stackbase, sizeof(Stktop));
+ }
return g;
}
void
newproc1(byte *fn, byte *argp, int32 narg, int32 nret)
{
- byte *stk, *sp;
+ byte *sp;
G *newg;
int32 siz;
newg->alllink = allg;
allg = newg;
}
- stk = newg->stack0;
-
- newg->stackguard = stk+StackGuard;
-
- sp = stk + 4096 - 4*8;
- newg->stackbase = sp;
+ sp = newg->stackbase;
sp -= siz;
mcpy(sp, argp, narg);
ret.cap = cap;
if((t->elem->kind&KindNoPointers))
- ret.array = mallocgc(size, RefNoPointers, 1);
+ ret.array = mallocgc(size, RefNoPointers, 1, 1);
else
ret.array = mal(size);
FLUSH(&ret);
if(debug) {
- printf("makeslice(%S, %d, %d); ret=",
+ printf("makeslice(%S, %d, %d); ret=",
*t->string, nel, cap);
·printslice(ret);
}