From: Keith Randall Date: Tue, 1 Jul 2014 02:48:08 +0000 (-0700) Subject: undo CL 104200047 / 318b04f28372 X-Git-Tag: go1.4beta1~1192 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=3cf83c182af504bcffb82f3fc78a0c8b0ffb3aaa;p=gostls13.git undo CL 104200047 / 318b04f28372 Breaks windows and race detector. TBR=rsc ««« original CL description runtime: stack allocator, separate from mallocgc In order to move malloc to Go, we need to have a separate stack allocator. If we run out of stack during malloc, malloc will not be available to allocate a new stack. Stacks are the last remaining FlagNoGC objects in the GC heap. Once they are out, we can get rid of the distinction between the allocated/blockboundary bits. (This will be in a separate change.) Fixes #7468 Fixes #7424 LGTM=rsc, dvyukov R=golang-codereviews, dvyukov, khr, dave, rsc CC=golang-codereviews https://golang.org/cl/104200047 »»» TBR=rsc CC=golang-codereviews https://golang.org/cl/101570044 --- diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h index 422df5405f..798c130ad5 100644 --- a/src/pkg/runtime/malloc.h +++ b/src/pkg/runtime/malloc.h @@ -116,12 +116,6 @@ enum MaxMHeapList = 1<<(20 - PageShift), // Maximum page length for fixed-size list in MHeap. HeapAllocChunk = 1<<20, // Chunk size for heap growth - // Per-P, per order stack segment cache size. - StackCacheSize = 32*1024, - // Number of orders that get caching. Order 0 is StackMin - // and each successive order is twice as large. - NumStackOrders = 3, - // Number of bits in page to span calculations (4k pages). // On Windows 64-bit we limit the arena to 32GB or 35 bits (see below for reason). // On other 64-bit platforms, we limit the arena to 128GB, or 37 bits. @@ -253,8 +247,8 @@ struct MStats // Statistics about allocation of low-level fixed-size structures. // Protected by FixAlloc locks. - uint64 stacks_inuse; // this number is included in heap_inuse above - uint64 stacks_sys; // always 0 in mstats + uint64 stacks_inuse; // bootstrap stacks + uint64 stacks_sys; uint64 mspan_inuse; // MSpan structures uint64 mspan_sys; uint64 mcache_inuse; // MCache structures @@ -311,13 +305,6 @@ struct MCacheList uint32 nlist; }; -typedef struct StackFreeList StackFreeList; -struct StackFreeList -{ - MLink *list; // linked list of free stacks - uintptr size; // total size of stacks in list -}; - // Per-thread (in Go, per-P) cache for small objects. // No locking needed because it is per-thread (per-P). struct MCache @@ -333,9 +320,6 @@ struct MCache // The rest is not accessed on every malloc. MSpan* alloc[NumSizeClasses]; // spans to allocate from MCacheList free[NumSizeClasses];// lists of explicitly freed objects - - StackFreeList stackcache[NumStackOrders]; - // Local allocator stats, flushed during GC. uintptr local_nlookup; // number of pointer lookups uintptr local_largefree; // bytes freed for large objects (>MaxSmallSize) @@ -346,7 +330,6 @@ struct MCache MSpan* runtime·MCache_Refill(MCache *c, int32 sizeclass); void runtime·MCache_Free(MCache *c, MLink *p, int32 sizeclass, uintptr size); void runtime·MCache_ReleaseAll(MCache *c); -void runtime·stackcache_clear(MCache *c); // MTypes describes the types of blocks allocated within a span. // The compression field describes the layout of the data. @@ -426,8 +409,7 @@ struct SpecialProfile // An MSpan is a run of pages. enum { - MSpanInUse = 0, // allocated for garbage collected heap - MSpanStack, // allocated for use by stack allocator + MSpanInUse = 0, MSpanFree, MSpanListHead, MSpanDead, @@ -543,9 +525,7 @@ extern MHeap runtime·mheap; void runtime·MHeap_Init(MHeap *h); MSpan* runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large, bool needzero); -MSpan* runtime·MHeap_AllocStack(MHeap *h, uintptr npage); void runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct); -void runtime·MHeap_FreeStack(MHeap *h, MSpan *s); MSpan* runtime·MHeap_Lookup(MHeap *h, void *v); MSpan* runtime·MHeap_LookupMaybe(MHeap *h, void *v); void runtime·MGetSizeClassInfo(int32 sizeclass, uintptr *size, int32 *npages, int32 *nobj); @@ -553,6 +533,7 @@ void* runtime·MHeap_SysAlloc(MHeap *h, uintptr n); void runtime·MHeap_MapBits(MHeap *h); void runtime·MHeap_MapSpans(MHeap *h); void runtime·MHeap_Scavenger(void); +void runtime·MHeap_SplitSpan(MHeap *h, MSpan *s); void* runtime·mallocgc(uintptr size, uintptr typ, uint32 flag); void* runtime·persistentalloc(uintptr size, uintptr align, uint64 *stat); diff --git a/src/pkg/runtime/mcache.c b/src/pkg/runtime/mcache.c index 92521a2c7a..13437a50cd 100644 --- a/src/pkg/runtime/mcache.c +++ b/src/pkg/runtime/mcache.c @@ -43,7 +43,6 @@ void runtime·freemcache(MCache *c) { runtime·MCache_ReleaseAll(c); - runtime·stackcache_clear(c); runtime·lock(&runtime·mheap); runtime·purgecachedstats(c); runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c); diff --git a/src/pkg/runtime/mcentral.c b/src/pkg/runtime/mcentral.c index 9291c24856..203558fca5 100644 --- a/src/pkg/runtime/mcentral.c +++ b/src/pkg/runtime/mcentral.c @@ -263,8 +263,6 @@ MCentral_Grow(MCentral *c) runtime·unlock(c); runtime·MGetSizeClassInfo(c->sizeclass, &size, &npages, &n); s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1); - if(s->next != nil || s->prev != nil) - runtime·throw("internal error: MSpan should not be in a list"); if(s == nil) { // TODO(rsc): Log out of memory runtime·lock(c); diff --git a/src/pkg/runtime/mem.go b/src/pkg/runtime/mem.go index 0fec501e7a..fa308b5d96 100644 --- a/src/pkg/runtime/mem.go +++ b/src/pkg/runtime/mem.go @@ -30,7 +30,7 @@ type MemStats struct { // Low-level fixed-size structure allocator statistics. // Inuse is bytes used now. // Sys is bytes obtained from system. - StackInuse uint64 // bytes used by stack allocator + StackInuse uint64 // bootstrap stacks StackSys uint64 MSpanInuse uint64 // mspan structures MSpanSys uint64 diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c index ff53c31d98..4ad8f3b08f 100644 --- a/src/pkg/runtime/mgc0.c +++ b/src/pkg/runtime/mgc0.c @@ -1252,12 +1252,12 @@ markroot(ParFor *desc, uint32 i) SpecialFinalizer *spf; s = allspans[spanidx]; - if(s->state != MSpanInUse) - continue; if(s->sweepgen != sg) { runtime·printf("sweep %d %d\n", s->sweepgen, sg); runtime·throw("gc: unswept span"); } + if(s->state != MSpanInUse) + continue; // The garbage collector ignores type pointers stored in MSpan.types: // - Compiler-generated types are stored outside of heap. // - The reflect package has runtime-generated types cached in its data structures. @@ -2124,7 +2124,6 @@ flushallmcaches(void) if(c==nil) continue; runtime·MCache_ReleaseAll(c); - runtime·stackcache_clear(c); } } @@ -2134,12 +2133,14 @@ runtime·updatememstats(GCStats *stats) M *mp; MSpan *s; int32 i; - uint64 smallfree; + uint64 stacks_inuse, smallfree; uint64 *src, *dst; if(stats) runtime·memclr((byte*)stats, sizeof(*stats)); + stacks_inuse = 0; for(mp=runtime·allm; mp; mp=mp->alllink) { + stacks_inuse += mp->stackinuse*FixedStack; if(stats) { src = (uint64*)&mp->gcstats; dst = (uint64*)stats; @@ -2148,6 +2149,7 @@ runtime·updatememstats(GCStats *stats) runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats)); } } + mstats.stacks_inuse = stacks_inuse; mstats.mcache_inuse = runtime·mheap.cachealloc.inuse; mstats.mspan_inuse = runtime·mheap.spanalloc.inuse; mstats.sys = mstats.heap_sys + mstats.stacks_sys + mstats.mspan_sys + @@ -2507,12 +2509,6 @@ runtime·ReadMemStats(MStats *stats) // Size of the trailing by_size array differs between Go and C, // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. runtime·memcopy(runtime·sizeof_C_MStats, stats, &mstats); - - // Stack numbers are part of the heap numbers, separate those out for user consumption - stats->stacks_sys = stats->stacks_inuse; - stats->heap_inuse -= stats->stacks_inuse; - stats->heap_sys -= stats->stacks_inuse; - g->m->gcing = 0; g->m->locks++; runtime·semrelease(&runtime·worldsema); diff --git a/src/pkg/runtime/mheap.c b/src/pkg/runtime/mheap.c index 62ae126f1e..961b32e504 100644 --- a/src/pkg/runtime/mheap.c +++ b/src/pkg/runtime/mheap.c @@ -9,16 +9,16 @@ // When a MSpan is in the heap free list, state == MSpanFree // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. // -// When a MSpan is allocated, state == MSpanInUse or MSpanStack +// When a MSpan is allocated, state == MSpanInUse // and heapmap(i) == span for all s->start <= i < s->start+s->npages. #include "runtime.h" #include "arch_GOARCH.h" #include "malloc.h" -static MSpan *MHeap_AllocSpanLocked(MHeap*, uintptr); -static void MHeap_FreeSpanLocked(MHeap*, MSpan*); +static MSpan *MHeap_AllocLocked(MHeap*, uintptr, int32); static bool MHeap_Grow(MHeap*, uintptr); +static void MHeap_FreeLocked(MHeap*, MSpan*); static MSpan *MHeap_AllocLarge(MHeap*, uintptr); static MSpan *BestFit(MSpan*, uintptr, MSpan*); @@ -165,38 +165,19 @@ MHeap_Reclaim(MHeap *h, uintptr npage) runtime·lock(h); } -// Allocate a new span of npage pages from the heap for GC'd memory +// Allocate a new span of npage pages from the heap // and record its size class in the HeapMap and HeapMapCache. -static MSpan* -mheap_alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large) +MSpan* +runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large, bool needzero) { MSpan *s; - if(g != g->m->g0) - runtime·throw("mheap_alloc not on M stack"); runtime·lock(h); - - // To prevent excessive heap growth, before allocating n pages - // we need to sweep and reclaim at least n pages. - if(!h->sweepdone) - MHeap_Reclaim(h, npage); - - // transfer stats from cache to global mstats.heap_alloc += g->m->mcache->local_cachealloc; g->m->mcache->local_cachealloc = 0; - - s = MHeap_AllocSpanLocked(h, npage); + s = MHeap_AllocLocked(h, npage, sizeclass); if(s != nil) { - // Record span info, because gc needs to be - // able to map interior pointer to containing span. - s->state = MSpanInUse; - s->ref = 0; - s->sizeclass = sizeclass; - s->elemsize = (sizeclass==0 ? s->npages<types.compression = MTypes_Empty; - s->sweepgen = h->sweepgen; - - // update stats, sweep lists + mstats.heap_inuse += npage<m->ptrarg[0]; - g->m->ptrarg[0] = nil; - s = mheap_alloc(h, g->m->scalararg[0], g->m->scalararg[1], g->m->scalararg[2]); - g->m->ptrarg[0] = s; - - runtime·gogo(&gp->sched); -} - -MSpan* -runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large, bool needzero) -{ - MSpan *s; - - // Don't do any operations that lock the heap on the G stack. - // It might trigger stack growth, and the stack growth code needs - // to be able to allocate heap. - if(g == g->m->g0) { - s = mheap_alloc(h, npage, sizeclass, large); - } else { - g->m->ptrarg[0] = h; - g->m->scalararg[0] = npage; - g->m->scalararg[1] = sizeclass; - g->m->scalararg[2] = large; - runtime·mcall(mheap_alloc_m); - s = g->m->ptrarg[0]; - g->m->ptrarg[0] = nil; - } if(s != nil) { if(needzero && s->needzero) runtime·memclr((byte*)(s->start<npages<m->g0) - runtime·throw("mheap_allocstack not on M stack"); - runtime·lock(h); - s = MHeap_AllocSpanLocked(h, npage); - if(s != nil) { - s->state = MSpanStack; - s->ref = 0; - mstats.stacks_inuse += s->npages<sweepdone) + MHeap_Reclaim(h, npage); + // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { if(!runtime·MSpanList_IsEmpty(&h->free[n])) { @@ -303,13 +232,13 @@ HaveSpan: if(s->npages < npage) runtime·throw("MHeap_AllocLocked - bad npages"); runtime·MSpanList_Remove(s); - if(s->next != nil || s->prev != nil) - runtime·throw("still in list"); - if(s->npreleased > 0) { + runtime·atomicstore(&s->sweepgen, h->sweepgen); + s->state = MSpanInUse; + mstats.heap_idle -= s->npages<npreleased<npreleased > 0) runtime·SysUsed((void*)(s->start<npages<npreleased<npreleased = 0; - } + s->npreleased = 0; if(s->npages > npage) { // Trim extra and put it back in the heap. @@ -323,25 +252,22 @@ HaveSpan: h->spans[p] = t; h->spans[p+t->npages-1] = t; t->needzero = s->needzero; - s->state = MSpanStack; // prevent coalescing with s - t->state = MSpanStack; - MHeap_FreeSpanLocked(h, t); - t->unusedsince = s->unusedsince; // preserve age (TODO: wrong: t is possibly merged and/or deallocated at this point) - s->state = MSpanFree; + runtime·atomicstore(&t->sweepgen, h->sweepgen); + t->state = MSpanInUse; + MHeap_FreeLocked(h, t); + t->unusedsince = s->unusedsince; // preserve age } s->unusedsince = 0; + // Record span info, because gc needs to be + // able to map interior pointer to containing span. + s->sizeclass = sizeclass; + s->elemsize = (sizeclass==0 ? s->npages<types.compression = MTypes_Empty; p = s->start; p -= ((uintptr)h->arena_start>>PageShift); for(n=0; nspans[p+n] = s; - - mstats.heap_inuse += npage<start << PageShift); - if(s->next != nil || s->prev != nil) - runtime·throw("still in list"); return s; } @@ -412,7 +338,7 @@ MHeap_Grow(MHeap *h, uintptr npage) h->spans[p + s->npages - 1] = s; runtime·atomicstore(&s->sweepgen, h->sweepgen); s->state = MSpanInUse; - MHeap_FreeSpanLocked(h, s); + MHeap_FreeLocked(h, s); return true; } @@ -454,83 +380,34 @@ runtime·MHeap_LookupMaybe(MHeap *h, void *v) } // Free the span back into the heap. -static void -mheap_free(MHeap *h, MSpan *s, int32 acct) +void +runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct) { - if(g != g->m->g0) - runtime·throw("mheap_free not on M stack"); runtime·lock(h); mstats.heap_alloc += g->m->mcache->local_cachealloc; g->m->mcache->local_cachealloc = 0; + mstats.heap_inuse -= s->npages<npages<types.compression = MTypes_Empty; - MHeap_FreeSpanLocked(h, s); + MHeap_FreeLocked(h, s); runtime·unlock(h); } static void -mheap_free_m(G *gp) -{ - MHeap *h; - MSpan *s; - - h = g->m->ptrarg[0]; - s = g->m->ptrarg[1]; - g->m->ptrarg[0] = nil; - g->m->ptrarg[1] = nil; - mheap_free(h, s, g->m->scalararg[0]); - runtime·gogo(&gp->sched); -} - -void -runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct) -{ - if(g == g->m->g0) { - mheap_free(h, s, acct); - } else { - g->m->ptrarg[0] = h; - g->m->ptrarg[1] = s; - g->m->scalararg[0] = acct; - runtime·mcall(mheap_free_m); - } -} - -void -runtime·MHeap_FreeStack(MHeap *h, MSpan *s) -{ - if(g != g->m->g0) - runtime·throw("mheap_freestack not on M stack"); - s->needzero = 1; - runtime·lock(h); - MHeap_FreeSpanLocked(h, s); - mstats.stacks_inuse -= s->npages<state) { - case MSpanStack: - break; - case MSpanInUse: - if(s->ref != 0 || s->sweepgen != h->sweepgen) { - runtime·printf("MHeap_FreeSpanLocked - span %p ptr %p ref %d sweepgen %d/%d\n", - s, s->start<ref, s->sweepgen, h->sweepgen); - runtime·throw("MHeap_FreeSpanLocked - invalid free"); - } - break; - default: - runtime·throw("MHeap_FreeSpanLocked - invalid span state"); - break; + s->types.compression = MTypes_Empty; + + if(s->state != MSpanInUse || s->ref != 0 || s->sweepgen != h->sweepgen) { + runtime·printf("MHeap_FreeLocked - span %p ptr %p state %d ref %d sweepgen %d/%d\n", + s, s->start<state, s->ref, s->sweepgen, h->sweepgen); + runtime·throw("MHeap_FreeLocked - invalid free"); } - mstats.heap_inuse -= s->npages<npages<state = MSpanFree; runtime·MSpanList_Remove(s); @@ -542,7 +419,7 @@ MHeap_FreeSpanLocked(MHeap *h, MSpan *s) // Coalesce with earlier, later spans. p = s->start; p -= (uintptr)h->arena_start >> PageShift; - if(p > 0 && (t = h->spans[p-1]) != nil && t->state != MSpanInUse && t->state != MSpanStack) { + if(p > 0 && (t = h->spans[p-1]) != nil && t->state != MSpanInUse) { s->start = t->start; s->npages += t->npages; s->npreleased = t->npreleased; // absorb released pages @@ -553,7 +430,7 @@ MHeap_FreeSpanLocked(MHeap *h, MSpan *s) t->state = MSpanDead; runtime·FixAlloc_Free(&h->spanalloc, t); } - if((p+s->npages)*sizeof(h->spans[0]) < h->spans_mapped && (t = h->spans[p+s->npages]) != nil && t->state != MSpanInUse && t->state != MSpanStack) { + if((p+s->npages)*sizeof(h->spans[0]) < h->spans_mapped && (t = h->spans[p+s->npages]) != nil && t->state != MSpanInUse) { s->npages += t->npages; s->npreleased += t->npreleased; s->needzero |= t->needzero; @@ -621,15 +498,6 @@ scavenge(int32 k, uint64 now, uint64 limit) } } -static void -scavenge_m(G *gp) -{ - runtime·lock(&runtime·mheap); - scavenge(g->m->scalararg[0], g->m->scalararg[1], g->m->scalararg[2]); - runtime·unlock(&runtime·mheap); - runtime·gogo(&gp->sched); -} - static FuncVal forcegchelperv = {(void(*)(void))forcegchelper}; // Release (part of) unused memory to OS. @@ -639,7 +507,7 @@ void runtime·MHeap_Scavenger(void) { MHeap *h; - uint64 tick, forcegc, limit; + uint64 tick, now, forcegc, limit; int64 unixnow; int32 k; Note note, *notep; @@ -678,11 +546,9 @@ runtime·MHeap_Scavenger(void) runtime·printf("scvg%d: GC forced\n", k); runtime·lock(h); } + now = runtime·nanotime(); + scavenge(k, now, limit); runtime·unlock(h); - g->m->scalararg[0] = k; - g->m->scalararg[1] = runtime·nanotime(); - g->m->scalararg[2] = limit; - runtime·mcall(scavenge_m); } } @@ -690,11 +556,9 @@ void runtime∕debug·freeOSMemory(void) { runtime·gc(2); // force GC and do eager sweep - - g->m->scalararg[0] = -1; - g->m->scalararg[1] = ~(uintptr)0; - g->m->scalararg[2] = 0; - runtime·mcall(scavenge_m); + runtime·lock(&runtime·mheap); + scavenge(-1, ~(uintptr)0, 0); + runtime·unlock(&runtime·mheap); } // Initialize a new span with the given start and npages. @@ -977,3 +841,92 @@ runtime·freeallspecials(MSpan *span, void *p, uintptr size) runtime·throw("can't explicitly free an object with a finalizer"); } } + +// Split an allocated span into two equal parts. +void +runtime·MHeap_SplitSpan(MHeap *h, MSpan *s) +{ + MSpan *t; + MCentral *c; + uintptr i; + uintptr npages; + PageID p; + + if(s->state != MSpanInUse) + runtime·throw("MHeap_SplitSpan on a free span"); + if(s->sizeclass != 0 && s->ref != 1) + runtime·throw("MHeap_SplitSpan doesn't have an allocated object"); + npages = s->npages; + + // remove the span from whatever list it is in now + if(s->sizeclass > 0) { + // must be in h->central[x].empty + c = &h->central[s->sizeclass]; + runtime·lock(c); + runtime·MSpanList_Remove(s); + runtime·unlock(c); + runtime·lock(h); + } else { + // must be in h->busy/busylarge + runtime·lock(h); + runtime·MSpanList_Remove(s); + } + // heap is locked now + + if(npages == 1) { + // convert span of 1 PageSize object to a span of 2 PageSize/2 objects. + s->ref = 2; + s->sizeclass = runtime·SizeToClass(PageSize/2); + s->elemsize = PageSize/2; + } else { + // convert span of n>1 pages into two spans of n/2 pages each. + if((s->npages & 1) != 0) + runtime·throw("MHeap_SplitSpan on an odd size span"); + + // compute position in h->spans + p = s->start; + p -= (uintptr)h->arena_start >> PageShift; + + // Allocate a new span for the first half. + t = runtime·FixAlloc_Alloc(&h->spanalloc); + runtime·MSpan_Init(t, s->start, npages/2); + t->limit = (byte*)((t->start + npages/2) << PageShift); + t->state = MSpanInUse; + t->elemsize = npages << (PageShift - 1); + t->sweepgen = s->sweepgen; + if(t->elemsize <= MaxSmallSize) { + t->sizeclass = runtime·SizeToClass(t->elemsize); + t->ref = 1; + } + + // the old span holds the second half. + s->start += npages/2; + s->npages = npages/2; + s->elemsize = npages << (PageShift - 1); + if(s->elemsize <= MaxSmallSize) { + s->sizeclass = runtime·SizeToClass(s->elemsize); + s->ref = 1; + } + + // update span lookup table + for(i = p; i < p + npages/2; i++) + h->spans[i] = t; + } + + // place the span into a new list + if(s->sizeclass > 0) { + runtime·unlock(h); + c = &h->central[s->sizeclass]; + runtime·lock(c); + // swept spans are at the end of the list + runtime·MSpanList_InsertBack(&c->empty, s); + runtime·unlock(c); + } else { + // Swept spans are at the end of lists. + if(s->npages < nelem(h->free)) + runtime·MSpanList_InsertBack(&h->busy[s->npages], s); + else + runtime·MSpanList_InsertBack(&h->busylarge, s); + runtime·unlock(h); + } +} diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c index ccaddee2eb..04808f2c50 100644 --- a/src/pkg/runtime/proc.c +++ b/src/pkg/runtime/proc.c @@ -152,7 +152,6 @@ runtime·schedinit(void) runtime·precisestack = true; // haveexperiment("precisestack"); runtime·symtabinit(); - runtime·stackinit(); runtime·mallocinit(); mcommoninit(g->m); @@ -1927,7 +1926,7 @@ gfput(P *p, G *gp) runtime·throw("gfput: bad stacksize"); } top = (Stktop*)gp->stackbase; - if(stksize != FixedStack) { + if(top->malloced) { // non-standard stack size - free it. runtime·stackfree(gp, (void*)gp->stack0, top); gp->stack0 = 0; diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index 4350f280d7..0f630abbf2 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -146,6 +146,13 @@ enum { PtrSize = sizeof(void*), }; +enum +{ + // Per-M stack segment cache size. + StackCacheSize = 32, + // Global <-> per-M stack segment cache transfer batch size. + StackCacheBatch = 16, +}; /* * structures */ @@ -319,6 +326,10 @@ struct M M* schedlink; uint32 machport; // Return address for Mach IPC (OS X) MCache* mcache; + int32 stackinuse; + uint32 stackcachepos; + uint32 stackcachecnt; + void* stackcache[StackCacheSize]; G* lockedg; uintptr createstack[32];// Stack that created this thread. uint32 freglo[16]; // D[i] lsb and F[i] @@ -335,8 +346,6 @@ struct M bool (*waitunlockf)(G*, void*); void* waitlock; uintptr forkstackguard; - uintptr scalararg[4]; // scalar argument/return for mcall - void* ptrarg[4]; // pointer argument/return for mcall #ifdef GOOS_windows void* thread; // thread handle // these are here because they are too large to be on the stack @@ -419,6 +428,7 @@ struct Stktop uint8* argp; // pointer to arguments in old frame bool panic; // is this frame the top of a panic? + bool malloced; }; struct SigTab { @@ -856,7 +866,6 @@ int32 runtime·funcarglen(Func*, uintptr); int32 runtime·funcspdelta(Func*, uintptr); int8* runtime·funcname(Func*); int32 runtime·pcdatavalue(Func*, int32, uintptr); -void runtime·stackinit(void); void* runtime·stackalloc(G*, uint32); void runtime·stackfree(G*, void*, Stktop*); void runtime·shrinkstack(G*); diff --git a/src/pkg/runtime/stack.c b/src/pkg/runtime/stack.c index 96ba515c68..a07042111e 100644 --- a/src/pkg/runtime/stack.c +++ b/src/pkg/runtime/stack.c @@ -21,163 +21,76 @@ enum StackDebug = 0, StackFromSystem = 0, // allocate stacks from system memory instead of the heap StackFaultOnFree = 0, // old stacks are mapped noaccess to detect use after free - - StackCache = 1, }; -// Global pool of spans that have free stacks. -// Stacks are assigned an order according to size. -// order = log_2(size/FixedStack) -// There is a free list for each order. -static MSpan stackpool[NumStackOrders]; -static Lock stackpoolmu; -// TODO: one lock per order? - -void -runtime·stackinit(void) -{ - int32 i; - - for(i = 0; i < NumStackOrders; i++) - runtime·MSpanList_Init(&stackpool[i]); -} - -// Allocates a stack from the free pool. Must be called with -// stackpoolmu held. -static MLink* -poolalloc(uint8 order) -{ - MSpan *list; - MSpan *s; - MLink *x; - uintptr i; - - list = &stackpool[order]; - s = list->next; - if(s == list) { - // no free stacks. Allocate another span worth. - s = runtime·MHeap_AllocStack(&runtime·mheap, StackCacheSize >> PageShift); - if(s == nil) - runtime·throw("out of memory"); - for(i = 0; i < StackCacheSize; i += FixedStack << order) { - x = (MLink*)((s->start << PageShift) + i); - x->next = s->freelist; - s->freelist = x; - } - } - x = s->freelist; - s->freelist = x->next; - s->ref--; - if(s->ref == 0) { - // all stacks in s are allocated. - runtime·MSpanList_Remove(s); - } - return x; -} - -// Adds stack x to the free pool. Must be called with stackpoolmu held. -static void -poolfree(MLink *x, uint8 order) +typedef struct StackCacheNode StackCacheNode; +struct StackCacheNode { - MSpan *s; + StackCacheNode *next; + void* batch[StackCacheBatch-1]; +}; - s = runtime·MHeap_Lookup(&runtime·mheap, x); - x->next = s->freelist; - s->freelist = x; - if(s->ref == 0) { - // s now has a free stack - runtime·MSpanList_Insert(&stackpool[order], s); - } - s->ref++; - if(s->ref == (StackCacheSize / FixedStack) >> order) { - // span is completely free - return to heap - runtime·MSpanList_Remove(s); - runtime·MHeap_FreeStack(&runtime·mheap, s); - } -} +static StackCacheNode *stackcache; +static Lock stackcachemu; -// stackcacherefill/stackcacherelease implement a global pool of stack segments. -// The pool is required to prevent unlimited growth of per-thread caches. +// stackcacherefill/stackcacherelease implement a global cache of stack segments. +// The cache is required to prevent unlimited growth of per-thread caches. static void -stackcacherefill(MCache *c, uint8 order) +stackcacherefill(void) { - MLink *x, *list; - uintptr size; - - if(StackDebug >= 1) - runtime·printf("stackcacherefill order=%d\n", order); - - // Grab some stacks from the global cache. - // Grab half of the allowed capacity (to prevent thrashing). - list = nil; - size = 0; - runtime·lock(&stackpoolmu); - while(size < StackCacheSize/2) { - x = poolalloc(order); - x->next = list; - list = x; - size += FixedStack << order; - } - runtime·unlock(&stackpoolmu); - - c->stackcache[order].list = list; - c->stackcache[order].size = size; + StackCacheNode *n; + int32 i, pos; + + runtime·lock(&stackcachemu); + n = stackcache; + if(n) + stackcache = n->next; + runtime·unlock(&stackcachemu); + if(n == nil) { + n = (StackCacheNode*)runtime·SysAlloc(FixedStack*StackCacheBatch, &mstats.stacks_sys); + if(n == nil) + runtime·throw("out of memory (stackcacherefill)"); + for(i = 0; i < StackCacheBatch-1; i++) + n->batch[i] = (byte*)n + (i+1)*FixedStack; + } + pos = g->m->stackcachepos; + for(i = 0; i < StackCacheBatch-1; i++) { + g->m->stackcache[pos] = n->batch[i]; + pos = (pos + 1) % StackCacheSize; + } + g->m->stackcache[pos] = n; + pos = (pos + 1) % StackCacheSize; + g->m->stackcachepos = pos; + g->m->stackcachecnt += StackCacheBatch; } static void -stackcacherelease(MCache *c, uint8 order) +stackcacherelease(void) { - MLink *x, *y; - uintptr size; - - if(StackDebug >= 1) - runtime·printf("stackcacherelease order=%d\n", order); - x = c->stackcache[order].list; - size = c->stackcache[order].size; - runtime·lock(&stackpoolmu); - while(size > StackCacheSize/2) { - y = x->next; - poolfree(x, order); - x = y; - size -= FixedStack << order; - } - runtime·unlock(&stackpoolmu); - c->stackcache[order].list = x; - c->stackcache[order].size = size; -} - -void -runtime·stackcache_clear(MCache *c) -{ - uint8 order; - MLink *x, *y; - - if(StackDebug >= 1) - runtime·printf("stackcache clear\n"); - runtime·lock(&stackpoolmu); - for(order = 0; order < NumStackOrders; order++) { - x = c->stackcache[order].list; - while(x != nil) { - y = x->next; - poolfree(x, order); - x = y; - } - c->stackcache[order].list = nil; - c->stackcache[order].size = 0; - } - runtime·unlock(&stackpoolmu); + StackCacheNode *n; + uint32 i, pos; + + pos = (g->m->stackcachepos - g->m->stackcachecnt) % StackCacheSize; + n = (StackCacheNode*)g->m->stackcache[pos]; + pos = (pos + 1) % StackCacheSize; + for(i = 0; i < StackCacheBatch-1; i++) { + n->batch[i] = g->m->stackcache[pos]; + pos = (pos + 1) % StackCacheSize; + } + g->m->stackcachecnt -= StackCacheBatch; + runtime·lock(&stackcachemu); + n->next = stackcache; + stackcache = n; + runtime·unlock(&stackcachemu); } void* runtime·stackalloc(G *gp, uint32 n) { - uint8 order; - uint32 n2; + uint32 pos; void *v; + bool malloced; Stktop *top; - MLink *x; - MSpan *s; - MCache *c; // Stackalloc must be called on scheduler stack, so that we // never try to grow the stack during the code that stackalloc runs. @@ -197,58 +110,41 @@ runtime·stackalloc(G *gp, uint32 n) return v; } - // Small stacks are allocated with a fixed-size free-list allocator. - // If we need a stack of a bigger size, we fall back on allocating - // a dedicated span. - if(StackCache && n < FixedStack << NumStackOrders) { - order = 0; - n2 = n; - while(n2 > FixedStack) { - order++; - n2 >>= 1; + // Minimum-sized stacks are allocated with a fixed-size free-list allocator, + // but if we need a stack of a bigger size, we fall back on malloc + // (assuming that inside malloc all the stack frames are small, + // so that we do not deadlock). + malloced = true; + if(n == FixedStack || g->m->mallocing) { + if(n != FixedStack) { + runtime·printf("stackalloc: in malloc, size=%d want %d\n", FixedStack, n); + runtime·throw("stackalloc"); } - c = g->m->mcache; - if(c == nil) { - // This can happen in the guts of exitsyscall or - // procresize. Just get a stack from the global pool. - runtime·lock(&stackpoolmu); - x = poolalloc(order); - runtime·unlock(&stackpoolmu); - } else { - x = c->stackcache[order].list; - if(x == nil) { - stackcacherefill(c, order); - x = c->stackcache[order].list; - } - c->stackcache[order].list = x->next; - c->stackcache[order].size -= n; - } - v = (byte*)x; - } else { - s = runtime·MHeap_AllocStack(&runtime·mheap, (n+PageSize-1) >> PageShift); - if(s == nil) - runtime·throw("out of memory"); - v = (byte*)(s->start<m->stackcachecnt == 0) + stackcacherefill(); + pos = g->m->stackcachepos; + pos = (pos - 1) % StackCacheSize; + v = g->m->stackcache[pos]; + g->m->stackcachepos = pos; + g->m->stackcachecnt--; + g->m->stackinuse++; + malloced = false; + } else + v = runtime·mallocgc(n, 0, FlagNoProfiling|FlagNoGC|FlagNoZero|FlagNoInvokeGC); + top = (Stktop*)((byte*)v+n-sizeof(Stktop)); runtime·memclr((byte*)top, sizeof(*top)); - if(StackDebug >= 1) - runtime·printf(" allocated %p\n", v); + top->malloced = malloced; return v; } void runtime·stackfree(G *gp, void *v, Stktop *top) { - uint8 order; - uintptr n, n2; - MSpan *s; - MLink *x; - MCache *c; + uint32 pos; + uintptr n; n = (uintptr)(top+1) - (uintptr)v; - if(n & (n-1)) - runtime·throw("stack not a power of 2"); if(StackDebug >= 1) runtime·printf("stackfree %p %d\n", v, (int32)n); gp->stacksize -= n; @@ -259,34 +155,19 @@ runtime·stackfree(G *gp, void *v, Stktop *top) runtime·SysFree(v, n, &mstats.stacks_sys); return; } - if(StackCache && n < FixedStack << NumStackOrders) { - order = 0; - n2 = n; - while(n2 > FixedStack) { - order++; - n2 >>= 1; - } - x = (MLink*)v; - c = g->m->mcache; - if(c == nil) { - runtime·lock(&stackpoolmu); - poolfree(x, order); - runtime·unlock(&stackpoolmu); - } else { - if(c->stackcache[order].size >= StackCacheSize) - stackcacherelease(c, order); - x->next = c->stackcache[order].list; - c->stackcache[order].list = x; - c->stackcache[order].size += n; - } - } else { - s = runtime·MHeap_Lookup(&runtime·mheap, v); - if(s->state != MSpanStack) { - runtime·printf("%p %p\n", s->start<malloced) { + runtime·free(v); + return; } + if(n != FixedStack) + runtime·throw("stackfree: bad fixed size"); + if(g->m->stackcachecnt == StackCacheSize) + stackcacherelease(); + pos = g->m->stackcachepos; + g->m->stackcache[pos] = v; + g->m->stackcachepos = (pos + 1) % StackCacheSize; + g->m->stackcachecnt++; + g->m->stackinuse--; } // Called from runtime·lessstack when returning from a function which @@ -718,6 +599,7 @@ copystack(G *gp, uintptr nframes, uintptr newsize) uintptr oldsize, used; AdjustInfo adjinfo; Stktop *oldtop, *newtop; + bool malloced; if(gp->syscallstack != 0) runtime·throw("can't handle stack copy in syscall yet"); @@ -731,9 +613,10 @@ copystack(G *gp, uintptr nframes, uintptr newsize) newstk = runtime·stackalloc(gp, newsize); newbase = newstk + newsize; newtop = (Stktop*)(newbase - sizeof(Stktop)); + malloced = newtop->malloced; if(StackDebug >= 1) - runtime·printf("copystack gp=%p [%p %p]/%d -> [%p %p]/%d\n", gp, oldstk, oldbase, (int32)oldsize, newstk, newbase, (int32)newsize); + runtime·printf("copystack [%p %p]/%d -> [%p %p]/%d\n", oldstk, oldbase, (int32)oldsize, newstk, newbase, (int32)newsize); USED(oldsize); // adjust pointers in the to-be-copied frames @@ -748,6 +631,7 @@ copystack(G *gp, uintptr nframes, uintptr newsize) // copy the stack (including Stktop) to the new location runtime·memmove(newbase - used, oldbase - used, used); + newtop->malloced = malloced; // Swap out old stack for new one gp->stackbase = (uintptr)newtop; @@ -908,7 +792,7 @@ runtime·newstack(void) top = (Stktop*)(stk+framesize-sizeof(*top)); if(StackDebug >= 1) { - runtime·printf("\t-> new stack gp=%p [%p, %p]\n", gp, stk, top); + runtime·printf("\t-> new stack [%p, %p]\n", stk, top); } top->stackbase = gp->stackbase; @@ -997,6 +881,7 @@ runtime·shrinkstack(G *gp) int32 nframes; byte *oldstk, *oldbase; uintptr used, oldsize, newsize; + MSpan *span; if(!runtime·copystack) return; @@ -1010,14 +895,53 @@ runtime·shrinkstack(G *gp) if(used >= oldsize / 4) return; // still using at least 1/4 of the segment. - if(gp->syscallstack != (uintptr)nil) // TODO: can we handle this case? - return; + // To shrink to less than 1/2 a page, we need to copy. + if(newsize < PageSize/2) { + if(gp->syscallstack != (uintptr)nil) // TODO: can we handle this case? + return; #ifdef GOOS_windows - if(gp->m != nil && gp->m->libcallsp != 0) - return; + if(gp->m != nil && gp->m->libcallsp != 0) + return; #endif - nframes = copyabletopsegment(gp); - if(nframes == -1) + nframes = copyabletopsegment(gp); + if(nframes == -1) + return; + copystack(gp, nframes, newsize); return; - copystack(gp, nframes, newsize); + } + + // To shrink a stack of one page size or more, we can shrink it + // without copying. Just deallocate the lower half. + span = runtime·MHeap_LookupMaybe(&runtime·mheap, oldstk); + if(span == nil) + return; // stack allocated outside heap. Can't shrink it. Can happen if stack is allocated while inside malloc. TODO: shrink by copying? + if(span->elemsize != oldsize) + runtime·throw("span element size doesn't match stack size"); + if((uintptr)oldstk != span->start << PageShift) + runtime·throw("stack not at start of span"); + + if(StackDebug) + runtime·printf("shrinking stack in place %p %X->%X\n", oldstk, oldsize, newsize); + + // new stack guard for smaller stack + gp->stackguard = (uintptr)oldstk + newsize + StackGuard; + gp->stackguard0 = (uintptr)oldstk + newsize + StackGuard; + if(gp->stack0 == (uintptr)oldstk) + gp->stack0 = (uintptr)oldstk + newsize; + gp->stacksize -= oldsize - newsize; + + // Free bottom half of the stack. + if(runtime·debug.efence || StackFromSystem) { + if(runtime·debug.efence || StackFaultOnFree) + runtime·SysFault(oldstk, newsize); + else + runtime·SysFree(oldstk, newsize, &mstats.stacks_sys); + return; + } + // First, we trick malloc into thinking + // we allocated the stack as two separate half-size allocs. Then the + // free() call does the rest of the work for us. + runtime·MSpan_EnsureSwept(span); + runtime·MHeap_SplitSpan(&runtime·mheap, span); + runtime·free(oldstk); } diff --git a/src/pkg/runtime/stack_test.go b/src/pkg/runtime/stack_test.go index 424a15b3e5..f0c599ac5d 100644 --- a/src/pkg/runtime/stack_test.go +++ b/src/pkg/runtime/stack_test.go @@ -281,52 +281,3 @@ func TestDeferPtrs(t *testing.T) { defer set(&y, 42) growStack() } - -// use about n KB of stack -func useStack(n int) { - if n == 0 { - return - } - var b [1024]byte // makes frame about 1KB - useStack(n - 1 + int(b[99])) -} - -func growing(c chan int, done chan struct{}) { - for n := range c { - useStack(n) - done <- struct{}{} - } - done <- struct{}{} -} - -func TestStackCache(t *testing.T) { - // Allocate a bunch of goroutines and grow their stacks. - // Repeat a few times to test the stack cache. - const ( - R = 4 - G = 200 - S = 5 - ) - for i := 0; i < R; i++ { - var reqchans [G]chan int - done := make(chan struct{}) - for j := 0; j < G; j++ { - reqchans[j] = make(chan int) - go growing(reqchans[j], done) - } - for s := 0; s < S; s++ { - for j := 0; j < G; j++ { - reqchans[j] <- 1 << uint(s) - } - for j := 0; j < G; j++ { - <-done - } - } - for j := 0; j < G; j++ { - close(reqchans[j]) - } - for j := 0; j < G; j++ { - <-done - } - } -}