From 8d321625fdae77d7e4a8c1681fe90bd893b9cdd2 Mon Sep 17 00:00:00 2001 From: Dmitriy Vyukov Date: Fri, 14 Mar 2014 23:25:48 +0400 Subject: [PATCH] runtime: fix spans corruption The problem was that spans end up in wrong lists after split (e.g. in h->busy instead of h->central->empty). Also the span can be non-swept before split, I don't know what it can cause, but it's safer to operate on swept spans. Fixes #7544. R=golang-codereviews, rsc CC=golang-codereviews, khr https://golang.org/cl/76160043 --- src/pkg/runtime/mgc0.c | 2 +- src/pkg/runtime/mheap.c | 96 ++++++++++++++++++++++++++++------------- src/pkg/runtime/stack.c | 12 +----- 3 files changed, 70 insertions(+), 40 deletions(-) diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c index f9d6face09..b7f6adf394 100644 --- a/src/pkg/runtime/mgc0.c +++ b/src/pkg/runtime/mgc0.c @@ -1679,7 +1679,7 @@ runtime·MSpan_EnsureSwept(MSpan *s) // Caller must disable preemption. // Otherwise when this function returns the span can become unswept again // (if GC is triggered on another goroutine). - if(m->locks == 0 && m->mallocing == 0) + if(m->locks == 0 && m->mallocing == 0 && g != m->g0) runtime·throw("MSpan_EnsureSwept: m is not locked"); sg = runtime·mheap.sweepgen; diff --git a/src/pkg/runtime/mheap.c b/src/pkg/runtime/mheap.c index 9d8375a5bb..93cf83f163 100644 --- a/src/pkg/runtime/mheap.c +++ b/src/pkg/runtime/mheap.c @@ -846,48 +846,86 @@ void runtime·MHeap_SplitSpan(MHeap *h, MSpan *s) { MSpan *t; + MCentral *c; uintptr i; uintptr npages; PageID p; - if((s->npages & 1) != 0) - runtime·throw("MHeap_SplitSpan on an odd size span"); if(s->state != MSpanInUse) runtime·throw("MHeap_SplitSpan on a free span"); if(s->sizeclass != 0 && s->ref != 1) runtime·throw("MHeap_SplitSpan doesn't have an allocated object"); npages = s->npages; - runtime·lock(h); + // remove the span from whatever list it is in now + if(s->sizeclass > 0) { + // must be in h->central[x].empty + c = &h->central[s->sizeclass]; + runtime·lock(c); + runtime·MSpanList_Remove(s); + runtime·unlock(c); + runtime·lock(h); + } else { + // must be in h->busy/busylarge + runtime·lock(h); + runtime·MSpanList_Remove(s); + } + // heap is locked now + + if(npages == 1) { + // convert span of 1 PageSize object to a span of 2 PageSize/2 objects. + s->ref = 2; + s->sizeclass = runtime·SizeToClass(PageSize/2); + s->elemsize = PageSize/2; + } else { + // convert span of n>1 pages into two spans of n/2 pages each. + if((s->npages & 1) != 0) + runtime·throw("MHeap_SplitSpan on an odd size span"); + + // compute position in h->spans + p = s->start; + p -= (uintptr)h->arena_start >> PageShift; + + // Allocate a new span for the first half. + t = runtime·FixAlloc_Alloc(&h->spanalloc); + runtime·MSpan_Init(t, s->start, npages/2); + t->limit = (byte*)((t->start + npages/2) << PageShift); + t->state = MSpanInUse; + t->elemsize = npages << (PageShift - 1); + t->sweepgen = s->sweepgen; + if(t->elemsize <= MaxSmallSize) { + t->sizeclass = runtime·SizeToClass(t->elemsize); + t->ref = 1; + } - // compute position in h->spans - p = s->start; - p -= (uintptr)h->arena_start >> PageShift; + // the old span holds the second half. + s->start += npages/2; + s->npages = npages/2; + s->elemsize = npages << (PageShift - 1); + if(s->elemsize <= MaxSmallSize) { + s->sizeclass = runtime·SizeToClass(s->elemsize); + s->ref = 1; + } - // Allocate a new span for the first half. - t = runtime·FixAlloc_Alloc(&h->spanalloc); - runtime·MSpan_Init(t, s->start, npages/2); - t->limit = (byte*)((t->start + npages/2) << PageShift); - t->state = MSpanInUse; - t->elemsize = npages << (PageShift - 1); - t->sweepgen = s->sweepgen; - if(t->elemsize <= MaxSmallSize) { - t->sizeclass = runtime·SizeToClass(t->elemsize); - t->ref = 1; + // update span lookup table + for(i = p; i < p + npages/2; i++) + h->spans[i] = t; } - // the old span holds the second half. - s->start += npages/2; - s->npages = npages/2; - s->elemsize = npages << (PageShift - 1); - if(s->elemsize <= MaxSmallSize) { - s->sizeclass = runtime·SizeToClass(s->elemsize); - s->ref = 1; + // place the span into a new list + if(s->sizeclass > 0) { + runtime·unlock(h); + c = &h->central[s->sizeclass]; + runtime·lock(c); + // swept spans are at the end of the list + runtime·MSpanList_InsertBack(&c->empty, s); + runtime·unlock(c); + } else { + // Swept spans are at the end of lists. + if(s->npages < nelem(h->free)) + runtime·MSpanList_InsertBack(&h->busy[s->npages], s); + else + runtime·MSpanList_InsertBack(&h->busylarge, s); + runtime·unlock(h); } - - // update span lookup table - for(i = p; i < p + npages/2; i++) - h->spans[i] = t; - - runtime·unlock(h); } diff --git a/src/pkg/runtime/stack.c b/src/pkg/runtime/stack.c index c0b98634d7..c73991470e 100644 --- a/src/pkg/runtime/stack.c +++ b/src/pkg/runtime/stack.c @@ -838,15 +838,7 @@ runtime·shrinkstack(G *gp) // First, we trick malloc into thinking // we allocated the stack as two separate half-size allocs. Then the // free() call does the rest of the work for us. - if(oldsize == PageSize) { - // convert span of 1 PageSize object to a span of 2 - // PageSize/2 objects. - span->ref = 2; - span->sizeclass = runtime·SizeToClass(PageSize/2); - span->elemsize = PageSize/2; - } else { - // convert span of n>1 pages into two spans of n/2 pages each. - runtime·MHeap_SplitSpan(&runtime·mheap, span); - } + runtime·MSpan_EnsureSwept(span); + runtime·MHeap_SplitSpan(&runtime·mheap, span); runtime·free(oldstk); } -- 2.50.0