From 23ad56311977b4a4bfff78fb5f674616e0272445 Mon Sep 17 00:00:00 2001 From: Dmitriy Vyukov Date: Wed, 15 May 2013 18:35:05 +0400 Subject: [PATCH] runtime: transfer whole span from MCentral to MCache Finer-grained transfers were relevant with per-M caches, with per-P caches they are not relevant and harmful for performance. For few small size classes where it makes difference, it's fine to grab the whole span (4K). benchmark old ns/op new ns/op delta BenchmarkMalloc 42 40 -4.45% R=golang-dev, bradfitz CC=golang-dev https://golang.org/cl/9374043 --- src/pkg/runtime/malloc.h | 6 +----- src/pkg/runtime/mcache.c | 5 ++--- src/pkg/runtime/mcentral.c | 34 ++++++++-------------------------- src/pkg/runtime/msize.c | 11 ----------- 4 files changed, 11 insertions(+), 45 deletions(-) diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h index 7474f85258..a4652beda1 100644 --- a/src/pkg/runtime/malloc.h +++ b/src/pkg/runtime/malloc.h @@ -267,14 +267,10 @@ extern MStats mstats; // class_to_size[i] = largest size in class i // class_to_allocnpages[i] = number of pages to allocate when // making new objects in class i -// class_to_transfercount[i] = number of objects to move when -// taking a bunch of objects out of the central lists -// and putting them in the thread free list. int32 runtime·SizeToClass(int32); extern int32 runtime·class_to_size[NumSizeClasses]; extern int32 runtime·class_to_allocnpages[NumSizeClasses]; -extern int32 runtime·class_to_transfercount[NumSizeClasses]; extern int8 runtime·size_to_class8[1024/8 + 1]; extern int8 runtime·size_to_class128[(MaxSmallSize-1024)/128 + 1]; extern void runtime·InitSizes(void); @@ -399,7 +395,7 @@ struct MCentral }; void runtime·MCentral_Init(MCentral *c, int32 sizeclass); -int32 runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **first); +int32 runtime·MCentral_AllocList(MCentral *c, MLink **first); void runtime·MCentral_FreeList(MCentral *c, int32 n, MLink *first); void runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end); diff --git a/src/pkg/runtime/mcache.c b/src/pkg/runtime/mcache.c index 64803e7037..20b062b65e 100644 --- a/src/pkg/runtime/mcache.c +++ b/src/pkg/runtime/mcache.c @@ -21,8 +21,7 @@ runtime·MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed) l = &c->list[sizeclass]; if(l->list == nil) { // Replenish using central lists. - n = runtime·MCentral_AllocList(&runtime·mheap->central[sizeclass], - runtime·class_to_transfercount[sizeclass], &first); + n = runtime·MCentral_AllocList(&runtime·mheap->central[sizeclass], &first); if(n == 0) runtime·throw("out of memory"); l->list = first; @@ -91,7 +90,7 @@ runtime·MCache_Free(MCache *c, void *v, int32 sizeclass, uintptr size) if(l->nlist >= MaxMCacheListLen) { // Release a chunk back. - ReleaseN(c, l, runtime·class_to_transfercount[sizeclass], sizeclass); + ReleaseN(c, l, l->nlist/2, sizeclass); } if(c->size >= MaxMCacheSize) { diff --git a/src/pkg/runtime/mcentral.c b/src/pkg/runtime/mcentral.c index ec2a91ad5c..063d1c501d 100644 --- a/src/pkg/runtime/mcentral.c +++ b/src/pkg/runtime/mcentral.c @@ -30,16 +30,15 @@ runtime·MCentral_Init(MCentral *c, int32 sizeclass) runtime·MSpanList_Init(&c->empty); } -// Allocate up to n objects from the central free list. +// Allocate a list of objects from the central free list. // Return the number of objects allocated. // The objects are linked together by their first words. // On return, *pstart points at the first object. int32 -runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **pfirst) +runtime·MCentral_AllocList(MCentral *c, MLink **pfirst) { MSpan *s; - MLink *first, *last; - int32 cap, avail, i; + int32 cap, n; runtime·lock(c); // Replenish central list if empty. @@ -52,31 +51,14 @@ runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **pfirst) } s = c->nonempty.next; cap = (s->npages << PageShift) / s->elemsize; - avail = cap - s->ref; - if(avail < n) - n = avail; - - // First one is guaranteed to work, because we just grew the list. - first = s->freelist; - last = first; - for(i=1; inext; - } - s->freelist = last->next; - last->next = nil; + n = cap - s->ref; + *pfirst = s->freelist; + s->freelist = nil; s->ref += n; c->nfree -= n; - - if(n == avail) { - if(s->freelist != nil || s->ref != cap) { - runtime·throw("invalid freelist"); - } - runtime·MSpanList_Remove(s); - runtime·MSpanList_Insert(&c->empty, s); - } - + runtime·MSpanList_Remove(s); + runtime·MSpanList_Insert(&c->empty, s); runtime·unlock(c); - *pfirst = first; return n; } diff --git a/src/pkg/runtime/msize.c b/src/pkg/runtime/msize.c index a81bc11aae..50b372b613 100644 --- a/src/pkg/runtime/msize.c +++ b/src/pkg/runtime/msize.c @@ -31,7 +31,6 @@ int32 runtime·class_to_size[NumSizeClasses]; int32 runtime·class_to_allocnpages[NumSizeClasses]; -int32 runtime·class_to_transfercount[NumSizeClasses]; // The SizeToClass lookup is implemented using two arrays, // one mapping sizes <= 1024 to their class and one mapping @@ -137,16 +136,6 @@ runtime·InitSizes(void) // Copy out for statistics table. for(i=0; i 32) - n = 32; - runtime·class_to_transfercount[sizeclass] = n; - } return; dump: -- 2.48.1