runtime: inline MCache_Alloc() into mallocgc()

benchmark                    old ns/op    new ns/op    delta
BenchmarkMalloc8                    68           62   -8.63%
BenchmarkMalloc16                   75           69   -7.94%
BenchmarkMallocTypeInfo8           102           98   -3.73%
BenchmarkMallocTypeInfo16          108          103   -4.63%

R=golang-dev, dave, khr
CC=golang-dev
https://golang.org/cl/9790043

diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc
index cf18e8c9e59d0ef35018fcf373bf4e7e231bf353..47eb005894f910f9b7019f2fe0a07702b4586eb0 100644 (file)
--- a/src/pkg/runtime/malloc.goc
+++ b/src/pkg/runtime/malloc.goc
@@ -31,9 +31,10 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
        int32 sizeclass;
        intgo rate;
        MCache *c;
+       MCacheList *l;
        uintptr npages;
        MSpan *s;
-       void *v;
+       MLink *v;
 
        if(runtime·gcwaiting && g != m->g0 && m->locks == 0 && dogc)
                runtime·gosched();
@@ -56,9 +57,20 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
                else
                        sizeclass = runtime·size_to_class128[(size-1024+127) >> 7];
                size = runtime·class_to_size[sizeclass];
-               v = runtime·MCache_Alloc(c, sizeclass, size, zeroed);
-               if(v == nil)
-                       runtime·throw("out of memory");
+               l = &c->list[sizeclass];
+               if(l->list == nil)
+                       runtime·MCache_Refill(c, sizeclass);
+               v = l->list;
+               l->list = v->next;
+               l->nlist--;
+               if(zeroed) {
+                       v->next = nil;
+                       // block is zeroed iff second word is zero ...
+                       if(size > sizeof(uintptr) && ((uintptr*)v)[1] != 0)
+                               runtime·memclr((byte*)v, size);
+               }
+               c->local_cachealloc += size;
+               c->local_objects++;
                c->local_alloc += size;
                c->local_total_alloc += size;
                c->local_by_size[sizeclass].nmalloc++;

diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h
index 99a1966071698ad6e3bbc1e0726b52a82453a82c..1085344ee1f83f0cd8ba18897b57e553b1d0c3c6 100644 (file)
--- a/src/pkg/runtime/malloc.h
+++ b/src/pkg/runtime/malloc.h
@@ -305,7 +305,7 @@ struct MCache
 
 };
 
-void*  runtime·MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed);
+void   runtime·MCache_Refill(MCache *c, int32 sizeclass);
 void   runtime·MCache_Free(MCache *c, void *p, int32 sizeclass, uintptr size);
 void   runtime·MCache_ReleaseAll(MCache *c);
 

diff --git a/src/pkg/runtime/mcache.c b/src/pkg/runtime/mcache.c
index 3df0450fea0b8b4ca5f1ebf31458b901f8c2478c..219eb8d4d67e5b04340d059f71efff084076ca26 100644 (file)
--- a/src/pkg/runtime/mcache.c
+++ b/src/pkg/runtime/mcache.c
@@ -10,35 +10,18 @@
 #include "arch_GOARCH.h"
 #include "malloc.h"
 
-void*
-runtime·MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed)
+void
+runtime·MCache_Refill(MCache *c, int32 sizeclass)
 {
        MCacheList *l;
-       MLink *v;
 
-       // Allocate from list.
+       // Replenish using central lists.
        l = &c->list[sizeclass];
-       if(l->list == nil) {
-               // Replenish using central lists.
-               l->nlist = runtime·MCentral_AllocList(&runtime·mheap->central[sizeclass], &l->list);
-               if(l->list == nil)
-                       runtime·throw("out of memory");
-       }
-       v = l->list;
-       l->list = v->next;
-       l->nlist--;
-
-       // v is zeroed except for the link pointer
-       // that we used above; zero that.
-       v->next = nil;
-       if(zeroed) {
-               // block is zeroed iff second word is zero ...
-               if(size > sizeof(uintptr) && ((uintptr*)v)[1] != 0)
-                       runtime·memclr((byte*)v, size);
-       }
-       c->local_cachealloc += size;
-       c->local_objects++;
-       return v;
+       if(l->list)
+               runtime·throw("MCache_Refill: the list is not empty");
+       l->nlist = runtime·MCentral_AllocList(&runtime·mheap->central[sizeclass], &l->list);
+       if(l->list == nil)
+               runtime·throw("out of memory");
 }
 
 // Take n elements off l and return them to the central free list.