runtime: faster mcentral alloc.

Reduce individual object handling by anticipating how much of them are servable.

Not a chunked transfer cache, but decent enough to make sure the bottleneck is not here.

Mac OSX, median of 10 runs:

benchmark                 old ns/op    new ns/op    delta
BenchmarkBinaryTree17    5358937333   4892813012   -8.70%
BenchmarkFannkuch11      3257752475   3315436116   +1.77%
BenchmarkGobDecode         23277349     23001114   -1.19%
BenchmarkGobEncode         14367327     14262925   -0.73%
BenchmarkGzip             441045541    440451719   -0.13%
BenchmarkGunzip           139117663    139622494   +0.36%
BenchmarkJSONEncode        45715854     45687802   -0.06%
BenchmarkJSONDecode       103949570    106530032   +2.48%
BenchmarkMandelbrot200      4542462      4548290   +0.13%
BenchmarkParse              7790558      7557540   -2.99%
BenchmarkRevcomp          831436684    832510381   +0.13%
BenchmarkTemplate         133789824    133007337   -0.58%

benchmark                  old MB/s     new MB/s  speedup
BenchmarkGobDecode            32.82        33.33    1.02x
BenchmarkGobEncode            53.42        53.86    1.01x
BenchmarkGzip                 43.70        44.01    1.01x
BenchmarkGunzip              139.09       139.14    1.00x
BenchmarkJSONEncode           42.69        42.56    1.00x
BenchmarkJSONDecode           18.78        17.91    0.95x
BenchmarkParse                 7.37         7.67    1.04x
BenchmarkRevcomp             306.83       305.70    1.00x
BenchmarkTemplate             14.57        14.56    1.00x

R=rsc, dvyukov
CC=golang-dev
https://golang.org/cl/7005055

diff --git a/src/pkg/runtime/mcentral.c b/src/pkg/runtime/mcentral.c
index 558b35b0e1d1e255091bb18342c6ed53fffc0bdb..b2bfa73b3d0f239844882250e6a072daa8866230 100644 (file)
--- a/src/pkg/runtime/mcentral.c
+++ b/src/pkg/runtime/mcentral.c
@@ -34,12 +34,13 @@ runtime·MCentral_Init(MCentral *c, int32 sizeclass)
 // Allocate up to n objects from the central free list.
 // Return the number of objects allocated.
 // The objects are linked together by their first words.
-// On return, *pstart points at the first object and *pend at the last.
+// On return, *pstart points at the first object.
 int32
 runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **pfirst)
 {
-       MLink *first, *last, *v;
-       int32 i;
+       MSpan *s;
+       MLink *first, *last;
+       int32 cap, avail, i;
 
        runtime·lock(c);
        // Replenish central list if empty.
@@ -50,41 +51,34 @@ runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **pfirst)
                        return 0;
                }
        }
+       s = c->nonempty.next;
+       cap = (s->npages << PageShift) / s->elemsize;
+       avail = cap - s->ref;
+       if(avail < n)
+               n = avail;
 
-       // Copy from list, up to n.
        // First one is guaranteed to work, because we just grew the list.
-       first = MCentral_Alloc(c);
+       first = s->freelist;
        last = first;
-       for(i=1; i<n && (v = MCentral_Alloc(c)) != nil; i++) {
-               last->next = v;
-               last = v;
+       for(i=1; i<n; i++) {
+               last = last->next;
        }
+       s->freelist = last->next;
        last->next = nil;
-       c->nfree -= i;
-
-       runtime·unlock(c);
-       *pfirst = first;
-       return i;
-}
+       s->ref += n;
+       c->nfree -= n;
 
-// Helper: allocate one object from the central free list.
-static void*
-MCentral_Alloc(MCentral *c)
-{
-       MSpan *s;
-       MLink *v;
-
-       if(runtime·MSpanList_IsEmpty(&c->nonempty))
-               return nil;
-       s = c->nonempty.next;
-       s->ref++;
-       v = s->freelist;
-       s->freelist = v->next;
-       if(s->freelist == nil) {
+       if(n == avail) {
+               if(s->freelist != nil || s->ref != cap) {
+                       runtime·throw("invalid freelist");
+               }
                runtime·MSpanList_Remove(s);
                runtime·MSpanList_Insert(&c->empty, s);
        }
-       return v;
+
+       runtime·unlock(c);
+       *pfirst = first;
+       return n;
 }
 
 // Free n objects back into the central free list.