runtime: prefetch next block in mallocgc

json-1
cputime                  99600000     98600000      -1.00%
time                    100005493     98859693      -1.15%

garbage-1
cputime                  15760000     15440000      -2.03%
time                     15791759     15471701      -2.03%

LGTM=khr
R=golang-codereviews, gobot, khr, dave
CC=bradfitz, golang-codereviews, iant
https://golang.org/cl/57310043

diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc
index 3dfa63dbec4ea53917af6d765b538ebe4d544963..b00d690aad8ce3fefc17931e93224a9e2c82199c 100644 (file)
--- a/src/pkg/runtime/malloc.goc
+++ b/src/pkg/runtime/malloc.goc
@@ -40,7 +40,7 @@ runtime·mallocgc(uintptr size, uintptr typ, uint32 flag)
        intgo rate;
        MCache *c;
        MCacheList *l;
-       MLink *v;
+       MLink *v, *next;
        byte *tiny;
 
        if(size == 0) {
@@ -120,7 +120,10 @@ runtime·mallocgc(uintptr size, uintptr typ, uint32 flag)
                        if(l->list == nil)
                                runtime·MCache_Refill(c, TinySizeClass);
                        v = l->list;
-                       l->list = v->next;
+                       next = v->next;
+                       if(next != nil)  // prefetching nil leads to a DTLB miss
+                               PREFETCH(next);
+                       l->list = next;
                        l->nlist--;
                        ((uint64*)v)[0] = 0;
                        ((uint64*)v)[1] = 0;
@@ -144,7 +147,10 @@ runtime·mallocgc(uintptr size, uintptr typ, uint32 flag)
                if(l->list == nil)
                        runtime·MCache_Refill(c, sizeclass);
                v = l->list;
-               l->list = v->next;
+               next = v->next;
+               if(next != nil)  // prefetching nil leads to a DTLB miss
+                       PREFETCH(next);
+               l->list = next;
                l->nlist--;
                if(!(flag & FlagNoZero)) {
                        v->next = nil;