runtime: add more specialized type algorithms

The change adds specialized type algorithms
for slices and types of size 8/16/32/64/128.
It significantly accelerates chan and map operations
for most builtin types as well as user structs.

benchmark                   old,ns/op   new,ns/op
BenchmarkChanUncontended          226          94
(on Intel Xeon E5620, 2.4GHz, Linux 64 bit)

R=golang-dev, rsc
CC=golang-dev
https://golang.org/cl/4815087

diff --git a/src/cmd/gc/go.h b/src/cmd/gc/go.h
index 449127292678586ce345736ee944ef0c3310dd2a..da0fb5146e83cadc7b4652ebec1e953d5e3ef84e 100644 (file)
--- a/src/cmd/gc/go.h
+++ b/src/cmd/gc/go.h
@@ -40,7 +40,17 @@ enum
        ASTRING,
        AINTER,
        ANILINTER,
-       AMEMWORD,
+       ASLICE,
+       AMEM8,
+       AMEM16,
+       AMEM32,
+       AMEM64,
+       AMEM128,
+       ANOEQ8,
+       ANOEQ16,
+       ANOEQ32,
+       ANOEQ64,
+       ANOEQ128,
 
        BADWIDTH        = -1000000000,
 };

diff --git a/src/cmd/gc/subr.c b/src/cmd/gc/subr.c
index 96675be3fd6240a7a8b99faeaa4740360f304869..1a05d43d0ec6ce38ce9c5b736aa158ddc40a5ab9 100644 (file)
--- a/src/cmd/gc/subr.c
+++ b/src/cmd/gc/subr.c
@@ -493,8 +493,16 @@ algtype(Type *t)
 
        if(issimple[t->etype] || isptr[t->etype] ||
                t->etype == TCHAN || t->etype == TFUNC || t->etype == TMAP) {
-               if(t->width == widthptr)
-                       a = AMEMWORD;
+               if(t->width == 1)
+                       a = AMEM8;
+               else if(t->width == 2)
+                       a = AMEM16;
+               else if(t->width == 4)
+                       a = AMEM32;
+               else if(t->width == 8)
+                       a = AMEM64;
+               else if(t->width == 16)
+                       a = AMEM128;
                else
                        a = AMEM;       // just bytes (int, ptr, etc)
        } else if(t->etype == TSTRING)
@@ -503,8 +511,22 @@ algtype(Type *t)
                a = ANILINTER;  // nil interface
        else if(t->etype == TINTER)
                a = AINTER;     // interface
-       else
-               a = ANOEQ;      // just bytes, but no hash/eq
+       else if(isslice(t))
+               a = ASLICE;     // slice
+       else {
+               if(t->width == 1)
+                       a = ANOEQ8;
+               else if(t->width == 2)
+                       a = ANOEQ16;
+               else if(t->width == 4)
+                       a = ANOEQ32;
+               else if(t->width == 8)
+                       a = ANOEQ64;
+               else if(t->width == 16)
+                       a = ANOEQ128;
+               else
+                       a = ANOEQ;      // just bytes, but no hash/eq
+       }
        return a;
 }
 

diff --git a/src/pkg/runtime/runtime.c b/src/pkg/runtime/runtime.c
index 57c0873306437acac65158896555070d0b420079..49aba7da018efbca7e0f93cfe9e7ec05c75e5323 100644 (file)
--- a/src/pkg/runtime/runtime.c
+++ b/src/pkg/runtime/runtime.c
@@ -398,21 +398,110 @@ memcopy(uint32 s, void *a, void *b)
 }
 
 static uint32
-memwordequal(uint32 s, void *a, void *b)
+memequal8(uint32 s, uint8 *a, uint8 *b)
 {
        USED(s);
-       return *(uintptr*)(a) == *(uintptr*)(b);
+       return *a == *b;
 }
 
 static void
-memwordcopy(uint32 s, void *a, void *b)
+memcopy8(uint32 s, uint8 *a, uint8 *b)
 {
        USED(s);
-       if (b == nil) {
-               *(uintptr*)(a) = 0;
+       if(b == nil) {
+               *a = 0;
+               return;
+       }
+       *a = *b;
+}
+
+static uint32
+memequal16(uint32 s, uint16 *a, uint16 *b)
+{
+       USED(s);
+       return *a == *b;
+}
+
+static void
+memcopy16(uint32 s, uint16 *a, uint16 *b)
+{
+       USED(s);
+       if(b == nil) {
+               *a = 0;
                return;
        }
-       *(uintptr*)(a) = *(uintptr*)(b);
+       *a = *b;
+}
+
+static uint32
+memequal32(uint32 s, uint32 *a, uint32 *b)
+{
+       USED(s);
+       return *a == *b;
+}
+
+static void
+memcopy32(uint32 s, uint32 *a, uint32 *b)
+{
+       USED(s);
+       if(b == nil) {
+               *a = 0;
+               return;
+       }
+       *a = *b;
+}
+
+static uint32
+memequal64(uint32 s, uint64 *a, uint64 *b)
+{
+       USED(s);
+       return *a == *b;
+}
+
+static void
+memcopy64(uint32 s, uint64 *a, uint64 *b)
+{
+       USED(s);
+       if(b == nil) {
+               *a = 0;
+               return;
+       }
+       *a = *b;
+}
+
+static uint32
+memequal128(uint32 s, uint64 *a, uint64 *b)
+{
+       USED(s);
+       return a[0] == b[0] && a[1] == b[1];
+}
+
+static void
+memcopy128(uint32 s, uint64 *a, uint64 *b)
+{
+       USED(s);
+       if(b == nil) {
+               a[0] = 0;
+               a[1] = 0;
+               return;
+       }
+       a[0] = b[0];
+       a[1] = b[1];
+}
+
+static void
+slicecopy(uint32 s, Slice *a, Slice *b)
+{
+       USED(s);
+       if(b == nil) {
+               a->array = 0;
+               a->len = 0;
+               a->cap = 0;
+               return;
+       }
+       a->array = b->array;
+       a->len = b->len;
+       a->cap = b->cap;
 }
 
 static uintptr
@@ -441,6 +530,19 @@ strprint(uint32 s, String *a)
        runtime·printstring(*a);
 }
 
+static void
+strcopy(uint32 s, String *a, String *b)
+{
+       USED(s);
+       if(b == nil) {
+               a->str = 0;
+               a->len = 0;
+               return;
+       }
+       a->str = b->str;
+       a->len = b->len;
+}
+
 static uintptr
 interhash(uint32 s, Iface *a)
 {
@@ -462,6 +564,19 @@ interequal(uint32 s, Iface *a, Iface *b)
        return runtime·ifaceeq_c(*a, *b);
 }
 
+static void
+intercopy(uint32 s, Iface *a, Iface *b)
+{
+       USED(s);
+       if(b == nil) {
+               a->tab = 0;
+               a->data = 0;
+               return;
+       }
+       a->tab = b->tab;
+       a->data = b->data;
+}
+
 static uintptr
 nilinterhash(uint32 s, Eface *a)
 {
@@ -483,6 +598,19 @@ nilinterequal(uint32 s, Eface *a, Eface *b)
        return runtime·efaceeq_c(*a, *b);
 }
 
+static void
+nilintercopy(uint32 s, Eface *a, Eface *b)
+{
+       USED(s);
+       if(b == nil) {
+               a->type = 0;
+               a->data = 0;
+               return;
+       }
+       a->type = b->type;
+       a->data = b->data;
+}
+
 uintptr
 runtime·nohash(uint32 s, void *a)
 {
@@ -507,10 +635,20 @@ runtime·algarray[] =
 {
 [AMEM] { memhash, memequal, memprint, memcopy },
 [ANOEQ]        { runtime·nohash, runtime·noequal, memprint, memcopy },
-[ASTRING]      { strhash, strequal, strprint, memcopy },
-[AINTER]               { interhash, interequal, interprint, memcopy },
-[ANILINTER]    { nilinterhash, nilinterequal, nilinterprint, memcopy },
-[AMEMWORD] { memhash, memwordequal, memprint, memwordcopy },
+[ASTRING]      { strhash, strequal, strprint, strcopy },
+[AINTER]               { interhash, interequal, interprint, intercopy },
+[ANILINTER]    { nilinterhash, nilinterequal, nilinterprint, nilintercopy },
+[ASLICE]       { runtime·nohash, runtime·noequal, memprint, slicecopy },
+[AMEM8]                { memhash, memequal8, memprint, memcopy8 },
+[AMEM16]       { memhash, memequal16, memprint, memcopy16 },
+[AMEM32]       { memhash, memequal32, memprint, memcopy32 },
+[AMEM64]       { memhash, memequal64, memprint, memcopy64 },
+[AMEM128]      { memhash, memequal128, memprint, memcopy128 },
+[ANOEQ8]       { runtime·nohash, runtime·noequal, memprint, memcopy8 },
+[ANOEQ16]      { runtime·nohash, runtime·noequal, memprint, memcopy16 },
+[ANOEQ32]      { runtime·nohash, runtime·noequal, memprint, memcopy32 },
+[ANOEQ64]      { runtime·nohash, runtime·noequal, memprint, memcopy64 },
+[ANOEQ128]     { runtime·nohash, runtime·noequal, memprint, memcopy128 },
 };
 
 int64

diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h
index d2e4378b59c20c4e9a5af2721ad7f62274c34a47..8c5403f444c54903a4b8097fc7c1c2c71af7972b 100644 (file)
--- a/src/pkg/runtime/runtime.h
+++ b/src/pkg/runtime/runtime.h
@@ -335,7 +335,17 @@ enum
        ASTRING,
        AINTER,
        ANILINTER,
-       AMEMWORD,
+       ASLICE,
+       AMEM8,
+       AMEM16,
+       AMEM32,
+       AMEM64,
+       AMEM128,
+       ANOEQ8,
+       ANOEQ16,
+       ANOEQ32,
+       ANOEQ64,
+       ANOEQ128,
        Amax
 };