cmd/gc: add 2-, 3-, 4-word write barrier specializations

Assignments of 2-, 3-, and 4-word values were handled
by individual MOV instructions (and for scalars still are).
But if there are pointers involved, those assignments now
go through the write barrier routine. Before this CL, they
went to writebarrierfat, which calls memmove.
Memmove is too much overhead for these small
amounts of data.

Instead, call writebarrierfat{2,3,4}, which are specialized
for the specific amount of data being copied.
Today the write barrier does not care which words are
pointers, so size alone is enough to distinguish the cases.
If we keep these distinctions in Go 1.5 we will need to
expand them for all the pointer-vs-scalar possibilities,
so the current 3 functions will become 3+7+15 = 25,
still not a large burden (we deleted more morestack
functions than that when we dropped segmented stacks).

BenchmarkBinaryTree17           3250972583  3123910344  -3.91%
BenchmarkFannkuch11             3067605223  2964737839  -3.35%
BenchmarkFmtFprintfEmpty        101         96.0        -4.95%
BenchmarkFmtFprintfString       267         235         -11.99%
BenchmarkFmtFprintfInt          261         253         -3.07%
BenchmarkFmtFprintfIntInt       444         402         -9.46%
BenchmarkFmtFprintfPrefixedInt  374         346         -7.49%
BenchmarkFmtFprintfFloat        472         449         -4.87%
BenchmarkFmtManyArgs            1537        1476        -3.97%
BenchmarkGobDecode              13986528    12432985    -11.11%
BenchmarkGobEncode              13120323    12537420    -4.44%
BenchmarkGzip                   451925758   437500578   -3.19%
BenchmarkGunzip                 113267612   110053644   -2.84%
BenchmarkHTTPClientServer       103151      77100       -25.26%
BenchmarkJSONEncode             25002733    23435278    -6.27%
BenchmarkJSONDecode             94213717    82568789    -12.36%
BenchmarkMandelbrot200          4804246     4713070     -1.90%
BenchmarkGoParse                4646114     4379456     -5.74%
BenchmarkRegexpMatchEasy0_32    163         158         -3.07%
BenchmarkRegexpMatchEasy0_1K    433         391         -9.70%
BenchmarkRegexpMatchEasy1_32    154         138         -10.39%
BenchmarkRegexpMatchEasy1_1K    1481        1132        -23.57%
BenchmarkRegexpMatchMedium_32   282         270         -4.26%
BenchmarkRegexpMatchMedium_1K   92421       86149       -6.79%
BenchmarkRegexpMatchHard_32     5209        4718        -9.43%
BenchmarkRegexpMatchHard_1K     158141      147921      -6.46%
BenchmarkRevcomp                699818791   642222464   -8.23%
BenchmarkTemplate               132402383   108269713   -18.23%
BenchmarkTimeParse              509         478         -6.09%
BenchmarkTimeFormat             462         456         -1.30%

LGTM=r
R=r
CC=golang-codereviews
https://golang.org/cl/156200043

diff --git a/src/cmd/gc/builtin.c b/src/cmd/gc/builtin.c
index ee1ac1da4280dd3c307aa10760f281898773a6db..17f80ebba46e5f9850a9eed3ba8eaaf6e18f72ef 100644 (file)
--- a/src/cmd/gc/builtin.c
+++ b/src/cmd/gc/builtin.c
@@ -84,9 +84,12 @@ char *runtimeimport =
        "func @\"\".chansend1 (@\"\".chanType·1 *byte, @\"\".hchan·2 chan<- any, @\"\".elem·3 *any)\n"
        "func @\"\".closechan (@\"\".hchan·1 any)\n"
        "func @\"\".writebarrierptr (@\"\".dst·1 *any, @\"\".src·2 any)\n"
-       "func @\"\".writebarrieriface (@\"\".dst·1 *any, @\"\".src·2 any)\n"
        "func @\"\".writebarrierstring (@\"\".dst·1 *any, @\"\".src·2 any)\n"
        "func @\"\".writebarrierslice (@\"\".dst·1 *any, @\"\".src·2 any)\n"
+       "func @\"\".writebarrieriface (@\"\".dst·1 *any, @\"\".src·2 any)\n"
+       "func @\"\".writebarrierfat2 (@\"\".dst·1 *any, @\"\".src·2 any)\n"
+       "func @\"\".writebarrierfat3 (@\"\".dst·1 *any, @\"\".src·2 any)\n"
+       "func @\"\".writebarrierfat4 (@\"\".dst·1 *any, @\"\".src·2 any)\n"
        "func @\"\".writebarrierfat (@\"\".typ·1 *byte, @\"\".dst·2 *any, @\"\".src·3 *any)\n"
        "func @\"\".selectnbsend (@\"\".chanType·2 *byte, @\"\".hchan·3 chan<- any, @\"\".elem·4 *any) (? bool)\n"
        "func @\"\".selectnbrecv (@\"\".chanType·2 *byte, @\"\".elem·3 *any, @\"\".hchan·4 <-chan any) (? bool)\n"

diff --git a/src/cmd/gc/runtime.go b/src/cmd/gc/runtime.go
index fa927a58ac7fc931823cc2b758fe55741daffcc7..6ee5e2e364e6ec1b9ffd2d748435337f978fe4eb 100644 (file)
--- a/src/cmd/gc/runtime.go
+++ b/src/cmd/gc/runtime.go
@@ -112,6 +112,9 @@ func writebarrierptr(dst *any, src any)
 func writebarrierstring(dst *any, src any)
 func writebarrierslice(dst *any, src any)
 func writebarrieriface(dst *any, src any)
+func writebarrierfat2(dst *any, src any)
+func writebarrierfat3(dst *any, src any)
+func writebarrierfat4(dst *any, src any)
 func writebarrierfat(typ *byte, dst *any, src *any)
 
 func selectnbsend(chanType *byte, hchan chan<- any, elem *any) bool

diff --git a/src/cmd/gc/walk.c b/src/cmd/gc/walk.c
index 713348c0c07522a26c33049d4e5ca1d5c1916921..5b5385d50c6569c5fe3c7da9fa914d38433932d9 100644 (file)
--- a/src/cmd/gc/walk.c
+++ b/src/cmd/gc/walk.c
@@ -2040,21 +2040,32 @@ static Node*
 applywritebarrier(Node *n, NodeList **init)
 {
        Node *l, *r;
+       Type *t;
 
        if(n->left && n->right && needwritebarrier(n->left, n->right)) {
+               t = n->left->type;
                l = nod(OADDR, n->left, N);
                l->etype = 1; // addr does not escape
-               if(n->left->type->width == widthptr) {
-                       n = mkcall1(writebarrierfn("writebarrierptr", n->left->type, n->right->type), T, init,
+               if(t->width == widthptr) {
+                       n = mkcall1(writebarrierfn("writebarrierptr", t, n->right->type), T, init,
+                               l, n->right);
+               } else if(t->etype == TSTRING) {
+                       n = mkcall1(writebarrierfn("writebarrierstring", t, n->right->type), T, init,
+                               l, n->right);
+               } else if(isslice(t)) {
+                       n = mkcall1(writebarrierfn("writebarrierslice", t, n->right->type), T, init,
+                               l, n->right);
+               } else if(isinter(t)) {
+                       n = mkcall1(writebarrierfn("writebarrieriface", t, n->right->type), T, init,
                                l, n->right);
-               } else if(n->left->type->etype == TSTRING) {
-                       n = mkcall1(writebarrierfn("writebarrierstring", n->left->type, n->right->type), T, init,
+               } else if(t->width == 2*widthptr) {
+                       n = mkcall1(writebarrierfn("writebarrierfat2", t, n->right->type), T, init,
                                l, n->right);
-               } else if(isslice(n->left->type)) {
-                       n = mkcall1(writebarrierfn("writebarrierslice", n->left->type, n->right->type), T, init,
+               } else if(t->width == 3*widthptr) {
+                       n = mkcall1(writebarrierfn("writebarrierfat3", t, n->right->type), T, init,
                                l, n->right);
-               } else if(isinter(n->left->type)) {
-                       n = mkcall1(writebarrierfn("writebarrieriface", n->left->type, n->right->type), T, init,
+               } else if(t->width == 4*widthptr) {
+                       n = mkcall1(writebarrierfn("writebarrierfat4", t, n->right->type), T, init,
                                l, n->right);
                } else {
                        r = n->right;
@@ -2062,8 +2073,9 @@ applywritebarrier(Node *n, NodeList **init)
                                r = r->left;
                        r = nod(OADDR, r, N);
                        r->etype = 1; // addr does not escape
-                       n = mkcall1(writebarrierfn("writebarrierfat", n->left->type, r->left->type), T, init,
-                               typename(n->left->type), l, r);
+                       //warnl(n->lineno, "writebarrierfat %T %N", t, r);
+                       n = mkcall1(writebarrierfn("writebarrierfat", t, r->left->type), T, init,
+                               typename(t), l, r);
                }
        }
        return n;

diff --git a/src/runtime/mgc0.go b/src/runtime/mgc0.go
index 0e17599c2ad6229333b0a68fd0f6676d62a5a4ed..3152b1fe1aa34615f33ed7e9c4ca73c7354afb87 100644 (file)
--- a/src/runtime/mgc0.go
+++ b/src/runtime/mgc0.go
@@ -109,6 +109,27 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
        dst[1] = src[1]
 }
 
+//go:nosplit
+func writebarrierfat2(dst *[2]uintptr, src [2]uintptr) {
+       dst[0] = src[0]
+       dst[1] = src[1]
+}
+
+//go:nosplit
+func writebarrierfat3(dst *[3]uintptr, src [3]uintptr) {
+       dst[0] = src[0]
+       dst[1] = src[1]
+       dst[2] = src[2]
+}
+
+//go:nosplit
+func writebarrierfat4(dst *[4]uintptr, src [4]uintptr) {
+       dst[0] = src[0]
+       dst[1] = src[1]
+       dst[2] = src[2]
+       dst[3] = src[3]
+}
+
 //go:nosplit
 func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
        memmove(dst, src, typ.size)