runtime: native xadd for 386/amd64

benchmark                          old ns/op    new ns/op    delta
BenchmarkSemaUncontended               37.40        34.10   -8.82%
BenchmarkSemaUncontended-2             18.90        17.70   -6.35%
BenchmarkSemaUncontended-4             11.90        10.90   -8.40%
BenchmarkSemaUncontended-8              6.26         5.19  -17.09%
BenchmarkSemaUncontended-16             4.39         3.91  -10.93%
BenchmarkSemaSyntNonblock              38.00        35.30   -7.11%
BenchmarkSemaSyntNonblock-2            83.00        46.70  -43.73%
BenchmarkSemaSyntNonblock-4           124.00       101.00  -18.55%
BenchmarkSemaSyntNonblock-8           124.00       116.00   -6.45%
BenchmarkSemaSyntNonblock-16          148.00       114.00  -22.97%

(on HP Z600 2 x Xeon E5620, 8 HT cores, 2.40GHz)

R=golang-dev, rsc
CC=golang-dev
https://golang.org/cl/4755041

diff --git a/src/pkg/runtime/386/asm.s b/src/pkg/runtime/386/asm.s
index 3aa5bdee55588ab9a6f9a3fe15a274377fc01c58..c6df4f069a7014e0be36eb6156b5f8789f79ca9f 100644 (file)
--- a/src/pkg/runtime/386/asm.s
+++ b/src/pkg/runtime/386/asm.s
@@ -318,6 +318,19 @@ TEXT runtime·casp(SB), 7, $0
        MOVL    $1, AX
        RET
 
+// uint32 xadd(uint32 volatile *val, int32 delta)
+// Atomically:
+//     *val += delta;
+//     return *val;
+TEXT runtime·xadd(SB), 7, $0
+       MOVL    4(SP), BX
+       MOVL    8(SP), AX
+       MOVL    AX, CX
+       LOCK
+       XADDL   AX, 0(BX)
+       ADDL    CX, AX
+       RET
+
 TEXT runtime·atomicstorep(SB), 7, $0
        MOVL    4(SP), BX
        MOVL    8(SP), AX

diff --git a/src/pkg/runtime/amd64/asm.s b/src/pkg/runtime/amd64/asm.s
index e03c9ebfdf991f91b7b460f7752c07f3c76b4ceb..2734ae1dca9a6f7b0901d0b4ea950921c96bf24c 100644 (file)
--- a/src/pkg/runtime/amd64/asm.s
+++ b/src/pkg/runtime/amd64/asm.s
@@ -364,6 +364,19 @@ TEXT runtime·casp(SB), 7, $0
        MOVL    $1, AX
        RET
 
+// uint32 xadd(uint32 volatile *val, int32 delta)
+// Atomically:
+//     *val += delta;
+//     return *val;
+TEXT runtime·xadd(SB), 7, $0
+       MOVQ    8(SP), BX
+       MOVL    16(SP), AX
+       MOVL    AX, CX
+       LOCK
+       XADDL   AX, 0(BX)
+       ADDL    CX, AX
+       RET
+
 TEXT runtime·atomicstorep(SB), 7, $0
        MOVQ    8(SP), BX
        MOVQ    16(SP), AX

diff --git a/src/pkg/runtime/arm/atomic.c b/src/pkg/runtime/arm/atomic.c
index 186ffcfd48a933714c283e339e8097d1edb27819..d229e8c347584d2dbc55fbda715f99380da355a5 100644 (file)
--- a/src/pkg/runtime/arm/atomic.c
+++ b/src/pkg/runtime/arm/atomic.c
@@ -4,6 +4,21 @@
 
 #include "runtime.h"
 
+// Atomic add and return new value.
+#pragma textflag 7
+uint32
+runtime·xadd(uint32 volatile *val, int32 delta)
+{
+       uint32 oval, nval;
+
+       for(;;){
+               oval = *val;
+               nval = oval + delta;
+               if(runtime·cas(val, oval, nval))
+                       return nval;
+       }
+}
+
 #pragma textflag 7
 uint32
 runtime·atomicload(uint32 volatile* addr)

diff --git a/src/pkg/runtime/runtime.c b/src/pkg/runtime/runtime.c
index 7e37d66d4109a5286f68ddc76006d1ea87bb1070..a16ca417e6f538b2756e2dd6bd2ab4735f713b99 100644 (file)
--- a/src/pkg/runtime/runtime.c
+++ b/src/pkg/runtime/runtime.c
@@ -207,20 +207,6 @@ runtime·goenvs_unix(void)
        os·Envs.cap = n;
 }
 
-// Atomic add and return new value.
-uint32
-runtime·xadd(uint32 volatile *val, int32 delta)
-{
-       uint32 oval, nval;
-
-       for(;;){
-               oval = *val;
-               nval = oval + delta;
-               if(runtime·cas(val, oval, nval))
-                       return nval;
-       }
-}
-
 byte*
 runtime·getenv(int8 *s)
 {