sync/atomic, runtime/internal/atomic: improve ppc64x atomics

The following performance improvements have been made to the
low-level atomic functions for ppc64le & ppc64:

- For those cases containing a lwarx and stwcx (or other sizes):
sync, lwarx, maybe something, stwcx, loop to sync, sync, isync
The sync is moved before (outside) the lwarx/stwcx loop, and the
 sync after is removed, so it becomes:
sync, lwarx, maybe something, stwcx, loop to lwarx, isync

- For the Or8 and And8, the shifting and manipulation of the
address to the word aligned version were removed and the
instructions were changed to use lbarx, stbcx instead of
register shifting, xor, then lwarx, stwcx.

- New instructions LWSYNC, LBAR, STBCC were tested and added.
runtime/atomic_ppc64x.s was changed to use the LWSYNC opcode
instead of the WORD encoding.

Fixes #15469

Ran some of the benchmarks in the runtime and sync directories.
Some results varied from run to run but the trend was improvement
based on best times for base and new:

runtime.test:
BenchmarkChanNonblocking-128         0.88          0.89          +1.14%
BenchmarkChanUncontended-128         569           511           -10.19%
BenchmarkChanContended-128           63110         53231         -15.65%
BenchmarkChanSync-128                691           598           -13.46%
BenchmarkChanSyncWork-128            11355         11649         +2.59%
BenchmarkChanProdCons0-128           2402          2090          -12.99%
BenchmarkChanProdCons10-128          1348          1363          +1.11%
BenchmarkChanProdCons100-128         1002          746           -25.55%
BenchmarkChanProdConsWork0-128       2554          2720          +6.50%
BenchmarkChanProdConsWork10-128      1909          1804          -5.50%
BenchmarkChanProdConsWork100-128     1624          1580          -2.71%
BenchmarkChanCreation-128            237           212           -10.55%
BenchmarkChanSem-128                 705           667           -5.39%
BenchmarkChanPopular-128             5081190       4497566       -11.49%

BenchmarkCreateGoroutines-128             532           473           -11.09%
BenchmarkCreateGoroutinesParallel-128     35.0          34.7          -0.86%
BenchmarkCreateGoroutinesCapture-128      4923          4200          -14.69%

sync.test:
BenchmarkUncontendedSemaphore-128      112           94.2          -15.89%
BenchmarkContendedSemaphore-128        133           128           -3.76%
BenchmarkMutexUncontended-128          1.90          1.67          -12.11%
BenchmarkMutex-128                     353           310           -12.18%
BenchmarkMutexSlack-128                304           283           -6.91%
BenchmarkMutexWork-128                 554           541           -2.35%
BenchmarkMutexWorkSlack-128            567           556           -1.94%
BenchmarkMutexNoSpin-128               275           242           -12.00%
BenchmarkMutexSpin-128                 1129          1030          -8.77%
BenchmarkOnce-128                      1.08          0.96          -11.11%
BenchmarkPool-128                      29.8          27.4          -8.05%
BenchmarkPoolOverflow-128              40564         36583         -9.81%
BenchmarkSemaUncontended-128           3.14          2.63          -16.24%
BenchmarkSemaSyntNonblock-128          1087          1069          -1.66%
BenchmarkSemaSyntBlock-128             897           893           -0.45%
BenchmarkSemaWorkNonblock-128          1034          1028          -0.58%
BenchmarkSemaWorkBlock-128             949           886           -6.64%

Change-Id: I4403fb29d3cd5254b7b1ce87a216bd11b391079e
Reviewed-on: https://go-review.googlesource.com/22549
Reviewed-by: Michael Munday <munday@ca.ibm.com>
Reviewed-by: Minux Ma <minux@golang.org>

diff --git a/src/cmd/internal/obj/ppc64/a.out.go b/src/cmd/internal/obj/ppc64/a.out.go
index 8cc984260e17639d0dda7f2947852c47efafee30..8e58c592383327b13641bd267f5fa58fbdcace3c 100644 (file)
--- a/src/cmd/internal/obj/ppc64/a.out.go
+++ b/src/cmd/internal/obj/ppc64/a.out.go
@@ -342,8 +342,10 @@ const (
        AFSUBS
        AFSUBSCC
        AMOVMW
+       ALBAR
        ALSW
        ALWAR
+       ALWSYNC
        AMOVWBR
        AMOVB
        AMOVBU
@@ -401,6 +403,7 @@ const (
        ASRAW
        ASRAWCC
        ASRWCC
+       ASTBCCC
        ASTSW
        ASTWCCC
        ASUB

diff --git a/src/cmd/internal/obj/ppc64/anames.go b/src/cmd/internal/obj/ppc64/anames.go
index 868700f0124aed36dd4c18669c4f9f368d03acf0..eb42c9a953e4324a9cb1db5ee63e9bf4b4feacc3 100644 (file)
--- a/src/cmd/internal/obj/ppc64/anames.go
+++ b/src/cmd/internal/obj/ppc64/anames.go
@@ -118,8 +118,10 @@ var Anames = []string{
        "FSUBS",
        "FSUBSCC",
        "MOVMW",
+       "LBAR",
        "LSW",
        "LWAR",
+       "LWSYNC",
        "MOVWBR",
        "MOVB",
        "MOVBU",
@@ -177,6 +179,7 @@ var Anames = []string{
        "SRAW",
        "SRAWCC",
        "SRWCC",
+       "STBCCC",
        "STSW",
        "STWCCC",
        "SUB",

diff --git a/src/cmd/internal/obj/ppc64/asm9.go b/src/cmd/internal/obj/ppc64/asm9.go
index e847ec341f513d990d1d88f48bfe8c97ceaa808e..5366809d2fc6e5df5a42f6d76736d0292133b04b 100644 (file)
--- a/src/cmd/internal/obj/ppc64/asm9.go
+++ b/src/cmd/internal/obj/ppc64/asm9.go
@@ -933,6 +933,7 @@ func buildop(ctxt *obj.Link) {
 
                case AECOWX: /* indexed store: op s,(b+a); op s,(b) */
                        opset(ASTWCCC, r0)
+                       opset(ASTBCCC, r0)
 
                        opset(ASTDCCC, r0)
 
@@ -1202,6 +1203,7 @@ func buildop(ctxt *obj.Link) {
 
                case ASYNC:
                        opset(AISYNC, r0)
+                       opset(ALWSYNC, r0)
                        opset(APTESYNC, r0)
                        opset(ATLBSYNC, r0)
 
@@ -1228,6 +1230,7 @@ func buildop(ctxt *obj.Link) {
                        opset(AFMOVSU, r0)
 
                case AECIWX:
+                       opset(ALBAR, r0)
                        opset(ALWAR, r0)
                        opset(ALDAR, r0)
 
@@ -3001,6 +3004,9 @@ func oprrr(ctxt *obj.Link, a obj.As) uint32 {
 
        case ASYNC:
                return OPVCC(31, 598, 0, 0)
+       case ALWSYNC:
+               return OPVCC(31, 598, 0, 0) | 1<<21
+
        case APTESYNC:
                return OPVCC(31, 598, 0, 0) | 2<<21
 
@@ -3246,6 +3252,8 @@ func oploadx(ctxt *obj.Link, a obj.As) uint32 {
                return OPVCC(31, 311, 0, 0) /* lhzux */
        case AECIWX:
                return OPVCC(31, 310, 0, 0) /* eciwx */
+       case ALBAR:
+               return OPVCC(31, 52, 0, 0) /* lbarx */
        case ALWAR:
                return OPVCC(31, 20, 0, 0) /* lwarx */
        case ALDAR:
@@ -3342,6 +3350,8 @@ func opstorex(ctxt *obj.Link, a obj.As) uint32 {
                return OPVCC(31, 661, 0, 0) /* stswx */
        case AMOVWBR:
                return OPVCC(31, 662, 0, 0) /* stwbrx */
+       case ASTBCCC:
+               return OPVCC(31, 694, 0, 1) /* stbcx. */
        case ASTWCCC:
                return OPVCC(31, 150, 0, 1) /* stwcx. */
        case ASTDCCC:

diff --git a/src/cmd/internal/obj/ppc64/obj9.go b/src/cmd/internal/obj/ppc64/obj9.go
index 43e4825af01cdf8472613812f288dd00009800b9..5f88307a2f1a107b31a1e54e214c488d3ae87fd0 100644 (file)
--- a/src/cmd/internal/obj/ppc64/obj9.go
+++ b/src/cmd/internal/obj/ppc64/obj9.go
@@ -301,6 +301,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym) {
                        }
 
                case ALWAR,
+                       ALBAR,
+                       ASTBCCC,
                        ASTWCCC,
                        AECIWX,
                        AECOWX,
@@ -323,6 +325,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym) {
                        ASYNC,
                        ATLBSYNC,
                        APTESYNC,
+                       ALWSYNC,
                        ATW,
                        AWORD,
                        ARFI,

diff --git a/src/runtime/atomic_ppc64x.s b/src/runtime/atomic_ppc64x.s
index 7cdb7466eb12ab974c639e26bf307583f3992f7d..57f672f330144963aec60fc89ff06704621883d4 100644 (file)
--- a/src/runtime/atomic_ppc64x.s
+++ b/src/runtime/atomic_ppc64x.s
@@ -10,5 +10,5 @@ TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
        // LWSYNC is the "export" barrier recommended by Power ISA
        // v2.07 book II, appendix B.2.2.2.
        // LWSYNC is a load/load, load/store, and store/store barrier.
-       WORD $0x7c2004ac        // LWSYNC
+       LWSYNC
        RET

diff --git a/src/runtime/internal/atomic/asm_ppc64x.s b/src/runtime/internal/atomic/asm_ppc64x.s
index 4a776787a29aefe427c5bdd1eaf4277d2a29c5ab..a928e400d2cf70413b4bfcf626171f2ad7703125 100644 (file)
--- a/src/runtime/internal/atomic/asm_ppc64x.s
+++ b/src/runtime/internal/atomic/asm_ppc64x.s
@@ -17,21 +17,20 @@ TEXT runtime∕internal∕atomic·Cas(SB), NOSPLIT, $0-17
        MOVD    ptr+0(FP), R3
        MOVWZ   old+8(FP), R4
        MOVWZ   new+12(FP), R5
-cas_again:
        SYNC
+cas_again:
        LWAR    (R3), R6
        CMPW    R6, R4
        BNE     cas_fail
        STWCCC  R5, (R3)
        BNE     cas_again
        MOVD    $1, R3
-       SYNC
        ISYNC
        MOVB    R3, ret+16(FP)
        RET
 cas_fail:
-       MOVD    $0, R3
-       BR      -5(PC)
+       MOVB    R0, ret+16(FP)
+       RET
 
 // bool        runtime∕internal∕atomic·Cas64(uint64 *ptr, uint64 old, uint64 new)
 // Atomically:
@@ -45,21 +44,20 @@ TEXT runtime∕internal∕atomic·Cas64(SB), NOSPLIT, $0-25
        MOVD    ptr+0(FP), R3
        MOVD    old+8(FP), R4
        MOVD    new+16(FP), R5
-cas64_again:
        SYNC
+cas64_again:
        LDAR    (R3), R6
        CMP     R6, R4
        BNE     cas64_fail
        STDCCC  R5, (R3)
        BNE     cas64_again
        MOVD    $1, R3
-       SYNC
        ISYNC
        MOVB    R3, ret+24(FP)
        RET
 cas64_fail:
-       MOVD    $0, R3
-       BR      -5(PC)
+       MOVB    R0, ret+24(FP)
+       RET
 
 TEXT runtime∕internal∕atomic·Casuintptr(SB), NOSPLIT, $0-25
        BR      runtime∕internal∕atomic·Cas64(SB)
@@ -103,8 +101,7 @@ TEXT runtime∕internal∕atomic·Xadd(SB), NOSPLIT, $0-20
        LWAR    (R4), R3
        ADD     R5, R3
        STWCCC  R3, (R4)
-       BNE     -4(PC)
-       SYNC
+       BNE     -3(PC)
        ISYNC
        MOVW    R3, ret+16(FP)
        RET
@@ -116,8 +113,7 @@ TEXT runtime∕internal∕atomic·Xadd64(SB), NOSPLIT, $0-24
        LDAR    (R4), R3
        ADD     R5, R3
        STDCCC  R3, (R4)
-       BNE     -4(PC)
-       SYNC
+       BNE     -3(PC)
        ISYNC
        MOVD    R3, ret+16(FP)
        RET
@@ -128,8 +124,7 @@ TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-20
        SYNC
        LWAR    (R4), R3
        STWCCC  R5, (R4)
-       BNE     -3(PC)
-       SYNC
+       BNE     -2(PC)
        ISYNC
        MOVW    R3, ret+16(FP)
        RET
@@ -140,8 +135,7 @@ TEXT runtime∕internal∕atomic·Xchg64(SB), NOSPLIT, $0-24
        SYNC
        LDAR    (R4), R3
        STDCCC  R5, (R4)
-       BNE     -3(PC)
-       SYNC
+       BNE     -2(PC)
        ISYNC
        MOVD    R3, ret+16(FP)
        RET
@@ -171,26 +165,12 @@ TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16
 TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-9
        MOVD    ptr+0(FP), R3
        MOVBZ   val+8(FP), R4
-       // Align ptr down to 4 bytes so we can use 32-bit load/store.
-       // R5 = (R3 << 0) & ~3
-       RLDCR   $0, R3, $~3, R5
-       // Compute val shift.
-#ifdef GOARCH_ppc64
-       // Big endian.  ptr = ptr ^ 3
-       XOR     $3, R3
-#endif
-       // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8)
-       RLDC    $3, R3, $(3*8), R6
-       // Shift val for aligned ptr. R4 = val << R6
-       SLD     R6, R4, R4
-
-again:
        SYNC
-       LWAR    (R5), R6
+again:
+       LBAR    (R3), R6
        OR      R4, R6
-       STWCCC  R6, (R5)
+       STBCCC  R6, (R3)
        BNE     again
-       SYNC
        ISYNC
        RET
 
@@ -198,28 +178,11 @@ again:
 TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-9
        MOVD    ptr+0(FP), R3
        MOVBZ   val+8(FP), R4
-       // Align ptr down to 4 bytes so we can use 32-bit load/store.
-       // R5 = (R3 << 0) & ~3
-       RLDCR   $0, R3, $~3, R5
-       // Compute val shift.
-#ifdef GOARCH_ppc64
-       // Big endian.  ptr = ptr ^ 3
-       XOR     $3, R3
-#endif
-       // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8)
-       RLDC    $3, R3, $(3*8), R6
-       // Shift val for aligned ptr. R4 = val << R6 | ^(0xFF << R6)
-       MOVD    $0xFF, R7
-       SLD     R6, R4
-       SLD     R6, R7
-       XOR $-1, R7
-       OR      R7, R4
-again:
        SYNC
-       LWAR    (R5), R6
+again:
+       LBAR    (R3), R6
        AND     R4, R6
-       STWCCC  R6, (R5)
+       STBCCC  R6, (R3)
        BNE     again
-       SYNC
        ISYNC
        RET

diff --git a/src/sync/atomic/asm_ppc64x.s b/src/sync/atomic/asm_ppc64x.s
index ed348458b49ceb08744d5ef136864a8bb9f77a61..2474e96435bedbb82bc305b7db938927db761370 100644 (file)
--- a/src/sync/atomic/asm_ppc64x.s
+++ b/src/sync/atomic/asm_ppc64x.s
@@ -15,8 +15,7 @@ TEXT ·SwapUint32(SB),NOSPLIT,$0-20
        SYNC
        LWAR    (R3), R5
        STWCCC  R4, (R3)
-       BNE     -3(PC)
-       SYNC
+       BNE     -2(PC)
        ISYNC
        MOVW    R5, old+16(FP)
        RET
@@ -30,8 +29,7 @@ TEXT ·SwapUint64(SB),NOSPLIT,$0-24
        SYNC
        LDAR    (R3), R5
        STDCCC  R4, (R3)
-       BNE     -3(PC)
-       SYNC
+       BNE     -2(PC)
        ISYNC
        MOVD    R5, old+16(FP)
        RET
@@ -49,10 +47,9 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-17
        SYNC
        LWAR    (R3), R6
        CMPW    R6, R4
-       BNE     8(PC)
+       BNE     7(PC)
        STWCCC  R5, (R3)
-       BNE     -5(PC)
-       SYNC
+       BNE     -4(PC)
        ISYNC
        MOVD    $1, R3
        MOVB    R3, swapped+16(FP)
@@ -73,10 +70,9 @@ TEXT ·CompareAndSwapUint64(SB),NOSPLIT,$0-25
        SYNC
        LDAR    (R3), R6
        CMP     R6, R4
-       BNE     8(PC)
+       BNE     7(PC)
        STDCCC  R5, (R3)
-       BNE     -5(PC)
-       SYNC
+       BNE     -4(PC)
        ISYNC
        MOVD    $1, R3
        MOVB    R3, swapped+24(FP)
@@ -94,8 +90,7 @@ TEXT ·AddUint32(SB),NOSPLIT,$0-20
        LWAR    (R3), R5
        ADD     R4, R5
        STWCCC  R5, (R3)
-       BNE     -4(PC)
-       SYNC
+       BNE     -3(PC)
        ISYNC
        MOVW    R5, ret+16(FP)
        RET
@@ -113,8 +108,7 @@ TEXT ·AddUint64(SB),NOSPLIT,$0-24
        LDAR    (R3), R5
        ADD     R4, R5
        STDCCC  R5, (R3)
-       BNE     -4(PC)
-       SYNC
+       BNE     -3(PC)
        ISYNC
        MOVD    R5, ret+16(FP)
        RET