From: Lynn Boger Date: Thu, 28 Apr 2016 12:16:08 +0000 (-0500) Subject: sync/atomic, runtime/internal/atomic: improve ppc64x atomics X-Git-Tag: go1.7beta1~321 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=eeca3ba92fdb07e44abf3e2bebfcede03e1eae12;p=gostls13.git sync/atomic, runtime/internal/atomic: improve ppc64x atomics The following performance improvements have been made to the low-level atomic functions for ppc64le & ppc64: - For those cases containing a lwarx and stwcx (or other sizes): sync, lwarx, maybe something, stwcx, loop to sync, sync, isync The sync is moved before (outside) the lwarx/stwcx loop, and the sync after is removed, so it becomes: sync, lwarx, maybe something, stwcx, loop to lwarx, isync - For the Or8 and And8, the shifting and manipulation of the address to the word aligned version were removed and the instructions were changed to use lbarx, stbcx instead of register shifting, xor, then lwarx, stwcx. - New instructions LWSYNC, LBAR, STBCC were tested and added. runtime/atomic_ppc64x.s was changed to use the LWSYNC opcode instead of the WORD encoding. Fixes #15469 Ran some of the benchmarks in the runtime and sync directories. Some results varied from run to run but the trend was improvement based on best times for base and new: runtime.test: BenchmarkChanNonblocking-128 0.88 0.89 +1.14% BenchmarkChanUncontended-128 569 511 -10.19% BenchmarkChanContended-128 63110 53231 -15.65% BenchmarkChanSync-128 691 598 -13.46% BenchmarkChanSyncWork-128 11355 11649 +2.59% BenchmarkChanProdCons0-128 2402 2090 -12.99% BenchmarkChanProdCons10-128 1348 1363 +1.11% BenchmarkChanProdCons100-128 1002 746 -25.55% BenchmarkChanProdConsWork0-128 2554 2720 +6.50% BenchmarkChanProdConsWork10-128 1909 1804 -5.50% BenchmarkChanProdConsWork100-128 1624 1580 -2.71% BenchmarkChanCreation-128 237 212 -10.55% BenchmarkChanSem-128 705 667 -5.39% BenchmarkChanPopular-128 5081190 4497566 -11.49% BenchmarkCreateGoroutines-128 532 473 -11.09% BenchmarkCreateGoroutinesParallel-128 35.0 34.7 -0.86% BenchmarkCreateGoroutinesCapture-128 4923 4200 -14.69% sync.test: BenchmarkUncontendedSemaphore-128 112 94.2 -15.89% BenchmarkContendedSemaphore-128 133 128 -3.76% BenchmarkMutexUncontended-128 1.90 1.67 -12.11% BenchmarkMutex-128 353 310 -12.18% BenchmarkMutexSlack-128 304 283 -6.91% BenchmarkMutexWork-128 554 541 -2.35% BenchmarkMutexWorkSlack-128 567 556 -1.94% BenchmarkMutexNoSpin-128 275 242 -12.00% BenchmarkMutexSpin-128 1129 1030 -8.77% BenchmarkOnce-128 1.08 0.96 -11.11% BenchmarkPool-128 29.8 27.4 -8.05% BenchmarkPoolOverflow-128 40564 36583 -9.81% BenchmarkSemaUncontended-128 3.14 2.63 -16.24% BenchmarkSemaSyntNonblock-128 1087 1069 -1.66% BenchmarkSemaSyntBlock-128 897 893 -0.45% BenchmarkSemaWorkNonblock-128 1034 1028 -0.58% BenchmarkSemaWorkBlock-128 949 886 -6.64% Change-Id: I4403fb29d3cd5254b7b1ce87a216bd11b391079e Reviewed-on: https://go-review.googlesource.com/22549 Reviewed-by: Michael Munday Reviewed-by: Minux Ma --- diff --git a/src/cmd/internal/obj/ppc64/a.out.go b/src/cmd/internal/obj/ppc64/a.out.go index 8cc984260e..8e58c59238 100644 --- a/src/cmd/internal/obj/ppc64/a.out.go +++ b/src/cmd/internal/obj/ppc64/a.out.go @@ -342,8 +342,10 @@ const ( AFSUBS AFSUBSCC AMOVMW + ALBAR ALSW ALWAR + ALWSYNC AMOVWBR AMOVB AMOVBU @@ -401,6 +403,7 @@ const ( ASRAW ASRAWCC ASRWCC + ASTBCCC ASTSW ASTWCCC ASUB diff --git a/src/cmd/internal/obj/ppc64/anames.go b/src/cmd/internal/obj/ppc64/anames.go index 868700f012..eb42c9a953 100644 --- a/src/cmd/internal/obj/ppc64/anames.go +++ b/src/cmd/internal/obj/ppc64/anames.go @@ -118,8 +118,10 @@ var Anames = []string{ "FSUBS", "FSUBSCC", "MOVMW", + "LBAR", "LSW", "LWAR", + "LWSYNC", "MOVWBR", "MOVB", "MOVBU", @@ -177,6 +179,7 @@ var Anames = []string{ "SRAW", "SRAWCC", "SRWCC", + "STBCCC", "STSW", "STWCCC", "SUB", diff --git a/src/cmd/internal/obj/ppc64/asm9.go b/src/cmd/internal/obj/ppc64/asm9.go index e847ec341f..5366809d2f 100644 --- a/src/cmd/internal/obj/ppc64/asm9.go +++ b/src/cmd/internal/obj/ppc64/asm9.go @@ -933,6 +933,7 @@ func buildop(ctxt *obj.Link) { case AECOWX: /* indexed store: op s,(b+a); op s,(b) */ opset(ASTWCCC, r0) + opset(ASTBCCC, r0) opset(ASTDCCC, r0) @@ -1202,6 +1203,7 @@ func buildop(ctxt *obj.Link) { case ASYNC: opset(AISYNC, r0) + opset(ALWSYNC, r0) opset(APTESYNC, r0) opset(ATLBSYNC, r0) @@ -1228,6 +1230,7 @@ func buildop(ctxt *obj.Link) { opset(AFMOVSU, r0) case AECIWX: + opset(ALBAR, r0) opset(ALWAR, r0) opset(ALDAR, r0) @@ -3001,6 +3004,9 @@ func oprrr(ctxt *obj.Link, a obj.As) uint32 { case ASYNC: return OPVCC(31, 598, 0, 0) + case ALWSYNC: + return OPVCC(31, 598, 0, 0) | 1<<21 + case APTESYNC: return OPVCC(31, 598, 0, 0) | 2<<21 @@ -3246,6 +3252,8 @@ func oploadx(ctxt *obj.Link, a obj.As) uint32 { return OPVCC(31, 311, 0, 0) /* lhzux */ case AECIWX: return OPVCC(31, 310, 0, 0) /* eciwx */ + case ALBAR: + return OPVCC(31, 52, 0, 0) /* lbarx */ case ALWAR: return OPVCC(31, 20, 0, 0) /* lwarx */ case ALDAR: @@ -3342,6 +3350,8 @@ func opstorex(ctxt *obj.Link, a obj.As) uint32 { return OPVCC(31, 661, 0, 0) /* stswx */ case AMOVWBR: return OPVCC(31, 662, 0, 0) /* stwbrx */ + case ASTBCCC: + return OPVCC(31, 694, 0, 1) /* stbcx. */ case ASTWCCC: return OPVCC(31, 150, 0, 1) /* stwcx. */ case ASTDCCC: diff --git a/src/cmd/internal/obj/ppc64/obj9.go b/src/cmd/internal/obj/ppc64/obj9.go index 43e4825af0..5f88307a2f 100644 --- a/src/cmd/internal/obj/ppc64/obj9.go +++ b/src/cmd/internal/obj/ppc64/obj9.go @@ -301,6 +301,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym) { } case ALWAR, + ALBAR, + ASTBCCC, ASTWCCC, AECIWX, AECOWX, @@ -323,6 +325,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym) { ASYNC, ATLBSYNC, APTESYNC, + ALWSYNC, ATW, AWORD, ARFI, diff --git a/src/runtime/atomic_ppc64x.s b/src/runtime/atomic_ppc64x.s index 7cdb7466eb..57f672f330 100644 --- a/src/runtime/atomic_ppc64x.s +++ b/src/runtime/atomic_ppc64x.s @@ -10,5 +10,5 @@ TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 // LWSYNC is the "export" barrier recommended by Power ISA // v2.07 book II, appendix B.2.2.2. // LWSYNC is a load/load, load/store, and store/store barrier. - WORD $0x7c2004ac // LWSYNC + LWSYNC RET diff --git a/src/runtime/internal/atomic/asm_ppc64x.s b/src/runtime/internal/atomic/asm_ppc64x.s index 4a776787a2..a928e400d2 100644 --- a/src/runtime/internal/atomic/asm_ppc64x.s +++ b/src/runtime/internal/atomic/asm_ppc64x.s @@ -17,21 +17,20 @@ TEXT runtime∕internal∕atomic·Cas(SB), NOSPLIT, $0-17 MOVD ptr+0(FP), R3 MOVWZ old+8(FP), R4 MOVWZ new+12(FP), R5 -cas_again: SYNC +cas_again: LWAR (R3), R6 CMPW R6, R4 BNE cas_fail STWCCC R5, (R3) BNE cas_again MOVD $1, R3 - SYNC ISYNC MOVB R3, ret+16(FP) RET cas_fail: - MOVD $0, R3 - BR -5(PC) + MOVB R0, ret+16(FP) + RET // bool runtime∕internal∕atomic·Cas64(uint64 *ptr, uint64 old, uint64 new) // Atomically: @@ -45,21 +44,20 @@ TEXT runtime∕internal∕atomic·Cas64(SB), NOSPLIT, $0-25 MOVD ptr+0(FP), R3 MOVD old+8(FP), R4 MOVD new+16(FP), R5 -cas64_again: SYNC +cas64_again: LDAR (R3), R6 CMP R6, R4 BNE cas64_fail STDCCC R5, (R3) BNE cas64_again MOVD $1, R3 - SYNC ISYNC MOVB R3, ret+24(FP) RET cas64_fail: - MOVD $0, R3 - BR -5(PC) + MOVB R0, ret+24(FP) + RET TEXT runtime∕internal∕atomic·Casuintptr(SB), NOSPLIT, $0-25 BR runtime∕internal∕atomic·Cas64(SB) @@ -103,8 +101,7 @@ TEXT runtime∕internal∕atomic·Xadd(SB), NOSPLIT, $0-20 LWAR (R4), R3 ADD R5, R3 STWCCC R3, (R4) - BNE -4(PC) - SYNC + BNE -3(PC) ISYNC MOVW R3, ret+16(FP) RET @@ -116,8 +113,7 @@ TEXT runtime∕internal∕atomic·Xadd64(SB), NOSPLIT, $0-24 LDAR (R4), R3 ADD R5, R3 STDCCC R3, (R4) - BNE -4(PC) - SYNC + BNE -3(PC) ISYNC MOVD R3, ret+16(FP) RET @@ -128,8 +124,7 @@ TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-20 SYNC LWAR (R4), R3 STWCCC R5, (R4) - BNE -3(PC) - SYNC + BNE -2(PC) ISYNC MOVW R3, ret+16(FP) RET @@ -140,8 +135,7 @@ TEXT runtime∕internal∕atomic·Xchg64(SB), NOSPLIT, $0-24 SYNC LDAR (R4), R3 STDCCC R5, (R4) - BNE -3(PC) - SYNC + BNE -2(PC) ISYNC MOVD R3, ret+16(FP) RET @@ -171,26 +165,12 @@ TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-9 MOVD ptr+0(FP), R3 MOVBZ val+8(FP), R4 - // Align ptr down to 4 bytes so we can use 32-bit load/store. - // R5 = (R3 << 0) & ~3 - RLDCR $0, R3, $~3, R5 - // Compute val shift. -#ifdef GOARCH_ppc64 - // Big endian. ptr = ptr ^ 3 - XOR $3, R3 -#endif - // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8) - RLDC $3, R3, $(3*8), R6 - // Shift val for aligned ptr. R4 = val << R6 - SLD R6, R4, R4 - -again: SYNC - LWAR (R5), R6 +again: + LBAR (R3), R6 OR R4, R6 - STWCCC R6, (R5) + STBCCC R6, (R3) BNE again - SYNC ISYNC RET @@ -198,28 +178,11 @@ again: TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-9 MOVD ptr+0(FP), R3 MOVBZ val+8(FP), R4 - // Align ptr down to 4 bytes so we can use 32-bit load/store. - // R5 = (R3 << 0) & ~3 - RLDCR $0, R3, $~3, R5 - // Compute val shift. -#ifdef GOARCH_ppc64 - // Big endian. ptr = ptr ^ 3 - XOR $3, R3 -#endif - // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8) - RLDC $3, R3, $(3*8), R6 - // Shift val for aligned ptr. R4 = val << R6 | ^(0xFF << R6) - MOVD $0xFF, R7 - SLD R6, R4 - SLD R6, R7 - XOR $-1, R7 - OR R7, R4 -again: SYNC - LWAR (R5), R6 +again: + LBAR (R3), R6 AND R4, R6 - STWCCC R6, (R5) + STBCCC R6, (R3) BNE again - SYNC ISYNC RET diff --git a/src/sync/atomic/asm_ppc64x.s b/src/sync/atomic/asm_ppc64x.s index ed348458b4..2474e96435 100644 --- a/src/sync/atomic/asm_ppc64x.s +++ b/src/sync/atomic/asm_ppc64x.s @@ -15,8 +15,7 @@ TEXT ·SwapUint32(SB),NOSPLIT,$0-20 SYNC LWAR (R3), R5 STWCCC R4, (R3) - BNE -3(PC) - SYNC + BNE -2(PC) ISYNC MOVW R5, old+16(FP) RET @@ -30,8 +29,7 @@ TEXT ·SwapUint64(SB),NOSPLIT,$0-24 SYNC LDAR (R3), R5 STDCCC R4, (R3) - BNE -3(PC) - SYNC + BNE -2(PC) ISYNC MOVD R5, old+16(FP) RET @@ -49,10 +47,9 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-17 SYNC LWAR (R3), R6 CMPW R6, R4 - BNE 8(PC) + BNE 7(PC) STWCCC R5, (R3) - BNE -5(PC) - SYNC + BNE -4(PC) ISYNC MOVD $1, R3 MOVB R3, swapped+16(FP) @@ -73,10 +70,9 @@ TEXT ·CompareAndSwapUint64(SB),NOSPLIT,$0-25 SYNC LDAR (R3), R6 CMP R6, R4 - BNE 8(PC) + BNE 7(PC) STDCCC R5, (R3) - BNE -5(PC) - SYNC + BNE -4(PC) ISYNC MOVD $1, R3 MOVB R3, swapped+24(FP) @@ -94,8 +90,7 @@ TEXT ·AddUint32(SB),NOSPLIT,$0-20 LWAR (R3), R5 ADD R4, R5 STWCCC R5, (R3) - BNE -4(PC) - SYNC + BNE -3(PC) ISYNC MOVW R5, ret+16(FP) RET @@ -113,8 +108,7 @@ TEXT ·AddUint64(SB),NOSPLIT,$0-24 LDAR (R3), R5 ADD R4, R5 STDCCC R5, (R3) - BNE -4(PC) - SYNC + BNE -3(PC) ISYNC MOVD R5, ret+16(FP) RET