runtime/internal/atomic: Use power5 compatible instructions for ppc64

This modifies a recent performance improvement to the
And8 and Or8 atomic functions which required both ppc64le
and ppc64 to use power8 instructions. Since then it was
decided that ppc64 (BE) should work for power5 and later.
This change uses instructions compatible with power5 for
ppc64 and uses power8 for ppc64le.

Fixes #16004

Change-Id: I623c75e8e6fd1fa063a53d250d86cdc9d0890dc7
Reviewed-on: https://go-review.googlesource.com/24181
Reviewed-by: Keith Randall <khr@golang.org>
Run-TryBot: Andrew Gerrand <adg@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

diff --git a/src/runtime/internal/atomic/asm_ppc64x.s b/src/runtime/internal/atomic/asm_ppc64x.s
index a928e400d2cf70413b4bfcf626171f2ad7703125..de4f895efded2a9287192c73a127a046fc58e0fe 100644 (file)
--- a/src/runtime/internal/atomic/asm_ppc64x.s
+++ b/src/runtime/internal/atomic/asm_ppc64x.s
@@ -161,28 +161,71 @@ TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16
        MOVD    R4, 0(R3)
        RET
 
-// void        runtime∕internal∕atomic·Or8(byte volatile*, byte);
+// void runtime∕internal∕atomic·Or8(byte volatile*, byte);
 TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-9
        MOVD    ptr+0(FP), R3
        MOVBZ   val+8(FP), R4
+#ifdef  GOARCH_ppc64
+       // Align ptr down to 4 bytes so we can use 32-bit load/store.
+       // R5 = (R3 << 0) & ~3
+       RLDCR   $0, R3, $~3, R5
+       // Compute val shift.
+       // Big endian.  ptr = ptr ^ 3
+       XOR     $3, R3
+       // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8)
+       RLDC    $3, R3, $(3*8), R6
+       // Shift val for aligned ptr.  R4 = val << R6
+       SLD     R6, R4, R4
+       SYNC
+
+again:
+       LWAR    (R5), R6
+       OR      R4, R6
+       STWCCC  R6, (R5)
+       BNE     again
+#else
        SYNC
 again:
        LBAR    (R3), R6
        OR      R4, R6
        STBCCC  R6, (R3)
        BNE     again
+#endif
        ISYNC
        RET
 
-// void        runtime∕internal∕atomic·And8(byte volatile*, byte);
+// void runtime∕internal∕atomic·And8(byte volatile*, byte);
 TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-9
        MOVD    ptr+0(FP), R3
        MOVBZ   val+8(FP), R4
+#ifdef  GOARCH_ppc64
+       // Align ptr down to 4 bytes so we can use 32-bit load/store.
+       // R5 = (R3 << 0) & ~3
+       RLDCR   $0, R3, $~3, R5
+       // Compute val shift.
+       // Big endian.  ptr = ptr ^ 3
+       XOR     $3, R3
+       // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8)
+       RLDC    $3, R3, $(3*8), R6
+       // Shift val for aligned ptr.  R4 = val << R6 | ^(0xFF << R6)
+       MOVD    $0xFF, R7
+       SLD     R6, R4
+       SLD     R6, R7
+       XOR     $-1, R7
+       OR      R7, R4
        SYNC
 again:
-       LBAR    (R3), R6
+       LWAR    (R5), R6
        AND     R4, R6
-       STBCCC  R6, (R3)
+       STWCCC  R6, (R5)
+       BNE     again
+#else
+       SYNC
+again:
+       LBAR    (R3),R6
+       AND     R4,R6
+       STBCCC  R6,(R3)
        BNE     again
+#endif
        ISYNC
        RET