cmd/compile: add internal/runtime/atomic.Xchg8 intrinsic for PPC64

This is minor extension of the existing support for 32 and
64 bit types.

For #69735

Change-Id: I6828ec223951d2b692e077dc507b000ac23c32a1
Reviewed-on: https://go-review.googlesource.com/c/go/+/617496
Reviewed-by: Rhys Hiltner <rhys.hiltner@gmail.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Reviewed-by: Keith Randall <khr@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Mauri de Souza Meneguzzo <mauri870@gmail.com>
Reviewed-by: Keith Randall <khr@google.com>

diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go
index 0c5137f97db4287fc28e12ca9fde2f954f032771..53ec4289c7bec20777596bb0583f396cfd08f71a 100644 (file)
--- a/src/cmd/compile/internal/ppc64/ssa.go
+++ b/src/cmd/compile/internal/ppc64/ssa.go
@@ -223,16 +223,21 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                        p5.From.Reg = out
                }
 
-       case ssa.OpPPC64LoweredAtomicExchange32,
+       case ssa.OpPPC64LoweredAtomicExchange8,
+               ssa.OpPPC64LoweredAtomicExchange32,
                ssa.OpPPC64LoweredAtomicExchange64:
                // LWSYNC
-               // LDAR/LWAR    (Rarg0), Rout
-               // STDCCC/STWCCC Rout, (Rarg0)
+               // LDAR/LWAR/LBAR        (Rarg0), Rout
+               // STDCCC/STWCCC/STBWCCC Rout, (Rarg0)
                // BNE         -2(PC)
                // ISYNC
                ld := ppc64.ALDAR
                st := ppc64.ASTDCCC
-               if v.Op == ssa.OpPPC64LoweredAtomicExchange32 {
+               switch v.Op {
+               case ssa.OpPPC64LoweredAtomicExchange8:
+                       ld = ppc64.ALBAR
+                       st = ppc64.ASTBCCC
+               case ssa.OpPPC64LoweredAtomicExchange32:
                        ld = ppc64.ALWAR
                        st = ppc64.ASTWCCC
                }
@@ -243,13 +248,13 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                // caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b.
                plwsync := s.Prog(ppc64.ALWSYNC)
                plwsync.To.Type = obj.TYPE_NONE
-               // LDAR or LWAR
+               // L[B|W|D]AR
                p := s.Prog(ld)
                p.From.Type = obj.TYPE_MEM
                p.From.Reg = r0
                p.To.Type = obj.TYPE_REG
                p.To.Reg = out
-               // STDCCC or STWCCC
+               // ST[B|W|D]CCC
                p1 := s.Prog(st)
                p1.From.Type = obj.TYPE_REG
                p1.From.Reg = r1

diff --git a/src/cmd/compile/internal/ssa/_gen/PPC64.rules b/src/cmd/compile/internal/ssa/_gen/PPC64.rules
index 1ff60823b481e095ed87e3ee1666d339250c0360..ebd152f578827a6acf67a11977fe25eb61e9e3a7 100644 (file)
--- a/src/cmd/compile/internal/ssa/_gen/PPC64.rules
+++ b/src/cmd/compile/internal/ssa/_gen/PPC64.rules
@@ -806,7 +806,7 @@
 (AtomicStore(8|32|64)    ptr val mem) => (LoweredAtomicStore(8|32|64) [1] ptr val mem)
 (AtomicStoreRel(32|64)   ptr val mem) => (LoweredAtomicStore(32|64) [0] ptr val mem)
 
-(AtomicExchange(32|64) ...) => (LoweredAtomicExchange(32|64) ...)
+(AtomicExchange(8|32|64) ...) => (LoweredAtomicExchange(8|32|64) ...)
 
 (AtomicAdd(32|64) ...) => (LoweredAtomicAdd(32|64) ...)
 

diff --git a/src/cmd/compile/internal/ssa/_gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/_gen/PPC64Ops.go
index 719bfeb6f4d597821a260646dc9a4f66974432a8..f4212c15af7f0de2c86ef44bd13f44ab8f45fedd 100644 (file)
--- a/src/cmd/compile/internal/ssa/_gen/PPC64Ops.go
+++ b/src/cmd/compile/internal/ssa/_gen/PPC64Ops.go
@@ -658,13 +658,14 @@ func init() {
                {name: "LoweredAtomicAdd32", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true},
                {name: "LoweredAtomicAdd64", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true},
 
-               // atomic exchange32, 64
+               // atomic exchange8, 32, 64
                // LWSYNC
                // LDAR         (Rarg0), Rout
                // STDCCC       Rarg1, (Rarg0)
                // BNE          -2(PC)
                // ISYNC
                // return old val
+               {name: "LoweredAtomicExchange8", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true},
                {name: "LoweredAtomicExchange32", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true},
                {name: "LoweredAtomicExchange64", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true},
 

diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go
index a96d07a9a3248b5ed9897dc264466c927f38a525..fbc9ae7ea55810a6ebbb6dd88d3c4d4f94dc6263 100644 (file)
--- a/src/cmd/compile/internal/ssa/opGen.go
+++ b/src/cmd/compile/internal/ssa/opGen.go
@@ -2349,6 +2349,7 @@ const (
        OpPPC64LoweredAtomicLoadPtr
        OpPPC64LoweredAtomicAdd32
        OpPPC64LoweredAtomicAdd64
+       OpPPC64LoweredAtomicExchange8
        OpPPC64LoweredAtomicExchange32
        OpPPC64LoweredAtomicExchange64
        OpPPC64LoweredAtomicCas64
@@ -31663,6 +31664,23 @@ var opcodeTable = [...]opInfo{
                        },
                },
        },
+       {
+               name:            "LoweredAtomicExchange8",
+               argLen:          3,
+               resultNotInArgs: true,
+               clobberFlags:    true,
+               faultOnNilArg0:  true,
+               hasSideEffects:  true,
+               reg: regInfo{
+                       inputs: []inputInfo{
+                               {1, 1073733624}, // R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29
+                               {0, 1073733630}, // SP SB R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29
+                       },
+                       outputs: []outputInfo{
+                               {0, 1073733624}, // R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29
+                       },
+               },
+       },
        {
                name:            "LoweredAtomicExchange32",
                argLen:          3,

diff --git a/src/cmd/compile/internal/ssa/rewritePPC64.go b/src/cmd/compile/internal/ssa/rewritePPC64.go
index 62bba98068bc1e8b7128385fe1c346294a18097e..5f8f2a2c99a6e944ad4601b96788e2c88d7ebf60 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewritePPC64.go
+++ b/src/cmd/compile/internal/ssa/rewritePPC64.go
@@ -73,6 +73,9 @@ func rewriteValuePPC64(v *Value) bool {
        case OpAtomicExchange64:
                v.Op = OpPPC64LoweredAtomicExchange64
                return true
+       case OpAtomicExchange8:
+               v.Op = OpPPC64LoweredAtomicExchange8
+               return true
        case OpAtomicLoad32:
                return rewriteValuePPC64_OpAtomicLoad32(v)
        case OpAtomicLoad64:

diff --git a/src/cmd/compile/internal/ssagen/intrinsics.go b/src/cmd/compile/internal/ssagen/intrinsics.go
index 4bf4d3006cb4a08d089e267be45faef4aa0ade9e..20581803d9121928fce5696f0597dadf987c854c 100644 (file)
--- a/src/cmd/compile/internal/ssagen/intrinsics.go
+++ b/src/cmd/compile/internal/ssagen/intrinsics.go
@@ -302,7 +302,7 @@ func initIntrinsics(cfg *intrinsicBuildConfig) {
                        s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
                        return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v)
                },
-               sys.AMD64)
+               sys.AMD64, sys.PPC64)
        addF("internal/runtime/atomic", "Xchg",
                func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
                        v := s.newValue3(ssa.OpAtomicExchange32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())

diff --git a/src/cmd/compile/internal/ssagen/intrinsics_test.go b/src/cmd/compile/internal/ssagen/intrinsics_test.go
index d5c55ae2788a74e6c19853702efe4ac9e877a587..579f346f4947b73fcf3374eda1182d7e700e5a91 100644 (file)
--- a/src/cmd/compile/internal/ssagen/intrinsics_test.go
+++ b/src/cmd/compile/internal/ssagen/intrinsics_test.go
@@ -783,6 +783,7 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{
        {"ppc64", "internal/runtime/atomic", "Xaddint32"}:          struct{}{},
        {"ppc64", "internal/runtime/atomic", "Xaddint64"}:          struct{}{},
        {"ppc64", "internal/runtime/atomic", "Xadduintptr"}:        struct{}{},
+       {"ppc64", "internal/runtime/atomic", "Xchg8"}:              struct{}{},
        {"ppc64", "internal/runtime/atomic", "Xchg"}:               struct{}{},
        {"ppc64", "internal/runtime/atomic", "Xchg64"}:             struct{}{},
        {"ppc64", "internal/runtime/atomic", "Xchgint32"}:          struct{}{},
@@ -903,6 +904,7 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{
        {"ppc64le", "internal/runtime/atomic", "Xaddint32"}:        struct{}{},
        {"ppc64le", "internal/runtime/atomic", "Xaddint64"}:        struct{}{},
        {"ppc64le", "internal/runtime/atomic", "Xadduintptr"}:      struct{}{},
+       {"ppc64le", "internal/runtime/atomic", "Xchg8"}:            struct{}{},
        {"ppc64le", "internal/runtime/atomic", "Xchg"}:             struct{}{},
        {"ppc64le", "internal/runtime/atomic", "Xchg64"}:           struct{}{},
        {"ppc64le", "internal/runtime/atomic", "Xchgint32"}:        struct{}{},

diff --git a/src/internal/runtime/atomic/atomic_ppc64x.go b/src/internal/runtime/atomic/atomic_ppc64x.go
index 33a92b53f47ff051fafe2c89dc2925b263f9cb1b..590ba03ecf5c9f94a4f2afab7b3beadce5845024 100644 (file)
--- a/src/internal/runtime/atomic/atomic_ppc64x.go
+++ b/src/internal/runtime/atomic/atomic_ppc64x.go
@@ -17,6 +17,9 @@ func Xadd64(ptr *uint64, delta int64) uint64
 //go:noescape
 func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
 
+//go:noescape
+func Xchg8(ptr *uint8, new uint8) uint8
+
 //go:noescape
 func Xchg(ptr *uint32, new uint32) uint32
 

diff --git a/src/internal/runtime/atomic/atomic_ppc64x.s b/src/internal/runtime/atomic/atomic_ppc64x.s
index 75635b933d7423a9972da23ac54e40639c399643..184a30c970c0c74bc9ca321d4ef33ae563d81a72 100644 (file)
--- a/src/internal/runtime/atomic/atomic_ppc64x.s
+++ b/src/internal/runtime/atomic/atomic_ppc64x.s
@@ -236,6 +236,22 @@ TEXT ·Xadd64(SB), NOSPLIT, $0-24
        MOVD    R3, ret+16(FP)
        RET
 
+// uint8 Xchg(ptr *uint8, new uint8)
+// Atomically:
+//     old := *ptr;
+//     *ptr = new;
+//     return old;
+TEXT ·Xchg8(SB), NOSPLIT, $0-17
+       MOVD    ptr+0(FP), R4
+       MOVB    new+8(FP), R5
+       LWSYNC
+       LBAR    (R4), R3
+       STBCCC  R5, (R4)
+       BNE     -2(PC)
+       ISYNC
+       MOVB    R3, ret+16(FP)
+       RET
+
 // uint32 Xchg(ptr *uint32, new uint32)
 // Atomically:
 //     old := *ptr;

diff --git a/src/internal/runtime/atomic/xchg8_test.go b/src/internal/runtime/atomic/xchg8_test.go
index 139062422ec5d4489221cd24121a59953d722293..b0b39c2dd7f6757a015320f24efd3552ca30fd8d 100644 (file)
--- a/src/internal/runtime/atomic/xchg8_test.go
+++ b/src/internal/runtime/atomic/xchg8_test.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build amd64
+//go:build amd64 || ppc64 || ppc64le
 
 package atomic_test