From 15618840f6d416cc9da34505a5ea5190ff15c3aa Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Wed, 2 Oct 2024 13:31:37 -0500 Subject: [PATCH] cmd/compile: add internal/runtime/atomic.Xchg8 intrinsic for PPC64 This is minor extension of the existing support for 32 and 64 bit types. For #69735 Change-Id: I6828ec223951d2b692e077dc507b000ac23c32a1 Reviewed-on: https://go-review.googlesource.com/c/go/+/617496 Reviewed-by: Rhys Hiltner Reviewed-by: Cherry Mui Reviewed-by: Keith Randall LUCI-TryBot-Result: Go LUCI Reviewed-by: Mauri de Souza Meneguzzo Reviewed-by: Keith Randall --- src/cmd/compile/internal/ppc64/ssa.go | 17 +++++++++++------ src/cmd/compile/internal/ssa/_gen/PPC64.rules | 2 +- src/cmd/compile/internal/ssa/_gen/PPC64Ops.go | 3 ++- src/cmd/compile/internal/ssa/opGen.go | 18 ++++++++++++++++++ src/cmd/compile/internal/ssa/rewritePPC64.go | 3 +++ src/cmd/compile/internal/ssagen/intrinsics.go | 2 +- .../compile/internal/ssagen/intrinsics_test.go | 2 ++ src/internal/runtime/atomic/atomic_ppc64x.go | 3 +++ src/internal/runtime/atomic/atomic_ppc64x.s | 16 ++++++++++++++++ src/internal/runtime/atomic/xchg8_test.go | 2 +- 10 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go index 0c5137f97d..53ec4289c7 100644 --- a/src/cmd/compile/internal/ppc64/ssa.go +++ b/src/cmd/compile/internal/ppc64/ssa.go @@ -223,16 +223,21 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { p5.From.Reg = out } - case ssa.OpPPC64LoweredAtomicExchange32, + case ssa.OpPPC64LoweredAtomicExchange8, + ssa.OpPPC64LoweredAtomicExchange32, ssa.OpPPC64LoweredAtomicExchange64: // LWSYNC - // LDAR/LWAR (Rarg0), Rout - // STDCCC/STWCCC Rout, (Rarg0) + // LDAR/LWAR/LBAR (Rarg0), Rout + // STDCCC/STWCCC/STBWCCC Rout, (Rarg0) // BNE -2(PC) // ISYNC ld := ppc64.ALDAR st := ppc64.ASTDCCC - if v.Op == ssa.OpPPC64LoweredAtomicExchange32 { + switch v.Op { + case ssa.OpPPC64LoweredAtomicExchange8: + ld = ppc64.ALBAR + st = ppc64.ASTBCCC + case ssa.OpPPC64LoweredAtomicExchange32: ld = ppc64.ALWAR st = ppc64.ASTWCCC } @@ -243,13 +248,13 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { // caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b. plwsync := s.Prog(ppc64.ALWSYNC) plwsync.To.Type = obj.TYPE_NONE - // LDAR or LWAR + // L[B|W|D]AR p := s.Prog(ld) p.From.Type = obj.TYPE_MEM p.From.Reg = r0 p.To.Type = obj.TYPE_REG p.To.Reg = out - // STDCCC or STWCCC + // ST[B|W|D]CCC p1 := s.Prog(st) p1.From.Type = obj.TYPE_REG p1.From.Reg = r1 diff --git a/src/cmd/compile/internal/ssa/_gen/PPC64.rules b/src/cmd/compile/internal/ssa/_gen/PPC64.rules index 1ff60823b4..ebd152f578 100644 --- a/src/cmd/compile/internal/ssa/_gen/PPC64.rules +++ b/src/cmd/compile/internal/ssa/_gen/PPC64.rules @@ -806,7 +806,7 @@ (AtomicStore(8|32|64) ptr val mem) => (LoweredAtomicStore(8|32|64) [1] ptr val mem) (AtomicStoreRel(32|64) ptr val mem) => (LoweredAtomicStore(32|64) [0] ptr val mem) -(AtomicExchange(32|64) ...) => (LoweredAtomicExchange(32|64) ...) +(AtomicExchange(8|32|64) ...) => (LoweredAtomicExchange(8|32|64) ...) (AtomicAdd(32|64) ...) => (LoweredAtomicAdd(32|64) ...) diff --git a/src/cmd/compile/internal/ssa/_gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/_gen/PPC64Ops.go index 719bfeb6f4..f4212c15af 100644 --- a/src/cmd/compile/internal/ssa/_gen/PPC64Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/PPC64Ops.go @@ -658,13 +658,14 @@ func init() { {name: "LoweredAtomicAdd32", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true}, {name: "LoweredAtomicAdd64", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true}, - // atomic exchange32, 64 + // atomic exchange8, 32, 64 // LWSYNC // LDAR (Rarg0), Rout // STDCCC Rarg1, (Rarg0) // BNE -2(PC) // ISYNC // return old val + {name: "LoweredAtomicExchange8", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true}, {name: "LoweredAtomicExchange32", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true}, {name: "LoweredAtomicExchange64", argLength: 3, reg: gpxchg, resultNotInArgs: true, clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true}, diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index a96d07a9a3..fbc9ae7ea5 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -2349,6 +2349,7 @@ const ( OpPPC64LoweredAtomicLoadPtr OpPPC64LoweredAtomicAdd32 OpPPC64LoweredAtomicAdd64 + OpPPC64LoweredAtomicExchange8 OpPPC64LoweredAtomicExchange32 OpPPC64LoweredAtomicExchange64 OpPPC64LoweredAtomicCas64 @@ -31663,6 +31664,23 @@ var opcodeTable = [...]opInfo{ }, }, }, + { + name: "LoweredAtomicExchange8", + argLen: 3, + resultNotInArgs: true, + clobberFlags: true, + faultOnNilArg0: true, + hasSideEffects: true, + reg: regInfo{ + inputs: []inputInfo{ + {1, 1073733624}, // R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + {0, 1073733630}, // SP SB R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + }, + outputs: []outputInfo{ + {0, 1073733624}, // R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + }, + }, + }, { name: "LoweredAtomicExchange32", argLen: 3, diff --git a/src/cmd/compile/internal/ssa/rewritePPC64.go b/src/cmd/compile/internal/ssa/rewritePPC64.go index 62bba98068..5f8f2a2c99 100644 --- a/src/cmd/compile/internal/ssa/rewritePPC64.go +++ b/src/cmd/compile/internal/ssa/rewritePPC64.go @@ -73,6 +73,9 @@ func rewriteValuePPC64(v *Value) bool { case OpAtomicExchange64: v.Op = OpPPC64LoweredAtomicExchange64 return true + case OpAtomicExchange8: + v.Op = OpPPC64LoweredAtomicExchange8 + return true case OpAtomicLoad32: return rewriteValuePPC64_OpAtomicLoad32(v) case OpAtomicLoad64: diff --git a/src/cmd/compile/internal/ssagen/intrinsics.go b/src/cmd/compile/internal/ssagen/intrinsics.go index 4bf4d3006c..20581803d9 100644 --- a/src/cmd/compile/internal/ssagen/intrinsics.go +++ b/src/cmd/compile/internal/ssagen/intrinsics.go @@ -302,7 +302,7 @@ func initIntrinsics(cfg *intrinsicBuildConfig) { s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v) }, - sys.AMD64) + sys.AMD64, sys.PPC64) addF("internal/runtime/atomic", "Xchg", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { v := s.newValue3(ssa.OpAtomicExchange32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem()) diff --git a/src/cmd/compile/internal/ssagen/intrinsics_test.go b/src/cmd/compile/internal/ssagen/intrinsics_test.go index d5c55ae278..579f346f49 100644 --- a/src/cmd/compile/internal/ssagen/intrinsics_test.go +++ b/src/cmd/compile/internal/ssagen/intrinsics_test.go @@ -783,6 +783,7 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{ {"ppc64", "internal/runtime/atomic", "Xaddint32"}: struct{}{}, {"ppc64", "internal/runtime/atomic", "Xaddint64"}: struct{}{}, {"ppc64", "internal/runtime/atomic", "Xadduintptr"}: struct{}{}, + {"ppc64", "internal/runtime/atomic", "Xchg8"}: struct{}{}, {"ppc64", "internal/runtime/atomic", "Xchg"}: struct{}{}, {"ppc64", "internal/runtime/atomic", "Xchg64"}: struct{}{}, {"ppc64", "internal/runtime/atomic", "Xchgint32"}: struct{}{}, @@ -903,6 +904,7 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{ {"ppc64le", "internal/runtime/atomic", "Xaddint32"}: struct{}{}, {"ppc64le", "internal/runtime/atomic", "Xaddint64"}: struct{}{}, {"ppc64le", "internal/runtime/atomic", "Xadduintptr"}: struct{}{}, + {"ppc64le", "internal/runtime/atomic", "Xchg8"}: struct{}{}, {"ppc64le", "internal/runtime/atomic", "Xchg"}: struct{}{}, {"ppc64le", "internal/runtime/atomic", "Xchg64"}: struct{}{}, {"ppc64le", "internal/runtime/atomic", "Xchgint32"}: struct{}{}, diff --git a/src/internal/runtime/atomic/atomic_ppc64x.go b/src/internal/runtime/atomic/atomic_ppc64x.go index 33a92b53f4..590ba03ecf 100644 --- a/src/internal/runtime/atomic/atomic_ppc64x.go +++ b/src/internal/runtime/atomic/atomic_ppc64x.go @@ -17,6 +17,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_ppc64x.s b/src/internal/runtime/atomic/atomic_ppc64x.s index 75635b933d..184a30c970 100644 --- a/src/internal/runtime/atomic/atomic_ppc64x.s +++ b/src/internal/runtime/atomic/atomic_ppc64x.s @@ -236,6 +236,22 @@ TEXT ·Xadd64(SB), NOSPLIT, $0-24 MOVD R3, ret+16(FP) RET +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVD ptr+0(FP), R4 + MOVB new+8(FP), R5 + LWSYNC + LBAR (R4), R3 + STBCCC R5, (R4) + BNE -2(PC) + ISYNC + MOVB R3, ret+16(FP) + RET + // uint32 Xchg(ptr *uint32, new uint32) // Atomically: // old := *ptr; diff --git a/src/internal/runtime/atomic/xchg8_test.go b/src/internal/runtime/atomic/xchg8_test.go index 139062422e..b0b39c2dd7 100644 --- a/src/internal/runtime/atomic/xchg8_test.go +++ b/src/internal/runtime/atomic/xchg8_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 +//go:build amd64 || ppc64 || ppc64le package atomic_test -- 2.48.1