cmd/compile, internal/runtime/atomic: add Xchg8 for arm64

For #69735

Change-Id: I61a2e561684c538eea705e60c8ebda6be3ef31a7
GitHub-Last-Rev: 3c7f4ec845182d3ef1a007319d91027433163db3
GitHub-Pull-Request: golang/go#69751
Reviewed-on: https://go-review.googlesource.com/c/go/+/617595
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Michael Knyszek <mknyszek@google.com>

diff --git a/src/cmd/compile/internal/arm64/ssa.go b/src/cmd/compile/internal/arm64/ssa.go
index 900e7016a3e091504c11d548644d9a28e2b24726..e2c487319231ac8557c37a8a395ed82fba89ba5c 100644 (file)
--- a/src/cmd/compile/internal/arm64/ssa.go
+++ b/src/cmd/compile/internal/arm64/ssa.go
@@ -578,15 +578,22 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p.To.Type = obj.TYPE_REG
                p.To.Reg = v.Reg()
        case ssa.OpARM64LoweredAtomicExchange64,
-               ssa.OpARM64LoweredAtomicExchange32:
+               ssa.OpARM64LoweredAtomicExchange32,
+               ssa.OpARM64LoweredAtomicExchange8:
                // LDAXR        (Rarg0), Rout
                // STLXR        Rarg1, (Rarg0), Rtmp
                // CBNZ         Rtmp, -2(PC)
-               ld := arm64.ALDAXR
-               st := arm64.ASTLXR
-               if v.Op == ssa.OpARM64LoweredAtomicExchange32 {
+               var ld, st obj.As
+               switch v.Op {
+               case ssa.OpARM64LoweredAtomicExchange8:
+                       ld = arm64.ALDAXRB
+                       st = arm64.ASTLXRB
+               case ssa.OpARM64LoweredAtomicExchange32:
                        ld = arm64.ALDAXRW
                        st = arm64.ASTLXRW
+               case ssa.OpARM64LoweredAtomicExchange64:
+                       ld = arm64.ALDAXR
+                       st = arm64.ASTLXR
                }
                r0 := v.Args[0].Reg()
                r1 := v.Args[1].Reg()
@@ -608,10 +615,16 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p2.To.Type = obj.TYPE_BRANCH
                p2.To.SetTarget(p)
        case ssa.OpARM64LoweredAtomicExchange64Variant,
-               ssa.OpARM64LoweredAtomicExchange32Variant:
-               swap := arm64.ASWPALD
-               if v.Op == ssa.OpARM64LoweredAtomicExchange32Variant {
+               ssa.OpARM64LoweredAtomicExchange32Variant,
+               ssa.OpARM64LoweredAtomicExchange8Variant:
+               var swap obj.As
+               switch v.Op {
+               case ssa.OpARM64LoweredAtomicExchange8Variant:
+                       swap = arm64.ASWPALB
+               case ssa.OpARM64LoweredAtomicExchange32Variant:
                        swap = arm64.ASWPALW
+               case ssa.OpARM64LoweredAtomicExchange64Variant:
+                       swap = arm64.ASWPALD
                }
                r0 := v.Args[0].Reg()
                r1 := v.Args[1].Reg()

diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64.rules b/src/cmd/compile/internal/ssa/_gen/ARM64.rules
index 9064b515f7422d1cfb63ae520f2edfc9179a2d3d..29dc258d9e37674a7bc13701601c66715dad2544 100644 (file)
--- a/src/cmd/compile/internal/ssa/_gen/ARM64.rules
+++ b/src/cmd/compile/internal/ssa/_gen/ARM64.rules
@@ -571,12 +571,12 @@
 (AtomicStore64      ...) => (STLR  ...)
 (AtomicStorePtrNoWB ...) => (STLR  ...)
 
-(AtomicExchange(32|64)       ...) => (LoweredAtomicExchange(32|64) ...)
+(AtomicExchange(8|32|64)       ...) => (LoweredAtomicExchange(8|32|64) ...)
 (AtomicAdd(32|64)            ...) => (LoweredAtomicAdd(32|64)      ...)
 (AtomicCompareAndSwap(32|64) ...) => (LoweredAtomicCas(32|64)      ...)
 
 (AtomicAdd(32|64)Variant            ...) => (LoweredAtomicAdd(32|64)Variant      ...)
-(AtomicExchange(32|64)Variant       ...) => (LoweredAtomicExchange(32|64)Variant ...)
+(AtomicExchange(8|32|64)Variant       ...) => (LoweredAtomicExchange(8|32|64)Variant ...)
 (AtomicCompareAndSwap(32|64)Variant ...) => (LoweredAtomicCas(32|64)Variant      ...)
 
 // Return old contents.

diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go
index d7fecf502d11da890968728efeb2ee9beb233b50..c9cb62cd17cee2d42ee149c96fa221f03d73af2b 100644 (file)
--- a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go
+++ b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go
@@ -651,12 +651,14 @@ func init() {
                // CBNZ         Rtmp, -2(PC)
                {name: "LoweredAtomicExchange64", argLength: 3, reg: gpxchg, resultNotInArgs: true, faultOnNilArg0: true, hasSideEffects: true, unsafePoint: true},
                {name: "LoweredAtomicExchange32", argLength: 3, reg: gpxchg, resultNotInArgs: true, faultOnNilArg0: true, hasSideEffects: true, unsafePoint: true},
+               {name: "LoweredAtomicExchange8", argLength: 3, reg: gpxchg, resultNotInArgs: true, faultOnNilArg0: true, hasSideEffects: true, unsafePoint: true},
 
                // atomic exchange variant.
                // store arg1 to arg0. arg2=mem. returns <old content of *arg0, memory>. auxint must be zero.
                // SWPALD       Rarg1, (Rarg0), Rout
                {name: "LoweredAtomicExchange64Variant", argLength: 3, reg: gpxchg, resultNotInArgs: true, faultOnNilArg0: true, hasSideEffects: true},
                {name: "LoweredAtomicExchange32Variant", argLength: 3, reg: gpxchg, resultNotInArgs: true, faultOnNilArg0: true, hasSideEffects: true},
+               {name: "LoweredAtomicExchange8Variant", argLength: 3, reg: gpxchg, resultNotInArgs: true, faultOnNilArg0: true, hasSideEffects: true, unsafePoint: true},
 
                // atomic add.
                // *arg0 += arg1. arg2=mem. returns <new content of *arg0, memory>. auxint must be zero.

diff --git a/src/cmd/compile/internal/ssa/_gen/genericOps.go b/src/cmd/compile/internal/ssa/_gen/genericOps.go
index 86bcef9980f979ae64224cadd0be0ffc77261c62..ceaff221d7bfe11efc96e3062828b244149d62aa 100644 (file)
--- a/src/cmd/compile/internal/ssa/_gen/genericOps.go
+++ b/src/cmd/compile/internal/ssa/_gen/genericOps.go
@@ -635,6 +635,7 @@ var genericOps = []opData{
        // These are not currently used on any other platform.
        {name: "AtomicAdd32Variant", argLength: 3, typ: "(UInt32,Mem)", hasSideEffects: true},          // Do *arg0 += arg1.  arg2=memory.  Returns sum and new memory.
        {name: "AtomicAdd64Variant", argLength: 3, typ: "(UInt64,Mem)", hasSideEffects: true},          // Do *arg0 += arg1.  arg2=memory.  Returns sum and new memory.
+       {name: "AtomicExchange8Variant", argLength: 3, typ: "(UInt8,Mem)", hasSideEffects: true},       // Store arg1 to *arg0.  arg2=memory.  Returns old contents of *arg0 and new memory.
        {name: "AtomicExchange32Variant", argLength: 3, typ: "(UInt32,Mem)", hasSideEffects: true},     // Store arg1 to *arg0.  arg2=memory.  Returns old contents of *arg0 and new memory.
        {name: "AtomicExchange64Variant", argLength: 3, typ: "(UInt64,Mem)", hasSideEffects: true},     // Store arg1 to *arg0.  arg2=memory.  Returns old contents of *arg0 and new memory.
        {name: "AtomicCompareAndSwap32Variant", argLength: 4, typ: "(Bool,Mem)", hasSideEffects: true}, // if *arg0==arg1, then set *arg0=arg2.  Returns true if store happens and new memory.

diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go
index fbc9ae7ea55810a6ebbb6dd88d3c4d4f94dc6263..8642d39e8b272902207ca496f98f4da4e821f7a1 100644 (file)
--- a/src/cmd/compile/internal/ssa/opGen.go
+++ b/src/cmd/compile/internal/ssa/opGen.go
@@ -1712,8 +1712,10 @@ const (
        OpARM64STLRW
        OpARM64LoweredAtomicExchange64
        OpARM64LoweredAtomicExchange32
+       OpARM64LoweredAtomicExchange8
        OpARM64LoweredAtomicExchange64Variant
        OpARM64LoweredAtomicExchange32Variant
+       OpARM64LoweredAtomicExchange8Variant
        OpARM64LoweredAtomicAdd64
        OpARM64LoweredAtomicAdd32
        OpARM64LoweredAtomicAdd64Variant
@@ -3270,6 +3272,7 @@ const (
        OpAtomicOr8value
        OpAtomicAdd32Variant
        OpAtomicAdd64Variant
+       OpAtomicExchange8Variant
        OpAtomicExchange32Variant
        OpAtomicExchange64Variant
        OpAtomicCompareAndSwap32Variant
@@ -22987,6 +22990,23 @@ var opcodeTable = [...]opInfo{
                        },
                },
        },
+       {
+               name:            "LoweredAtomicExchange8",
+               argLen:          3,
+               resultNotInArgs: true,
+               faultOnNilArg0:  true,
+               hasSideEffects:  true,
+               unsafePoint:     true,
+               reg: regInfo{
+                       inputs: []inputInfo{
+                               {1, 805044223},           // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30
+                               {0, 9223372038733561855}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 SP SB
+                       },
+                       outputs: []outputInfo{
+                               {0, 670826495}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 R30
+                       },
+               },
+       },
        {
                name:            "LoweredAtomicExchange64Variant",
                argLen:          3,
@@ -23019,6 +23039,23 @@ var opcodeTable = [...]opInfo{
                        },
                },
        },
+       {
+               name:            "LoweredAtomicExchange8Variant",
+               argLen:          3,
+               resultNotInArgs: true,
+               faultOnNilArg0:  true,
+               hasSideEffects:  true,
+               unsafePoint:     true,
+               reg: regInfo{
+                       inputs: []inputInfo{
+                               {1, 805044223},           // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30
+                               {0, 9223372038733561855}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 SP SB
+                       },
+                       outputs: []outputInfo{
+                               {0, 670826495}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 R30
+                       },
+               },
+       },
        {
                name:            "LoweredAtomicAdd64",
                argLen:          3,
@@ -41281,6 +41318,12 @@ var opcodeTable = [...]opInfo{
                hasSideEffects: true,
                generic:        true,
        },
+       {
+               name:           "AtomicExchange8Variant",
+               argLen:         3,
+               hasSideEffects: true,
+               generic:        true,
+       },
        {
                name:           "AtomicExchange32Variant",
                argLen:         3,

diff --git a/src/cmd/compile/internal/ssa/rewriteARM64.go b/src/cmd/compile/internal/ssa/rewriteARM64.go
index 571046b5b8331ce7d158831911b1fbfeac42949f..edb17cedf3875fd46cfe538dadd473f50e386d30 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteARM64.go
+++ b/src/cmd/compile/internal/ssa/rewriteARM64.go
@@ -513,6 +513,12 @@ func rewriteValueARM64(v *Value) bool {
        case OpAtomicExchange64Variant:
                v.Op = OpARM64LoweredAtomicExchange64Variant
                return true
+       case OpAtomicExchange8:
+               v.Op = OpARM64LoweredAtomicExchange8
+               return true
+       case OpAtomicExchange8Variant:
+               v.Op = OpARM64LoweredAtomicExchange8Variant
+               return true
        case OpAtomicLoad32:
                v.Op = OpARM64LDARW
                return true

diff --git a/src/cmd/compile/internal/ssagen/intrinsics.go b/src/cmd/compile/internal/ssagen/intrinsics.go
index 20581803d9121928fce5696f0597dadf987c854c..df5862f718eda40743c492cc807de449a8903d97 100644 (file)
--- a/src/cmd/compile/internal/ssagen/intrinsics.go
+++ b/src/cmd/compile/internal/ssagen/intrinsics.go
@@ -373,6 +373,9 @@ func initIntrinsics(cfg *intrinsicBuildConfig) {
                        s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
                }
        }
+       addF("internal/runtime/atomic", "Xchg8",
+               makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange8, ssa.OpAtomicExchange8Variant, types.TUINT8, atomicEmitterARM64),
+               sys.ARM64)
        addF("internal/runtime/atomic", "Xchg",
                makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange32, ssa.OpAtomicExchange32Variant, types.TUINT32, atomicEmitterARM64),
                sys.ARM64)

diff --git a/src/cmd/compile/internal/ssagen/intrinsics_test.go b/src/cmd/compile/internal/ssagen/intrinsics_test.go
index 579f346f4947b73fcf3374eda1182d7e700e5a91..60f11c980f30887ae87b80f75ee2864eed9f7890 100644 (file)
--- a/src/cmd/compile/internal/ssagen/intrinsics_test.go
+++ b/src/cmd/compile/internal/ssagen/intrinsics_test.go
@@ -249,6 +249,7 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{
        {"arm64", "internal/runtime/atomic", "Xaddint32"}:          struct{}{},
        {"arm64", "internal/runtime/atomic", "Xaddint64"}:          struct{}{},
        {"arm64", "internal/runtime/atomic", "Xadduintptr"}:        struct{}{},
+       {"arm64", "internal/runtime/atomic", "Xchg8"}:              struct{}{},
        {"arm64", "internal/runtime/atomic", "Xchg"}:               struct{}{},
        {"arm64", "internal/runtime/atomic", "Xchg64"}:             struct{}{},
        {"arm64", "internal/runtime/atomic", "Xchgint32"}:          struct{}{},

diff --git a/src/internal/runtime/atomic/atomic_arm64.go b/src/internal/runtime/atomic/atomic_arm64.go
index c4c56ae8951c6b548f1bb0eee1e58692d4d1aa19..f4aef19388d562cdbb77b8fc323bb4301dff6499 100644 (file)
--- a/src/internal/runtime/atomic/atomic_arm64.go
+++ b/src/internal/runtime/atomic/atomic_arm64.go
@@ -24,6 +24,9 @@ func Xadd64(ptr *uint64, delta int64) uint64
 //go:noescape
 func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
 
+//go:noescape
+func Xchg8(ptr *uint8, new uint8) uint8
+
 //go:noescape
 func Xchg(ptr *uint32, new uint32) uint32
 

diff --git a/src/internal/runtime/atomic/atomic_arm64.s b/src/internal/runtime/atomic/atomic_arm64.s
index ede56538b880cb748ea964f7d4db41b34018ea82..09f3b53c5cdd1860776fec603027da1003f34de6 100644 (file)
--- a/src/internal/runtime/atomic/atomic_arm64.s
+++ b/src/internal/runtime/atomic/atomic_arm64.s
@@ -120,6 +120,30 @@ TEXT ·Store64(SB), NOSPLIT, $0-16
        STLR    R1, (R0)
        RET
 
+// uint8 Xchg(ptr *uint8, new uint8)
+// Atomically:
+//     old := *ptr;
+//     *ptr = new;
+//     return old;
+TEXT ·Xchg8(SB), NOSPLIT, $0-17
+       MOVD    ptr+0(FP), R0
+       MOVB    new+8(FP), R1
+#ifndef GOARM64_LSE
+       MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
+       CBZ     R4, load_store_loop
+#endif
+       SWPALB  R1, (R0), R2
+       MOVB    R2, ret+16(FP)
+       RET
+#ifndef GOARM64_LSE
+load_store_loop:
+       LDAXRB  (R0), R2
+       STLXRB  R1, (R0), R3
+       CBNZ    R3, load_store_loop
+       MOVB    R2, ret+16(FP)
+       RET
+#endif
+
 // uint32 Xchg(ptr *uint32, new uint32)
 // Atomically:
 //     old := *ptr;

diff --git a/src/internal/runtime/atomic/xchg8_test.go b/src/internal/runtime/atomic/xchg8_test.go
index b0b39c2dd7f6757a015320f24efd3552ca30fd8d..a04fcfc4bdbdc693f6b6df18473a83c583b41ada 100644 (file)
--- a/src/internal/runtime/atomic/xchg8_test.go
+++ b/src/internal/runtime/atomic/xchg8_test.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build amd64 || ppc64 || ppc64le
+//go:build amd64 || arm64 || ppc64 || ppc64le
 
 package atomic_test