cmd/compile/internal/ssa: intrinsify atomic.Xchg8 on amd64

For #68578

Change-Id: Ia9580579bfc4709945bfcf6ec3803d5d11812187
Reviewed-on: https://go-review.googlesource.com/c/go/+/606901
Reviewed-by: Mauri de Souza Meneguzzo <mauri870@gmail.com>
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Martin Möhrmann <moehrmann@google.com>
Auto-Submit: Rhys Hiltner <rhys.hiltner@gmail.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64.rules b/src/cmd/compile/internal/ssa/_gen/AMD64.rules
index 7ac9d5ca16826357d41aa09c6a0fe90cbaa6fb25..ce9a6e99140a29e4b991653c46cdef21cc4ef78a 100644 (file)
--- a/src/cmd/compile/internal/ssa/_gen/AMD64.rules
+++ b/src/cmd/compile/internal/ssa/_gen/AMD64.rules
@@ -563,6 +563,7 @@
 (AtomicStorePtrNoWB ptr val mem) => (Select1 (XCHGQ <types.NewTuple(typ.BytePtr,types.TypeMem)> val ptr mem))
 
 // Atomic exchanges.
+(AtomicExchange8 ptr val mem) => (XCHGB val ptr mem)
 (AtomicExchange32 ptr val mem) => (XCHGL val ptr mem)
 (AtomicExchange64 ptr val mem) => (XCHGQ val ptr mem)
 

diff --git a/src/cmd/compile/internal/ssa/_gen/genericOps.go b/src/cmd/compile/internal/ssa/_gen/genericOps.go
index 9baceb19203faee3e21439fe29657d8f28b2e244..86bcef9980f979ae64224cadd0be0ffc77261c62 100644 (file)
--- a/src/cmd/compile/internal/ssa/_gen/genericOps.go
+++ b/src/cmd/compile/internal/ssa/_gen/genericOps.go
@@ -604,6 +604,7 @@ var genericOps = []opData{
        {name: "AtomicStorePtrNoWB", argLength: 3, typ: "Mem", hasSideEffects: true},               // Store arg1 to *arg0.  arg2=memory.  Returns memory.
        {name: "AtomicStoreRel32", argLength: 3, typ: "Mem", hasSideEffects: true},                 // Store arg1 to *arg0.  arg2=memory.  Lock release, returns memory.
        {name: "AtomicStoreRel64", argLength: 3, typ: "Mem", hasSideEffects: true},                 // Store arg1 to *arg0.  arg2=memory.  Lock release, returns memory.
+       {name: "AtomicExchange8", argLength: 3, typ: "(UInt8,Mem)", hasSideEffects: true},          // Store arg1 to *arg0.  arg2=memory.  Returns old contents of *arg0 and new memory.
        {name: "AtomicExchange32", argLength: 3, typ: "(UInt32,Mem)", hasSideEffects: true},        // Store arg1 to *arg0.  arg2=memory.  Returns old contents of *arg0 and new memory.
        {name: "AtomicExchange64", argLength: 3, typ: "(UInt64,Mem)", hasSideEffects: true},        // Store arg1 to *arg0.  arg2=memory.  Returns old contents of *arg0 and new memory.
        {name: "AtomicAdd32", argLength: 3, typ: "(UInt32,Mem)", hasSideEffects: true},             // Do *arg0 += arg1.  arg2=memory.  Returns sum and new memory.

diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go
index 574bbbdc613e30a2113eca0504a41e683cb0a205..a96d07a9a3248b5ed9897dc264466c927f38a525 100644 (file)
--- a/src/cmd/compile/internal/ssa/opGen.go
+++ b/src/cmd/compile/internal/ssa/opGen.go
@@ -3249,6 +3249,7 @@ const (
        OpAtomicStorePtrNoWB
        OpAtomicStoreRel32
        OpAtomicStoreRel64
+       OpAtomicExchange8
        OpAtomicExchange32
        OpAtomicExchange64
        OpAtomicAdd32
@@ -41142,6 +41143,12 @@ var opcodeTable = [...]opInfo{
                hasSideEffects: true,
                generic:        true,
        },
+       {
+               name:           "AtomicExchange8",
+               argLen:         3,
+               hasSideEffects: true,
+               generic:        true,
+       },
        {
                name:           "AtomicExchange32",
                argLen:         3,

diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go
index 77d53997f6e6e795bb7d84a0eafc0c5efae46b62..f17c4be516077523df7ab1cda771944574cc2122 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteAMD64.go
+++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go
@@ -587,6 +587,8 @@ func rewriteValueAMD64(v *Value) bool {
                return rewriteValueAMD64_OpAtomicExchange32(v)
        case OpAtomicExchange64:
                return rewriteValueAMD64_OpAtomicExchange64(v)
+       case OpAtomicExchange8:
+               return rewriteValueAMD64_OpAtomicExchange8(v)
        case OpAtomicLoad32:
                return rewriteValueAMD64_OpAtomicLoad32(v)
        case OpAtomicLoad64:
@@ -23990,6 +23992,21 @@ func rewriteValueAMD64_OpAtomicExchange64(v *Value) bool {
                return true
        }
 }
+func rewriteValueAMD64_OpAtomicExchange8(v *Value) bool {
+       v_2 := v.Args[2]
+       v_1 := v.Args[1]
+       v_0 := v.Args[0]
+       // match: (AtomicExchange8 ptr val mem)
+       // result: (XCHGB val ptr mem)
+       for {
+               ptr := v_0
+               val := v_1
+               mem := v_2
+               v.reset(OpAMD64XCHGB)
+               v.AddArg3(val, ptr, mem)
+               return true
+       }
+}
 func rewriteValueAMD64_OpAtomicLoad32(v *Value) bool {
        v_1 := v.Args[1]
        v_0 := v.Args[0]

diff --git a/src/cmd/compile/internal/ssagen/intrinsics.go b/src/cmd/compile/internal/ssagen/intrinsics.go
index 39a72533df8d8438303ee19b9802e08d3da497bb..4bf4d3006cb4a08d089e267be45faef4aa0ade9e 100644 (file)
--- a/src/cmd/compile/internal/ssagen/intrinsics.go
+++ b/src/cmd/compile/internal/ssagen/intrinsics.go
@@ -296,6 +296,13 @@ func initIntrinsics(cfg *intrinsicBuildConfig) {
                },
                sys.PPC64)
 
+       addF("internal/runtime/atomic", "Xchg8",
+               func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
+                       v := s.newValue3(ssa.OpAtomicExchange8, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], args[1], s.mem())
+                       s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
+                       return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v)
+               },
+               sys.AMD64)
        addF("internal/runtime/atomic", "Xchg",
                func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
                        v := s.newValue3(ssa.OpAtomicExchange32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())

diff --git a/src/cmd/compile/internal/ssagen/intrinsics_test.go b/src/cmd/compile/internal/ssagen/intrinsics_test.go
index 888063cbda0984de72a4bdeec7f6153c79f02061..d5c55ae2788a74e6c19853702efe4ac9e877a587 100644 (file)
--- a/src/cmd/compile/internal/ssagen/intrinsics_test.go
+++ b/src/cmd/compile/internal/ssagen/intrinsics_test.go
@@ -84,6 +84,7 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{
        {"amd64", "internal/runtime/atomic", "Xadduintptr"}:        struct{}{},
        {"amd64", "internal/runtime/atomic", "Xchg"}:               struct{}{},
        {"amd64", "internal/runtime/atomic", "Xchg64"}:             struct{}{},
+       {"amd64", "internal/runtime/atomic", "Xchg8"}:              struct{}{},
        {"amd64", "internal/runtime/atomic", "Xchgint32"}:          struct{}{},
        {"amd64", "internal/runtime/atomic", "Xchgint64"}:          struct{}{},
        {"amd64", "internal/runtime/atomic", "Xchguintptr"}:        struct{}{},