cmd/compile, runtime/internal/atomic: add lwsync for false ppc64 cas

This CL changes ppc64 atomic compare-and-swap (cas). Before this CL,
if the cas failed--if the value in memory was not the value expected
by the cas call--the atomic function would not synchronize memory.

In the note code in runtime/lock_sema.go, used on BSD systems,
notesleep and notetsleep first try a cas on the key. If that cas fails,
something has already called notewakeup, and the sleep completes.
However, because the cas did not synchronize memory on failure,
this meant that notesleep/notetsleep could return to a core that was
unable to see the memory changes that the notewakeup was reporting.

Fixes #30189
Fixes #63384

Change-Id: I9b921de5c1c09b10a37df6b3206b9003c3f32986
Reviewed-on: https://go-review.googlesource.com/c/go/+/533118
Run-TryBot: Ian Lance Taylor <iant@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Paul Murphy <murp@ibm.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Lynn Boger <laboger@linux.vnet.ibm.com>
Auto-Submit: Ian Lance Taylor <iant@google.com>
Reviewed-by: Ian Lance Taylor <iant@google.com>

diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go
index 7745352c98006894312930dfac0717476181a6e5..9ba66b35f39167c80cf51a02b5b5e82b3a1858f9 100644 (file)
--- a/src/cmd/compile/internal/ppc64/ssa.go
+++ b/src/cmd/compile/internal/ppc64/ssa.go
@@ -347,9 +347,9 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                // BNE         end
                // STDCCC      Rarg2, (Rarg0)
                // BNE         loop
-               // LWSYNC      // Only for sequential consistency; not required in CasRel.
                // MOVD        $1, Rout
                // end:
+               // LWSYNC      // Only for sequential consistency; not required in CasRel.
                ld := ppc64.ALDAR
                st := ppc64.ASTDCCC
                cmp := ppc64.ACMP
@@ -402,22 +402,24 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p4 := s.Prog(ppc64.ABNE)
                p4.To.Type = obj.TYPE_BRANCH
                p4.To.SetTarget(p0)
+               // return value true
+               p5 := s.Prog(ppc64.AMOVD)
+               p5.From.Type = obj.TYPE_CONST
+               p5.From.Offset = 1
+               p5.To.Type = obj.TYPE_REG
+               p5.To.Reg = out
                // LWSYNC - Assuming shared data not write-through-required nor
                // caching-inhibited. See Appendix B.2.1.1 in the ISA 2.07b.
                // If the operation is a CAS-Release, then synchronization is not necessary.
                if v.AuxInt != 0 {
                        plwsync2 := s.Prog(ppc64.ALWSYNC)
                        plwsync2.To.Type = obj.TYPE_NONE
+                       p2.To.SetTarget(plwsync2)
+               } else {
+                       // done (label)
+                       p6 := s.Prog(obj.ANOP)
+                       p2.To.SetTarget(p6)
                }
-               // return value true
-               p5 := s.Prog(ppc64.AMOVD)
-               p5.From.Type = obj.TYPE_CONST
-               p5.From.Offset = 1
-               p5.To.Type = obj.TYPE_REG
-               p5.To.Reg = out
-               // done (label)
-               p6 := s.Prog(obj.ANOP)
-               p2.To.SetTarget(p6)
 
        case ssa.OpPPC64LoweredPubBarrier:
                // LWSYNC

diff --git a/src/runtime/internal/atomic/atomic_ppc64x.s b/src/runtime/internal/atomic/atomic_ppc64x.s
index 144439ec2353ebda02376f94835baf280c3b3d20..75635b933d7423a9972da23ac54e40639c399643 100644 (file)
--- a/src/runtime/internal/atomic/atomic_ppc64x.s
+++ b/src/runtime/internal/atomic/atomic_ppc64x.s
@@ -101,6 +101,7 @@ cas_again:
        MOVB    R3, ret+16(FP)
        RET
 cas_fail:
+       LWSYNC
        MOVB    R0, ret+16(FP)
        RET
 
@@ -128,6 +129,7 @@ cas64_again:
        MOVB    R3, ret+24(FP)
        RET
 cas64_fail:
+       LWSYNC
        MOVB    R0, ret+24(FP)
        RET