runtime/internal/atomic: improve ARM atomics

This is a follow-up of CL 93637. There, when we redirect sync/atomic
to runtime/internal/atomic, a few good implementations of ARM atomics
were lost. This CL brings most of them back, with some improvements.

- Change atomic Store to a plain store with memory barrier, as we
  already changed atomic Load to plain load with memory barrier.

- Use native 64-bit atomics on ARMv7, jump to Go implementations
  on older machines. But drop the kernel helper. In particular,
  for Load64, just do loads, not using Cas on the address being
  load from, so it works also for read-only memory (since we have
  already fixed 32-bit Load).

Change-Id: I725cd65cf945ae5200db81a35be3f251c9f7af14
Reviewed-on: https://go-review.googlesource.com/111315
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/internal/atomic/asm_arm.s b/src/runtime/internal/atomic/asm_arm.s
index 95373faab104ed8b3d8667cb5c2e76235014eb2f..09724c1c34a62c70402c103cafac6551c7b5ca85 100644 (file)
--- a/src/runtime/internal/atomic/asm_arm.s
+++ b/src/runtime/internal/atomic/asm_arm.s
@@ -27,8 +27,8 @@ casl:
        CMP     R0, R2
        BNE     casfail
 
-       MOVB    runtime·goarm(SB), R11
-       CMP     $7, R11
+       MOVB    runtime·goarm(SB), R8
+       CMP     $7, R8
        BLT     2(PC)
        DMB     MB_ISHST
 
@@ -37,8 +37,7 @@ casl:
        BNE     casl
        MOVW    $1, R0
 
-       MOVB    runtime·goarm(SB), R11
-       CMP     $7, R11
+       CMP     $7, R8
        BLT     2(PC)
        DMB     MB_ISH
 
@@ -49,12 +48,17 @@ casfail:
        MOVB    R0, ret+12(FP)
        RET
 
+// stubs
+
 TEXT runtime∕internal∕atomic·Loadp(SB),NOSPLIT|NOFRAME,$0-8
        B runtime∕internal∕atomic·Load(SB)
 
 TEXT runtime∕internal∕atomic·Casuintptr(SB),NOSPLIT,$0-13
        B       runtime∕internal∕atomic·Cas(SB)
 
+TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0-13
+       B       runtime∕internal∕atomic·Cas(SB)
+
 TEXT runtime∕internal∕atomic·Loaduintptr(SB),NOSPLIT,$0-8
        B       runtime∕internal∕atomic·Load(SB)
 
@@ -64,6 +68,9 @@ TEXT runtime∕internal∕atomic·Loaduint(SB),NOSPLIT,$0-8
 TEXT runtime∕internal∕atomic·Storeuintptr(SB),NOSPLIT,$0-8
        B       runtime∕internal∕atomic·Store(SB)
 
+TEXT runtime∕internal∕atomic·StorepNoWB(SB),NOSPLIT,$0-8
+       B       runtime∕internal∕atomic·Store(SB)
+
 TEXT runtime∕internal∕atomic·Xadduintptr(SB),NOSPLIT,$0-12
        B       runtime∕internal∕atomic·Xadd(SB)
 
@@ -72,3 +79,162 @@ TEXT runtime∕internal∕atomic·Loadint64(SB),NOSPLIT,$0-12
 
 TEXT runtime∕internal∕atomic·Xaddint64(SB),NOSPLIT,$0-20
        B       runtime∕internal∕atomic·Xadd64(SB)
+
+// 64-bit atomics
+// The native ARM implementations use LDREXD/STREXD, which are
+// available on ARMv6k or later. We use them only on ARMv7.
+// On older ARM, we use Go implementations which simulate 64-bit
+// atomics with locks.
+
+TEXT   armCas64<>(SB),NOSPLIT,$0-21
+       MOVW    addr+0(FP), R1
+       // make unaligned atomic access panic
+       AND.S   $7, R1, R2
+       BEQ     2(PC)
+       MOVW    R2, (R2)        // crash. AND.S above left only low 3 bits in R2.
+       MOVW    old_lo+4(FP), R2
+       MOVW    old_hi+8(FP), R3
+       MOVW    new_lo+12(FP), R4
+       MOVW    new_hi+16(FP), R5
+cas64loop:
+       LDREXD  (R1), R6        // loads R6 and R7
+       CMP     R2, R6
+       BNE     cas64fail
+       CMP     R3, R7
+       BNE     cas64fail
+
+       DMB     MB_ISHST
+
+       STREXD  R4, (R1), R0    // stores R4 and R5
+       CMP     $0, R0
+       BNE     cas64loop
+       MOVW    $1, R0
+
+       DMB     MB_ISH
+
+       MOVBU   R0, swapped+20(FP)
+       RET
+cas64fail:
+       MOVW    $0, R0
+       MOVBU   R0, swapped+20(FP)
+       RET
+
+TEXT   armXadd64<>(SB),NOSPLIT,$0-20
+       MOVW    addr+0(FP), R1
+       // make unaligned atomic access panic
+       AND.S   $7, R1, R2
+       BEQ     2(PC)
+       MOVW    R2, (R2)        // crash. AND.S above left only low 3 bits in R2.
+       MOVW    delta_lo+4(FP), R2
+       MOVW    delta_hi+8(FP), R3
+
+add64loop:
+       LDREXD  (R1), R4        // loads R4 and R5
+       ADD.S   R2, R4
+       ADC     R3, R5
+
+       DMB     MB_ISHST
+
+       STREXD  R4, (R1), R0    // stores R4 and R5
+       CMP     $0, R0
+       BNE     add64loop
+
+       DMB     MB_ISH
+
+       MOVW    R4, new_lo+12(FP)
+       MOVW    R5, new_hi+16(FP)
+       RET
+
+TEXT   armXchg64<>(SB),NOSPLIT,$0-20
+       MOVW    addr+0(FP), R1
+       // make unaligned atomic access panic
+       AND.S   $7, R1, R2
+       BEQ     2(PC)
+       MOVW    R2, (R2)        // crash. AND.S above left only low 3 bits in R2.
+       MOVW    new_lo+4(FP), R2
+       MOVW    new_hi+8(FP), R3
+
+swap64loop:
+       LDREXD  (R1), R4        // loads R4 and R5
+
+       DMB     MB_ISHST
+
+       STREXD  R2, (R1), R0    // stores R2 and R3
+       CMP     $0, R0
+       BNE     swap64loop
+
+       DMB     MB_ISH
+
+       MOVW    R4, old_lo+12(FP)
+       MOVW    R5, old_hi+16(FP)
+       RET
+
+TEXT   armLoad64<>(SB),NOSPLIT,$0-12
+       MOVW    addr+0(FP), R1
+       // make unaligned atomic access panic
+       AND.S   $7, R1, R2
+       BEQ     2(PC)
+       MOVW    R2, (R2)        // crash. AND.S above left only low 3 bits in R2.
+
+       LDREXD  (R1), R2        // loads R2 and R3
+       DMB     MB_ISH
+
+       MOVW    R2, val_lo+4(FP)
+       MOVW    R3, val_hi+8(FP)
+       RET
+
+TEXT   armStore64<>(SB),NOSPLIT,$0-12
+       MOVW    addr+0(FP), R1
+       // make unaligned atomic access panic
+       AND.S   $7, R1, R2
+       BEQ     2(PC)
+       MOVW    R2, (R2)        // crash. AND.S above left only low 3 bits in R2.
+       MOVW    val_lo+4(FP), R2
+       MOVW    val_hi+8(FP), R3
+
+store64loop:
+       LDREXD  (R1), R4        // loads R4 and R5
+
+       DMB     MB_ISHST
+
+       STREXD  R2, (R1), R0    // stores R2 and R3
+       CMP     $0, R0
+       BNE     store64loop
+
+       DMB     MB_ISH
+       RET
+
+TEXT   ·Cas64(SB),NOSPLIT,$0-21
+       MOVB    runtime·goarm(SB), R11
+       CMP     $7, R11
+       BLT     2(PC)
+       JMP     armCas64<>(SB)
+       JMP     ·goCas64(SB)
+
+TEXT   ·Xadd64(SB),NOSPLIT,$0-20
+       MOVB    runtime·goarm(SB), R11
+       CMP     $7, R11
+       BLT     2(PC)
+       JMP     armXadd64<>(SB)
+       JMP     ·goXadd64(SB)
+
+TEXT   ·Xchg64(SB),NOSPLIT,$0-20
+       MOVB    runtime·goarm(SB), R11
+       CMP     $7, R11
+       BLT     2(PC)
+       JMP     armXchg64<>(SB)
+       JMP     ·goXchg64(SB)
+
+TEXT   ·Load64(SB),NOSPLIT,$0-12
+       MOVB    runtime·goarm(SB), R11
+       CMP     $7, R11
+       BLT     2(PC)
+       JMP     armLoad64<>(SB)
+       JMP     ·goLoad64(SB)
+
+TEXT   ·Store64(SB),NOSPLIT,$0-12
+       MOVB    runtime·goarm(SB), R11
+       CMP     $7, R11
+       BLT     2(PC)
+       JMP     armStore64<>(SB)
+       JMP     ·goStore64(SB)

diff --git a/src/runtime/internal/atomic/atomic_arm.go b/src/runtime/internal/atomic/atomic_arm.go
index bbf19feea925d7873012d8dffaf679caf03129b6..4ed7e991fe294ab840b08dbe6136366d4d411bc0 100644 (file)
--- a/src/runtime/internal/atomic/atomic_arm.go
+++ b/src/runtime/internal/atomic/atomic_arm.go
@@ -68,28 +68,14 @@ func Xchguintptr(addr *uintptr, v uintptr) uintptr {
        return uintptr(Xchg((*uint32)(unsafe.Pointer(addr)), uint32(v)))
 }
 
-//go:nosplit
-func StorepNoWB(addr unsafe.Pointer, v unsafe.Pointer) {
-       for {
-               old := *(*unsafe.Pointer)(addr)
-               if Casp1((*unsafe.Pointer)(addr), old, v) {
-                       return
-               }
-       }
-}
+// Not noescape -- it installs a pointer to addr.
+func StorepNoWB(addr unsafe.Pointer, v unsafe.Pointer)
 
-//go:nosplit
-func Store(addr *uint32, v uint32) {
-       for {
-               old := *addr
-               if Cas(addr, old, v) {
-                       return
-               }
-       }
-}
+//go:noescape
+func Store(addr *uint32, v uint32)
 
 //go:nosplit
-func Cas64(addr *uint64, old, new uint64) bool {
+func goCas64(addr *uint64, old, new uint64) bool {
        if uintptr(unsafe.Pointer(addr))&7 != 0 {
                *(*int)(nil) = 0 // crash on unaligned uint64
        }
@@ -105,7 +91,7 @@ func Cas64(addr *uint64, old, new uint64) bool {
 }
 
 //go:nosplit
-func Xadd64(addr *uint64, delta int64) uint64 {
+func goXadd64(addr *uint64, delta int64) uint64 {
        if uintptr(unsafe.Pointer(addr))&7 != 0 {
                *(*int)(nil) = 0 // crash on unaligned uint64
        }
@@ -119,7 +105,7 @@ func Xadd64(addr *uint64, delta int64) uint64 {
 }
 
 //go:nosplit
-func Xchg64(addr *uint64, v uint64) uint64 {
+func goXchg64(addr *uint64, v uint64) uint64 {
        if uintptr(unsafe.Pointer(addr))&7 != 0 {
                *(*int)(nil) = 0 // crash on unaligned uint64
        }
@@ -133,7 +119,7 @@ func Xchg64(addr *uint64, v uint64) uint64 {
 }
 
 //go:nosplit
-func Load64(addr *uint64) uint64 {
+func goLoad64(addr *uint64) uint64 {
        if uintptr(unsafe.Pointer(addr))&7 != 0 {
                *(*int)(nil) = 0 // crash on unaligned uint64
        }
@@ -146,7 +132,7 @@ func Load64(addr *uint64) uint64 {
 }
 
 //go:nosplit
-func Store64(addr *uint64, v uint64) {
+func goStore64(addr *uint64, v uint64) {
        if uintptr(unsafe.Pointer(addr))&7 != 0 {
                *(*int)(nil) = 0 // crash on unaligned uint64
        }
@@ -194,3 +180,18 @@ func Load(addr *uint32) uint32
 
 //go:noescape
 func Loadp(addr unsafe.Pointer) unsafe.Pointer
+
+//go:noescape
+func Cas64(addr *uint64, old, new uint64) bool
+
+//go:noescape
+func Xadd64(addr *uint64, delta int64) uint64
+
+//go:noescape
+func Xchg64(addr *uint64, v uint64) uint64
+
+//go:noescape
+func Load64(addr *uint64) uint64
+
+//go:noescape
+func Store64(addr *uint64, v uint64)

diff --git a/src/runtime/internal/atomic/sys_linux_arm.s b/src/runtime/internal/atomic/sys_linux_arm.s
index 7e234d8f2678fabd7213f628b8834c1604eed8be..715231fbaa6ad4d7047064083b335ce050e31d5a 100644 (file)
--- a/src/runtime/internal/atomic/sys_linux_arm.s
+++ b/src/runtime/internal/atomic/sys_linux_arm.s
@@ -55,9 +55,6 @@ check:
        MOVB    R0, ret+12(FP)
        RET
 
-TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0
-       B       runtime∕internal∕atomic·Cas(SB)
-
 // As for cas, memory barriers are complicated on ARM, but the kernel
 // provides a user helper. ARMv5 does not support SMP and has no
 // memory barrier instruction at all. ARMv6 added SMP support and has
@@ -70,7 +67,7 @@ TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0
 TEXT memory_barrier<>(SB),NOSPLIT|NOFRAME,$0
        MOVW    $0xffff0fa0, R15 // R15 is hardware PC.
 
-TEXT runtime∕internal∕atomic·Load(SB),NOSPLIT,$0-8
+TEXT   ·Load(SB),NOSPLIT,$0-8
        MOVW    addr+0(FP), R0
        MOVW    (R0), R1
 
@@ -78,10 +75,32 @@ TEXT runtime∕internal∕atomic·Load(SB),NOSPLIT,$0-8
        CMP     $7, R11
        BGE     native_barrier
        BL      memory_barrier<>(SB)
-       B       prolog
+       B       end
 native_barrier:
        DMB     MB_ISH
-
-prolog:
+end:
        MOVW    R1, ret+4(FP)
        RET
+
+TEXT   ·Store(SB),NOSPLIT,$0-8
+       MOVW    addr+0(FP), R1
+       MOVW    v+4(FP), R2
+
+       MOVB    runtime·goarm(SB), R8
+       CMP     $7, R8
+       BGE     native_barrier
+       BL      memory_barrier<>(SB)
+       B       store
+native_barrier:
+       DMB     MB_ISH
+
+store:
+       MOVW    R2, (R1)
+
+       CMP     $7, R8
+       BGE     native_barrier2
+       BL      memory_barrier<>(SB)
+       RET
+native_barrier2:
+       DMB     MB_ISH
+       RET

diff --git a/src/runtime/internal/atomic/sys_nonlinux_arm.s b/src/runtime/internal/atomic/sys_nonlinux_arm.s
index 9026b66e0981ae6dbddde376849ade7fbb88350a..e593b3c92b2239a653941459d69c0c9d220a55e0 100644 (file)
--- a/src/runtime/internal/atomic/sys_nonlinux_arm.s
+++ b/src/runtime/internal/atomic/sys_nonlinux_arm.s
@@ -17,10 +17,11 @@
 TEXT   ·Cas(SB),NOSPLIT,$0
        JMP     ·armcas(SB)
 
-TEXT   ·Casp1(SB),NOSPLIT,$0
-       JMP     ·Cas(SB)
+// Non-linux OSes support only single processor machines before ARMv7.
+// So we don't need memory barriers if goarm < 7. And we fail loud at
+// startup (runtime.checkgoarm) if it is a multi-processor but goarm < 7.
 
-TEXT runtime∕internal∕atomic·Load(SB),NOSPLIT|NOFRAME,$0-8
+TEXT   ·Load(SB),NOSPLIT|NOFRAME,$0-8
        MOVW    addr+0(FP), R0
        MOVW    (R0), R1
 
@@ -31,3 +32,19 @@ TEXT runtime∕internal∕atomic·Load(SB),NOSPLIT|NOFRAME,$0-8
 
        MOVW    R1, ret+4(FP)
        RET
+
+TEXT   ·Store(SB),NOSPLIT,$0-8
+       MOVW    addr+0(FP), R1
+       MOVW    v+4(FP), R2
+
+       MOVB    runtime·goarm(SB), R8
+       CMP     $7, R8
+       BLT     2(PC)
+       DMB     MB_ISH
+
+       MOVW    R2, (R1)
+
+       CMP     $7, R8
+       BLT     2(PC)
+       DMB     MB_ISH
+       RET