From d2101e54908dc6899863be0772658dbd7e0bbc71 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 23 Oct 2019 09:47:53 -0400 Subject: [PATCH] runtime/internal/atomic: add Store8 We already have Load8, And8, and Or8. For #10958, #24543, but makes sense on its own. Change-Id: I478529fc643edc57efdeccaae413c99edd19b2eb Reviewed-on: https://go-review.googlesource.com/c/go/+/203283 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/runtime/internal/atomic/asm_386.s | 6 +++++ src/runtime/internal/atomic/asm_amd64.s | 6 +++++ src/runtime/internal/atomic/asm_mips64x.s | 8 +++++++ src/runtime/internal/atomic/asm_mipsx.s | 8 +++++++ src/runtime/internal/atomic/asm_ppc64x.s | 7 ++++++ src/runtime/internal/atomic/asm_s390x.s | 8 +++++++ src/runtime/internal/atomic/atomic_386.go | 3 +++ src/runtime/internal/atomic/atomic_amd64.go | 3 +++ src/runtime/internal/atomic/atomic_arm.go | 3 +++ src/runtime/internal/atomic/atomic_arm64.go | 3 +++ src/runtime/internal/atomic/atomic_arm64.s | 6 +++++ src/runtime/internal/atomic/atomic_mips64x.go | 3 +++ src/runtime/internal/atomic/atomic_mipsx.go | 3 +++ src/runtime/internal/atomic/atomic_ppc64x.go | 3 +++ src/runtime/internal/atomic/atomic_s390x.go | 3 +++ src/runtime/internal/atomic/atomic_wasm.go | 6 +++++ src/runtime/internal/atomic/sys_linux_arm.s | 22 +++++++++++++++++++ .../internal/atomic/sys_nonlinux_arm.s | 17 ++++++++++++++ 18 files changed, 118 insertions(+) diff --git a/src/runtime/internal/atomic/asm_386.s b/src/runtime/internal/atomic/asm_386.s index 13289a88d0..9b9dc14a60 100644 --- a/src/runtime/internal/atomic/asm_386.s +++ b/src/runtime/internal/atomic/asm_386.s @@ -229,3 +229,9 @@ TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-5 LOCK ANDB BX, (AX) RET + +TEXT runtime∕internal∕atomic·Store8(SB), NOSPLIT, $0-5 + MOVL ptr+0(FP), BX + MOVB val+4(FP), AX + XCHGB AX, 0(BX) + RET diff --git a/src/runtime/internal/atomic/asm_amd64.s b/src/runtime/internal/atomic/asm_amd64.s index e18aee7d59..90c56424c9 100644 --- a/src/runtime/internal/atomic/asm_amd64.s +++ b/src/runtime/internal/atomic/asm_amd64.s @@ -136,6 +136,12 @@ TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-12 TEXT runtime∕internal∕atomic·StoreRel(SB), NOSPLIT, $0-12 JMP runtime∕internal∕atomic·Store(SB) +TEXT runtime∕internal∕atomic·Store8(SB), NOSPLIT, $0-9 + MOVQ ptr+0(FP), BX + MOVB val+8(FP), AX + XCHGB AX, 0(BX) + RET + TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 MOVQ ptr+0(FP), BX MOVQ val+8(FP), AX diff --git a/src/runtime/internal/atomic/asm_mips64x.s b/src/runtime/internal/atomic/asm_mips64x.s index 9cb10371b7..3290fb726a 100644 --- a/src/runtime/internal/atomic/asm_mips64x.s +++ b/src/runtime/internal/atomic/asm_mips64x.s @@ -166,6 +166,14 @@ TEXT ·Store(SB), NOSPLIT, $0-12 SYNC RET +TEXT ·Store8(SB), NOSPLIT, $0-9 + MOVV ptr+0(FP), R1 + MOVB val+8(FP), R2 + SYNC + MOVB R2, 0(R1) + SYNC + RET + TEXT ·Store64(SB), NOSPLIT, $0-16 MOVV ptr+0(FP), R1 MOVV val+8(FP), R2 diff --git a/src/runtime/internal/atomic/asm_mipsx.s b/src/runtime/internal/atomic/asm_mipsx.s index af6bce57d6..62811a6599 100644 --- a/src/runtime/internal/atomic/asm_mipsx.s +++ b/src/runtime/internal/atomic/asm_mipsx.s @@ -32,6 +32,14 @@ TEXT ·Store(SB),NOSPLIT,$0-8 SYNC RET +TEXT ·Store8(SB),NOSPLIT,$0-5 + MOVW ptr+0(FP), R1 + MOVB val+4(FP), R2 + SYNC + MOVB R2, 0(R1) + SYNC + RET + TEXT ·Load(SB),NOSPLIT,$0-8 MOVW ptr+0(FP), R1 SYNC diff --git a/src/runtime/internal/atomic/asm_ppc64x.s b/src/runtime/internal/atomic/asm_ppc64x.s index 052b031cfb..06dc931bf4 100644 --- a/src/runtime/internal/atomic/asm_ppc64x.s +++ b/src/runtime/internal/atomic/asm_ppc64x.s @@ -170,6 +170,13 @@ TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-12 MOVW R4, 0(R3) RET +TEXT runtime∕internal∕atomic·Store8(SB), NOSPLIT, $0-9 + MOVD ptr+0(FP), R3 + MOVB val+8(FP), R4 + SYNC + MOVB R4, 0(R3) + RET + TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 MOVD ptr+0(FP), R3 MOVD val+8(FP), R4 diff --git a/src/runtime/internal/atomic/asm_s390x.s b/src/runtime/internal/atomic/asm_s390x.s index 084f5b5163..78abd48afa 100644 --- a/src/runtime/internal/atomic/asm_s390x.s +++ b/src/runtime/internal/atomic/asm_s390x.s @@ -12,6 +12,14 @@ TEXT ·Store(SB), NOSPLIT, $0 SYNC RET +// func Store8(ptr *uint8, val uint8) +TEXT ·Store8(SB), NOSPLIT, $0 + MOVD ptr+0(FP), R2 + MOVB val+8(FP), R3 + MOVB R3, 0(R2) + SYNC + RET + // func Store64(ptr *uint64, val uint64) TEXT ·Store64(SB), NOSPLIT, $0 MOVD ptr+0(FP), R2 diff --git a/src/runtime/internal/atomic/atomic_386.go b/src/runtime/internal/atomic/atomic_386.go index d7f82cc752..8d002ebfe3 100644 --- a/src/runtime/internal/atomic/atomic_386.go +++ b/src/runtime/internal/atomic/atomic_386.go @@ -74,6 +74,9 @@ func CasRel(ptr *uint32, old, new uint32) bool //go:noescape func Store(ptr *uint32, val uint32) +//go:noescape +func Store8(ptr *uint8, val uint8) + //go:noescape func Store64(ptr *uint64, val uint64) diff --git a/src/runtime/internal/atomic/atomic_amd64.go b/src/runtime/internal/atomic/atomic_amd64.go index fc865e892d..14b8101720 100644 --- a/src/runtime/internal/atomic/atomic_amd64.go +++ b/src/runtime/internal/atomic/atomic_amd64.go @@ -76,6 +76,9 @@ func CasRel(ptr *uint32, old, new uint32) bool //go:noescape func Store(ptr *uint32, val uint32) +//go:noescape +func Store8(ptr *uint8, val uint8) + //go:noescape func Store64(ptr *uint64, val uint64) diff --git a/src/runtime/internal/atomic/atomic_arm.go b/src/runtime/internal/atomic/atomic_arm.go index c1fc1f727f..95713afcc1 100644 --- a/src/runtime/internal/atomic/atomic_arm.go +++ b/src/runtime/internal/atomic/atomic_arm.go @@ -209,5 +209,8 @@ func Xchg64(addr *uint64, v uint64) uint64 //go:noescape func Load64(addr *uint64) uint64 +//go:noescape +func Store8(addr *uint8, v uint8) + //go:noescape func Store64(addr *uint64, v uint64) diff --git a/src/runtime/internal/atomic/atomic_arm64.go b/src/runtime/internal/atomic/atomic_arm64.go index 0182f309cc..26ca94d54c 100644 --- a/src/runtime/internal/atomic/atomic_arm64.go +++ b/src/runtime/internal/atomic/atomic_arm64.go @@ -56,6 +56,9 @@ func CasRel(ptr *uint32, old, new uint32) bool //go:noescape func Store(ptr *uint32, val uint32) +//go:noescape +func Store8(ptr *uint8, val uint8) + //go:noescape func Store64(ptr *uint64, val uint64) diff --git a/src/runtime/internal/atomic/atomic_arm64.s b/src/runtime/internal/atomic/atomic_arm64.s index a7e8c35449..d95689fe2d 100644 --- a/src/runtime/internal/atomic/atomic_arm64.s +++ b/src/runtime/internal/atomic/atomic_arm64.s @@ -48,6 +48,12 @@ TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-12 STLRW R1, (R0) RET +TEXT runtime∕internal∕atomic·Store8(SB), NOSPLIT, $0-9 + MOVD ptr+0(FP), R0 + MOVB val+8(FP), R1 + STLRB R1, (R0) + RET + TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 MOVD ptr+0(FP), R0 MOVD val+8(FP), R1 diff --git a/src/runtime/internal/atomic/atomic_mips64x.go b/src/runtime/internal/atomic/atomic_mips64x.go index ce11e38a96..1d9977850b 100644 --- a/src/runtime/internal/atomic/atomic_mips64x.go +++ b/src/runtime/internal/atomic/atomic_mips64x.go @@ -58,6 +58,9 @@ func CasRel(ptr *uint32, old, new uint32) bool //go:noescape func Store(ptr *uint32, val uint32) +//go:noescape +func Store8(ptr *uint8, val uint8) + //go:noescape func Store64(ptr *uint64, val uint64) diff --git a/src/runtime/internal/atomic/atomic_mipsx.go b/src/runtime/internal/atomic/atomic_mipsx.go index 6e39262c15..0e2d77ade1 100644 --- a/src/runtime/internal/atomic/atomic_mipsx.go +++ b/src/runtime/internal/atomic/atomic_mipsx.go @@ -141,6 +141,9 @@ func Or8(ptr *uint8, val uint8) //go:noescape func Store(ptr *uint32, val uint32) +//go:noescape +func Store8(ptr *uint8, val uint8) + // NO go:noescape annotation; see atomic_pointer.go. func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/internal/atomic/atomic_ppc64x.go b/src/runtime/internal/atomic/atomic_ppc64x.go index 13805a5275..a48ecf5ee8 100644 --- a/src/runtime/internal/atomic/atomic_ppc64x.go +++ b/src/runtime/internal/atomic/atomic_ppc64x.go @@ -58,6 +58,9 @@ func CasRel(ptr *uint32, old, new uint32) bool //go:noescape func Store(ptr *uint32, val uint32) +//go:noescape +func Store8(ptr *uint8, val uint8) + //go:noescape func Store64(ptr *uint64, val uint64) diff --git a/src/runtime/internal/atomic/atomic_s390x.go b/src/runtime/internal/atomic/atomic_s390x.go index 25fd890524..4d73b39baf 100644 --- a/src/runtime/internal/atomic/atomic_s390x.go +++ b/src/runtime/internal/atomic/atomic_s390x.go @@ -44,6 +44,9 @@ func LoadAcq(ptr *uint32) uint32 { //go:noescape func Store(ptr *uint32, val uint32) +//go:noescape +func Store8(ptr *uint8, val uint8) + //go:noescape func Store64(ptr *uint64, val uint64) diff --git a/src/runtime/internal/atomic/atomic_wasm.go b/src/runtime/internal/atomic/atomic_wasm.go index 0731763ac1..9037c2f7c8 100644 --- a/src/runtime/internal/atomic/atomic_wasm.go +++ b/src/runtime/internal/atomic/atomic_wasm.go @@ -141,6 +141,12 @@ func StoreRel(ptr *uint32, val uint32) { *ptr = val } +//go:nosplit +//go:noinline +func Store8(ptr *uint8, val uint8) { + *ptr = val +} + //go:nosplit //go:noinline func Store64(ptr *uint64, val uint64) { diff --git a/src/runtime/internal/atomic/sys_linux_arm.s b/src/runtime/internal/atomic/sys_linux_arm.s index df62f6c8ad..0c1cc3dc86 100644 --- a/src/runtime/internal/atomic/sys_linux_arm.s +++ b/src/runtime/internal/atomic/sys_linux_arm.s @@ -120,3 +120,25 @@ end: MOVB R1, ret+4(FP) RET +TEXT ·Store8(SB),NOSPLIT,$0-5 + MOVW addr+0(FP), R1 + MOVB v+4(FP), R2 + + MOVB runtime·goarm(SB), R8 + CMP $7, R8 + BGE native_barrier + BL memory_barrier<>(SB) + B store +native_barrier: + DMB MB_ISH + +store: + MOVB R2, (R1) + + CMP $7, R8 + BGE native_barrier2 + BL memory_barrier<>(SB) + RET +native_barrier2: + DMB MB_ISH + RET diff --git a/src/runtime/internal/atomic/sys_nonlinux_arm.s b/src/runtime/internal/atomic/sys_nonlinux_arm.s index 9d81334791..57568b2238 100644 --- a/src/runtime/internal/atomic/sys_nonlinux_arm.s +++ b/src/runtime/internal/atomic/sys_nonlinux_arm.s @@ -60,3 +60,20 @@ TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-5 MOVB R1, ret+4(FP) RET + +TEXT ·Store8(SB),NOSPLIT,$0-5 + MOVW addr+0(FP), R1 + MOVB v+4(FP), R2 + + MOVB runtime·goarm(SB), R8 + CMP $7, R8 + BLT 2(PC) + DMB MB_ISH + + MOVB R2, (R1) + + CMP $7, R8 + BLT 2(PC) + DMB MB_ISH + RET + -- 2.48.1