runtime: ensure GC sees type-safe memory on weak machines

Currently its possible for the garbage collector to observe
uninitialized memory or stale heap bitmap bits on weakly ordered
architectures such as ARM and PPC. On such architectures, the stores
that zero newly allocated memory and initialize its heap bitmap may
move after a store in user code that makes the allocated object
observable by the garbage collector.

To fix this, add a "publication barrier" (also known as an "export
barrier") before returning from mallocgc. This is a store/store
barrier that ensures any write done by user code that makes the
returned object observable to the garbage collector will be ordered
after the initialization performed by mallocgc. No barrier is
necessary on the reading side because of the data dependency between
loading the pointer and loading the contents of the object.

Fixes one of the issues raised in #9984.

Change-Id: Ia3d96ad9c5fc7f4d342f5e05ec0ceae700cd17c8
Reviewed-on: https://go-review.googlesource.com/11083
Reviewed-by: Rick Hudson <rlh@golang.org>
Reviewed-by: Dmitry Vyukov <dvyukov@google.com>
Reviewed-by: Minux Ma <minux@golang.org>
Reviewed-by: Martin Capitanio <capnm9@gmail.com>
Reviewed-by: Russ Cox <rsc@golang.org>

diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index b8a4054931d5cb6c7ebffacb48067c0b800f0878..eb9ca6350a325b9a8dc819e7dde9b39783d603e4 100644 (file)
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -632,6 +632,11 @@ TEXT runtime·atomicand8(SB), NOSPLIT, $0-5
        ANDB    BX, (AX)
        RET
 
+TEXT ·publicationBarrier(SB),NOSPLIT,$0-0
+       // Stores are already ordered on x86, so this is just a
+       // compile barrier.
+       RET
+
 // void jmpdefer(fn, sp);
 // called from deferreturn.
 // 1. pop the caller

diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index 13cca8e460834b0b0bf6a0efcd547cab8cfcced0..3b4ca4d012f88fdbc099e19fd3581bdf3356d674 100644 (file)
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -615,6 +615,11 @@ TEXT runtime·atomicand8(SB), NOSPLIT, $0-9
        ANDB    BX, (AX)
        RET
 
+TEXT ·publicationBarrier(SB),NOSPLIT,$0-0
+       // Stores are already ordered on x86, so this is just a
+       // compile barrier.
+       RET
+
 // void jmpdefer(fn, sp);
 // called from deferreturn.
 // 1. pop the caller

diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s
index c058bde420ebafacef7760f762f1f01b69a8c06b..a5d6e8155ad2217e32631da52ebf6577e5065543 100644 (file)
--- a/src/runtime/asm_amd64p32.s
+++ b/src/runtime/asm_amd64p32.s
@@ -569,6 +569,11 @@ TEXT runtime·atomicand8(SB), NOSPLIT, $0-5
        ANDB    AX, 0(BX)
        RET
 
+TEXT ·publicationBarrier(SB),NOSPLIT,$0-0
+       // Stores are already ordered on x86, so this is just a
+       // compile barrier.
+       RET
+
 // void jmpdefer(fn, sp);
 // called from deferreturn.
 // 1. pop the caller

diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s
index 874dc4fe55b9ae84042d79ffbb5d92bef3256ffb..661538c02472706aaadb492337fcfa46c98778f2 100644 (file)
--- a/src/runtime/asm_arm.s
+++ b/src/runtime/asm_arm.s
@@ -736,6 +736,17 @@ TEXT runtime·atomicloaduint(SB),NOSPLIT,$0-8
 TEXT runtime·atomicstoreuintptr(SB),NOSPLIT,$0-8
        B       runtime·atomicstore(SB)
 
+// armPublicationBarrier is a native store/store barrier for ARMv7+.
+// To implement publiationBarrier in sys_$GOOS_arm.s using the native
+// instructions, use:
+//
+//     TEXT ·publicationBarrier(SB),NOSPLIT,$-4-0
+//             B       runtime·armPublicationBarrier(SB)
+//
+TEXT runtime·armPublicationBarrier(SB),NOSPLIT,$-4-0
+       WORD $0xf57ff05e        // DMB ST
+       RET
+
 // AES hashing not implemented for ARM
 TEXT runtime·aeshash(SB),NOSPLIT,$-4-0
        MOVW    $0, R0

diff --git a/src/runtime/atomic_arm64.s b/src/runtime/atomic_arm64.s
index acd0a62f4d11215b8df1dc5100543f01023e3f2f..d3ab2a121ca28eb0ea978f5fd503a86b836c0200 100644 (file)
--- a/src/runtime/atomic_arm64.s
+++ b/src/runtime/atomic_arm64.s
@@ -111,3 +111,7 @@ again:
 
 TEXT runtime·xchguintptr(SB), NOSPLIT, $0-24
        B       runtime·xchg64(SB)
+
+TEXT ·publicationBarrier(SB),NOSPLIT,$-8-0
+       DMB     $0xe    // DMB ST
+       RET

diff --git a/src/runtime/atomic_ppc64x.s b/src/runtime/atomic_ppc64x.s
index d84865efd67ce91a5a28b80c59431341c6746f82..28c5bf3729009453b6c27dd29707db6731d42357 100644 (file)
--- a/src/runtime/atomic_ppc64x.s
+++ b/src/runtime/atomic_ppc64x.s
@@ -38,3 +38,10 @@ TEXT ·atomicloadp(SB),NOSPLIT,$-8-16
        ISYNC
        MOVD    R3, ret+8(FP)
        RET
+
+TEXT ·publicationBarrier(SB),NOSPLIT,$-8-0
+       // LWSYNC is the "export" barrier recommended by Power ISA
+       // v2.07 book II, appendix B.2.2.2.
+       // LWSYNC is a load/load, load/store, and store/store barrier.
+       WORD $0x7c2004ac        // LWSYNC
+       RET

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 5872a3752e43b95366ac87159034954239d0835c..37d3a1eea18de34a488dd964482896d5a97d91c4 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -657,6 +657,14 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
                } else {
                        c.local_scan += typ.ptrdata
                }
+
+               // Ensure that the stores above that initialize x to
+               // type-safe memory and set the heap bits occur before
+               // the caller can make x observable to the garbage
+               // collector. Otherwise, on weakly ordered machines,
+               // the garbage collector could follow a pointer to x,
+               // but see uninitialized memory or stale heap bits.
+               publicationBarrier()
        }
 
        // GCmarkterminate allocates black

diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go
index f116dc3e9f1a46aacac47ae1482f298509a4b144..cd9a22336f05aa368d92beff9ef1aac773ae9a00 100644 (file)
--- a/src/runtime/stubs.go
+++ b/src/runtime/stubs.go
@@ -167,6 +167,23 @@ func xaddint64(ptr *int64, delta int64) int64 {
        return int64(xadd64((*uint64)(unsafe.Pointer(ptr)), delta))
 }
 
+// publicationBarrier performs a store/store barrier (a "publication"
+// or "export" barrier). Some form of synchronization is required
+// between initializing an object and making that object accessible to
+// another processor. Without synchronization, the initialization
+// writes and the "publication" write may be reordered, allowing the
+// other processor to follow the pointer and observe an uninitialized
+// object. In general, higher-level synchronization should be used,
+// such as locking or an atomic pointer write. publicationBarrier is
+// for when those aren't an option, such as in the implementation of
+// the memory manager.
+//
+// There's no corresponding barrier for the read side because the read
+// side naturally has a data dependency order. All architectures that
+// Go supports or seems likely to ever support automatically enforce
+// data dependency ordering.
+func publicationBarrier()
+
 //go:noescape
 func setcallerpc(argp unsafe.Pointer, pc uintptr)
 

diff --git a/src/runtime/sys_darwin_arm.s b/src/runtime/sys_darwin_arm.s
index b4c1b27530222c7169c0b4967d88aab869439198..55ae8f3a46b5e32fa8b819b4233734460ec623c2 100644 (file)
--- a/src/runtime/sys_darwin_arm.s
+++ b/src/runtime/sys_darwin_arm.s
@@ -301,6 +301,9 @@ TEXT runtime·cas(SB),NOSPLIT,$0
 TEXT runtime·casp1(SB),NOSPLIT,$0
        B       runtime·cas(SB)
 
+TEXT ·publicationBarrier(SB),NOSPLIT,$-4-0
+       B       runtime·armPublicationBarrier(SB)
+
 TEXT runtime·sysctl(SB),NOSPLIT,$0
        MOVW    mib+0(FP), R0
        MOVW    miblen+4(FP), R1

diff --git a/src/runtime/sys_freebsd_arm.s b/src/runtime/sys_freebsd_arm.s
index 2b5d754590b0a613c9392a234945bd226413d05f..3dd04cf97325f0d4b635763c4bc87a13686012ce 100644 (file)
--- a/src/runtime/sys_freebsd_arm.s
+++ b/src/runtime/sys_freebsd_arm.s
@@ -381,6 +381,10 @@ TEXT runtime·casp1(SB),NOSPLIT,$0
 TEXT runtime·cas(SB),NOSPLIT,$0
        B runtime·armcas(SB)
 
+// TODO: this is only valid for ARMv7+
+TEXT ·publicationBarrier(SB),NOSPLIT,$-4-0
+       B       runtime·armPublicationBarrier(SB)
+
 // TODO(minux): this only supports ARMv6K+.
 TEXT runtime·read_tls_fallback(SB),NOSPLIT,$-4
        WORD $0xee1d0f70 // mrc p15, 0, r0, c13, c0, 3

diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s
index 50f074a234bc7d7aaef1ec1e2b4a8418f9a09d56..b68b81af3e1f6f0a3605fc7434ce6046b84e63ac 100644 (file)
--- a/src/runtime/sys_linux_arm.s
+++ b/src/runtime/sys_linux_arm.s
@@ -416,6 +416,22 @@ check:
 TEXT runtime·casp1(SB),NOSPLIT,$0
        B       runtime·cas(SB)
 
+// As for cas, memory barriers are complicated on ARM, but the kernel
+// provides a user helper. ARMv5 does not support SMP and has no
+// memory barrier instruction at all. ARMv6 added SMP support and has
+// a memory barrier, but it requires writing to a coprocessor
+// register. ARMv7 introduced the DMB instruction, but it's expensive
+// even on single-core devices. The kernel helper takes care of all of
+// this for us.
+
+TEXT publicationBarrier<>(SB),NOSPLIT,$0
+       // void __kuser_memory_barrier(void);
+       MOVW    $0xffff0fa0, R15 // R15 is hardware PC.
+
+TEXT ·publicationBarrier(SB),NOSPLIT,$0
+       BL      publicationBarrier<>(SB)
+       RET
+
 TEXT runtime·osyield(SB),NOSPLIT,$0
        MOVW    $SYS_sched_yield, R7
        SWI     $0

diff --git a/src/runtime/sys_nacl_arm.s b/src/runtime/sys_nacl_arm.s
index 39ef25a618b1a3db823fa6fdbf760028ac5014e8..cf4804fe14738917562dc4e116e9d36fbd314530 100644 (file)
--- a/src/runtime/sys_nacl_arm.s
+++ b/src/runtime/sys_nacl_arm.s
@@ -323,5 +323,9 @@ TEXT runtime·casp1(SB),NOSPLIT,$0
 TEXT runtime·cas(SB),NOSPLIT,$0
        B runtime·armcas(SB)
 
+// Likewise, this is only valid for ARMv7+, but that's okay.
+TEXT ·publicationBarrier(SB),NOSPLIT,$-4-0
+       B       runtime·armPublicationBarrier(SB)
+
 TEXT runtime·read_tls_fallback(SB),NOSPLIT,$-4
        WORD $0xe7fedef0 // NACL_INSTR_ARM_ABORT_NOW (UDF #0xEDE0)

diff --git a/src/runtime/sys_netbsd_arm.s b/src/runtime/sys_netbsd_arm.s
index d275d6d0b6629b19cc8fcd803501d18d6a252077..5832f6d15c2666ac0bdfda1afaec2f15cbcf994d 100644 (file)
--- a/src/runtime/sys_netbsd_arm.s
+++ b/src/runtime/sys_netbsd_arm.s
@@ -349,6 +349,10 @@ TEXT runtime·casp1(SB),NOSPLIT,$0
 TEXT runtime·cas(SB),NOSPLIT,$0
        B runtime·armcas(SB)
 
+// TODO: this is only valid for ARMv7+
+TEXT ·publicationBarrier(SB),NOSPLIT,$-4-0
+       B       runtime·armPublicationBarrier(SB)
+
 TEXT runtime·read_tls_fallback(SB),NOSPLIT,$-4
        MOVM.WP [R1, R2, R3, R12], (R13)
        SWI $0x00a0013c // _lwp_getprivate

diff --git a/src/runtime/sys_openbsd_arm.s b/src/runtime/sys_openbsd_arm.s
index e28d43eeaad65dedea3c7adfe0e8db3c013857f3..d231f0fdb3ddc26887819566cb8dffc97bc4a974 100644 (file)
--- a/src/runtime/sys_openbsd_arm.s
+++ b/src/runtime/sys_openbsd_arm.s
@@ -374,6 +374,9 @@ TEXT runtime·casp1(SB),NOSPLIT,$0
 TEXT runtime·cas(SB),NOSPLIT,$0
        B       runtime·armcas(SB)
 
+TEXT ·publicationBarrier(SB),NOSPLIT,$-4-0
+       B       runtime·armPublicationBarrier(SB)
+
 // TODO(jsing): Implement.
 TEXT runtime·read_tls_fallback(SB),NOSPLIT,$-4
        MOVW    $5, R0