func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
atomicstorep1(noescape(ptr), new)
writebarrierptr_nostore((*uintptr)(ptr), uintptr(new))
- if mheap_.shadow_enabled {
- writebarrierptr_noshadow((*uintptr)(noescape(ptr)))
- }
}
//go:nosplit
func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
old := xchgp1(noescape(ptr), new)
writebarrierptr_nostore((*uintptr)(ptr), uintptr(new))
- if mheap_.shadow_enabled {
- writebarrierptr_noshadow((*uintptr)(noescape(ptr)))
- }
return old
}
return false
}
writebarrierptr_nostore((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
- if mheap_.shadow_enabled {
- writebarrierptr_noshadow((*uintptr)(noescape(unsafe.Pointer(ptr))))
- }
return true
}
sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
atomicstorep1(noescape(unsafe.Pointer(ptr)), new)
writebarrierptr_nostore((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
- if mheap_.shadow_enabled {
- writebarrierptr_noshadow((*uintptr)(noescape(unsafe.Pointer(ptr))))
- }
}
//go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr
func sync_atomic_SwapPointer(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(ptr)), uintptr(new)))
writebarrierptr_nostore((*uintptr)(ptr), uintptr(new))
- if mheap_.shadow_enabled {
- writebarrierptr_noshadow((*uintptr)(noescape(ptr)))
- }
return old
}
return false
}
writebarrierptr_nostore((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
- if mheap_.shadow_enabled {
- writebarrierptr_noshadow((*uintptr)(noescape(unsafe.Pointer(ptr))))
- }
return true
}
// implementation, markwb, and the various wrappers called by the
// compiler to implement pointer assignment, slice assignment,
// typed memmove, and so on.
-//
-// To check for missed write barriers, the GODEBUG=wbshadow debugging
-// mode allocates a second copy of the heap. Write barrier-based pointer
-// updates make changes to both the real heap and the shadow, and both
-// the pointer updates and the GC look for inconsistencies between the two,
-// indicating pointer writes that bypassed the barrier.
package runtime
// but if we do that, Go inserts a write barrier on *dst = src.
//go:nosplit
func writebarrierptr(dst *uintptr, src uintptr) {
+ *dst = src
if !writeBarrierEnabled {
- *dst = src
return
}
-
if src != 0 && (src < _PhysPageSize || src == poisonStack) {
systemstack(func() { throw("bad pointer in write barrier") })
}
-
- if mheap_.shadow_enabled {
- writebarrierptr_shadow(dst, src)
- }
-
- *dst = src
writebarrierptr_nostore1(dst, src)
}
-//go:nosplit
-func writebarrierptr_shadow(dst *uintptr, src uintptr) {
- systemstack(func() {
- addr := uintptr(unsafe.Pointer(dst))
- shadow := shadowptr(addr)
- if shadow == nil {
- return
- }
- // There is a race here but only if the program is using
- // racy writes instead of sync/atomic. In that case we
- // don't mind crashing.
- if *shadow != *dst && *shadow != noShadow && istrackedptr(*dst) {
- mheap_.shadow_enabled = false
- print("runtime: write barrier dst=", dst, " old=", hex(*dst), " shadow=", shadow, " old=", hex(*shadow), " new=", hex(src), "\n")
- throw("missed write barrier")
- }
- *shadow = src
- })
-}
-
// Like writebarrierptr, but the store has already been applied.
// Do not reapply.
//go:nosplit
if !writeBarrierEnabled {
return
}
-
if src != 0 && (src < _PhysPageSize || src == poisonStack) {
systemstack(func() { throw("bad pointer in write barrier") })
}
-
- // Apply changes to shadow.
- // Since *dst has been overwritten already, we cannot check
- // whether there were any missed updates, but writebarrierptr_nostore
- // is only rarely used.
- if mheap_.shadow_enabled {
- systemstack(func() {
- addr := uintptr(unsafe.Pointer(dst))
- shadow := shadowptr(addr)
- if shadow == nil {
- return
- }
- *shadow = src
- })
- }
-
writebarrierptr_nostore1(dst, src)
}
-// writebarrierptr_noshadow records that the value in *dst
-// has been written to using an atomic operation and the shadow
-// has not been updated. (In general if dst must be manipulated
-// atomically we cannot get the right bits for use in the shadow.)
-//go:nosplit
-func writebarrierptr_noshadow(dst *uintptr) {
- addr := uintptr(unsafe.Pointer(dst))
- shadow := shadowptr(addr)
- if shadow == nil {
- return
- }
-
- *shadow = noShadow
-}
-
//go:nosplit
func writebarrierstring(dst *[2]uintptr, src [2]uintptr) {
writebarrierptr(&dst[0], src[0])
func reflect_typedslicecopy(elemType *_type, dst, src slice) int {
return typedslicecopy(elemType, dst, src)
}
-
-// Shadow heap for detecting missed write barriers.
-
-// noShadow is stored in as the shadow pointer to mark that there is no
-// shadow word recorded. It matches any actual pointer word.
-// noShadow is used when it is impossible to know the right word
-// to store in the shadow heap, such as when the real heap word
-// is being manipulated atomically.
-const noShadow uintptr = 1
-
-func wbshadowinit() {
- // Initialize write barrier shadow heap if we were asked for it
- // and we have enough address space (not on 32-bit).
- if debug.wbshadow == 0 {
- return
- }
- if ptrSize != 8 {
- print("runtime: GODEBUG=wbshadow=1 disabled on 32-bit system\n")
- return
- }
-
- var reserved bool
- p1 := sysReserveHigh(mheap_.arena_end-mheap_.arena_start, &reserved)
- if p1 == nil {
- throw("cannot map shadow heap")
- }
- mheap_.shadow_heap = uintptr(p1) - mheap_.arena_start
- sysMap(p1, mheap_.arena_used-mheap_.arena_start, reserved, &memstats.other_sys)
- memmove(p1, unsafe.Pointer(mheap_.arena_start), mheap_.arena_used-mheap_.arena_start)
-
- mheap_.shadow_reserved = reserved
-
- for datap := &firstmoduledata; datap != nil; datap = datap.next {
- start := ^uintptr(0)
- end := uintptr(0)
- if start > datap.noptrdata {
- start = datap.noptrdata
- }
- if start > datap.data {
- start = datap.data
- }
- if start > datap.noptrbss {
- start = datap.noptrbss
- }
- if start > datap.bss {
- start = datap.bss
- }
- if end < datap.enoptrdata {
- end = datap.enoptrdata
- }
- if end < datap.edata {
- end = datap.edata
- }
- if end < datap.enoptrbss {
- end = datap.enoptrbss
- }
- if end < datap.ebss {
- end = datap.ebss
- }
- start &^= _PhysPageSize - 1
- end = round(end, _PhysPageSize)
- datap.data_start = start
- datap.data_end = end
- reserved = false
- p1 = sysReserveHigh(end-start, &reserved)
- if p1 == nil {
- throw("cannot map shadow data")
- }
- datap.shadow_data = uintptr(p1) - start
- sysMap(p1, end-start, reserved, &memstats.other_sys)
- memmove(p1, unsafe.Pointer(start), end-start)
- }
-
- mheap_.shadow_enabled = true
- writeBarrierEnabled = true
-}
-
-// shadowptr returns a pointer to the shadow value for addr.
-//go:nosplit
-func shadowptr(addr uintptr) *uintptr {
- for datap := &firstmoduledata; datap != nil; datap = datap.next {
- if datap.data_start <= addr && addr < datap.data_end {
- return (*uintptr)(unsafe.Pointer(addr + datap.shadow_data))
- }
- }
- if inheap(addr) {
- return (*uintptr)(unsafe.Pointer(addr + mheap_.shadow_heap))
- }
- return nil
-}
-
-// istrackedptr reports whether the pointer value p requires a write barrier
-// when stored into the heap.
-func istrackedptr(p uintptr) bool {
- return inheap(p)
-}
-
-// checkwbshadow checks that p matches its shadow word.
-// The garbage collector calls checkwbshadow for each pointer during the checkmark phase.
-// It is only called when mheap_.shadow_enabled is true.
-func checkwbshadow(p *uintptr) {
- addr := uintptr(unsafe.Pointer(p))
- shadow := shadowptr(addr)
- if shadow == nil {
- return
- }
- // There is no race on the accesses here, because the world is stopped,
- // but there may be racy writes that lead to the shadow and the
- // heap being inconsistent. If so, we will detect that here as a
- // missed write barrier and crash. We don't mind.
- // Code should use sync/atomic instead of racy pointer writes.
- if *shadow != *p && *shadow != noShadow && istrackedptr(*p) {
- mheap_.shadow_enabled = false
- print("runtime: checkwritebarrier p=", p, " *p=", hex(*p), " shadow=", shadow, " *shadow=", hex(*shadow), "\n")
- throw("missed write barrier")
- }
-}
-
-// clearshadow clears the shadow copy associated with the n bytes of memory at addr.
-func clearshadow(addr, n uintptr) {
- if !mheap_.shadow_enabled {
- return
- }
- p := shadowptr(addr)
- if p == nil || n <= ptrSize {
- return
- }
- memclr(unsafe.Pointer(p), n)
-}