}
}
-// needwb reports whether a write barrier is needed now
-// (otherwise the write can be made directly).
-//go:nosplit
-func needwb() bool {
- return gcphase == _GCmark || gcphase == _GCmarktermination || mheap_.shadow_enabled
-}
-
// Write barrier calls must not happen during critical GC and scheduler
// related operations. In particular there are times when the GC assumes
// that the world is stopped but scheduler related code is still being
// but if we do that, Go inserts a write barrier on *dst = src.
//go:nosplit
func writebarrierptr(dst *uintptr, src uintptr) {
- if !needwb() {
+ if !writeBarrierEnabled {
*dst = src
return
}
// Do not reapply.
//go:nosplit
func writebarrierptr_nostore(dst *uintptr, src uintptr) {
- if !needwb() {
+ if !writeBarrierEnabled {
return
}
// typedmemmove copies a value of type t to dst from src.
//go:nosplit
func typedmemmove(typ *_type, dst, src unsafe.Pointer) {
- if !needwb() || (typ.kind&kindNoPointers) != 0 {
+ if !writeBarrierEnabled || (typ.kind&kindNoPointers) != 0 {
memmove(dst, src, typ.size)
return
}
// dst and src point off bytes into the value and only copies size bytes.
//go:linkname reflect_typedmemmovepartial reflect.typedmemmovepartial
func reflect_typedmemmovepartial(typ *_type, dst, src unsafe.Pointer, off, size uintptr) {
- if !needwb() || (typ.kind&kindNoPointers) != 0 || size < ptrSize {
+ if !writeBarrierEnabled || (typ.kind&kindNoPointers) != 0 || size < ptrSize {
memmove(dst, src, size)
return
}
// not to be preempted before the write barriers have been run.
//go:nosplit
func callwritebarrier(typ *_type, frame unsafe.Pointer, framesize, retoffset uintptr) {
- if !needwb() || typ == nil || (typ.kind&kindNoPointers) != 0 || framesize-retoffset < ptrSize {
+ if !writeBarrierEnabled || typ == nil || (typ.kind&kindNoPointers) != 0 || framesize-retoffset < ptrSize {
return
}
racereadrangepc(srcp, uintptr(n)*typ.size, callerpc, pc)
}
- if !needwb() {
+ if !writeBarrierEnabled {
memmove(dstp, srcp, uintptr(n)*typ.size)
return n
}
}
mheap_.shadow_enabled = true
+ writeBarrierEnabled = true
}
// shadowptr returns a pointer to the shadow value for addr.
return out
}
+// Garbage collector phase.
+// Indicates to write barrier and sychronization task to preform.
+var gcphase uint32
+var writeBarrierEnabled bool // compiler emits references to this in write barriers
+
+// gcBlackenEnabled is 1 if mutator assists and background mark
+// workers are allowed to blacken objects. This must only be set when
+// gcphase == _GCmark.
+var gcBlackenEnabled uint32
+
+const (
+ _GCoff = iota // GC not running, write barrier disabled
+ _GCquiesce // unused state
+ _GCstw // unused state
+ _GCscan // GC collecting roots into workbufs, write barrier disabled
+ _GCmark // GC marking from workbufs, write barrier ENABLED
+ _GCmarktermination // GC mark termination: allocate black, P's help GC, write barrier ENABLED
+ _GCsweep // GC mark completed; sweeping in background, write barrier disabled
+)
+
+//go:nosplit
+func setGCPhase(x uint32) {
+ atomicstore(&gcphase, x)
+ writeBarrierEnabled = gcphase == _GCmark || gcphase == _GCmarktermination || mheap_.shadow_enabled
+}
+
// gcMarkWorkerMode represents the mode that a concurrent mark worker
// should operate in.
//
heapGoal = gcController.heapGoal
systemstack(func() {
- gcphase = _GCscan
+ setGCPhase(_GCscan)
// Concurrent scan.
starttheworld()
if debug.gctrace > 0 {
tInstallWB = nanotime()
}
- atomicstore(&gcphase, _GCmark)
+ setGCPhase(_GCmark)
// Ensure all Ps have observed the phase
// change and have write barriers enabled
// before any blackening occurs.
// World is stopped.
// Start marktermination which includes enabling the write barrier.
atomicstore(&gcBlackenEnabled, 0)
- gcphase = _GCmarktermination
+ setGCPhase(_GCmarktermination)
if debug.gctrace > 0 {
heap1 = memstats.heap_live
}
// marking is complete so we can turn the write barrier off
- gcphase = _GCoff
+ setGCPhase(_GCoff)
gcSweep(mode)
if debug.gctrace > 1 {
// Still in STW but gcphase is _GCoff, reset to _GCmarktermination
// At this point all objects will be found during the gcMark which
// does a complete STW mark and object scan.
- gcphase = _GCmarktermination
+ setGCPhase(_GCmarktermination)
gcMark(startTime)
- gcphase = _GCoff // marking is done, turn off wb.
+ setGCPhase(_GCoff) // marking is done, turn off wb.
gcSweep(mode)
}
})
pushcnt uintptr
}
-// Indicates to write barrier and sychronization task to preform.
-const (
- _GCoff = iota // GC not running, write barrier disabled
- _GCquiesce // unused state
- _GCstw // unused state
- _GCscan // GC collecting roots into workbufs, write barrier disabled
- _GCmark // GC marking from workbufs, write barrier ENABLED
- _GCmarktermination // GC mark termination: allocate black, P's help GC, write barrier ENABLED
- _GCsweep // GC mark completed; sweeping in background, write barrier disabled
-)
-
type forcegcstate struct {
lock mutex
g *g
idle uint32
}
-var gcphase uint32
-
-// gcBlackenEnabled is 1 if mutator assists and background mark
-// workers are allowed to blacken objects. This must only be set when
-// gcphase == _GCmark.
-var gcBlackenEnabled uint32
-
/*
* known to compiler
*/
memclr(add(p, lenmem), capmem-lenmem)
} else {
// Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan unitialized memory.
- // TODO(rsc): Use memmove when !needwb().
+ // TODO(rsc): Use memmove when !writeBarrierEnabled.
p = newarray(et, uintptr(newcap))
for i := 0; i < old.len; i++ {
typedmemmove(et, add(p, uintptr(i)*et.size), add(old.array, uintptr(i)*et.size))