runtime: flush on every write barrier while debugging

Currently, we flush the write barrier buffer on every write barrier
once throwOnGCWork is set, but not during the mark completion
algorithm itself. As seen in recent failures like

  https://build.golang.org/log/317369853b803b4ee762b27653f367e1aa445ac1

by the time we actually catch a late gcWork put, the write barrier
buffer is full-size again.

As a result, we're probably not catching the actual problematic write
barrier, which is probably somewhere in the buffer.

Fix this by using the gcWork pause generation to also keep the write
barrier buffer small between the mark completion flushes it and when
mark completion is done.

For #27993.

Change-Id: I77618169441d42a7d562fb2a998cfaa89891edb2
Reviewed-on: https://go-review.googlesource.com/c/154638
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>
Reviewed-by: Michael Knyszek <mknyszek@google.com>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 36d48d2561c4f1fbebe50d9ce3971e3d56ff4b01..9d21dc4fa0c9fc369d4f82f02c266f2a8ed47715 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1434,6 +1434,7 @@ top:
                        if debugCachedWork {
                                b := &_p_.wbBuf
                                b.end = uintptr(unsafe.Pointer(&b.buf[wbBufEntryPointers]))
+                               b.debugGen = gcWorkPauseGen
                        }
                        // Flush the gcWork, since this may create global work
                        // and set the flushedWork flag.

diff --git a/src/runtime/mwbbuf.go b/src/runtime/mwbbuf.go
index 78ce54452db1c181271f893ec08ac1d3588d0d48..f444452bab502abe00a0a9d1adaa8dfd441161a7 100644 (file)
--- a/src/runtime/mwbbuf.go
+++ b/src/runtime/mwbbuf.go
@@ -23,6 +23,7 @@
 package runtime
 
 import (
+       "runtime/internal/atomic"
        "runtime/internal/sys"
        "unsafe"
 )
@@ -56,6 +57,12 @@ type wbBuf struct {
        // on. This must be a multiple of wbBufEntryPointers because
        // the write barrier only checks for overflow once per entry.
        buf [wbBufEntryPointers * wbBufEntries]uintptr
+
+       // debugGen causes the write barrier buffer to flush after
+       // every write barrier if equal to gcWorkPauseGen. This is for
+       // debugging #27993. This is only set if debugCachedWork is
+       // set.
+       debugGen uint32
 }
 
 const (
@@ -79,7 +86,7 @@ const (
 func (b *wbBuf) reset() {
        start := uintptr(unsafe.Pointer(&b.buf[0]))
        b.next = start
-       if writeBarrier.cgo || (debugCachedWork && throwOnGCWork) {
+       if writeBarrier.cgo || (debugCachedWork && (throwOnGCWork || b.debugGen == atomic.Load(&gcWorkPauseGen))) {
                // Effectively disable the buffer by forcing a flush
                // on every barrier.
                b.end = uintptr(unsafe.Pointer(&b.buf[wbBufEntryPointers]))