[dev.garbage] runtime: reduce stack footprint of write barriers

This is going to hurt a bit but we'll make it better later.
Now the race detector can be run again.

I added the write barrier optimizations from
CL 183020043 to try to make it hurt a little less.

TBR=rlh
CC=golang-codereviews
https://golang.org/cl/185070043

diff --git a/src/run.bash b/src/run.bash
index b8ce417a0a823089fc13102d8ff589a67fe8c4a5..ff98d09f514909b606c7431b1eb0251090184f2c 100755 (executable)
--- a/src/run.bash
+++ b/src/run.bash
@@ -163,7 +163,7 @@ esac
 # and only on amd64, and only when cgo is enabled.
 # Delayed until here so we know whether to try external linking.
 # DISABLED until we get garbage collection working.
-case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED-XXX-DISABLED" in
+case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED" in
 linux-linux-amd64-1 | freebsd-freebsd-amd64-1 | darwin-darwin-amd64-1)
        echo
        echo '# Testing race detector.'

diff --git a/src/runtime/mgc0.go b/src/runtime/mgc0.go
index 7797894fc5fbb9b50135a5a94e0f517d535278be..d6e786165b8a4a1de838856b5786e122187b8a8d 100644 (file)
--- a/src/runtime/mgc0.go
+++ b/src/runtime/mgc0.go
@@ -102,19 +102,25 @@ const (
        _PoisonStack = 0x6868686868686868 & (1<<(8*ptrSize) - 1)
 )
 
+func needwb() bool {
+       return gcphase == _GCmark || gcphase == _GCmarktermination
+}
+
 // NOTE: Really dst *unsafe.Pointer, src unsafe.Pointer,
 // but if we do that, Go inserts a write barrier on *dst = src.
 //go:nosplit
 func writebarrierptr(dst *uintptr, src uintptr) {
        *dst = src
-       writebarrierptr_nostore(dst, src)
+       if needwb() {
+               writebarrierptr_nostore(dst, src)
+       }
 }
 
 // Like writebarrierptr, but the store has already been applied.
 // Do not reapply.
 //go:nosplit
 func writebarrierptr_nostore(dst *uintptr, src uintptr) {
-       if getg() == nil { // very low-level startup
+       if getg() == nil || !needwb() { // very low-level startup
                return
        }
 
@@ -162,29 +168,36 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
 
 //go:nosplit
 func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
-       mask := loadPtrMask(typ)
-       nptr := typ.size / ptrSize
-       for i := uintptr(0); i < nptr; i += 2 {
-               bits := mask[i/2]
-               if (bits>>2)&_BitsMask == _BitsPointer {
-                       writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
-               } else {
-                       *(*uintptr)(dst) = *(*uintptr)(src)
-               }
-               dst = add(dst, ptrSize)
-               src = add(src, ptrSize)
-               if i+1 == nptr {
-                       break
-               }
-               bits >>= 4
-               if (bits>>2)&_BitsMask == _BitsPointer {
-                       writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
-               } else {
-                       *(*uintptr)(dst) = *(*uintptr)(src)
-               }
-               dst = add(dst, ptrSize)
-               src = add(src, ptrSize)
+       if !needwb() {
+               memmove(dst, src, typ.size)
+               return
        }
+
+       systemstack(func() {
+               mask := loadPtrMask(typ)
+               nptr := typ.size / ptrSize
+               for i := uintptr(0); i < nptr; i += 2 {
+                       bits := mask[i/2]
+                       if (bits>>2)&_BitsMask == _BitsPointer {
+                               writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
+                       } else {
+                               *(*uintptr)(dst) = *(*uintptr)(src)
+                       }
+                       dst = add(dst, ptrSize)
+                       src = add(src, ptrSize)
+                       if i+1 == nptr {
+                               break
+                       }
+                       bits >>= 4
+                       if (bits>>2)&_BitsMask == _BitsPointer {
+                               writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
+                       } else {
+                               *(*uintptr)(dst) = *(*uintptr)(src)
+                       }
+                       dst = add(dst, ptrSize)
+                       src = add(src, ptrSize)
+               }
+       })
 }
 
 //go:nosplit
@@ -199,33 +212,40 @@ func writebarriercopy(typ *_type, dst, src slice) int {
        dstp := unsafe.Pointer(dst.array)
        srcp := unsafe.Pointer(src.array)
 
-       if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) {
-               // Overlap with src before dst.
-               // Copy backward, being careful not to move dstp/srcp
-               // out of the array they point into.
-               dstp = add(dstp, uintptr(n-1)*typ.size)
-               srcp = add(srcp, uintptr(n-1)*typ.size)
-               i := uint(0)
-               for {
-                       writebarrierfat(typ, dstp, srcp)
-                       if i++; i >= n {
-                               break
+       if !needwb() {
+               memmove(dstp, srcp, uintptr(n)*typ.size)
+               return int(n)
+       }
+
+       systemstack(func() {
+               if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) {
+                       // Overlap with src before dst.
+                       // Copy backward, being careful not to move dstp/srcp
+                       // out of the array they point into.
+                       dstp = add(dstp, uintptr(n-1)*typ.size)
+                       srcp = add(srcp, uintptr(n-1)*typ.size)
+                       i := uint(0)
+                       for {
+                               writebarrierfat(typ, dstp, srcp)
+                               if i++; i >= n {
+                                       break
+                               }
+                               dstp = add(dstp, -typ.size)
+                               srcp = add(srcp, -typ.size)
                        }
-                       dstp = add(dstp, -typ.size)
-                       srcp = add(srcp, -typ.size)
-               }
-       } else {
-               // Copy forward, being careful not to move dstp/srcp
-               // out of the array they point into.
-               i := uint(0)
-               for {
-                       writebarrierfat(typ, dstp, srcp)
-                       if i++; i >= n {
-                               break
+               } else {
+                       // Copy forward, being careful not to move dstp/srcp
+                       // out of the array they point into.
+                       i := uint(0)
+                       for {
+                               writebarrierfat(typ, dstp, srcp)
+                               if i++; i >= n {
+                                       break
+                               }
+                               dstp = add(dstp, typ.size)
+                               srcp = add(srcp, typ.size)
                        }
-                       dstp = add(dstp, typ.size)
-                       srcp = add(srcp, typ.size)
                }
-       }
+       })
        return int(n)
 }