runtime: merge inline mark bits with gcmarkBits 8 bytes at a time

Currently, with Green Tea GC, we need to copy (really bitwise-or) mark
bits back into mspan.gcmarkBits, so that it can propagate to
mspan.allocBits at sweep time. This function does actually seem to make
sweeping small spans a good bit more expensive, though sweeping is still
relatively cheap. There's some low-hanging fruit here though, in that
the merge is performed one byte at a time, but this is pretty
inefficient. We can almost as easily perform this merge one word at a
time instead, which seems to make this operation about 33% faster.

For #73581.

Change-Id: I170d36e7a2193199c423dcd556cba048ebd698af
Reviewed-on: https://go-review.googlesource.com/c/go/+/687935
Reviewed-by: Michael Pratt <mpratt@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Auto-Submit: Michael Knyszek <mknyszek@google.com>

diff --git a/src/runtime/mgcmark_greenteagc.go b/src/runtime/mgcmark_greenteagc.go
index ac2b1732f930265aac262ba3e3726c1a1b4e848c..3a368438d4b57745ff12f487a8612742808f3584 100644 (file)
--- a/src/runtime/mgcmark_greenteagc.go
+++ b/src/runtime/mgcmark_greenteagc.go
@@ -192,9 +192,13 @@ func (s *mspan) mergeInlineMarks(dst *gcBits) {
        }
        bytes := divRoundUp(uintptr(s.nelems), 8)
        imb := s.inlineMarkBits()
-       _ = imb.marks[bytes-1]
-       for i := uintptr(0); i < bytes; i++ {
-               *dst.bytep(i) |= imb.marks[i]
+       imbMarks := (*gc.ObjMask)(unsafe.Pointer(&imb.marks))
+       for i := uintptr(0); i < bytes; i += goarch.PtrSize {
+               marks := bswapIfBigEndian(imbMarks[i/goarch.PtrSize])
+               if i/goarch.PtrSize == uintptr(len(imb.marks)+1)/goarch.PtrSize-1 {
+                       marks &^= 0xff << ((goarch.PtrSize - 1) * 8) // mask out class
+               }
+               *(*uintptr)(unsafe.Pointer(dst.bytep(i))) |= bswapIfBigEndian(marks)
        }
        if doubleCheckGreenTea && !s.spanclass.noscan() && imb.marks != imb.scans {
                throw("marks don't match scans for span with pointer")
@@ -652,7 +656,7 @@ func spanSetScans(spanBase uintptr, nelems uint16, imb *spanInlineMarkBits, toSc
                marks := imbMarks[i/goarch.PtrSize]
                scans = bswapIfBigEndian(scans)
                marks = bswapIfBigEndian(marks)
-               if i/goarch.PtrSize == 64/goarch.PtrSize-1 {
+               if i/goarch.PtrSize == uintptr(len(imb.marks)+1)/goarch.PtrSize-1 {
                        scans &^= 0xff << ((goarch.PtrSize - 1) * 8) // mask out owned
                        marks &^= 0xff << ((goarch.PtrSize - 1) * 8) // mask out class
                }