runtime: mark span when marking any object on the span

This adds a mark bit for each span that is set if any objects on the
span are marked. This will be used for sweeping.

For #18155.

The impact of this is negligible for most benchmarks, and < 1% for
GC-heavy benchmarks.

name                       old time/op  new time/op  delta
Garbage/benchmem-MB=64-12  2.18ms ± 0%  2.20ms ± 1%  +0.88%  (p=0.000 n=16+18)

(https://perf.golang.org/search?q=upload:20180928.1)

name                      old time/op    new time/op    delta
BinaryTree17-12              2.68s ± 1%     2.68s ± 1%    ~     (p=0.707 n=17+19)
Fannkuch11-12                2.28s ± 0%     2.39s ± 0%  +4.95%  (p=0.000 n=19+18)
FmtFprintfEmpty-12          40.3ns ± 4%    39.4ns ± 2%  -2.27%  (p=0.000 n=17+18)
FmtFprintfString-12         67.9ns ± 1%    68.3ns ± 1%  +0.55%  (p=0.000 n=18+19)
FmtFprintfInt-12            75.7ns ± 1%    76.1ns ± 1%  +0.44%  (p=0.005 n=18+19)
FmtFprintfIntInt-12          123ns ± 1%     121ns ± 1%  -1.00%  (p=0.000 n=18+18)
FmtFprintfPrefixedInt-12     150ns ± 0%     148ns ± 0%  -1.33%  (p=0.000 n=16+13)
FmtFprintfFloat-12           208ns ± 0%     204ns ± 0%  -1.92%  (p=0.000 n=13+17)
FmtManyArgs-12               501ns ± 1%     498ns ± 0%  -0.55%  (p=0.000 n=19+17)
GobDecode-12                6.24ms ± 0%    6.25ms ± 1%    ~     (p=0.113 n=20+19)
GobEncode-12                5.33ms ± 0%    5.29ms ± 1%  -0.72%  (p=0.000 n=20+18)
Gzip-12                      220ms ± 1%     218ms ± 1%  -1.02%  (p=0.000 n=19+19)
Gunzip-12                   35.5ms ± 0%    35.7ms ± 0%  +0.45%  (p=0.000 n=16+18)
HTTPClientServer-12         77.9µs ± 1%    77.7µs ± 1%  -0.30%  (p=0.047 n=20+19)
JSONEncode-12               8.82ms ± 0%    8.93ms ± 0%  +1.20%  (p=0.000 n=18+17)
JSONDecode-12               47.3ms ± 0%    47.0ms ± 0%  -0.49%  (p=0.000 n=17+18)
Mandelbrot200-12            3.69ms ± 0%    3.68ms ± 0%  -0.25%  (p=0.000 n=19+18)
GoParse-12                  3.13ms ± 1%    3.13ms ± 1%    ~     (p=0.640 n=20+20)
RegexpMatchEasy0_32-12      76.2ns ± 1%    76.2ns ± 1%    ~     (p=0.818 n=20+19)
RegexpMatchEasy0_1K-12       226ns ± 0%     226ns ± 0%  -0.22%  (p=0.001 n=17+18)
RegexpMatchEasy1_32-12      71.9ns ± 1%    72.0ns ± 1%    ~     (p=0.653 n=18+18)
RegexpMatchEasy1_1K-12       355ns ± 1%     356ns ± 1%    ~     (p=0.160 n=18+19)
RegexpMatchMedium_32-12      106ns ± 1%     106ns ± 1%    ~     (p=0.325 n=17+20)
RegexpMatchMedium_1K-12     31.1µs ± 2%    31.2µs ± 0%  +0.59%  (p=0.007 n=19+15)
RegexpMatchHard_32-12       1.54µs ± 2%    1.53µs ± 2%  -0.78%  (p=0.021 n=17+18)
RegexpMatchHard_1K-12       46.0µs ± 1%    45.9µs ± 1%  -0.31%  (p=0.025 n=17+19)
Revcomp-12                   391ms ± 1%     394ms ± 2%  +0.80%  (p=0.000 n=17+19)
Template-12                 59.9ms ± 1%    59.9ms ± 1%    ~     (p=0.428 n=20+19)
TimeParse-12                 304ns ± 1%     312ns ± 0%  +2.88%  (p=0.000 n=20+17)
TimeFormat-12                318ns ± 0%     326ns ± 0%  +2.64%  (p=0.000 n=20+17)

(https://perf.golang.org/search?q=upload:20180928.2)

Change-Id: I336b9bf054113580a24103192904c8c76593e90e
Reviewed-on: https://go-review.googlesource.com/c/138958
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Rick Hudson <rlh@golang.org>
Reviewed-by: Michael Knyszek <mknyszek@google.com>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index f4646db67aac5f99c4a8deed69caeb3ca0d5372f..0215a2c0c2d4fa9e021d907b15aeb382b3b6d9b2 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -2023,6 +2023,18 @@ func gcResetMarkState() {
        }
        unlock(&allglock)
 
+       // Clear page marks. This is just 1MB per 64GB of heap, so the
+       // time here is pretty trivial.
+       lock(&mheap_.lock)
+       arenas := mheap_.allArenas
+       unlock(&mheap_.lock)
+       for _, ai := range arenas {
+               ha := mheap_.arenas[ai.l1()][ai.l2()]
+               for i := range ha.pageMarks {
+                       ha.pageMarks[i] = 0
+               }
+       }
+
        work.bytesMarked = 0
        work.initialHeapLive = atomic.Load64(&memstats.heap_live)
 }

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 28260ab7060bb875f8a19d4161255a55c1ffe7d5..03c64c4b113752f14b2c21b73710d3d30e747029 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -1229,6 +1229,13 @@ func greyobject(obj, base, off uintptr, span *mspan, gcw *gcWork, objIndex uintp
                        return
                }
                mbits.setMarked()
+
+               // Mark span.
+               arena, pageIdx, pageMask := pageIndexOf(span.base())
+               if arena.pageMarks[pageIdx]&pageMask == 0 {
+                       atomic.Or8(&arena.pageMarks[pageIdx], pageMask)
+               }
+
                // If this is a noscan object, fast-track it to black
                // instead of greying it.
                if span.spanclass.noscan() {

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index 12868075d47d58d4628fdc041ff514e16cced41d..d183268b54fd6ff9fe4c54d8c647fbc5592b3050 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -201,6 +201,21 @@ type heapArena struct {
        //
        // Writes are protected by mheap_.lock.
        pageInUse [pagesPerArena / 8]uint8
+
+       // pageMarks is a bitmap that indicates which spans have any
+       // marked objects on them. Like pageInUse, only the bit
+       // corresponding to the first page in each span is used.
+       //
+       // Writes are done atomically during marking. Reads are
+       // non-atomic and lock-free since they only occur during
+       // sweeping (and hence never race with writes).
+       //
+       // This is used to quickly find whole spans that can be freed.
+       //
+       // TODO(austin): It would be nice if this was uint64 for
+       // faster scanning, but we don't have 64-bit atomic bit
+       // operations.
+       pageMarks [pagesPerArena / 8]uint8
 }
 
 // arenaHint is a hint for where to grow the heap arenas. See