runtime: track the number of free unscavenged huge pages

This change tracks the number of potential free and unscavenged huge
pages which will be used to inform the rate at which scavenging should
occur.

For #30333.

Change-Id: I47663e5ffb64cac44ffa10db158486783f707479
Reviewed-on: https://go-review.googlesource.com/c/go/+/170860
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go
index 852f37409edb86cb46266b0df8b7392eff230e33..3c3e110f898f0c19e4b98654d80056d22eff70f1 100644 (file)
--- a/src/runtime/export_test.go
+++ b/src/runtime/export_test.go
@@ -36,6 +36,8 @@ var Atoi32 = atoi32
 
 var Nanotime = nanotime
 
+var PhysHugePageSize = physHugePageSize
+
 type LFNode struct {
        Next    uint64
        Pushcnt uintptr
@@ -516,6 +518,26 @@ func MapTombstoneCheck(m map[int]int) {
        }
 }
 
+// UnscavHugePagesSlow returns the value of mheap_.freeHugePages
+// and the number of unscavenged huge pages calculated by
+// scanning the heap.
+func UnscavHugePagesSlow() (uintptr, uintptr) {
+       var base, slow uintptr
+       // Run on the system stack to avoid deadlock from stack growth
+       // trying to acquire the heap lock.
+       systemstack(func() {
+               lock(&mheap_.lock)
+               base = mheap_.free.unscavHugePages
+               for _, s := range mheap_.allspans {
+                       if s.state == mSpanFree && !s.scavenged {
+                               slow += s.hugePages()
+                       }
+               }
+               unlock(&mheap_.lock)
+       })
+       return base, slow
+}
+
 // Span is a safe wrapper around an mspan, whose memory
 // is managed manually.
 type Span struct {

diff --git a/src/runtime/gc_test.go b/src/runtime/gc_test.go
index 51e8ea4d313935924fc0d78bd68ed7717df6bc7d..d55a934519491534ba2a3066bc9d0b4dee4f2d85 100644 (file)
--- a/src/runtime/gc_test.go
+++ b/src/runtime/gc_test.go
@@ -470,6 +470,25 @@ func TestReadMemStats(t *testing.T) {
        }
 }
 
+func TestUnscavHugePages(t *testing.T) {
+       // Allocate 20 MiB and immediately free it a few times to increase
+       // the chance that unscavHugePages isn't zero and that some kind of
+       // accounting had to happen in the runtime.
+       for j := 0; j < 3; j++ {
+               var large [][]byte
+               for i := 0; i < 5; i++ {
+                       large = append(large, make([]byte, runtime.PhysHugePageSize))
+               }
+               runtime.KeepAlive(large)
+               runtime.GC()
+       }
+       base, slow := runtime.UnscavHugePagesSlow()
+       if base != slow {
+               logDiff(t, "unscavHugePages", reflect.ValueOf(base), reflect.ValueOf(slow))
+               t.Fatal("unscavHugePages mismatch")
+       }
+}
+
 func logDiff(t *testing.T, prefix string, got, want reflect.Value) {
        typ := got.Type()
        switch typ.Kind() {

diff --git a/src/runtime/mgclarge.go b/src/runtime/mgclarge.go
index b1e7c23e25e1124db6c298f3266b30dc42fe47c2..857bc6108a8fc356571240c9475a88431aaa6d05 100644 (file)
--- a/src/runtime/mgclarge.go
+++ b/src/runtime/mgclarge.go
@@ -40,7 +40,8 @@ import (
 
 //go:notinheap
 type mTreap struct {
-       treap *treapNode
+       treap           *treapNode
+       unscavHugePages uintptr // number of unscavenged huge pages in the treap
 }
 
 //go:notinheap
@@ -378,6 +379,9 @@ func (root *mTreap) end(mask, match treapIterType) treapIter {
 
 // insert adds span to the large span treap.
 func (root *mTreap) insert(span *mspan) {
+       if !span.scavenged {
+               root.unscavHugePages += span.hugePages()
+       }
        base := span.base()
        var last *treapNode
        pt := &root.treap
@@ -435,6 +439,9 @@ func (root *mTreap) insert(span *mspan) {
 }
 
 func (root *mTreap) removeNode(t *treapNode) {
+       if !t.span.scavenged {
+               root.unscavHugePages -= t.span.hugePages()
+       }
        if t.span.base() != t.key {
                throw("span and treap node base addresses do not match")
        }

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index b14a28fc13e9f77e44474466f2281b6fe3d48688..d033a9d026a7283e12785359e6b111efd55d573b 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -59,7 +59,7 @@ type mheap struct {
        // on the swept stack.
        sweepSpans [2]gcSweepBuf
 
-       // _ uint32 // align uint64 fields on 32-bit for atomics
+       _ uint32 // align uint64 fields on 32-bit for atomics
 
        // Proportional sweep
        //