runtime: ensure free and unscavenged spans may be backed by huge pages

This change adds a new sysHugePage function to provide the equivalent of
Linux's madvise(MADV_HUGEPAGE) support to the runtime. It then uses
sysHugePage to mark a newly-coalesced free span as backable by huge
pages to make the freeHugePages approximation a bit more accurate.

The problem being solved here is that if a large free span is composed
of many small spans which were coalesced together, then there's a chance
that they have had madvise(MADV_NOHUGEPAGE) called on them at some point,
which makes freeHugePages less accurate.

For #30333.

Change-Id: Idd4b02567619fc8d45647d9abd18da42f96f0522
Reviewed-on: https://go-review.googlesource.com/c/go/+/173338
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/mem_aix.go b/src/runtime/mem_aix.go
index 660861a9f1fa50030ae342c7041cd3998bf4a0d0..eeebfa73adcc86b36f82f1e88925c447f0c26aa7 100644 (file)
--- a/src/runtime/mem_aix.go
+++ b/src/runtime/mem_aix.go
@@ -35,6 +35,9 @@ func sysUnused(v unsafe.Pointer, n uintptr) {
 func sysUsed(v unsafe.Pointer, n uintptr) {
 }
 
+func sysHugePage(v unsafe.Pointer, n uintptr) {
+}
+
 // Don't split the stack as this function may be invoked without a valid G,
 // which prevents us from allocating more stack.
 //go:nosplit

diff --git a/src/runtime/mem_bsd.go b/src/runtime/mem_bsd.go
index 3977e4ae9eff96167e006ac41fbb40f7bd2e07ff..08a2391610b0bd4e78e0a8f6183aaa06ae5ae191 100644 (file)
--- a/src/runtime/mem_bsd.go
+++ b/src/runtime/mem_bsd.go
@@ -29,6 +29,9 @@ func sysUnused(v unsafe.Pointer, n uintptr) {
 func sysUsed(v unsafe.Pointer, n uintptr) {
 }
 
+func sysHugePage(v unsafe.Pointer, n uintptr) {
+}
+
 // Don't split the stack as this function may be invoked without a valid G,
 // which prevents us from allocating more stack.
 //go:nosplit

diff --git a/src/runtime/mem_darwin.go b/src/runtime/mem_darwin.go
index fd5bba9aa7392b4b7a67234a201b3e1f2855be3b..86d9fca85ac2e0e07bc58a061365167109bba0d7 100644 (file)
--- a/src/runtime/mem_darwin.go
+++ b/src/runtime/mem_darwin.go
@@ -33,6 +33,9 @@ func sysUsed(v unsafe.Pointer, n uintptr) {
        madvise(v, n, _MADV_FREE_REUSE)
 }
 
+func sysHugePage(v unsafe.Pointer, n uintptr) {
+}
+
 // Don't split the stack as this function may be invoked without a valid G,
 // which prevents us from allocating more stack.
 //go:nosplit

diff --git a/src/runtime/mem_js.go b/src/runtime/mem_js.go
index 7da4beda2a75319f45544bb56c87f2ffe9d6e2c2..de90f5305f2bec6047f31d485d1a6678013fcf5b 100644 (file)
--- a/src/runtime/mem_js.go
+++ b/src/runtime/mem_js.go
@@ -26,6 +26,9 @@ func sysUnused(v unsafe.Pointer, n uintptr) {
 func sysUsed(v unsafe.Pointer, n uintptr) {
 }
 
+func sysHugePage(v unsafe.Pointer, n uintptr) {
+}
+
 // Don't split the stack as this function may be invoked without a valid G,
 // which prevents us from allocating more stack.
 //go:nosplit

diff --git a/src/runtime/mem_linux.go b/src/runtime/mem_linux.go
index bf399227a1215dde54dffcfa246d480ba48a4e15..cda2c78eaf932899f5d67270e65c828c3d649261 100644 (file)
--- a/src/runtime/mem_linux.go
+++ b/src/runtime/mem_linux.go
@@ -117,16 +117,19 @@ func sysUnused(v unsafe.Pointer, n uintptr) {
 }
 
 func sysUsed(v unsafe.Pointer, n uintptr) {
-       if physHugePageSize != 0 {
-               // Partially undo the NOHUGEPAGE marks from sysUnused
-               // for whole huge pages between v and v+n. This may
-               // leave huge pages off at the end points v and v+n
-               // even though allocations may cover these entire huge
-               // pages. We could detect this and undo NOHUGEPAGE on
-               // the end points as well, but it's probably not worth
-               // the cost because when neighboring allocations are
-               // freed sysUnused will just set NOHUGEPAGE again.
+       // Partially undo the NOHUGEPAGE marks from sysUnused
+       // for whole huge pages between v and v+n. This may
+       // leave huge pages off at the end points v and v+n
+       // even though allocations may cover these entire huge
+       // pages. We could detect this and undo NOHUGEPAGE on
+       // the end points as well, but it's probably not worth
+       // the cost because when neighboring allocations are
+       // freed sysUnused will just set NOHUGEPAGE again.
+       sysHugePage(v, n)
+}
 
+func sysHugePage(v unsafe.Pointer, n uintptr) {
+       if physHugePageSize != 0 {
                // Round v up to a huge page boundary.
                beg := (uintptr(v) + (physHugePageSize - 1)) &^ (physHugePageSize - 1)
                // Round v+n down to a huge page boundary.

diff --git a/src/runtime/mem_plan9.go b/src/runtime/mem_plan9.go
index 2359f138bc208d14f0ea095a0e72788303645d0a..688cdd31ca3d33cf0dad1995dbf1328fd18facc7 100644 (file)
--- a/src/runtime/mem_plan9.go
+++ b/src/runtime/mem_plan9.go
@@ -173,6 +173,9 @@ func sysUnused(v unsafe.Pointer, n uintptr) {
 func sysUsed(v unsafe.Pointer, n uintptr) {
 }
 
+func sysHugePage(v unsafe.Pointer, n uintptr) {
+}
+
 func sysMap(v unsafe.Pointer, n uintptr, sysStat *uint64) {
        // sysReserve has already allocated all heap memory,
        // but has not adjusted stats.

diff --git a/src/runtime/mem_windows.go b/src/runtime/mem_windows.go
index fc52ec59a05e0f082d207649901dc48e6793b7ed..f752136706c882a955f012d0eab5152026d80473 100644 (file)
--- a/src/runtime/mem_windows.go
+++ b/src/runtime/mem_windows.go
@@ -81,6 +81,9 @@ func sysUsed(v unsafe.Pointer, n uintptr) {
        }
 }
 
+func sysHugePage(v unsafe.Pointer, n uintptr) {
+}
+
 // Don't split the stack as this function may be invoked without a valid G,
 // which prevents us from allocating more stack.
 //go:nosplit

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index 1aea52966e28742825dd5e4423e9aaba1a231b2d..60220874792012dfc12cc87c28d7e683b029be8d 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -502,6 +502,8 @@ func (h *mheap) coalesce(s *mspan) {
                h.free.insert(other)
        }
 
+       hpBefore := s.hugePages()
+
        // Coalesce with earlier, later spans.
        if before := spanOf(s.base() - 1); before != nil && before.state == mSpanFree {
                if s.scavenged == before.scavenged {
@@ -519,6 +521,18 @@ func (h *mheap) coalesce(s *mspan) {
                        realign(s, after, after)
                }
        }
+
+       if !s.scavenged && s.hugePages() > hpBefore {
+               // If s has grown such that it now may contain more huge pages than it
+               // did before, then mark the whole region as huge-page-backable.
+               //
+               // Otherwise, on systems where we break up huge pages (like Linux)
+               // s may not be backed by huge pages because it could be made up of
+               // pieces which are broken up in the underlying VMA. The primary issue
+               // with this is that it can lead to a poor estimate of the amount of
+               // free memory backed by huge pages for determining the scavenging rate.
+               sysHugePage(unsafe.Pointer(s.base()), s.npages*pageSize)
+       }
 }
 
 // hugePages returns the number of aligned physical huge pages in the memory