This change adds physHugePageShift which is defined such that
1 << physHugePageShift == physHugePageSize. The purpose of this variable
is to avoid doing expensive divisions in key functions, such as
(*mspan).hugePages.
This change also does a sweep of any place we might do a division or mod
operation with physHugePageSize and turns it into bit shifts and other
bitwise operations.
Finally, this change adds a check to mallocinit which ensures that
physHugePageSize is always a power of two. osinit might choose to ignore
non-powers-of-two for the value and replace it with zero, but mallocinit
will fail if it's not a power of two (or zero). It also derives
physHugePageShift from physHugePageSize.
This change helps improve the performance of most applications because
of how often (*mspan).hugePages is called.
Updates #32828.
Change-Id: I1a6db113d52d563f59ae8fd4f0e130858859e68f
Reviewed-on: https://go-review.googlesource.com/c/go/+/186598
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
var physPageSize uintptr
// physHugePageSize is the size in bytes of the OS's default physical huge
-// page size whose allocation is opaque to the application.
+// page size whose allocation is opaque to the application. It is assumed
+// and verified to be a power of two.
//
// If set, this must be set by the OS init code (typically in osinit) before
// mallocinit. However, setting it at all is optional, and leaving the default
// value is always safe (though potentially less efficient).
-var physHugePageSize uintptr
+//
+// Since physHugePageSize is always assumed to be a power of two,
+// physHugePageShift is defined as physHugePageSize == 1 << physHugePageShift.
+// The purpose of physHugePageShift is to avoid doing divisions in
+// performance critical functions.
+var (
+ physHugePageSize uintptr
+ physHugePageShift uint
+)
// OS memory management abstraction layer
//
print("system page size (", physPageSize, ") must be a power of 2\n")
throw("bad system page size")
}
+ if physHugePageSize&(physHugePageSize-1) != 0 {
+ print("system huge page size (", physHugePageSize, ") must be a power of 2\n")
+ throw("bad system huge page size")
+ }
+ if physHugePageSize != 0 {
+ // Since physHugePageSize is a power of 2, it suffices to increase
+ // physHugePageShift until 1<<physHugePageShift == physHugePageSize.
+ for 1<<physHugePageShift != physHugePageSize {
+ physHugePageShift++
+ }
+ }
// Initialize the heap.
mheap_.init()
// flag on the huge pages containing v and v+n-1, and
// only if those aren't aligned.
var head, tail uintptr
- if uintptr(v)%physHugePageSize != 0 {
+ if uintptr(v)&(physHugePageSize-1) != 0 {
// Compute huge page containing v.
head = uintptr(v) &^ (physHugePageSize - 1)
}
- if (uintptr(v)+n)%physHugePageSize != 0 {
+ if (uintptr(v)+n)&(physHugePageSize-1) != 0 {
// Compute huge page containing v+n-1.
tail = (uintptr(v) + n - 1) &^ (physHugePageSize - 1)
}
if physHugePageSize != 0 {
// Start by computing the amount of free memory we have in huge pages
// in total. Trivially, this is all the huge page work we need to do.
- hugeWork := uint64(mheap_.free.unscavHugePages * physHugePageSize)
+ hugeWork := uint64(mheap_.free.unscavHugePages) << physHugePageShift
// ...but it could turn out that there's more huge work to do than
// total work, so cap it at total work. This might happen for very large
// that there are free chunks of memory larger than a huge page that we don't want
// to scavenge.
if hugeWork >= totalWork {
- hugePages := totalWork / uint64(physHugePageSize)
- hugeWork = hugePages * uint64(physHugePageSize)
+ hugePages := totalWork >> physHugePageShift
+ hugeWork = hugePages << physHugePageShift
}
// Everything that's not huge work is regular work. At this point we
// know huge work so we can calculate how much time that will take
// based on scavengePageRate (which applies to pages of any size).
regularWork = totalWork - hugeWork
- hugeTime = hugeWork / uint64(physHugePageSize) * scavengeHugePagePeriod
+ hugeTime = (hugeWork >> physHugePageShift) * scavengeHugePagePeriod
}
// Finally, we can compute how much time it'll take to do the regular work
// and the total time to do all the work.
end &^= physHugePageSize - 1
}
if start < end {
- return (end - start) / physHugePageSize
+ return (end - start) >> physHugePageShift
}
return 0
}