]> Cypherpunks repositories - gostls13.git/commitdiff
runtime: mark pages we return to kernel as NOHUGEPAGE
authorKeith Randall <khr@golang.org>
Mon, 23 Feb 2015 21:58:05 +0000 (13:58 -0800)
committerKeith Randall <khr@golang.org>
Wed, 25 Feb 2015 21:16:18 +0000 (21:16 +0000)
We return memory to the kernel with madvise(..., DONTNEED).
Also mark returned memory with NOHUGEPAGE to keep the kernel from
merging this memory into a huge page, effectively reallocating it.

Only known to be a problem on linux/{386,amd64,amd64p32} at the moment.
It may come up on other os/arch combinations in the future.

Fixes #8832

Change-Id: Ifffc6627a0296926e3f189a8a9b6e4bdb54c79eb
Reviewed-on: https://go-review.googlesource.com/5660
Reviewed-by: Dmitry Vyukov <dvyukov@google.com>
12 files changed:
src/runtime/arch1_386.go
src/runtime/arch1_amd64.go
src/runtime/arch1_amd64p32.go
src/runtime/arch1_arm.go
src/runtime/arch1_ppc64.go
src/runtime/arch1_ppc64le.go
src/runtime/defs_linux_386.go
src/runtime/defs_linux_amd64.go
src/runtime/defs_linux_arm.go
src/runtime/defs_linux_ppc64.go
src/runtime/defs_linux_ppc64le.go
src/runtime/mem_linux.go

index a73e207eddce825ef40d28884d743daa07d975df..b024d7a51f25a85cc3c9ce6faf7bf6e8822287dd 100644 (file)
@@ -12,4 +12,5 @@ const (
        _PhysPageSize     = goos_nacl*65536 + (1-goos_nacl)*4096 // 4k normally; 64k on NaCl
        _PCQuantum        = 1
        _Int64Align       = 4
+       hugePageSize      = 1 << 21
 )
index 199686db6f512a833391e706dc76cb41dbb7830a..932b2b7c555df691d52e541008ac80c4583874d5 100644 (file)
@@ -12,4 +12,5 @@ const (
        _PhysPageSize     = 4096
        _PCQuantum        = 1
        _Int64Align       = 8
+       hugePageSize      = 1 << 21
 )
index 2cee21f0ca5ac0bf83d7244f0f05b93db74f170c..79421e848a8411ff2f1b9624de85ef376eb3441f 100644 (file)
@@ -12,4 +12,5 @@ const (
        _PhysPageSize     = 65536*goos_nacl + 4096*(1-goos_nacl)
        _PCQuantum        = 1
        _Int64Align       = 8
+       hugePageSize      = 1 << 21
 )
index 6662eaeac396a8938aa1daf412eb592abd2f98e7..c3fe4f0cb3f22ef8f915c34a0d74cd5261e06708 100644 (file)
@@ -12,4 +12,5 @@ const (
        _PhysPageSize     = 65536*goos_nacl + 4096*(1-goos_nacl)
        _PCQuantum        = 4
        _Int64Align       = 4
+       hugePageSize      = 0
 )
index 6996d0fac00accfeec0b941447de89ee6930a5f1..ee453c09f255e6b6913ddf4cfb246b43d4c96ef5 100644 (file)
@@ -12,4 +12,5 @@ const (
        _PhysPageSize     = 65536
        _PCQuantum        = 4
        _Int64Align       = 8
+       hugePageSize      = 0
 )
index 7d5dac46a07c6ae851a53c7970c406d570797164..aa028a10f3753092dc12a18c81fe5e563ddf361a 100644 (file)
@@ -12,4 +12,5 @@ const (
        _PhysPageSize     = 65536
        _PCQuantum        = 4
        _Int64Align       = 8
+       hugePageSize      = 0
 )
index f55924b61cbb75657590df47c8b53812fcc00fa6..7cf57c84523fb9cb3a65734f45d59e7120a24e7b 100644 (file)
@@ -17,7 +17,9 @@ const (
        _MAP_PRIVATE = 0x2
        _MAP_FIXED   = 0x10
 
-       _MADV_DONTNEED = 0x4
+       _MADV_DONTNEED   = 0x4
+       _MADV_HUGEPAGE   = 0xe
+       _MADV_NOHUGEPAGE = 0xf
 
        _SA_RESTART  = 0x10000000
        _SA_ONSTACK  = 0x8000000
index a73f475148201ec3899ca4f852be1bdac3400c51..48aeb8020334889555e273b16fe3d689967efdff 100644 (file)
@@ -17,7 +17,9 @@ const (
        _MAP_PRIVATE = 0x2
        _MAP_FIXED   = 0x10
 
-       _MADV_DONTNEED = 0x4
+       _MADV_DONTNEED   = 0x4
+       _MADV_HUGEPAGE   = 0xe
+       _MADV_NOHUGEPAGE = 0xf
 
        _SA_RESTART  = 0x10000000
        _SA_ONSTACK  = 0x8000000
index 3940240898b0ca0ca44d8a01ec81deba99411d97..b68b9642a9c1592c5daf2d3e897132723c7bb2ff 100644 (file)
@@ -6,14 +6,19 @@ const (
        _ENOMEM = 0xc
        _EAGAIN = 0xb
 
-       _PROT_NONE      = 0
-       _PROT_READ      = 0x1
-       _PROT_WRITE     = 0x2
-       _PROT_EXEC      = 0x4
-       _MAP_ANON       = 0x20
-       _MAP_PRIVATE    = 0x2
-       _MAP_FIXED      = 0x10
-       _MADV_DONTNEED  = 0x4
+       _PROT_NONE  = 0
+       _PROT_READ  = 0x1
+       _PROT_WRITE = 0x2
+       _PROT_EXEC  = 0x4
+
+       _MAP_ANON    = 0x20
+       _MAP_PRIVATE = 0x2
+       _MAP_FIXED   = 0x10
+
+       _MADV_DONTNEED   = 0x4
+       _MADV_HUGEPAGE   = 0xe
+       _MADV_NOHUGEPAGE = 0xf
+
        _SA_RESTART     = 0x10000000
        _SA_ONSTACK     = 0x8000000
        _SA_RESTORER    = 0 // unused on ARM
index f90b84874bd758a6901b78bdb009f5c4775f47d6..317a764a707a95672ffda092648d6a5c7d4948a5 100644 (file)
@@ -17,7 +17,9 @@ const (
        _MAP_PRIVATE = 0x2
        _MAP_FIXED   = 0x10
 
-       _MADV_DONTNEED = 0x4
+       _MADV_DONTNEED   = 0x4
+       _MADV_HUGEPAGE   = 0xe
+       _MADV_NOHUGEPAGE = 0xf
 
        _SA_RESTART = 0x10000000
        _SA_ONSTACK = 0x8000000
index f90b84874bd758a6901b78bdb009f5c4775f47d6..317a764a707a95672ffda092648d6a5c7d4948a5 100644 (file)
@@ -17,7 +17,9 @@ const (
        _MAP_PRIVATE = 0x2
        _MAP_FIXED   = 0x10
 
-       _MADV_DONTNEED = 0x4
+       _MADV_DONTNEED   = 0x4
+       _MADV_HUGEPAGE   = 0xe
+       _MADV_NOHUGEPAGE = 0xf
 
        _SA_RESTART = 0x10000000
        _SA_ONSTACK = 0x8000000
index 920fbcf6d6be3d42826deb167edc16c8874286dd..74de11caea29d247fcfa5c220a2d7ace65483717 100644 (file)
@@ -67,10 +67,29 @@ func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer {
 }
 
 func sysUnused(v unsafe.Pointer, n uintptr) {
+       if hugePageSize != 0 && (uintptr(v)%hugePageSize != 0 || n%hugePageSize != 0) {
+               // See issue 8832
+               // Linux kernel bug: https://bugzilla.kernel.org/show_bug.cgi?id=93111
+               // Mark the region as NOHUGEPAGE so the kernel's khugepaged
+               // doesn't undo our DONTNEED request.  khugepaged likes to migrate
+               // regions which are only partially mapped to huge pages, including
+               // regions with some DONTNEED marks.  That needlessly allocates physical
+               // memory for our DONTNEED regions.
+               madvise(v, n, _MADV_NOHUGEPAGE)
+       }
        madvise(v, n, _MADV_DONTNEED)
 }
 
 func sysUsed(v unsafe.Pointer, n uintptr) {
+       if hugePageSize != 0 {
+               // Undo the NOHUGEPAGE marks from sysUnused.  There is no alignment check
+               // around this call as spans may have been merged in the interim.
+               // Note that this might enable huge pages for regions which were
+               // previously disabled.  Unfortunately there is no easy way to detect
+               // what the previous state was, and in any case we probably want huge
+               // pages to back our heap if the kernel can arrange that.
+               madvise(v, n, _MADV_HUGEPAGE)
+       }
 }
 
 func sysFree(v unsafe.Pointer, n uintptr, stat *uint64) {