runtime: optimize permission changes with mprotect

On Linux, both mprotect() and mmap() acquire the mmap_lock (in writer mode),
posing scalability challenges.

The mmap_lock (formerly called mmap_sem) is a reader/writer lock that controls
access to a process's address space; before making changes there (mapping in a
new range, for example), the kernel must acquire that lock.

Page-fault handling must also acquire mmap_lock (in reader mode) to ensure that
the address space doesn't change in surprising ways while a fault is being resolved.

A process can have a large address space and many threads running (and incurring
page faults) concurrently, turning mmap_lock into a significant bottleneck.

While both mmap() and mprotect() are protected by the mmap_lock, the shorter
duration of mprotect system call, due to their simpler nature, results in a reduced
locking time for the mmap_lock.

Change-Id: I7f929544904e31eab34d0d8a9e368abe4de64637
GitHub-Last-Rev: 6f27a216b4fb789181d00316561b44358a118b19
GitHub-Pull-Request: golang/go#65038
Reviewed-on: https://go-review.googlesource.com/c/go/+/554935
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Mauri de Souza Meneguzzo <mauri870@gmail.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Pratt <mpratt@google.com>

diff --git a/src/internal/runtime/syscall/defs_linux_386.go b/src/internal/runtime/syscall/defs_linux_386.go
index 613dc77d59d7dac2beb501992c7e613f8933fed8..68e687fb14b71b607279b4320faa6893520819e7 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_386.go
+++ b/src/internal/runtime/syscall/defs_linux_386.go
@@ -6,6 +6,7 @@ package syscall
 
 const (
        SYS_FCNTL         = 55
+       SYS_MPROTECT      = 125
        SYS_EPOLL_CTL     = 255
        SYS_EPOLL_PWAIT   = 319
        SYS_EPOLL_CREATE1 = 329

diff --git a/src/internal/runtime/syscall/defs_linux_amd64.go b/src/internal/runtime/syscall/defs_linux_amd64.go
index 2ba31288139e469bd410685c11e26dbaba3eb5e5..ec480f5817e03177a62e745bbc3712b5f6d79dbf 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_amd64.go
+++ b/src/internal/runtime/syscall/defs_linux_amd64.go
@@ -5,6 +5,7 @@
 package syscall
 
 const (
+       SYS_MPROTECT      = 10
        SYS_FCNTL         = 72
        SYS_EPOLL_CTL     = 233
        SYS_EPOLL_PWAIT   = 281

diff --git a/src/internal/runtime/syscall/defs_linux_arm.go b/src/internal/runtime/syscall/defs_linux_arm.go
index af3e0510b19b41055cd6ef7000c7a969f532a9a8..c5d1503012304c3076997e7501469f236932179e 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_arm.go
+++ b/src/internal/runtime/syscall/defs_linux_arm.go
@@ -6,6 +6,7 @@ package syscall
 
 const (
        SYS_FCNTL         = 55
+       SYS_MPROTECT      = 125
        SYS_EPOLL_CTL     = 251
        SYS_EPOLL_PWAIT   = 346
        SYS_EPOLL_CREATE1 = 357

diff --git a/src/internal/runtime/syscall/defs_linux_arm64.go b/src/internal/runtime/syscall/defs_linux_arm64.go
index c924f6211a32e275fcac68b5e6b73f37e8c2bfef..f743fe31a58d1a5413ddaf37559099a90cbb796d 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_arm64.go
+++ b/src/internal/runtime/syscall/defs_linux_arm64.go
@@ -9,6 +9,7 @@ const (
        SYS_EPOLL_CTL     = 21
        SYS_EPOLL_PWAIT   = 22
        SYS_FCNTL         = 25
+       SYS_MPROTECT      = 226
        SYS_EPOLL_PWAIT2  = 441
        SYS_EVENTFD2      = 19
 

diff --git a/src/internal/runtime/syscall/defs_linux_loong64.go b/src/internal/runtime/syscall/defs_linux_loong64.go
index c1a5649a4210fbfa0daf33949ffaa81439d76249..82218d15099aa5d35e7882ae6c0b8ce17547ac22 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_loong64.go
+++ b/src/internal/runtime/syscall/defs_linux_loong64.go
@@ -9,6 +9,7 @@ const (
        SYS_EPOLL_CTL     = 21
        SYS_EPOLL_PWAIT   = 22
        SYS_FCNTL         = 25
+       SYS_MPROTECT      = 226
        SYS_EPOLL_PWAIT2  = 441
        SYS_EVENTFD2      = 19
 

diff --git a/src/internal/runtime/syscall/defs_linux_mips64x.go b/src/internal/runtime/syscall/defs_linux_mips64x.go
index 07c0aba5398c1f40a630f48d9b319a08d9681e7b..4e0fd1f5d1b10a62caa3ecae47389f26e54b5cff 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_mips64x.go
+++ b/src/internal/runtime/syscall/defs_linux_mips64x.go
@@ -7,6 +7,7 @@
 package syscall
 
 const (
+       SYS_MPROTECT      = 5010
        SYS_FCNTL         = 5070
        SYS_EPOLL_CTL     = 5208
        SYS_EPOLL_PWAIT   = 5272

diff --git a/src/internal/runtime/syscall/defs_linux_mipsx.go b/src/internal/runtime/syscall/defs_linux_mipsx.go
index a1bb5d720a58cdac7b04ea8f9ffaa67f50452336..b87a355093e4894dadfa21412f71e646b7fe889c 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_mipsx.go
+++ b/src/internal/runtime/syscall/defs_linux_mipsx.go
@@ -8,6 +8,7 @@ package syscall
 
 const (
        SYS_FCNTL         = 4055
+       SYS_MPROTECT      = 4125
        SYS_EPOLL_CTL     = 4249
        SYS_EPOLL_PWAIT   = 4313
        SYS_EPOLL_CREATE1 = 4326

diff --git a/src/internal/runtime/syscall/defs_linux_ppc64x.go b/src/internal/runtime/syscall/defs_linux_ppc64x.go
index 78558b360fadd18c7b390b6d841bdb22b97d19f8..8235edd795fa85a88823eb71169ddfc387b1cf54 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_ppc64x.go
+++ b/src/internal/runtime/syscall/defs_linux_ppc64x.go
@@ -8,6 +8,7 @@ package syscall
 
 const (
        SYS_FCNTL         = 55
+       SYS_MPROTECT      = 125
        SYS_EPOLL_CTL     = 237
        SYS_EPOLL_PWAIT   = 303
        SYS_EPOLL_CREATE1 = 315

diff --git a/src/internal/runtime/syscall/defs_linux_riscv64.go b/src/internal/runtime/syscall/defs_linux_riscv64.go
index c1a5649a4210fbfa0daf33949ffaa81439d76249..82218d15099aa5d35e7882ae6c0b8ce17547ac22 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_riscv64.go
+++ b/src/internal/runtime/syscall/defs_linux_riscv64.go
@@ -9,6 +9,7 @@ const (
        SYS_EPOLL_CTL     = 21
        SYS_EPOLL_PWAIT   = 22
        SYS_FCNTL         = 25
+       SYS_MPROTECT      = 226
        SYS_EPOLL_PWAIT2  = 441
        SYS_EVENTFD2      = 19
 

diff --git a/src/internal/runtime/syscall/defs_linux_s390x.go b/src/internal/runtime/syscall/defs_linux_s390x.go
index b539b2d22a99d70bfa1cff81b963b15d8a602d28..08073c01f091b4177d6f16103d5796e81bd823ad 100644 (file)
--- a/src/internal/runtime/syscall/defs_linux_s390x.go
+++ b/src/internal/runtime/syscall/defs_linux_s390x.go
@@ -6,6 +6,7 @@ package syscall
 
 const (
        SYS_FCNTL         = 55
+       SYS_MPROTECT      = 125
        SYS_EPOLL_CTL     = 250
        SYS_EPOLL_PWAIT   = 312
        SYS_EPOLL_CREATE1 = 327

diff --git a/src/runtime/mem_linux.go b/src/runtime/mem_linux.go
index d63c38c209d881ffcd0b549e0146523dba009c68..15a406d97af22bce3842c7250b616a2b8522e6e5 100644 (file)
--- a/src/runtime/mem_linux.go
+++ b/src/runtime/mem_linux.go
@@ -150,7 +150,8 @@ func sysFreeOS(v unsafe.Pointer, n uintptr) {
 }
 
 func sysFaultOS(v unsafe.Pointer, n uintptr) {
-       mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0)
+       mprotect(v, n, _PROT_NONE)
+       madvise(v, n, _MADV_DONTNEED)
 }
 
 func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer {

diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go
index ddacaa585c5f536831e9f041b01b27fffb2170ca..c4f503c8c92fbfc108bfa8c6f94781e0f5f62f2e 100644 (file)
--- a/src/runtime/os_linux.go
+++ b/src/runtime/os_linux.go
@@ -891,3 +891,9 @@ func (c *sigctxt) sigFromUser() bool {
        code := int32(c.sigcode())
        return code == _SI_USER || code == _SI_TKILL
 }
+
+//go:nosplit
+func mprotect(addr unsafe.Pointer, n uintptr, prot int32) (ret int32, errno int32) {
+       r, _, err := syscall.Syscall6(syscall.SYS_MPROTECT, uintptr(addr), n, uintptr(prot), 0, 0, 0)
+       return int32(r), int32(err)
+}