runtime: use private futexes on Linux

By default futexes are permitted in shared memory regions, which
requires the kernel to translate the memory address. Since our futexes
are never in shared memory, set FUTEX_PRIVATE_FLAG, which makes futex
operations slightly more efficient.

Change-Id: I2a82365ed27d5cd8d53c5382ebaca1a720a80952
Reviewed-on: https://go-review.googlesource.com/80144
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Reviewed-by: David Crawshaw <crawshaw@golang.org>

diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go
index 2e442192cc818f467967db5bf7f05680eb8f2e1b..7231cf1226a1123acc1a9ea385dd4cbc66c0d797 100644 (file)
--- a/src/runtime/os_linux.go
+++ b/src/runtime/os_linux.go
@@ -24,8 +24,9 @@ func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer,
 // Futexsleep is allowed to wake up spuriously.
 
 const (
-       _FUTEX_WAIT = 0
-       _FUTEX_WAKE = 1
+       _FUTEX_PRIVATE_FLAG = 128
+       _FUTEX_WAIT_PRIVATE = 0 | _FUTEX_PRIVATE_FLAG
+       _FUTEX_WAKE_PRIVATE = 1 | _FUTEX_PRIVATE_FLAG
 )
 
 // Atomically,
@@ -42,7 +43,7 @@ func futexsleep(addr *uint32, val uint32, ns int64) {
        // here, and so can we: as it says a few lines up,
        // spurious wakeups are allowed.
        if ns < 0 {
-               futex(unsafe.Pointer(addr), _FUTEX_WAIT, val, nil, nil, 0)
+               futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, nil, nil, 0)
                return
        }
 
@@ -59,13 +60,13 @@ func futexsleep(addr *uint32, val uint32, ns int64) {
                ts.tv_nsec = 0
                ts.set_sec(int64(timediv(ns, 1000000000, (*int32)(unsafe.Pointer(&ts.tv_nsec)))))
        }
-       futex(unsafe.Pointer(addr), _FUTEX_WAIT, val, unsafe.Pointer(&ts), nil, 0)
+       futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0)
 }
 
 // If any procs are sleeping on addr, wake up at most cnt.
 //go:nosplit
 func futexwakeup(addr *uint32, cnt uint32) {
-       ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE, cnt, nil, nil, 0)
+       ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0)
        if ret >= 0 {
                return
        }