runtime: add support for VDSO on ppc64x for use in walltime/nanotime

This change adds support for VDSO on ppc64x, making it possible to
avoid a syscall in walltime and nanotime.

BenchmarkClockVDSOAndFallbackPaths/vDSO-192             20000000    66.0 ns/op
BenchmarkClockVDSOAndFallbackPaths/Fallback-192          1000000    1456 ns/op

Change-Id: I3373bd804b6f122961de3ae9d034e6ccf35748e6
Reviewed-on: https://go-review.googlesource.com/131135
Run-TryBot: Lynn Boger <laboger@linux.vnet.ibm.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Lynn Boger <laboger@linux.vnet.ibm.com>

diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go
index ee4a7a95c2aff0a2ae5a2e964a0c08ac749be4cd..e54c1c4dc138f9a92b163c3a7602e702a43c2c14 100644 (file)
--- a/src/runtime/os_linux_novdso.go
+++ b/src/runtime/os_linux_novdso.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build !386,!amd64,!arm,!arm64
+// +build !386,!amd64,!arm,!arm64,!ppc64,!ppc64le
 
 package runtime
 

diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s
index 483cb8ef9aefd72f039fc7e78341f6b927ba8de6..075adf2368439e18c003ffbb79eb89f71280702b 100644 (file)
--- a/src/runtime/sys_linux_ppc64x.s
+++ b/src/runtime/sys_linux_ppc64x.s
@@ -154,21 +154,87 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28
 
 // func walltime() (sec int64, nsec int32)
 TEXT runtime·walltime(SB),NOSPLIT,$16
-       MOVD    $0, R3 // CLOCK_REALTIME
-       MOVD    $0(R1), R4
-       SYSCALL $SYS_clock_gettime
-       MOVD    0(R1), R3       // sec
-       MOVD    8(R1), R5       // nsec
+       MOVD    R1, R15         // R15 is unchanged by C code
+       MOVD    g_m(g), R21     // R21 = m
+
+       MOVD    $0, R3          // CLOCK_REALTIME
+
+       MOVD    runtime·vdsoClockgettimeSym(SB), R12   // Check for VDSO availability
+       CMP     R12, R0
+       BEQ     fallback
+
+       // Set vdsoPC and vdsoSP for SIGPROF traceback.
+       MOVD    LR, R14
+       MOVD    R14, m_vdsoPC(R21)
+       MOVD    R15, m_vdsoSP(R21)
+
+       MOVD    m_curg(R21), R6
+       CMP     g, R6
+       BNE     noswitch
+
+       MOVD    m_g0(R21), R7
+       MOVD    (g_sched+gobuf_sp)(R7), R1      // Set SP to g0 stack
+
+noswitch:
+       SUB     $16, R1                 // Space for results
+       RLDICR  $0, R1, $59, R1         // Align for C code
+       MOVD    R12, CTR
+       MOVD    R1, R4
+       BL      (CTR)                   // Call from VDSO
+       MOVD    $0, R0                  // Restore R0
+       MOVD    R0, m_vdsoSP(R21)       // Clear vdsoSP
+       MOVD    0(R1), R3               // sec
+       MOVD    8(R1), R5               // nsec
+       MOVD    R15, R1                 // Restore SP
+
+finish:
        MOVD    R3, sec+0(FP)
        MOVW    R5, nsec+8(FP)
        RET
 
+       // Syscall fallback
+fallback:
+       ADD     $32, R1, R4
+       SYSCALL $SYS_clock_gettime
+       MOVD    32(R1), R3
+       MOVD    40(R1), R5
+       JMP     finish
+
 TEXT runtime·nanotime(SB),NOSPLIT,$16
-       MOVW    $1, R3 // CLOCK_MONOTONIC
-       MOVD    $0(R1), R4
-       SYSCALL $SYS_clock_gettime
-       MOVD    0(R1), R3       // sec
-       MOVD    8(R1), R5       // nsec
+       MOVD    $1, R3          // CLOCK_MONOTONIC
+
+       MOVD    R1, R15         // R15 is unchanged by C code
+       MOVD    g_m(g), R21     // R21 = m
+
+       MOVD    runtime·vdsoClockgettimeSym(SB), R12   // Check for VDSO availability
+       CMP     R12, R0
+       BEQ     fallback
+
+       // Set vdsoPC and vdsoSP for SIGPROF traceback.
+       MOVD    LR, R14         // R14 is unchanged by C code
+       MOVD    R14, m_vdsoPC(R21)
+       MOVD    R15, m_vdsoSP(R21)
+
+       MOVD    m_curg(R21), R6
+       CMP     g, R6
+       BNE     noswitch
+
+       MOVD    m_g0(R21), R7
+       MOVD    (g_sched+gobuf_sp)(R7), R1      // Set SP to g0 stack
+
+noswitch:
+       SUB     $16, R1                 // Space for results
+       RLDICR  $0, R1, $59, R1         // Align for C code
+       MOVD    R12, CTR
+       MOVD    R1, R4
+       BL      (CTR)                   // Call from VDSO
+       MOVD    $0, R0                  // Restore R0
+       MOVD    $0, m_vdsoSP(R21)       // Clear vdsoSP
+       MOVD    0(R1), R3               // sec
+       MOVD    8(R1), R5               // nsec
+       MOVD    R15, R1                 // Restore SP
+
+finish:
        // sec is in R3, nsec in R5
        // return nsec in R3
        MOVD    $1000000000, R4
@@ -177,6 +243,14 @@ TEXT runtime·nanotime(SB),NOSPLIT,$16
        MOVD    R3, ret+0(FP)
        RET
 
+       // Syscall fallback
+fallback:
+       ADD     $32, R1, R4
+       SYSCALL $SYS_clock_gettime
+       MOVD    32(R1), R3
+       MOVD    48(R1), R5
+       JMP     finish
+
 TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28
        MOVW    how+0(FP), R3
        MOVD    new+8(FP), R4

diff --git a/src/runtime/vdso_elf64.go b/src/runtime/vdso_elf64.go
index 851025006586c22147fe2581f05b39c899d23f53..7c9bd962779077e85294cf9cbd534f7fada64fcc 100644 (file)
--- a/src/runtime/vdso_elf64.go
+++ b/src/runtime/vdso_elf64.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build amd64 arm64
+// +build amd64 arm64 ppc64 ppc64le
 
 package runtime
 

diff --git a/src/runtime/vdso_in_none.go b/src/runtime/vdso_in_none.go
index 34cfac56d124565cb90c0c73b24df86359ff1268..f2d6bb55d9cba177bad023d1a3b3eff1c43cac0c 100644 (file)
--- a/src/runtime/vdso_in_none.go
+++ b/src/runtime/vdso_in_none.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux,!386,!amd64,!arm,!arm64 !linux
+// +build linux,!386,!amd64,!arm,!arm64,!ppc64,!ppc64le !linux
 
 package runtime
 

diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go
index f6a285efd73f26a93d3ac60d588ae4a33f47ca8f..9827874beabcfb34450ec0b7b0a75aec15374048 100644 (file)
--- a/src/runtime/vdso_linux.go
+++ b/src/runtime/vdso_linux.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386 amd64 arm arm64
+// +build 386 amd64 arm arm64 ppc64 ppc64le
 
 package runtime
 
@@ -42,6 +42,8 @@ const (
 
        _STT_FUNC = 2 /* Symbol is a code object */
 
+       _STT_NOTYPE = 0 /* Symbol type is not specified */
+
        _STB_GLOBAL = 1 /* Global symbol */
        _STB_WEAK   = 2 /* Weak symbol */
 
@@ -212,7 +214,8 @@ func vdsoParseSymbols(info *vdsoInfo, version int32) {
                sym := &info.symtab[symIndex]
                typ := _ELF_ST_TYPE(sym.st_info)
                bind := _ELF_ST_BIND(sym.st_info)
-               if typ != _STT_FUNC || bind != _STB_GLOBAL && bind != _STB_WEAK || sym.st_shndx == _SHN_UNDEF {
+               // On ppc64x, VDSO functions are of type _STT_NOTYPE.
+               if typ != _STT_FUNC && typ != _STT_NOTYPE || bind != _STB_GLOBAL && bind != _STB_WEAK || sym.st_shndx == _SHN_UNDEF {
                        return false
                }
                if k.name != gostringnocopy(&info.symstrings[sym.st_name]) {

diff --git a/src/runtime/vdso_linux_ppc64x.go b/src/runtime/vdso_linux_ppc64x.go
new file mode 100644 (file)
index 0000000..f30946e
--- /dev/null
+++ b/src/runtime/vdso_linux_ppc64x.go
@@ -0,0 +1,25 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+// +build ppc64 ppc64le
+
+package runtime
+
+const (
+       // vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+       // See cmd/compile/internal/ppc64/galign.go arch.MAXWIDTH initialization.
+       vdsoArrayMax = 1<<50 - 1
+)
+
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.15", 0x75fcba5}
+
+var vdsoSymbolKeys = []vdsoSymbolKey{
+       {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym},
+}
+
+// initialize with vsyscall fallbacks
+var (
+       vdsoClockgettimeSym uintptr = 0
+)

diff --git a/src/runtime/vdso_linux_test.go b/src/runtime/vdso_linux_test.go
index b5221f90b71e4a6f05208c3b097397d8a6c88308..ad083c61b443f334909ecae45c6d29586929136e 100644 (file)
--- a/src/runtime/vdso_linux_test.go
+++ b/src/runtime/vdso_linux_test.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386 amd64 arm arm64
+// +build 386 amd64 arm arm64 ppc64 ppc64le
 
 package runtime_test