runtime: use vDSO clock_gettime on linux/mips64x

Speed up nanotime1 and walltime1 on MIPS64 with vDSO, just like the
other vDSO-enabled targets.

Benchmark numbers on Loongson 3A3000 (GOARCH=mips64le, 1.4GHz) against
current master:

benchmark                old ns/op     new ns/op     delta
BenchmarkNow             868           293           -66.24%
BenchmarkNowUnixNano     851           296           -65.22%

Performance hit on fallback case, tested by using a wrong vDSO symbol name:

benchmark                old ns/op     new ns/op     delta
BenchmarkNow             868           889           +2.42%
BenchmarkNowUnixNano     851           893           +4.94%

Change-Id: Ibfb48893cd060536359863ffee7624c00def646b
GitHub-Last-Rev: 03a58ac2e4e036a4f61227cfd013082871e92863
GitHub-Pull-Request: golang/go#35181
Reviewed-on: https://go-review.googlesource.com/c/go/+/203578
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go
index e54c1c4dc138f9a92b163c3a7602e702a43c2c14..155f415e71cadea9568dcf41e6d7883c670f318c 100644 (file)
--- a/src/runtime/os_linux_novdso.go
+++ b/src/runtime/os_linux_novdso.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build !386,!amd64,!arm,!arm64,!ppc64,!ppc64le
+// +build !386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le
 
 package runtime
 

diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s
index e4d02a39533a09ae9cd7ec9543ceef603e91058d..723cfe43d90c17e875b85f3fea63876335903655 100644 (file)
--- a/src/runtime/sys_linux_mips64x.s
+++ b/src/runtime/sys_linux_mips64x.s
@@ -211,23 +211,88 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28
 
 // func walltime1() (sec int64, nsec int32)
 TEXT runtime·walltime1(SB),NOSPLIT,$16
+       MOVV    R29, R16        // R16 is unchanged by C code
+       MOVV    R29, R1
+
+       MOVV    g_m(g), R17     // R17 = m
+
+       // Set vdsoPC and vdsoSP for SIGPROF traceback.
+       MOVV    R31, m_vdsoPC(R17)
+       MOVV    R29, m_vdsoSP(R17)
+
+       MOVV    m_curg(R17), R4
+       MOVV    g, R5
+       BNE     R4, R5, noswitch
+
+       MOVV    m_g0(R17), R4
+       MOVV    (g_sched+gobuf_sp)(R4), R1      // Set SP to g0 stack
+
+noswitch:
+       SUBV    $16, R1
+       AND     $~15, R1        // Align for C code
+       MOVV    R1, R29
+
        MOVW    $0, R4 // CLOCK_REALTIME
        MOVV    $0(R29), R5
-       MOVV    $SYS_clock_gettime, R2
-       SYSCALL
+
+       MOVV    runtime·vdsoClockgettimeSym(SB), R25
+       BEQ     R25, fallback
+
+       JAL     (R25)
+
+finish:
        MOVV    0(R29), R3      // sec
        MOVV    8(R29), R5      // nsec
+
+       MOVV    R16, R29        // restore SP
+       MOVV    R0, m_vdsoSP(R17)       // clear vdsoSP
+
        MOVV    R3, sec+0(FP)
        MOVW    R5, nsec+8(FP)
        RET
 
+fallback:
+       MOVV    $SYS_clock_gettime, R2
+       SYSCALL
+       JMP finish
+
 TEXT runtime·nanotime1(SB),NOSPLIT,$16
+       MOVV    R29, R16        // R16 is unchanged by C code
+       MOVV    R29, R1
+
+       MOVV    g_m(g), R17     // R17 = m
+
+       // Set vdsoPC and vdsoSP for SIGPROF traceback.
+       MOVV    R31, m_vdsoPC(R17)
+       MOVV    R29, m_vdsoSP(R17)
+
+       MOVV    m_curg(R17), R4
+       MOVV    g, R5
+       BNE     R4, R5, noswitch
+
+       MOVV    m_g0(R17), R4
+       MOVV    (g_sched+gobuf_sp)(R4), R1      // Set SP to g0 stack
+
+noswitch:
+       SUBV    $16, R1
+       AND     $~15, R1        // Align for C code
+       MOVV    R1, R29
+
        MOVW    $1, R4 // CLOCK_MONOTONIC
        MOVV    $0(R29), R5
-       MOVV    $SYS_clock_gettime, R2
-       SYSCALL
+
+       MOVV    runtime·vdsoClockgettimeSym(SB), R25
+       BEQ     R25, fallback
+
+       JAL     (R25)
+
+finish:
        MOVV    0(R29), R3      // sec
        MOVV    8(R29), R5      // nsec
+
+       MOVV    R16, R29        // restore SP
+       MOVV    R0, m_vdsoSP(R17)       // clear vdsoSP
+
        // sec is in R3, nsec in R5
        // return nsec in R3
        MOVV    $1000000000, R4
@@ -237,6 +302,11 @@ TEXT runtime·nanotime1(SB),NOSPLIT,$16
        MOVV    R3, ret+0(FP)
        RET
 
+fallback:
+       MOVV    $SYS_clock_gettime, R2
+       SYSCALL
+       JMP     finish
+
 TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28
        MOVW    how+0(FP), R4
        MOVV    new+8(FP), R5

diff --git a/src/runtime/vdso_elf64.go b/src/runtime/vdso_elf64.go
index 7c9bd962779077e85294cf9cbd534f7fada64fcc..6ded9d621a67e83cb783f8fc5cb96db1bb18b3a6 100644 (file)
--- a/src/runtime/vdso_elf64.go
+++ b/src/runtime/vdso_elf64.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build amd64 arm64 ppc64 ppc64le
+// +build amd64 arm64 mips64 mips64le ppc64 ppc64le
 
 package runtime
 

diff --git a/src/runtime/vdso_in_none.go b/src/runtime/vdso_in_none.go
index f2d6bb55d9cba177bad023d1a3b3eff1c43cac0c..7f4019c0d62689542ea497639d777ddaee8b1202 100644 (file)
--- a/src/runtime/vdso_in_none.go
+++ b/src/runtime/vdso_in_none.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux,!386,!amd64,!arm,!arm64,!ppc64,!ppc64le !linux
+// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le !linux
 
 package runtime
 

diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go
index 8518276867e3949f69926ae3fa6e14b644552965..6e2942498dde1778f431ce80e74633b9caa64f89 100644 (file)
--- a/src/runtime/vdso_linux.go
+++ b/src/runtime/vdso_linux.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386 amd64 arm arm64 ppc64 ppc64le
+// +build 386 amd64 arm arm64 mips64 mips64le ppc64 ppc64le
 
 package runtime
 

diff --git a/src/runtime/vdso_linux_mips64x.go b/src/runtime/vdso_linux_mips64x.go
new file mode 100644 (file)
index 0000000..3a0f947
--- /dev/null
+++ b/src/runtime/vdso_linux_mips64x.go
@@ -0,0 +1,28 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+// +build mips64 mips64le
+
+package runtime
+
+const (
+       // vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+       // See cmd/compile/internal/mips64/galign.go arch.MAXWIDTH initialization.
+       vdsoArrayMax = 1<<50 - 1
+)
+
+// see man 7 vdso : mips
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
+
+// The symbol name is not __kernel_clock_gettime as suggested by the manpage;
+// according to Linux source code it should be __vdso_clock_gettime instead.
+var vdsoSymbolKeys = []vdsoSymbolKey{
+       {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
+}
+
+// initialize to fall back to syscall
+var (
+       vdsoClockgettimeSym uintptr = 0
+)