runtime: use vDSO clock_gettime on linux/riscv64

Speed up nanotime1 and walltime on riscv64 with vDSO, just like the
other vDSO-enabled targets.

name          old time/op  new time/op  delta
Now           1.53µs ± 1%  1.05µs ± 3%  -31.74%  (p=0.000 n=10+10)
NowUnixNano   1.54µs ± 0%  1.05µs ± 2%  -31.91%  (p=0.000 n=9+10)
NowUnixMilli  1.56µs ± 1%  1.06µs ± 1%  -31.73%  (p=0.000 n=10+10)
NowUnixMicro  1.56µs ± 1%  1.05µs ± 1%  -32.80%  (p=0.000 n=9+10)

Change-Id: I69b3fe3cc57685a826c53b366c0992e8048399bb
Reviewed-on: https://go-review.googlesource.com/c/go/+/328509
Trust: Meng Zhuo <mzh@golangcn.org>
Run-TryBot: Meng Zhuo <mzh@golangcn.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Cherry Mui <cherryyz@google.com>

diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go
index 8104f63627f7486b462adbac1c9f8f8cddf48c9c..7e93d2ba830197b31e2a82540755a331e16c2b50 100644 (file)
--- a/src/runtime/os_linux_novdso.go
+++ b/src/runtime/os_linux_novdso.go
@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le
-// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le
+//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64
+// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le,!riscv64
 
 package runtime
 

diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go
index 8117582855db02d55e1b874a4f71630427f8ed30..cab5c879d39b6ab61de0b14757642e751935e995 100644 (file)
--- a/src/runtime/signal_unix.go
+++ b/src/runtime/signal_unix.go
@@ -383,7 +383,7 @@ func preemptM(mp *m) {
 //go:nosplit
 func sigFetchG(c *sigctxt) *g {
        switch GOARCH {
-       case "arm", "arm64", "ppc64", "ppc64le":
+       case "arm", "arm64", "ppc64", "ppc64le", "riscv64":
                if !iscgo && inVDSOPage(c.sigpc()) {
                        // When using cgo, we save the g on TLS and load it from there
                        // in sigtramp. Just use that.

diff --git a/src/runtime/sys_linux_riscv64.s b/src/runtime/sys_linux_riscv64.s
index 54b7a3f61c56d8823878998e0a40c03bbf583ddf..ebcdd56a459072d379931b2e937f034956b4bf2f 100644 (file)
--- a/src/runtime/sys_linux_riscv64.s
+++ b/src/runtime/sys_linux_riscv64.s
@@ -10,6 +10,8 @@
 #include "go_asm.h"
 
 #define AT_FDCWD -100
+#define CLOCK_REALTIME 0
+#define CLOCK_MONOTONIC 1
 
 #define SYS_brk                        214
 #define SYS_clock_gettime      113
@@ -210,8 +212,68 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28
        RET
 
 // func walltime() (sec int64, nsec int32)
-TEXT runtime·walltime(SB),NOSPLIT,$24-12
-       MOV     $0, A0 // CLOCK_REALTIME
+TEXT runtime·walltime(SB),NOSPLIT,$40-12
+       MOV     $CLOCK_REALTIME, A0
+
+       MOV     runtime·vdsoClockgettimeSym(SB), A7
+       BEQZ    A7, fallback
+       MOV     X2, S2 // S2,S3,S4 is unchanged by C code
+       MOV     g_m(g), S3 // S3 = m
+
+       // Save the old values on stack for reentrant
+       MOV     m_vdsoPC(S3), T0
+       MOV     T0, 24(X2)
+       MOV     m_vdsoSP(S3), T0
+       MOV     T0, 32(X2)
+
+       MOV     RA, m_vdsoPC(S3)
+       MOV     $ret-8(FP), T1 // caller's SP
+       MOV     T1, m_vdsoSP(S3)
+
+       MOV     m_curg(S3), T1
+       BNE     g, T1, noswitch
+
+       MOV     m_g0(S3), T1
+       MOV     (g_sched+gobuf_sp)(T1), X2
+
+noswitch:
+       ADDI    $-24, X2 // Space for result
+       ANDI    $~7, X2 // Align for C code
+       MOV     $8(X2), A1
+
+       // Store g on gsignal's stack, see sys_linux_arm64.s for detail
+       MOVBU   runtime·iscgo(SB), S4
+       BNEZ    S4, nosaveg
+       MOV     m_gsignal(S3), S4 // g.m.gsignal
+       BEQZ    S4, nosaveg
+       BEQ     g, S4, nosaveg
+       MOV     (g_stack+stack_lo)(S4), S4 // g.m.gsignal.stack.lo
+       MOV     g, (S4)
+
+       JALR    RA, A7
+
+       MOV     ZERO, (S4)
+       JMP     finish
+
+nosaveg:
+       JALR    RA, A7
+
+finish:
+       MOV     8(X2), T0       // sec
+       MOV     16(X2), T1      // nsec
+
+       MOV     S2, X2  // restore stack
+       MOV     24(X2), A2
+       MOV     A2, m_vdsoPC(S3)
+
+       MOV     32(X2), A3
+       MOV     A3, m_vdsoSP(S3)
+
+       MOV     T0, sec+0(FP)
+       MOVW    T1, nsec+8(FP)
+       RET
+
+fallback:
        MOV     $8(X2), A1
        MOV     $SYS_clock_gettime, A7
        ECALL
@@ -222,13 +284,62 @@ TEXT runtime·walltime(SB),NOSPLIT,$24-12
        RET
 
 // func nanotime1() int64
-TEXT runtime·nanotime1(SB),NOSPLIT,$24-8
-       MOV     $1, A0 // CLOCK_MONOTONIC
+TEXT runtime·nanotime1(SB),NOSPLIT,$40-8
+       MOV     $CLOCK_MONOTONIC, A0
+
+       MOV     runtime·vdsoClockgettimeSym(SB), A7
+       BEQZ    A7, fallback
+
+       MOV     X2, S2 // S2 = RSP, S2 is unchanged by C code
+       MOV     g_m(g), S3 // S3 = m
+       // Save the old values on stack for reentrant
+       MOV     m_vdsoPC(S3), T0
+       MOV     T0, 24(X2)
+       MOV     m_vdsoSP(S3), T0
+       MOV     T0, 32(X2)
+
+       MOV     RA, m_vdsoPC(S3)
+       MOV     $ret-8(FP), T0 // caller's SP
+       MOV     T0, m_vdsoSP(S3)
+
+       MOV     m_curg(S3), T1
+       BNE     g, T1, noswitch
+
+       MOV     m_g0(S3), T1
+       MOV     (g_sched+gobuf_sp)(T1), X2
+
+noswitch:
+       ADDI    $-24, X2 // Space for result
+       ANDI    $~7, X2 // Align for C code
        MOV     $8(X2), A1
-       MOV     $SYS_clock_gettime, A7
-       ECALL
+
+       // Store g on gsignal's stack, see sys_linux_arm64.s for detail
+       MOVBU   runtime·iscgo(SB), S4
+       BNEZ    S4, nosaveg
+       MOV     m_gsignal(S3), S4 // g.m.gsignal
+       BEQZ    S4, nosaveg
+       BEQ     g, S4, nosaveg
+       MOV     (g_stack+stack_lo)(S4), S4 // g.m.gsignal.stack.lo
+       MOV     g, (S4)
+
+       JALR    RA, A7
+
+       MOV     ZERO, (S4)
+       JMP     finish
+
+nosaveg:
+       JALR    RA, A7
+
+finish:
        MOV     8(X2), T0       // sec
        MOV     16(X2), T1      // nsec
+       // restore stack
+       MOV     S2, X2
+       MOV     24(X2), T2
+       MOV     T2, m_vdsoPC(S3)
+
+       MOV     32(X2), T2
+       MOV     T2, m_vdsoSP(S3)
        // sec is in T0, nsec in T1
        // return nsec in T0
        MOV     $1000000000, T2
@@ -237,6 +348,18 @@ TEXT runtime·nanotime1(SB),NOSPLIT,$24-8
        MOV     T0, ret+0(FP)
        RET
 
+fallback:
+       MOV     $8(X2), A1
+       MOV     $SYS_clock_gettime, A7
+       ECALL
+       MOV     8(X2), T0       // sec
+       MOV     16(X2), T1      // nsec
+       MOV     $1000000000, T2
+       MUL     T2, T0
+       ADD     T1, T0
+       MOV     T0, ret+0(FP)
+       RET
+
 // func rtsigprocmask(how int32, new, old *sigset, size int32)
 TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28
        MOVW    how+0(FP), A0

diff --git a/src/runtime/vdso_elf64.go b/src/runtime/vdso_elf64.go
index 9923bd46971e5f2bc971c3cd448e5bfc7ccc19c1..aecc84dcfe5746f7d1e26b56ae147e30f1ce8209 100644 (file)
--- a/src/runtime/vdso_elf64.go
+++ b/src/runtime/vdso_elf64.go
@@ -2,9 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le)
+//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64)
 // +build linux
-// +build amd64 arm64 mips64 mips64le ppc64 ppc64le
+// +build amd64 arm64 mips64 mips64le ppc64 ppc64le riscv64
 
 package runtime
 

diff --git a/src/runtime/vdso_in_none.go b/src/runtime/vdso_in_none.go
index c66fbf821669fd6425e3739c07b1df0b9643c0a3..3e98b08b1519e098d65cc256d5cc451c3fcb2781 100644 (file)
--- a/src/runtime/vdso_in_none.go
+++ b/src/runtime/vdso_in_none.go
@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le) || !linux
-// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le !linux
+//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64) || !linux
+// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le,!riscv64 !linux
 
 package runtime
 

diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go
index ae211f96b1b058c2766c923b3001b395682d9f57..20c8db78bee88adaa2c3d3ce3a8cfcd3e9135b23 100644 (file)
--- a/src/runtime/vdso_linux.go
+++ b/src/runtime/vdso_linux.go
@@ -2,9 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le)
+//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64)
 // +build linux
-// +build 386 amd64 arm arm64 mips64 mips64le ppc64 ppc64le
+// +build 386 amd64 arm arm64 mips64 mips64le ppc64 ppc64le riscv64
 
 package runtime
 

diff --git a/src/runtime/vdso_linux_riscv64.go b/src/runtime/vdso_linux_riscv64.go
new file mode 100644 (file)
index 0000000..f427124
--- /dev/null
+++ b/src/runtime/vdso_linux_riscv64.go
@@ -0,0 +1,21 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+const (
+       // vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+       // See cmd/compile/internal/riscv64/galign.go arch.MAXWIDTH initialization.
+       vdsoArrayMax = 1<<50 - 1
+)
+
+// key and version at man 7 vdso : riscv
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_4.15", 0xae77f75}
+
+var vdsoSymbolKeys = []vdsoSymbolKey{
+       {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
+}
+
+// initialize to fall back to syscall
+var vdsoClockgettimeSym uintptr = 0