From: Meng Zhuo Date: Thu, 26 Aug 2021 01:32:38 +0000 (+0800) Subject: runtime: use vDSO clock_gettime on linux/riscv64 X-Git-Tag: go1.18beta1~1521 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=1bd35fab05;p=gostls13.git runtime: use vDSO clock_gettime on linux/riscv64 Speed up nanotime1 and walltime on riscv64 with vDSO, just like the other vDSO-enabled targets. name old time/op new time/op delta Now 1.53µs ± 1% 1.05µs ± 3% -31.74% (p=0.000 n=10+10) NowUnixNano 1.54µs ± 0% 1.05µs ± 2% -31.91% (p=0.000 n=9+10) NowUnixMilli 1.56µs ± 1% 1.06µs ± 1% -31.73% (p=0.000 n=10+10) NowUnixMicro 1.56µs ± 1% 1.05µs ± 1% -32.80% (p=0.000 n=9+10) Change-Id: I69b3fe3cc57685a826c53b366c0992e8048399bb Reviewed-on: https://go-review.googlesource.com/c/go/+/328509 Trust: Meng Zhuo Run-TryBot: Meng Zhuo TryBot-Result: Go Bot Reviewed-by: Cherry Mui --- diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go index 8104f63627..7e93d2ba83 100644 --- a/src/runtime/os_linux_novdso.go +++ b/src/runtime/os_linux_novdso.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le -// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le +//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64 +// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le,!riscv64 package runtime diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index 8117582855..cab5c879d3 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -383,7 +383,7 @@ func preemptM(mp *m) { //go:nosplit func sigFetchG(c *sigctxt) *g { switch GOARCH { - case "arm", "arm64", "ppc64", "ppc64le": + case "arm", "arm64", "ppc64", "ppc64le", "riscv64": if !iscgo && inVDSOPage(c.sigpc()) { // When using cgo, we save the g on TLS and load it from there // in sigtramp. Just use that. diff --git a/src/runtime/sys_linux_riscv64.s b/src/runtime/sys_linux_riscv64.s index 54b7a3f61c..ebcdd56a45 100644 --- a/src/runtime/sys_linux_riscv64.s +++ b/src/runtime/sys_linux_riscv64.s @@ -10,6 +10,8 @@ #include "go_asm.h" #define AT_FDCWD -100 +#define CLOCK_REALTIME 0 +#define CLOCK_MONOTONIC 1 #define SYS_brk 214 #define SYS_clock_gettime 113 @@ -210,8 +212,68 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28 RET // func walltime() (sec int64, nsec int32) -TEXT runtime·walltime(SB),NOSPLIT,$24-12 - MOV $0, A0 // CLOCK_REALTIME +TEXT runtime·walltime(SB),NOSPLIT,$40-12 + MOV $CLOCK_REALTIME, A0 + + MOV runtime·vdsoClockgettimeSym(SB), A7 + BEQZ A7, fallback + MOV X2, S2 // S2,S3,S4 is unchanged by C code + MOV g_m(g), S3 // S3 = m + + // Save the old values on stack for reentrant + MOV m_vdsoPC(S3), T0 + MOV T0, 24(X2) + MOV m_vdsoSP(S3), T0 + MOV T0, 32(X2) + + MOV RA, m_vdsoPC(S3) + MOV $ret-8(FP), T1 // caller's SP + MOV T1, m_vdsoSP(S3) + + MOV m_curg(S3), T1 + BNE g, T1, noswitch + + MOV m_g0(S3), T1 + MOV (g_sched+gobuf_sp)(T1), X2 + +noswitch: + ADDI $-24, X2 // Space for result + ANDI $~7, X2 // Align for C code + MOV $8(X2), A1 + + // Store g on gsignal's stack, see sys_linux_arm64.s for detail + MOVBU runtime·iscgo(SB), S4 + BNEZ S4, nosaveg + MOV m_gsignal(S3), S4 // g.m.gsignal + BEQZ S4, nosaveg + BEQ g, S4, nosaveg + MOV (g_stack+stack_lo)(S4), S4 // g.m.gsignal.stack.lo + MOV g, (S4) + + JALR RA, A7 + + MOV ZERO, (S4) + JMP finish + +nosaveg: + JALR RA, A7 + +finish: + MOV 8(X2), T0 // sec + MOV 16(X2), T1 // nsec + + MOV S2, X2 // restore stack + MOV 24(X2), A2 + MOV A2, m_vdsoPC(S3) + + MOV 32(X2), A3 + MOV A3, m_vdsoSP(S3) + + MOV T0, sec+0(FP) + MOVW T1, nsec+8(FP) + RET + +fallback: MOV $8(X2), A1 MOV $SYS_clock_gettime, A7 ECALL @@ -222,13 +284,62 @@ TEXT runtime·walltime(SB),NOSPLIT,$24-12 RET // func nanotime1() int64 -TEXT runtime·nanotime1(SB),NOSPLIT,$24-8 - MOV $1, A0 // CLOCK_MONOTONIC +TEXT runtime·nanotime1(SB),NOSPLIT,$40-8 + MOV $CLOCK_MONOTONIC, A0 + + MOV runtime·vdsoClockgettimeSym(SB), A7 + BEQZ A7, fallback + + MOV X2, S2 // S2 = RSP, S2 is unchanged by C code + MOV g_m(g), S3 // S3 = m + // Save the old values on stack for reentrant + MOV m_vdsoPC(S3), T0 + MOV T0, 24(X2) + MOV m_vdsoSP(S3), T0 + MOV T0, 32(X2) + + MOV RA, m_vdsoPC(S3) + MOV $ret-8(FP), T0 // caller's SP + MOV T0, m_vdsoSP(S3) + + MOV m_curg(S3), T1 + BNE g, T1, noswitch + + MOV m_g0(S3), T1 + MOV (g_sched+gobuf_sp)(T1), X2 + +noswitch: + ADDI $-24, X2 // Space for result + ANDI $~7, X2 // Align for C code MOV $8(X2), A1 - MOV $SYS_clock_gettime, A7 - ECALL + + // Store g on gsignal's stack, see sys_linux_arm64.s for detail + MOVBU runtime·iscgo(SB), S4 + BNEZ S4, nosaveg + MOV m_gsignal(S3), S4 // g.m.gsignal + BEQZ S4, nosaveg + BEQ g, S4, nosaveg + MOV (g_stack+stack_lo)(S4), S4 // g.m.gsignal.stack.lo + MOV g, (S4) + + JALR RA, A7 + + MOV ZERO, (S4) + JMP finish + +nosaveg: + JALR RA, A7 + +finish: MOV 8(X2), T0 // sec MOV 16(X2), T1 // nsec + // restore stack + MOV S2, X2 + MOV 24(X2), T2 + MOV T2, m_vdsoPC(S3) + + MOV 32(X2), T2 + MOV T2, m_vdsoSP(S3) // sec is in T0, nsec in T1 // return nsec in T0 MOV $1000000000, T2 @@ -237,6 +348,18 @@ TEXT runtime·nanotime1(SB),NOSPLIT,$24-8 MOV T0, ret+0(FP) RET +fallback: + MOV $8(X2), A1 + MOV $SYS_clock_gettime, A7 + ECALL + MOV 8(X2), T0 // sec + MOV 16(X2), T1 // nsec + MOV $1000000000, T2 + MUL T2, T0 + ADD T1, T0 + MOV T0, ret+0(FP) + RET + // func rtsigprocmask(how int32, new, old *sigset, size int32) TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28 MOVW how+0(FP), A0 diff --git a/src/runtime/vdso_elf64.go b/src/runtime/vdso_elf64.go index 9923bd4697..aecc84dcfe 100644 --- a/src/runtime/vdso_elf64.go +++ b/src/runtime/vdso_elf64.go @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le) +//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64) // +build linux -// +build amd64 arm64 mips64 mips64le ppc64 ppc64le +// +build amd64 arm64 mips64 mips64le ppc64 ppc64le riscv64 package runtime diff --git a/src/runtime/vdso_in_none.go b/src/runtime/vdso_in_none.go index c66fbf8216..3e98b08b15 100644 --- a/src/runtime/vdso_in_none.go +++ b/src/runtime/vdso_in_none.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le) || !linux -// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le !linux +//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64) || !linux +// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le,!riscv64 !linux package runtime diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go index ae211f96b1..20c8db78be 100644 --- a/src/runtime/vdso_linux.go +++ b/src/runtime/vdso_linux.go @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le) +//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64) // +build linux -// +build 386 amd64 arm arm64 mips64 mips64le ppc64 ppc64le +// +build 386 amd64 arm arm64 mips64 mips64le ppc64 ppc64le riscv64 package runtime diff --git a/src/runtime/vdso_linux_riscv64.go b/src/runtime/vdso_linux_riscv64.go new file mode 100644 index 0000000000..f427124c3c --- /dev/null +++ b/src/runtime/vdso_linux_riscv64.go @@ -0,0 +1,21 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + // vdsoArrayMax is the byte-size of a maximally sized array on this architecture. + // See cmd/compile/internal/riscv64/galign.go arch.MAXWIDTH initialization. + vdsoArrayMax = 1<<50 - 1 +) + +// key and version at man 7 vdso : riscv +var vdsoLinuxVersion = vdsoVersionKey{"LINUX_4.15", 0xae77f75} + +var vdsoSymbolKeys = []vdsoSymbolKey{ + {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym}, +} + +// initialize to fall back to syscall +var vdsoClockgettimeSym uintptr = 0