From: Tobias Klauser Date: Fri, 2 Mar 2018 10:27:15 +0000 (+0100) Subject: runtime: use vDSO for clock_gettime on linux/arm X-Git-Tag: go1.11beta1~1360 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=51b027116c2c90e7cb938362b0134ff710fea54e;p=gostls13.git runtime: use vDSO for clock_gettime on linux/arm Use the __vdso_clock_gettime fast path via the vDSO on linux/arm to speed up nanotime and walltime. This results in the following performance improvement for time.Now on a RaspberryPi 3 (running 32bit Raspbian, i.e. GOOS=linux/GOARCH=arm): name old time/op new time/op delta TimeNow 0.99µs ± 0% 0.39µs ± 1% -60.74% (p=0.000 n=12+20) Change-Id: I3598278a6c88d7f6a6ce66c56b9d25f9dd2f4c9a Reviewed-on: https://go-review.googlesource.com/98095 Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot --- diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index 7ebca7f856..d8c1592a1d 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -268,6 +268,7 @@ func sysauxv(auxv []uintptr) int { } archauxv(tag, val) + vdsoauxv(tag, val) } return i / 2 } diff --git a/src/runtime/os_linux_noauxv.go b/src/runtime/os_linux_noauxv.go index db6e5a0530..d6174c33fe 100644 --- a/src/runtime/os_linux_noauxv.go +++ b/src/runtime/os_linux_noauxv.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !386,!amd64,!arm,!arm64,!mips,!mipsle,!mips64,!mips64le,!s390x,!ppc64,!ppc64le +// +build !arm,!arm64,!mips,!mipsle,!mips64,!mips64le,!s390x,!ppc64,!ppc64le package runtime diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go new file mode 100644 index 0000000000..41acd957cf --- /dev/null +++ b/src/runtime/os_linux_novdso.go @@ -0,0 +1,10 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !386,!amd64,!arm + +package runtime + +func vdsoauxv(tag, val uintptr) { +} diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index ef72ae0308..39046bb5c5 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -10,6 +10,9 @@ #include "go_tls.h" #include "textflag.h" +#define CLOCK_REALTIME 0 +#define CLOCK_MONOTONIC 1 + // for EABI, as we don't support OABI #define SYS_BASE 0x0 @@ -209,31 +212,90 @@ TEXT runtime·mincore(SB),NOSPLIT,$0 MOVW R0, ret+12(FP) RET -TEXT runtime·walltime(SB), NOSPLIT, $32 - MOVW $0, R0 // CLOCK_REALTIME - MOVW $8(R13), R1 // timespec +TEXT runtime·walltime(SB),NOSPLIT,$0-12 + // We don't know how much stack space the VDSO code will need, + // so switch to g0. + + // Save old SP. Use R13 instead of SP to avoid linker rewriting the offsets. + MOVW R13, R4 // R4 is unchanged by C code. + + MOVW g_m(g), R1 + MOVW m_curg(R1), R0 + + CMP R1, R0 // Only switch if on curg. + B.NE noswitch + + MOVW m_g0(R1), R0 + MOVW (g_sched+gobuf_sp)(R0), R13 // Set SP to g0 stack + +noswitch: + SUB $24, R13 // Space for results + BIC $0x7, R13 // Align for C code + + MOVW $CLOCK_REALTIME, R0 + MOVW $8(R13), R1 // timespec + MOVW runtime·__vdso_clock_gettime_sym(SB), R11 + CMP $0, R11 + B.EQ fallback + + BL (R11) + JMP finish + +fallback: MOVW $SYS_clock_gettime, R7 SWI $0 - + +finish: MOVW 8(R13), R0 // sec MOVW 12(R13), R2 // nsec - + + MOVW R4, R13 // Restore real SP + MOVW R0, sec_lo+0(FP) MOVW $0, R1 MOVW R1, sec_hi+4(FP) MOVW R2, nsec+8(FP) - RET + RET // int64 nanotime(void) -TEXT runtime·nanotime(SB),NOSPLIT,$32 - MOVW $1, R0 // CLOCK_MONOTONIC - MOVW $8(R13), R1 // timespec +TEXT runtime·nanotime(SB),NOSPLIT,$0-8 + // Switch to g0 stack. See comment above in runtime·walltime. + + // Save old SP. Use R13 instead of SP to avoid linker rewriting the offsets. + MOVW R13, R4 // R4 is unchanged by C code. + + MOVW g_m(g), R1 + MOVW m_curg(R1), R0 + + CMP R1, R0 // Only switch if on curg. + B.NE noswitch + + MOVW m_g0(R1), R0 + MOVW (g_sched+gobuf_sp)(R0), R13 // Set SP to g0 stack + +noswitch: + SUB $24, R13 // Space for results + BIC $0x7, R13 // Align for C code + + MOVW $CLOCK_MONOTONIC, R0 + MOVW $8(R13), R1 // timespec + MOVW runtime·__vdso_clock_gettime_sym(SB), R11 + CMP $0, R11 + B.EQ fallback + + BL (R11) + JMP finish + +fallback: MOVW $SYS_clock_gettime, R7 SWI $0 - - MOVW 8(R13), R0 // sec - MOVW 12(R13), R2 // nsec - + +finish: + MOVW 8(R13), R0 // sec + MOVW 12(R13), R2 // nsec + + MOVW R4, R13 // Restore real SP + MOVW $1000000000, R3 MULLU R0, R3, (R1, R0) MOVW $0, R4 diff --git a/src/runtime/vdso_elf32.go b/src/runtime/vdso_elf32.go index e1a693db90..2720f33eed 100644 --- a/src/runtime/vdso_elf32.go +++ b/src/runtime/vdso_elf32.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build linux -// +build 386 +// +build 386 arm package runtime diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go index 61872e3b27..46b92869ab 100644 --- a/src/runtime/vdso_linux.go +++ b/src/runtime/vdso_linux.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build linux -// +build 386 amd64 +// +build 386 amd64 arm package runtime @@ -264,7 +264,7 @@ func vdso_parse_symbols(info *vdso_info, version int32) { } } -func archauxv(tag, val uintptr) { +func vdsoauxv(tag, val uintptr) { switch tag { case _AT_SYSINFO_EHDR: if val == 0 { diff --git a/src/runtime/vdso_linux_arm.go b/src/runtime/vdso_linux_arm.go new file mode 100644 index 0000000000..0f5fd5a38c --- /dev/null +++ b/src/runtime/vdso_linux_arm.go @@ -0,0 +1,19 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + // vdsoArrayMax is the byte-size of a maximally sized array on this architecture. + // See cmd/compile/internal/arm/galign.go arch.MAXWIDTH initialization, but must also + // be constrained to max +ve int. + vdsoArrayMax = 1<<31 - 1 +) + +var sym_keys = []symbol_key{ + {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &__vdso_clock_gettime_sym}, +} + +// initialize to fall back to syscall +var __vdso_clock_gettime_sym uintptr = 0 diff --git a/src/runtime/vdso_linux_test.go b/src/runtime/vdso_linux_test.go index f507ee98ee..890f907d22 100644 --- a/src/runtime/vdso_linux_test.go +++ b/src/runtime/vdso_linux_test.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build linux -// +build 386 amd64 +// +build 386 amd64 arm package runtime_test