Use the __vdso_clock_gettime fast path via the vDSO on linux/arm64 to
speed up nanotime and walltime. This results in the following
performance improvement for time.Now on Cavium ThunderX:
name old time/op new time/op delta
TimeNow 442ns ± 0% 163ns ± 0% -63.16% (p=0.000 n=10+10)
And benchmarks on VDSO
BenchmarkClockVDSOAndFallbackPaths/vDSO
10000000 166 ns/op
BenchmarkClockVDSOAndFallbackPaths/Fallback
3000000 456 ns/op
Change-Id: I326118c6dff865eaa0569fc45d1fc1ff95cb74f6
Reviewed-on: https://go-review.googlesource.com/99855
Run-TryBot: Tobias Klauser <tobias.klauser@gmail.com>
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>
// license that can be found in the LICENSE file.
// +build linux
-// +build !386,!amd64,!arm
+// +build !386,!amd64,!arm,!arm64
package runtime
#define AT_FDCWD -100
+#define CLOCK_REALTIME 0
+#define CLOCK_MONOTONIC 1
+
#define SYS_exit 93
#define SYS_read 63
#define SYS_write 64
// func walltime() (sec int64, nsec int32)
TEXT runtime·walltime(SB),NOSPLIT,$24-12
- MOVW $0, R0 // CLOCK_REALTIME
+ MOVD RSP, R20 // R20 is unchanged by C code
MOVD RSP, R1
+
+ MOVD g_m(g), R21 // R21 = m
+
+ // Set vdsoPC and vdsoSP for SIGPROF traceback.
+ MOVD LR, m_vdsoPC(R21)
+ MOVD R20, m_vdsoSP(R21)
+
+ MOVD m_curg(R21), R0
+ CMP g, R0
+ BNE noswitch
+
+ MOVD m_g0(R21), R3
+ MOVD (g_sched+gobuf_sp)(R3), R1 // Set RSP to g0 stack
+
+noswitch:
+ SUB $16, R1
+ BIC $15, R1 // Align for C code
+ MOVD R1, RSP
+
+ MOVW $CLOCK_REALTIME, R0
+ MOVD runtime·vdsoClockgettimeSym(SB), R2
+ CBZ R2, fallback
+ BL (R2)
+ B finish
+
+fallback:
MOVD $SYS_clock_gettime, R8
SVC
+
+finish:
MOVD 0(RSP), R3 // sec
MOVD 8(RSP), R5 // nsec
+
+ MOVD R20, RSP // restore SP
+ MOVD $0, m_vdsoSP(R21) // clear vdsoSP
+
MOVD R3, sec+0(FP)
MOVW R5, nsec+8(FP)
RET
TEXT runtime·nanotime(SB),NOSPLIT,$24-8
- MOVW $1, R0 // CLOCK_MONOTONIC
+ MOVD RSP, R20 // R20 is unchanged by C code
MOVD RSP, R1
+
+ MOVD g_m(g), R21 // R21 = m
+
+ // Set vdsoPC and vdsoSP for SIGPROF traceback.
+ MOVD LR, m_vdsoPC(R21)
+ MOVD R20, m_vdsoSP(R21)
+
+ MOVD m_curg(R21), R0
+ CMP g, R0
+ BNE noswitch
+
+ MOVD m_g0(R21), R3
+ MOVD (g_sched+gobuf_sp)(R3), R1 // Set RSP to g0 stack
+
+noswitch:
+ SUB $16, R1
+ BIC $15, R1
+ MOVD R1, RSP
+
+ MOVW $CLOCK_MONOTONIC, R0
+ MOVD runtime·vdsoClockgettimeSym(SB), R2
+ CBZ R2, fallback
+ BL (R2)
+ B finish
+
+fallback:
MOVD $SYS_clock_gettime, R8
SVC
+
+finish:
MOVD 0(RSP), R3 // sec
MOVD 8(RSP), R5 // nsec
+
+ MOVD R20, RSP // restore SP
+ MOVD $0, m_vdsoSP(R21) // clear vdsoSP
+
// sec is in R3, nsec in R5
// return nsec in R3
MOVD $1000000000, R4
// license that can be found in the LICENSE file.
// +build linux
-// +build amd64
+// +build amd64 arm64
package runtime
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// +build linux,!386,!amd64,!arm !linux
+// +build linux,!386,!amd64,!arm,!arm64 !linux
package runtime
// license that can be found in the LICENSE file.
// +build linux
-// +build 386 amd64 arm
+// +build 386 amd64 arm arm64
package runtime
verdef *elfVerdef
}
-var linux26 = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
-
// see vdso_linux_*.go for vdsoSymbolKeys[] and vdso*Sym vars
func vdsoInitFromSysinfoEhdr(info *vdsoInfo, hdr *elfEhdr) {
if k.name != gostringnocopy(&info.symstrings[sym.st_name]) {
return false
}
-
// Check symbol version.
if info.versym != nil && version != 0 && int32(info.versym[symIndex]&0x7fff) != version {
return false
// when passed to the three functions below.
info1 := (*vdsoInfo)(noescape(unsafe.Pointer(&info)))
vdsoInitFromSysinfoEhdr(info1, (*elfEhdr)(unsafe.Pointer(val)))
- vdsoParseSymbols(info1, vdsoFindVersion(info1, &linux26))
+ vdsoParseSymbols(info1, vdsoFindVersion(info1, &vdsoLinuxVersion))
}
}
vdsoArrayMax = 1<<31 - 1
)
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
+
var vdsoSymbolKeys = []vdsoSymbolKey{
{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
}
// initialize to fall back to syscall
-var (
- vdsoClockgettimeSym uintptr = 0
-)
+var vdsoClockgettimeSym uintptr = 0
vdsoArrayMax = 1<<50 - 1
)
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
+
var vdsoSymbolKeys = []vdsoSymbolKey{
{"__vdso_gettimeofday", 0x315ca59, 0xb01bca00, &vdsoGettimeofdaySym},
{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
vdsoArrayMax = 1<<31 - 1
)
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
+
var vdsoSymbolKeys = []vdsoSymbolKey{
{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+const (
+ // vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+ // See cmd/compile/internal/arm64/galign.go arch.MAXWIDTH initialization.
+ vdsoArrayMax = 1<<50 - 1
+)
+
+// key and version at man 7 vdso : aarch64
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.39", 0x75fcb89}
+
+var vdsoSymbolKeys = []vdsoSymbolKey{
+ {"__kernel_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
+}
+
+// initialize to fall back to syscall
+var vdsoClockgettimeSym uintptr = 0
// license that can be found in the LICENSE file.
// +build linux
-// +build 386 amd64 arm
+// +build 386 amd64 arm arm64
package runtime_test