Use AT_TIMEKEEP ELF aux entry to access a kernel mapped ring of timehands structs.
The timehands are updated by the kernel periodically, but for accurate measure the
timecounter still needs to be queried.
Currently the fast path is used only when kern.timecounter.hardware==TSC-low
or kern.timecounter.hardware=='ARM MPCore Timecounter',
other timecounters revert back to regular system call.
TODO: add support for HPET timecounter on 386/amd64.
Change-Id: I321ca4e92be63ba21a2574b758ef5c1e729086ad
Reviewed-on: https://go-review.googlesource.com/93156
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>
#include <sys/sysctl.h>
#include <sys/cpuset.h>
#include <sys/param.h>
+#include <sys/vdso.h>
*/
import "C"
SA_ONSTACK = C.SA_ONSTACK
CLOCK_MONOTONIC = C.CLOCK_MONOTONIC
+ CLOCK_REALTIME = C.CLOCK_REALTIME
UMTX_OP_WAIT_UINT = C.UMTX_OP_WAIT_UINT
UMTX_OP_WAIT_UINT_PRIVATE = C.UMTX_OP_WAIT_UINT_PRIVATE
type Umtx_time C.struct__umtx_time
type Kevent C.struct_kevent
+
+type bintime C.struct_bintime
+type vdsoTimehands C.struct_vdso_timehands
+type vdsoTimekeep C.struct_vdso_timekeep
+
+const (
+ _VDSO_TK_VER_CURR = C.VDSO_TK_VER_CURR
+
+ vdsoTimehandsSize = C.sizeof_struct_vdso_timehands
+ vdsoTimekeepSize = C.sizeof_struct_vdso_timekeep
+)
_SA_ONSTACK = 0x1
_CLOCK_MONOTONIC = 0x4
+ _CLOCK_REALTIME = 0x0
_UMTX_OP_WAIT_UINT = 0xb
_UMTX_OP_WAIT_UINT_PRIVATE = 0xf
data int32
udata *byte
}
+
+type bintime struct {
+ sec int32
+ frac uint64
+}
+
+type vdsoTimehands struct {
+ algo uint32
+ gen uint32
+ scale uint64
+ offset_count uint32
+ counter_mask uint32
+ offset bintime
+ boottime bintime
+ x86_shift uint32
+ x86_hpet_idx uint32
+ res [6]uint32
+}
+
+type vdsoTimekeep struct {
+ ver uint32
+ enabled uint32
+ current uint32
+}
+
+const (
+ _VDSO_TK_VER_CURR = 0x1
+
+ vdsoTimehandsSize = 0x50
+ vdsoTimekeepSize = 0xc
+)
_SA_ONSTACK = 0x1
_CLOCK_MONOTONIC = 0x4
+ _CLOCK_REALTIME = 0x0
_UMTX_OP_WAIT_UINT = 0xb
_UMTX_OP_WAIT_UINT_PRIVATE = 0xf
data int64
udata *byte
}
+
+type bintime struct {
+ sec int64
+ frac uint64
+}
+
+type vdsoTimehands struct {
+ algo uint32
+ gen uint32
+ scale uint64
+ offset_count uint32
+ counter_mask uint32
+ offset bintime
+ boottime bintime
+ x86_shift uint32
+ x86_hpet_idx uint32
+ res [6]uint32
+}
+
+type vdsoTimekeep struct {
+ ver uint32
+ enabled uint32
+ current uint32
+ pad_cgo_0 [4]byte
+}
+
+const (
+ _VDSO_TK_VER_CURR = 0x1
+
+ vdsoTimehandsSize = 0x58
+ vdsoTimekeepSize = 0x10
+)
_SA_ONSTACK = 0x1
_CLOCK_MONOTONIC = 0x4
+ _CLOCK_REALTIME = 0x0
_UMTX_OP_WAIT_UINT = 0xb
_UMTX_OP_WAIT_UINT_PRIVATE = 0xf
data int32
udata *byte
}
+
+type bintime struct {
+ sec int64
+ frac uint64
+}
+
+type vdsoTimehands struct {
+ algo uint32
+ gen uint32
+ scale uint64
+ offset_count uint32
+ counter_mask uint32
+ offset bintime
+ boottime bintime
+ physical uint32
+ res [7]uint32
+}
+
+type vdsoTimekeep struct {
+ ver uint32
+ enabled uint32
+ current uint32
+ pad_cgo_0 [4]byte
+}
+
+const (
+ _VDSO_TK_VER_CURR = 0x1
+
+ vdsoTimehandsSize = 0x58
+ vdsoTimekeepSize = 0x10
+)
}
const (
- _AT_NULL = 0 // Terminates the vector
- _AT_PAGESZ = 6 // Page size in bytes
- _AT_HWCAP = 26 // CPU feature flags
+ _AT_NULL = 0 // Terminates the vector
+ _AT_PAGESZ = 6 // Page size in bytes
+ _AT_TIMEKEEP = 22 // Pointer to timehands.
+ _AT_HWCAP = 26 // CPU feature flags
)
func sysauxv(auxv []uintptr) {
// _AT_NCPUS from auxv shouldn't be used due to golang.org/issue/15206
case _AT_PAGESZ:
physPageSize = val
+ case _AT_TIMEKEEP:
+ timekeepSharedPage = (*vdsoTimekeep)(unsafe.Pointer(val))
}
archauxv(tag, val)
func closefd(fd int32) int32
func exit(code int32)
-func nanotime() int64
func usleep(usec uint32)
//go:noescape
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !plan9
+// +build !solaris
+// +build !windows
+// +build !nacl
+// +build !freebsd
+
+package runtime
+
+func nanotime() int64
INT $0x80
RET
-// func walltime() (sec int64, nsec int32)
-TEXT runtime·walltime(SB), NOSPLIT, $32
+// func fallback_walltime() (sec int64, nsec int32)
+TEXT runtime·fallback_walltime(SB), NOSPLIT, $32-12
MOVL $232, AX // clock_gettime
LEAL 12(SP), BX
MOVL $0, 4(SP) // CLOCK_REALTIME
MOVL BX, nsec+8(FP)
RET
-// int64 nanotime(void) so really
-// void nanotime(int64 *nsec)
-TEXT runtime·nanotime(SB), NOSPLIT, $32
+// func fallback_nanotime() int64
+TEXT runtime·fallback_nanotime(SB), NOSPLIT, $32-8
MOVL $232, AX
LEAL 12(SP), BX
- // We can use CLOCK_MONOTONIC_FAST here when we drop
- // support for FreeBSD 8-STABLE.
MOVL $4, 4(SP) // CLOCK_MONOTONIC
MOVL BX, 8(SP)
INT $0x80
SYSCALL
RET
-// func walltime() (sec int64, nsec int32)
-TEXT runtime·walltime(SB), NOSPLIT, $32
- MOVL $232, AX // clock_gettime
+// func fallback_walltime() (sec int64, nsec int32)
+TEXT runtime·fallback_walltime(SB), NOSPLIT, $32-12
+ MOVL $232, AX // clock_gettime
MOVQ $0, DI // CLOCK_REALTIME
LEAQ 8(SP), SI
SYSCALL
MOVL DX, nsec+8(FP)
RET
-TEXT runtime·nanotime(SB), NOSPLIT, $32
+TEXT runtime·fallback_nanotime(SB), NOSPLIT, $32-8
MOVL $232, AX
- // We can use CLOCK_MONOTONIC_FAST here when we drop
- // support for FreeBSD 8-STABLE.
MOVQ $4, DI // CLOCK_MONOTONIC
LEAQ 8(SP), SI
SYSCALL
SWI $0
RET
-// func walltime() (sec int64, nsec int32)
-TEXT runtime·walltime(SB), NOSPLIT, $32
+// func fallback_walltime() (sec int64, nsec int32)
+TEXT runtime·fallback_walltime(SB), NOSPLIT, $32-12
MOVW $0, R0 // CLOCK_REALTIME
MOVW $8(R13), R1
MOVW $SYS_clock_gettime, R7
MOVW R2, nsec+8(FP)
RET
-// int64 nanotime(void) so really
-// void nanotime(int64 *nsec)
-TEXT runtime·nanotime(SB), NOSPLIT, $32
- // We can use CLOCK_MONOTONIC_FAST here when we drop
- // support for FreeBSD 8-STABLE.
+// func fallback_nanotime() int64
+TEXT runtime·fallback_nanotime(SB), NOSPLIT, $32
MOVW $4, R0 // CLOCK_MONOTONIC
MOVW $8(R13), R1
MOVW $SYS_clock_gettime, R7
SUB $20, R13
MOVW R0, ret+24(FP)
RET
+
+// func getCntxct(physical bool) uint32
+TEXT runtime·getCntxct(SB),NOSPLIT|NOFRAME,$0-8
+ MOVB runtime·goarm(SB), R11
+ CMP $7, R11
+ BLT 2(PC)
+ DMB
+
+ MOVB physical+0(FP), R0
+ CMP $1, R0
+ B.NE 3(PC)
+
+ // get CNTPCT (Physical Count Register) into R0(low) R1(high)
+ // mrrc 15, 0, r0, r1, cr14
+ WORD $0xec510f0e
+ B 2(PC)
+
+ // get CNTVCT (Virtual Count Register) into R0(low) R1(high)
+ // mrrc 15, 1, r0, r1, cr14
+ WORD $0xec510f1e
+
+ MOVW R0, ret+4(FP)
+ RET
import _ "unsafe" // for go:linkname
-func walltime() (sec int64, nsec int32)
-
//go:linkname time_now time.now
func time_now() (sec int64, nsec int32, mono int64) {
sec, nsec = walltime()
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !darwin !amd64,!386
+// +build !windows
+// +build !freebsd
+
+package runtime
+
+func walltime() (sec int64, nsec int32)
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build freebsd
+
+package runtime
+
+import (
+ "runtime/internal/atomic"
+ "unsafe"
+)
+
+const _VDSO_TH_NUM = 4 // defined in <sys/vdso.h> #ifdef _KERNEL
+
+var timekeepSharedPage *vdsoTimekeep
+
+//go:nosplit
+func (bt bintime) Add(bt2 bintime) bintime {
+ u := bt.frac
+ bt.frac += bt2.frac
+ if u > bt.frac {
+ bt.sec++
+ }
+ bt.sec += bt2.sec
+ return bt
+}
+
+//go:nosplit
+func (bt bintime) AddX(x uint64) bintime {
+ u := bt.frac
+ bt.frac += x
+ if u > bt.frac {
+ bt.sec++
+ }
+ return bt
+}
+
+var binuptimeDummy uint32
+
+// based on /usr/src/lib/libc/sys/__vdso_gettimeofday.c
+//
+//go:nosplit
+func binuptime(abs bool) (bintime, bool) {
+ var bt bintime
+ timehands := (*[_VDSO_TH_NUM]vdsoTimehands)(add(unsafe.Pointer(timekeepSharedPage), vdsoTimekeepSize))
+ for {
+ if timekeepSharedPage.enabled == 0 {
+ return bt, false
+ }
+
+ curr := atomic.Load(&timekeepSharedPage.current) // atomic_load_acq_32
+ th := &timehands[curr]
+ gen := atomic.Load(&th.gen) // atomic_load_acq_32
+ bt = th.offset
+
+ if tc, ok := th.getTimecounter(); !ok {
+ return bt, false
+ } else {
+ delta := (tc - th.offset_count) & th.counter_mask
+ bt = bt.AddX(th.scale * uint64(delta))
+ }
+ if abs {
+ bt = bt.Add(th.boottime)
+ }
+
+ atomic.Load(&binuptimeDummy) // atomic_thread_fence_acq()
+ if curr == timekeepSharedPage.current && gen != 0 && gen == th.gen {
+ break
+ }
+ }
+ return bt, true
+}
+
+//go:nosplit
+func vdsoClockGettime(clockID int32) (bintime, bool) {
+ if timekeepSharedPage == nil || timekeepSharedPage.ver != _VDSO_TK_VER_CURR {
+ return bintime{}, false
+ }
+ abs := false
+ switch clockID {
+ case _CLOCK_MONOTONIC:
+ /* ok */
+ case _CLOCK_REALTIME:
+ abs = true
+ default:
+ return bintime{}, false
+ }
+
+ return binuptime(abs)
+}
+
+func fallback_nanotime() int64
+func fallback_walltime() (sec int64, nsec int32)
+
+//go:nosplit
+func nanotime() int64 {
+ bt, ok := vdsoClockGettime(_CLOCK_MONOTONIC)
+ if !ok {
+ return fallback_nanotime()
+ }
+ return int64((1e9 * uint64(bt.sec)) + ((1e9 * uint64(bt.frac>>32)) >> 32))
+}
+
+func walltime() (sec int64, nsec int32) {
+ bt, ok := vdsoClockGettime(_CLOCK_REALTIME)
+ if !ok {
+ return fallback_walltime()
+ }
+ return int64(bt.sec), int32((1e9 * uint64(bt.frac>>32)) >> 32)
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+const (
+ _VDSO_TH_ALGO_ARM_GENTIM = 1
+)
+
+func getCntxct(physical bool) uint32
+
+//go:nosplit
+func (th *vdsoTimehands) getTimecounter() (uint32, bool) {
+ switch th.algo {
+ case _VDSO_TH_ALGO_ARM_GENTIM:
+ return getCntxct(th.physical != 0), true
+ default:
+ return 0, false
+ }
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build freebsd
+// +build 386 amd64
+
+package runtime
+
+const (
+ _VDSO_TH_ALGO_X86_TSC = 1
+)
+
+//go:nosplit
+func (th *vdsoTimehands) getTSCTimecounter() uint32 {
+ tsc := cputicks()
+ if th.x86_shift > 0 {
+ tsc >>= th.x86_shift
+ }
+ return uint32(tsc)
+}
+
+//go:nosplit
+func (th *vdsoTimehands) getTimecounter() (uint32, bool) {
+ switch th.algo {
+ case _VDSO_TH_ALGO_X86_TSC:
+ return th.getTSCTimecounter(), true
+ default:
+ return 0, false
+ }
+}