From: Yuval Pavel Zholkover Date: Thu, 19 Apr 2018 14:52:14 +0000 (+0300) Subject: runtime: FreeBSD fast clock_gettime HPET timecounter support X-Git-Tag: go1.11beta1~670 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=58c231f244feda52f66dd4692229f740d3c3b988;p=gostls13.git runtime: FreeBSD fast clock_gettime HPET timecounter support This is a followup for CL 93156. Fixes #22942. Change-Id: Ic6e2de44011d041b91454353a6f2e3b0cf590060 Reviewed-on: https://go-review.googlesource.com/108095 Run-TryBot: Ian Lance Taylor Reviewed-by: Ian Lance Taylor --- diff --git a/src/runtime/defs_freebsd.go b/src/runtime/defs_freebsd.go index f8eaf4167c..29a6ec20a5 100644 --- a/src/runtime/defs_freebsd.go +++ b/src/runtime/defs_freebsd.go @@ -53,6 +53,7 @@ const ( PROT_EXEC = C.PROT_EXEC MAP_ANON = C.MAP_ANON + MAP_SHARED = C.MAP_SHARED MAP_PRIVATE = C.MAP_PRIVATE MAP_FIXED = C.MAP_FIXED diff --git a/src/runtime/defs_freebsd_386.go b/src/runtime/defs_freebsd_386.go index 3e56a9f4d6..afdf54055f 100644 --- a/src/runtime/defs_freebsd_386.go +++ b/src/runtime/defs_freebsd_386.go @@ -22,6 +22,7 @@ const ( _PROT_EXEC = 0x4 _MAP_ANON = 0x1000 + _MAP_SHARED = 0x1 _MAP_PRIVATE = 0x2 _MAP_FIXED = 0x10 diff --git a/src/runtime/defs_freebsd_amd64.go b/src/runtime/defs_freebsd_amd64.go index 4f470fcc38..c88c0c55c7 100644 --- a/src/runtime/defs_freebsd_amd64.go +++ b/src/runtime/defs_freebsd_amd64.go @@ -22,6 +22,7 @@ const ( _PROT_EXEC = 0x4 _MAP_ANON = 0x1000 + _MAP_SHARED = 0x1 _MAP_PRIVATE = 0x2 _MAP_FIXED = 0x10 diff --git a/src/runtime/defs_freebsd_arm.go b/src/runtime/defs_freebsd_arm.go index 7b8f0d997d..0c21ea6cff 100644 --- a/src/runtime/defs_freebsd_arm.go +++ b/src/runtime/defs_freebsd_arm.go @@ -22,6 +22,7 @@ const ( _PROT_EXEC = 0x4 _MAP_ANON = 0x1000 + _MAP_SHARED = 0x1 _MAP_PRIVATE = 0x2 _MAP_FIXED = 0x10 diff --git a/src/runtime/vdso_freebsd.go b/src/runtime/vdso_freebsd.go index cefbb5df1c..4e5891976d 100644 --- a/src/runtime/vdso_freebsd.go +++ b/src/runtime/vdso_freebsd.go @@ -16,37 +16,41 @@ const _VDSO_TH_NUM = 4 // defined in #ifdef _KERNEL var timekeepSharedPage *vdsoTimekeep //go:nosplit -func (bt bintime) Add(bt2 bintime) bintime { +func (bt *bintime) Add(bt2 *bintime) { u := bt.frac bt.frac += bt2.frac if u > bt.frac { bt.sec++ } bt.sec += bt2.sec - return bt } //go:nosplit -func (bt bintime) AddX(x uint64) bintime { +func (bt *bintime) AddX(x uint64) { u := bt.frac bt.frac += x if u > bt.frac { bt.sec++ } - return bt } -var binuptimeDummy uint32 +var ( + // binuptimeDummy is used in binuptime as the address of an atomic.Load, to simulate + // an atomic_thread_fence_acq() call which behaves as an instruction reordering and + // memory barrier. + binuptimeDummy uint32 + + zeroBintime bintime +) // based on /usr/src/lib/libc/sys/__vdso_gettimeofday.c // //go:nosplit -func binuptime(abs bool) (bintime, bool) { - var bt bintime +func binuptime(abs bool) (bt bintime) { timehands := (*[_VDSO_TH_NUM]vdsoTimehands)(add(unsafe.Pointer(timekeepSharedPage), vdsoTimekeepSize)) for { if timekeepSharedPage.enabled == 0 { - return bt, false + return zeroBintime } curr := atomic.Load(&timekeepSharedPage.current) // atomic_load_acq_32 @@ -55,13 +59,13 @@ func binuptime(abs bool) (bintime, bool) { bt = th.offset if tc, ok := th.getTimecounter(); !ok { - return bt, false + return zeroBintime } else { delta := (tc - th.offset_count) & th.counter_mask - bt = bt.AddX(th.scale * uint64(delta)) + bt.AddX(th.scale * uint64(delta)) } if abs { - bt = bt.Add(th.boottime) + bt.Add(&th.boottime) } atomic.Load(&binuptimeDummy) // atomic_thread_fence_acq() @@ -69,13 +73,13 @@ func binuptime(abs bool) (bintime, bool) { break } } - return bt, true + return bt } //go:nosplit -func vdsoClockGettime(clockID int32) (bintime, bool) { +func vdsoClockGettime(clockID int32) bintime { if timekeepSharedPage == nil || timekeepSharedPage.ver != _VDSO_TK_VER_CURR { - return bintime{}, false + return zeroBintime } abs := false switch clockID { @@ -84,9 +88,8 @@ func vdsoClockGettime(clockID int32) (bintime, bool) { case _CLOCK_REALTIME: abs = true default: - return bintime{}, false + return zeroBintime } - return binuptime(abs) } @@ -95,16 +98,16 @@ func fallback_walltime() (sec int64, nsec int32) //go:nosplit func nanotime() int64 { - bt, ok := vdsoClockGettime(_CLOCK_MONOTONIC) - if !ok { + bt := vdsoClockGettime(_CLOCK_MONOTONIC) + if bt == zeroBintime { return fallback_nanotime() } return int64((1e9 * uint64(bt.sec)) + ((1e9 * uint64(bt.frac>>32)) >> 32)) } func walltime() (sec int64, nsec int32) { - bt, ok := vdsoClockGettime(_CLOCK_REALTIME) - if !ok { + bt := vdsoClockGettime(_CLOCK_REALTIME) + if bt == zeroBintime { return fallback_walltime() } return int64(bt.sec), int32((1e9 * uint64(bt.frac>>32)) >> 32) diff --git a/src/runtime/vdso_freebsd_x86.go b/src/runtime/vdso_freebsd_x86.go index e3cff68c6b..1b1be5f925 100644 --- a/src/runtime/vdso_freebsd_x86.go +++ b/src/runtime/vdso_freebsd_x86.go @@ -7,8 +7,35 @@ package runtime +import ( + "runtime/internal/atomic" + "unsafe" +) + +const ( + _VDSO_TH_ALGO_X86_TSC = 1 + _VDSO_TH_ALGO_X86_HPET = 2 +) + const ( - _VDSO_TH_ALGO_X86_TSC = 1 + _HPET_DEV_MAP_MAX = 10 + _HPET_MAIN_COUNTER = 0xf0 /* Main counter register */ +) + +var ( + hpetDevMap [_HPET_DEV_MAP_MAX]uintptr + hpetDevPath = [_HPET_DEV_MAP_MAX][11]byte{ + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '0', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '1', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '2', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '3', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '4', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '5', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '6', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '7', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '8', 0}, + {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '9', 0}, + } ) //go:nosplit @@ -20,11 +47,45 @@ func (th *vdsoTimehands) getTSCTimecounter() uint32 { return uint32(tsc) } +//go:nosplit +func (th *vdsoTimehands) getHPETTimecounter() (uint32, bool) { + idx := int(th.x86_hpet_idx) + if idx >= len(hpetDevMap) { + return 0, false + } + + p := atomic.Loaduintptr(&hpetDevMap[idx]) + if p == 0 { + fd := open(&hpetDevPath[idx][0], 0 /* O_RDONLY */, 0) + if fd < 0 { + atomic.Casuintptr(&hpetDevMap[idx], 0, ^uintptr(0)) + return 0, false + } + + addr, mmapErr := mmap(nil, physPageSize, _PROT_READ, _MAP_SHARED, fd, 0) + closefd(fd) + newP := uintptr(addr) + if mmapErr != 0 { + newP = ^uintptr(0) + } + if !atomic.Casuintptr(&hpetDevMap[idx], 0, newP) && mmapErr == 0 { + munmap(addr, physPageSize) + } + p = atomic.Loaduintptr(&hpetDevMap[idx]) + } + if p == ^uintptr(0) { + return 0, false + } + return *(*uint32)(unsafe.Pointer(p + _HPET_MAIN_COUNTER)), true +} + //go:nosplit func (th *vdsoTimehands) getTimecounter() (uint32, bool) { switch th.algo { case _VDSO_TH_ALGO_X86_TSC: return th.getTSCTimecounter(), true + case _VDSO_TH_ALGO_X86_HPET: + return th.getHPETTimecounter() default: return 0, false }