runtime: FreeBSD fast clock_gettime HPET timecounter support

This is a followup for CL 93156.

Fixes #22942.

Change-Id: Ic6e2de44011d041b91454353a6f2e3b0cf590060
Reviewed-on: https://go-review.googlesource.com/108095
Run-TryBot: Ian Lance Taylor <iant@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

diff --git a/src/runtime/defs_freebsd.go b/src/runtime/defs_freebsd.go
index f8eaf4167c2e3e13920a1f1a77baf0d38087abcf..29a6ec20a56ba3af8e32bd8cb347590995a5142d 100644 (file)
--- a/src/runtime/defs_freebsd.go
+++ b/src/runtime/defs_freebsd.go
@@ -53,6 +53,7 @@ const (
        PROT_EXEC  = C.PROT_EXEC
 
        MAP_ANON    = C.MAP_ANON
+       MAP_SHARED  = C.MAP_SHARED
        MAP_PRIVATE = C.MAP_PRIVATE
        MAP_FIXED   = C.MAP_FIXED
 

diff --git a/src/runtime/defs_freebsd_386.go b/src/runtime/defs_freebsd_386.go
index 3e56a9f4d6aa35836c6e136f8c7095879a2890de..afdf54055f7e00bde435d7b134bef2d8aff8f40c 100644 (file)
--- a/src/runtime/defs_freebsd_386.go
+++ b/src/runtime/defs_freebsd_386.go
@@ -22,6 +22,7 @@ const (
        _PROT_EXEC  = 0x4
 
        _MAP_ANON    = 0x1000
+       _MAP_SHARED  = 0x1
        _MAP_PRIVATE = 0x2
        _MAP_FIXED   = 0x10
 

diff --git a/src/runtime/defs_freebsd_amd64.go b/src/runtime/defs_freebsd_amd64.go
index 4f470fcc388a2b25c5d670fa61db3df937ac4e6a..c88c0c55c7f096e80694d361b8be6b49550c23ec 100644 (file)
--- a/src/runtime/defs_freebsd_amd64.go
+++ b/src/runtime/defs_freebsd_amd64.go
@@ -22,6 +22,7 @@ const (
        _PROT_EXEC  = 0x4
 
        _MAP_ANON    = 0x1000
+       _MAP_SHARED  = 0x1
        _MAP_PRIVATE = 0x2
        _MAP_FIXED   = 0x10
 

diff --git a/src/runtime/defs_freebsd_arm.go b/src/runtime/defs_freebsd_arm.go
index 7b8f0d997daa5f7435e48a5328f4b72516d32199..0c21ea6cff1e26ec163dfbfa918067607a70fb86 100644 (file)
--- a/src/runtime/defs_freebsd_arm.go
+++ b/src/runtime/defs_freebsd_arm.go
@@ -22,6 +22,7 @@ const (
        _PROT_EXEC  = 0x4
 
        _MAP_ANON    = 0x1000
+       _MAP_SHARED  = 0x1
        _MAP_PRIVATE = 0x2
        _MAP_FIXED   = 0x10
 

diff --git a/src/runtime/vdso_freebsd.go b/src/runtime/vdso_freebsd.go
index cefbb5df1c74bf8d237fdf306065da8dace222db..4e5891976dd707b78bc83d123a62007aafe83c3c 100644 (file)
--- a/src/runtime/vdso_freebsd.go
+++ b/src/runtime/vdso_freebsd.go
@@ -16,37 +16,41 @@ const _VDSO_TH_NUM = 4 // defined in <sys/vdso.h> #ifdef _KERNEL
 var timekeepSharedPage *vdsoTimekeep
 
 //go:nosplit
-func (bt bintime) Add(bt2 bintime) bintime {
+func (bt *bintime) Add(bt2 *bintime) {
        u := bt.frac
        bt.frac += bt2.frac
        if u > bt.frac {
                bt.sec++
        }
        bt.sec += bt2.sec
-       return bt
 }
 
 //go:nosplit
-func (bt bintime) AddX(x uint64) bintime {
+func (bt *bintime) AddX(x uint64) {
        u := bt.frac
        bt.frac += x
        if u > bt.frac {
                bt.sec++
        }
-       return bt
 }
 
-var binuptimeDummy uint32
+var (
+       // binuptimeDummy is used in binuptime as the address of an atomic.Load, to simulate
+       // an atomic_thread_fence_acq() call which behaves as an instruction reordering and
+       // memory barrier.
+       binuptimeDummy uint32
+
+       zeroBintime bintime
+)
 
 // based on /usr/src/lib/libc/sys/__vdso_gettimeofday.c
 //
 //go:nosplit
-func binuptime(abs bool) (bintime, bool) {
-       var bt bintime
+func binuptime(abs bool) (bt bintime) {
        timehands := (*[_VDSO_TH_NUM]vdsoTimehands)(add(unsafe.Pointer(timekeepSharedPage), vdsoTimekeepSize))
        for {
                if timekeepSharedPage.enabled == 0 {
-                       return bt, false
+                       return zeroBintime
                }
 
                curr := atomic.Load(&timekeepSharedPage.current) // atomic_load_acq_32
@@ -55,13 +59,13 @@ func binuptime(abs bool) (bintime, bool) {
                bt = th.offset
 
                if tc, ok := th.getTimecounter(); !ok {
-                       return bt, false
+                       return zeroBintime
                } else {
                        delta := (tc - th.offset_count) & th.counter_mask
-                       bt = bt.AddX(th.scale * uint64(delta))
+                       bt.AddX(th.scale * uint64(delta))
                }
                if abs {
-                       bt = bt.Add(th.boottime)
+                       bt.Add(&th.boottime)
                }
 
                atomic.Load(&binuptimeDummy) // atomic_thread_fence_acq()
@@ -69,13 +73,13 @@ func binuptime(abs bool) (bintime, bool) {
                        break
                }
        }
-       return bt, true
+       return bt
 }
 
 //go:nosplit
-func vdsoClockGettime(clockID int32) (bintime, bool) {
+func vdsoClockGettime(clockID int32) bintime {
        if timekeepSharedPage == nil || timekeepSharedPage.ver != _VDSO_TK_VER_CURR {
-               return bintime{}, false
+               return zeroBintime
        }
        abs := false
        switch clockID {
@@ -84,9 +88,8 @@ func vdsoClockGettime(clockID int32) (bintime, bool) {
        case _CLOCK_REALTIME:
                abs = true
        default:
-               return bintime{}, false
+               return zeroBintime
        }
-
        return binuptime(abs)
 }
 
@@ -95,16 +98,16 @@ func fallback_walltime() (sec int64, nsec int32)
 
 //go:nosplit
 func nanotime() int64 {
-       bt, ok := vdsoClockGettime(_CLOCK_MONOTONIC)
-       if !ok {
+       bt := vdsoClockGettime(_CLOCK_MONOTONIC)
+       if bt == zeroBintime {
                return fallback_nanotime()
        }
        return int64((1e9 * uint64(bt.sec)) + ((1e9 * uint64(bt.frac>>32)) >> 32))
 }
 
 func walltime() (sec int64, nsec int32) {
-       bt, ok := vdsoClockGettime(_CLOCK_REALTIME)
-       if !ok {
+       bt := vdsoClockGettime(_CLOCK_REALTIME)
+       if bt == zeroBintime {
                return fallback_walltime()
        }
        return int64(bt.sec), int32((1e9 * uint64(bt.frac>>32)) >> 32)

diff --git a/src/runtime/vdso_freebsd_x86.go b/src/runtime/vdso_freebsd_x86.go
index e3cff68c6b85bd6e886ea1de8b6b8d6195e35e58..1b1be5f9256b4c3714f285d5c79eebfc0b069a3b 100644 (file)
--- a/src/runtime/vdso_freebsd_x86.go
+++ b/src/runtime/vdso_freebsd_x86.go
@@ -7,8 +7,35 @@
 
 package runtime
 
+import (
+       "runtime/internal/atomic"
+       "unsafe"
+)
+
+const (
+       _VDSO_TH_ALGO_X86_TSC  = 1
+       _VDSO_TH_ALGO_X86_HPET = 2
+)
+
 const (
-       _VDSO_TH_ALGO_X86_TSC = 1
+       _HPET_DEV_MAP_MAX  = 10
+       _HPET_MAIN_COUNTER = 0xf0 /* Main counter register */
+)
+
+var (
+       hpetDevMap  [_HPET_DEV_MAP_MAX]uintptr
+       hpetDevPath = [_HPET_DEV_MAP_MAX][11]byte{
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '0', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '1', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '2', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '3', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '4', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '5', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '6', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '7', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '8', 0},
+               {'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '9', 0},
+       }
 )
 
 //go:nosplit
@@ -20,11 +47,45 @@ func (th *vdsoTimehands) getTSCTimecounter() uint32 {
        return uint32(tsc)
 }
 
+//go:nosplit
+func (th *vdsoTimehands) getHPETTimecounter() (uint32, bool) {
+       idx := int(th.x86_hpet_idx)
+       if idx >= len(hpetDevMap) {
+               return 0, false
+       }
+
+       p := atomic.Loaduintptr(&hpetDevMap[idx])
+       if p == 0 {
+               fd := open(&hpetDevPath[idx][0], 0 /* O_RDONLY */, 0)
+               if fd < 0 {
+                       atomic.Casuintptr(&hpetDevMap[idx], 0, ^uintptr(0))
+                       return 0, false
+               }
+
+               addr, mmapErr := mmap(nil, physPageSize, _PROT_READ, _MAP_SHARED, fd, 0)
+               closefd(fd)
+               newP := uintptr(addr)
+               if mmapErr != 0 {
+                       newP = ^uintptr(0)
+               }
+               if !atomic.Casuintptr(&hpetDevMap[idx], 0, newP) && mmapErr == 0 {
+                       munmap(addr, physPageSize)
+               }
+               p = atomic.Loaduintptr(&hpetDevMap[idx])
+       }
+       if p == ^uintptr(0) {
+               return 0, false
+       }
+       return *(*uint32)(unsafe.Pointer(p + _HPET_MAIN_COUNTER)), true
+}
+
 //go:nosplit
 func (th *vdsoTimehands) getTimecounter() (uint32, bool) {
        switch th.algo {
        case _VDSO_TH_ALGO_X86_TSC:
                return th.getTSCTimecounter(), true
+       case _VDSO_TH_ALGO_X86_HPET:
+               return th.getHPETTimecounter()
        default:
                return 0, false
        }