runtime: use vDSO for clock_gettime on linux/arm

Use the __vdso_clock_gettime fast path via the vDSO on linux/arm to
speed up nanotime and walltime. This results in the following
performance improvement for time.Now on a RaspberryPi 3 (running
32bit Raspbian, i.e. GOOS=linux/GOARCH=arm):

name     old time/op  new time/op  delta
TimeNow  0.99µs ± 0%  0.39µs ± 1%  -60.74%  (p=0.000 n=12+20)

Change-Id: I3598278a6c88d7f6a6ce66c56b9d25f9dd2f4c9a
Reviewed-on: https://go-review.googlesource.com/98095
Reviewed-by: Ian Lance Taylor <iant@golang.org>
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go
index 7ebca7f8569c64f64f2432d7461cd706565f95ef..d8c1592a1d784703dbdbbeaee9dd719b0da3c3b3 100644 (file)
--- a/src/runtime/os_linux.go
+++ b/src/runtime/os_linux.go
@@ -268,6 +268,7 @@ func sysauxv(auxv []uintptr) int {
                }
 
                archauxv(tag, val)
+               vdsoauxv(tag, val)
        }
        return i / 2
 }

diff --git a/src/runtime/os_linux_noauxv.go b/src/runtime/os_linux_noauxv.go
index db6e5a0530fc268f71e36c953cb6a34f91a6d543..d6174c33fe6e5806c2946d7befbbf91c141dc119 100644 (file)
--- a/src/runtime/os_linux_noauxv.go
+++ b/src/runtime/os_linux_noauxv.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !386,!amd64,!arm,!arm64,!mips,!mipsle,!mips64,!mips64le,!s390x,!ppc64,!ppc64le
+// +build !arm,!arm64,!mips,!mipsle,!mips64,!mips64le,!s390x,!ppc64,!ppc64le
 
 package runtime
 

diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go
new file mode 100644 (file)
index 0000000..41acd95
--- /dev/null
+++ b/src/runtime/os_linux_novdso.go
@@ -0,0 +1,10 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !386,!amd64,!arm
+
+package runtime
+
+func vdsoauxv(tag, val uintptr) {
+}

diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s
index ef72ae0308431285d957c4f9d38fdc0e358aa99e..39046bb5c5968971b9368e9eb506eb81ea6cfd7e 100644 (file)
--- a/src/runtime/sys_linux_arm.s
+++ b/src/runtime/sys_linux_arm.s
@@ -10,6 +10,9 @@
 #include "go_tls.h"
 #include "textflag.h"
 
+#define CLOCK_REALTIME 0
+#define CLOCK_MONOTONIC        1
+
 // for EABI, as we don't support OABI
 #define SYS_BASE 0x0
 
@@ -209,31 +212,90 @@ TEXT runtime·mincore(SB),NOSPLIT,$0
        MOVW    R0, ret+12(FP)
        RET
 
-TEXT runtime·walltime(SB), NOSPLIT, $32
-       MOVW    $0, R0  // CLOCK_REALTIME
-       MOVW    $8(R13), R1  // timespec
+TEXT runtime·walltime(SB),NOSPLIT,$0-12
+       // We don't know how much stack space the VDSO code will need,
+       // so switch to g0.
+
+       // Save old SP. Use R13 instead of SP to avoid linker rewriting the offsets.
+       MOVW    R13, R4 // R4 is unchanged by C code.
+
+       MOVW    g_m(g), R1
+       MOVW    m_curg(R1), R0
+
+       CMP     R1, R0          // Only switch if on curg.
+       B.NE    noswitch
+
+       MOVW    m_g0(R1), R0
+       MOVW    (g_sched+gobuf_sp)(R0), R13      // Set SP to g0 stack
+
+noswitch:
+       SUB     $24, R13        // Space for results
+       BIC     $0x7, R13       // Align for C code
+
+       MOVW    $CLOCK_REALTIME, R0
+       MOVW    $8(R13), R1     // timespec
+       MOVW    runtime·__vdso_clock_gettime_sym(SB), R11
+       CMP     $0, R11
+       B.EQ    fallback
+
+       BL      (R11)
+       JMP     finish
+
+fallback:
        MOVW    $SYS_clock_gettime, R7
        SWI     $0
-       
+
+finish:
        MOVW    8(R13), R0  // sec
        MOVW    12(R13), R2  // nsec
-       
+
+       MOVW    R4, R13         // Restore real SP
+
        MOVW    R0, sec_lo+0(FP)
        MOVW    $0, R1
        MOVW    R1, sec_hi+4(FP)
        MOVW    R2, nsec+8(FP)
-       RET     
+       RET
 
 // int64 nanotime(void)
-TEXT runtime·nanotime(SB),NOSPLIT,$32
-       MOVW    $1, R0  // CLOCK_MONOTONIC
-       MOVW    $8(R13), R1  // timespec
+TEXT runtime·nanotime(SB),NOSPLIT,$0-8
+       // Switch to g0 stack. See comment above in runtime·walltime.
+
+       // Save old SP. Use R13 instead of SP to avoid linker rewriting the offsets.
+       MOVW    R13, R4 // R4 is unchanged by C code.
+
+       MOVW    g_m(g), R1
+       MOVW    m_curg(R1), R0
+
+       CMP     R1, R0          // Only switch if on curg.
+       B.NE    noswitch
+
+       MOVW    m_g0(R1), R0
+       MOVW    (g_sched+gobuf_sp)(R0), R13     // Set SP to g0 stack
+
+noswitch:
+       SUB     $24, R13        // Space for results
+       BIC     $0x7, R13       // Align for C code
+
+       MOVW    $CLOCK_MONOTONIC, R0
+       MOVW    $8(R13), R1     // timespec
+       MOVW    runtime·__vdso_clock_gettime_sym(SB), R11
+       CMP     $0, R11
+       B.EQ    fallback
+
+       BL      (R11)
+       JMP     finish
+
+fallback:
        MOVW    $SYS_clock_gettime, R7
        SWI     $0
-       
-       MOVW    8(R13), R0  // sec
-       MOVW    12(R13), R2  // nsec
-       
+
+finish:
+       MOVW    8(R13), R0      // sec
+       MOVW    12(R13), R2     // nsec
+
+       MOVW    R4, R13         // Restore real SP
+
        MOVW    $1000000000, R3
        MULLU   R0, R3, (R1, R0)
        MOVW    $0, R4

diff --git a/src/runtime/vdso_elf32.go b/src/runtime/vdso_elf32.go
index e1a693db902738b2a6b4aac81a4bb7e386df042e..2720f33eedec8065452a7df5119fa7bd14b60512 100644 (file)
--- a/src/runtime/vdso_elf32.go
+++ b/src/runtime/vdso_elf32.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386
+// +build 386 arm
 
 package runtime
 

diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go
index 61872e3b27bfe35d03b869b921940c7d3558b1cc..46b92869ab26d5617ce8977c8a76cd46b0356471 100644 (file)
--- a/src/runtime/vdso_linux.go
+++ b/src/runtime/vdso_linux.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386 amd64
+// +build 386 amd64 arm
 
 package runtime
 
@@ -264,7 +264,7 @@ func vdso_parse_symbols(info *vdso_info, version int32) {
        }
 }
 
-func archauxv(tag, val uintptr) {
+func vdsoauxv(tag, val uintptr) {
        switch tag {
        case _AT_SYSINFO_EHDR:
                if val == 0 {

diff --git a/src/runtime/vdso_linux_arm.go b/src/runtime/vdso_linux_arm.go
new file mode 100644 (file)
index 0000000..0f5fd5a
--- /dev/null
+++ b/src/runtime/vdso_linux_arm.go
@@ -0,0 +1,19 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+const (
+       // vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+       // See cmd/compile/internal/arm/galign.go arch.MAXWIDTH initialization, but must also
+       // be constrained to max +ve int.
+       vdsoArrayMax = 1<<31 - 1
+)
+
+var sym_keys = []symbol_key{
+       {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &__vdso_clock_gettime_sym},
+}
+
+// initialize to fall back to syscall
+var __vdso_clock_gettime_sym uintptr = 0

diff --git a/src/runtime/vdso_linux_test.go b/src/runtime/vdso_linux_test.go
index f507ee98ee9ff9dd4fa9743192c599915fcaf798..890f907d22aa432479a317416b2f1c4dad028b00 100644 (file)
--- a/src/runtime/vdso_linux_test.go
+++ b/src/runtime/vdso_linux_test.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386 amd64
+// +build 386 amd64 arm
 
 package runtime_test