runtime: enable vDSO support for s390x architecture

This change adds support for vDSO for s390x architecture. This avoids the use of system calls in nanotime and walltime and accelerates them by factor 4-5.

Benchmarks:
100,000,000 x time.Now():
syscall fallback	13923ms		139.23 ns/op
vDSO enabled		2640ms	 	26.40 ns/op

Change-Id: Ic679fe31048379e59ccf83b400140f13c9d49696
GitHub-Last-Rev: 8f6e918a45cf8c5aadc5c203949a8ce4e372086f
GitHub-Pull-Request: golang/go#49717
Reviewed-on: https://go-review.googlesource.com/c/go/+/365995
Run-TryBot: Paul Murphy <murp@ibm.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: Paul Murphy <murp@ibm.com>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Jonathan Albrecht <jonathan.albrecht@ibm.com>
Reviewed-by: Heschi Kreinick <heschi@google.com>
Reviewed-by: Bill O'Farrell <billotosyr@gmail.com>

diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go
index b06716dc6a1c7e1e568800ecaddfcb0c8e0138db..1882b90c5e708ef84b5f7764bb4cb1df6db7e1fc 100644 (file)
--- a/src/runtime/os_linux_novdso.go
+++ b/src/runtime/os_linux_novdso.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64
+//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64 && !s390x
 
 package runtime
 

diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go
index 66a5c941a38307ade791ac13d40c19ccb518e2be..69a4103948cb2b07c2bd979cd736f76e69e8821a 100644 (file)
--- a/src/runtime/signal_unix.go
+++ b/src/runtime/signal_unix.go
@@ -397,7 +397,7 @@ func preemptM(mp *m) {
 //go:nosplit
 func sigFetchG(c *sigctxt) *g {
        switch GOARCH {
-       case "arm", "arm64", "ppc64", "ppc64le", "riscv64":
+       case "arm", "arm64", "ppc64", "ppc64le", "riscv64", "s390x":
                if !iscgo && inVDSOPage(c.sigpc()) {
                        // When using cgo, we save the g on TLS and load it from there
                        // in sigtramp. Just use that.

diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s
index 91ce1b3c8db4d3fc5ae4eb527344e2dd6a823b7d..c82cb9b4aab12634de4e39d607b4a66f5bd35c5d 100644 (file)
--- a/src/runtime/sys_linux_s390x.s
+++ b/src/runtime/sys_linux_s390x.s
@@ -215,29 +215,177 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28
        RET
 
 // func walltime() (sec int64, nsec int32)
-TEXT runtime·walltime(SB),NOSPLIT,$16
-       MOVW    $0, R2 // CLOCK_REALTIME
+TEXT runtime·walltime(SB),NOSPLIT,$32-12
+       MOVW    $0, R2                  // CLOCK_REALTIME
+       MOVD    R15, R7                 // Backup stack pointer
+
+       MOVD    g_m(g), R6              //m
+
+       MOVD    runtime·vdsoClockgettimeSym(SB), R9    // Check for VDSO availability
+       CMPBEQ  R9, $0, fallback
+
+       MOVD    m_vdsoPC(R6), R4
+       MOVD    R4, 16(R15)
+       MOVD    m_vdsoSP(R6), R4
+       MOVD    R4, 24(R15)
+
+       MOVD    R14, R8                 // Backup return address
+       MOVD    $sec+0(FP), R4  // return parameter caller
+
+       MOVD    R8, m_vdsoPC(R6)
+       MOVD    R4, m_vdsoSP(R6)
+
+       MOVD    m_curg(R6), R5
+       CMP             g, R5
+       BNE             noswitch
+
+       MOVD    m_g0(R6), R4
+       MOVD    (g_sched+gobuf_sp)(R4), R15     // Set SP to g0 stack
+
+noswitch:
+       SUB             $16, R15                // reserve 2x 8 bytes for parameters
+       MOVD    $~7, R4                 // align to 8 bytes because of gcc ABI
+       AND             R4, R15
+       MOVD    R15, R3                 // R15 needs to be in R3 as expected by kernel_clock_gettime
+
+       MOVB    runtime·iscgo(SB),R12
+       CMPBNE  R12, $0, nosaveg
+
+       MOVD    m_gsignal(R6), R12      // g.m.gsignal
+       CMPBEQ  R12, $0, nosaveg
+
+       CMPBEQ  g, R12, nosaveg
+       MOVD    (g_stack+stack_lo)(R12), R12 // g.m.gsignal.stack.lo
+       MOVD    g, (R12)
+
+       BL      R9 // to vdso lookup
+
+       MOVD    $0, (R12)
+
+       JMP     finish
+
+nosaveg:
+       BL      R9                                      // to vdso lookup
+
+finish:
+       MOVD    0(R15), R3              // sec
+       MOVD    8(R15), R5              // nsec
+       MOVD    R7, R15                 // Restore SP
+
+       // Restore vdsoPC, vdsoSP
+       // We don't worry about being signaled between the two stores.
+       // If we are not in a signal handler, we'll restore vdsoSP to 0,
+       // and no one will care about vdsoPC. If we are in a signal handler,
+       // we cannot receive another signal.
+       MOVD    24(R15), R12
+       MOVD    R12, m_vdsoSP(R6)
+       MOVD    16(R15), R12
+       MOVD    R12, m_vdsoPC(R6)
+
+return:
+       // sec is in R3, nsec in R5
+       // return nsec in R3
+       MOVD    R3, sec+0(FP)
+       MOVW    R5, nsec+8(FP)
+       RET
+
+       // Syscall fallback
+fallback:
        MOVD    $tp-16(SP), R3
        MOVW    $SYS_clock_gettime, R1
        SYSCALL
-       LMG     tp-16(SP), R2, R3
+       LMG             tp-16(SP), R2, R3
        // sec is in R2, nsec in R3
        MOVD    R2, sec+0(FP)
        MOVW    R3, nsec+8(FP)
        RET
 
-TEXT runtime·nanotime1(SB),NOSPLIT,$16
-       MOVW    $1, R2 // CLOCK_MONOTONIC
+TEXT runtime·nanotime1(SB),NOSPLIT,$32-8
+       MOVW    $1, R2                  // CLOCK_MONOTONIC
+
+       MOVD    R15, R7                 // Backup stack pointer
+
+       MOVD    g_m(g), R6              //m
+
+       MOVD    runtime·vdsoClockgettimeSym(SB), R9    // Check for VDSO availability
+       CMPBEQ  R9, $0, fallback
+
+       MOVD    m_vdsoPC(R6), R4
+       MOVD    R4, 16(R15)
+       MOVD    m_vdsoSP(R6), R4
+       MOVD    R4, 24(R15)
+
+       MOVD    R14, R8                 // Backup return address
+       MOVD    $ret+0(FP), R4  // caller's SP
+
+       MOVD    R8, m_vdsoPC(R6)
+       MOVD    R4, m_vdsoSP(R6)
+
+       MOVD    m_curg(R6), R5
+       CMP             g, R5
+       BNE             noswitch
+
+       MOVD    m_g0(R6), R4
+       MOVD    (g_sched+gobuf_sp)(R4), R15     // Set SP to g0 stack
+
+noswitch:
+       SUB             $16, R15                // reserve 2x 8 bytes for parameters
+       MOVD    $~7, R4                 // align to 8 bytes because of gcc ABI
+       AND             R4, R15
+       MOVD    R15, R3                 // R15 needs to be in R3 as expected by kernel_clock_gettime
+
+       MOVB    runtime·iscgo(SB),R12
+       CMPBNE  R12, $0, nosaveg
+
+       MOVD    m_gsignal(R6), R12      // g.m.gsignal
+       CMPBEQ  R12, $0, nosaveg
+
+       CMPBEQ  g, R12, nosaveg
+       MOVD    (g_stack+stack_lo)(R12), R12    // g.m.gsignal.stack.lo
+       MOVD    g, (R12)
+
+       BL      R9                                      // to vdso lookup
+
+       MOVD $0, (R12)
+
+       JMP     finish
+
+nosaveg:
+       BL      R9                                      // to vdso lookup
+
+finish:
+       MOVD    0(R15), R3              // sec
+       MOVD    8(R15), R5              // nsec
+       MOVD    R7, R15                 // Restore SP
+
+       // Restore vdsoPC, vdsoSP
+       // We don't worry about being signaled between the two stores.
+       // If we are not in a signal handler, we'll restore vdsoSP to 0,
+       // and no one will care about vdsoPC. If we are in a signal handler,
+       // we cannot receive another signal.
+
+       MOVD    24(R15), R12
+       MOVD    R12, m_vdsoSP(R6)
+       MOVD    16(R15), R12
+       MOVD    R12, m_vdsoPC(R6)
+
+return:
+       // sec is in R3, nsec in R5
+       // return nsec in R3
+       MULLD   $1000000000, R3
+       ADD             R5, R3
+       MOVD    R3, ret+0(FP)
+       RET
+
+       // Syscall fallback
+fallback:
        MOVD    $tp-16(SP), R3
-       MOVW    $SYS_clock_gettime, R1
+       MOVD    $SYS_clock_gettime, R1
        SYSCALL
-       LMG     tp-16(SP), R2, R3
-       // sec is in R2, nsec in R3
-       // return nsec in R2
-       MULLD   $1000000000, R2
-       ADD     R3, R2
-       MOVD    R2, ret+0(FP)
-       RET
+       LMG             tp-16(SP), R2, R3
+       MOVD    R3, R5
+       MOVD    R2, R3
+       JMP     return
 
 TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28
        MOVW    how+0(FP), R2

diff --git a/src/runtime/vdso_elf64.go b/src/runtime/vdso_elf64.go
index d46d6f8c34da7599191832ab451830b18031763d..b50e58f333def70eec2c3b182032ecb6e9841c0d 100644 (file)
--- a/src/runtime/vdso_elf64.go
+++ b/src/runtime/vdso_elf64.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64)
+//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x)
 
 package runtime
 

diff --git a/src/runtime/vdso_in_none.go b/src/runtime/vdso_in_none.go
index 618bd39b42b0f79a82cba8632d758e8dd142729c..a11ecb00efd449695c26cc7adb3c63f8ec3e035c 100644 (file)
--- a/src/runtime/vdso_in_none.go
+++ b/src/runtime/vdso_in_none.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64) || !linux
+//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64 && !s390x) || !linux
 
 package runtime
 

diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go
index 2ebdd44e94d481020b99a0478ac01219a2cd370e..36b9f426c1b0b60fe0ffddf259ea191a0cca5c68 100644 (file)
--- a/src/runtime/vdso_linux.go
+++ b/src/runtime/vdso_linux.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64)
+//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x)
 
 package runtime
 
@@ -232,9 +232,11 @@ func vdsoParseSymbols(info *vdsoInfo, version int32) {
        if !info.isGNUHash {
                // Old-style DT_HASH table.
                for _, k := range vdsoSymbolKeys {
-                       for chain := info.bucket[k.symHash%uint32(len(info.bucket))]; chain != 0; chain = info.chain[chain] {
-                               if apply(chain, k) {
-                                       break
+                       if len(info.bucket) > 0 {
+                               for chain := info.bucket[k.symHash%uint32(len(info.bucket))]; chain != 0; chain = info.chain[chain] {
+                                       if apply(chain, k) {
+                                               break
+                                       }
                                }
                        }
                }

diff --git a/src/runtime/vdso_linux_s390x.go b/src/runtime/vdso_linux_s390x.go
new file mode 100644 (file)
index 0000000..85f78bb
--- /dev/null
+++ b/src/runtime/vdso_linux_s390x.go
@@ -0,0 +1,26 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && s390x
+// +build linux
+// +build s390x
+
+package runtime
+
+const (
+       // vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+       // See cmd/compile/internal/s390x/galign.go arch.MAXWIDTH initialization.
+       vdsoArrayMax = 1<<50 - 1
+)
+
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.29", 0x75fcbb9}
+
+var vdsoSymbolKeys = []vdsoSymbolKey{
+       {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym},
+}
+
+// initialize with vsyscall fallbacks
+var (
+       vdsoClockgettimeSym uintptr = 0
+)