From: Axel Busch <94176305+abuschIBM@users.noreply.github.com> Date: Tue, 10 May 2022 13:48:19 +0000 (+0000) Subject: runtime: enable vDSO support for s390x architecture X-Git-Tag: go1.19beta1~307 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=636c5f02082e4be5ef2d9a66c99c9e31b72246a3;p=gostls13.git runtime: enable vDSO support for s390x architecture This change adds support for vDSO for s390x architecture. This avoids the use of system calls in nanotime and walltime and accelerates them by factor 4-5. Benchmarks: 100,000,000 x time.Now(): syscall fallback 13923ms 139.23 ns/op vDSO enabled 2640ms 26.40 ns/op Change-Id: Ic679fe31048379e59ccf83b400140f13c9d49696 GitHub-Last-Rev: 8f6e918a45cf8c5aadc5c203949a8ce4e372086f GitHub-Pull-Request: golang/go#49717 Reviewed-on: https://go-review.googlesource.com/c/go/+/365995 Run-TryBot: Paul Murphy TryBot-Result: Gopher Robot Reviewed-by: David Chase Reviewed-by: Paul Murphy Reviewed-by: Michael Knyszek Reviewed-by: Jonathan Albrecht Reviewed-by: Heschi Kreinick Reviewed-by: Bill O'Farrell --- diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go index b06716dc6a..1882b90c5e 100644 --- a/src/runtime/os_linux_novdso.go +++ b/src/runtime/os_linux_novdso.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64 +//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64 && !s390x package runtime diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index 66a5c941a3..69a4103948 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -397,7 +397,7 @@ func preemptM(mp *m) { //go:nosplit func sigFetchG(c *sigctxt) *g { switch GOARCH { - case "arm", "arm64", "ppc64", "ppc64le", "riscv64": + case "arm", "arm64", "ppc64", "ppc64le", "riscv64", "s390x": if !iscgo && inVDSOPage(c.sigpc()) { // When using cgo, we save the g on TLS and load it from there // in sigtramp. Just use that. diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s index 91ce1b3c8d..c82cb9b4aa 100644 --- a/src/runtime/sys_linux_s390x.s +++ b/src/runtime/sys_linux_s390x.s @@ -215,29 +215,177 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28 RET // func walltime() (sec int64, nsec int32) -TEXT runtime·walltime(SB),NOSPLIT,$16 - MOVW $0, R2 // CLOCK_REALTIME +TEXT runtime·walltime(SB),NOSPLIT,$32-12 + MOVW $0, R2 // CLOCK_REALTIME + MOVD R15, R7 // Backup stack pointer + + MOVD g_m(g), R6 //m + + MOVD runtime·vdsoClockgettimeSym(SB), R9 // Check for VDSO availability + CMPBEQ R9, $0, fallback + + MOVD m_vdsoPC(R6), R4 + MOVD R4, 16(R15) + MOVD m_vdsoSP(R6), R4 + MOVD R4, 24(R15) + + MOVD R14, R8 // Backup return address + MOVD $sec+0(FP), R4 // return parameter caller + + MOVD R8, m_vdsoPC(R6) + MOVD R4, m_vdsoSP(R6) + + MOVD m_curg(R6), R5 + CMP g, R5 + BNE noswitch + + MOVD m_g0(R6), R4 + MOVD (g_sched+gobuf_sp)(R4), R15 // Set SP to g0 stack + +noswitch: + SUB $16, R15 // reserve 2x 8 bytes for parameters + MOVD $~7, R4 // align to 8 bytes because of gcc ABI + AND R4, R15 + MOVD R15, R3 // R15 needs to be in R3 as expected by kernel_clock_gettime + + MOVB runtime·iscgo(SB),R12 + CMPBNE R12, $0, nosaveg + + MOVD m_gsignal(R6), R12 // g.m.gsignal + CMPBEQ R12, $0, nosaveg + + CMPBEQ g, R12, nosaveg + MOVD (g_stack+stack_lo)(R12), R12 // g.m.gsignal.stack.lo + MOVD g, (R12) + + BL R9 // to vdso lookup + + MOVD $0, (R12) + + JMP finish + +nosaveg: + BL R9 // to vdso lookup + +finish: + MOVD 0(R15), R3 // sec + MOVD 8(R15), R5 // nsec + MOVD R7, R15 // Restore SP + + // Restore vdsoPC, vdsoSP + // We don't worry about being signaled between the two stores. + // If we are not in a signal handler, we'll restore vdsoSP to 0, + // and no one will care about vdsoPC. If we are in a signal handler, + // we cannot receive another signal. + MOVD 24(R15), R12 + MOVD R12, m_vdsoSP(R6) + MOVD 16(R15), R12 + MOVD R12, m_vdsoPC(R6) + +return: + // sec is in R3, nsec in R5 + // return nsec in R3 + MOVD R3, sec+0(FP) + MOVW R5, nsec+8(FP) + RET + + // Syscall fallback +fallback: MOVD $tp-16(SP), R3 MOVW $SYS_clock_gettime, R1 SYSCALL - LMG tp-16(SP), R2, R3 + LMG tp-16(SP), R2, R3 // sec is in R2, nsec in R3 MOVD R2, sec+0(FP) MOVW R3, nsec+8(FP) RET -TEXT runtime·nanotime1(SB),NOSPLIT,$16 - MOVW $1, R2 // CLOCK_MONOTONIC +TEXT runtime·nanotime1(SB),NOSPLIT,$32-8 + MOVW $1, R2 // CLOCK_MONOTONIC + + MOVD R15, R7 // Backup stack pointer + + MOVD g_m(g), R6 //m + + MOVD runtime·vdsoClockgettimeSym(SB), R9 // Check for VDSO availability + CMPBEQ R9, $0, fallback + + MOVD m_vdsoPC(R6), R4 + MOVD R4, 16(R15) + MOVD m_vdsoSP(R6), R4 + MOVD R4, 24(R15) + + MOVD R14, R8 // Backup return address + MOVD $ret+0(FP), R4 // caller's SP + + MOVD R8, m_vdsoPC(R6) + MOVD R4, m_vdsoSP(R6) + + MOVD m_curg(R6), R5 + CMP g, R5 + BNE noswitch + + MOVD m_g0(R6), R4 + MOVD (g_sched+gobuf_sp)(R4), R15 // Set SP to g0 stack + +noswitch: + SUB $16, R15 // reserve 2x 8 bytes for parameters + MOVD $~7, R4 // align to 8 bytes because of gcc ABI + AND R4, R15 + MOVD R15, R3 // R15 needs to be in R3 as expected by kernel_clock_gettime + + MOVB runtime·iscgo(SB),R12 + CMPBNE R12, $0, nosaveg + + MOVD m_gsignal(R6), R12 // g.m.gsignal + CMPBEQ R12, $0, nosaveg + + CMPBEQ g, R12, nosaveg + MOVD (g_stack+stack_lo)(R12), R12 // g.m.gsignal.stack.lo + MOVD g, (R12) + + BL R9 // to vdso lookup + + MOVD $0, (R12) + + JMP finish + +nosaveg: + BL R9 // to vdso lookup + +finish: + MOVD 0(R15), R3 // sec + MOVD 8(R15), R5 // nsec + MOVD R7, R15 // Restore SP + + // Restore vdsoPC, vdsoSP + // We don't worry about being signaled between the two stores. + // If we are not in a signal handler, we'll restore vdsoSP to 0, + // and no one will care about vdsoPC. If we are in a signal handler, + // we cannot receive another signal. + + MOVD 24(R15), R12 + MOVD R12, m_vdsoSP(R6) + MOVD 16(R15), R12 + MOVD R12, m_vdsoPC(R6) + +return: + // sec is in R3, nsec in R5 + // return nsec in R3 + MULLD $1000000000, R3 + ADD R5, R3 + MOVD R3, ret+0(FP) + RET + + // Syscall fallback +fallback: MOVD $tp-16(SP), R3 - MOVW $SYS_clock_gettime, R1 + MOVD $SYS_clock_gettime, R1 SYSCALL - LMG tp-16(SP), R2, R3 - // sec is in R2, nsec in R3 - // return nsec in R2 - MULLD $1000000000, R2 - ADD R3, R2 - MOVD R2, ret+0(FP) - RET + LMG tp-16(SP), R2, R3 + MOVD R3, R5 + MOVD R2, R3 + JMP return TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28 MOVW how+0(FP), R2 diff --git a/src/runtime/vdso_elf64.go b/src/runtime/vdso_elf64.go index d46d6f8c34..b50e58f333 100644 --- a/src/runtime/vdso_elf64.go +++ b/src/runtime/vdso_elf64.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64) +//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x) package runtime diff --git a/src/runtime/vdso_in_none.go b/src/runtime/vdso_in_none.go index 618bd39b42..a11ecb00ef 100644 --- a/src/runtime/vdso_in_none.go +++ b/src/runtime/vdso_in_none.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64) || !linux +//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64 && !s390x) || !linux package runtime diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go index 2ebdd44e94..36b9f426c1 100644 --- a/src/runtime/vdso_linux.go +++ b/src/runtime/vdso_linux.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64) +//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x) package runtime @@ -232,9 +232,11 @@ func vdsoParseSymbols(info *vdsoInfo, version int32) { if !info.isGNUHash { // Old-style DT_HASH table. for _, k := range vdsoSymbolKeys { - for chain := info.bucket[k.symHash%uint32(len(info.bucket))]; chain != 0; chain = info.chain[chain] { - if apply(chain, k) { - break + if len(info.bucket) > 0 { + for chain := info.bucket[k.symHash%uint32(len(info.bucket))]; chain != 0; chain = info.chain[chain] { + if apply(chain, k) { + break + } } } } diff --git a/src/runtime/vdso_linux_s390x.go b/src/runtime/vdso_linux_s390x.go new file mode 100644 index 0000000000..85f78bbef2 --- /dev/null +++ b/src/runtime/vdso_linux_s390x.go @@ -0,0 +1,26 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && s390x +// +build linux +// +build s390x + +package runtime + +const ( + // vdsoArrayMax is the byte-size of a maximally sized array on this architecture. + // See cmd/compile/internal/s390x/galign.go arch.MAXWIDTH initialization. + vdsoArrayMax = 1<<50 - 1 +) + +var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.29", 0x75fcbb9} + +var vdsoSymbolKeys = []vdsoSymbolKey{ + {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym}, +} + +// initialize with vsyscall fallbacks +var ( + vdsoClockgettimeSym uintptr = 0 +)