From 876d477b0e3b27ae136a7e8dfceaf0ec72bddea6 Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Thu, 4 Nov 2021 17:21:13 -0400 Subject: [PATCH] runtime: refactor ARM VDSO call setup to helper We have a very complex process to make VDSO calls on ARM. Create a wrapper helper function which reduces duplication and allows for additional calls from other packages. vdsoCall has a few differences from the original code in walltime/nanotime: * It does not use R0-R3, as they are passed through as arguments to fn. * It does not save g if g.m.gsignal.stack.lo is zero. This may occur if it called at startup on g0 between assigning g0.m.gsignal and setting its stack. For #49182 Change-Id: I51aca514b4835b71142011341d2f09125334d30f Reviewed-on: https://go-review.googlesource.com/c/go/+/362795 Run-TryBot: Michael Pratt Reviewed-by: Michael Knyszek Reviewed-by: Cherry Mui TryBot-Result: Gopher Robot --- src/runtime/os_linux_arm.go | 2 + src/runtime/sys_linux_arm.s | 205 ++++++++++++++---------------------- 2 files changed, 82 insertions(+), 125 deletions(-) diff --git a/src/runtime/os_linux_arm.go b/src/runtime/os_linux_arm.go index b590da750f..bd3ab44a11 100644 --- a/src/runtime/os_linux_arm.go +++ b/src/runtime/os_linux_arm.go @@ -11,6 +11,8 @@ const ( _HWCAP_VFPv3 = 1 << 13 // introduced in 2.6.30 ) +func vdsoCall() + func checkgoarm() { // On Android, /proc/self/auxv might be unreadable and hwcap won't // reflect the CPU capabilities. Assume that every Android arm device diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index 66bf40328e..9ac64328c1 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -262,167 +262,132 @@ TEXT runtime·mincore(SB),NOSPLIT,$0 MOVW R0, ret+12(FP) RET -TEXT runtime·walltime(SB),NOSPLIT,$8-12 +// Call a VDSO function. +// +// R0-R3: arguments to VDSO function (C calling convention) +// R4: uintptr function to call +// +// There is no return value. +TEXT runtime·vdsoCall(SB),NOSPLIT,$8-0 + // R0-R3 may be arguments to fn, do not touch. + // R4 is function to call. + // R5-R9 are available as locals. They are unchanged by the C call + // (callee-save). + // We don't know how much stack space the VDSO code will need, // so switch to g0. // Save old SP. Use R13 instead of SP to avoid linker rewriting the offsets. - MOVW R13, R4 // R4 is unchanged by C code. + MOVW R13, R5 - MOVW g_m(g), R5 // R5 is unchanged by C code. + MOVW g_m(g), R6 // Set vdsoPC and vdsoSP for SIGPROF traceback. // Save the old values on stack and restore them on exit, // so this function is reentrant. - MOVW m_vdsoPC(R5), R1 - MOVW m_vdsoSP(R5), R2 - MOVW R1, 4(R13) - MOVW R2, 8(R13) + MOVW m_vdsoPC(R6), R7 + MOVW m_vdsoSP(R6), R8 + MOVW R7, 4(R13) + MOVW R8, 8(R13) - MOVW $ret-4(FP), R2 // caller's SP - MOVW LR, m_vdsoPC(R5) - MOVW R2, m_vdsoSP(R5) + MOVW $sp-4(FP), R7 // caller's SP + MOVW LR, m_vdsoPC(R6) + MOVW R7, m_vdsoSP(R6) - MOVW m_curg(R5), R0 + MOVW m_curg(R6), R7 - CMP g, R0 // Only switch if on curg. + CMP g, R7 // Only switch if on curg. B.NE noswitch - MOVW m_g0(R5), R0 - MOVW (g_sched+gobuf_sp)(R0), R13 // Set SP to g0 stack + MOVW m_g0(R6), R7 + MOVW (g_sched+gobuf_sp)(R7), R13 // Set SP to g0 stack noswitch: - SUB $24, R13 // Space for results BIC $0x7, R13 // Align for C code - MOVW $CLOCK_REALTIME, R0 - MOVW $8(R13), R1 // timespec - MOVW runtime·vdsoClockgettimeSym(SB), R2 - CMP $0, R2 - B.EQ fallback - // Store g on gsignal's stack, so if we receive a signal // during VDSO code we can find the g. - // If we don't have a signal stack, we won't receive signal, - // so don't bother saving g. - // When using cgo, we already saved g on TLS, also don't save - // g here. - // Also don't save g if we are already on the signal stack. - // We won't get a nested signal. - MOVB runtime·iscgo(SB), R6 - CMP $0, R6 + + // When using cgo, we already saved g on TLS, also don't save g here. + MOVB runtime·iscgo(SB), R7 + CMP $0, R7 BNE nosaveg - MOVW m_gsignal(R5), R6 // g.m.gsignal - CMP $0, R6 + // If we don't have a signal stack, we won't receive signal, so don't + // bother saving g. + MOVW m_gsignal(R6), R7 // g.m.gsignal + CMP $0, R7 + BEQ nosaveg + // Don't save g if we are already on the signal stack, as we won't get + // a nested signal. + CMP g, R7 BEQ nosaveg - CMP g, R6 + // If we don't have a signal stack, we won't receive signal, so don't + // bother saving g. + MOVW (g_stack+stack_lo)(R7), R7 // g.m.gsignal.stack.lo + CMP $0, R7 BEQ nosaveg - MOVW (g_stack+stack_lo)(R6), R6 // g.m.gsignal.stack.lo - MOVW g, (R6) + MOVW g, (R7) - BL (R2) + BL (R4) - MOVW $0, R1 - MOVW R1, (R6) // clear g slot, R6 is unchanged by C code + MOVW $0, R8 + MOVW R8, (R7) // clear g slot JMP finish nosaveg: - BL (R2) - JMP finish - -fallback: - MOVW $SYS_clock_gettime, R7 - SWI $0 + BL (R4) finish: - MOVW 8(R13), R0 // sec - MOVW 12(R13), R2 // nsec - - MOVW R4, R13 // Restore real SP + MOVW R5, R13 // Restore real SP // Restore vdsoPC, vdsoSP // We don't worry about being signaled between the two stores. // If we are not in a signal handler, we'll restore vdsoSP to 0, // and no one will care about vdsoPC. If we are in a signal handler, // we cannot receive another signal. - MOVW 8(R13), R1 - MOVW R1, m_vdsoSP(R5) - MOVW 4(R13), R1 - MOVW R1, m_vdsoPC(R5) - - MOVW R0, sec_lo+0(FP) - MOVW $0, R1 - MOVW R1, sec_hi+4(FP) - MOVW R2, nsec+8(FP) + MOVW 8(R13), R7 + MOVW R7, m_vdsoSP(R6) + MOVW 4(R13), R7 + MOVW R7, m_vdsoPC(R6) RET -// int64 nanotime1(void) -TEXT runtime·nanotime1(SB),NOSPLIT,$8-8 - // Switch to g0 stack. See comment above in runtime·walltime. - - // Save old SP. Use R13 instead of SP to avoid linker rewriting the offsets. - MOVW R13, R4 // R4 is unchanged by C code. - - MOVW g_m(g), R5 // R5 is unchanged by C code. +TEXT runtime·walltime(SB),NOSPLIT,$12-12 + MOVW $CLOCK_REALTIME, R0 + MOVW $spec-12(SP), R1 // timespec - // Set vdsoPC and vdsoSP for SIGPROF traceback. - // Save the old values on stack and restore them on exit, - // so this function is reentrant. - MOVW m_vdsoPC(R5), R1 - MOVW m_vdsoSP(R5), R2 - MOVW R1, 4(R13) - MOVW R2, 8(R13) + MOVW runtime·vdsoClockgettimeSym(SB), R4 + CMP $0, R4 + B.EQ fallback - MOVW $ret-4(FP), R2 // caller's SP - MOVW LR, m_vdsoPC(R5) - MOVW R2, m_vdsoSP(R5) + BL runtime·vdsoCall(SB) - MOVW m_curg(R5), R0 + JMP finish - CMP g, R0 // Only switch if on curg. - B.NE noswitch +fallback: + MOVW $SYS_clock_gettime, R7 + SWI $0 - MOVW m_g0(R5), R0 - MOVW (g_sched+gobuf_sp)(R0), R13 // Set SP to g0 stack +finish: + MOVW sec-12(SP), R0 // sec + MOVW nsec-8(SP), R2 // nsec -noswitch: - SUB $24, R13 // Space for results - BIC $0x7, R13 // Align for C code + MOVW R0, sec_lo+0(FP) + MOVW $0, R1 + MOVW R1, sec_hi+4(FP) + MOVW R2, nsec+8(FP) + RET +// func nanotime1() int64 +TEXT runtime·nanotime1(SB),NOSPLIT,$12-8 MOVW $CLOCK_MONOTONIC, R0 - MOVW $8(R13), R1 // timespec - MOVW runtime·vdsoClockgettimeSym(SB), R2 - CMP $0, R2 - B.EQ fallback - - // Store g on gsignal's stack, so if we receive a signal - // during VDSO code we can find the g. - // If we don't have a signal stack, we won't receive signal, - // so don't bother saving g. - // When using cgo, we already saved g on TLS, also don't save - // g here. - // Also don't save g if we are already on the signal stack. - // We won't get a nested signal. - MOVB runtime·iscgo(SB), R6 - CMP $0, R6 - BNE nosaveg - MOVW m_gsignal(R5), R6 // g.m.gsignal - CMP $0, R6 - BEQ nosaveg - CMP g, R6 - BEQ nosaveg - MOVW (g_stack+stack_lo)(R6), R6 // g.m.gsignal.stack.lo - MOVW g, (R6) - - BL (R2) + MOVW $spec-12(SP), R1 // timespec - MOVW $0, R1 - MOVW R1, (R6) // clear g slot, R6 is unchanged by C code + MOVW runtime·vdsoClockgettimeSym(SB), R4 + CMP $0, R4 + B.EQ fallback - JMP finish + BL runtime·vdsoCall(SB) -nosaveg: - BL (R2) JMP finish fallback: @@ -430,19 +395,8 @@ fallback: SWI $0 finish: - MOVW 8(R13), R0 // sec - MOVW 12(R13), R2 // nsec - - MOVW R4, R13 // Restore real SP - // Restore vdsoPC, vdsoSP - // We don't worry about being signaled between the two stores. - // If we are not in a signal handler, we'll restore vdsoSP to 0, - // and no one will care about vdsoPC. If we are in a signal handler, - // we cannot receive another signal. - MOVW 8(R13), R4 - MOVW R4, m_vdsoSP(R5) - MOVW 4(R13), R4 - MOVW R4, m_vdsoPC(R5) + MOVW sec-12(SP), R0 // sec + MOVW nsec-8(SP), R2 // nsec MOVW $1000000000, R3 MULLU R0, R3, (R1, R0) @@ -451,6 +405,7 @@ finish: MOVW R0, ret_lo+0(FP) MOVW R1, ret_hi+4(FP) + RET // int32 futex(int32 *uaddr, int32 op, int32 val, -- 2.50.0