From c5d6c42d35059bb911cf9f4a77704438ab4f9de0 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Sat, 24 Feb 2018 10:50:38 -0800 Subject: [PATCH] runtime: improve 386/amd64 systemstack Minor improvements, noticed while investigating other things. Shorten the prologue. Make branch direction better for static branch prediction; the most common case by far is switching stacks (g==curg). Change-Id: Ib2211d3efecb60446355cda56194221ccb78057d Reviewed-on: https://go-review.googlesource.com/97377 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/asm_386.s | 21 ++++++++++----------- src/runtime/asm_amd64.s | 20 ++++++++++---------- src/runtime/asm_amd64p32.s | 21 ++++++++++----------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index ee6d768c23..a8de5976ac 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -424,24 +424,17 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-4 MOVL g(CX), AX // AX = g MOVL g_m(AX), BX // BX = m - MOVL m_gsignal(BX), DX // DX = gsignal - CMPL AX, DX + CMPL AX, m_gsignal(BX) JEQ noswitch MOVL m_g0(BX), DX // DX = g0 CMPL AX, DX JEQ noswitch - MOVL m_curg(BX), BP - CMPL AX, BP - JEQ switch - - // Bad: g is not gsignal, not g0, not curg. What is it? - // Hide call from linker nosplit analysis. - MOVL $runtime·badsystemstack(SB), AX - CALL AX + CMPL AX, m_curg(BX) + JNE bad -switch: + // switch stacks // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. MOVL $runtime·systemstack_switch(SB), (g_sched+gobuf_pc)(AX) @@ -481,6 +474,12 @@ noswitch: MOVL 0(DI), DI JMP DI +bad: + // Bad: g is not gsignal, not g0, not curg. What is it? + // Hide call from linker nosplit analysis. + MOVL $runtime·badsystemstack(SB), AX + CALL AX + /* * support for morestack */ diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 82b7832ae3..2376fe0aae 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -369,23 +369,17 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 MOVQ g(CX), AX // AX = g MOVQ g_m(AX), BX // BX = m - MOVQ m_gsignal(BX), DX // DX = gsignal - CMPQ AX, DX + CMPQ AX, m_gsignal(BX) JEQ noswitch MOVQ m_g0(BX), DX // DX = g0 CMPQ AX, DX JEQ noswitch - MOVQ m_curg(BX), R8 - CMPQ AX, R8 - JEQ switch - - // Bad: g is not gsignal, not g0, not curg. What is it? - MOVQ $runtime·badsystemstack(SB), AX - CALL AX + CMPQ AX, m_curg(BX) + JNE bad -switch: + // switch stacks // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. MOVQ $runtime·systemstack_switch(SB), SI @@ -426,6 +420,12 @@ noswitch: MOVQ 0(DI), DI JMP DI +bad: + // Bad: g is not gsignal, not g0, not curg. What is it? + MOVQ $runtime·badsystemstack(SB), AX + CALL AX + + /* * support for morestack */ diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s index cdf2cd7f00..a59ba6ad91 100644 --- a/src/runtime/asm_amd64p32.s +++ b/src/runtime/asm_amd64p32.s @@ -260,24 +260,17 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-4 MOVL g(CX), AX // AX = g MOVL g_m(AX), BX // BX = m - MOVL m_gsignal(BX), DX // DX = gsignal - CMPL AX, DX + CMPL AX, m_gsignal(BX) JEQ noswitch MOVL m_g0(BX), DX // DX = g0 CMPL AX, DX JEQ noswitch - MOVL m_curg(BX), R8 - CMPL AX, R8 - JEQ switch - - // Not g0, not curg. Must be gsignal, but that's not allowed. - // Hide call from linker nosplit analysis. - MOVL $runtime·badsystemstack(SB), AX - CALL AX + CMPL AX, m_curg(BX) + JNE bad -switch: + // switch stacks // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. MOVL $runtime·systemstack_switch(SB), SI @@ -312,6 +305,12 @@ noswitch: MOVL 0(DI), DI JMP DI +bad: + // Not g0, not curg. Must be gsignal, but that's not allowed. + // Hide call from linker nosplit analysis. + MOVL $runtime·badsystemstack(SB), AX + CALL AX + /* * support for morestack */ -- 2.50.0