runtime: make asmcgocall g0/gsignal checks consistent

In asmcgocall() we need to switch to the g0 stack if we're not already on
the g0 stack or the gsignal stack. The prefered way of doing this is to
check gsignal first, then g0, since if we are going to switch to g0 we will
need g0 handy (thus avoiding a second load).

Rewrite/reorder 386 and amd64 to check gsignal first - this shaves a few
assembly instructions off and makes the order consistent with arm, arm64,
mips64 and ppc64. Add missing gsignal checks to mips, riscv64 and s390x.

Change-Id: I1b027bf393c25e0c33e1d8eb80de67e4a0a3f561
Reviewed-on: https://go-review.googlesource.com/c/go/+/335869
Trust: Joel Sing <joel@sing.id.au>
Run-TryBot: Joel Sing <joel@sing.id.au>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Cherry Mui <cherryyz@google.com>

diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index 11c60309f4e07e55895bff83d97737fa8a1ba288..571aa28a9e1acb754f45b98a981c36c9afb9a49d 100644 (file)
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -633,18 +633,18 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        get_tls(CX)
-       MOVL    g(CX), BP
-       CMPL    BP, $0
-       JEQ     nosave  // Don't even have a G yet.
-       MOVL    g_m(BP), BP
-       MOVL    m_g0(BP), SI
        MOVL    g(CX), DI
-       CMPL    SI, DI
-       JEQ     noswitch
+       CMPL    DI, $0
+       JEQ     nosave  // Don't even have a G yet.
+       MOVL    g_m(DI), BP
        CMPL    DI, m_gsignal(BP)
        JEQ     noswitch
+       MOVL    m_g0(BP), SI
+       CMPL    DI, SI
+       JEQ     noswitch
        CALL    gosave_systemstack_switch<>(SB)
        get_tls(CX)
        MOVL    SI, g(CX)

diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index 2083ecb53e8bfc40d612df2f17fcdd786c91e89c..a6b321aa4250ab746d67d1f283d093764be16fcf 100644 (file)
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -667,22 +667,21 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        get_tls(CX)
-       MOVQ    g(CX), R8
-       CMPQ    R8, $0
-       JEQ     nosave
-       MOVQ    g_m(R8), R8
-       MOVQ    m_g0(R8), SI
        MOVQ    g(CX), DI
-       CMPQ    SI, DI
+       CMPQ    DI, $0
        JEQ     nosave
+       MOVQ    g_m(DI), R8
        MOVQ    m_gsignal(R8), SI
-       CMPQ    SI, DI
+       CMPQ    DI, SI
+       JEQ     nosave
+       MOVQ    m_g0(R8), SI
+       CMPQ    DI, SI
        JEQ     nosave
 
        // Switch to system stack.
-       MOVQ    m_g0(R8), SI
        CALL    gosave_systemstack_switch<>(SB)
        MOVQ    SI, g(CX)
        MOVQ    (g_sched+gobuf_sp)(SI), SP

diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s
index a1164781d2eff652e8eafd1d523c44d329dd192e..b47184e36bc9801e1c2c2080b9a9ef629122304a 100644 (file)
--- a/src/runtime/asm_arm.s
+++ b/src/runtime/asm_arm.s
@@ -556,7 +556,8 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        MOVW    g_m(g), R8
        MOVW    m_gsignal(R8), R3
        CMP     R3, g

diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s
index e51ce2f8316b4010b952f7b4e5f99eb7b1a2b31d..8cbd17fa7506df985bfc59cf25c411a9bfc6ee6e 100644 (file)
--- a/src/runtime/asm_arm64.s
+++ b/src/runtime/asm_arm64.s
@@ -1027,7 +1027,8 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        MOVD    g_m(g), R8
        MOVD    m_gsignal(R8), R3
        CMP     R3, g

diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s
index 940a38a0d6fcfac2e020207c16f4be0075ed90a5..e0e5cbb7048c918de43d2696f7d692d707a60329 100644 (file)
--- a/src/runtime/asm_mips64x.s
+++ b/src/runtime/asm_mips64x.s
@@ -424,7 +424,8 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        MOVV    g_m(g), R5
        MOVV    m_gsignal(R5), R6
        BEQ     R6, g, g0

diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s
index 87a1344e8f6de574193f56089fe0dc0ffe4d97c9..1b550719d17eb4b5e3ef5ff320a814783315a960 100644 (file)
--- a/src/runtime/asm_mipsx.s
+++ b/src/runtime/asm_mipsx.s
@@ -413,8 +413,11 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        MOVW    g_m(g), R5
+       MOVW    m_gsignal(R5), R6
+       BEQ     R6, g, g0
        MOVW    m_g0(R5), R6
        BEQ     R6, g, g0
 

diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s
index 5dc96c594732ffc86899fb964914cc98e2a2cda5..7270abbdee211adcb863537677ef80188e34358c 100644 (file)
--- a/src/runtime/asm_ppc64x.s
+++ b/src/runtime/asm_ppc64x.s
@@ -541,9 +541,8 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
-       // Moreover, if it's called inside the signal handler, it must not switch
-       // to g0 as it can be in use by another syscall.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        MOVD    g_m(g), R8
        MOVD    m_gsignal(R8), R6
        CMP     R6, g

diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s
index 9927a817f7274a86761b711bce09b7b8b5282c6c..d7ab90d1abfbf4b7387367bc2a0a2214799f8eea 100644 (file)
--- a/src/runtime/asm_riscv64.s
+++ b/src/runtime/asm_riscv64.s
@@ -310,8 +310,11 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        MOV     g_m(g), X6
+       MOV     m_gsignal(X6), X7
+       BEQ     X7, g, g0
        MOV     m_g0(X6), X7
        BEQ     X7, g, g0
 

diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s
index d4110d563f444efc1142eaba94b0f0c76b2dfab0..5894fe5783113e6717233aaf1a09afcfebdce4d1 100644 (file)
--- a/src/runtime/asm_s390x.s
+++ b/src/runtime/asm_s390x.s
@@ -513,12 +513,15 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
-       // come in on the m->g0 stack already.
+       // come in on the m->g0 stack already. Or we might already
+       // be on the m->gsignal stack.
        MOVD    g_m(g), R6
-       MOVD    m_g0(R6), R6
-       CMPBEQ  R6, g, g0
+       MOVD    m_gsignal(R6), R7
+       CMPBEQ  R7, g, g0
+       MOVD    m_g0(R6), R7
+       CMPBEQ  R7, g, g0
        BL      gosave_systemstack_switch<>(SB)
-       MOVD    R6, g
+       MOVD    R7, g
        BL      runtime·save_g(SB)
        MOVD    (g_sched+gobuf_sp)(g), R15