runtime: improve 386/amd64 systemstack

Minor improvements, noticed while investigating other things.

Shorten the prologue.

Make branch direction better for static branch prediction;
the most common case by far is switching stacks (g==curg).

Change-Id: Ib2211d3efecb60446355cda56194221ccb78057d
Reviewed-on: https://go-review.googlesource.com/97377
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index ee6d768c234b4e090606099a0381c0a3bc85b203..a8de5976acd0d522a6b0e4a4f75e2d00a5436e46 100644 (file)
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -424,24 +424,17 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-4
        MOVL    g(CX), AX       // AX = g
        MOVL    g_m(AX), BX     // BX = m
 
-       MOVL    m_gsignal(BX), DX       // DX = gsignal
-       CMPL    AX, DX
+       CMPL    AX, m_gsignal(BX)
        JEQ     noswitch
 
        MOVL    m_g0(BX), DX    // DX = g0
        CMPL    AX, DX
        JEQ     noswitch
 
-       MOVL    m_curg(BX), BP
-       CMPL    AX, BP
-       JEQ     switch
-       
-       // Bad: g is not gsignal, not g0, not curg. What is it?
-       // Hide call from linker nosplit analysis.
-       MOVL    $runtime·badsystemstack(SB), AX
-       CALL    AX
+       CMPL    AX, m_curg(BX)
+       JNE     bad
 
-switch:
+       // switch stacks
        // save our state in g->sched. Pretend to
        // be systemstack_switch if the G stack is scanned.
        MOVL    $runtime·systemstack_switch(SB), (g_sched+gobuf_pc)(AX)
@@ -481,6 +474,12 @@ noswitch:
        MOVL    0(DI), DI
        JMP     DI
 
+bad:
+       // Bad: g is not gsignal, not g0, not curg. What is it?
+       // Hide call from linker nosplit analysis.
+       MOVL    $runtime·badsystemstack(SB), AX
+       CALL    AX
+
 /*
  * support for morestack
  */

diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index 82b7832ae3e1f46a2e8f0a676e3940ff3ef19590..2376fe0aae10a45cf309d29e9f672ed3e0e6c0dd 100644 (file)
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -369,23 +369,17 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8
        MOVQ    g(CX), AX       // AX = g
        MOVQ    g_m(AX), BX     // BX = m
 
-       MOVQ    m_gsignal(BX), DX       // DX = gsignal
-       CMPQ    AX, DX
+       CMPQ    AX, m_gsignal(BX)
        JEQ     noswitch
 
        MOVQ    m_g0(BX), DX    // DX = g0
        CMPQ    AX, DX
        JEQ     noswitch
 
-       MOVQ    m_curg(BX), R8
-       CMPQ    AX, R8
-       JEQ     switch
-       
-       // Bad: g is not gsignal, not g0, not curg. What is it?
-       MOVQ    $runtime·badsystemstack(SB), AX
-       CALL    AX
+       CMPQ    AX, m_curg(BX)
+       JNE     bad
 
-switch:
+       // switch stacks
        // save our state in g->sched. Pretend to
        // be systemstack_switch if the G stack is scanned.
        MOVQ    $runtime·systemstack_switch(SB), SI
@@ -426,6 +420,12 @@ noswitch:
        MOVQ    0(DI), DI
        JMP     DI
 
+bad:
+       // Bad: g is not gsignal, not g0, not curg. What is it?
+       MOVQ    $runtime·badsystemstack(SB), AX
+       CALL    AX
+
+
 /*
  * support for morestack
  */

diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s
index cdf2cd7f008609331146ad1caa1c4253573c471f..a59ba6ad914fdf66fac10417f458b59669519056 100644 (file)
--- a/src/runtime/asm_amd64p32.s
+++ b/src/runtime/asm_amd64p32.s
@@ -260,24 +260,17 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-4
        MOVL    g(CX), AX       // AX = g
        MOVL    g_m(AX), BX     // BX = m
 
-       MOVL    m_gsignal(BX), DX       // DX = gsignal
-       CMPL    AX, DX
+       CMPL    AX, m_gsignal(BX)
        JEQ     noswitch
 
        MOVL    m_g0(BX), DX    // DX = g0
        CMPL    AX, DX
        JEQ     noswitch
 
-       MOVL    m_curg(BX), R8
-       CMPL    AX, R8
-       JEQ     switch
-       
-       // Not g0, not curg. Must be gsignal, but that's not allowed.
-       // Hide call from linker nosplit analysis.
-       MOVL    $runtime·badsystemstack(SB), AX
-       CALL    AX
+       CMPL    AX, m_curg(BX)
+       JNE     bad
 
-switch:
+       // switch stacks
        // save our state in g->sched. Pretend to
        // be systemstack_switch if the G stack is scanned.
        MOVL    $runtime·systemstack_switch(SB), SI
@@ -312,6 +305,12 @@ noswitch:
        MOVL    0(DI), DI
        JMP     DI
 
+bad:
+       // Not g0, not curg. Must be gsignal, but that's not allowed.
+       // Hide call from linker nosplit analysis.
+       MOVL    $runtime·badsystemstack(SB), AX
+       CALL    AX
+
 /*
  * support for morestack
  */