runtime: add deletion barriers on gobuf.ctxt

gobuf.ctxt is set to nil from many places in assembly code and these
assignments require write barriers with the hybrid barrier.

Conveniently, in most of these places ctxt should already be nil, in
which case we don't need the barrier. This commit changes these places
to assert that ctxt is already nil.

gogo is more complicated, since ctxt may not already be nil. For gogo,
we manually perform the write barrier if ctxt is not nil.

Updates #17503.

Change-Id: I9d75e27c75a1b7f8b715ad112fc5d45ffa856d30
Reviewed-on: https://go-review.googlesource.com/31764
Reviewed-by: Cherry Zhang <cherryyz@google.com>

diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index 68d1e512655ea7b2e777f7ba9e6cb5919147e8fe..3d0b74ce926e34996dba96b611a8ac001691ac5d 100644 (file)
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -209,7 +209,11 @@ TEXT runtime·gosave(SB), NOSPLIT, $0-4
        MOVL    0(SP), BX               // caller's PC
        MOVL    BX, gobuf_pc(AX)
        MOVL    $0, gobuf_ret(AX)
-       MOVL    $0, gobuf_ctxt(AX)
+       // Assert ctxt is zero. See func save.
+       MOVL    gobuf_ctxt(AX), BX
+       TESTL   BX, BX
+       JZ      2(PC)
+       CALL    runtime·badctxt(SB)
        get_tls(CX)
        MOVL    g(CX), BX
        MOVL    BX, gobuf_g(AX)
@@ -217,8 +221,20 @@ TEXT runtime·gosave(SB), NOSPLIT, $0-4
 
 // void gogo(Gobuf*)
 // restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB), NOSPLIT, $0-4
+TEXT runtime·gogo(SB), NOSPLIT, $8-4
        MOVL    buf+0(FP), BX           // gobuf
+
+       // If ctxt is not nil, invoke deletion barrier before overwriting.
+       MOVL    gobuf_ctxt(BX), DX
+       TESTL   DX, DX
+       JZ      nilctxt
+       LEAL    gobuf_ctxt(BX), AX
+       MOVL    AX, 0(SP)
+       MOVL    $0, 4(SP)
+       CALL    runtime·writebarrierptr_prewrite(SB)
+       MOVL    buf+0(FP), BX
+
+nilctxt:
        MOVL    gobuf_g(BX), DX
        MOVL    0(DX), CX               // make sure g != nil
        get_tls(CX)
@@ -572,7 +588,11 @@ TEXT gosave<>(SB),NOSPLIT,$0
        MOVL    -4(AX), AX
        MOVL    AX, (g_sched+gobuf_pc)(BX)
        MOVL    $0, (g_sched+gobuf_ret)(BX)
-       MOVL    $0, (g_sched+gobuf_ctxt)(BX)
+       // Assert ctxt is zero. See func save.
+       MOVL    (g_sched+gobuf_ctxt)(BX), AX
+       TESTL   AX, AX
+       JZ      2(PC)
+       CALL    runtime·badctxt(SB)
        POPL    BX
        POPL    AX
        RET

diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index bcc9cad655a18b46145abe543b908e6c413c73f2..9ffd297d842eab9b5ec98059c36b798137888969 100644 (file)
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -182,8 +182,12 @@ TEXT runtime·gosave(SB), NOSPLIT, $0-8
        MOVQ    0(SP), BX               // caller's PC
        MOVQ    BX, gobuf_pc(AX)
        MOVQ    $0, gobuf_ret(AX)
-       MOVQ    $0, gobuf_ctxt(AX)
        MOVQ    BP, gobuf_bp(AX)
+       // Assert ctxt is zero. See func save.
+       MOVQ    gobuf_ctxt(AX), BX
+       TESTQ   BX, BX
+       JZ      2(PC)
+       CALL    runtime·badctxt(SB)
        get_tls(CX)
        MOVQ    g(CX), BX
        MOVQ    BX, gobuf_g(AX)
@@ -191,8 +195,20 @@ TEXT runtime·gosave(SB), NOSPLIT, $0-8
 
 // void gogo(Gobuf*)
 // restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB), NOSPLIT, $0-8
+TEXT runtime·gogo(SB), NOSPLIT, $16-8
        MOVQ    buf+0(FP), BX           // gobuf
+
+       // If ctxt is not nil, invoke deletion barrier before overwriting.
+       MOVQ    gobuf_ctxt(BX), AX
+       TESTQ   AX, AX
+       JZ      nilctxt
+       LEAQ    gobuf_ctxt(BX), AX
+       MOVQ    AX, 0(SP)
+       MOVQ    $0, 8(SP)
+       CALL    runtime·writebarrierptr_prewrite(SB)
+       MOVQ    buf+0(FP), BX
+
+nilctxt:
        MOVQ    gobuf_g(BX), DX
        MOVQ    0(DX), CX               // make sure g != nil
        get_tls(CX)
@@ -546,8 +562,12 @@ TEXT gosave<>(SB),NOSPLIT,$0
        LEAQ    8(SP), R9
        MOVQ    R9, (g_sched+gobuf_sp)(R8)
        MOVQ    $0, (g_sched+gobuf_ret)(R8)
-       MOVQ    $0, (g_sched+gobuf_ctxt)(R8)
        MOVQ    BP, (g_sched+gobuf_bp)(R8)
+       // Assert ctxt is zero. See func save.
+       MOVQ    (g_sched+gobuf_ctxt)(R8), R9
+       TESTQ   R9, R9
+       JZ      2(PC)
+       CALL    runtime·badctxt(SB)
        RET
 
 // func asmcgocall(fn, arg unsafe.Pointer) int32

diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s
index ab73508a595451c708573fe9d4a48e21864a1b22..3081ca7b12dab91c9452d1a328fb618fbc0e9ea0 100644 (file)
--- a/src/runtime/asm_amd64p32.s
+++ b/src/runtime/asm_amd64p32.s
@@ -107,8 +107,12 @@ TEXT runtime·gosave(SB), NOSPLIT, $0-4
        MOVL    BX, gobuf_sp(AX)
        MOVL    0(SP), BX               // caller's PC
        MOVL    BX, gobuf_pc(AX)
-       MOVL    $0, gobuf_ctxt(AX)
        MOVQ    $0, gobuf_ret(AX)
+       // Assert ctxt is zero. See func save.
+       MOVL    gobuf_ctxt(AX), BX
+       TESTL   BX, BX
+       JZ      2(PC)
+       CALL    runtime·badctxt(SB)
        get_tls(CX)
        MOVL    g(CX), BX
        MOVL    BX, gobuf_g(AX)
@@ -116,8 +120,20 @@ TEXT runtime·gosave(SB), NOSPLIT, $0-4
 
 // void gogo(Gobuf*)
 // restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB), NOSPLIT, $0-4
+TEXT runtime·gogo(SB), NOSPLIT, $8-4
        MOVL    buf+0(FP), BX           // gobuf
+
+       // If ctxt is not nil, invoke deletion barrier before overwriting.
+       MOVL    gobuf_ctxt(BX), DX
+       TESTL   DX, DX
+       JZ      nilctxt
+       LEAL    gobuf_ctxt(BX), AX
+       MOVL    AX, 0(SP)
+       MOVL    $0, 4(SP)
+       CALL    runtime·writebarrierptr_prewrite(SB)
+       MOVL    buf+0(FP), BX
+
+nilctxt:
        MOVL    gobuf_g(BX), DX
        MOVL    0(DX), CX               // make sure g != nil
        get_tls(CX)

diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s
index aa7b74827a6b3f6ad92aa258f559af716b27a37b..79c28a81789f37fea6c5cf04490bd86b609f9e00 100644 (file)
--- a/src/runtime/asm_arm.s
+++ b/src/runtime/asm_arm.s
@@ -118,13 +118,30 @@ TEXT runtime·gosave(SB),NOSPLIT,$-4-4
        MOVW    $0, R11
        MOVW    R11, gobuf_lr(R0)
        MOVW    R11, gobuf_ret(R0)
-       MOVW    R11, gobuf_ctxt(R0)
+       // Assert ctxt is zero. See func save.
+       MOVW    gobuf_ctxt(R0), R0
+       CMP     R0, R11
+       B.EQ    2(PC)
+       CALL    runtime·badctxt(SB)
        RET
 
 // void gogo(Gobuf*)
 // restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB),NOSPLIT,$-4-4
+TEXT runtime·gogo(SB),NOSPLIT,$8-4
+       MOVW    buf+0(FP), R1
+
+       // If ctxt is not nil, invoke deletion barrier before overwriting.
+       MOVW    gobuf_ctxt(R1), R0
+       CMP     $0, R0
+       B.EQ    nilctxt
+       MOVW    $gobuf_ctxt(R1), R0
+       MOVW    R0, 4(R13)
+       MOVW    $0, R0
+       MOVW    R0, 8(R13)
+       BL      runtime·writebarrierptr_prewrite(SB)
        MOVW    buf+0(FP), R1
+
+nilctxt:
        MOVW    gobuf_g(R1), R0
        BL      setg<>(SB)
 
@@ -476,13 +493,18 @@ TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8
        B       (R1)
 
 // Save state of caller into g->sched. Smashes R11.
-TEXT gosave<>(SB),NOSPLIT,$0
+TEXT gosave<>(SB),NOSPLIT,$-4
        MOVW    LR, (g_sched+gobuf_pc)(g)
        MOVW    R13, (g_sched+gobuf_sp)(g)
        MOVW    $0, R11
        MOVW    R11, (g_sched+gobuf_lr)(g)
        MOVW    R11, (g_sched+gobuf_ret)(g)
        MOVW    R11, (g_sched+gobuf_ctxt)(g)
+       // Assert ctxt is zero. See func save.
+       MOVW    (g_sched+gobuf_ctxt)(g), R11
+       CMP     $0, R11
+       B.EQ    2(PC)
+       CALL    runtime·badctxt(SB)
        RET
 
 // func asmcgocall(fn, arg unsafe.Pointer) int32

diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s
index 675abb51d37f2c212f5d1553d15e6c38163c38be..0e286d484fae169c8acc686ddf07ce67e376cae3 100644 (file)
--- a/src/runtime/asm_arm64.s
+++ b/src/runtime/asm_arm64.s
@@ -111,13 +111,29 @@ TEXT runtime·gosave(SB), NOSPLIT, $-8-8
        MOVD    g, gobuf_g(R3)
        MOVD    ZR, gobuf_lr(R3)
        MOVD    ZR, gobuf_ret(R3)
-       MOVD    ZR, gobuf_ctxt(R3)
+       // Assert ctxt is zero. See func save.
+       MOVD    gobuf_ctxt(R3), R0
+       CMP     $0, R0
+       BEQ     2(PC)
+       CALL    runtime·badctxt(SB)
        RET
 
 // void gogo(Gobuf*)
 // restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB), NOSPLIT, $-8-8
+TEXT runtime·gogo(SB), NOSPLIT, $24-8
        MOVD    buf+0(FP), R5
+
+       // If ctxt is not nil, invoke deletion barrier before overwriting.
+       MOVD    gobuf_ctxt(R5), R0
+       CMP     $0, R0
+       BEQ     nilctxt
+       MOVD    $gobuf_ctxt(R5), R0
+       MOVD    R0, 8(RSP)
+       MOVD    ZR, 16(RSP)
+       BL      runtime·writebarrierptr_prewrite(SB)
+       MOVD    buf+0(FP), R5
+
+nilctxt:
        MOVD    gobuf_g(R5), g
        BL      runtime·save_g(SB)
 
@@ -483,7 +499,11 @@ TEXT gosave<>(SB),NOSPLIT,$-8
        MOVD    R0, (g_sched+gobuf_sp)(g)
        MOVD    $0, (g_sched+gobuf_lr)(g)
        MOVD    $0, (g_sched+gobuf_ret)(g)
-       MOVD    $0, (g_sched+gobuf_ctxt)(g)
+       // Assert ctxt is zero. See func save.
+       MOVD    (g_sched+gobuf_ctxt)(g), R0
+       CMP     $0, R0
+       BEQ     2(PC)
+       CALL    runtime·badctxt(SB)
        RET
 
 // func asmcgocall(fn, arg unsafe.Pointer) int32

diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s
index 4666741f28bd355bd0fa6fb80d11c0814437fc06..c2d991d36d3410cf4592c2e191d479a1f51e239d 100644 (file)
--- a/src/runtime/asm_mips64x.s
+++ b/src/runtime/asm_mips64x.s
@@ -98,13 +98,27 @@ TEXT runtime·gosave(SB), NOSPLIT, $-8-8
        MOVV    g, gobuf_g(R1)
        MOVV    R0, gobuf_lr(R1)
        MOVV    R0, gobuf_ret(R1)
-       MOVV    R0, gobuf_ctxt(R1)
+       // Assert ctxt is zero. See func save.
+       MOVV    gobuf_ctxt(R1), R1
+       BEQ     R1, 2(PC)
+       JAL     runtime·badctxt(SB)
        RET
 
 // void gogo(Gobuf*)
 // restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB), NOSPLIT, $-8-8
+TEXT runtime·gogo(SB), NOSPLIT, $16-8
        MOVV    buf+0(FP), R3
+
+       // If ctxt is not nil, invoke deletion barrier before overwriting.
+       MOVV    gobuf_ctxt(R3), R1
+       BEQ     R1, nilctxt
+       MOVV    $gobuf_ctxt(R3), R1
+       MOVV    R1, 8(R29)
+       MOVV    R0, 16(R29)
+       JAL     runtime·writebarrierptr_prewrite(SB)
+       MOVV    buf+0(FP), R3
+
+nilctxt:
        MOVV    gobuf_g(R3), g  // make sure g is not nil
        JAL     runtime·save_g(SB)
 
@@ -429,13 +443,16 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $-8-16
        MOVV    0(REGCTXT), R4
        JMP     (R4)
 
-// Save state of caller into g->sched. Smashes R31.
+// Save state of caller into g->sched. Smashes R1.
 TEXT gosave<>(SB),NOSPLIT,$-8
        MOVV    R31, (g_sched+gobuf_pc)(g)
        MOVV    R29, (g_sched+gobuf_sp)(g)
        MOVV    R0, (g_sched+gobuf_lr)(g)
        MOVV    R0, (g_sched+gobuf_ret)(g)
-       MOVV    R0, (g_sched+gobuf_ctxt)(g)
+       // Assert ctxt is zero. See func save.
+       MOVV    (g_sched+gobuf_ctxt)(g), R1
+       BEQ     R1, 2(PC)
+       JAL     runtime·badctxt(SB)
        RET
 
 // func asmcgocall(fn, arg unsafe.Pointer) int32

diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s
index 75711169572352b842ef6700c25e458ac9b9e72a..1d6adcc553c1c8ceb72aa55b6701897525e8041c 100644 (file)
--- a/src/runtime/asm_ppc64x.s
+++ b/src/runtime/asm_ppc64x.s
@@ -122,13 +122,29 @@ TEXT runtime·gosave(SB), NOSPLIT|NOFRAME, $0-8
        MOVD    g, gobuf_g(R3)
        MOVD    R0, gobuf_lr(R3)
        MOVD    R0, gobuf_ret(R3)
-       MOVD    R0, gobuf_ctxt(R3)
+       // Assert ctxt is zero. See func save.
+       MOVD    gobuf_ctxt(R3), R3
+       CMP     R0, R3
+       BEQ     2(PC)
+       BL      runtime·badctxt(SB)
        RET
 
 // void gogo(Gobuf*)
 // restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
+TEXT runtime·gogo(SB), NOSPLIT, $16-8
        MOVD    buf+0(FP), R5
+
+       // If ctxt is not nil, invoke deletion barrier before overwriting.
+       MOVD    gobuf_ctxt(R5), R3
+       CMP     R0, R3
+       BEQ     nilctxt
+       MOVD    $gobuf_ctxt(R5), R3
+       MOVD    R3, FIXED_FRAME+0(R1)
+       MOVD    R0, FIXED_FRAME+8(R1)
+       BL      runtime·writebarrierptr_prewrite(SB)
+       MOVD    buf+0(FP), R5
+
+nilctxt:
        MOVD    gobuf_g(R5), g  // make sure g is not nil
        BL      runtime·save_g(SB)
 
@@ -497,7 +513,11 @@ TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0
        MOVD    R1, (g_sched+gobuf_sp)(g)
        MOVD    R0, (g_sched+gobuf_lr)(g)
        MOVD    R0, (g_sched+gobuf_ret)(g)
-       MOVD    R0, (g_sched+gobuf_ctxt)(g)
+       // Assert ctxt is zero. See func save.
+       MOVD    (g_sched+gobuf_ctxt)(g), R31
+       CMP     R0, R31
+       BEQ     2(PC)
+       BL      runtime·badctxt(SB)
        RET
 
 // func asmcgocall(fn, arg unsafe.Pointer) int32

diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s
index 1dcee7cf4e3c598f970b2bb3e27094f9ec0c945b..614a79943275ed113ffb4e47c5ad177d669f1e7e 100644 (file)
--- a/src/runtime/asm_s390x.s
+++ b/src/runtime/asm_s390x.s
@@ -106,13 +106,27 @@ TEXT runtime·gosave(SB), NOSPLIT, $-8-8
        MOVD    g, gobuf_g(R3)
        MOVD    $0, gobuf_lr(R3)
        MOVD    $0, gobuf_ret(R3)
-       MOVD    $0, gobuf_ctxt(R3)
+       // Assert ctxt is zero. See func save.
+       MOVD    gobuf_ctxt(R3), R3
+       CMPBEQ  R3, $0, 2(PC)
+       BL      runtime·badctxt(SB)
        RET
 
 // void gogo(Gobuf*)
 // restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB), NOSPLIT, $-8-8
+TEXT runtime·gogo(SB), NOSPLIT, $16-8
        MOVD    buf+0(FP), R5
+
+       // If ctxt is not nil, invoke deletion barrier before overwriting.
+       MOVD    gobuf_ctxt(R5), R1
+       CMPBEQ  R1, $0, nilctxt
+       MOVD    $gobuf_ctxt(R5), R1
+       MOVD    R1, 8(R15)
+       MOVD    R0, 16(R15)
+       BL      runtime·writebarrierptr_prewrite(SB)
+       MOVD    buf+0(FP), R5
+
+nilctxt:
        MOVD    gobuf_g(R5), g  // make sure g is not nil
        BL      runtime·save_g(SB)
 
@@ -447,13 +461,16 @@ TEXT runtime·jmpdefer(SB),NOSPLIT|NOFRAME,$0-16
        MOVD    0(R12), R3
        BR      (R3)
 
-// Save state of caller into g->sched. Smashes R31.
+// Save state of caller into g->sched. Smashes R1.
 TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0
        MOVD    LR, (g_sched+gobuf_pc)(g)
        MOVD    R15, (g_sched+gobuf_sp)(g)
        MOVD    $0, (g_sched+gobuf_lr)(g)
        MOVD    $0, (g_sched+gobuf_ret)(g)
-       MOVD    $0, (g_sched+gobuf_ctxt)(g)
+       // Assert ctxt is zero. See func save.
+       MOVD    (g_sched+gobuf_ctxt)(g), R1
+       CMPBEQ  R1, $0, 2(PC)
+       BL      runtime·badctxt(SB)
        RET
 
 // func asmcgocall(fn, arg unsafe.Pointer) int32

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index 49f6e6f64983277abaa2927ef44bfc38884c21b7..696ea81e00f1556383852a2d2d71eaadbe16a604 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -239,6 +239,15 @@ func setMNoWB(mp **m, new *m) {
 
 type gobuf struct {
        // The offsets of sp, pc, and g are known to (hard-coded in) libmach.
+       //
+       // ctxt is unusual with respect to GC: it may be a
+       // heap-allocated funcval so write require a write barrier,
+       // but gobuf needs to be cleared from assembly. We take
+       // advantage of the fact that the only path that uses a
+       // non-nil ctxt is morestack. As a result, gogo is the only
+       // place where it may not already be nil, so gogo uses an
+       // explicit write barrier. Everywhere else that resets the
+       // gobuf asserts that ctxt is already nil.
        sp   uintptr
        pc   uintptr
        g    guintptr