runtime: buffered write barrier for amd64p32

Updates #22460.

Change-Id: I6656d478625e5e54aa2eaa38d99dfb0f71ea1fdd
Reviewed-on: https://go-review.googlesource.com/92697
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go
index 34ab29777d86a12efb668dfcd8c1d6d9d1ac848b..71ba1f61eb98aa3634e98cb3e80321670b2e6be8 100644 (file)
--- a/src/cmd/compile/internal/gc/main.go
+++ b/src/cmd/compile/internal/gc/main.go
@@ -408,7 +408,7 @@ func Main(archInit func(*Arch)) {
        }
 
        switch objabi.GOARCH {
-       case "amd64", "386":
+       case "amd64", "amd64p32", "386":
        default:
                // Other architectures don't support the buffered
                // write barrier yet.

diff --git a/src/cmd/compile/internal/ssa/config.go b/src/cmd/compile/internal/ssa/config.go
index a4f4d7edfc0f4108b62579c6f5f87551d3689056..725cdcd9a4666d50f014c4911177b5a2f426b382 100644 (file)
--- a/src/cmd/compile/internal/ssa/config.go
+++ b/src/cmd/compile/internal/ssa/config.go
@@ -298,6 +298,9 @@ func NewConfig(arch string, types Types, ctxt *obj.Link, optimize bool) *Config
                // Returns clobber BP on nacl/386, so the write
                // barrier does.
                opcodeTable[Op386LoweredWB].reg.clobbers |= 1 << 5 // BP
+
+               // ... and SI on nacl/amd64.
+               opcodeTable[OpAMD64LoweredWB].reg.clobbers |= 1 << 6 // SI
        }
 
        if ctxt.Flag_shared {

diff --git a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
index 0c3b2efa305d7f11a703fb8f7c98b940bc371147..f3eaf789bf450299f4a859fe9e8ff299e339aa22 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
@@ -567,7 +567,7 @@ func init() {
 
                // LoweredWB invokes runtime.gcWriteBarrier. arg0=destptr, arg1=srcptr, arg2=mem, aux=runtime.gcWriteBarrier
                // It saves all GP registers if necessary, but may clobber others.
-               {name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("DI"), ax}, clobbers: callerSave ^ gp}, clobberFlags: true, aux: "Sym", symEffect: "None"},
+               {name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("DI"), ax}, clobbers: callerSave &^ gp}, clobberFlags: true, aux: "Sym", symEffect: "None"},
 
                // MOVQconvert converts between pointers and integers.
                // We have a special op for this so as to not confuse GC

diff --git a/src/cmd/vet/all/whitelist/nacl_amd64p32.txt b/src/cmd/vet/all/whitelist/nacl_amd64p32.txt
index 4b2aad2aacd9305464d426c0489433206f301881..5c6b3344a0815f6b677b9d61b400eee871bcfc91 100644 (file)
--- a/src/cmd/vet/all/whitelist/nacl_amd64p32.txt
+++ b/src/cmd/vet/all/whitelist/nacl_amd64p32.txt
@@ -27,3 +27,5 @@ runtime/asm_amd64p32.s: [amd64p32] indexbytebody: function indexbytebody missing
 runtime/asm_amd64p32.s: [amd64p32] asmcgocall: RET without writing to 4-byte ret+8(FP)
 
 runtime/asm_amd64p32.s: [amd64p32] stackcheck: function stackcheck missing Go declaration
+
+runtime/asm_ARCHSUFF.s: [GOARCH] gcWriteBarrier: function gcWriteBarrier missing Go declaration

diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s
index 7fee79aefbc6c86b553b52c025737bfd9eddfe24..cdf2cd7f008609331146ad1caa1c4253573c471f 100644 (file)
--- a/src/runtime/asm_amd64p32.s
+++ b/src/runtime/asm_amd64p32.s
@@ -973,3 +973,84 @@ TEXT runtime·goexit(SB),NOSPLIT,$0-0
 TEXT ·checkASM(SB),NOSPLIT,$0-1
        MOVB    $1, ret+0(FP)
        RET
+
+// gcWriteBarrier performs a heap pointer write and informs the GC.
+//
+// gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
+// - DI is the destination of the write
+// - AX is the value being written at DI
+// It clobbers FLAGS and SI. It does not clobber any other general-purpose registers,
+// but may clobber others (e.g., SSE registers).
+TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$88
+       // Save the registers clobbered by the fast path. This is slightly
+       // faster than having the caller spill these.
+       MOVQ    R14, 72(SP)
+       MOVQ    R13, 80(SP)
+       // TODO: Consider passing g.m.p in as an argument so they can be shared
+       // across a sequence of write barriers.
+       get_tls(R13)
+       MOVL    g(R13), R13
+       MOVL    g_m(R13), R13
+       MOVL    m_p(R13), R13
+       MOVL    (p_wbBuf+wbBuf_next)(R13), R14
+       // Increment wbBuf.next position.
+       LEAL    8(R14), R14
+       MOVL    R14, (p_wbBuf+wbBuf_next)(R13)
+       CMPL    R14, (p_wbBuf+wbBuf_end)(R13)
+       // Record the write.
+       MOVL    AX, -8(R14)     // Record value
+       MOVL    (DI), R13       // TODO: This turns bad writes into bad reads.
+       MOVL    R13, -4(R14)    // Record *slot
+       // Is the buffer full? (flags set in CMPL above)
+       JEQ     flush
+ret:
+       MOVQ    72(SP), R14
+       MOVQ    80(SP), R13
+       // Do the write.
+       MOVL    AX, (DI)
+       RET                     // Clobbers SI on NaCl
+
+flush:
+       // Save all general purpose registers since these could be
+       // clobbered by wbBufFlush and were not saved by the caller.
+       // It is possible for wbBufFlush to clobber other registers
+       // (e.g., SSE registers), but the compiler takes care of saving
+       // those in the caller if necessary. This strikes a balance
+       // with registers that are likely to be used.
+       //
+       // We don't have type information for these, but all code under
+       // here is NOSPLIT, so nothing will observe these.
+       //
+       // TODO: We could strike a different balance; e.g., saving X0
+       // and not saving GP registers that are less likely to be used.
+       MOVL    DI, 0(SP)       // Also first argument to wbBufFlush
+       MOVL    AX, 4(SP)       // Also second argument to wbBufFlush
+       MOVQ    BX, 8(SP)
+       MOVQ    CX, 16(SP)
+       MOVQ    DX, 24(SP)
+       // DI already saved
+       // SI is always clobbered on nacl
+       // BP is reserved on nacl
+       MOVQ    R8, 32(SP)
+       MOVQ    R9, 40(SP)
+       MOVQ    R10, 48(SP)
+       MOVQ    R11, 56(SP)
+       MOVQ    R12, 64(SP)
+       // R13 already saved
+       // R14 already saved
+       // R15 is reserved on nacl
+
+       // This takes arguments DI and AX
+       CALL    runtime·wbBufFlush(SB)
+
+       MOVL    0(SP), DI
+       MOVL    4(SP), AX
+       MOVQ    8(SP), BX
+       MOVQ    16(SP), CX
+       MOVQ    24(SP), DX
+       MOVQ    32(SP), R8
+       MOVQ    40(SP), R9
+       MOVQ    48(SP), R10
+       MOVQ    56(SP), R11
+       MOVQ    64(SP), R12
+       JMP     ret