cmd/compile: use typedmemclr for zeroing if there are pointers

Currently, zeroing generates an ssa.OpZero, which never has write
barriers, even if the assignment is an OASWB. The hybrid barrier
requires write barriers on zeroing, so change OASWB to generate an
ssa.OpZeroWB when assigning the zero value, which turns into a
typedmemclr.

Updates #17503.

Change-Id: Ib37ac5e39f578447dbd6b36a6a54117d5624784d
Reviewed-on: https://go-review.googlesource.com/31451
Reviewed-by: Cherry Zhang <cherryyz@google.com>

diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go
index 1afbce28351766dfe2e54de625998a5c757a8bab..9863c18b29549f88cd033dbef94d167affcfc8f9 100644 (file)
--- a/src/cmd/compile/internal/gc/ssa.go
+++ b/src/cmd/compile/internal/gc/ssa.go
@@ -686,7 +686,7 @@ func (s *state) stmt(n *Node) {
                }
                var r *ssa.Value
                var isVolatile bool
-               needwb := n.Op == OASWB && rhs != nil
+               needwb := n.Op == OASWB
                deref := !canSSAType(t)
                if deref {
                        if rhs == nil {
@@ -2390,14 +2390,14 @@ func (s *state) assign(left *Node, right *ssa.Value, wb, deref bool, line int32,
        }
        if deref {
                // Treat as a mem->mem move.
-               if right == nil {
-                       s.vars[&memVar] = s.newValue2I(ssa.OpZero, ssa.TypeMem, sizeAlignAuxInt(t), addr, s.mem())
-                       return
-               }
                if wb && !ssa.IsStackAddr(addr) {
                        s.insertWBmove(t, addr, right, line, rightIsVolatile)
                        return
                }
+               if right == nil {
+                       s.vars[&memVar] = s.newValue2I(ssa.OpZero, ssa.TypeMem, sizeAlignAuxInt(t), addr, s.mem())
+                       return
+               }
                s.vars[&memVar] = s.newValue3I(ssa.OpMove, ssa.TypeMem, sizeAlignAuxInt(t), addr, right, s.mem())
                return
        }
@@ -3295,12 +3295,21 @@ func (s *state) rtcall(fn *Node, returns bool, results []*Type, args ...*ssa.Val
 
 // insertWBmove inserts the assignment *left = *right including a write barrier.
 // t is the type being assigned.
+// If right == nil, then we're zeroing *left.
 func (s *state) insertWBmove(t *Type, left, right *ssa.Value, line int32, rightIsVolatile bool) {
        // if writeBarrier.enabled {
        //   typedmemmove(&t, left, right)
        // } else {
        //   *left = *right
        // }
+       //
+       // or
+       //
+       // if writeBarrier.enabled {
+       //   typedmemclr(&t, left)
+       // } else {
+       //   *left = zeroValue
+       // }
 
        if s.noWB {
                s.Error("write barrier prohibited")
@@ -3309,15 +3318,20 @@ func (s *state) insertWBmove(t *Type, left, right *ssa.Value, line int32, rightI
                s.WBLineno = left.Line
        }
 
-       var op ssa.Op
-       if rightIsVolatile {
-               op = ssa.OpMoveWBVolatile
+       var val *ssa.Value
+       if right == nil {
+               val = s.newValue2I(ssa.OpZeroWB, ssa.TypeMem, sizeAlignAuxInt(t), left, s.mem())
        } else {
-               op = ssa.OpMoveWB
+               var op ssa.Op
+               if rightIsVolatile {
+                       op = ssa.OpMoveWBVolatile
+               } else {
+                       op = ssa.OpMoveWB
+               }
+               val = s.newValue3I(op, ssa.TypeMem, sizeAlignAuxInt(t), left, right, s.mem())
        }
-       move := s.newValue3I(op, ssa.TypeMem, sizeAlignAuxInt(t), left, right, s.mem())
-       move.Aux = &ssa.ExternSymbol{Typ: Types[TUINTPTR], Sym: typenamesym(t)}
-       s.vars[&memVar] = move
+       val.Aux = &ssa.ExternSymbol{Typ: Types[TUINTPTR], Sym: typenamesym(t)}
+       s.vars[&memVar] = val
 
        // WB ops will be expanded to branches at writebarrier phase.
        // To make it easy, we put WB ops at the end of a block, so

diff --git a/src/cmd/compile/internal/ssa/gen/genericOps.go b/src/cmd/compile/internal/ssa/gen/genericOps.go
index 5a570c40c154f062dc1c8a63d12611bc305512d9..d935e74b9fedbe2160925ee9298ccacedc3d5595 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/genericOps.go
+++ b/src/cmd/compile/internal/ssa/gen/genericOps.go
@@ -320,7 +320,7 @@ var genericOps = []opData{
        {name: "StoreWB", argLength: 3, typ: "Mem", aux: "Int64"},                  // Store arg1 to arg0. arg2=memory, auxint=size.  Returns memory.
        {name: "MoveWB", argLength: 3, typ: "Mem", aux: "SymSizeAndAlign"},         // arg0=destptr, arg1=srcptr, arg2=mem, auxint=size+alignment, aux=symbol-of-type (for typedmemmove).  Returns memory.
        {name: "MoveWBVolatile", argLength: 3, typ: "Mem", aux: "SymSizeAndAlign"}, // arg0=destptr, arg1=srcptr, arg2=mem, auxint=size+alignment, aux=symbol-of-type (for typedmemmove).  Returns memory. Src is volatile, i.e. needs to move to a temp space before calling typedmemmove.
-       // maybe we'll need a ZeroWB for the new barrier
+       {name: "ZeroWB", argLength: 2, typ: "Mem", aux: "SymSizeAndAlign"},         // arg0=destptr, arg1=mem, auxint=size+alignment, aux=symbol-of-type. Returns memory.
 
        // Function calls. Arguments to the call have already been written to the stack.
        // Return values appear on the stack. The method receiver, if any, is treated

diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go
index 1178175b80a643626697d20a8d4e0f04a8b30a95..e889787c4e04267305e9379fd1bcb8f1501d5501 100644 (file)
--- a/src/cmd/compile/internal/ssa/opGen.go
+++ b/src/cmd/compile/internal/ssa/opGen.go
@@ -1658,6 +1658,7 @@ const (
        OpStoreWB
        OpMoveWB
        OpMoveWBVolatile
+       OpZeroWB
        OpClosureCall
        OpStaticCall
        OpDeferCall
@@ -19404,6 +19405,12 @@ var opcodeTable = [...]opInfo{
                argLen:  3,
                generic: true,
        },
+       {
+               name:    "ZeroWB",
+               auxType: auxSymSizeAndAlign,
+               argLen:  2,
+               generic: true,
+       },
        {
                name:    "ClosureCall",
                auxType: auxInt64,

diff --git a/src/cmd/compile/internal/ssa/writebarrier.go b/src/cmd/compile/internal/ssa/writebarrier.go
index 2eb549ce59ba809ed82fd1074668566a427023c5..b914154b48e02d23670aec99e5c4ad69ef97c613 100644 (file)
--- a/src/cmd/compile/internal/ssa/writebarrier.go
+++ b/src/cmd/compile/internal/ssa/writebarrier.go
@@ -28,7 +28,7 @@ import "fmt"
 // number of blocks as fuse merges blocks introduced in this phase.
 func writebarrier(f *Func) {
        var sb, sp, wbaddr *Value
-       var writebarrierptr, typedmemmove interface{} // *gc.Sym
+       var writebarrierptr, typedmemmove, typedmemclr interface{} // *gc.Sym
        var storeWBs, others []*Value
        var wbs *sparseSet
        for _, b := range f.Blocks { // range loop is safe since the blocks we added contain no WB stores
@@ -43,6 +43,9 @@ func writebarrier(f *Func) {
                                        case OpMoveWB, OpMoveWBVolatile:
                                                v.Op = OpMove
                                                v.Aux = nil
+                                       case OpZeroWB:
+                                               v.Op = OpZero
+                                               v.Aux = nil
                                        }
                                        continue
                                }
@@ -69,6 +72,7 @@ func writebarrier(f *Func) {
                                        wbaddr = f.Entry.NewValue1A(initln, OpAddr, f.Config.fe.TypeUInt32().PtrTo(), wbsym, sb)
                                        writebarrierptr = f.Config.fe.Syslook("writebarrierptr")
                                        typedmemmove = f.Config.fe.Syslook("typedmemmove")
+                                       typedmemclr = f.Config.fe.Syslook("typedmemclr")
 
                                        wbs = f.newSparseSet(f.NumValues())
                                        defer f.retSparseSet(wbs)
@@ -82,7 +86,7 @@ func writebarrier(f *Func) {
                                others = others[:0]
                                wbs.clear()
                                for _, w := range b.Values[i:] {
-                                       if w.Op == OpStoreWB || w.Op == OpMoveWB || w.Op == OpMoveWBVolatile {
+                                       if w.Op == OpStoreWB || w.Op == OpMoveWB || w.Op == OpMoveWBVolatile || w.Op == OpZeroWB {
                                                storeWBs = append(storeWBs, w)
                                                wbs.add(w.ID)
                                        } else {
@@ -92,7 +96,7 @@ func writebarrier(f *Func) {
 
                                // make sure that no value in this block depends on WB stores
                                for _, w := range b.Values {
-                                       if w.Op == OpStoreWB || w.Op == OpMoveWB || w.Op == OpMoveWBVolatile {
+                                       if w.Op == OpStoreWB || w.Op == OpMoveWB || w.Op == OpMoveWBVolatile || w.Op == OpZeroWB {
                                                continue
                                        }
                                        for _, a := range w.Args {
@@ -136,10 +140,10 @@ func writebarrier(f *Func) {
                                memThen := mem
                                memElse := mem
                                for _, w := range storeWBs {
+                                       var val *Value
                                        ptr := w.Args[0]
-                                       val := w.Args[1]
                                        siz := w.AuxInt
-                                       typ := w.Aux // only non-nil for MoveWB, MoveWBVolatile
+                                       typ := w.Aux // only non-nil for MoveWB, MoveWBVolatile, ZeroWB
 
                                        var op Op
                                        var fn interface{} // *gc.Sym
@@ -147,16 +151,25 @@ func writebarrier(f *Func) {
                                        case OpStoreWB:
                                                op = OpStore
                                                fn = writebarrierptr
+                                               val = w.Args[1]
                                        case OpMoveWB, OpMoveWBVolatile:
                                                op = OpMove
                                                fn = typedmemmove
+                                               val = w.Args[1]
+                                       case OpZeroWB:
+                                               op = OpZero
+                                               fn = typedmemclr
                                        }
 
                                        // then block: emit write barrier call
                                        memThen = wbcall(line, bThen, fn, typ, ptr, val, memThen, sp, sb, w.Op == OpMoveWBVolatile)
 
                                        // else block: normal store
-                                       memElse = bElse.NewValue3I(line, op, TypeMem, siz, ptr, val, memElse)
+                                       if op == OpZero {
+                                               memElse = bElse.NewValue2I(line, op, TypeMem, siz, ptr, memElse)
+                                       } else {
+                                               memElse = bElse.NewValue3I(line, op, TypeMem, siz, ptr, val, memElse)
+                                       }
                                }
 
                                // merge memory
@@ -226,10 +239,12 @@ func wbcall(line int32, b *Block, fn interface{}, typ interface{}, ptr, val, mem
        mem = b.NewValue3I(line, OpStore, TypeMem, ptr.Type.Size(), arg, ptr, mem)
        off += ptr.Type.Size()
 
-       off = round(off, val.Type.Alignment())
-       arg = b.NewValue1I(line, OpOffPtr, val.Type.PtrTo(), off, sp)
-       mem = b.NewValue3I(line, OpStore, TypeMem, val.Type.Size(), arg, val, mem)
-       off += val.Type.Size()
+       if val != nil {
+               off = round(off, val.Type.Alignment())
+               arg = b.NewValue1I(line, OpOffPtr, val.Type.PtrTo(), off, sp)
+               mem = b.NewValue3I(line, OpStore, TypeMem, val.Type.Size(), arg, val, mem)
+               off += val.Type.Size()
+       }
        off = round(off, config.PtrSize)
 
        // issue call