Replace calls to memmove with known (constant) size, with OpMove.
Do it only if it is safe from aliasing point of view.
Helps with code like this:
append(buf,"const str"...)
In strconv this provides nice benefit:
Quote-6 731ns ± 2% 647ns ± 3% -11.41% (p=0.000 n=10+10)
QuoteRune-6 117ns ± 5% 111ns ± 1% -4.54% (p=0.000 n=10+10)
AppendQuote-6 475ns ± 0% 396ns ± 0% -16.59% (p=0.000 n=9+10)
AppendQuoteRune-6 32.0ns ± 0% 27.4ns ± 0% -14.41% (p=0.000 n=8+9)
Change-Id: I7704f5c51b46aed2d8f033de74c75140fc35036c
Reviewed-on: https://go-review.googlesource.com/54394
Run-TryBot: Ilya Tocar <ilya.tocar@intel.com>
Reviewed-by: Keith Randall <khr@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
`,
pos: []string{"\tANDQ\t\\$4095,"},
},
+ {
+ // Test that small memmove was replaced with direct movs
+ fn: `
+ func $() {
+ x := [...]byte{1, 2, 3, 4, 5, 6, 7}
+ copy(x[1:], x[:])
+ }
+ `,
+ neg: []string{"memmove"},
+ },
+ {
+ // Same as above but with different size
+ fn: `
+ func $() {
+ x := [...]byte{1, 2, 3, 4}
+ copy(x[1:], x[:])
+ }
+ `,
+ neg: []string{"memmove"},
+ },
+ {
+ // Same as above but with different size
+ fn: `
+ func $() {
+ x := [...]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+ copy(x[1:], x[:])
+ }
+ `,
+ neg: []string{"memmove"},
+ },
}
var linux386Tests = []*asmTest{
`,
pos: []string{"\tANDL\t\\$4095,"},
},
+ {
+ // Test that small memmove was replaced with direct movs
+ fn: `
+ func $() {
+ x := [...]byte{1, 2, 3, 4, 5, 6, 7}
+ copy(x[1:], x[:])
+ }
+ `,
+ neg: []string{"memmove"},
+ },
+ {
+ // Same as above but with different size
+ fn: `
+ func $() {
+ x := [...]byte{1, 2, 3, 4}
+ copy(x[1:], x[:])
+ }
+ `,
+ neg: []string{"memmove"},
+ },
}
var linuxS390XTests = []*asmTest{
(EqPtr x x) -> (ConstBool [1])
(EqPtr (Addr {a} x) (Addr {b} x)) -> (ConstBool [b2i(a == b)])
+// Inline small runtime.memmove calls with constant length.
+(StaticCall {sym} s1:(Store _ (Const64 [sz]) s2:(Store _ src s3:(Store {t} _ dst mem))))
+ && isSameSym(sym,"runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz,config)
+ -> (Move {t.(*types.Type).Elem()} [sz] dst src mem)
+(StaticCall {sym} s1:(Store _ (Const32 [sz]) s2:(Store _ src s3:(Store {t} _ dst mem))))
+ && isSameSym(sym,"runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz,config)
+ -> (Move {t.(*types.Type).Elem()} [sz] dst src mem)
+
// De-virtualize interface calls into static calls.
// Note that (ITab (IMake)) doesn't get
// rewritten until after the first opt pass,
{name: "Invalid"}, // unused value
// Memory operations
- {name: "Load", argLength: 2}, // Load from arg0. arg1=memory
- {name: "Store", argLength: 3, typ: "Mem", aux: "Typ"}, // Store arg1 to arg0. arg2=memory, aux=type. Returns memory.
+ {name: "Load", argLength: 2}, // Load from arg0. arg1=memory
+ {name: "Store", argLength: 3, typ: "Mem", aux: "Typ"}, // Store arg1 to arg0. arg2=memory, aux=type. Returns memory.
+ // The source and destination of Move may overlap in some cases. See e.g.
+ // memmove inlining in generic.rules. When inlineablememmovesize (in ../rewrite.go)
+ // returns true, we must do all loads before all stores, when lowering Move.
{name: "Move", argLength: 3, typ: "Mem", aux: "TypSize"}, // arg0=destptr, arg1=srcptr, arg2=mem, auxint=size, aux=type. Returns memory.
{name: "Zero", argLength: 2, typ: "Mem", aux: "TypSize"}, // arg0=destptr, arg1=mem, auxint=size, aux=type. Returns memory.
}
return false
}
+
+// inlineablememmovesize reports whether the given arch performs OpMove of the given size
+// faster than memmove and in a safe way when src and dst overlap.
+// This is used as a check for replacing memmove with OpMove.
+func isInlinableMemmoveSize(sz int64, c *Config) bool {
+ switch c.arch {
+ case "amd64", "amd64p32":
+ return sz <= 16
+ case "386", "ppc64", "s390x", "ppc64le":
+ return sz <= 8
+ case "arm", "mips", "mips64", "mipsle", "mips64le":
+ return sz <= 4
+ }
+ return false
+}
}
}
}
+
+// We generate memmove for copy(x[1:], x[:]), however we may change it to OpMove,
+// because size is known. Check that OpMove is alias-safe, or we did call memmove.
+func TestMove(t *testing.T) {
+ x := [...]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40}
+ copy(x[1:], x[:])
+ for i := 1; i < len(x); i++ {
+ if int(x[i]) != i {
+ t.Errorf("Memmove got converted to OpMove in alias-unsafe way. Got %d insted of %d in position %d", int(x[i]), i, i+1)
+ }
+ }
+}
+
+func TestMoveSmall(t *testing.T) {
+ x := [...]byte{1, 2, 3, 4, 5, 6, 7}
+ copy(x[1:], x[:])
+ for i := 1; i < len(x); i++ {
+ if int(x[i]) != i {
+ t.Errorf("Memmove got converted to OpMove in alias-unsafe way. Got %d instead of %d in position %d", int(x[i]), i, i+1)
+ }
+ }
+}
return rewriteValuegeneric_OpSlicemask_0(v)
case OpSqrt:
return rewriteValuegeneric_OpSqrt_0(v)
+ case OpStaticCall:
+ return rewriteValuegeneric_OpStaticCall_0(v)
case OpStore:
return rewriteValuegeneric_OpStore_0(v) || rewriteValuegeneric_OpStore_10(v)
case OpStringLen:
}
return false
}
+func rewriteValuegeneric_OpStaticCall_0(v *Value) bool {
+ b := v.Block
+ _ = b
+ config := b.Func.Config
+ _ = config
+ // match: (StaticCall {sym} s1:(Store _ (Const64 [sz]) s2:(Store _ src s3:(Store {t} _ dst mem))))
+ // cond: isSameSym(sym,"runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz,config)
+ // result: (Move {t.(*types.Type).Elem()} [sz] dst src mem)
+ for {
+ sym := v.Aux
+ s1 := v.Args[0]
+ if s1.Op != OpStore {
+ break
+ }
+ _ = s1.Args[2]
+ s1_1 := s1.Args[1]
+ if s1_1.Op != OpConst64 {
+ break
+ }
+ sz := s1_1.AuxInt
+ s2 := s1.Args[2]
+ if s2.Op != OpStore {
+ break
+ }
+ _ = s2.Args[2]
+ src := s2.Args[1]
+ s3 := s2.Args[2]
+ if s3.Op != OpStore {
+ break
+ }
+ t := s3.Aux
+ _ = s3.Args[2]
+ dst := s3.Args[1]
+ mem := s3.Args[2]
+ if !(isSameSym(sym, "runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz, config)) {
+ break
+ }
+ v.reset(OpMove)
+ v.AuxInt = sz
+ v.Aux = t.(*types.Type).Elem()
+ v.AddArg(dst)
+ v.AddArg(src)
+ v.AddArg(mem)
+ return true
+ }
+ // match: (StaticCall {sym} s1:(Store _ (Const32 [sz]) s2:(Store _ src s3:(Store {t} _ dst mem))))
+ // cond: isSameSym(sym,"runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz,config)
+ // result: (Move {t.(*types.Type).Elem()} [sz] dst src mem)
+ for {
+ sym := v.Aux
+ s1 := v.Args[0]
+ if s1.Op != OpStore {
+ break
+ }
+ _ = s1.Args[2]
+ s1_1 := s1.Args[1]
+ if s1_1.Op != OpConst32 {
+ break
+ }
+ sz := s1_1.AuxInt
+ s2 := s1.Args[2]
+ if s2.Op != OpStore {
+ break
+ }
+ _ = s2.Args[2]
+ src := s2.Args[1]
+ s3 := s2.Args[2]
+ if s3.Op != OpStore {
+ break
+ }
+ t := s3.Aux
+ _ = s3.Args[2]
+ dst := s3.Args[1]
+ mem := s3.Args[2]
+ if !(isSameSym(sym, "runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz, config)) {
+ break
+ }
+ v.reset(OpMove)
+ v.AuxInt = sz
+ v.Aux = t.(*types.Type).Elem()
+ v.AddArg(dst)
+ v.AddArg(src)
+ v.AddArg(mem)
+ return true
+ }
+ return false
+}
func rewriteValuegeneric_OpStore_0(v *Value) bool {
b := v.Block
_ = b