runtime: use memmove during slice append

The effect of this CL:

name                   old mean              new mean              delta
BinaryTree17            5.97s × (0.96,1.04)   5.95s × (0.98,1.02)    ~    (p=0.697)
Fannkuch11              4.39s × (1.00,1.01)   4.41s × (1.00,1.01)  +0.52% (p=0.015)
FmtFprintfEmpty        90.8ns × (0.97,1.05)  89.4ns × (0.94,1.13)    ~    (p=0.571)
FmtFprintfString        305ns × (0.99,1.01)   292ns × (0.98,1.05)  -4.35% (p=0.000)
FmtFprintfInt           278ns × (0.96,1.03)   279ns × (0.98,1.04)    ~    (p=0.741)
FmtFprintfIntInt        489ns × (0.99,1.02)   482ns × (0.98,1.03)  -1.43% (p=0.024)
FmtFprintfPrefixedInt   402ns × (0.98,1.02)   395ns × (0.98,1.03)  -1.67% (p=0.014)
FmtFprintfFloat         578ns × (1.00,1.00)   569ns × (0.99,1.01)  -1.48% (p=0.000)
FmtManyArgs            1.88µs × (0.99,1.01)  1.88µs × (1.00,1.01)    ~    (p=0.055)
GobDecode              15.3ms × (0.99,1.01)  15.2ms × (1.00,1.01)  -0.61% (p=0.007)
GobEncode              11.8ms × (0.98,1.05)  11.6ms × (0.99,1.01)    ~    (p=0.075)
Gzip                    647ms × (0.99,1.01)   647ms × (1.00,1.00)    ~    (p=0.790)
Gunzip                  143ms × (1.00,1.00)   142ms × (1.00,1.00)    ~    (p=0.370)
HTTPClientServer       91.2µs × (0.99,1.01)  91.7µs × (0.99,1.02)    ~    (p=0.233)
JSONEncode             31.5ms × (0.98,1.01)  31.8ms × (0.99,1.02)  +1.09% (p=0.015)
JSONDecode              110ms × (0.99,1.01)   110ms × (0.99,1.02)    ~    (p=0.577)
Mandelbrot200          6.00ms × (1.00,1.00)  6.02ms × (1.00,1.00)  +0.24% (p=0.001)
GoParse                6.68ms × (0.98,1.02)  6.61ms × (0.99,1.01)  -1.10% (p=0.027)
RegexpMatchEasy0_32     162ns × (1.00,1.00)   161ns × (1.00,1.01)  -0.66% (p=0.001)
RegexpMatchEasy0_1K     539ns × (1.00,1.00)   539ns × (0.99,1.01)    ~    (p=0.509)
RegexpMatchEasy1_32     140ns × (0.99,1.02)   139ns × (0.99,1.02)    ~    (p=0.163)
RegexpMatchEasy1_1K     886ns × (1.00,1.00)   887ns × (1.00,1.00)    ~    (p=0.408)
RegexpMatchMedium_32    252ns × (1.00,1.00)   255ns × (0.99,1.01)  +1.01% (p=0.000)
RegexpMatchMedium_1K   72.6µs × (1.00,1.00)  72.6µs × (1.00,1.00)    ~    (p=0.176)
RegexpMatchHard_32     3.84µs × (1.00,1.00)  3.84µs × (1.00,1.00)    ~    (p=0.403)
RegexpMatchHard_1K      117µs × (1.00,1.00)   117µs × (1.00,1.00)    ~    (p=0.351)
Revcomp                 926ms × (0.99,1.01)   925ms × (0.99,1.01)    ~    (p=0.541)
Template                126ms × (0.99,1.02)   130ms × (0.99,1.01)  +3.42% (p=0.000)
TimeParse               632ns × (0.99,1.01)   626ns × (1.00,1.00)  -0.88% (p=0.000)
TimeFormat              658ns × (0.99,1.01)   662ns × (0.99,1.02)    ~    (p=0.111)

The effect of this CL combined with CL 9886:

name                   old mean              new mean              delta
BinaryTree17            5.90s × (0.98,1.03)   5.95s × (0.98,1.02)    ~    (p=0.175)
Fannkuch11              4.34s × (1.00,1.00)   4.41s × (1.00,1.01)  +1.69% (p=0.000)
FmtFprintfEmpty        87.3ns × (0.97,1.17)  89.4ns × (0.94,1.13)    ~    (p=0.499)
FmtFprintfString        288ns × (0.98,1.04)   292ns × (0.98,1.05)    ~    (p=0.292)
FmtFprintfInt           290ns × (0.98,1.05)   279ns × (0.98,1.04)  -3.76% (p=0.001)
FmtFprintfIntInt        493ns × (0.98,1.04)   482ns × (0.98,1.03)  -2.27% (p=0.017)
FmtFprintfPrefixedInt   399ns × (0.98,1.02)   395ns × (0.98,1.03)    ~    (p=0.159)
FmtFprintfFloat         569ns × (1.00,1.00)   569ns × (0.99,1.01)    ~    (p=0.847)
FmtManyArgs            1.90µs × (0.99,1.03)  1.88µs × (1.00,1.01)  -1.14% (p=0.009)
GobDecode              15.2ms × (1.00,1.01)  15.2ms × (1.00,1.01)    ~    (p=0.170)
GobEncode              11.8ms × (0.99,1.02)  11.6ms × (0.99,1.01)  -1.47% (p=0.003)
Gzip                    649ms × (0.99,1.00)   647ms × (1.00,1.00)    ~    (p=0.200)
Gunzip                  144ms × (0.99,1.01)   142ms × (1.00,1.00)  -1.04% (p=0.000)
HTTPClientServer       91.1µs × (0.98,1.03)  91.7µs × (0.99,1.02)    ~    (p=0.345)
JSONEncode             31.5ms × (0.99,1.01)  31.8ms × (0.99,1.02)  +0.98% (p=0.021)
JSONDecode              110ms × (1.00,1.01)   110ms × (0.99,1.02)    ~    (p=0.259)
Mandelbrot200          6.02ms × (1.00,1.01)  6.02ms × (1.00,1.00)    ~    (p=0.500)
GoParse                6.68ms × (1.00,1.01)  6.61ms × (0.99,1.01)  -1.17% (p=0.001)
RegexpMatchEasy0_32     161ns × (1.00,1.00)   161ns × (1.00,1.01)  -0.39% (p=0.033)
RegexpMatchEasy0_1K     539ns × (1.00,1.00)   539ns × (0.99,1.01)    ~    (p=0.445)
RegexpMatchEasy1_32     138ns × (1.00,1.01)   139ns × (0.99,1.02)    ~    (p=0.281)
RegexpMatchEasy1_1K     887ns × (1.00,1.01)   887ns × (1.00,1.00)    ~    (p=0.610)
RegexpMatchMedium_32    251ns × (1.00,1.02)   255ns × (0.99,1.01)  +1.42% (p=0.000)
RegexpMatchMedium_1K   72.7µs × (1.00,1.00)  72.6µs × (1.00,1.00)    ~    (p=0.097)
RegexpMatchHard_32     3.85µs × (1.00,1.00)  3.84µs × (1.00,1.00)  -0.31% (p=0.000)
RegexpMatchHard_1K      117µs × (1.00,1.00)   117µs × (1.00,1.00)    ~    (p=0.704)
Revcomp                 923ms × (0.98,1.02)   925ms × (0.99,1.01)    ~    (p=0.574)
Template                126ms × (0.98,1.03)   130ms × (0.99,1.01)  +3.28% (p=0.000)
TimeParse               631ns × (0.99,1.02)   626ns × (1.00,1.00)    ~    (p=0.053)
TimeFormat              660ns × (0.99,1.01)   662ns × (0.99,1.02)    ~    (p=0.398)

Change-Id: I59c03d329fe7bc178a31477c6f1f01062b881041
Reviewed-on: https://go-review.googlesource.com/9993
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go
index 409c1948c6af77d40ead220e2bdf142598978ce4..d3e4809737970cc884793ca5eb9ba71bc9bc1e2d 100644 (file)
--- a/src/runtime/mbarrier.go
+++ b/src/runtime/mbarrier.go
@@ -200,6 +200,8 @@ func callwritebarrier(typ *_type, frame unsafe.Pointer, framesize, retoffset uin
 
 //go:nosplit
 func typedslicecopy(typ *_type, dst, src slice) int {
+       // TODO(rsc): If typedslicecopy becomes faster than calling
+       // typedmemmove repeatedly, consider using during func growslice.
        n := dst.len
        if n > src.len {
                n = src.len
@@ -217,6 +219,10 @@ func typedslicecopy(typ *_type, dst, src slice) int {
                racereadrangepc(srcp, uintptr(n)*typ.size, callerpc, pc)
        }
 
+       // Note: No point in checking typ.kind&kindNoPointers here:
+       // compiler only emits calls to typedslicecopy for types with pointers,
+       // and growslice and reflect_typedslicecopy check for pointers
+       // before calling typedslicecopy.
        if !writeBarrierEnabled {
                memmove(dstp, srcp, uintptr(n)*typ.size)
                return n
@@ -257,5 +263,13 @@ func typedslicecopy(typ *_type, dst, src slice) int {
 
 //go:linkname reflect_typedslicecopy reflect.typedslicecopy
 func reflect_typedslicecopy(elemType *_type, dst, src slice) int {
+       if elemType.kind&kindNoPointers != 0 {
+               n := dst.len
+               if n > src.len {
+                       n = src.len
+               }
+               memmove(dst.array, src.array, uintptr(n)*elemType.size)
+               return n
+       }
        return typedslicecopy(elemType, dst, src)
 }

diff --git a/src/runtime/slice.go b/src/runtime/slice.go
index 5ccc6592bfe772ae1b906688fea6a51153adbab4..79b611839d49ac66240f81e01e736f4e971c6e32 100644 (file)
--- a/src/runtime/slice.go
+++ b/src/runtime/slice.go
@@ -84,10 +84,13 @@ func growslice(t *slicetype, old slice, n int) slice {
                memclr(add(p, lenmem), capmem-lenmem)
        } else {
                // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan unitialized memory.
-               // TODO(rsc): Use memmove when !writeBarrierEnabled.
                p = newarray(et, uintptr(newcap))
-               for i := 0; i < old.len; i++ {
-                       typedmemmove(et, add(p, uintptr(i)*et.size), add(old.array, uintptr(i)*et.size))
+               if !writeBarrierEnabled {
+                       memmove(p, old.array, lenmem)
+               } else {
+                       for i := uintptr(0); i < lenmem; i += et.size {
+                               typedmemmove(et, add(p, i), add(old.array, i))
+                       }
                }
        }