MOVL to+0(FP), DI
MOVL fr+4(FP), SI
MOVL n+8(FP), BX
+
+ // REP instructions have a high startup cost, so we handle small sizes
+ // with some straightline code. The REP MOVSL instruction is really fast
+ // for large sizes. The cutover is approximately 1K. We implement up to
+ // 128 because that is the maximum SSE register load (loading all data
+ // into registers lets us ignore copy direction).
+tail:
+ TESTL BX, BX
+ JEQ move_0
+ CMPL BX, $2
+ JBE move_1or2
+ CMPL BX, $4
+ JBE move_3or4
+ CMPL BX, $8
+ JBE move_5through8
+ CMPL BX, $16
+ JBE move_9through16
+ TESTL $0x4000000, runtime·cpuid_edx(SB) // check for sse2
+ JEQ nosse2
+ CMPL BX, $32
+ JBE move_17through32
+ CMPL BX, $64
+ JBE move_33through64
+ CMPL BX, $128
+ JBE move_65through128
+ // TODO: use branch table and BSR to make this just a single dispatch
+
+nosse2:
/*
* check and set for backwards
*/
ANDL $3, BX
REP; MOVSL
- MOVL BX, CX
- REP; MOVSB
-
- MOVL to+0(FP),AX
- RET
+ JMP tail
/*
* check overlap
*/
SUBL $4, SI
REP; MOVSL
- ADDL $3, DI
- ADDL $3, SI
- MOVL BX, CX
- REP; MOVSB
-
CLD
- MOVL to+0(FP),AX
- RET
+ ADDL $4, DI
+ ADDL $4, SI
+ SUBL BX, DI
+ SUBL BX, SI
+ JMP tail
+move_1or2:
+ MOVB (SI), AX
+ MOVB -1(SI)(BX*1), CX
+ MOVB AX, (DI)
+ MOVB CX, -1(DI)(BX*1)
+move_0:
+ RET
+move_3or4:
+ MOVW (SI), AX
+ MOVW -2(SI)(BX*1), CX
+ MOVW AX, (DI)
+ MOVW CX, -2(DI)(BX*1)
+ RET
+move_5through8:
+ MOVL (SI), AX
+ MOVL -4(SI)(BX*1), CX
+ MOVL AX, (DI)
+ MOVL CX, -4(DI)(BX*1)
+ RET
+move_9through16:
+ MOVL (SI), AX
+ MOVL 4(SI), CX
+ MOVL -8(SI)(BX*1), DX
+ MOVL -4(SI)(BX*1), BP
+ MOVL AX, (DI)
+ MOVL CX, 4(DI)
+ MOVL DX, -8(DI)(BX*1)
+ MOVL BP, -4(DI)(BX*1)
+ RET
+move_17through32:
+ MOVOU (SI), X0
+ MOVOU -16(SI)(BX*1), X1
+ MOVOU X0, (DI)
+ MOVOU X1, -16(DI)(BX*1)
+ RET
+move_33through64:
+ MOVOU (SI), X0
+ MOVOU 16(SI), X1
+ MOVOU -32(SI)(BX*1), X2
+ MOVOU -16(SI)(BX*1), X3
+ MOVOU X0, (DI)
+ MOVOU X1, 16(DI)
+ MOVOU X2, -32(DI)(BX*1)
+ MOVOU X3, -16(DI)(BX*1)
+ RET
+move_65through128:
+ MOVOU (SI), X0
+ MOVOU 16(SI), X1
+ MOVOU 32(SI), X2
+ MOVOU 48(SI), X3
+ MOVOU -64(SI)(BX*1), X4
+ MOVOU -48(SI)(BX*1), X5
+ MOVOU -32(SI)(BX*1), X6
+ MOVOU -16(SI)(BX*1), X7
+ MOVOU X0, (DI)
+ MOVOU X1, 16(DI)
+ MOVOU X2, 32(DI)
+ MOVOU X3, 48(DI)
+ MOVOU X4, -64(DI)(BX*1)
+ MOVOU X5, -48(DI)(BX*1)
+ MOVOU X6, -32(DI)(BX*1)
+ MOVOU X7, -16(DI)(BX*1)
+ RET
MOVQ fr+8(FP), SI
MOVQ n+16(FP), BX
+ // REP instructions have a high startup cost, so we handle small sizes
+ // with some straightline code. The REP MOVSQ instruction is really fast
+ // for large sizes. The cutover is approximately 1K. We implement up to
+ // 256 because that is the maximum SSE register load (loading all data
+ // into registers lets us ignore copy direction).
+tail:
+ TESTQ BX, BX
+ JEQ move_0
+ CMPQ BX, $2
+ JBE move_1or2
+ CMPQ BX, $4
+ JBE move_3or4
+ CMPQ BX, $8
+ JBE move_5through8
+ CMPQ BX, $16
+ JBE move_9through16
+ CMPQ BX, $32
+ JBE move_17through32
+ CMPQ BX, $64
+ JBE move_33through64
+ CMPQ BX, $128
+ JBE move_65through128
+ CMPQ BX, $256
+ JBE move_129through256
+ // TODO: use branch table and BSR to make this just a single dispatch
+
/*
* check and set for backwards
*/
ANDQ $7, BX
REP; MOVSQ
- MOVQ BX, CX
- REP; MOVSB
+ JMP tail
- MOVQ to+0(FP),AX
- RET
back:
/*
* check overlap
SUBQ $8, SI
REP; MOVSQ
- ADDQ $7, DI
- ADDQ $7, SI
- MOVQ BX, CX
- REP; MOVSB
-
CLD
- MOVQ to+0(FP),AX
- RET
+ ADDQ $8, DI
+ ADDQ $8, SI
+ SUBQ BX, DI
+ SUBQ BX, SI
+ JMP tail
+move_1or2:
+ MOVB (SI), AX
+ MOVB -1(SI)(BX*1), CX
+ MOVB AX, (DI)
+ MOVB CX, -1(DI)(BX*1)
+move_0:
+ RET
+move_3or4:
+ MOVW (SI), AX
+ MOVW -2(SI)(BX*1), CX
+ MOVW AX, (DI)
+ MOVW CX, -2(DI)(BX*1)
+ RET
+move_5through8:
+ MOVL (SI), AX
+ MOVL -4(SI)(BX*1), CX
+ MOVL AX, (DI)
+ MOVL CX, -4(DI)(BX*1)
+ RET
+move_9through16:
+ MOVQ (SI), AX
+ MOVQ -8(SI)(BX*1), CX
+ MOVQ AX, (DI)
+ MOVQ CX, -8(DI)(BX*1)
+ RET
+move_17through32:
+ MOVOU (SI), X0
+ MOVOU -16(SI)(BX*1), X1
+ MOVOU X0, (DI)
+ MOVOU X1, -16(DI)(BX*1)
+ RET
+move_33through64:
+ MOVOU (SI), X0
+ MOVOU 16(SI), X1
+ MOVOU -32(SI)(BX*1), X2
+ MOVOU -16(SI)(BX*1), X3
+ MOVOU X0, (DI)
+ MOVOU X1, 16(DI)
+ MOVOU X2, -32(DI)(BX*1)
+ MOVOU X3, -16(DI)(BX*1)
+ RET
+move_65through128:
+ MOVOU (SI), X0
+ MOVOU 16(SI), X1
+ MOVOU 32(SI), X2
+ MOVOU 48(SI), X3
+ MOVOU -64(SI)(BX*1), X4
+ MOVOU -48(SI)(BX*1), X5
+ MOVOU -32(SI)(BX*1), X6
+ MOVOU -16(SI)(BX*1), X7
+ MOVOU X0, (DI)
+ MOVOU X1, 16(DI)
+ MOVOU X2, 32(DI)
+ MOVOU X3, 48(DI)
+ MOVOU X4, -64(DI)(BX*1)
+ MOVOU X5, -48(DI)(BX*1)
+ MOVOU X6, -32(DI)(BX*1)
+ MOVOU X7, -16(DI)(BX*1)
+ RET
+move_129through256:
+ MOVOU (SI), X0
+ MOVOU 16(SI), X1
+ MOVOU 32(SI), X2
+ MOVOU 48(SI), X3
+ MOVOU 64(SI), X4
+ MOVOU 80(SI), X5
+ MOVOU 96(SI), X6
+ MOVOU 112(SI), X7
+ MOVOU -128(SI)(BX*1), X8
+ MOVOU -112(SI)(BX*1), X9
+ MOVOU -96(SI)(BX*1), X10
+ MOVOU -80(SI)(BX*1), X11
+ MOVOU -64(SI)(BX*1), X12
+ MOVOU -48(SI)(BX*1), X13
+ MOVOU -32(SI)(BX*1), X14
+ MOVOU -16(SI)(BX*1), X15
+ MOVOU X0, (DI)
+ MOVOU X1, 16(DI)
+ MOVOU X2, 32(DI)
+ MOVOU X3, 48(DI)
+ MOVOU X4, 64(DI)
+ MOVOU X5, 80(DI)
+ MOVOU X6, 96(DI)
+ MOVOU X7, 112(DI)
+ MOVOU X8, -128(DI)(BX*1)
+ MOVOU X9, -112(DI)(BX*1)
+ MOVOU X10, -96(DI)(BX*1)
+ MOVOU X11, -80(DI)(BX*1)
+ MOVOU X12, -64(DI)(BX*1)
+ MOVOU X13, -48(DI)(BX*1)
+ MOVOU X14, -32(DI)(BX*1)
+ MOVOU X15, -16(DI)(BX*1)
+ RET
--- /dev/null
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime_test
+
+import (
+ "testing"
+)
+
+func TestMemmove(t *testing.T) {
+ size := 256
+ if testing.Short() {
+ size = 128 + 16
+ }
+ src := make([]byte, size)
+ dst := make([]byte, size)
+ for i := 0; i < size; i++ {
+ src[i] = byte(128 + (i & 127))
+ }
+ for i := 0; i < size; i++ {
+ dst[i] = byte(i & 127)
+ }
+ for n := 0; n <= size; n++ {
+ for x := 0; x <= size-n; x++ { // offset in src
+ for y := 0; y <= size-n; y++ { // offset in dst
+ copy(dst[y:y+n], src[x:x+n])
+ for i := 0; i < y; i++ {
+ if dst[i] != byte(i&127) {
+ t.Fatalf("prefix dst[%d] = %d", i, dst[i])
+ }
+ }
+ for i := y; i < y+n; i++ {
+ if dst[i] != byte(128+((i-y+x)&127)) {
+ t.Fatalf("copied dst[%d] = %d", i, dst[i])
+ }
+ dst[i] = byte(i & 127) // reset dst
+ }
+ for i := y + n; i < size; i++ {
+ if dst[i] != byte(i&127) {
+ t.Fatalf("suffix dst[%d] = %d", i, dst[i])
+ }
+ }
+ }
+ }
+ }
+}
+
+func TestMemmoveAlias(t *testing.T) {
+ size := 256
+ if testing.Short() {
+ size = 128 + 16
+ }
+ buf := make([]byte, size)
+ for i := 0; i < size; i++ {
+ buf[i] = byte(i)
+ }
+ for n := 0; n <= size; n++ {
+ for x := 0; x <= size-n; x++ { // src offset
+ for y := 0; y <= size-n; y++ { // dst offset
+ copy(buf[y:y+n], buf[x:x+n])
+ for i := 0; i < y; i++ {
+ if buf[i] != byte(i) {
+ t.Fatalf("prefix buf[%d] = %d", i, buf[i])
+ }
+ }
+ for i := y; i < y+n; i++ {
+ if buf[i] != byte(i-y+x) {
+ t.Fatalf("copied buf[%d] = %d", i, buf[i])
+ }
+ buf[i] = byte(i) // reset buf
+ }
+ for i := y + n; i < size; i++ {
+ if buf[i] != byte(i) {
+ t.Fatalf("suffix buf[%d] = %d", i, buf[i])
+ }
+ }
+ }
+ }
+ }
+}
+
+func bmMemmove(n int, b *testing.B) {
+ x := make([]byte, n)
+ y := make([]byte, n)
+ b.SetBytes(int64(n))
+ for i := 0; i < b.N; i++ {
+ copy(x, y)
+ }
+}
+
+func BenchmarkMemmove0(b *testing.B) { bmMemmove(0, b) }
+func BenchmarkMemmove1(b *testing.B) { bmMemmove(1, b) }
+func BenchmarkMemmove2(b *testing.B) { bmMemmove(2, b) }
+func BenchmarkMemmove3(b *testing.B) { bmMemmove(3, b) }
+func BenchmarkMemmove4(b *testing.B) { bmMemmove(4, b) }
+func BenchmarkMemmove5(b *testing.B) { bmMemmove(5, b) }
+func BenchmarkMemmove6(b *testing.B) { bmMemmove(6, b) }
+func BenchmarkMemmove7(b *testing.B) { bmMemmove(7, b) }
+func BenchmarkMemmove8(b *testing.B) { bmMemmove(8, b) }
+func BenchmarkMemmove9(b *testing.B) { bmMemmove(9, b) }
+func BenchmarkMemmove10(b *testing.B) { bmMemmove(10, b) }
+func BenchmarkMemmove11(b *testing.B) { bmMemmove(11, b) }
+func BenchmarkMemmove12(b *testing.B) { bmMemmove(12, b) }
+func BenchmarkMemmove13(b *testing.B) { bmMemmove(13, b) }
+func BenchmarkMemmove14(b *testing.B) { bmMemmove(14, b) }
+func BenchmarkMemmove15(b *testing.B) { bmMemmove(15, b) }
+func BenchmarkMemmove16(b *testing.B) { bmMemmove(16, b) }
+func BenchmarkMemmove32(b *testing.B) { bmMemmove(32, b) }
+func BenchmarkMemmove64(b *testing.B) { bmMemmove(64, b) }
+func BenchmarkMemmove128(b *testing.B) { bmMemmove(128, b) }
+func BenchmarkMemmove256(b *testing.B) { bmMemmove(256, b) }
+func BenchmarkMemmove512(b *testing.B) { bmMemmove(512, b) }
+func BenchmarkMemmove1024(b *testing.B) { bmMemmove(1024, b) }
+func BenchmarkMemmove2048(b *testing.B) { bmMemmove(2048, b) }
+func BenchmarkMemmove4096(b *testing.B) { bmMemmove(4096, b) }