internal/bytealg: improve asm for memequal on ppc64x

This includes two changes to the memequal function.

Previously the asm implementation on ppc64x for Equal called the internal
function memequal using a BL, whereas the other asm implementations for
bytes functions on ppc64x used BR. The BR is preferred because the BL
causes the calling function to stack a frame. This changes Equal so it
uses BR and is consistent with the others.

This also uses vsx instructions where possible to improve performance
of the compares for sizes over 32.

Here are results from the sizes affected:

Equal/32             8.40ns ± 0%     7.66ns ± 0%    -8.81%  (p=0.029 n=4+4)
Equal/4K              193ns ± 0%      144ns ± 0%   -25.39%  (p=0.029 n=4+4)
Equal/4M              346µs ± 0%      277µs ± 0%   -20.08%  (p=0.029 n=4+4)
Equal/64M            7.66ms ± 1%     7.27ms ± 0%    -5.10%  (p=0.029 n=4+4)

Change-Id: Ib6ee2cdc3e5d146e2705e3338858b8e965d25420
Reviewed-on: https://go-review.googlesource.com/c/143060
Run-TryBot: Lynn Boger <laboger@linux.vnet.ibm.com>
Reviewed-by: Carlos Eduardo Seo <cseo@linux.vnet.ibm.com>
Reviewed-by: David Chase <drchase@google.com>

diff --git a/src/internal/bytealg/equal_ppc64x.s b/src/internal/bytealg/equal_ppc64x.s
index 9c9cf77588ad1991dfd8be5f14117f270f12e726..34d2a2574b7d5715dc09defa2c26a15fd6c482da 100644 (file)
--- a/src/internal/bytealg/equal_ppc64x.s
+++ b/src/internal/bytealg/equal_ppc64x.s
@@ -7,17 +7,15 @@
 #include "go_asm.h"
 #include "textflag.h"
 
-TEXT ·Equal(SB),NOSPLIT,$0-49
+TEXT ·Equal(SB),NOSPLIT|NOFRAME,$0-49
        MOVD    a_len+8(FP), R4
        MOVD    b_len+32(FP), R5
        CMP     R5, R4          // unequal lengths are not equal
        BNE     noteq
        MOVD    a_base+0(FP), R3
        MOVD    b_base+24(FP), R4
-       BL      memeqbody<>(SB)
-
-       MOVBZ   R9,ret+48(FP)
-       RET
+       MOVD    $ret+48(FP), R10
+       BR      memeqbody<>(SB)
 
 noteq:
        MOVBZ   $0,ret+48(FP)
@@ -28,7 +26,7 @@ equal:
        MOVBZ   R3,ret+48(FP)
        RET
 
-TEXT bytes·Equal(SB),NOSPLIT,$0-49
+TEXT bytes·Equal(SB),NOSPLIT|NOFRAME,$0-49
        FUNCDATA $0, ·Equal·args_stackmap(SB)
        MOVD    a_len+8(FP), R4
        MOVD    b_len+32(FP), R5
@@ -36,10 +34,8 @@ TEXT bytes·Equal(SB),NOSPLIT,$0-49
        BNE     noteq
        MOVD    a_base+0(FP), R3
        MOVD    b_base+24(FP), R4
-       BL      memeqbody<>(SB)
-
-       MOVBZ   R9,ret+48(FP)
-       RET
+       MOVD    $ret+48(FP), R10
+       BR      memeqbody<>(SB)
 
 noteq:
        MOVBZ   $0,ret+48(FP)
@@ -51,25 +47,23 @@ equal:
        RET
 
 // memequal(a, b unsafe.Pointer, size uintptr) bool
-TEXT runtime·memequal(SB),NOSPLIT,$0-25
+TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25
        MOVD    a+0(FP), R3
        MOVD    b+8(FP), R4
        MOVD    size+16(FP), R5
+       MOVD    $ret+24(FP), R10
 
-       BL      memeqbody<>(SB)
-       MOVB    R9, ret+24(FP)
-       RET
+       BR      memeqbody<>(SB)
 
 // memequal_varlen(a, b unsafe.Pointer) bool
-TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17
+TEXT runtime·memequal_varlen(SB),NOSPLIT|NOFRAME,$0-17
        MOVD    a+0(FP), R3
        MOVD    b+8(FP), R4
        CMP     R3, R4
        BEQ     eq
        MOVD    8(R11), R5    // compiler stores size at offset 8 in the closure
-       BL      memeqbody<>(SB)
-       MOVB    R9, ret+16(FP)
-       RET
+       MOVD    $ret+16(FP), R10
+       BR      memeqbody<>(SB)
 eq:
        MOVD    $1, R3
        MOVB    R3, ret+16(FP)
@@ -79,7 +73,7 @@ eq:
 // R3 = s1
 // R4 = s2
 // R5 = len
-// R9 = return value
+// R10 = addr of return value (byte)
 TEXT memeqbody<>(SB),NOSPLIT|NOFRAME,$0-0
        MOVD    R5,CTR
        CMP     R5,$8           // only optimize >=8
@@ -92,26 +86,19 @@ TEXT memeqbody<>(SB),NOSPLIT|NOFRAME,$0-0
 setup32a:                       // 8 byte aligned, >= 32 bytes
        SRADCC  $5,R5,R6        // number of 32 byte chunks to compare
        MOVD    R6,CTR
+       MOVD    $16,R14         // index for VSX loads and stores
 loop32a:
-       MOVD    0(R3),R6        // doublewords to compare
-       MOVD    0(R4),R7
-       MOVD    8(R3),R8        //
-       MOVD    8(R4),R9
-       CMP     R6,R7           // bytes batch?
-       BNE     noteq
-       MOVD    16(R3),R6
-       MOVD    16(R4),R7
-       CMP     R8,R9           // bytes match?
-       MOVD    24(R3),R8
-       MOVD    24(R4),R9
-       BNE     noteq
-       CMP     R6,R7           // bytes match?
-       BNE     noteq
+       LXVD2X  (R3+R0), VS32   // VS32 = V0
+       LXVD2X  (R4+R0), VS33   // VS33 = V1
+       VCMPEQUBCC V0, V1, V2   // compare, setting CR6
+       BGE     CR6, noteq
+       LXVD2X  (R3+R14), VS32
+       LXVD2X  (R4+R14), VS33
+       VCMPEQUBCC V0, V1, V2
+       BGE     CR6, noteq
        ADD     $32,R3          // bump up to next 32
        ADD     $32,R4
-       CMP     R8,R9           // bytes match?
-       BC      8,2,loop32a     // br ctr and cr
-       BNE     noteq
+       BC      16, 0, loop32a  // br ctr and cr
        ANDCC   $24,R5,R6       // Any 8 byte chunks?
        BEQ     leftover        // and result is 0
 setup8a:
@@ -145,9 +132,10 @@ simple:
        BNE     noteq
        BR      equal
 noteq:
-       MOVD    $0, R9
+       MOVB    $0, (R10)
        RET
 equal:
-       MOVD    $1, R9
+       MOVD    $1, R3
+       MOVB    R3, (R10)
        RET