crypto/sha256, cmd/internal/notsha256: improve PPC64 sha256

This minimizes addi usage inside vector heavy loops. This
results in a small performance uptick on P9 ppc64le/linux.
Likewise, cleanup some minor whitespace issues around comments.

The implementation from crypto/sha256 is also shared with notsha256.
It is copied, but preserves notsha256's go:build directives. They are
otherwise identical now. Previously, bootstrap restrictions required
workarounds to support XXLOR on older toolchains. This is not needed
anymore as the minimum bootstrap (1.17) compiler will support XXLOR.

name               old speed      new speed      delta
Hash8Bytes/New     28.8MB/s ± 0%  30.5MB/s ± 0%  +5.98%
Hash8Bytes/Sum224  29.5MB/s ± 0%  31.3MB/s ± 0%  +6.17%
Hash8Bytes/Sum256  29.5MB/s ± 0%  31.2MB/s ± 0%  +5.80%
Hash1K/New          287MB/s ± 0%   312MB/s ± 0%  +8.60%
Hash1K/Sum224       289MB/s ± 0%   312MB/s ± 0%  +7.99%
Hash1K/Sum256       289MB/s ± 0%   312MB/s ± 0%  +7.98%
Hash8K/New          313MB/s ± 0%   338MB/s ± 0%  +8.12%
Hash8K/Sum224       313MB/s ± 0%   338MB/s ± 0%  +8.20%
Hash8K/Sum256       313MB/s ± 0%   338MB/s ± 0%  +8.12%

Change-Id: Ib386d6306673b4e6553ee745ec2e1b53a9722df1
Reviewed-on: https://go-review.googlesource.com/c/go/+/441815
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>
Reviewed-by: Lynn Boger <laboger@linux.vnet.ibm.com>
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: Archana Ravindar <aravind5@in.ibm.com>
Run-TryBot: Paul Murphy <murp@ibm.com>

diff --git a/src/cmd/internal/notsha256/sha256block_ppc64x.s b/src/cmd/internal/notsha256/sha256block_ppc64x.s
index e907d3b71beca7bef8951b2ad6a5dddbd9ebc55c..ea4417d6c6b6d74d77bde65d7a0ba71eb003eb67 100644 (file)
--- a/src/cmd/internal/notsha256/sha256block_ppc64x.s
+++ b/src/cmd/internal/notsha256/sha256block_ppc64x.s
@@ -65,13 +65,31 @@
 #define CTX    R3
 #define INP    R4
 #define END    R5
-#define TBL    R6
-#define IDX    R7
+#define TBL    R6 // Pointer into kcon table
 #define LEN    R9
 #define TEMP   R12
 
-#define HEX00  R0
-#define HEX10  R10
+#define TBL_STRT       R7 // Pointer to start of kcon table.
+
+#define R_x000 R0
+#define R_x010 R8
+#define R_x020 R10
+#define R_x030 R11
+#define R_x040 R14
+#define R_x050 R15
+#define R_x060 R16
+#define R_x070 R17
+#define R_x080 R18
+#define R_x090 R19
+#define R_x0a0 R20
+#define R_x0b0 R21
+#define R_x0c0 R22
+#define R_x0d0 R23
+#define R_x0e0 R24
+#define R_x0f0 R25
+#define R_x100 R26
+#define R_x110 R27
+
 
 // V0-V7 are A-H
 // V8-V23 are used for the message schedule
@@ -81,7 +99,7 @@
 #define S1     V27
 #define s0     V28
 #define s1     V29
-#define LEMASK V31     // Permutation control register for little endian
+#define LEMASK V31 // Permutation control register for little endian
 
 // 4 copies of each Kt, to fill all 4 words of a vector register
 DATA  ·kcon+0x000(SB)/8, $0x428a2f98428a2f98
@@ -216,7 +234,7 @@ DATA  ·kcon+0x400(SB)/8, $0x0000000000000000
 DATA  ·kcon+0x408(SB)/8, $0x0000000000000000
 
 #ifdef GOARCH_ppc64le
-DATA  ·kcon+0x410(SB)/8, $0x1011121310111213  // permutation control vectors
+DATA  ·kcon+0x410(SB)/8, $0x1011121310111213 // permutation control vectors
 DATA  ·kcon+0x418(SB)/8, $0x1011121300010203
 DATA  ·kcon+0x420(SB)/8, $0x1011121310111213
 DATA  ·kcon+0x428(SB)/8, $0x0405060700010203
@@ -224,7 +242,7 @@ DATA  ·kcon+0x430(SB)/8, $0x1011121308090a0b
 DATA  ·kcon+0x438(SB)/8, $0x0405060700010203
 #else
 DATA  ·kcon+0x410(SB)/8, $0x1011121300010203
-DATA  ·kcon+0x418(SB)/8, $0x1011121310111213  // permutation control vectors
+DATA  ·kcon+0x418(SB)/8, $0x1011121310111213 // permutation control vectors
 DATA  ·kcon+0x420(SB)/8, $0x0405060700010203
 DATA  ·kcon+0x428(SB)/8, $0x1011121310111213
 DATA  ·kcon+0x430(SB)/8, $0x0001020304050607
@@ -233,7 +251,7 @@ DATA  ·kcon+0x438(SB)/8, $0x08090a0b10111213
 
 GLOBL ·kcon(SB), RODATA, $1088
 
-#define SHA256ROUND0(a, b, c, d, e, f, g, h, xi) \
+#define SHA256ROUND0(a, b, c, d, e, f, g, h, xi, idx) \
        VSEL            g, f, e, FUNC; \
        VSHASIGMAW      $15, e, $1, S1; \
        VADDUWM         xi, h, h; \
@@ -245,11 +263,10 @@ GLOBL ·kcon(SB), RODATA, $1088
        VADDUWM         KI, g, g; \
        VADDUWM         h, d, d; \
        VADDUWM         FUNC, S0, S0; \
-       LVX             (TBL)(IDX), KI; \
-       ADD             $16, IDX; \
+       LVX             (TBL)(idx), KI; \
        VADDUWM         S0, h, h
 
-#define SHA256ROUND1(a, b, c, d, e, f, g, h, xi, xj, xj_1, xj_9, xj_14) \
+#define SHA256ROUND1(a, b, c, d, e, f, g, h, xi, xj, xj_1, xj_9, xj_14, idx) \
        VSHASIGMAW      $0, xj_1, $0, s0; \
        VSEL            g, f, e, FUNC; \
        VSHASIGMAW      $15, e, $1, S1; \
@@ -265,8 +282,7 @@ GLOBL ·kcon(SB), RODATA, $1088
        VADDUWM         h, d, d; \
        VADDUWM         FUNC, S0, S0; \
        VADDUWM         s0, xj, xj; \
-       LVX             (TBL)(IDX), KI; \
-       ADD             $16, IDX; \
+       LVX             (TBL)(idx), KI; \
        VADDUWM         S0, h, h; \
        VADDUWM         s1, xj, xj
 
@@ -289,18 +305,18 @@ TEXT ·block(SB),0,$0-32
        CMP     INP, END
        BEQ     end
 
-       MOVD    $·kcon(SB), TBL
-       MOVWZ   $0x10, HEX10
-       MOVWZ   $8, IDX
+       MOVD    $·kcon(SB), TBL_STRT
+       MOVD    $0x10, R_x010
 
 #ifdef GOARCH_ppc64le
-       LVSL    (IDX)(R0), LEMASK
+       MOVWZ   $8, TEMP
+       LVSL    (TEMP)(R0), LEMASK
        VSPLTISB        $0x0F, KI
        VXOR    KI, LEMASK, LEMASK
 #endif
 
-       LXVW4X  (CTX)(HEX00), VS32      // v0 = vs32
-       LXVW4X  (CTX)(HEX10), VS36      // v4 = vs36
+       LXVW4X  (CTX)(R_x000), V0
+       LXVW4X  (CTX)(R_x010), V4
 
        // unpack the input values into vector registers
        VSLDOI  $4, V0, V0, V1
@@ -310,121 +326,135 @@ TEXT ·block(SB),0,$0-32
        VSLDOI  $8, V4, V4, V6
        VSLDOI  $12, V4, V4, V7
 
+       MOVD    $0x020, R_x020
+       MOVD    $0x030, R_x030
+       MOVD    $0x040, R_x040
+       MOVD    $0x050, R_x050
+       MOVD    $0x060, R_x060
+       MOVD    $0x070, R_x070
+       MOVD    $0x080, R_x080
+       MOVD    $0x090, R_x090
+       MOVD    $0x0a0, R_x0a0
+       MOVD    $0x0b0, R_x0b0
+       MOVD    $0x0c0, R_x0c0
+       MOVD    $0x0d0, R_x0d0
+       MOVD    $0x0e0, R_x0e0
+       MOVD    $0x0f0, R_x0f0
+       MOVD    $0x100, R_x100
+       MOVD    $0x110, R_x110
+
 loop:
-       LVX     (TBL)(HEX00), KI
-       MOVWZ   $16, IDX
+       MOVD    TBL_STRT, TBL
+       LVX     (TBL)(R_x000), KI
 
-       LXVD2X  (INP)(R0), VS40 // load v8 (=vs40) in advance
-       ADD     $16, INP
+       LXVD2X  (INP)(R_x000), V8 // load v8 in advance
 
        // Offload to VSR24-31 (aka FPR24-31)
-       XXLORQ  VS32, VS32, VS24
-       XXLORQ  VS33, VS33, VS25
-       XXLORQ  VS34, VS34, VS26
-       XXLORQ  VS35, VS35, VS27
-       XXLORQ  VS36, VS36, VS28
-       XXLORQ  VS37, VS37, VS29
-       XXLORQ  VS38, VS38, VS30
-       XXLORQ  VS39, VS39, VS31
-
-       VADDUWM KI, V7, V7      // h+K[i]
-       LVX     (TBL)(IDX), KI
-       ADD     $16, IDX
+       XXLOR   V0, V0, VS24
+       XXLOR   V1, V1, VS25
+       XXLOR   V2, V2, VS26
+       XXLOR   V3, V3, VS27
+       XXLOR   V4, V4, VS28
+       XXLOR   V5, V5, VS29
+       XXLOR   V6, V6, VS30
+       XXLOR   V7, V7, VS31
+
+       VADDUWM KI, V7, V7        // h+K[i]
+       LVX     (TBL)(R_x010), KI
 
        VPERMLE(V8, V8, LEMASK, V8)
-       SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V8)
+       SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V8, R_x020)
        VSLDOI  $4, V8, V8, V9
-       SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V9)
+       SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V9, R_x030)
        VSLDOI  $4, V9, V9, V10
-       SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V10)
-       LXVD2X  (INP)(R0), VS44 // load v12 (=vs44) in advance
-       ADD     $16, INP, INP
+       SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V10, R_x040)
+       LXVD2X  (INP)(R_x010), V12 // load v12 in advance
        VSLDOI  $4, V10, V10, V11
-       SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V11)
+       SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V11, R_x050)
        VPERMLE(V12, V12, LEMASK, V12)
-       SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V12)
+       SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V12, R_x060)
        VSLDOI  $4, V12, V12, V13
-       SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V13)
+       SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V13, R_x070)
        VSLDOI  $4, V13, V13, V14
-       SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V14)
-       LXVD2X  (INP)(R0), VS48 // load v16 (=vs48) in advance
-       ADD     $16, INP, INP
+       SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V14, R_x080)
+       LXVD2X  (INP)(R_x020), V16 // load v16 in advance
        VSLDOI  $4, V14, V14, V15
-       SHA256ROUND0(V1, V2, V3, V4, V5, V6, V7, V0, V15)
+       SHA256ROUND0(V1, V2, V3, V4, V5, V6, V7, V0, V15, R_x090)
        VPERMLE(V16, V16, LEMASK, V16)
-       SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V16)
+       SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V16, R_x0a0)
        VSLDOI  $4, V16, V16, V17
-       SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V17)
+       SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V17, R_x0b0)
        VSLDOI  $4, V17, V17, V18
-       SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V18)
+       SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V18, R_x0c0)
        VSLDOI  $4, V18, V18, V19
-       LXVD2X  (INP)(R0), VS52 // load v20 (=vs52) in advance
-       ADD     $16, INP, INP
-       SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V19)
+       LXVD2X  (INP)(R_x030), V20 // load v20 in advance
+       SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V19, R_x0d0)
        VPERMLE(V20, V20, LEMASK, V20)
-       SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V20)
+       SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V20, R_x0e0)
        VSLDOI  $4, V20, V20, V21
-       SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V21)
+       SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V21, R_x0f0)
        VSLDOI  $4, V21, V21, V22
-       SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V22)
+       SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V22, R_x100)
        VSLDOI  $4, V22, V22, V23
-       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22)
+       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22, R_x110)
 
-       MOVWZ   $3, TEMP
-       MOVWZ   TEMP, CTR
+       MOVD    $3, TEMP
+       MOVD    TEMP, CTR
+       ADD     $0x120, TBL
+       ADD     $0x40, INP
 
 L16_xx:
-       SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V18, V23)
-       SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V9, V10, V11, V19, V8)
-       SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V10, V11, V12, V20, V9)
-       SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V11, V12, V13, V21, V10)
-       SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V12, V13, V14, V22, V11)
-       SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V13, V14, V15, V23, V12)
-       SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V14, V15, V16, V8, V13)
-       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V15, V16, V17, V9, V14)
-       SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V16, V17, V18, V10, V15)
-       SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V17, V18, V19, V11, V16)
-       SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V18, V19, V20, V12, V17)
-       SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V19, V20, V21, V13, V18)
-       SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V20, V21, V22, V14, V19)
-       SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V21, V22, V23, V15, V20)
-       SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V22, V23, V8, V16, V21)
-       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22)
-
-       BC      0x10, 0, L16_xx         // bdnz
-
-       XXLORQ  VS24, VS24, VS42
-
-       XXLORQ  VS25, VS25, VS43
+       SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V18, V23, R_x000)
+       SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V9, V10, V11, V19, V8, R_x010)
+       SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V10, V11, V12, V20, V9, R_x020)
+       SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V11, V12, V13, V21, V10, R_x030)
+       SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V12, V13, V14, V22, V11, R_x040)
+       SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V13, V14, V15, V23, V12, R_x050)
+       SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V14, V15, V16, V8, V13, R_x060)
+       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V15, V16, V17, V9, V14, R_x070)
+       SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V16, V17, V18, V10, V15, R_x080)
+       SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V17, V18, V19, V11, V16, R_x090)
+       SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V18, V19, V20, V12, V17, R_x0a0)
+       SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V19, V20, V21, V13, V18, R_x0b0)
+       SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V20, V21, V22, V14, V19, R_x0c0)
+       SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V21, V22, V23, V15, V20, R_x0d0)
+       SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V22, V23, V8, V16, V21, R_x0e0)
+       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22, R_x0f0)
+       ADD     $0x100, TBL
+
+       BDNZ    L16_xx
+
+       XXLOR   VS24, VS24, V10
+
+       XXLOR   VS25, VS25, V11
        VADDUWM V10, V0, V0
-       XXLORQ  VS26, VS26, VS44
+       XXLOR   VS26, VS26, V12
        VADDUWM V11, V1, V1
-       XXLORQ  VS27, VS27, VS45
+       XXLOR   VS27, VS27, V13
        VADDUWM V12, V2, V2
-       XXLORQ  VS28, VS28, VS46
+       XXLOR   VS28, VS28, V14
        VADDUWM V13, V3, V3
-       XXLORQ  VS29, VS29, VS47
+       XXLOR   VS29, VS29, V15
        VADDUWM V14, V4, V4
-       XXLORQ  VS30, VS30, VS48
+       XXLOR   VS30, VS30, V16
        VADDUWM V15, V5, V5
-       XXLORQ  VS31, VS31, VS49
+       XXLOR   VS31, VS31, V17
        VADDUWM V16, V6, V6
        VADDUWM V17, V7, V7
 
        CMPU    INP, END
        BLT     loop
 
-       LVX     (TBL)(IDX), V8
-       ADD     $16, IDX
+       LVX     (TBL)(R_x000), V8
        VPERM   V0, V1, KI, V0
-       LVX     (TBL)(IDX), V9
+       LVX     (TBL)(R_x010), V9
        VPERM   V4, V5, KI, V4
        VPERM   V0, V2, V8, V0
        VPERM   V4, V6, V8, V4
        VPERM   V0, V3, V9, V0
        VPERM   V4, V7, V9, V4
-       STXVD2X VS32, (CTX+HEX00)       // v0 = vs32
-       STXVD2X VS36, (CTX+HEX10)       // v4 = vs36
+       STXVD2X V0, (CTX+R_x000)
+       STXVD2X V4, (CTX+R_x010)
 
 end:
        RET

diff --git a/src/crypto/sha256/sha256block_ppc64x.s b/src/crypto/sha256/sha256block_ppc64x.s
index 617d42e1d735fc0be5808f8893dfca2562236b92..b229ef619a35bcb729967492c31ed4c27b506a1e 100644 (file)
--- a/src/crypto/sha256/sha256block_ppc64x.s
+++ b/src/crypto/sha256/sha256block_ppc64x.s
@@ -57,13 +57,31 @@
 #define CTX    R3
 #define INP    R4
 #define END    R5
-#define TBL    R6
-#define IDX    R7
+#define TBL    R6 // Pointer into kcon table
 #define LEN    R9
 #define TEMP   R12
 
-#define HEX00  R0
-#define HEX10  R10
+#define TBL_STRT       R7 // Pointer to start of kcon table.
+
+#define R_x000 R0
+#define R_x010 R8
+#define R_x020 R10
+#define R_x030 R11
+#define R_x040 R14
+#define R_x050 R15
+#define R_x060 R16
+#define R_x070 R17
+#define R_x080 R18
+#define R_x090 R19
+#define R_x0a0 R20
+#define R_x0b0 R21
+#define R_x0c0 R22
+#define R_x0d0 R23
+#define R_x0e0 R24
+#define R_x0f0 R25
+#define R_x100 R26
+#define R_x110 R27
+
 
 // V0-V7 are A-H
 // V8-V23 are used for the message schedule
@@ -73,7 +91,7 @@
 #define S1     V27
 #define s0     V28
 #define s1     V29
-#define LEMASK V31     // Permutation control register for little endian
+#define LEMASK V31 // Permutation control register for little endian
 
 // 4 copies of each Kt, to fill all 4 words of a vector register
 DATA  ·kcon+0x000(SB)/8, $0x428a2f98428a2f98
@@ -208,7 +226,7 @@ DATA  ·kcon+0x400(SB)/8, $0x0000000000000000
 DATA  ·kcon+0x408(SB)/8, $0x0000000000000000
 
 #ifdef GOARCH_ppc64le
-DATA  ·kcon+0x410(SB)/8, $0x1011121310111213  // permutation control vectors
+DATA  ·kcon+0x410(SB)/8, $0x1011121310111213 // permutation control vectors
 DATA  ·kcon+0x418(SB)/8, $0x1011121300010203
 DATA  ·kcon+0x420(SB)/8, $0x1011121310111213
 DATA  ·kcon+0x428(SB)/8, $0x0405060700010203
@@ -216,7 +234,7 @@ DATA  ·kcon+0x430(SB)/8, $0x1011121308090a0b
 DATA  ·kcon+0x438(SB)/8, $0x0405060700010203
 #else
 DATA  ·kcon+0x410(SB)/8, $0x1011121300010203
-DATA  ·kcon+0x418(SB)/8, $0x1011121310111213  // permutation control vectors
+DATA  ·kcon+0x418(SB)/8, $0x1011121310111213 // permutation control vectors
 DATA  ·kcon+0x420(SB)/8, $0x0405060700010203
 DATA  ·kcon+0x428(SB)/8, $0x1011121310111213
 DATA  ·kcon+0x430(SB)/8, $0x0001020304050607
@@ -225,7 +243,7 @@ DATA  ·kcon+0x438(SB)/8, $0x08090a0b10111213
 
 GLOBL ·kcon(SB), RODATA, $1088
 
-#define SHA256ROUND0(a, b, c, d, e, f, g, h, xi) \
+#define SHA256ROUND0(a, b, c, d, e, f, g, h, xi, idx) \
        VSEL            g, f, e, FUNC; \
        VSHASIGMAW      $15, e, $1, S1; \
        VADDUWM         xi, h, h; \
@@ -237,11 +255,10 @@ GLOBL ·kcon(SB), RODATA, $1088
        VADDUWM         KI, g, g; \
        VADDUWM         h, d, d; \
        VADDUWM         FUNC, S0, S0; \
-       LVX             (TBL)(IDX), KI; \
-       ADD             $16, IDX; \
+       LVX             (TBL)(idx), KI; \
        VADDUWM         S0, h, h
 
-#define SHA256ROUND1(a, b, c, d, e, f, g, h, xi, xj, xj_1, xj_9, xj_14) \
+#define SHA256ROUND1(a, b, c, d, e, f, g, h, xi, xj, xj_1, xj_9, xj_14, idx) \
        VSHASIGMAW      $0, xj_1, $0, s0; \
        VSEL            g, f, e, FUNC; \
        VSHASIGMAW      $15, e, $1, S1; \
@@ -257,8 +274,7 @@ GLOBL ·kcon(SB), RODATA, $1088
        VADDUWM         h, d, d; \
        VADDUWM         FUNC, S0, S0; \
        VADDUWM         s0, xj, xj; \
-       LVX             (TBL)(IDX), KI; \
-       ADD             $16, IDX; \
+       LVX             (TBL)(idx), KI; \
        VADDUWM         S0, h, h; \
        VADDUWM         s1, xj, xj
 
@@ -281,18 +297,18 @@ TEXT ·block(SB),0,$0-32
        CMP     INP, END
        BEQ     end
 
-       MOVD    $·kcon(SB), TBL
-       MOVWZ   $0x10, HEX10
-       MOVWZ   $8, IDX
+       MOVD    $·kcon(SB), TBL_STRT
+       MOVD    $0x10, R_x010
 
 #ifdef GOARCH_ppc64le
-       LVSL    (IDX)(R0), LEMASK
+       MOVWZ   $8, TEMP
+       LVSL    (TEMP)(R0), LEMASK
        VSPLTISB        $0x0F, KI
        VXOR    KI, LEMASK, LEMASK
 #endif
 
-       LXVW4X  (CTX)(HEX00), VS32      // v0 = vs32
-       LXVW4X  (CTX)(HEX10), VS36      // v4 = vs36
+       LXVW4X  (CTX)(R_x000), V0
+       LXVW4X  (CTX)(R_x010), V4
 
        // unpack the input values into vector registers
        VSLDOI  $4, V0, V0, V1
@@ -302,12 +318,28 @@ TEXT ·block(SB),0,$0-32
        VSLDOI  $8, V4, V4, V6
        VSLDOI  $12, V4, V4, V7
 
+       MOVD    $0x020, R_x020
+       MOVD    $0x030, R_x030
+       MOVD    $0x040, R_x040
+       MOVD    $0x050, R_x050
+       MOVD    $0x060, R_x060
+       MOVD    $0x070, R_x070
+       MOVD    $0x080, R_x080
+       MOVD    $0x090, R_x090
+       MOVD    $0x0a0, R_x0a0
+       MOVD    $0x0b0, R_x0b0
+       MOVD    $0x0c0, R_x0c0
+       MOVD    $0x0d0, R_x0d0
+       MOVD    $0x0e0, R_x0e0
+       MOVD    $0x0f0, R_x0f0
+       MOVD    $0x100, R_x100
+       MOVD    $0x110, R_x110
+
 loop:
-       LVX     (TBL)(HEX00), KI
-       MOVWZ   $16, IDX
+       MOVD    TBL_STRT, TBL
+       LVX     (TBL)(R_x000), KI
 
-       LXVD2X  (INP)(R0), VS40 // load v8 (=vs40) in advance
-       ADD     $16, INP
+       LXVD2X  (INP)(R_x000), V8 // load v8 in advance
 
        // Offload to VSR24-31 (aka FPR24-31)
        XXLOR   V0, V0, VS24
@@ -319,71 +351,70 @@ loop:
        XXLOR   V6, V6, VS30
        XXLOR   V7, V7, VS31
 
-       VADDUWM KI, V7, V7      // h+K[i]
-       LVX     (TBL)(IDX), KI
-       ADD     $16, IDX
+       VADDUWM KI, V7, V7        // h+K[i]
+       LVX     (TBL)(R_x010), KI
 
        VPERMLE(V8, V8, LEMASK, V8)
-       SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V8)
+       SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V8, R_x020)
        VSLDOI  $4, V8, V8, V9
-       SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V9)
+       SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V9, R_x030)
        VSLDOI  $4, V9, V9, V10
-       SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V10)
-       LXVD2X  (INP)(R0), VS44 // load v12 (=vs44) in advance
-       ADD     $16, INP, INP
+       SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V10, R_x040)
+       LXVD2X  (INP)(R_x010), V12 // load v12 in advance
        VSLDOI  $4, V10, V10, V11
-       SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V11)
+       SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V11, R_x050)
        VPERMLE(V12, V12, LEMASK, V12)
-       SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V12)
+       SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V12, R_x060)
        VSLDOI  $4, V12, V12, V13
-       SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V13)
+       SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V13, R_x070)
        VSLDOI  $4, V13, V13, V14
-       SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V14)
-       LXVD2X  (INP)(R0), VS48 // load v16 (=vs48) in advance
-       ADD     $16, INP, INP
+       SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V14, R_x080)
+       LXVD2X  (INP)(R_x020), V16 // load v16 in advance
        VSLDOI  $4, V14, V14, V15
-       SHA256ROUND0(V1, V2, V3, V4, V5, V6, V7, V0, V15)
+       SHA256ROUND0(V1, V2, V3, V4, V5, V6, V7, V0, V15, R_x090)
        VPERMLE(V16, V16, LEMASK, V16)
-       SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V16)
+       SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V16, R_x0a0)
        VSLDOI  $4, V16, V16, V17
-       SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V17)
+       SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V17, R_x0b0)
        VSLDOI  $4, V17, V17, V18
-       SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V18)
+       SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V18, R_x0c0)
        VSLDOI  $4, V18, V18, V19
-       LXVD2X  (INP)(R0), VS52 // load v20 (=vs52) in advance
-       ADD     $16, INP, INP
-       SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V19)
+       LXVD2X  (INP)(R_x030), V20 // load v20 in advance
+       SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V19, R_x0d0)
        VPERMLE(V20, V20, LEMASK, V20)
-       SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V20)
+       SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V20, R_x0e0)
        VSLDOI  $4, V20, V20, V21
-       SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V21)
+       SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V21, R_x0f0)
        VSLDOI  $4, V21, V21, V22
-       SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V22)
+       SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V22, R_x100)
        VSLDOI  $4, V22, V22, V23
-       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22)
+       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22, R_x110)
 
-       MOVWZ   $3, TEMP
-       MOVWZ   TEMP, CTR
+       MOVD    $3, TEMP
+       MOVD    TEMP, CTR
+       ADD     $0x120, TBL
+       ADD     $0x40, INP
 
 L16_xx:
-       SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V18, V23)
-       SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V9, V10, V11, V19, V8)
-       SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V10, V11, V12, V20, V9)
-       SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V11, V12, V13, V21, V10)
-       SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V12, V13, V14, V22, V11)
-       SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V13, V14, V15, V23, V12)
-       SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V14, V15, V16, V8, V13)
-       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V15, V16, V17, V9, V14)
-       SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V16, V17, V18, V10, V15)
-       SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V17, V18, V19, V11, V16)
-       SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V18, V19, V20, V12, V17)
-       SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V19, V20, V21, V13, V18)
-       SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V20, V21, V22, V14, V19)
-       SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V21, V22, V23, V15, V20)
-       SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V22, V23, V8, V16, V21)
-       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22)
-
-       BC      0x10, 0, L16_xx         // bdnz
+       SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V18, V23, R_x000)
+       SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V9, V10, V11, V19, V8, R_x010)
+       SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V10, V11, V12, V20, V9, R_x020)
+       SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V11, V12, V13, V21, V10, R_x030)
+       SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V12, V13, V14, V22, V11, R_x040)
+       SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V13, V14, V15, V23, V12, R_x050)
+       SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V14, V15, V16, V8, V13, R_x060)
+       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V15, V16, V17, V9, V14, R_x070)
+       SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V16, V17, V18, V10, V15, R_x080)
+       SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V17, V18, V19, V11, V16, R_x090)
+       SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V18, V19, V20, V12, V17, R_x0a0)
+       SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V19, V20, V21, V13, V18, R_x0b0)
+       SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V20, V21, V22, V14, V19, R_x0c0)
+       SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V21, V22, V23, V15, V20, R_x0d0)
+       SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V22, V23, V8, V16, V21, R_x0e0)
+       SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22, R_x0f0)
+       ADD     $0x100, TBL
+
+       BDNZ    L16_xx
 
        XXLOR   VS24, VS24, V10
 
@@ -406,17 +437,16 @@ L16_xx:
        CMPU    INP, END
        BLT     loop
 
-       LVX     (TBL)(IDX), V8
-       ADD     $16, IDX
+       LVX     (TBL)(R_x000), V8
        VPERM   V0, V1, KI, V0
-       LVX     (TBL)(IDX), V9
+       LVX     (TBL)(R_x010), V9
        VPERM   V4, V5, KI, V4
        VPERM   V0, V2, V8, V0
        VPERM   V4, V6, V8, V4
        VPERM   V0, V3, V9, V0
        VPERM   V4, V7, V9, V4
-       STXVD2X VS32, (CTX+HEX00)       // v0 = vs32
-       STXVD2X VS36, (CTX+HEX10)       // v4 = vs36
+       STXVD2X V0, (CTX+R_x000)
+       STXVD2X V4, (CTX+R_x010)
 
 end:
        RET