// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+// This is a derived work from OpenSSL of SHA-2 using assembly optimizations. The
+// original code was written by Andy Polyakov <appro@openssl.org> and it's dual
+// licensed under OpenSSL and CRYPTOGAMS licenses depending on where you obtain
+// it. For further details see http://www.openssl.org/~appro/cryptogams/.
+
#include "textflag.h"
// SHA256 block routine. See sha256block.go for Go equivalent.
// H6 = g + H6
// H7 = h + H7
-// Wt = Mt; for 0 <= t <= 15
-#define MSGSCHEDULE0(index) \
- MOVWZ (index*4)(R26), R7; \
- RLWNM $24, R7, $-1, R11; \
- RLWMI $8, R7, $0x00FF0000, R11; \
- RLWMI $8, R7, $0x000000FF, R11; \
- MOVWZ R11, R7; \
- MOVWZ R7, (index*4)(R27)
-
-// Wt = SIGMA1(Wt-2) + Wt-7 + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63
-// SIGMA0(x) = ROTR(7,x) XOR ROTR(18,x) XOR SHR(3,x)
-// SIGMA1(x) = ROTR(17,x) XOR ROTR(19,x) XOR SHR(10,x)
-#define MSGSCHEDULE1(index) \
- MOVWZ ((index-2)*4)(R27), R7; \
- MOVWZ R7, R9; \
- RLWNM $32-17, R7, $-1, R7; \
- MOVWZ R9, R10; \
- RLWNM $32-19, R9, $-1, R9; \
- SRW $10, R10; \
- MOVWZ ((index-15)*4)(R27), R8; \
- XOR R9, R7; \
- MOVWZ R8, R9; \
- XOR R10, R7; \
- RLWNM $32-7, R8, $-1, R8; \
- MOVWZ R9, R10; \
- SRW $3, R10; \
- RLWNM $32-18, R9, $-1, R9; \
- MOVWZ ((index-7)*4)(R27), R11; \
- ADD R11, R7; \
- XOR R9, R8; \
- XOR R10, R8; \
- MOVWZ ((index-16)*4)(R27), R11; \
- ADD R11, R8; \
- ADD R8, R7; \
- MOVWZ R7, ((index)*4)(R27)
-
-// T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + Kt + Wt
-// BIGSIGMA1(x) = ROTR(6,x) XOR ROTR(11,x) XOR ROTR(25,x)
-// Ch(x, y, z) = (x AND y) XOR (NOT x AND z)
-#define SHA256T1(const, e, f, g, h) \
- ADD R7, h; \
- MOVWZ e, R7; \
- ADD $const, h; \
- MOVWZ e, R9; \
- RLWNM $32-6, R7, $-1, R7; \
- MOVWZ e, R10; \
- RLWNM $32-11, R9, $-1, R9; \
- XOR R9, R7; \
- MOVWZ e, R9; \
- RLWNM $32-25, R10, $-1, R10; \
- AND f, R9; \
- XOR R7, R10; \
- MOVWZ e, R7; \
- NOR R7, R7, R7; \
- ADD R10, h; \
- AND g, R7; \
- XOR R9, R7; \
- ADD h, R7
-
-// T2 = BIGSIGMA0(a) + Maj(a, b, c)
-// BIGSIGMA0(x) = ROTR(2,x) XOR ROTR(13,x) XOR ROTR(22,x)
-// Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z)
-#define SHA256T2(a, b, c) \
- MOVWZ a, R28; \
- MOVWZ c, R8; \
- RLWNM $32-2, R28, $-1, R28; \
- MOVWZ a, R10; \
- AND b, R8; \
- RLWNM $32-13, R10, $-1, R10; \
- MOVWZ a, R9; \
- AND c, R9; \
- XOR R10, R28; \
- XOR R9, R8; \
- MOVWZ a, R10; \
- MOVWZ b, R9; \
- RLWNM $32-22, R10, $-1, R10; \
- AND a, R9; \
- XOR R9, R8; \
- XOR R10, R28; \
- ADD R28, R8
-
-// Calculate T1 and T2, then e = d + T1 and a = T1 + T2.
-// The values for e and a are stored in d and h, ready for rotation.
-#define SHA256ROUND(index, const, a, b, c, d, e, f, g, h) \
- SHA256T1(const, e, f, g, h); \
- SHA256T2(a, b, c); \
- MOVWZ R8, h; \
- ADD R7, d; \
- ADD R7, h
-
-#define SHA256ROUND0(index, const, a, b, c, d, e, f, g, h) \
- MSGSCHEDULE0(index); \
- SHA256ROUND(index, const, a, b, c, d, e, f, g, h)
-
-#define SHA256ROUND1(index, const, a, b, c, d, e, f, g, h) \
- MSGSCHEDULE1(index); \
- SHA256ROUND(index, const, a, b, c, d, e, f, g, h)
+#define CTX R3
+#define INP R4
+#define END R5
+#define TBL R6
+#define IDX R7
+#define CNT R8
+#define LEN R9
+#define OFFLOAD R11
+#define TEMP R12
+
+#define HEX00 R0
+#define HEX10 R10
+#define HEX20 R25
+#define HEX30 R26
+#define HEX40 R27
+#define HEX50 R28
+#define HEX60 R29
+#define HEX70 R31
+
+// V0-V7 are A-H
+// V8-V23 are used for the message schedule
+#define KI V24
+#define FUNC V25
+#define S0 V26
+#define S1 V27
+#define s0 V28
+#define s1 V29
+#define LEMASK V31 // Permutation control register for little endian
+
+// 4 copies of each Kt, to fill all 4 words of a vector register
+DATA ·kcon+0x000(SB)/8, $0x428a2f98428a2f98
+DATA ·kcon+0x008(SB)/8, $0x428a2f98428a2f98
+DATA ·kcon+0x010(SB)/8, $0x7137449171374491
+DATA ·kcon+0x018(SB)/8, $0x7137449171374491
+DATA ·kcon+0x020(SB)/8, $0xb5c0fbcfb5c0fbcf
+DATA ·kcon+0x028(SB)/8, $0xb5c0fbcfb5c0fbcf
+DATA ·kcon+0x030(SB)/8, $0xe9b5dba5e9b5dba5
+DATA ·kcon+0x038(SB)/8, $0xe9b5dba5e9b5dba5
+DATA ·kcon+0x040(SB)/8, $0x3956c25b3956c25b
+DATA ·kcon+0x048(SB)/8, $0x3956c25b3956c25b
+DATA ·kcon+0x050(SB)/8, $0x59f111f159f111f1
+DATA ·kcon+0x058(SB)/8, $0x59f111f159f111f1
+DATA ·kcon+0x060(SB)/8, $0x923f82a4923f82a4
+DATA ·kcon+0x068(SB)/8, $0x923f82a4923f82a4
+DATA ·kcon+0x070(SB)/8, $0xab1c5ed5ab1c5ed5
+DATA ·kcon+0x078(SB)/8, $0xab1c5ed5ab1c5ed5
+DATA ·kcon+0x080(SB)/8, $0xd807aa98d807aa98
+DATA ·kcon+0x088(SB)/8, $0xd807aa98d807aa98
+DATA ·kcon+0x090(SB)/8, $0x12835b0112835b01
+DATA ·kcon+0x098(SB)/8, $0x12835b0112835b01
+DATA ·kcon+0x0A0(SB)/8, $0x243185be243185be
+DATA ·kcon+0x0A8(SB)/8, $0x243185be243185be
+DATA ·kcon+0x0B0(SB)/8, $0x550c7dc3550c7dc3
+DATA ·kcon+0x0B8(SB)/8, $0x550c7dc3550c7dc3
+DATA ·kcon+0x0C0(SB)/8, $0x72be5d7472be5d74
+DATA ·kcon+0x0C8(SB)/8, $0x72be5d7472be5d74
+DATA ·kcon+0x0D0(SB)/8, $0x80deb1fe80deb1fe
+DATA ·kcon+0x0D8(SB)/8, $0x80deb1fe80deb1fe
+DATA ·kcon+0x0E0(SB)/8, $0x9bdc06a79bdc06a7
+DATA ·kcon+0x0E8(SB)/8, $0x9bdc06a79bdc06a7
+DATA ·kcon+0x0F0(SB)/8, $0xc19bf174c19bf174
+DATA ·kcon+0x0F8(SB)/8, $0xc19bf174c19bf174
+DATA ·kcon+0x100(SB)/8, $0xe49b69c1e49b69c1
+DATA ·kcon+0x108(SB)/8, $0xe49b69c1e49b69c1
+DATA ·kcon+0x110(SB)/8, $0xefbe4786efbe4786
+DATA ·kcon+0x118(SB)/8, $0xefbe4786efbe4786
+DATA ·kcon+0x120(SB)/8, $0x0fc19dc60fc19dc6
+DATA ·kcon+0x128(SB)/8, $0x0fc19dc60fc19dc6
+DATA ·kcon+0x130(SB)/8, $0x240ca1cc240ca1cc
+DATA ·kcon+0x138(SB)/8, $0x240ca1cc240ca1cc
+DATA ·kcon+0x140(SB)/8, $0x2de92c6f2de92c6f
+DATA ·kcon+0x148(SB)/8, $0x2de92c6f2de92c6f
+DATA ·kcon+0x150(SB)/8, $0x4a7484aa4a7484aa
+DATA ·kcon+0x158(SB)/8, $0x4a7484aa4a7484aa
+DATA ·kcon+0x160(SB)/8, $0x5cb0a9dc5cb0a9dc
+DATA ·kcon+0x168(SB)/8, $0x5cb0a9dc5cb0a9dc
+DATA ·kcon+0x170(SB)/8, $0x76f988da76f988da
+DATA ·kcon+0x178(SB)/8, $0x76f988da76f988da
+DATA ·kcon+0x180(SB)/8, $0x983e5152983e5152
+DATA ·kcon+0x188(SB)/8, $0x983e5152983e5152
+DATA ·kcon+0x190(SB)/8, $0xa831c66da831c66d
+DATA ·kcon+0x198(SB)/8, $0xa831c66da831c66d
+DATA ·kcon+0x1A0(SB)/8, $0xb00327c8b00327c8
+DATA ·kcon+0x1A8(SB)/8, $0xb00327c8b00327c8
+DATA ·kcon+0x1B0(SB)/8, $0xbf597fc7bf597fc7
+DATA ·kcon+0x1B8(SB)/8, $0xbf597fc7bf597fc7
+DATA ·kcon+0x1C0(SB)/8, $0xc6e00bf3c6e00bf3
+DATA ·kcon+0x1C8(SB)/8, $0xc6e00bf3c6e00bf3
+DATA ·kcon+0x1D0(SB)/8, $0xd5a79147d5a79147
+DATA ·kcon+0x1D8(SB)/8, $0xd5a79147d5a79147
+DATA ·kcon+0x1E0(SB)/8, $0x06ca635106ca6351
+DATA ·kcon+0x1E8(SB)/8, $0x06ca635106ca6351
+DATA ·kcon+0x1F0(SB)/8, $0x1429296714292967
+DATA ·kcon+0x1F8(SB)/8, $0x1429296714292967
+DATA ·kcon+0x200(SB)/8, $0x27b70a8527b70a85
+DATA ·kcon+0x208(SB)/8, $0x27b70a8527b70a85
+DATA ·kcon+0x210(SB)/8, $0x2e1b21382e1b2138
+DATA ·kcon+0x218(SB)/8, $0x2e1b21382e1b2138
+DATA ·kcon+0x220(SB)/8, $0x4d2c6dfc4d2c6dfc
+DATA ·kcon+0x228(SB)/8, $0x4d2c6dfc4d2c6dfc
+DATA ·kcon+0x230(SB)/8, $0x53380d1353380d13
+DATA ·kcon+0x238(SB)/8, $0x53380d1353380d13
+DATA ·kcon+0x240(SB)/8, $0x650a7354650a7354
+DATA ·kcon+0x248(SB)/8, $0x650a7354650a7354
+DATA ·kcon+0x250(SB)/8, $0x766a0abb766a0abb
+DATA ·kcon+0x258(SB)/8, $0x766a0abb766a0abb
+DATA ·kcon+0x260(SB)/8, $0x81c2c92e81c2c92e
+DATA ·kcon+0x268(SB)/8, $0x81c2c92e81c2c92e
+DATA ·kcon+0x270(SB)/8, $0x92722c8592722c85
+DATA ·kcon+0x278(SB)/8, $0x92722c8592722c85
+DATA ·kcon+0x280(SB)/8, $0xa2bfe8a1a2bfe8a1
+DATA ·kcon+0x288(SB)/8, $0xa2bfe8a1a2bfe8a1
+DATA ·kcon+0x290(SB)/8, $0xa81a664ba81a664b
+DATA ·kcon+0x298(SB)/8, $0xa81a664ba81a664b
+DATA ·kcon+0x2A0(SB)/8, $0xc24b8b70c24b8b70
+DATA ·kcon+0x2A8(SB)/8, $0xc24b8b70c24b8b70
+DATA ·kcon+0x2B0(SB)/8, $0xc76c51a3c76c51a3
+DATA ·kcon+0x2B8(SB)/8, $0xc76c51a3c76c51a3
+DATA ·kcon+0x2C0(SB)/8, $0xd192e819d192e819
+DATA ·kcon+0x2C8(SB)/8, $0xd192e819d192e819
+DATA ·kcon+0x2D0(SB)/8, $0xd6990624d6990624
+DATA ·kcon+0x2D8(SB)/8, $0xd6990624d6990624
+DATA ·kcon+0x2E0(SB)/8, $0xf40e3585f40e3585
+DATA ·kcon+0x2E8(SB)/8, $0xf40e3585f40e3585
+DATA ·kcon+0x2F0(SB)/8, $0x106aa070106aa070
+DATA ·kcon+0x2F8(SB)/8, $0x106aa070106aa070
+DATA ·kcon+0x300(SB)/8, $0x19a4c11619a4c116
+DATA ·kcon+0x308(SB)/8, $0x19a4c11619a4c116
+DATA ·kcon+0x310(SB)/8, $0x1e376c081e376c08
+DATA ·kcon+0x318(SB)/8, $0x1e376c081e376c08
+DATA ·kcon+0x320(SB)/8, $0x2748774c2748774c
+DATA ·kcon+0x328(SB)/8, $0x2748774c2748774c
+DATA ·kcon+0x330(SB)/8, $0x34b0bcb534b0bcb5
+DATA ·kcon+0x338(SB)/8, $0x34b0bcb534b0bcb5
+DATA ·kcon+0x340(SB)/8, $0x391c0cb3391c0cb3
+DATA ·kcon+0x348(SB)/8, $0x391c0cb3391c0cb3
+DATA ·kcon+0x350(SB)/8, $0x4ed8aa4a4ed8aa4a
+DATA ·kcon+0x358(SB)/8, $0x4ed8aa4a4ed8aa4a
+DATA ·kcon+0x360(SB)/8, $0x5b9cca4f5b9cca4f
+DATA ·kcon+0x368(SB)/8, $0x5b9cca4f5b9cca4f
+DATA ·kcon+0x370(SB)/8, $0x682e6ff3682e6ff3
+DATA ·kcon+0x378(SB)/8, $0x682e6ff3682e6ff3
+DATA ·kcon+0x380(SB)/8, $0x748f82ee748f82ee
+DATA ·kcon+0x388(SB)/8, $0x748f82ee748f82ee
+DATA ·kcon+0x390(SB)/8, $0x78a5636f78a5636f
+DATA ·kcon+0x398(SB)/8, $0x78a5636f78a5636f
+DATA ·kcon+0x3A0(SB)/8, $0x84c8781484c87814
+DATA ·kcon+0x3A8(SB)/8, $0x84c8781484c87814
+DATA ·kcon+0x3B0(SB)/8, $0x8cc702088cc70208
+DATA ·kcon+0x3B8(SB)/8, $0x8cc702088cc70208
+DATA ·kcon+0x3C0(SB)/8, $0x90befffa90befffa
+DATA ·kcon+0x3C8(SB)/8, $0x90befffa90befffa
+DATA ·kcon+0x3D0(SB)/8, $0xa4506ceba4506ceb
+DATA ·kcon+0x3D8(SB)/8, $0xa4506ceba4506ceb
+DATA ·kcon+0x3E0(SB)/8, $0xbef9a3f7bef9a3f7
+DATA ·kcon+0x3E8(SB)/8, $0xbef9a3f7bef9a3f7
+DATA ·kcon+0x3F0(SB)/8, $0xc67178f2c67178f2
+DATA ·kcon+0x3F8(SB)/8, $0xc67178f2c67178f2
+DATA ·kcon+0x400(SB)/8, $0x0000000000000000
+DATA ·kcon+0x408(SB)/8, $0x0000000000000000
+DATA ·kcon+0x410(SB)/8, $0x1011121310111213 // permutation control vectors
+DATA ·kcon+0x418(SB)/8, $0x1011121300010203
+DATA ·kcon+0x420(SB)/8, $0x1011121310111213
+DATA ·kcon+0x428(SB)/8, $0x0405060700010203
+DATA ·kcon+0x430(SB)/8, $0x1011121308090a0b
+DATA ·kcon+0x438(SB)/8, $0x0405060700010203
+GLOBL ·kcon(SB), RODATA, $1088
+
+#define SHA256ROUND0(a, b, c, d, e, f, g, h, xi) \
+ VSEL g, f, e, FUNC; \
+ VSHASIGMAW $15, e, $1, S1; \
+ VADDUWM xi, h, h; \
+ VSHASIGMAW $0, a, $1, S0; \
+ VADDUWM FUNC, h, h; \
+ VXOR b, a, FUNC; \
+ VADDUWM S1, h, h; \
+ VSEL b, c, FUNC, FUNC; \
+ VADDUWM KI, g, g; \
+ VADDUWM h, d, d; \
+ VADDUWM FUNC, S0, S0; \
+ LVX (TBL)(IDX), KI; \
+ ADD $16, IDX; \
+ VADDUWM S0, h, h
+
+#define SHA256ROUND1(a, b, c, d, e, f, g, h, xi, xj, xj_1, xj_9, xj_14) \
+ VSHASIGMAW $0, xj_1, $0, s0; \
+ VSEL g, f, e, FUNC; \
+ VSHASIGMAW $15, e, $1, S1; \
+ VADDUWM xi, h, h; \
+ VSHASIGMAW $0, a, $1, S0; \
+ VSHASIGMAW $15, xj_14, $0, s1; \
+ VADDUWM FUNC, h, h; \
+ VXOR b, a, FUNC; \
+ VADDUWM xj_9, xj, xj; \
+ VADDUWM S1, h, h; \
+ VSEL b, c, FUNC, FUNC; \
+ VADDUWM KI, g, g; \
+ VADDUWM h, d, d; \
+ VADDUWM FUNC, S0, S0; \
+ VADDUWM s0, xj, xj; \
+ LVX (TBL)(IDX), KI; \
+ ADD $16, IDX; \
+ VADDUWM S0, h, h; \
+ VADDUWM s1, xj, xj
// func block(dig *digest, p []byte)
-TEXT ·block(SB),0,$296-32
- MOVD p_base+8(FP), R26
- MOVD p_len+16(FP), R29
- SRD $6, R29
- SLD $6, R29
+TEXT ·block(SB),0,$128-32
+ MOVD dig+0(FP), CTX
+ MOVD p_base+8(FP), INP
+ MOVD p_len+16(FP), LEN
+
+ SRD $6, LEN
+ SLD $6, LEN
- ADD R26, R29, R28
+ ADD INP, LEN, END
- MOVD R28, 256(R1)
- CMP R26, R28
+ CMP INP, END
BEQ end
- MOVD dig+0(FP), R27
- MOVWZ (0*4)(R27), R14 // a = H0
- MOVWZ (1*4)(R27), R15 // b = H1
- MOVWZ (2*4)(R27), R16 // c = H2
- MOVWZ (3*4)(R27), R17 // d = H3
- MOVWZ (4*4)(R27), R18 // e = H4
- MOVWZ (5*4)(R27), R19 // f = H5
- MOVWZ (6*4)(R27), R20 // g = H6
- MOVWZ (7*4)(R27), R21 // h = H7
+ MOVD $·kcon(SB), TBL
+ MOVD R1, OFFLOAD
+
+ MOVD R0, CNT
+ MOVWZ $0x10, HEX10
+ MOVWZ $0x20, HEX20
+ MOVWZ $0x30, HEX30
+ MOVWZ $0x40, HEX40
+ MOVWZ $0x50, HEX50
+ MOVWZ $0x60, HEX60
+ MOVWZ $0x70, HEX70
+
+ MOVWZ $8, IDX
+ LVSL (IDX)(R0), LEMASK
+ VSPLTISB $0x0F, KI
+ VXOR KI, LEMASK, LEMASK
+
+ LXVW4X (CTX)(HEX00), VS32 // v0 = vs32
+ LXVW4X (CTX)(HEX10), VS36 // v4 = vs36
+
+ // unpack the input values into vector registers
+ VSLDOI $4, V0, V0, V1
+ VSLDOI $8, V0, V0, V2
+ VSLDOI $12, V0, V0, V3
+ VSLDOI $4, V4, V4, V5
+ VSLDOI $8, V4, V4, V6
+ VSLDOI $12, V4, V4, V7
loop:
- MOVD R1, R27 // R27: message schedule
-
- SHA256ROUND0(0, 0x428a2f98, R14, R15, R16, R17, R18, R19, R20, R21)
- SHA256ROUND0(1, 0x71374491, R21, R14, R15, R16, R17, R18, R19, R20)
- SHA256ROUND0(2, 0xb5c0fbcf, R20, R21, R14, R15, R16, R17, R18, R19)
- SHA256ROUND0(3, 0xe9b5dba5, R19, R20, R21, R14, R15, R16, R17, R18)
- SHA256ROUND0(4, 0x3956c25b, R18, R19, R20, R21, R14, R15, R16, R17)
- SHA256ROUND0(5, 0x59f111f1, R17, R18, R19, R20, R21, R14, R15, R16)
- SHA256ROUND0(6, 0x923f82a4, R16, R17, R18, R19, R20, R21, R14, R15)
- SHA256ROUND0(7, 0xab1c5ed5, R15, R16, R17, R18, R19, R20, R21, R14)
- SHA256ROUND0(8, 0xd807aa98, R14, R15, R16, R17, R18, R19, R20, R21)
- SHA256ROUND0(9, 0x12835b01, R21, R14, R15, R16, R17, R18, R19, R20)
- SHA256ROUND0(10, 0x243185be, R20, R21, R14, R15, R16, R17, R18, R19)
- SHA256ROUND0(11, 0x550c7dc3, R19, R20, R21, R14, R15, R16, R17, R18)
- SHA256ROUND0(12, 0x72be5d74, R18, R19, R20, R21, R14, R15, R16, R17)
- SHA256ROUND0(13, 0x80deb1fe, R17, R18, R19, R20, R21, R14, R15, R16)
- SHA256ROUND0(14, 0x9bdc06a7, R16, R17, R18, R19, R20, R21, R14, R15)
- SHA256ROUND0(15, 0xc19bf174, R15, R16, R17, R18, R19, R20, R21, R14)
-
- SHA256ROUND1(16, 0xe49b69c1, R14, R15, R16, R17, R18, R19, R20, R21)
- SHA256ROUND1(17, 0xefbe4786, R21, R14, R15, R16, R17, R18, R19, R20)
- SHA256ROUND1(18, 0x0fc19dc6, R20, R21, R14, R15, R16, R17, R18, R19)
- SHA256ROUND1(19, 0x240ca1cc, R19, R20, R21, R14, R15, R16, R17, R18)
- SHA256ROUND1(20, 0x2de92c6f, R18, R19, R20, R21, R14, R15, R16, R17)
- SHA256ROUND1(21, 0x4a7484aa, R17, R18, R19, R20, R21, R14, R15, R16)
- SHA256ROUND1(22, 0x5cb0a9dc, R16, R17, R18, R19, R20, R21, R14, R15)
- SHA256ROUND1(23, 0x76f988da, R15, R16, R17, R18, R19, R20, R21, R14)
- SHA256ROUND1(24, 0x983e5152, R14, R15, R16, R17, R18, R19, R20, R21)
- SHA256ROUND1(25, 0xa831c66d, R21, R14, R15, R16, R17, R18, R19, R20)
- SHA256ROUND1(26, 0xb00327c8, R20, R21, R14, R15, R16, R17, R18, R19)
- SHA256ROUND1(27, 0xbf597fc7, R19, R20, R21, R14, R15, R16, R17, R18)
- SHA256ROUND1(28, 0xc6e00bf3, R18, R19, R20, R21, R14, R15, R16, R17)
- SHA256ROUND1(29, 0xd5a79147, R17, R18, R19, R20, R21, R14, R15, R16)
- SHA256ROUND1(30, 0x06ca6351, R16, R17, R18, R19, R20, R21, R14, R15)
- SHA256ROUND1(31, 0x14292967, R15, R16, R17, R18, R19, R20, R21, R14)
- SHA256ROUND1(32, 0x27b70a85, R14, R15, R16, R17, R18, R19, R20, R21)
- SHA256ROUND1(33, 0x2e1b2138, R21, R14, R15, R16, R17, R18, R19, R20)
- SHA256ROUND1(34, 0x4d2c6dfc, R20, R21, R14, R15, R16, R17, R18, R19)
- SHA256ROUND1(35, 0x53380d13, R19, R20, R21, R14, R15, R16, R17, R18)
- SHA256ROUND1(36, 0x650a7354, R18, R19, R20, R21, R14, R15, R16, R17)
- SHA256ROUND1(37, 0x766a0abb, R17, R18, R19, R20, R21, R14, R15, R16)
- SHA256ROUND1(38, 0x81c2c92e, R16, R17, R18, R19, R20, R21, R14, R15)
- SHA256ROUND1(39, 0x92722c85, R15, R16, R17, R18, R19, R20, R21, R14)
- SHA256ROUND1(40, 0xa2bfe8a1, R14, R15, R16, R17, R18, R19, R20, R21)
- SHA256ROUND1(41, 0xa81a664b, R21, R14, R15, R16, R17, R18, R19, R20)
- SHA256ROUND1(42, 0xc24b8b70, R20, R21, R14, R15, R16, R17, R18, R19)
- SHA256ROUND1(43, 0xc76c51a3, R19, R20, R21, R14, R15, R16, R17, R18)
- SHA256ROUND1(44, 0xd192e819, R18, R19, R20, R21, R14, R15, R16, R17)
- SHA256ROUND1(45, 0xd6990624, R17, R18, R19, R20, R21, R14, R15, R16)
- SHA256ROUND1(46, 0xf40e3585, R16, R17, R18, R19, R20, R21, R14, R15)
- SHA256ROUND1(47, 0x106aa070, R15, R16, R17, R18, R19, R20, R21, R14)
- SHA256ROUND1(48, 0x19a4c116, R14, R15, R16, R17, R18, R19, R20, R21)
- SHA256ROUND1(49, 0x1e376c08, R21, R14, R15, R16, R17, R18, R19, R20)
- SHA256ROUND1(50, 0x2748774c, R20, R21, R14, R15, R16, R17, R18, R19)
- SHA256ROUND1(51, 0x34b0bcb5, R19, R20, R21, R14, R15, R16, R17, R18)
- SHA256ROUND1(52, 0x391c0cb3, R18, R19, R20, R21, R14, R15, R16, R17)
- SHA256ROUND1(53, 0x4ed8aa4a, R17, R18, R19, R20, R21, R14, R15, R16)
- SHA256ROUND1(54, 0x5b9cca4f, R16, R17, R18, R19, R20, R21, R14, R15)
- SHA256ROUND1(55, 0x682e6ff3, R15, R16, R17, R18, R19, R20, R21, R14)
- SHA256ROUND1(56, 0x748f82ee, R14, R15, R16, R17, R18, R19, R20, R21)
- SHA256ROUND1(57, 0x78a5636f, R21, R14, R15, R16, R17, R18, R19, R20)
- SHA256ROUND1(58, 0x84c87814, R20, R21, R14, R15, R16, R17, R18, R19)
- SHA256ROUND1(59, 0x8cc70208, R19, R20, R21, R14, R15, R16, R17, R18)
- SHA256ROUND1(60, 0x90befffa, R18, R19, R20, R21, R14, R15, R16, R17)
- SHA256ROUND1(61, 0xa4506ceb, R17, R18, R19, R20, R21, R14, R15, R16)
- SHA256ROUND1(62, 0xbef9a3f7, R16, R17, R18, R19, R20, R21, R14, R15)
- SHA256ROUND1(63, 0xc67178f2, R15, R16, R17, R18, R19, R20, R21, R14)
-
- MOVD dig+0(FP), R27
- MOVWZ (0*4)(R27), R11
- ADD R11, R14 // H0 = a + H0
- MOVWZ R14, (0*4)(R27)
- MOVWZ (1*4)(R27), R11
- ADD R11, R15 // H1 = b + H1
- MOVWZ R15, (1*4)(R27)
- MOVWZ (2*4)(R27), R11
- ADD R11, R16 // H2 = c + H2
- MOVWZ R16, (2*4)(R27)
- MOVWZ (3*4)(R27), R11
- ADD R11, R17 // H3 = d + H3
- MOVWZ R17, (3*4)(R27)
- MOVWZ (4*4)(R27), R11
- ADD R11, R18 // H4 = e + H4
- MOVWZ R18, (4*4)(R27)
- MOVWZ (5*4)(R27), R11
- ADD R11, R19 // H5 = f + H5
- MOVWZ R19, (5*4)(R27)
- MOVWZ (6*4)(R27), R11
- ADD R11, R20 // H6 = g + H6
- MOVWZ R20, (6*4)(R27)
- MOVWZ (7*4)(R27), R11
- ADD R11, R21 // H7 = h + H7
- MOVWZ R21, (7*4)(R27)
-
- ADD $64, R26
- MOVD 256(R1), R11
- CMPU R26, R11
+ LVX (TBL)(HEX00), KI
+ MOVWZ $16, IDX
+
+ LXVD2X (INP)(R0), VS40 // load v8 (=vs40) in advance
+ ADD $16, INP
+
+ STVX V0, (OFFLOAD+HEX00)
+ STVX V1, (OFFLOAD+HEX10)
+ STVX V2, (OFFLOAD+HEX20)
+ STVX V3, (OFFLOAD+HEX30)
+ STVX V4, (OFFLOAD+HEX40)
+ STVX V5, (OFFLOAD+HEX50)
+ STVX V6, (OFFLOAD+HEX60)
+ STVX V7, (OFFLOAD+HEX70)
+
+ VADDUWM KI, V7, V7 // h+K[i]
+ LVX (TBL)(IDX), KI
+ ADD $16, IDX
+
+ VPERM V8, V8, LEMASK, V8
+ SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V8)
+ VSLDOI $4, V8, V8, V9
+ SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V9)
+ VSLDOI $4, V9, V9, V10
+ SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V10)
+ LXVD2X (INP)(R0), VS44 // load v12 (=vs44) in advance
+ ADD $16, INP, INP
+ VSLDOI $4, V10, V10, V11
+ SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V11)
+ VPERM V12, V12, LEMASK, V12
+ SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V12)
+ VSLDOI $4, V12, V12, V13
+ SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V13)
+ VSLDOI $4, V13, V13, V14
+ SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V14)
+ LXVD2X (INP)(R0), VS48 // load v16 (=vs48) in advance
+ ADD $16, INP, INP
+ VSLDOI $4, V14, V14, V15
+ SHA256ROUND0(V1, V2, V3, V4, V5, V6, V7, V0, V15)
+ VPERM V16, V16, LEMASK, V16
+ SHA256ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V16)
+ VSLDOI $4, V16, V16, V17
+ SHA256ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V17)
+ VSLDOI $4, V17, V17, V18
+ SHA256ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V18)
+ VSLDOI $4, V18, V18, V19
+ LXVD2X (INP)(R0), VS52 // load v20 (=vs52) in advance
+ ADD $16, INP, INP
+ SHA256ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V19)
+ VPERM V20, V20, LEMASK, V20
+ SHA256ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V20)
+ VSLDOI $4, V20, V20, V21
+ SHA256ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V21)
+ VSLDOI $4, V21, V21, V22
+ SHA256ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V22)
+ VSLDOI $4, V22, V22, V23
+ SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22)
+
+ MOVWZ $3, TEMP
+ MOVWZ TEMP, CTR
+
+L16_xx:
+ SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V18, V23)
+ SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V9, V10, V11, V19, V8)
+ SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V10, V11, V12, V20, V9)
+ SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V11, V12, V13, V21, V10)
+ SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V12, V13, V14, V22, V11)
+ SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V13, V14, V15, V23, V12)
+ SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V14, V15, V16, V8, V13)
+ SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V15, V16, V17, V9, V14)
+ SHA256ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V16, V17, V18, V10, V15)
+ SHA256ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V17, V18, V19, V11, V16)
+ SHA256ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V18, V19, V20, V12, V17)
+ SHA256ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V19, V20, V21, V13, V18)
+ SHA256ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V20, V21, V22, V14, V19)
+ SHA256ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V21, V22, V23, V15, V20)
+ SHA256ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V22, V23, V8, V16, V21)
+ SHA256ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22)
+
+ BC 0x10, 0, L16_xx // bdnz
+
+ LVX (OFFLOAD)(HEX00), V10
+
+ LVX (OFFLOAD)(HEX10), V11
+ VADDUWM V10, V0, V0
+ LVX (OFFLOAD)(HEX20), V12
+ VADDUWM V11, V1, V1
+ LVX (OFFLOAD)(HEX30), V13
+ VADDUWM V12, V2, V2
+ LVX (OFFLOAD)(HEX40), V14
+ VADDUWM V13, V3, V3
+ LVX (OFFLOAD)(HEX50), V15
+ VADDUWM V14, V4, V4
+ LVX (OFFLOAD)(HEX60), V16
+ VADDUWM V15, V5, V5
+ LVX (OFFLOAD)(HEX70), V17
+ VADDUWM V16, V6, V6
+ VADDUWM V17, V7, V7
+
+ CMPU INP, END
BLT loop
+ LVX (TBL)(IDX), V8
+ ADD $16, IDX
+ VPERM V0, V1, KI, V0
+ LVX (TBL)(IDX), V9
+ VPERM V4, V5, KI, V4
+ VPERM V0, V2, V8, V0
+ VPERM V4, V6, V8, V4
+ VPERM V0, V3, V9, V0
+ VPERM V4, V7, V9, V4
+ STXVD2X VS32, (CTX+HEX00) // v0 = vs32
+ STXVD2X VS36, (CTX+HEX10) // v4 = vs36
+
end:
RET
+
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+// This is a derived work from OpenSSL of SHA-2 using assembly optimizations. The
+// original code was written by Andy Polyakov <appro@openssl.org> and it's dual
+// licensed under OpenSSL and CRYPTOGAMS licenses depending on where you obtain
+// it. For further details see http://www.openssl.org/~appro/cryptogams/.
+
#include "textflag.h"
// SHA512 block routine. See sha512block.go for Go equivalent.
// H6 = g + H6
// H7 = h + H7
-// Wt = Mt; for 0 <= t <= 15
-#define MSGSCHEDULE0(index) \
- MOVD (index*8)(R6), R14; \
- RLWNM $24, R14, $-1, R21; \
- RLWMI $8, R14, $0x00FF0000, R21; \
- RLWMI $8, R14, $0x000000FF, R21; \
- SLD $32, R21; \
- SRD $32, R14, R20; \
- RLWNM $24, R20, $-1, R14; \
- RLWMI $8, R20, $0x00FF0000, R14; \
- RLWMI $8, R20, $0x000000FF, R14; \
- OR R21, R14; \
- MOVD R14, (index*8)(R9)
-
-// Wt = SIGMA1(Wt-2) + Wt-7 + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 79
-// SIGMA0(x) = ROTR(1,x) XOR ROTR(8,x) XOR SHR(7,x)
-// SIGMA1(x) = ROTR(19,x) XOR ROTR(61,x) XOR SHR(6,x)
-#define MSGSCHEDULE1(index) \
- MOVD ((index-2)*8)(R9), R14; \
- MOVD R14, R16; \
- RLDCL $64-19, R14, $-1, R14; \
- MOVD R16, R17; \
- RLDCL $64-61, R16, $-1, R16; \
- SRD $6, R17; \
- MOVD ((index-15)*8)(R9), R15; \
- XOR R16, R14; \
- MOVD R15, R16; \
- XOR R17, R14; \
- RLDCL $64-1, R15, $-1, R15; \
- MOVD R16, R17; \
- SRD $7, R17; \
- RLDCL $64-8, R16, $-1, R16; \
- MOVD ((index-7)*8)(R9), R21; \
- ADD R21, R14; \
- XOR R16, R15; \
- XOR R17, R15; \
- MOVD ((index-16)*8)(R9), R21; \
- ADD R21, R15; \
- ADD R15, R14; \
- MOVD R14, ((index)*8)(R9)
+#define CTX R3
+#define INP R4
+#define END R5
+#define TBL R6
+#define IDX R7
+#define CNT R8
+#define LEN R9
+#define OFFLOAD R11
+#define TEMP R12
-// Calculate T1 in R14 - uses R14, R16 and R17 registers.
-// h is also used as an accumulator. Wt is passed in R14.
-// T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + Kt + Wt
-// BIGSIGMA1(x) = ROTR(14,x) XOR ROTR(18,x) XOR ROTR(41,x)
-// Ch(x, y, z) = (x AND y) XOR (NOT x AND z)
-#define SHA512T1(const, e, f, g, h) \
- MOVD $const, R17; \
- ADD R14, h; \
- MOVD e, R14; \
- ADD R17, h; \
- MOVD e, R16; \
- RLDCL $64-14, R14, $-1, R14; \
- MOVD e, R17; \
- RLDCL $64-18, R16, $-1, R16; \
- XOR R16, R14; \
- MOVD e, R16; \
- RLDCL $64-41, R17, $-1, R17; \
- AND f, R16; \
- XOR R14, R17; \
- MOVD e, R14; \
- NOR R14, R14, R14; \
- ADD R17, h; \
- AND g, R14; \
- XOR R16, R14; \
- ADD h, R14
+#define HEX00 R0
+#define HEX10 R10
+#define HEX20 R25
+#define HEX30 R26
+#define HEX40 R27
+#define HEX50 R28
+#define HEX60 R29
+#define HEX70 R31
-// Calculate T2 in R15 - uses R15, R16, R17 and R8 registers.
-// T2 = BIGSIGMA0(a) + Maj(a, b, c)
-// BIGSIGMA0(x) = ROTR(28,x) XOR ROTR(34,x) XOR ROTR(39,x)
-// Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z)
-#define SHA512T2(a, b, c) \
- MOVD a, R8; \
- MOVD c, R15; \
- RLDCL $64-28, R8, $-1, R8; \
- MOVD a, R17; \
- AND b, R15; \
- RLDCL $64-34, R17, $-1, R17; \
- MOVD a, R16; \
- AND c, R16; \
- XOR R17, R8; \
- XOR R16, R15; \
- MOVD a, R17; \
- MOVD b, R16; \
- RLDCL $64-39, R17, $-1, R17; \
- AND a, R16; \
- XOR R16, R15; \
- XOR R17, R8; \
- ADD R8, R15
+// V0-V7 are A-H
+// V8-V23 are used for the message schedule
+#define KI V24
+#define FUNC V25
+#define S0 V26
+#define S1 V27
+#define s0 V28
+#define s1 V29
+#define LEMASK V31 // Permutation control register for little endian
-// Calculate T1 and T2, then e = d + T1 and a = T1 + T2.
-// The values for e and a are stored in d and h, ready for rotation.
-#define SHA512ROUND(index, const, a, b, c, d, e, f, g, h) \
- SHA512T1(const, e, f, g, h); \
- SHA512T2(a, b, c); \
- MOVD R15, h; \
- ADD R14, d; \
- ADD R14, h
+// 2 copies of each Kt, to fill both doublewords of a vector register
+DATA ·kcon+0x000(SB)/8, $0x428a2f98d728ae22
+DATA ·kcon+0x008(SB)/8, $0x428a2f98d728ae22
+DATA ·kcon+0x010(SB)/8, $0x7137449123ef65cd
+DATA ·kcon+0x018(SB)/8, $0x7137449123ef65cd
+DATA ·kcon+0x020(SB)/8, $0xb5c0fbcfec4d3b2f
+DATA ·kcon+0x028(SB)/8, $0xb5c0fbcfec4d3b2f
+DATA ·kcon+0x030(SB)/8, $0xe9b5dba58189dbbc
+DATA ·kcon+0x038(SB)/8, $0xe9b5dba58189dbbc
+DATA ·kcon+0x040(SB)/8, $0x3956c25bf348b538
+DATA ·kcon+0x048(SB)/8, $0x3956c25bf348b538
+DATA ·kcon+0x050(SB)/8, $0x59f111f1b605d019
+DATA ·kcon+0x058(SB)/8, $0x59f111f1b605d019
+DATA ·kcon+0x060(SB)/8, $0x923f82a4af194f9b
+DATA ·kcon+0x068(SB)/8, $0x923f82a4af194f9b
+DATA ·kcon+0x070(SB)/8, $0xab1c5ed5da6d8118
+DATA ·kcon+0x078(SB)/8, $0xab1c5ed5da6d8118
+DATA ·kcon+0x080(SB)/8, $0xd807aa98a3030242
+DATA ·kcon+0x088(SB)/8, $0xd807aa98a3030242
+DATA ·kcon+0x090(SB)/8, $0x12835b0145706fbe
+DATA ·kcon+0x098(SB)/8, $0x12835b0145706fbe
+DATA ·kcon+0x0A0(SB)/8, $0x243185be4ee4b28c
+DATA ·kcon+0x0A8(SB)/8, $0x243185be4ee4b28c
+DATA ·kcon+0x0B0(SB)/8, $0x550c7dc3d5ffb4e2
+DATA ·kcon+0x0B8(SB)/8, $0x550c7dc3d5ffb4e2
+DATA ·kcon+0x0C0(SB)/8, $0x72be5d74f27b896f
+DATA ·kcon+0x0C8(SB)/8, $0x72be5d74f27b896f
+DATA ·kcon+0x0D0(SB)/8, $0x80deb1fe3b1696b1
+DATA ·kcon+0x0D8(SB)/8, $0x80deb1fe3b1696b1
+DATA ·kcon+0x0E0(SB)/8, $0x9bdc06a725c71235
+DATA ·kcon+0x0E8(SB)/8, $0x9bdc06a725c71235
+DATA ·kcon+0x0F0(SB)/8, $0xc19bf174cf692694
+DATA ·kcon+0x0F8(SB)/8, $0xc19bf174cf692694
+DATA ·kcon+0x100(SB)/8, $0xe49b69c19ef14ad2
+DATA ·kcon+0x108(SB)/8, $0xe49b69c19ef14ad2
+DATA ·kcon+0x110(SB)/8, $0xefbe4786384f25e3
+DATA ·kcon+0x118(SB)/8, $0xefbe4786384f25e3
+DATA ·kcon+0x120(SB)/8, $0x0fc19dc68b8cd5b5
+DATA ·kcon+0x128(SB)/8, $0x0fc19dc68b8cd5b5
+DATA ·kcon+0x130(SB)/8, $0x240ca1cc77ac9c65
+DATA ·kcon+0x138(SB)/8, $0x240ca1cc77ac9c65
+DATA ·kcon+0x140(SB)/8, $0x2de92c6f592b0275
+DATA ·kcon+0x148(SB)/8, $0x2de92c6f592b0275
+DATA ·kcon+0x150(SB)/8, $0x4a7484aa6ea6e483
+DATA ·kcon+0x158(SB)/8, $0x4a7484aa6ea6e483
+DATA ·kcon+0x160(SB)/8, $0x5cb0a9dcbd41fbd4
+DATA ·kcon+0x168(SB)/8, $0x5cb0a9dcbd41fbd4
+DATA ·kcon+0x170(SB)/8, $0x76f988da831153b5
+DATA ·kcon+0x178(SB)/8, $0x76f988da831153b5
+DATA ·kcon+0x180(SB)/8, $0x983e5152ee66dfab
+DATA ·kcon+0x188(SB)/8, $0x983e5152ee66dfab
+DATA ·kcon+0x190(SB)/8, $0xa831c66d2db43210
+DATA ·kcon+0x198(SB)/8, $0xa831c66d2db43210
+DATA ·kcon+0x1A0(SB)/8, $0xb00327c898fb213f
+DATA ·kcon+0x1A8(SB)/8, $0xb00327c898fb213f
+DATA ·kcon+0x1B0(SB)/8, $0xbf597fc7beef0ee4
+DATA ·kcon+0x1B8(SB)/8, $0xbf597fc7beef0ee4
+DATA ·kcon+0x1C0(SB)/8, $0xc6e00bf33da88fc2
+DATA ·kcon+0x1C8(SB)/8, $0xc6e00bf33da88fc2
+DATA ·kcon+0x1D0(SB)/8, $0xd5a79147930aa725
+DATA ·kcon+0x1D8(SB)/8, $0xd5a79147930aa725
+DATA ·kcon+0x1E0(SB)/8, $0x06ca6351e003826f
+DATA ·kcon+0x1E8(SB)/8, $0x06ca6351e003826f
+DATA ·kcon+0x1F0(SB)/8, $0x142929670a0e6e70
+DATA ·kcon+0x1F8(SB)/8, $0x142929670a0e6e70
+DATA ·kcon+0x200(SB)/8, $0x27b70a8546d22ffc
+DATA ·kcon+0x208(SB)/8, $0x27b70a8546d22ffc
+DATA ·kcon+0x210(SB)/8, $0x2e1b21385c26c926
+DATA ·kcon+0x218(SB)/8, $0x2e1b21385c26c926
+DATA ·kcon+0x220(SB)/8, $0x4d2c6dfc5ac42aed
+DATA ·kcon+0x228(SB)/8, $0x4d2c6dfc5ac42aed
+DATA ·kcon+0x230(SB)/8, $0x53380d139d95b3df
+DATA ·kcon+0x238(SB)/8, $0x53380d139d95b3df
+DATA ·kcon+0x240(SB)/8, $0x650a73548baf63de
+DATA ·kcon+0x248(SB)/8, $0x650a73548baf63de
+DATA ·kcon+0x250(SB)/8, $0x766a0abb3c77b2a8
+DATA ·kcon+0x258(SB)/8, $0x766a0abb3c77b2a8
+DATA ·kcon+0x260(SB)/8, $0x81c2c92e47edaee6
+DATA ·kcon+0x268(SB)/8, $0x81c2c92e47edaee6
+DATA ·kcon+0x270(SB)/8, $0x92722c851482353b
+DATA ·kcon+0x278(SB)/8, $0x92722c851482353b
+DATA ·kcon+0x280(SB)/8, $0xa2bfe8a14cf10364
+DATA ·kcon+0x288(SB)/8, $0xa2bfe8a14cf10364
+DATA ·kcon+0x290(SB)/8, $0xa81a664bbc423001
+DATA ·kcon+0x298(SB)/8, $0xa81a664bbc423001
+DATA ·kcon+0x2A0(SB)/8, $0xc24b8b70d0f89791
+DATA ·kcon+0x2A8(SB)/8, $0xc24b8b70d0f89791
+DATA ·kcon+0x2B0(SB)/8, $0xc76c51a30654be30
+DATA ·kcon+0x2B8(SB)/8, $0xc76c51a30654be30
+DATA ·kcon+0x2C0(SB)/8, $0xd192e819d6ef5218
+DATA ·kcon+0x2C8(SB)/8, $0xd192e819d6ef5218
+DATA ·kcon+0x2D0(SB)/8, $0xd69906245565a910
+DATA ·kcon+0x2D8(SB)/8, $0xd69906245565a910
+DATA ·kcon+0x2E0(SB)/8, $0xf40e35855771202a
+DATA ·kcon+0x2E8(SB)/8, $0xf40e35855771202a
+DATA ·kcon+0x2F0(SB)/8, $0x106aa07032bbd1b8
+DATA ·kcon+0x2F8(SB)/8, $0x106aa07032bbd1b8
+DATA ·kcon+0x300(SB)/8, $0x19a4c116b8d2d0c8
+DATA ·kcon+0x308(SB)/8, $0x19a4c116b8d2d0c8
+DATA ·kcon+0x310(SB)/8, $0x1e376c085141ab53
+DATA ·kcon+0x318(SB)/8, $0x1e376c085141ab53
+DATA ·kcon+0x320(SB)/8, $0x2748774cdf8eeb99
+DATA ·kcon+0x328(SB)/8, $0x2748774cdf8eeb99
+DATA ·kcon+0x330(SB)/8, $0x34b0bcb5e19b48a8
+DATA ·kcon+0x338(SB)/8, $0x34b0bcb5e19b48a8
+DATA ·kcon+0x340(SB)/8, $0x391c0cb3c5c95a63
+DATA ·kcon+0x348(SB)/8, $0x391c0cb3c5c95a63
+DATA ·kcon+0x350(SB)/8, $0x4ed8aa4ae3418acb
+DATA ·kcon+0x358(SB)/8, $0x4ed8aa4ae3418acb
+DATA ·kcon+0x360(SB)/8, $0x5b9cca4f7763e373
+DATA ·kcon+0x368(SB)/8, $0x5b9cca4f7763e373
+DATA ·kcon+0x370(SB)/8, $0x682e6ff3d6b2b8a3
+DATA ·kcon+0x378(SB)/8, $0x682e6ff3d6b2b8a3
+DATA ·kcon+0x380(SB)/8, $0x748f82ee5defb2fc
+DATA ·kcon+0x388(SB)/8, $0x748f82ee5defb2fc
+DATA ·kcon+0x390(SB)/8, $0x78a5636f43172f60
+DATA ·kcon+0x398(SB)/8, $0x78a5636f43172f60
+DATA ·kcon+0x3A0(SB)/8, $0x84c87814a1f0ab72
+DATA ·kcon+0x3A8(SB)/8, $0x84c87814a1f0ab72
+DATA ·kcon+0x3B0(SB)/8, $0x8cc702081a6439ec
+DATA ·kcon+0x3B8(SB)/8, $0x8cc702081a6439ec
+DATA ·kcon+0x3C0(SB)/8, $0x90befffa23631e28
+DATA ·kcon+0x3C8(SB)/8, $0x90befffa23631e28
+DATA ·kcon+0x3D0(SB)/8, $0xa4506cebde82bde9
+DATA ·kcon+0x3D8(SB)/8, $0xa4506cebde82bde9
+DATA ·kcon+0x3E0(SB)/8, $0xbef9a3f7b2c67915
+DATA ·kcon+0x3E8(SB)/8, $0xbef9a3f7b2c67915
+DATA ·kcon+0x3F0(SB)/8, $0xc67178f2e372532b
+DATA ·kcon+0x3F8(SB)/8, $0xc67178f2e372532b
+DATA ·kcon+0x400(SB)/8, $0xca273eceea26619c
+DATA ·kcon+0x408(SB)/8, $0xca273eceea26619c
+DATA ·kcon+0x410(SB)/8, $0xd186b8c721c0c207
+DATA ·kcon+0x418(SB)/8, $0xd186b8c721c0c207
+DATA ·kcon+0x420(SB)/8, $0xeada7dd6cde0eb1e
+DATA ·kcon+0x428(SB)/8, $0xeada7dd6cde0eb1e
+DATA ·kcon+0x430(SB)/8, $0xf57d4f7fee6ed178
+DATA ·kcon+0x438(SB)/8, $0xf57d4f7fee6ed178
+DATA ·kcon+0x440(SB)/8, $0x06f067aa72176fba
+DATA ·kcon+0x448(SB)/8, $0x06f067aa72176fba
+DATA ·kcon+0x450(SB)/8, $0x0a637dc5a2c898a6
+DATA ·kcon+0x458(SB)/8, $0x0a637dc5a2c898a6
+DATA ·kcon+0x460(SB)/8, $0x113f9804bef90dae
+DATA ·kcon+0x468(SB)/8, $0x113f9804bef90dae
+DATA ·kcon+0x470(SB)/8, $0x1b710b35131c471b
+DATA ·kcon+0x478(SB)/8, $0x1b710b35131c471b
+DATA ·kcon+0x480(SB)/8, $0x28db77f523047d84
+DATA ·kcon+0x488(SB)/8, $0x28db77f523047d84
+DATA ·kcon+0x490(SB)/8, $0x32caab7b40c72493
+DATA ·kcon+0x498(SB)/8, $0x32caab7b40c72493
+DATA ·kcon+0x4A0(SB)/8, $0x3c9ebe0a15c9bebc
+DATA ·kcon+0x4A8(SB)/8, $0x3c9ebe0a15c9bebc
+DATA ·kcon+0x4B0(SB)/8, $0x431d67c49c100d4c
+DATA ·kcon+0x4B8(SB)/8, $0x431d67c49c100d4c
+DATA ·kcon+0x4C0(SB)/8, $0x4cc5d4becb3e42b6
+DATA ·kcon+0x4C8(SB)/8, $0x4cc5d4becb3e42b6
+DATA ·kcon+0x4D0(SB)/8, $0x597f299cfc657e2a
+DATA ·kcon+0x4D8(SB)/8, $0x597f299cfc657e2a
+DATA ·kcon+0x4E0(SB)/8, $0x5fcb6fab3ad6faec
+DATA ·kcon+0x4E8(SB)/8, $0x5fcb6fab3ad6faec
+DATA ·kcon+0x4F0(SB)/8, $0x6c44198c4a475817
+DATA ·kcon+0x4F8(SB)/8, $0x6c44198c4a475817
+DATA ·kcon+0x500(SB)/8, $0x0000000000000000
+DATA ·kcon+0x508(SB)/8, $0x0000000000000000
+DATA ·kcon+0x510(SB)/8, $0x1011121314151617
+DATA ·kcon+0x518(SB)/8, $0x0001020304050607
+GLOBL ·kcon(SB), RODATA, $1312
-#define SHA512ROUND0(index, const, a, b, c, d, e, f, g, h) \
- MSGSCHEDULE0(index); \
- SHA512ROUND(index, const, a, b, c, d, e, f, g, h)
+#define SHA512ROUND0(a, b, c, d, e, f, g, h, xi) \
+ VSEL g, f, e, FUNC; \
+ VSHASIGMAD $15, e, $1, S1; \
+ VADDUDM xi, h, h; \
+ VSHASIGMAD $0, a, $1, S0; \
+ VADDUDM FUNC, h, h; \
+ VXOR b, a, FUNC; \
+ VADDUDM S1, h, h; \
+ VSEL b, c, FUNC, FUNC; \
+ VADDUDM KI, g, g; \
+ VADDUDM h, d, d; \
+ VADDUDM FUNC, S0, S0; \
+ LVX (TBL)(IDX), KI; \
+ ADD $16, IDX; \
+ VADDUDM S0, h, h
-#define SHA512ROUND1(index, const, a, b, c, d, e, f, g, h) \
- MSGSCHEDULE1(index); \
- SHA512ROUND(index, const, a, b, c, d, e, f, g, h)
+#define SHA512ROUND1(a, b, c, d, e, f, g, h, xi, xj, xj_1, xj_9, xj_14) \
+ VSHASIGMAD $0, xj_1, $0, s0; \
+ VSEL g, f, e, FUNC; \
+ VSHASIGMAD $15, e, $1, S1; \
+ VADDUDM xi, h, h; \
+ VSHASIGMAD $0, a, $1, S0; \
+ VSHASIGMAD $15, xj_14, $0, s1; \
+ VADDUDM FUNC, h, h; \
+ VXOR b, a, FUNC; \
+ VADDUDM xj_9, xj, xj; \
+ VADDUDM S1, h, h; \
+ VSEL b, c, FUNC, FUNC; \
+ VADDUDM KI, g, g; \
+ VADDUDM h, d, d; \
+ VADDUDM FUNC, S0, S0; \
+ VADDUDM s0, xj, xj; \
+ LVX (TBL)(IDX), KI; \
+ ADD $16, IDX; \
+ VADDUDM S0, h, h; \
+ VADDUDM s1, xj, xj
// func block(dig *digest, p []byte)
-TEXT ·block(SB),0,$680-32
- MOVD p_base+8(FP), R6
- MOVD p_len+16(FP), R7
- SRD $7, R7
- SLD $7, R7
+TEXT ·block(SB),0,$128-32
+ MOVD dig+0(FP), CTX
+ MOVD p_base+8(FP), INP
+ MOVD p_len+16(FP), LEN
+
+ SRD $6, LEN
+ SLD $6, LEN
- ADD R6, R7, R8
- MOVD R8, 640(R1)
- CMP R6, R8
+ ADD INP, LEN, END
+
+ CMP INP, END
BEQ end
- MOVD dig+0(FP), R9
- MOVD (0*8)(R9), R22 // a = H0
- MOVD (1*8)(R9), R23 // b = H1
- MOVD (2*8)(R9), R24 // c = H2
- MOVD (3*8)(R9), R25 // d = H3
- MOVD (4*8)(R9), R26 // e = H4
- MOVD (5*8)(R9), R27 // f = H5
- MOVD (6*8)(R9), R28 // g = H6
- MOVD (7*8)(R9), R29 // h = H7
+ MOVD $·kcon(SB), TBL
+ MOVD R1, OFFLOAD
+
+ MOVD R0, CNT
+ MOVWZ $0x10, HEX10
+ MOVWZ $0x20, HEX20
+ MOVWZ $0x30, HEX30
+ MOVWZ $0x40, HEX40
+ MOVWZ $0x50, HEX50
+ MOVWZ $0x60, HEX60
+ MOVWZ $0x70, HEX70
+
+ MOVWZ $8, IDX
+ LVSL (IDX)(R0), LEMASK
+ VSPLTISB $0x0F, KI
+ VXOR KI, LEMASK, LEMASK
+
+ LXVD2X (CTX)(HEX00), VS32 // v0 = vs32
+ LXVD2X (CTX)(HEX10), VS34 // v2 = vs34
+ LXVD2X (CTX)(HEX20), VS36 // v4 = vs36
+ // unpack the input values into vector registers
+ VSLDOI $8, V0, V0, V1
+ LXVD2X (CTX)(HEX30), VS38 // v6 = vs38
+ VSLDOI $8, V2, V2, V3
+ VSLDOI $8, V4, V4, V5
+ VSLDOI $8, V6, V6, V7
loop:
- MOVD R1, R9 // R9: message schedule
+ LVX (TBL)(HEX00), KI
+ MOVWZ $16, IDX
- SHA512ROUND0(0, 0x428a2f98d728ae22, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND0(1, 0x7137449123ef65cd, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND0(2, 0xb5c0fbcfec4d3b2f, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND0(3, 0xe9b5dba58189dbbc, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND0(4, 0x3956c25bf348b538, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND0(5, 0x59f111f1b605d019, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND0(6, 0x923f82a4af194f9b, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND0(7, 0xab1c5ed5da6d8118, R23, R24, R25, R26, R27, R28, R29, R22)
- SHA512ROUND0(8, 0xd807aa98a3030242, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND0(9, 0x12835b0145706fbe, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND0(10, 0x243185be4ee4b28c, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND0(11, 0x550c7dc3d5ffb4e2, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND0(12, 0x72be5d74f27b896f, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND0(13, 0x80deb1fe3b1696b1, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND0(14, 0x9bdc06a725c71235, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND0(15, 0xc19bf174cf692694, R23, R24, R25, R26, R27, R28, R29, R22)
+ LXVD2X (INP)(R0), VS40 // load v8 (=vs40) in advance
+ ADD $16, INP
- SHA512ROUND1(16, 0xe49b69c19ef14ad2, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND1(17, 0xefbe4786384f25e3, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND1(18, 0x0fc19dc68b8cd5b5, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND1(19, 0x240ca1cc77ac9c65, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND1(20, 0x2de92c6f592b0275, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND1(21, 0x4a7484aa6ea6e483, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND1(22, 0x5cb0a9dcbd41fbd4, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND1(23, 0x76f988da831153b5, R23, R24, R25, R26, R27, R28, R29, R22)
- SHA512ROUND1(24, 0x983e5152ee66dfab, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND1(25, 0xa831c66d2db43210, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND1(26, 0xb00327c898fb213f, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND1(27, 0xbf597fc7beef0ee4, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND1(28, 0xc6e00bf33da88fc2, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND1(29, 0xd5a79147930aa725, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND1(30, 0x06ca6351e003826f, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND1(31, 0x142929670a0e6e70, R23, R24, R25, R26, R27, R28, R29, R22)
- SHA512ROUND1(32, 0x27b70a8546d22ffc, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND1(33, 0x2e1b21385c26c926, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND1(34, 0x4d2c6dfc5ac42aed, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND1(35, 0x53380d139d95b3df, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND1(36, 0x650a73548baf63de, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND1(37, 0x766a0abb3c77b2a8, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND1(38, 0x81c2c92e47edaee6, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND1(39, 0x92722c851482353b, R23, R24, R25, R26, R27, R28, R29, R22)
- SHA512ROUND1(40, 0xa2bfe8a14cf10364, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND1(41, 0xa81a664bbc423001, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND1(42, 0xc24b8b70d0f89791, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND1(43, 0xc76c51a30654be30, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND1(44, 0xd192e819d6ef5218, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND1(45, 0xd69906245565a910, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND1(46, 0xf40e35855771202a, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND1(47, 0x106aa07032bbd1b8, R23, R24, R25, R26, R27, R28, R29, R22)
- SHA512ROUND1(48, 0x19a4c116b8d2d0c8, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND1(49, 0x1e376c085141ab53, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND1(50, 0x2748774cdf8eeb99, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND1(51, 0x34b0bcb5e19b48a8, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND1(52, 0x391c0cb3c5c95a63, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND1(53, 0x4ed8aa4ae3418acb, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND1(54, 0x5b9cca4f7763e373, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND1(55, 0x682e6ff3d6b2b8a3, R23, R24, R25, R26, R27, R28, R29, R22)
- SHA512ROUND1(56, 0x748f82ee5defb2fc, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND1(57, 0x78a5636f43172f60, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND1(58, 0x84c87814a1f0ab72, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND1(59, 0x8cc702081a6439ec, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND1(60, 0x90befffa23631e28, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND1(61, 0xa4506cebde82bde9, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND1(62, 0xbef9a3f7b2c67915, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND1(63, 0xc67178f2e372532b, R23, R24, R25, R26, R27, R28, R29, R22)
- SHA512ROUND1(64, 0xca273eceea26619c, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND1(65, 0xd186b8c721c0c207, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND1(66, 0xeada7dd6cde0eb1e, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND1(67, 0xf57d4f7fee6ed178, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND1(68, 0x06f067aa72176fba, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND1(69, 0x0a637dc5a2c898a6, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND1(70, 0x113f9804bef90dae, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND1(71, 0x1b710b35131c471b, R23, R24, R25, R26, R27, R28, R29, R22)
- SHA512ROUND1(72, 0x28db77f523047d84, R22, R23, R24, R25, R26, R27, R28, R29)
- SHA512ROUND1(73, 0x32caab7b40c72493, R29, R22, R23, R24, R25, R26, R27, R28)
- SHA512ROUND1(74, 0x3c9ebe0a15c9bebc, R28, R29, R22, R23, R24, R25, R26, R27)
- SHA512ROUND1(75, 0x431d67c49c100d4c, R27, R28, R29, R22, R23, R24, R25, R26)
- SHA512ROUND1(76, 0x4cc5d4becb3e42b6, R26, R27, R28, R29, R22, R23, R24, R25)
- SHA512ROUND1(77, 0x597f299cfc657e2a, R25, R26, R27, R28, R29, R22, R23, R24)
- SHA512ROUND1(78, 0x5fcb6fab3ad6faec, R24, R25, R26, R27, R28, R29, R22, R23)
- SHA512ROUND1(79, 0x6c44198c4a475817, R23, R24, R25, R26, R27, R28, R29, R22)
+ STVX V0, (OFFLOAD+HEX00)
+ STVX V1, (OFFLOAD+HEX10)
+ STVX V2, (OFFLOAD+HEX20)
+ STVX V3, (OFFLOAD+HEX30)
+ STVX V4, (OFFLOAD+HEX40)
+ STVX V5, (OFFLOAD+HEX50)
+ STVX V6, (OFFLOAD+HEX60)
+ STVX V7, (OFFLOAD+HEX70)
- MOVD dig+0(FP), R9
- MOVD (0*8)(R9), R21
- ADD R21, R22 // H0 = a + H0
- MOVD R22, (0*8)(R9)
- MOVD (1*8)(R9), R21
- ADD R21, R23 // H1 = b + H1
- MOVD R23, (1*8)(R9)
- MOVD (2*8)(R9), R21
- ADD R21, R24 // H2 = c + H2
- MOVD R24, (2*8)(R9)
- MOVD (3*8)(R9), R21
- ADD R21, R25 // H3 = d + H3
- MOVD R25, (3*8)(R9)
- MOVD (4*8)(R9), R21
- ADD R21, R26 // H4 = e + H4
- MOVD R26, (4*8)(R9)
- MOVD (5*8)(R9), R21
- ADD R21, R27 // H5 = f + H5
- MOVD R27, (5*8)(R9)
- MOVD (6*8)(R9), R21
- ADD R21, R28 // H6 = g + H6
- MOVD R28, (6*8)(R9)
- MOVD (7*8)(R9), R21
- ADD R21, R29 // H7 = h + H7
- MOVD R29, (7*8)(R9)
+ VADDUDM KI, V7, V7 // h+K[i]
+ LVX (TBL)(IDX), KI
+ ADD $16, IDX
- ADD $128, R6
- MOVD 640(R1), R21
- CMPU R6, R21
+ VPERM V8, V8, LEMASK, V8
+ SHA512ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V8)
+ LXVD2X (INP)(R0), VS42 // load v10 (=vs42) in advance
+ ADD $16, INP, INP
+ VSLDOI $8, V8, V8, V9
+ SHA512ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V9)
+ VPERM V10, V10, LEMASK, V10
+ SHA512ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V10)
+ LXVD2X (INP)(R0), VS44 // load v12 (=vs44) in advance
+ ADD $16, INP, INP
+ VSLDOI $8, V10, V10, V11
+ SHA512ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V11)
+ VPERM V12, V12, LEMASK, V12
+ SHA512ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V12)
+ LXVD2X (INP)(R0), VS46 // load v14 (=vs46) in advance
+ ADD $16, INP, INP
+ VSLDOI $8, V12, V12, V13
+ SHA512ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V13)
+ VPERM V14, V14, LEMASK, V14
+ SHA512ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V14)
+ LXVD2X (INP)(R0), VS48 // load v16 (=vs48) in advance
+ ADD $16, INP, INP
+ VSLDOI $8, V14, V14, V15
+ SHA512ROUND0(V1, V2, V3, V4, V5, V6, V7, V0, V15)
+ VPERM V16, V16, LEMASK, V16
+ SHA512ROUND0(V0, V1, V2, V3, V4, V5, V6, V7, V16)
+ LXVD2X (INP)(R0), VS50 // load v18 (=vs50) in advance
+ ADD $16, INP, INP
+ VSLDOI $8, V16, V16, V17
+ SHA512ROUND0(V7, V0, V1, V2, V3, V4, V5, V6, V17)
+ VPERM V18, V18, LEMASK, V18
+ SHA512ROUND0(V6, V7, V0, V1, V2, V3, V4, V5, V18)
+ LXVD2X (INP)(R0), VS52 // load v20 (=vs52) in advance
+ ADD $16, INP, INP
+ VSLDOI $8, V18, V18, V19
+ SHA512ROUND0(V5, V6, V7, V0, V1, V2, V3, V4, V19)
+ VPERM V20, V20, LEMASK, V20
+ SHA512ROUND0(V4, V5, V6, V7, V0, V1, V2, V3, V20)
+ LXVD2X (INP)(R0), VS54 // load v22 (=vs54) in advance
+ ADD $16, INP, INP
+ VSLDOI $8, V20, V20, V21
+ SHA512ROUND0(V3, V4, V5, V6, V7, V0, V1, V2, V21)
+ VPERM V22, V22, LEMASK, V22
+ SHA512ROUND0(V2, V3, V4, V5, V6, V7, V0, V1, V22)
+ VSLDOI $8, V22, V22, V23
+ SHA512ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22)
+
+ MOVWZ $4, TEMP
+ MOVWZ TEMP, CTR
+
+L16_xx:
+ SHA512ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V18, V23)
+ SHA512ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V9, V10, V11, V19, V8)
+ SHA512ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V10, V11, V12, V20, V9)
+ SHA512ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V11, V12, V13, V21, V10)
+ SHA512ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V12, V13, V14, V22, V11)
+ SHA512ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V13, V14, V15, V23, V12)
+ SHA512ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V14, V15, V16, V8, V13)
+ SHA512ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V15, V16, V17, V9, V14)
+ SHA512ROUND1(V0, V1, V2, V3, V4, V5, V6, V7, V16, V17, V18, V10, V15)
+ SHA512ROUND1(V7, V0, V1, V2, V3, V4, V5, V6, V17, V18, V19, V11, V16)
+ SHA512ROUND1(V6, V7, V0, V1, V2, V3, V4, V5, V18, V19, V20, V12, V17)
+ SHA512ROUND1(V5, V6, V7, V0, V1, V2, V3, V4, V19, V20, V21, V13, V18)
+ SHA512ROUND1(V4, V5, V6, V7, V0, V1, V2, V3, V20, V21, V22, V14, V19)
+ SHA512ROUND1(V3, V4, V5, V6, V7, V0, V1, V2, V21, V22, V23, V15, V20)
+ SHA512ROUND1(V2, V3, V4, V5, V6, V7, V0, V1, V22, V23, V8, V16, V21)
+ SHA512ROUND1(V1, V2, V3, V4, V5, V6, V7, V0, V23, V8, V9, V17, V22)
+
+ BC 0x10, 0, L16_xx // bdnz
+
+ LVX (OFFLOAD)(HEX00), V10
+
+ LVX (OFFLOAD)(HEX10), V11
+ VADDUDM V10, V0, V0
+ LVX (OFFLOAD)(HEX20), V12
+ VADDUDM V11, V1, V1
+ LVX (OFFLOAD)(HEX30), V13
+ VADDUDM V12, V2, V2
+ LVX (OFFLOAD)(HEX40), V14
+ VADDUDM V13, V3, V3
+ LVX (OFFLOAD)(HEX50), V15
+ VADDUDM V14, V4, V4
+ LVX (OFFLOAD)(HEX60), V16
+ VADDUDM V15, V5, V5
+ LVX (OFFLOAD)(HEX70), V17
+ VADDUDM V16, V6, V6
+ VADDUDM V17, V7, V7
+
+ CMPU INP, END
BLT loop
+ VPERM V0, V1, KI, V0
+ VPERM V2, V3, KI, V2
+ VPERM V4, V5, KI, V4
+ VPERM V6, V7, KI, V6
+ STXVD2X VS32, (CTX+HEX00) // v0 = vs32
+ STXVD2X VS34, (CTX+HEX10) // v2 = vs34
+ STXVD2X VS36, (CTX+HEX20) // v4 = vs36
+ STXVD2X VS38, (CTX+HEX30) // v6 = vs38
+
end:
RET
+