From: Xiaolin Zhao Date: Tue, 4 Jun 2024 03:21:16 +0000 (+0800) Subject: crypto/sha512: implement sha512block in hardware on loong64 X-Git-Tag: go1.24rc1~1001 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=137e91832aae514dda65dbd34baa438424c1cbf8;p=gostls13.git crypto/sha512: implement sha512block in hardware on loong64 goos: linux goarch: loong64 pkg: crypto/sha512 cpu: Loongson-3A6000 @ 2500.00MHz │ bench.old │ bench.new │ │ sec/op │ sec/op vs base │ Hash8Bytes/New 942.3n ± 0% 627.5n ± 0% -33.41% (p=0.000 n=20) Hash8Bytes/Sum384 951.3n ± 0% 661.1n ± 0% -30.51% (p=0.000 n=20) Hash8Bytes/Sum512 1032.0n ± 0% 631.9n ± 0% -38.77% (p=0.000 n=20) Hash1K/New 6.355µ ± 0% 3.285µ ± 0% -48.31% (p=0.000 n=20) Hash1K/Sum384 6.333µ ± 0% 3.320µ ± 0% -47.58% (p=0.000 n=20) Hash1K/Sum512 6.416µ ± 0% 3.293µ ± 0% -48.68% (p=0.000 n=20) Hash8K/New 43.91µ ± 0% 22.01µ ± 0% -49.89% (p=0.000 n=20) Hash8K/Sum384 43.77µ ± 0% 22.05µ ± 0% -49.61% (p=0.000 n=20) Hash8K/Sum512 43.87µ ± 0% 22.01µ ± 0% -49.83% (p=0.000 n=20) geomean 6.480µ 3.596µ -44.50% │ bench.old │ bench.new │ │ B/s │ B/s vs base │ Hash8Bytes/New 8.097Mi ± 0% 12.159Mi ± 0% +50.18% (p=0.000 n=20) Hash8Bytes/Sum384 8.020Mi ± 0% 11.539Mi ± 0% +43.88% (p=0.000 n=20) Hash8Bytes/Sum512 7.391Mi ± 0% 12.074Mi ± 0% +63.35% (p=0.000 n=20) Hash1K/New 153.7Mi ± 0% 297.2Mi ± 0% +93.43% (p=0.000 n=20) Hash1K/Sum384 154.2Mi ± 0% 294.1Mi ± 0% +90.74% (p=0.000 n=20) Hash1K/Sum512 152.2Mi ± 0% 296.6Mi ± 0% +94.84% (p=0.000 n=20) Hash8K/New 177.9Mi ± 0% 355.0Mi ± 0% +99.55% (p=0.000 n=20) Hash8K/Sum384 178.5Mi ± 0% 354.3Mi ± 0% +98.46% (p=0.000 n=20) Hash8K/Sum512 178.1Mi ± 0% 355.0Mi ± 0% +99.34% (p=0.000 n=20) geomean 59.81Mi 107.8Mi +80.19% goos: linux goarch: loong64 pkg: crypto/sha512 cpu: Loongson-3A5000 @ 2500.00MHz │ bench.old │ bench.new │ │ sec/op │ sec/op vs base │ Hash8Bytes/New 1224.0n ± 0% 730.1n ± 0% -40.36% (p=0.000 n=20) Hash8Bytes/Sum384 1265.0n ± 0% 763.2n ± 0% -39.67% (p=0.000 n=20) Hash8Bytes/Sum512 1257.0n ± 0% 753.5n ± 0% -40.06% (p=0.000 n=20) Hash1K/New 8.218µ ± 0% 3.785µ ± 0% -53.94% (p=0.000 n=20) Hash1K/Sum384 8.248µ ± 0% 3.813µ ± 0% -53.77% (p=0.000 n=20) Hash1K/Sum512 8.235µ ± 0% 3.807µ ± 0% -53.77% (p=0.000 n=20) Hash8K/New 56.83µ ± 0% 25.37µ ± 0% -55.35% (p=0.000 n=20) Hash8K/Sum384 56.85µ ± 0% 25.39µ ± 0% -55.34% (p=0.000 n=20) Hash8K/Sum512 56.84µ ± 0% 25.38µ ± 0% -55.36% (p=0.000 n=20) geomean 8.360µ 4.165µ -50.18% │ bench.old │ bench.new │ │ B/s │ B/s vs base │ Hash8Bytes/New 6.232Mi ± 0% 10.452Mi ± 0% +67.71% (p=0.000 n=20) Hash8Bytes/Sum384 6.027Mi ± 0% 9.995Mi ± 0% +65.82% (p=0.000 n=20) Hash8Bytes/Sum512 6.065Mi ± 0% 10.123Mi ± 0% +66.90% (p=0.000 n=20) Hash1K/New 118.8Mi ± 0% 258.0Mi ± 0% +117.12% (p=0.000 n=20) Hash1K/Sum384 118.4Mi ± 0% 256.1Mi ± 0% +116.29% (p=0.000 n=20) Hash1K/Sum512 118.6Mi ± 0% 256.5Mi ± 0% +116.30% (p=0.000 n=20) Hash8K/New 137.5Mi ± 0% 307.9Mi ± 0% +123.96% (p=0.000 n=20) Hash8K/Sum384 137.4Mi ± 0% 307.8Mi ± 0% +123.93% (p=0.000 n=20) Hash8K/Sum512 137.4Mi ± 0% 307.9Mi ± 0% +124.01% (p=0.000 n=20) geomean 46.35Mi 93.05Mi +100.76% Change-Id: I0d764df16872598b8d2fd92d7253cf3fdbfdfdf2 Reviewed-on: https://go-review.googlesource.com/c/go/+/590156 LUCI-TryBot-Result: Go LUCI Reviewed-by: Filippo Valsorda Reviewed-by: Dmitri Shuralyov Reviewed-by: Roland Shoemaker Reviewed-by: abner chenc Reviewed-by: Meidan Li Reviewed-by: Qiqi Huang --- diff --git a/src/crypto/sha512/sha512block_decl.go b/src/crypto/sha512/sha512block_decl.go index 324659e62c..b8a7854e4d 100644 --- a/src/crypto/sha512/sha512block_decl.go +++ b/src/crypto/sha512/sha512block_decl.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (ppc64le || ppc64 || riscv64 || s390x) && !purego +//go:build (loong64 || ppc64 || ppc64le || riscv64 || s390x) && !purego package sha512 diff --git a/src/crypto/sha512/sha512block_generic.go b/src/crypto/sha512/sha512block_generic.go index 9177722be1..5d556606ed 100644 --- a/src/crypto/sha512/sha512block_generic.go +++ b/src/crypto/sha512/sha512block_generic.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (!amd64 && !arm64 && !ppc64 && !ppc64le && !riscv64 && !s390x) || purego +//go:build (!amd64 && !arm64 && !loong64 && !ppc64 && !ppc64le && !riscv64 && !s390x) || purego package sha512 diff --git a/src/crypto/sha512/sha512block_loong64.s b/src/crypto/sha512/sha512block_loong64.s new file mode 100644 index 0000000000..e508f23c58 --- /dev/null +++ b/src/crypto/sha512/sha512block_loong64.s @@ -0,0 +1,238 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !purego + +#include "textflag.h" + +// SHA512 block routine. See sha512block.go for Go equivalent. + +#define REGTMP R30 +#define REGTMP1 R16 +#define REGTMP2 R17 +#define REGTMP3 R18 +#define REGTMP4 R7 +#define REGTMP5 R6 + +// W[i] = M[i]; for 0 <= i <= 15 +#define LOAD0(index) \ + MOVV (index*8)(R5), REGTMP4; \ + WORD $0x3ce7; \ //REVBV REGTMP4, REGTMP4 + MOVV REGTMP4, (index*8)(R3) + +// W[i] = SIGMA1(W[i-2]) + W[i-7] + SIGMA0(W[i-15]) + W[i-16]; for 16 <= i <= 79 +// SIGMA0(x) = ROTR(1,x) XOR ROTR(8,x) XOR SHR(7,x) +// SIGMA1(x) = ROTR(19,x) XOR ROTR(61,x) XOR SHR(6,x) +#define LOAD1(index) \ + MOVV (((index-2)&0xf)*8)(R3), REGTMP4; \ + MOVV (((index-15)&0xf)*8)(R3), REGTMP1; \ + MOVV (((index-7)&0xf)*8)(R3), REGTMP; \ + MOVV REGTMP4, REGTMP2; \ + MOVV REGTMP4, REGTMP3; \ + ROTRV $19, REGTMP4; \ + ROTRV $61, REGTMP2; \ + SRLV $6, REGTMP3; \ + XOR REGTMP2, REGTMP4; \ + XOR REGTMP3, REGTMP4; \ + ROTRV $1, REGTMP1, REGTMP5; \ + SRLV $7, REGTMP1, REGTMP2; \ + ROTRV $8, REGTMP1; \ + ADDV REGTMP, REGTMP4; \ + MOVV (((index-16)&0xf)*8)(R3), REGTMP; \ + XOR REGTMP1, REGTMP5; \ + XOR REGTMP2, REGTMP5; \ + ADDV REGTMP, REGTMP5; \ + ADDV REGTMP5, REGTMP4; \ + MOVV REGTMP4, ((index&0xf)*8)(R3) + +// h is also used as an accumulator. Wt is passed in REGTMP4. +// T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + K[i] + W[i] +// BIGSIGMA1(x) = ROTR(14,x) XOR ROTR(18,x) XOR ROTR(41,x) +// Ch(x, y, z) = (x AND y) XOR (NOT x AND z) +// Calculate T1 in REGTMP4 +#define SHA512T1(const, e, f, g, h) \ + ADDV $const, h; \ + ADDV REGTMP4, h; \ + ROTRV $14, e, REGTMP4; \ + ROTRV $18, e, REGTMP; \ + ROTRV $41, e, REGTMP3; \ + AND f, e, REGTMP2; \ + XOR REGTMP, REGTMP4; \ + MOVV $0xffffffffffffffff, REGTMP; \ + XOR REGTMP4, REGTMP3; \ + XOR REGTMP, e, REGTMP5; \ + ADDV REGTMP3, h; \ + AND g, REGTMP5; \ + XOR REGTMP2, REGTMP5; \ + ADDV h, REGTMP5, REGTMP4 + +// T2 = BIGSIGMA0(a) + Maj(a, b, c) +// BIGSIGMA0(x) = ROTR(28,x) XOR ROTR(34,x) XOR ROTR(39,x) +// Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z) +// Calculate T2 in REGTMP1 +#define SHA512T2(a, b, c) \ + ROTRV $28, a, REGTMP5; \ + AND b, c, REGTMP1; \ + ROTRV $34, a, REGTMP3; \ + AND c, a, REGTMP; \ + XOR REGTMP3, REGTMP5; \ + XOR REGTMP, REGTMP1; \ + ROTRV $39, a, REGTMP2; \ + AND a, b, REGTMP3; \ + XOR REGTMP3, REGTMP1; \ + XOR REGTMP2, REGTMP5; \ + ADDV REGTMP5, REGTMP1 + +// Calculate T1 and T2, then e = d + T1 and a = T1 + T2. +// The values for e and a are stored in d and h, ready for rotation. +#define SHA512ROUND(const, a, b, c, d, e, f, g, h) \ + SHA512T1(const, e, f, g, h); \ + SHA512T2(a, b, c); \ + ADDV REGTMP4, d; \ + ADDV REGTMP1, REGTMP4, h + +#define SHA512ROUND0(index, const, a, b, c, d, e, f, g, h) \ + LOAD0(index); \ + SHA512ROUND(const, a, b, c, d, e, f, g, h) + +#define SHA512ROUND1(index, const, a, b, c, d, e, f, g, h) \ + LOAD1(index); \ + SHA512ROUND(const, a, b, c, d, e, f, g, h) + +// A stack frame size of 128 bytes is required here, because +// the frame size used for data expansion is 128 bytes. +// See the definition of the macro LOAD1 above (8 bytes * 16 entries). +// +// func block(dig *digest, p []byte) +TEXT ·block(SB),NOSPLIT,$128-32 + MOVV p_len+16(FP), R6 + MOVV p_base+8(FP), R5 + AND $~127, R6 + BEQ R6, end + + // p_len >= 128 + MOVV dig+0(FP), R4 + ADDV R5, R6, R25 + MOVV (0*8)(R4), R8 // a = H0 + MOVV (1*8)(R4), R9 // b = H1 + MOVV (2*8)(R4), R10 // c = H2 + MOVV (3*8)(R4), R11 // d = H3 + MOVV (4*8)(R4), R12 // e = H4 + MOVV (5*8)(R4), R13 // f = H5 + MOVV (6*8)(R4), R14 // g = H6 + MOVV (7*8)(R4), R15 // h = H7 + +loop: + SHA512ROUND0( 0, 0x428a2f98d728ae22, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND0( 1, 0x7137449123ef65cd, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND0( 2, 0xb5c0fbcfec4d3b2f, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND0( 3, 0xe9b5dba58189dbbc, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND0( 4, 0x3956c25bf348b538, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND0( 5, 0x59f111f1b605d019, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND0( 6, 0x923f82a4af194f9b, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND0( 7, 0xab1c5ed5da6d8118, R9, R10, R11, R12, R13, R14, R15, R8) + SHA512ROUND0( 8, 0xd807aa98a3030242, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND0( 9, 0x12835b0145706fbe, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND0(10, 0x243185be4ee4b28c, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND0(11, 0x550c7dc3d5ffb4e2, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND0(12, 0x72be5d74f27b896f, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND0(13, 0x80deb1fe3b1696b1, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND0(14, 0x9bdc06a725c71235, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND0(15, 0xc19bf174cf692694, R9, R10, R11, R12, R13, R14, R15, R8) + + SHA512ROUND1(16, 0xe49b69c19ef14ad2, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND1(17, 0xefbe4786384f25e3, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND1(18, 0x0fc19dc68b8cd5b5, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND1(19, 0x240ca1cc77ac9c65, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND1(20, 0x2de92c6f592b0275, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND1(21, 0x4a7484aa6ea6e483, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND1(22, 0x5cb0a9dcbd41fbd4, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND1(23, 0x76f988da831153b5, R9, R10, R11, R12, R13, R14, R15, R8) + SHA512ROUND1(24, 0x983e5152ee66dfab, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND1(25, 0xa831c66d2db43210, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND1(26, 0xb00327c898fb213f, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND1(27, 0xbf597fc7beef0ee4, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND1(28, 0xc6e00bf33da88fc2, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND1(29, 0xd5a79147930aa725, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND1(30, 0x06ca6351e003826f, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND1(31, 0x142929670a0e6e70, R9, R10, R11, R12, R13, R14, R15, R8) + SHA512ROUND1(32, 0x27b70a8546d22ffc, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND1(33, 0x2e1b21385c26c926, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND1(34, 0x4d2c6dfc5ac42aed, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND1(35, 0x53380d139d95b3df, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND1(36, 0x650a73548baf63de, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND1(37, 0x766a0abb3c77b2a8, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND1(38, 0x81c2c92e47edaee6, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND1(39, 0x92722c851482353b, R9, R10, R11, R12, R13, R14, R15, R8) + SHA512ROUND1(40, 0xa2bfe8a14cf10364, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND1(41, 0xa81a664bbc423001, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND1(42, 0xc24b8b70d0f89791, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND1(43, 0xc76c51a30654be30, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND1(44, 0xd192e819d6ef5218, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND1(45, 0xd69906245565a910, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND1(46, 0xf40e35855771202a, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND1(47, 0x106aa07032bbd1b8, R9, R10, R11, R12, R13, R14, R15, R8) + SHA512ROUND1(48, 0x19a4c116b8d2d0c8, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND1(49, 0x1e376c085141ab53, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND1(50, 0x2748774cdf8eeb99, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND1(51, 0x34b0bcb5e19b48a8, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND1(52, 0x391c0cb3c5c95a63, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND1(53, 0x4ed8aa4ae3418acb, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND1(54, 0x5b9cca4f7763e373, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND1(55, 0x682e6ff3d6b2b8a3, R9, R10, R11, R12, R13, R14, R15, R8) + SHA512ROUND1(56, 0x748f82ee5defb2fc, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND1(57, 0x78a5636f43172f60, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND1(58, 0x84c87814a1f0ab72, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND1(59, 0x8cc702081a6439ec, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND1(60, 0x90befffa23631e28, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND1(61, 0xa4506cebde82bde9, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND1(62, 0xbef9a3f7b2c67915, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND1(63, 0xc67178f2e372532b, R9, R10, R11, R12, R13, R14, R15, R8) + SHA512ROUND1(64, 0xca273eceea26619c, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND1(65, 0xd186b8c721c0c207, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND1(66, 0xeada7dd6cde0eb1e, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND1(67, 0xf57d4f7fee6ed178, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND1(68, 0x06f067aa72176fba, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND1(69, 0x0a637dc5a2c898a6, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND1(70, 0x113f9804bef90dae, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND1(71, 0x1b710b35131c471b, R9, R10, R11, R12, R13, R14, R15, R8) + SHA512ROUND1(72, 0x28db77f523047d84, R8, R9, R10, R11, R12, R13, R14, R15) + SHA512ROUND1(73, 0x32caab7b40c72493, R15, R8, R9, R10, R11, R12, R13, R14) + SHA512ROUND1(74, 0x3c9ebe0a15c9bebc, R14, R15, R8, R9, R10, R11, R12, R13) + SHA512ROUND1(75, 0x431d67c49c100d4c, R13, R14, R15, R8, R9, R10, R11, R12) + SHA512ROUND1(76, 0x4cc5d4becb3e42b6, R12, R13, R14, R15, R8, R9, R10, R11) + SHA512ROUND1(77, 0x597f299cfc657e2a, R11, R12, R13, R14, R15, R8, R9, R10) + SHA512ROUND1(78, 0x5fcb6fab3ad6faec, R10, R11, R12, R13, R14, R15, R8, R9) + SHA512ROUND1(79, 0x6c44198c4a475817, R9, R10, R11, R12, R13, R14, R15, R8) + + MOVV (0*8)(R4), REGTMP + MOVV (1*8)(R4), REGTMP1 + MOVV (2*8)(R4), REGTMP2 + MOVV (3*8)(R4), REGTMP3 + ADDV REGTMP, R8 // H0 = a + H0 + ADDV REGTMP1, R9 // H1 = b + H1 + ADDV REGTMP2, R10 // H2 = c + H2 + ADDV REGTMP3, R11 // H3 = d + H3 + MOVV R8, (0*8)(R4) + MOVV R9, (1*8)(R4) + MOVV R10, (2*8)(R4) + MOVV R11, (3*8)(R4) + MOVV (4*8)(R4), REGTMP + MOVV (5*8)(R4), REGTMP1 + MOVV (6*8)(R4), REGTMP2 + MOVV (7*8)(R4), REGTMP3 + ADDV REGTMP, R12 // H4 = e + H4 + ADDV REGTMP1, R13 // H5 = f + H5 + ADDV REGTMP2, R14 // H6 = g + H6 + ADDV REGTMP3, R15 // H7 = h + H7 + MOVV R12, (4*8)(R4) + MOVV R13, (5*8)(R4) + MOVV R14, (6*8)(R4) + MOVV R15, (7*8)(R4) + + ADDV $128, R5 + BNE R5, R25, loop + +end: + RET