From 1bc795633ece35a6429284cee4d7cec43b136bf7 Mon Sep 17 00:00:00 2001 From: Xiaolin Zhao Date: Tue, 4 Jun 2024 09:40:14 +0800 Subject: [PATCH] crypto/sha256: implement sha256block in hardware on loong64 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit goos: linux goarch: loong64 pkg: crypto/sha256 cpu: Loongson-3A6000 @ 2500.00MHz │ bench.old │ bench.new │ │ sec/op │ sec/op vs base │ Hash8Bytes/New 674.0n ± 0% 433.3n ± 0% -35.71% (p=0.000 n=20) Hash8Bytes/Sum224 695.3n ± 0% 445.8n ± 0% -35.88% (p=0.000 n=20) Hash8Bytes/Sum256 676.0n ± 0% 475.5n ± 0% -29.66% (p=0.000 n=20) Hash1K/New 8.982µ ± 0% 4.963µ ± 0% -44.74% (p=0.000 n=20) Hash1K/Sum224 8.999µ ± 0% 4.975µ ± 0% -44.72% (p=0.000 n=20) Hash1K/Sum256 8.979µ ± 0% 5.005µ ± 0% -44.26% (p=0.000 n=20) Hash8K/New 66.97µ ± 0% 36.78µ ± 0% -45.08% (p=0.000 n=20) Hash8K/Sum224 66.99µ ± 0% 36.79µ ± 0% -45.09% (p=0.000 n=20) Hash8K/Sum256 66.97µ ± 0% 36.82µ ± 0% -45.03% (p=0.000 n=20) geomean 7.431µ 4.357µ -41.37% │ bench.old │ bench.new │ │ B/s │ B/s vs base │ Hash8Bytes/New 11.32Mi ± 0% 17.60Mi ± 0% +55.52% (p=0.000 n=20) Hash8Bytes/Sum224 10.98Mi ± 0% 17.11Mi ± 0% +55.86% (p=0.000 n=20) Hash8Bytes/Sum256 11.28Mi ± 0% 16.04Mi ± 0% +42.18% (p=0.000 n=20) Hash1K/New 108.7Mi ± 0% 196.8Mi ± 0% +80.98% (p=0.000 n=20) Hash1K/Sum224 108.5Mi ± 0% 196.3Mi ± 0% +80.89% (p=0.000 n=20) Hash1K/Sum256 108.8Mi ± 0% 195.1Mi ± 0% +79.42% (p=0.000 n=20) Hash8K/New 116.7Mi ± 0% 212.4Mi ± 0% +82.09% (p=0.000 n=20) Hash8K/Sum224 116.6Mi ± 0% 212.4Mi ± 0% +82.09% (p=0.000 n=20) Hash8K/Sum256 116.7Mi ± 0% 212.2Mi ± 0% +81.90% (p=0.000 n=20) geomean 52.15Mi 88.95Mi +70.55% goos: linux goarch: loong64 pkg: crypto/sha256 cpu: Loongson-3A5000 @ 2500.00MHz │ bench.old │ bench.new │ │ sec/op │ sec/op vs base │ Hash8Bytes/New 855.9n ± 0% 521.1n ± 0% -39.12% (p=0.000 n=20) Hash8Bytes/Sum224 875.2n ± 0% 532.7n ± 1% -39.13% (p=0.000 n=20) Hash8Bytes/Sum256 909.4n ± 0% 553.9n ± 0% -39.09% (p=0.000 n=20) Hash1K/New 11.304µ ± 0% 5.684µ ± 0% -49.72% (p=0.000 n=20) Hash1K/Sum224 11.323µ ± 0% 5.690µ ± 0% -49.75% (p=0.000 n=20) Hash1K/Sum256 11.341µ ± 0% 5.714µ ± 0% -49.62% (p=0.000 n=20) Hash8K/New 84.26µ ± 0% 41.97µ ± 0% -50.19% (p=0.000 n=20) Hash8K/Sum224 84.27µ ± 0% 41.99µ ± 0% -50.18% (p=0.000 n=20) Hash8K/Sum256 84.32µ ± 0% 42.01µ ± 0% -50.18% (p=0.000 n=20) geomean 9.434µ 5.041µ -46.56% │ bench.old │ bench.new │ │ B/s │ B/s vs base │ Hash8Bytes/New 8.917Mi ± 0% 14.639Mi ± 0% +64.17% (p=0.000 n=20) Hash8Bytes/Sum224 8.717Mi ± 0% 14.319Mi ± 1% +64.28% (p=0.000 n=20) Hash8Bytes/Sum256 8.392Mi ± 0% 13.771Mi ± 0% +64.09% (p=0.000 n=20) Hash1K/New 86.39Mi ± 0% 171.81Mi ± 0% +98.89% (p=0.000 n=20) Hash1K/Sum224 86.25Mi ± 0% 171.64Mi ± 0% +99.00% (p=0.000 n=20) Hash1K/Sum256 86.11Mi ± 0% 170.92Mi ± 0% +98.49% (p=0.000 n=20) Hash8K/New 92.72Mi ± 0% 186.13Mi ± 0% +100.75% (p=0.000 n=20) Hash8K/Sum224 92.71Mi ± 0% 186.07Mi ± 0% +100.71% (p=0.000 n=20) Hash8K/Sum256 92.65Mi ± 0% 185.99Mi ± 0% +100.74% (p=0.000 n=20) geomean 41.08Mi 76.87Mi +87.12% Change-Id: Ib41d19d136b8593339af94a822942c102238891b Reviewed-on: https://go-review.googlesource.com/c/go/+/590155 Reviewed-by: Meidan Li Reviewed-by: abner chenc Reviewed-by: Dmitri Shuralyov Reviewed-by: Filippo Valsorda Reviewed-by: Roland Shoemaker Reviewed-by: Qiqi Huang LUCI-TryBot-Result: Go LUCI --- src/crypto/sha256/sha256block_decl.go | 2 +- src/crypto/sha256/sha256block_generic.go | 2 +- src/crypto/sha256/sha256block_loong64.s | 259 +++++++++++++++++++++++ 3 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 src/crypto/sha256/sha256block_loong64.s diff --git a/src/crypto/sha256/sha256block_decl.go b/src/crypto/sha256/sha256block_decl.go index d622c3b174..e793039387 100644 --- a/src/crypto/sha256/sha256block_decl.go +++ b/src/crypto/sha256/sha256block_decl.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (386 || amd64 || s390x || ppc64le || ppc64 || riscv64) && !purego +//go:build (386 || amd64 || loong64 || ppc64 || ppc64le || riscv64 || s390x) && !purego package sha256 diff --git a/src/crypto/sha256/sha256block_generic.go b/src/crypto/sha256/sha256block_generic.go index 4d21bdffbd..8ca8401f65 100644 --- a/src/crypto/sha256/sha256block_generic.go +++ b/src/crypto/sha256/sha256block_generic.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (!386 && !amd64 && !arm64 && !ppc64 && !ppc64le && !riscv64 && !s390x) || purego +//go:build (!386 && !amd64 && !arm64 && !loong64 && !ppc64 && !ppc64le && !riscv64 && !s390x) || purego package sha256 diff --git a/src/crypto/sha256/sha256block_loong64.s b/src/crypto/sha256/sha256block_loong64.s new file mode 100644 index 0000000000..2a2fbe6833 --- /dev/null +++ b/src/crypto/sha256/sha256block_loong64.s @@ -0,0 +1,259 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !purego + +#include "textflag.h" + +// SHA256 block routine. See sha256block.go for Go equivalent. +// +// The algorithm is detailed in FIPS 180-4: +// +// https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf +// +// W[i] = M[i]; for 0 <= i <= 15 +// W[i] = SIGMA1(W[i-2]) + W[i-7] + SIGMA0(W[i-15]) + W[i-16]; for 16 <= i <= 63 +// +// a = H0 +// b = H1 +// c = H2 +// d = H3 +// e = H4 +// f = H5 +// g = H6 +// h = H7 +// +// for i = 0 to 63 { +// T1 = h + BIGSIGMA1(e) + Ch(e,f,g) + K[i] + W[i] +// T2 = BIGSIGMA0(a) + Maj(a,b,c) +// h = g +// g = f +// f = e +// e = d + T1 +// d = c +// c = b +// b = a +// a = T1 + T2 +// } +// +// H0 = a + H0 +// H1 = b + H1 +// H2 = c + H2 +// H3 = d + H3 +// H4 = e + H4 +// H5 = f + H5 +// H6 = g + H6 +// H7 = h + H7 + +#define REGTMP R30 +#define REGTMP1 R16 +#define REGTMP2 R17 +#define REGTMP3 R18 +#define REGTMP4 R7 +#define REGTMP5 R6 + +// W[i] = M[i]; for 0 <= i <= 15 +#define LOAD0(index) \ + MOVW (index*4)(R5), REGTMP4; \ + WORD $0x38e7; \ // REVB2W REGTMP4, REGTMP4 to big-endian + MOVW REGTMP4, (index*4)(R3) + +// W[i] = SIGMA1(W[i-2]) + W[i-7] + SIGMA0(W[i-15]) + W[i-16]; for 16 <= i <= 63 +// SIGMA0(x) = ROTR(7,x) XOR ROTR(18,x) XOR SHR(3,x) +// SIGMA1(x) = ROTR(17,x) XOR ROTR(19,x) XOR SHR(10,x) +#define LOAD1(index) \ + MOVW (((index-2)&0xf)*4)(R3), REGTMP4; \ + MOVW (((index-15)&0xf)*4)(R3), REGTMP1; \ + MOVW (((index-7)&0xf)*4)(R3), REGTMP; \ + MOVW REGTMP4, REGTMP2; \ + MOVW REGTMP4, REGTMP3; \ + ROTR $17, REGTMP4; \ + ROTR $19, REGTMP2; \ + SRL $10, REGTMP3; \ + XOR REGTMP2, REGTMP4; \ + XOR REGTMP3, REGTMP4; \ + ROTR $7, REGTMP1, REGTMP5; \ + SRL $3, REGTMP1, REGTMP3; \ + ROTR $18, REGTMP1, REGTMP2; \ + ADD REGTMP, REGTMP4; \ + MOVW (((index-16)&0xf)*4)(R3), REGTMP; \ + XOR REGTMP3, REGTMP5; \ + XOR REGTMP2, REGTMP5; \ + ADD REGTMP, REGTMP5; \ + ADD REGTMP5, REGTMP4; \ + MOVW REGTMP4, ((index&0xf)*4)(R3) + +// T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + K[i] + W[i] +// BIGSIGMA1(x) = ROTR(6,x) XOR ROTR(11,x) XOR ROTR(25,x) +// Ch(x, y, z) = (x AND y) XOR (NOT x AND z) +// Calculate T1 in REGTMP4 +#define SHA256T1(const, e, f, g, h) \ + ADDV $const, h; \ + ADD REGTMP4, h; \ + ROTR $6, e, REGTMP4; \ + ROTR $11, e, REGTMP; \ + ROTR $25, e, REGTMP3; \ + AND f, e, REGTMP2; \ + XOR REGTMP, REGTMP4; \ + MOVV $0xffffffff, REGTMP; \ + XOR REGTMP4, REGTMP3; \ + XOR REGTMP, e, REGTMP5; \ + ADD REGTMP3, h; \ + AND g, REGTMP5; \ + XOR REGTMP2, REGTMP5; \ + ADD h, REGTMP5, REGTMP4 + +// T2 = BIGSIGMA0(a) + Maj(a, b, c) +// BIGSIGMA0(x) = ROTR(2,x) XOR ROTR(13,x) XOR ROTR(22,x) +// Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z) +// Calculate T2 in REGTMP1 +#define SHA256T2(a, b, c) \ + ROTR $2, a, REGTMP5; \ + AND b, c, REGTMP1; \ + ROTR $13, a, REGTMP3; \ + AND c, a, REGTMP; \ + XOR REGTMP3, REGTMP5; \ + XOR REGTMP, REGTMP1; \ + ROTR $22, a, REGTMP2; \ + AND a, b, REGTMP3; \ + XOR REGTMP2, REGTMP5; \ + XOR REGTMP3, REGTMP1; \ + ADD REGTMP5, REGTMP1 + +// Calculate T1 and T2, then e = d + T1 and a = T1 + T2. +// The values for e and a are stored in d and h, ready for rotation. +#define SHA256ROUND(const, a, b, c, d, e, f, g, h) \ + SHA256T1(const, e, f, g, h); \ + SHA256T2(a, b, c); \ + ADD REGTMP4, d; \ + ADD REGTMP1, REGTMP4, h + +#define SHA256ROUND0(index, const, a, b, c, d, e, f, g, h) \ + LOAD0(index); \ + SHA256ROUND(const, a, b, c, d, e, f, g, h) + +#define SHA256ROUND1(index, const, a, b, c, d, e, f, g, h) \ + LOAD1(index); \ + SHA256ROUND(const, a, b, c, d, e, f, g, h) + +// A stack frame size of 64 bytes is required here, because +// the frame size used for data expansion is 64 bytes. +// See the definition of the macro LOAD1 above (4 bytes * 16 entries). +// +//func block(dig *digest, p []byte) +TEXT ·block(SB),NOSPLIT,$64-32 + MOVV p_base+8(FP), R5 + MOVV p_len+16(FP), R6 + AND $~63, R6 + BEQ R6, end + + // p_len >= 64 + MOVV dig+0(FP), R4 + ADDV R5, R6, R25 + MOVW (0*4)(R4), R8 // a = H0 + MOVW (1*4)(R4), R9 // b = H1 + MOVW (2*4)(R4), R10 // c = H2 + MOVW (3*4)(R4), R11 // d = H3 + MOVW (4*4)(R4), R12 // e = H4 + MOVW (5*4)(R4), R13 // f = H5 + MOVW (6*4)(R4), R14 // g = H6 + MOVW (7*4)(R4), R15 // h = H7 + +loop: + SHA256ROUND0(0, 0x428a2f98, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND0(1, 0x71374491, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND0(2, 0xb5c0fbcf, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND0(3, 0xe9b5dba5, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND0(4, 0x3956c25b, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND0(5, 0x59f111f1, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND0(6, 0x923f82a4, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND0(7, 0xab1c5ed5, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND0(8, 0xd807aa98, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND0(9, 0x12835b01, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND0(10, 0x243185be, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND0(11, 0x550c7dc3, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND0(12, 0x72be5d74, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND0(13, 0x80deb1fe, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND0(14, 0x9bdc06a7, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND0(15, 0xc19bf174, R9, R10, R11, R12, R13, R14, R15, R8) + + SHA256ROUND1(16, 0xe49b69c1, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(17, 0xefbe4786, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(18, 0x0fc19dc6, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(19, 0x240ca1cc, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(20, 0x2de92c6f, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(21, 0x4a7484aa, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(22, 0x5cb0a9dc, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(23, 0x76f988da, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(24, 0x983e5152, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(25, 0xa831c66d, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(26, 0xb00327c8, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(27, 0xbf597fc7, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(28, 0xc6e00bf3, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(29, 0xd5a79147, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(30, 0x06ca6351, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(31, 0x14292967, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(32, 0x27b70a85, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(33, 0x2e1b2138, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(34, 0x4d2c6dfc, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(35, 0x53380d13, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(36, 0x650a7354, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(37, 0x766a0abb, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(38, 0x81c2c92e, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(39, 0x92722c85, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(40, 0xa2bfe8a1, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(41, 0xa81a664b, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(42, 0xc24b8b70, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(43, 0xc76c51a3, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(44, 0xd192e819, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(45, 0xd6990624, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(46, 0xf40e3585, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(47, 0x106aa070, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(48, 0x19a4c116, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(49, 0x1e376c08, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(50, 0x2748774c, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(51, 0x34b0bcb5, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(52, 0x391c0cb3, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(53, 0x4ed8aa4a, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(54, 0x5b9cca4f, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(55, 0x682e6ff3, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(56, 0x748f82ee, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(57, 0x78a5636f, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(58, 0x84c87814, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(59, 0x8cc70208, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(60, 0x90befffa, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(61, 0xa4506ceb, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(62, 0xbef9a3f7, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(63, 0xc67178f2, R9, R10, R11, R12, R13, R14, R15, R8) + + MOVW (0*4)(R4), REGTMP + MOVW (1*4)(R4), REGTMP1 + MOVW (2*4)(R4), REGTMP2 + MOVW (3*4)(R4), REGTMP3 + ADD REGTMP, R8 // H0 = a + H0 + ADD REGTMP1, R9 // H1 = b + H1 + ADD REGTMP2, R10 // H2 = c + H2 + ADD REGTMP3, R11 // H3 = d + H3 + MOVW R8, (0*4)(R4) + MOVW R9, (1*4)(R4) + MOVW R10, (2*4)(R4) + MOVW R11, (3*4)(R4) + MOVW (4*4)(R4), REGTMP + MOVW (5*4)(R4), REGTMP1 + MOVW (6*4)(R4), REGTMP2 + MOVW (7*4)(R4), REGTMP3 + ADD REGTMP, R12 // H4 = e + H4 + ADD REGTMP1, R13 // H5 = f + H5 + ADD REGTMP2, R14 // H6 = g + H6 + ADD REGTMP3, R15 // H7 = h + H7 + MOVW R12, (4*4)(R4) + MOVW R13, (5*4)(R4) + MOVW R14, (6*4)(R4) + MOVW R15, (7*4)(R4) + + ADDV $64, R5 + BNE R5, R25, loop + +end: + RET -- 2.48.1