From: Julian Zhu Date: Tue, 3 Jun 2025 17:06:20 +0000 (+0800) Subject: crypto/sha256: use const table for key loading on loong64 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=dda9d780e25feed39fd41f2e4f68a10489b5ec58;p=gostls13.git crypto/sha256: use const table for key loading on loong64 Load constant keys from a static memory table rather than loading immediates into registers on loong64. Benchmark for Loongson-3A5000: goos: linux goarch: loong64 pkg: crypto/sha256 cpu: Loongson-3A5000-HV @ 2500.00MHz │ sha256o │ sha256n │ │ sec/op │ sec/op vs base │ Hash8Bytes/New-4 356.1n ± 0% 347.0n ± 0% -2.54% (p=0.000 n=8) Hash8Bytes/Sum224-4 368.7n ± 0% 359.5n ± 0% -2.50% (p=0.000 n=8) Hash8Bytes/Sum256-4 367.7n ± 0% 358.9n ± 0% -2.41% (p=0.000 n=8) Hash1K/New-4 4.741µ ± 0% 4.578µ ± 0% -3.44% (p=0.000 n=8) Hash1K/Sum224-4 4.755µ ± 0% 4.591µ ± 0% -3.44% (p=0.000 n=8) Hash1K/Sum256-4 4.753µ ± 0% 4.589µ ± 0% -3.46% (p=0.000 n=8) Hash8K/New-4 35.42µ ± 0% 34.19µ ± 0% -3.45% (p=0.000 n=8) Hash8K/Sum224-4 35.43µ ± 0% 34.21µ ± 0% -3.44% (p=0.000 n=8) Hash8K/Sum256-4 35.46µ ± 0% 34.22µ ± 0% -3.48% (p=0.000 n=8) Hash256K/New-4 1.138m ± 0% 1.098m ± 0% -3.54% (p=0.000 n=8) Hash256K/Sum224-4 1.138m ± 0% 1.098m ± 0% -3.53% (p=0.000 n=8) Hash256K/Sum256-4 1.139m ± 0% 1.099m ± 0% -3.48% (p=0.000 n=8) Hash1M/New-4 4.488m ± 0% 4.388m ± 0% -2.22% (p=0.000 n=8) Hash1M/Sum224-4 4.488m ± 0% 4.387m ± 0% -2.24% (p=0.000 n=8) Hash1M/Sum256-4 4.489m ± 0% 4.388m ± 0% -2.25% (p=0.000 n=8) geomean 50.02µ 48.50µ -3.03% │ sha256o │ sha256n │ │ B/s │ B/s vs base │ Hash8Bytes/New-4 21.42Mi ± 0% 21.99Mi ± 0% +2.63% (p=0.000 n=8) Hash8Bytes/Sum224-4 20.69Mi ± 0% 21.22Mi ± 0% +2.56% (p=0.000 n=8) Hash8Bytes/Sum256-4 20.74Mi ± 0% 21.26Mi ± 0% +2.48% (p=0.000 n=8) Hash1K/New-4 206.0Mi ± 0% 213.3Mi ± 0% +3.57% (p=0.000 n=8) Hash1K/Sum224-4 205.4Mi ± 0% 212.7Mi ± 0% +3.57% (p=0.000 n=8) Hash1K/Sum256-4 205.5Mi ± 0% 212.8Mi ± 0% +3.58% (p=0.000 n=8) Hash8K/New-4 220.6Mi ± 0% 228.5Mi ± 0% +3.58% (p=0.000 n=8) Hash8K/Sum224-4 220.5Mi ± 0% 228.4Mi ± 0% +3.56% (p=0.000 n=8) Hash8K/Sum256-4 220.3Mi ± 0% 228.3Mi ± 0% +3.61% (p=0.000 n=8) Hash256K/New-4 219.7Mi ± 0% 227.7Mi ± 0% +3.67% (p=0.000 n=8) Hash256K/Sum224-4 219.6Mi ± 0% 227.6Mi ± 0% +3.66% (p=0.000 n=8) Hash256K/Sum256-4 219.6Mi ± 0% 227.5Mi ± 0% +3.60% (p=0.000 n=8) Hash1M/New-4 222.8Mi ± 0% 227.9Mi ± 0% +2.27% (p=0.000 n=8) Hash1M/Sum224-4 222.8Mi ± 0% 227.9Mi ± 0% +2.29% (p=0.000 n=8) Hash1M/Sum256-4 222.8Mi ± 0% 227.9Mi ± 0% +2.30% (p=0.000 n=8) geomean 136.0Mi 140.2Mi +3.13% Benchmark for Loongson-3A6000: goos: linux goarch: loong64 pkg: crypto/sha256 cpu: Loongson-3A6000 @ 2500.00MHz │ sha256.old │ sha256.new │ │ sec/op │ sec/op vs base │ Hash8Bytes/New-8 294.5n ± 0% 288.6n ± 0% -2.00% (p=0.000 n=10) Hash8Bytes/Sum224-8 305.0n ± 0% 299.7n ± 0% -1.74% (p=0.000 n=10) Hash8Bytes/Sum256-8 302.0n ± 0% 296.8n ± 0% -1.74% (p=0.000 n=10) Hash1K/New-8 4.186µ ± 0% 4.096µ ± 0% -2.15% (p=0.000 n=10) Hash1K/Sum224-8 4.193µ ± 0% 4.104µ ± 0% -2.12% (p=0.000 n=10) Hash1K/Sum256-8 4.194µ ± 0% 4.108µ ± 0% -2.04% (p=0.000 n=10) Hash8K/New-8 31.44µ ± 0% 30.76µ ± 0% -2.17% (p=0.000 n=10) Hash8K/Sum224-8 31.45µ ± 0% 30.79µ ± 0% -2.10% (p=0.000 n=10) Hash8K/Sum256-8 31.45µ ± 0% 30.78µ ± 0% -2.12% (p=0.000 n=10) Hash256K/New-8 996.7µ ± 0% 975.6µ ± 0% -2.12% (p=0.000 n=10) Hash256K/Sum224-8 996.8µ ± 0% 975.8µ ± 0% -2.11% (p=0.000 n=10) Hash256K/Sum256-8 996.8µ ± 0% 975.6µ ± 0% -2.12% (p=0.000 n=10) Hash1M/New-8 3.987m ± 0% 3.904m ± 0% -2.08% (p=0.000 n=10) Hash1M/Sum224-8 3.990m ± 0% 3.902m ± 0% -2.20% (p=0.000 n=10) Hash1M/Sum256-8 3.987m ± 0% 3.903m ± 0% -2.10% (p=0.000 n=10) geomean 43.59µ 42.69µ -2.06% │ sha256.old │ sha256.new │ │ B/s │ B/s vs base │ Hash8Bytes/New-8 25.90Mi ± 0% 26.44Mi ± 0% +2.06% (p=0.000 n=10) Hash8Bytes/Sum224-8 25.01Mi ± 0% 25.46Mi ± 0% +1.77% (p=0.000 n=10) Hash8Bytes/Sum256-8 25.26Mi ± 0% 25.72Mi ± 0% +1.79% (p=0.000 n=10) Hash1K/New-8 233.3Mi ± 0% 238.5Mi ± 0% +2.19% (p=0.000 n=10) Hash1K/Sum224-8 232.9Mi ± 0% 238.0Mi ± 0% +2.17% (p=0.000 n=10) Hash1K/Sum256-8 232.9Mi ± 0% 237.7Mi ± 0% +2.07% (p=0.000 n=10) Hash8K/New-8 248.5Mi ± 0% 254.0Mi ± 0% +2.22% (p=0.000 n=10) Hash8K/Sum224-8 248.4Mi ± 0% 253.7Mi ± 0% +2.14% (p=0.000 n=10) Hash8K/Sum256-8 248.4Mi ± 0% 253.8Mi ± 0% +2.17% (p=0.000 n=10) Hash256K/New-8 250.8Mi ± 0% 256.3Mi ± 0% +2.17% (p=0.000 n=10) Hash256K/Sum224-8 250.8Mi ± 0% 256.2Mi ± 0% +2.16% (p=0.000 n=10) Hash256K/Sum256-8 250.8Mi ± 0% 256.2Mi ± 0% +2.17% (p=0.000 n=10) Hash1M/New-8 250.8Mi ± 0% 256.2Mi ± 0% +2.12% (p=0.000 n=10) Hash1M/Sum224-8 250.6Mi ± 0% 256.3Mi ± 0% +2.25% (p=0.000 n=10) Hash1M/Sum256-8 250.8Mi ± 0% 256.2Mi ± 0% +2.14% (p=0.000 n=10) geomean 156.0Mi 159.3Mi +2.11% Change-Id: Ib72cf3c746d4ad73e52e5d31f6b4a834fd36d934 Reviewed-on: https://go-review.googlesource.com/c/go/+/678435 LUCI-TryBot-Result: Go LUCI Reviewed-by: Dmitri Shuralyov Reviewed-by: sophie zhao Reviewed-by: Mark Freeman Reviewed-by: abner chenc --- diff --git a/src/crypto/internal/fips140/sha256/sha256block_loong64.s b/src/crypto/internal/fips140/sha256/sha256block_loong64.s index e171d93e0b..ad03cd9931 100644 --- a/src/crypto/internal/fips140/sha256/sha256block_loong64.s +++ b/src/crypto/internal/fips140/sha256/sha256block_loong64.s @@ -52,6 +52,7 @@ #define REGTMP3 R18 #define REGTMP4 R7 #define REGTMP5 R6 +#define REG_KT R19 // W[i] = M[i]; for 0 <= i <= 15 #define LOAD0(index) \ @@ -89,8 +90,9 @@ // Ch(x, y, z) = (x AND y) XOR (NOT x AND z) // = ((y XOR z) AND x) XOR z // Calculate T1 in REGTMP4 -#define SHA256T1(const, e, f, g, h) \ - ADDV $const, h; \ +#define SHA256T1(index, e, f, g, h) \ + MOVW (index*4)(REG_KT), REGTMP5; \ + ADDV REGTMP5, h; \ ADD REGTMP4, h; \ ROTR $6, e, REGTMP5; \ ROTR $11, e, REGTMP; \ @@ -122,19 +124,19 @@ // Calculate T1 and T2, then e = d + T1 and a = T1 + T2. // The values for e and a are stored in d and h, ready for rotation. -#define SHA256ROUND(const, a, b, c, d, e, f, g, h) \ - SHA256T1(const, e, f, g, h); \ +#define SHA256ROUND(index, a, b, c, d, e, f, g, h) \ + SHA256T1(index, e, f, g, h); \ SHA256T2(a, b, c); \ ADD REGTMP4, d; \ ADD REGTMP1, REGTMP4, h -#define SHA256ROUND0(index, const, a, b, c, d, e, f, g, h) \ +#define SHA256ROUND0(index, a, b, c, d, e, f, g, h) \ LOAD0(index); \ - SHA256ROUND(const, a, b, c, d, e, f, g, h) + SHA256ROUND(index, a, b, c, d, e, f, g, h) -#define SHA256ROUND1(index, const, a, b, c, d, e, f, g, h) \ +#define SHA256ROUND1(index, a, b, c, d, e, f, g, h) \ LOAD1(index); \ - SHA256ROUND(const, a, b, c, d, e, f, g, h) + SHA256ROUND(index, a, b, c, d, e, f, g, h) // A stack frame size of 64 bytes is required here, because // the frame size used for data expansion is 64 bytes. @@ -147,6 +149,8 @@ TEXT ·block(SB),NOSPLIT,$64-32 AND $~63, R6 BEQ R6, end + MOVV $·_K(SB), REG_KT // const table + // p_len >= 64 MOVV dig+0(FP), R4 ADDV R5, R6, R25 @@ -160,71 +164,71 @@ TEXT ·block(SB),NOSPLIT,$64-32 MOVW (7*4)(R4), R15 // h = H7 loop: - SHA256ROUND0(0, 0x428a2f98, R8, R9, R10, R11, R12, R13, R14, R15) - SHA256ROUND0(1, 0x71374491, R15, R8, R9, R10, R11, R12, R13, R14) - SHA256ROUND0(2, 0xb5c0fbcf, R14, R15, R8, R9, R10, R11, R12, R13) - SHA256ROUND0(3, 0xe9b5dba5, R13, R14, R15, R8, R9, R10, R11, R12) - SHA256ROUND0(4, 0x3956c25b, R12, R13, R14, R15, R8, R9, R10, R11) - SHA256ROUND0(5, 0x59f111f1, R11, R12, R13, R14, R15, R8, R9, R10) - SHA256ROUND0(6, 0x923f82a4, R10, R11, R12, R13, R14, R15, R8, R9) - SHA256ROUND0(7, 0xab1c5ed5, R9, R10, R11, R12, R13, R14, R15, R8) - SHA256ROUND0(8, 0xd807aa98, R8, R9, R10, R11, R12, R13, R14, R15) - SHA256ROUND0(9, 0x12835b01, R15, R8, R9, R10, R11, R12, R13, R14) - SHA256ROUND0(10, 0x243185be, R14, R15, R8, R9, R10, R11, R12, R13) - SHA256ROUND0(11, 0x550c7dc3, R13, R14, R15, R8, R9, R10, R11, R12) - SHA256ROUND0(12, 0x72be5d74, R12, R13, R14, R15, R8, R9, R10, R11) - SHA256ROUND0(13, 0x80deb1fe, R11, R12, R13, R14, R15, R8, R9, R10) - SHA256ROUND0(14, 0x9bdc06a7, R10, R11, R12, R13, R14, R15, R8, R9) - SHA256ROUND0(15, 0xc19bf174, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND0(0, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND0(1, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND0(2, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND0(3, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND0(4, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND0(5, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND0(6, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND0(7, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND0(8, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND0(9, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND0(10, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND0(11, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND0(12, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND0(13, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND0(14, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND0(15, R9, R10, R11, R12, R13, R14, R15, R8) - SHA256ROUND1(16, 0xe49b69c1, R8, R9, R10, R11, R12, R13, R14, R15) - SHA256ROUND1(17, 0xefbe4786, R15, R8, R9, R10, R11, R12, R13, R14) - SHA256ROUND1(18, 0x0fc19dc6, R14, R15, R8, R9, R10, R11, R12, R13) - SHA256ROUND1(19, 0x240ca1cc, R13, R14, R15, R8, R9, R10, R11, R12) - SHA256ROUND1(20, 0x2de92c6f, R12, R13, R14, R15, R8, R9, R10, R11) - SHA256ROUND1(21, 0x4a7484aa, R11, R12, R13, R14, R15, R8, R9, R10) - SHA256ROUND1(22, 0x5cb0a9dc, R10, R11, R12, R13, R14, R15, R8, R9) - SHA256ROUND1(23, 0x76f988da, R9, R10, R11, R12, R13, R14, R15, R8) - SHA256ROUND1(24, 0x983e5152, R8, R9, R10, R11, R12, R13, R14, R15) - SHA256ROUND1(25, 0xa831c66d, R15, R8, R9, R10, R11, R12, R13, R14) - SHA256ROUND1(26, 0xb00327c8, R14, R15, R8, R9, R10, R11, R12, R13) - SHA256ROUND1(27, 0xbf597fc7, R13, R14, R15, R8, R9, R10, R11, R12) - SHA256ROUND1(28, 0xc6e00bf3, R12, R13, R14, R15, R8, R9, R10, R11) - SHA256ROUND1(29, 0xd5a79147, R11, R12, R13, R14, R15, R8, R9, R10) - SHA256ROUND1(30, 0x06ca6351, R10, R11, R12, R13, R14, R15, R8, R9) - SHA256ROUND1(31, 0x14292967, R9, R10, R11, R12, R13, R14, R15, R8) - SHA256ROUND1(32, 0x27b70a85, R8, R9, R10, R11, R12, R13, R14, R15) - SHA256ROUND1(33, 0x2e1b2138, R15, R8, R9, R10, R11, R12, R13, R14) - SHA256ROUND1(34, 0x4d2c6dfc, R14, R15, R8, R9, R10, R11, R12, R13) - SHA256ROUND1(35, 0x53380d13, R13, R14, R15, R8, R9, R10, R11, R12) - SHA256ROUND1(36, 0x650a7354, R12, R13, R14, R15, R8, R9, R10, R11) - SHA256ROUND1(37, 0x766a0abb, R11, R12, R13, R14, R15, R8, R9, R10) - SHA256ROUND1(38, 0x81c2c92e, R10, R11, R12, R13, R14, R15, R8, R9) - SHA256ROUND1(39, 0x92722c85, R9, R10, R11, R12, R13, R14, R15, R8) - SHA256ROUND1(40, 0xa2bfe8a1, R8, R9, R10, R11, R12, R13, R14, R15) - SHA256ROUND1(41, 0xa81a664b, R15, R8, R9, R10, R11, R12, R13, R14) - SHA256ROUND1(42, 0xc24b8b70, R14, R15, R8, R9, R10, R11, R12, R13) - SHA256ROUND1(43, 0xc76c51a3, R13, R14, R15, R8, R9, R10, R11, R12) - SHA256ROUND1(44, 0xd192e819, R12, R13, R14, R15, R8, R9, R10, R11) - SHA256ROUND1(45, 0xd6990624, R11, R12, R13, R14, R15, R8, R9, R10) - SHA256ROUND1(46, 0xf40e3585, R10, R11, R12, R13, R14, R15, R8, R9) - SHA256ROUND1(47, 0x106aa070, R9, R10, R11, R12, R13, R14, R15, R8) - SHA256ROUND1(48, 0x19a4c116, R8, R9, R10, R11, R12, R13, R14, R15) - SHA256ROUND1(49, 0x1e376c08, R15, R8, R9, R10, R11, R12, R13, R14) - SHA256ROUND1(50, 0x2748774c, R14, R15, R8, R9, R10, R11, R12, R13) - SHA256ROUND1(51, 0x34b0bcb5, R13, R14, R15, R8, R9, R10, R11, R12) - SHA256ROUND1(52, 0x391c0cb3, R12, R13, R14, R15, R8, R9, R10, R11) - SHA256ROUND1(53, 0x4ed8aa4a, R11, R12, R13, R14, R15, R8, R9, R10) - SHA256ROUND1(54, 0x5b9cca4f, R10, R11, R12, R13, R14, R15, R8, R9) - SHA256ROUND1(55, 0x682e6ff3, R9, R10, R11, R12, R13, R14, R15, R8) - SHA256ROUND1(56, 0x748f82ee, R8, R9, R10, R11, R12, R13, R14, R15) - SHA256ROUND1(57, 0x78a5636f, R15, R8, R9, R10, R11, R12, R13, R14) - SHA256ROUND1(58, 0x84c87814, R14, R15, R8, R9, R10, R11, R12, R13) - SHA256ROUND1(59, 0x8cc70208, R13, R14, R15, R8, R9, R10, R11, R12) - SHA256ROUND1(60, 0x90befffa, R12, R13, R14, R15, R8, R9, R10, R11) - SHA256ROUND1(61, 0xa4506ceb, R11, R12, R13, R14, R15, R8, R9, R10) - SHA256ROUND1(62, 0xbef9a3f7, R10, R11, R12, R13, R14, R15, R8, R9) - SHA256ROUND1(63, 0xc67178f2, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(16, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(17, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(18, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(19, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(20, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(21, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(22, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(23, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(24, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(25, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(26, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(27, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(28, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(29, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(30, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(31, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(32, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(33, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(34, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(35, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(36, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(37, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(38, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(39, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(40, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(41, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(42, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(43, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(44, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(45, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(46, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(47, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(48, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(49, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(50, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(51, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(52, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(53, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(54, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(55, R9, R10, R11, R12, R13, R14, R15, R8) + SHA256ROUND1(56, R8, R9, R10, R11, R12, R13, R14, R15) + SHA256ROUND1(57, R15, R8, R9, R10, R11, R12, R13, R14) + SHA256ROUND1(58, R14, R15, R8, R9, R10, R11, R12, R13) + SHA256ROUND1(59, R13, R14, R15, R8, R9, R10, R11, R12) + SHA256ROUND1(60, R12, R13, R14, R15, R8, R9, R10, R11) + SHA256ROUND1(61, R11, R12, R13, R14, R15, R8, R9, R10) + SHA256ROUND1(62, R10, R11, R12, R13, R14, R15, R8, R9) + SHA256ROUND1(63, R9, R10, R11, R12, R13, R14, R15, R8) MOVW (0*4)(R4), REGTMP MOVW (1*4)(R4), REGTMP1