From: Russ Cox Date: Thu, 5 Dec 2024 03:08:35 +0000 (-0500) Subject: crypto/internal/fips140/bigmod: add more //go:norace annotations X-Git-Tag: go1.24rc1~23 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=c3a706e1d2ea900fbaa715fb0100fb034abb1414;p=gostls13.git crypto/internal/fips140/bigmod: add more //go:norace annotations //go:norace does not carry over when a function is inlined. Add //go:norace to functions that inline loops over Nat words. Improves race tests, also asan, msan. These are with -race: goos: darwin goarch: arm64 pkg: crypto/internal/fips140/bigmod cpu: Apple M3 Pro │ old │ new │ │ sec/op │ sec/op vs base │ ModAdd-12 172.4n ± 3% 117.9n ± 3% -31.62% (p=0.000 n=20) ModSub-12 147.1n ± 2% 111.4n ± 3% -24.27% (p=0.000 n=20) NewModulus-12 12.966µ ± 1% 9.743µ ± 2% -24.86% (p=0.000 n=20) MontgomeryRepr-12 1305.5n ± 1% 986.3n ± 0% -24.45% (p=0.000 n=20) MontgomeryMul-12 1304.0n ± 1% 976.8n ± 0% -25.10% (p=0.000 n=20) ModMul-12 2.893µ ± 1% 2.055µ ± 3% -28.97% (p=0.000 n=20) ExpBig-12 2.784m ± 0% 2.789m ± 0% +0.17% (p=0.008 n=20) Exp-12 3.468m ± 0% 2.620m ± 0% -24.45% (p=0.000 n=20) geomean 7.930µ 6.073µ -23.41% pkg: crypto/rsa │ old │ new │ │ sec/op │ sec/op vs base │ DecryptPKCS1v15/2048-12 1.795m ± 1% 1.175m ± 1% -34.52% (p=0.000 n=20) DecryptPKCS1v15/3072-12 3.836m ± 2% 2.647m ± 0% -31.01% (p=0.000 n=20) DecryptPKCS1v15/4096-12 7.316m ± 0% 5.437m ± 0% -25.68% (p=0.000 n=20) EncryptPKCS1v15/2048-12 45.85µ ± 1% 34.78µ ± 0% -24.15% (p=0.000 n=20) DecryptOAEP/2048-12 1.793m ± 2% 1.188m ± 1% -33.76% (p=0.000 n=20) EncryptOAEP/2048-12 55.11µ ± 1% 43.91µ ± 1% -20.32% (p=0.000 n=20) SignPKCS1v15/2048-12 1.797m ± 2% 1.193m ± 0% -33.62% (p=0.000 n=20) VerifyPKCS1v15/2048-12 45.16µ ± 1% 34.51µ ± 0% -23.57% (p=0.000 n=20) SignPSS/2048-12 1.826m ± 2% 1.213m ± 0% -33.55% (p=0.000 n=20) VerifyPSS/2048-12 53.25µ ± 1% 42.40µ ± 1% -20.36% (p=0.000 n=20) GenerateKey/2048-12 323.7m ± 33% 209.0m ± 17% -35.43% (p=0.000 n=20) ParsePKCS8PrivateKey/2048-12 105.26µ ± 0% 94.74µ ± 0% -9.99% (p=0.000 n=20) geomean 792.5µ 574.3µ -27.53% Change-Id: I1f1986cf2bac126d7346799b08b17d356b28d956 Reviewed-on: https://go-review.googlesource.com/c/go/+/633995 LUCI-TryBot-Result: Go LUCI Reviewed-by: Roland Shoemaker --- diff --git a/src/crypto/internal/fips140/bigmod/nat.go b/src/crypto/internal/fips140/bigmod/nat.go index bcf827b1b5..2f17f896b3 100644 --- a/src/crypto/internal/fips140/bigmod/nat.go +++ b/src/crypto/internal/fips140/bigmod/nat.go @@ -18,6 +18,15 @@ const ( _S = _W / 8 ) +// Note: These functions make many loops over all the words in a Nat. +// These loops used to be in assembly, invisible to -race, -asan, and -msan, +// but now they are in Go and incur significant overhead in those modes. +// To bring the old performance back, we mark all functions that loop +// over Nat words with //go:norace. Because //go:norace does not +// propagate across inlining, we must also mark functions that inline +// //go:norace functions - specifically, those that inline add, addMulVVW, +// assign, cmpGeq, rshift1, and sub. + // choice represents a constant-time boolean. The value of choice is always // either 1 or 0. We use an int instead of bool in order to make decisions in // constant time by turning it into a mask. @@ -152,6 +161,8 @@ func (x *Nat) Bytes(m *Modulus) []byte { // SetBytes returns an error if b >= m. // // The output will be resized to the size of m and overwritten. +// +//go:norace func (x *Nat) SetBytes(b []byte, m *Modulus) (*Nat, error) { x.resetFor(m) if err := x.setBytes(b); err != nil { @@ -283,8 +294,6 @@ func (x *Nat) IsMinusOne(m *Modulus) choice { } // IsOdd returns 1 if x is odd, and 0 otherwise. -// -//go:norace func (x *Nat) IsOdd() choice { if len(x.limbs) == 0 { return no @@ -538,6 +547,8 @@ func NewModulus(b []byte) (*Modulus, error) { // NewModulusProduct creates a new Modulus from the product of two numbers // represented as big-endian byte slices. The result must be greater than one. +// +//go:norace func NewModulusProduct(a, b []byte) (*Modulus, error) { x := NewNat().resetToBytes(a) y := NewNat().resetToBytes(b) @@ -624,6 +635,8 @@ func (x *Nat) shiftIn(y uint, m *Modulus) *Nat { // This works regardless how large the value of x is. // // The output will be resized to the size of m and overwritten. +// +//go:norace func (out *Nat) Mod(x *Nat, m *Modulus) *Nat { out.resetFor(m) // Working our way from the most significant to the least significant limb, @@ -673,6 +686,8 @@ func (out *Nat) resetFor(m *Modulus) *Nat { // overflowed its size, meaning abstractly x > 2^_W*n > m even if x < m. // // x and m operands must have the same announced length. +// +//go:norace func (x *Nat) maybeSubtractModulus(always choice, m *Modulus) { t := NewNat().set(x) underflow := t.sub(m.nat) @@ -686,6 +701,8 @@ func (x *Nat) maybeSubtractModulus(always choice, m *Modulus) { // // The length of both operands must be the same as the modulus. Both operands // must already be reduced modulo m. +// +//go:norace func (x *Nat) Sub(y *Nat, m *Modulus) *Nat { underflow := x.sub(y) // If the subtraction underflowed, add m. @@ -710,6 +727,8 @@ func (x *Nat) SubOne(m *Modulus) *Nat { // // The length of both operands must be the same as the modulus. Both operands // must already be reduced modulo m. +// +//go:norace func (x *Nat) Add(y *Nat, m *Modulus) *Nat { overflow := x.add(y) x.maybeSubtractModulus(choice(overflow), m) @@ -747,6 +766,8 @@ func (x *Nat) montgomeryReduction(m *Modulus) *Nat { // // All inputs should be the same length and already reduced modulo m. // x will be resized to the size of m and overwritten. +// +//go:norace func (x *Nat) montgomeryMul(a *Nat, b *Nat, m *Modulus) *Nat { n := len(m.nat.limbs) mLimbs := m.nat.limbs[:n] @@ -890,6 +911,8 @@ func addMulVVW(z, x []uint, y uint) (carry uint) { // // The length of both operands must be the same as the modulus. Both operands // must already be reduced modulo m. +// +//go:norace func (x *Nat) Mul(y *Nat, m *Modulus) *Nat { if m.odd { // A Montgomery multiplication by a value out of the Montgomery domain @@ -951,6 +974,8 @@ func (x *Nat) Mul(y *Nat, m *Modulus) *Nat { // to the size of m and overwritten. x must already be reduced modulo m. // // m must be odd, or Exp will panic. +// +//go:norace func (out *Nat) Exp(x *Nat, e []byte, m *Modulus) *Nat { if !m.odd { panic("bigmod: modulus for Exp must be odd") @@ -1030,6 +1055,8 @@ func (out *Nat) ExpShortVarTime(x *Nat, e uint, m *Modulus) *Nat { // // a must be reduced modulo m, but doesn't need to have the same size. The // output will be resized to the size of m and overwritten. +// +//go:norace func (x *Nat) InverseVarTime(a *Nat, m *Modulus) (*Nat, bool) { // This is the extended binary GCD algorithm described in the Handbook of // Applied Cryptography, Algorithm 14.61, adapted by BoringSSL to bound