crypto/rsa: fix salt length calculation with PSSSaltLengthAuto

When PSSSaltLength is set, the maximum salt length must equal:

    (modulus_key_size - 1 + 7)/8 - hash_length - 2
and for example, with a 4096 bit modulus key, and a SHA-1 hash,
it should be:

     (4096 -1 + 7)/8 - 20 - 2 = 490
Previously we'd encounter this error:

     crypto/rsa: key size too small for PSS signature

Fixes #42741

Change-Id: I18bb82c41c511d564b3f4c443f4b3a38ab010ac5
Reviewed-on: https://go-review.googlesource.com/c/go/+/302230
Reviewed-by: Emmanuel Odeke <emmanuel@orijtech.com>
Reviewed-by: Filippo Valsorda <filippo@golang.org>
Trust: Emmanuel Odeke <emmanuel@orijtech.com>
Run-TryBot: Emmanuel Odeke <emmanuel@orijtech.com>
TryBot-Result: Go Bot <gobot@golang.org>

diff --git a/src/crypto/rsa/pss.go b/src/crypto/rsa/pss.go
index b2adbedb28fa859aa9ee0a23ac5313ca53370e02..814522de8181fc326d29d925a7d6df8e53af7996 100644 (file)
--- a/src/crypto/rsa/pss.go
+++ b/src/crypto/rsa/pss.go
@@ -269,7 +269,7 @@ func SignPSS(rand io.Reader, priv *PrivateKey, hash crypto.Hash, digest []byte,
        saltLength := opts.saltLength()
        switch saltLength {
        case PSSSaltLengthAuto:
-               saltLength = priv.Size() - 2 - hash.Size()
+               saltLength = (priv.N.BitLen()-1+7)/8 - 2 - hash.Size()
        case PSSSaltLengthEqualsHash:
                saltLength = hash.Size()
        }

diff --git a/src/crypto/rsa/pss_test.go b/src/crypto/rsa/pss_test.go
index dfa8d8bb5ad02e6feb3420348680c146b03740ad..c3a6d468497cd44ef38023e3d97a4b2e6dbc38d1 100644 (file)
--- a/src/crypto/rsa/pss_test.go
+++ b/src/crypto/rsa/pss_test.go
@@ -12,7 +12,7 @@ import (
        _ "crypto/md5"
        "crypto/rand"
        "crypto/sha1"
-       _ "crypto/sha256"
+       "crypto/sha256"
        "encoding/hex"
        "math/big"
        "os"
@@ -233,6 +233,24 @@ func TestPSSSigning(t *testing.T) {
        }
 }
 
+func TestSignWithPSSSaltLengthAuto(t *testing.T) {
+       key, err := GenerateKey(rand.Reader, 513)
+       if err != nil {
+               t.Fatal(err)
+       }
+       digest := sha256.Sum256([]byte("message"))
+       signature, err := key.Sign(rand.Reader, digest[:], &PSSOptions{
+               SaltLength: PSSSaltLengthAuto,
+               Hash:       crypto.SHA256,
+       })
+       if err != nil {
+               t.Fatal(err)
+       }
+       if len(signature) == 0 {
+               t.Fatal("empty signature returned")
+       }
+}
+
 func bigFromHex(hex string) *big.Int {
        n, ok := new(big.Int).SetString(hex, 16)
        if !ok {