From: Michael Munday Date: Fri, 31 Mar 2017 15:08:40 +0000 (-0400) Subject: crypto/aes: use s390x KMA instruction for AES-GCM if available X-Git-Tag: go1.10beta1~390 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=9f3991714af63043b01fa0004ae30b42b3c30e2f;p=gostls13.git crypto/aes: use s390x KMA instruction for AES-GCM if available Adds support for the cipher message with authentication (KMA) instruction added in message-security-assist extension 8. This instruction encapsulates most of the operations required for AES-GCM and is faster than executing the operations independently. name old speed new speed delta AESGCMSeal1K 1.96GB/s ± 0% 6.79GB/s ± 0% +246.47% (p=0.000 n=8+10) AESGCMOpen1K 1.85GB/s ± 0% 5.76GB/s ± 0% +211.18% (p=0.000 n=10+10) AESGCMSign8K 12.0GB/s ± 0% 14.5GB/s ± 0% +20.43% (p=0.000 n=10+8) AESGCMSeal8K 3.75GB/s ± 0% 14.16GB/s ± 0% +277.57% (p=0.000 n=9+10) AESGCMOpen8K 3.70GB/s ± 0% 13.57GB/s ± 0% +266.50% (p=0.000 n=10+9) Change-Id: I57c46573fc5a0bd63c32ce5cba6e37cab85e3de6 Reviewed-on: https://go-review.googlesource.com/73550 Run-TryBot: Michael Munday Reviewed-by: Bill O'Farrell Reviewed-by: Volodymyr Paprotski Reviewed-by: Adam Langley TryBot-Result: Gobot Gobot --- diff --git a/src/crypto/aes/asm_s390x.s b/src/crypto/aes/asm_s390x.s index 2cf3dddea8..cbeb622ace 100644 --- a/src/crypto/aes/asm_s390x.s +++ b/src/crypto/aes/asm_s390x.s @@ -150,3 +150,65 @@ loop: MVC $16, (R1), (R8) MOVD $0, R0 RET + +// func supportsKMA() bool +TEXT ·supportsKMA(SB),NOSPLIT,$24-1 + MOVD $tmp-24(SP), R1 + MOVD $2, R0 // store 24-bytes + XC $24, (R1), (R1) + WORD $0xb2b01000 // STFLE (R1) + MOVWZ 16(R1), R2 + ANDW $(1<<13), R2 // test bit 146 (message-security-assist 8) + BEQ no + + MOVD $0, R0 // KMA-Query + XC $16, (R1), (R1) + WORD $0xb9296024 // kma %r6,%r2,%r4 + MOVWZ (R1), R2 + WORD $0xa7213800 // TMLL R2, $0x3800 + BVS yes +no: + MOVB $0, ret+0(FP) + RET +yes: + MOVB $1, ret+0(FP) + RET + +// func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount) +TEXT ·kmaGCM(SB),NOSPLIT,$112-120 + MOVD fn+0(FP), R0 + MOVD $params-112(SP), R1 + + // load ptr/len pairs + LMG dst+32(FP), R2, R3 // R2=base R3=len + LMG src+56(FP), R4, R5 // R4=base R5=len + LMG aad+80(FP), R6, R7 // R6=base R7=len + + // setup parameters + MOVD cnt+112(FP), R8 + XC $12, (R1), (R1) // reserved + MVC $4, 12(R8), 12(R1) // set chain value + MVC $16, (R8), 64(R1) // set initial counter value + XC $32, 16(R1), 16(R1) // set hash subkey and tag + SLD $3, R7, R12 + MOVD R12, 48(R1) // set total AAD length + SLD $3, R5, R12 + MOVD R12, 56(R1) // set total plaintext/ciphertext length + + LMG key+8(FP), R8, R9 // R8=base R9=len + MVC $16, (R8), 80(R1) // set key + CMPBEQ R9, $16, kma + MVC $8, 16(R8), 96(R1) + CMPBEQ R9, $24, kma + MVC $8, 24(R8), 104(R1) + +kma: + WORD $0xb9296024 // kma %r6,%r2,%r4 + BVS kma + + MOVD tag+104(FP), R2 + MVC $16, 16(R1), 0(R2) // copy tag to output + MOVD cnt+112(FP), R8 + MVC $4, 12(R1), 12(R8) // update counter value + + RET diff --git a/src/crypto/aes/gcm_s390x.go b/src/crypto/aes/gcm_s390x.go index 438310d3de..055a9a927d 100644 --- a/src/crypto/aes/gcm_s390x.go +++ b/src/crypto/aes/gcm_s390x.go @@ -10,6 +10,11 @@ import ( "errors" ) +// This file contains two implementations of AES-GCM. The first implementation +// (gcmAsm) uses the KMCTR instruction to encrypt using AES in counter mode and +// the KIMD instruction for GHASH. The second implementation (gcmKMA) uses the +// newer KMA instruction which performs both operations. + // gcmCount represents a 16-byte big-endian count value. type gcmCount [16]byte @@ -71,12 +76,16 @@ var _ gcmAble = (*aesCipherAsm)(nil) func (c *aesCipherAsm) NewGCM(nonceSize int) (cipher.AEAD, error) { var hk gcmHashKey c.Encrypt(hk[:], hk[:]) - g := &gcmAsm{ + g := gcmAsm{ block: c, hashKey: hk, nonceSize: nonceSize, } - return g, nil + if hasKMA { + g := gcmKMA{g} + return &g, nil + } + return &g, nil } func (g *gcmAsm) NonceSize() int { @@ -268,3 +277,90 @@ func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { g.counterCrypt(out, ciphertext, &counter) return ret, nil } + +// supportsKMA reports whether the message-security-assist 8 facility is available. +// This function call may be expensive so hasKMA should be queried instead. +func supportsKMA() bool + +// hasKMA contains the result of supportsKMA. +var hasKMA = supportsKMA() + +// gcmKMA implements the cipher.AEAD interface using the KMA instruction. It should +// only be used if hasKMA is true. +type gcmKMA struct { + gcmAsm +} + +// flags for the KMA instruction +const ( + kmaHS = 1 << 10 // hash subkey supplied + kmaLAAD = 1 << 9 // last series of additional authenticated data + kmaLPC = 1 << 8 // last series of plaintext or ciphertext blocks + kmaDecrypt = 1 << 7 // decrypt +) + +// kmaGCM executes the encryption or decryption operation given by fn. The tag +// will be calculated and written to tag. cnt should contain the current +// counter state and will be overwritten with the updated counter state. +// TODO(mundaym): could pass in hash subkey +//go:noescape +func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount) + +// Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for +// details. +func (g *gcmKMA) Seal(dst, nonce, plaintext, data []byte) []byte { + if len(nonce) != g.nonceSize { + panic("cipher: incorrect nonce length given to GCM") + } + if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize { + panic("cipher: message too large for GCM") + } + + ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize) + + counter := g.deriveCounter(nonce) + fc := g.block.function | kmaLAAD | kmaLPC + + var tag [gcmTagSize]byte + kmaGCM(fc, g.block.key, out[:len(plaintext)], plaintext, data, &tag, &counter) + copy(out[len(plaintext):], tag[:]) + + return ret +} + +// Open authenticates and decrypts ciphertext. See the cipher.AEAD interface +// for details. +func (g *gcmKMA) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { + if len(nonce) != g.nonceSize { + panic("cipher: incorrect nonce length given to GCM") + } + if len(ciphertext) < gcmTagSize { + return nil, errOpen + } + if uint64(len(ciphertext)) > ((1<<32)-2)*BlockSize+gcmTagSize { + return nil, errOpen + } + + tag := ciphertext[len(ciphertext)-gcmTagSize:] + ciphertext = ciphertext[:len(ciphertext)-gcmTagSize] + ret, out := sliceForAppend(dst, len(ciphertext)) + + counter := g.deriveCounter(nonce) + fc := g.block.function | kmaLAAD | kmaLPC | kmaDecrypt + + var expectedTag [gcmTagSize]byte + kmaGCM(fc, g.block.key, out[:len(ciphertext)], ciphertext, data, &expectedTag, &counter) + + if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 { + // The AESNI code decrypts and authenticates concurrently, and + // so overwrites dst in the event of a tag mismatch. That + // behavior is mimicked here in order to be consistent across + // platforms. + for i := range out { + out[i] = 0 + } + return nil, errOpen + } + + return ret, nil +}