)
// The following functions are defined in gcm_amd64.s.
-func hasGCMAsm() bool
//go:noescape
func aesEncBlock(dst, src *[16]byte, ks []uint32)
MOVD $0, R0
RET
-// func supportsKMA() bool
-TEXT ·supportsKMA(SB),NOSPLIT,$24-1
- MOVD $tmp-24(SP), R1
- MOVD $2, R0 // store 24-bytes
- XC $24, (R1), (R1)
- WORD $0xb2b01000 // STFLE (R1)
- MOVWZ 16(R1), R2
- ANDW $(1<<13), R2 // test bit 146 (message-security-assist 8)
- BEQ no
-
- MOVD $0, R0 // KMA-Query
- XC $16, (R1), (R1)
- WORD $0xb9296024 // kma %r6,%r2,%r4
- MOVWZ (R1), R2
- WORD $0xa7213800 // TMLL R2, $0x3800
- BVS yes
-no:
- MOVB $0, ret+0(FP)
- RET
-yes:
- MOVB $1, ret+0(FP)
- RET
-
// func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount)
TEXT ·kmaGCM(SB),NOSPLIT,$112-120
MOVD fn+0(FP), R0
import (
"crypto/cipher"
- "crypto/internal/cipherhw"
+ "internal/cpu"
)
// defined in asm_amd64.s
+
func encryptBlockAsm(nr int, xk *uint32, dst, src *byte)
func decryptBlockAsm(nr int, xk *uint32, dst, src *byte)
func expandKeyAsm(nr int, key *byte, enc *uint32, dec *uint32)
aesCipher
}
-var useAsm = cipherhw.AESGCMSupport()
-
func newCipher(key []byte) (cipher.Block, error) {
- if !useAsm {
+ if !cpu.X86.HasAES {
return newCipherGeneric(key)
}
n := len(key) + 28
case 256 / 8:
rounds = 14
}
+
expandKeyAsm(rounds, &key[0], &c.enc[0], &c.dec[0])
- if hasGCMAsm() {
+ if cpu.X86.HasAES && cpu.X86.HasPCLMULQDQ {
return &aesCipherGCM{c}, nil
}
// expandKey is used by BenchmarkExpand to ensure that the asm implementation
// of key expansion is used for the benchmark when it is available.
func expandKey(key []byte, enc, dec []uint32) {
- if useAsm {
+ if cpu.X86.HasAES {
rounds := 10 // rounds needed for AES128
switch len(key) {
case 192 / 8:
// defined in asm_ppc64le.s
//go:noescape
-
func setEncryptKeyAsm(key *byte, keylen int, enc *uint32) int
//go:noescape
-
func setDecryptKeyAsm(key *byte, keylen int, dec *uint32) int
//go:noescape
-
func doEncryptKeyAsm(key *byte, keylen int, dec *uint32) int
//go:noescape
-
func encryptBlockAsm(dst, src *byte, enc *uint32)
//go:noescape
-
func decryptBlockAsm(dst, src *byte, dec *uint32)
type aesCipherAsm struct {
import (
"crypto/cipher"
- "crypto/internal/cipherhw"
+ "internal/cpu"
)
type code int
)
type aesCipherAsm struct {
- function code // code for cipher message instruction
- key []byte // key (128, 192 or 256 bytes)
- storage [256]byte // array backing key slice
+ function code // code for cipher message instruction
+ key []byte // key (128, 192 or 256 bits)
+ storage [32]byte // array backing key slice
}
// cryptBlocks invokes the cipher message (KM) instruction with
//go:noescape
func cryptBlocks(c code, key, dst, src *byte, length int)
-var useAsm = cipherhw.AESGCMSupport()
-
func newCipher(key []byte) (cipher.Block, error) {
- if !useAsm {
+ // Strictly speaking, this check should be for HasKM.
+ // The check for HasKMC and HasKMCTR provides compatibility
+ // with the existing optimized s390x CBC and CTR implementations
+ // in this package, which already assert that they meet the
+ // cbcEncAble, cbcDecAble, and ctrAble interfaces
+ if !(cpu.S390X.HasKM && cpu.S390X.HasKMC && cpu.S390X.HasKMCTR) {
return newCipherGeneric(key)
}
GLOBL gcmPoly<>(SB), (NOPTR+RODATA), $16
GLOBL andMask<>(SB), (NOPTR+RODATA), $240
-// func hasGCMAsm() bool
-// returns whether AES-NI AND CLMUL-NI are supported
-TEXT ·hasGCMAsm(SB),NOSPLIT,$0
- XORQ AX, AX
- INCL AX
- CPUID
- MOVQ CX, DX
- SHRQ $25, CX
- SHRQ $1, DX
- ANDQ DX, CX
- ANDQ $1, CX
- MOVB CX, ret+0(FP)
- RET
-
// func aesEncBlock(dst, src *[16]byte, ks []uint32)
TEXT ·aesEncBlock(SB),NOSPLIT,$0
MOVQ dst+0(FP), DI
"crypto/cipher"
"crypto/subtle"
"errors"
+ "internal/cpu"
)
// This file contains two implementations of AES-GCM. The first implementation
nonceSize: nonceSize,
tagSize: tagSize,
}
- if hasKMA {
+ if cpu.S390X.HasKMA {
g := gcmKMA{g}
return &g, nil
}
return ret, nil
}
-// supportsKMA reports whether the message-security-assist 8 facility is available.
-// This function call may be expensive so hasKMA should be queried instead.
-func supportsKMA() bool
-
-// hasKMA contains the result of supportsKMA.
-var hasKMA = supportsKMA()
-
// gcmKMA implements the cipher.AEAD interface using the KMA instruction. It should
// only be used if hasKMA is true.
type gcmKMA struct {
+++ /dev/null
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build amd64,!gccgo,!appengine
-
-#include "textflag.h"
-
-// func hasAESNI() bool
-TEXT ·hasAESNI(SB),NOSPLIT,$0
- XORQ AX, AX
- INCL AX
- CPUID
- SHRQ $25, CX
- ANDQ $1, CX
- MOVB CX, ret+0(FP)
- RET
+++ /dev/null
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build s390x,!gccgo,!appengine
-
-#include "textflag.h"
-
-// func hasHWSupport() bool
-TEXT ·hasHWSupport(SB),NOSPLIT,$16-1
- XOR R0, R0 // set function code to 0 (query)
- LA mask-16(SP), R1 // 16-byte stack variable for mask
- MOVD $(0x38<<40), R3 // mask for bits 18-20 (big endian)
-
- // check for KM AES functions
- WORD $0xB92E0024 // cipher message (KM)
- MOVD mask-16(SP), R2
- AND R3, R2
- CMPBNE R2, R3, notfound
-
- // check for KMC AES functions
- WORD $0xB92F0024 // cipher message with chaining (KMC)
- MOVD mask-16(SP), R2
- AND R3, R2
- CMPBNE R2, R3, notfound
-
- // check for KMCTR AES functions
- WORD $0xB92D4024 // cipher message with counter (KMCTR)
- MOVD mask-16(SP), R2
- AND R3, R2
- CMPBNE R2, R3, notfound
-
- // check for KIMD GHASH function
- WORD $0xB93E0024 // compute intermediate message digest (KIMD)
- MOVD mask-8(SP), R2 // bits 64-127
- MOVD $(1<<62), R5
- AND R5, R2
- CMPBNE R2, R5, notfound
-
- MOVB $1, ret+0(FP)
- RET
-notfound:
- MOVB $0, ret+0(FP)
- RET
+++ /dev/null
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build amd64,!gccgo,!appengine
-
-package cipherhw
-
-// defined in asm_amd64.s
-func hasAESNI() bool
-
-// AESGCMSupport returns true if the Go standard library supports AES-GCM in
-// hardware.
-func AESGCMSupport() bool {
- return hasAESNI()
-}
+++ /dev/null
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build s390x,!gccgo,!appengine
-
-package cipherhw
-
-// hasHWSupport reports whether the AES-128, AES-192 and AES-256 cipher message
-// (KM) function codes are supported. Note that this function is expensive.
-// defined in asm_s390x.s
-func hasHWSupport() bool
-
-var hwSupport = hasHWSupport()
-
-func AESGCMSupport() bool {
- return hwSupport
-}
+++ /dev/null
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package cipherhw exposes common functions for detecting whether hardware
-// support for certain ciphers and authenticators is present.
-package cipherhw
+++ /dev/null
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !amd64,!s390x gccgo appengine
-
-package cipherhw
-
-func AESGCMSupport() bool {
- return false
-}
import (
"container/list"
"crypto"
- "crypto/internal/cipherhw"
"crypto/rand"
"crypto/sha512"
"crypto/x509"
"errors"
"fmt"
+ "internal/cpu"
"io"
"math/big"
"net"
func initDefaultCipherSuites() {
var topCipherSuites []uint16
- if cipherhw.AESGCMSupport() {
+
+ // Check the cpu flags for each platform that has optimized GCM implementations.
+ // Worst case, these variables will just all be false
+ hasGCMAsmAMD64 := cpu.X86.HasAES && cpu.X86.HasPCLMULQDQ
+
+ // TODO: enable the arm64 HasAES && HasPMULL feature check after the
+ // optimized AES-GCM implementation for arm64 is merged (CL 107298).
+ // This is explicitly set to false for now to prevent misprioritization
+ // of AES-GCM based cipher suites, which will be slower than chacha20-poly1305
+ hasGCMAsmARM64 := false
+ // hasGCMAsmARM64 := cpu.ARM64.HasAES && cpu.ARM64.HasPMULL
+
+ hasGCMAsmS390X := cpu.S390X.HasKM && (cpu.S390X.HasKMA || (cpu.S390X.HasKMCTR && cpu.S390X.HasKIMD))
+
+ hasGCMAsm := hasGCMAsmAMD64 || hasGCMAsmARM64 || hasGCMAsmS390X
+
+ if hasGCMAsm {
// If AES-GCM hardware is provided then prioritise AES-GCM
// cipher suites.
topCipherSuites = []uint16{
"L2",
"crypto",
"crypto/cipher",
- "crypto/internal/cipherhw",
"crypto/subtle",
"encoding/base32",
"encoding/base64",
type s390x struct {
_ [CacheLineSize]byte
HasVX bool // vector facility. Note: the runtime sets this when it processes auxv records.
- _ [CacheLineSize]byte
+ HasKM bool // cipher message (KM)
+ HasKMA bool // cipher message assist (KMA)
+ HasKMC bool // cipher message with chaining (KMC)
+ HasKMCTR bool // cipher message with counter (KMCTR)
+ HasKIMD bool // compute intermediate message digest (KIMD)
+ _ [CacheLineSize]byte
}
// initialize examines the processor and sets the relevant variables above.
package cpu
const CacheLineSize = 256
+
+// the following cpu feature detection functions are defined in cpu_s390x.s
+func hasKM() bool
+func hasKMC() bool
+func hasKMCTR() bool
+func hasKMA() bool
+func hasKIMD() bool
+
+func init() {
+ S390X.HasKM = hasKM()
+ S390X.HasKMC = hasKMC()
+ S390X.HasKMCTR = hasKMCTR()
+ S390X.HasKMA = hasKMA()
+ S390X.HasKIMD = hasKIMD()
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.\r
+// Use of this source code is governed by a BSD-style\r
+// license that can be found in the LICENSE file.\r
+\r
+#include "textflag.h"\r
+\r
+// func hasKM() bool\r
+TEXT ·hasKM(SB),NOSPLIT,$16-1\r
+ XOR R0, R0 // set function code to 0 (query)\r
+ LA mask-16(SP), R1 // 16-byte stack variable for mask\r
+ MOVD $(0x38<<40), R3 // mask for bits 18-20 (big endian)\r
+\r
+ // check for KM AES functions\r
+ WORD $0xB92E0024 // cipher message (KM)\r
+ MOVD mask-16(SP), R2\r
+ AND R3, R2\r
+ CMPBNE R2, R3, notfound\r
+\r
+ MOVB $1, ret+0(FP)\r
+ RET\r
+notfound:\r
+ MOVB $0, ret+0(FP)\r
+ RET\r
+\r
+// func hasKMC() bool\r
+TEXT ·hasKMC(SB),NOSPLIT,$16-1\r
+ XOR R0, R0 // set function code to 0 (query)\r
+ LA mask-16(SP), R1 // 16-byte stack variable for mask\r
+ MOVD $(0x38<<40), R3 // mask for bits 18-20 (big endian)\r
+\r
+ // check for KMC AES functions\r
+ WORD $0xB92F0024 // cipher message with chaining (KMC)\r
+ MOVD mask-16(SP), R2\r
+ AND R3, R2\r
+ CMPBNE R2, R3, notfound\r
+\r
+ MOVB $1, ret+0(FP)\r
+ RET\r
+notfound:\r
+ MOVB $0, ret+0(FP)\r
+ RET\r
+\r
+// func hasKMCTR() bool\r
+TEXT ·hasKMCTR(SB),NOSPLIT,$16-1\r
+ XOR R0, R0 // set function code to 0 (query)\r
+ LA mask-16(SP), R1 // 16-byte stack variable for mask\r
+ MOVD $(0x38<<40), R3 // mask for bits 18-20 (big endian)\r
+\r
+ // check for KMCTR AES functions\r
+ WORD $0xB92D4024 // cipher message with counter (KMCTR)\r
+ MOVD mask-16(SP), R2\r
+ AND R3, R2\r
+ CMPBNE R2, R3, notfound\r
+\r
+ MOVB $1, ret+0(FP)\r
+ RET\r
+notfound:\r
+ MOVB $0, ret+0(FP)\r
+ RET\r
+\r
+// func hasKMA() bool\r
+TEXT ·hasKMA(SB),NOSPLIT,$24-1\r
+ MOVD $tmp-24(SP), R1\r
+ MOVD $2, R0 // store 24-bytes\r
+ XC $24, (R1), (R1)\r
+ WORD $0xb2b01000 // STFLE (R1)\r
+ MOVWZ 16(R1), R2\r
+ ANDW $(1<<13), R2 // test bit 146 (message-security-assist 8)\r
+ BEQ no\r
+\r
+ MOVD $0, R0 // KMA-Query\r
+ XC $16, (R1), (R1)\r
+ WORD $0xb9296024 // kma %r6,%r2,%r4\r
+ MOVWZ (R1), R2\r
+ WORD $0xa7213800 // TMLL R2, $0x3800\r
+ BVS yes\r
+no:\r
+ MOVB $0, ret+0(FP)\r
+ RET\r
+yes:\r
+ MOVB $1, ret+0(FP)\r
+ RET\r
+\r
+// func hasKIMD() bool\r
+TEXT ·hasKIMD(SB),NOSPLIT,$16-1\r
+ XOR R0, R0 // set function code to 0 (query)\r
+ LA mask-16(SP), R1 // 16-byte stack variable for mask\r
+ MOVD $(0x38<<40), R3 // mask for bits 18-20 (big endian)\r
+\r
+ // check for KIMD GHASH function\r
+ WORD $0xB93E0024 // compute intermediate message digest (KIMD)\r
+ MOVD mask-8(SP), R2 // bits 64-127\r
+ MOVD $(1<<62), R5\r
+ AND R5, R2\r
+ CMPBNE R2, R5, notfound\r
+\r
+ MOVB $1, ret+0(FP)\r
+ RET\r
+notfound:\r
+ MOVB $0, ret+0(FP)\r
+ RET\r