internal/cpu: add darwin/arm64 CPU feature detection support

Fixes #42747

Change-Id: I6b1679348c77161f075f0678818bb003fc0e8c86
Reviewed-on: https://go-review.googlesource.com/c/go/+/271989
Trust: Martin Möhrmann <moehrmann@google.com>
Run-TryBot: Martin Möhrmann <martisch@uos.de>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

diff --git a/src/internal/cpu/cpu_android.go b/src/internal/cpu/cpu_android.go
deleted file mode 100644 (file)
index d995e8d..0000000
--- a/src/internal/cpu/cpu_android.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// Copyright 2020 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cpu
-
-const GOOS = "android"

diff --git a/src/internal/cpu/cpu_arm64.go b/src/internal/cpu/cpu_arm64.go
index a8f7b2b458bac9fa333e3da93ebf52f995f4e0e4..f64d9e4dd3100a709aa0c682d61932a3d6259297 100644 (file)
--- a/src/internal/cpu/cpu_arm64.go
+++ b/src/internal/cpu/cpu_arm64.go
@@ -6,21 +6,6 @@ package cpu
 
 const CacheLinePadSize = 64
 
-// HWCap may be initialized by archauxv and
-// should not be changed after it was initialized.
-var HWCap uint
-
-// HWCAP bits. These are exposed by Linux.
-const (
-       hwcap_AES     = 1 << 3
-       hwcap_PMULL   = 1 << 4
-       hwcap_SHA1    = 1 << 5
-       hwcap_SHA2    = 1 << 6
-       hwcap_CRC32   = 1 << 7
-       hwcap_ATOMICS = 1 << 8
-       hwcap_CPUID   = 1 << 11
-)
-
 func doinit() {
        options = []option{
                {Name: "aes", Feature: &ARM64.HasAES},
@@ -34,86 +19,8 @@ func doinit() {
                {Name: "isZeus", Feature: &ARM64.IsZeus},
        }
 
-       switch GOOS {
-       case "linux", "android":
-               // HWCap was populated by the runtime from the auxiliary vector.
-               // Use HWCap information since reading aarch64 system registers
-               // is not supported in user space on older linux kernels.
-               ARM64.HasAES = isSet(HWCap, hwcap_AES)
-               ARM64.HasPMULL = isSet(HWCap, hwcap_PMULL)
-               ARM64.HasSHA1 = isSet(HWCap, hwcap_SHA1)
-               ARM64.HasSHA2 = isSet(HWCap, hwcap_SHA2)
-               ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32)
-               ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID)
-
-               // The Samsung S9+ kernel reports support for atomics, but not all cores
-               // actually support them, resulting in SIGILL. See issue #28431.
-               // TODO(elias.naur): Only disable the optimization on bad chipsets on android.
-               ARM64.HasATOMICS = isSet(HWCap, hwcap_ATOMICS) && GOOS != "android"
-
-               // Check to see if executing on a NeoverseN1 and in order to do that,
-               // check the AUXV for the CPUID bit. The getMIDR function executes an
-               // instruction which would normally be an illegal instruction, but it's
-               // trapped by the kernel, the value sanitized and then returned. Without
-               // the CPUID bit the kernel will not trap the instruction and the process
-               // will be terminated with SIGILL.
-               if ARM64.HasCPUID {
-                       midr := getMIDR()
-                       part_num := uint16((midr >> 4) & 0xfff)
-                       implementor := byte((midr >> 24) & 0xff)
-
-                       if implementor == 'A' && part_num == 0xd0c {
-                               ARM64.IsNeoverseN1 = true
-                       }
-                       if implementor == 'A' && part_num == 0xd40 {
-                               ARM64.IsZeus = true
-                       }
-               }
-
-       case "freebsd":
-               // Retrieve info from system register ID_AA64ISAR0_EL1.
-               isar0 := getisar0()
-
-               // ID_AA64ISAR0_EL1
-               switch extractBits(isar0, 4, 7) {
-               case 1:
-                       ARM64.HasAES = true
-               case 2:
-                       ARM64.HasAES = true
-                       ARM64.HasPMULL = true
-               }
-
-               switch extractBits(isar0, 8, 11) {
-               case 1:
-                       ARM64.HasSHA1 = true
-               }
-
-               switch extractBits(isar0, 12, 15) {
-               case 1, 2:
-                       ARM64.HasSHA2 = true
-               }
-
-               switch extractBits(isar0, 16, 19) {
-               case 1:
-                       ARM64.HasCRC32 = true
-               }
-
-               switch extractBits(isar0, 20, 23) {
-               case 2:
-                       ARM64.HasATOMICS = true
-               }
-       default:
-               // Other operating systems do not support reading HWCap from auxiliary vector
-               // or reading privileged aarch64 system registers in user space.
-       }
-}
-
-func extractBits(data uint64, start, end uint) uint {
-       return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
-}
-
-func isSet(hwc uint, value uint) bool {
-       return hwc&value != 0
+       // arm64 uses different ways to detect CPU features at runtime depending on the operating system.
+       osInit()
 }
 
 func getisar0() uint64

diff --git a/src/internal/cpu/cpu_linux.go b/src/internal/cpu/cpu_arm64_android.go
similarity index 75%
rename from src/internal/cpu/cpu_linux.go
rename to src/internal/cpu/cpu_arm64_android.go
index ec0b84c510f7c80e695be7995efc7303f581a25b..3c9e57c52aa14ce874d45a94d7036f97fd4c430a 100644 (file)
--- a/src/internal/cpu/cpu_linux.go
+++ b/src/internal/cpu/cpu_arm64_android.go
@@ -2,8 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !android
+// +build arm64
 
 package cpu
 
-const GOOS = "linux"
+func osInit() {
+       hwcapInit("android")
+}

diff --git a/src/internal/cpu/cpu_arm64_darwin.go b/src/internal/cpu/cpu_arm64_darwin.go
new file mode 100644 (file)
index 0000000..e094b97
--- /dev/null
+++ b/src/internal/cpu/cpu_arm64_darwin.go
@@ -0,0 +1,34 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build darwin
+// +build !ios
+
+package cpu
+
+func osInit() {
+       ARM64.HasATOMICS = sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00"))
+       ARM64.HasCRC32 = sysctlEnabled([]byte("hw.optional.armv8_crc32\x00"))
+
+       // There are no hw.optional sysctl values for the below features on Mac OS 11.0
+       // to detect their supported state dynamically. Assume the CPU features that
+       // Apple Silicon M1 supports to be available as a minimal set of features
+       // to all Go programs running on darwin/arm64.
+       ARM64.HasAES = true
+       ARM64.HasPMULL = true
+       ARM64.HasSHA1 = true
+       ARM64.HasSHA2 = true
+}
+
+//go:noescape
+func getsysctlbyname(name []byte) (int32, int32)
+
+func sysctlEnabled(name []byte) bool {
+       ret, value := getsysctlbyname(name)
+       if ret < 0 {
+               return false
+       }
+       return value > 0
+}

diff --git a/src/internal/cpu/cpu_arm64_freebsd.go b/src/internal/cpu/cpu_arm64_freebsd.go
new file mode 100644 (file)
index 0000000..9de2005
--- /dev/null
+++ b/src/internal/cpu/cpu_arm64_freebsd.go
@@ -0,0 +1,45 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+
+package cpu
+
+func osInit() {
+       // Retrieve info from system register ID_AA64ISAR0_EL1.
+       isar0 := getisar0()
+
+       // ID_AA64ISAR0_EL1
+       switch extractBits(isar0, 4, 7) {
+       case 1:
+               ARM64.HasAES = true
+       case 2:
+               ARM64.HasAES = true
+               ARM64.HasPMULL = true
+       }
+
+       switch extractBits(isar0, 8, 11) {
+       case 1:
+               ARM64.HasSHA1 = true
+       }
+
+       switch extractBits(isar0, 12, 15) {
+       case 1, 2:
+               ARM64.HasSHA2 = true
+       }
+
+       switch extractBits(isar0, 16, 19) {
+       case 1:
+               ARM64.HasCRC32 = true
+       }
+
+       switch extractBits(isar0, 20, 23) {
+       case 2:
+               ARM64.HasATOMICS = true
+       }
+}
+
+func extractBits(data uint64, start, end uint) uint {
+       return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
+}

diff --git a/src/internal/cpu/cpu_arm64_hwcap.go b/src/internal/cpu/cpu_arm64_hwcap.go
new file mode 100644 (file)
index 0000000..fdaf43e
--- /dev/null
+++ b/src/internal/cpu/cpu_arm64_hwcap.go
@@ -0,0 +1,63 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build linux
+
+package cpu
+
+// HWCap may be initialized by archauxv and
+// should not be changed after it was initialized.
+var HWCap uint
+
+// HWCAP bits. These are exposed by Linux.
+const (
+       hwcap_AES     = 1 << 3
+       hwcap_PMULL   = 1 << 4
+       hwcap_SHA1    = 1 << 5
+       hwcap_SHA2    = 1 << 6
+       hwcap_CRC32   = 1 << 7
+       hwcap_ATOMICS = 1 << 8
+       hwcap_CPUID   = 1 << 11
+)
+
+func hwcapInit(os string) {
+       // HWCap was populated by the runtime from the auxiliary vector.
+       // Use HWCap information since reading aarch64 system registers
+       // is not supported in user space on older linux kernels.
+       ARM64.HasAES = isSet(HWCap, hwcap_AES)
+       ARM64.HasPMULL = isSet(HWCap, hwcap_PMULL)
+       ARM64.HasSHA1 = isSet(HWCap, hwcap_SHA1)
+       ARM64.HasSHA2 = isSet(HWCap, hwcap_SHA2)
+       ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32)
+       ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID)
+
+       // The Samsung S9+ kernel reports support for atomics, but not all cores
+       // actually support them, resulting in SIGILL. See issue #28431.
+       // TODO(elias.naur): Only disable the optimization on bad chipsets on android.
+       ARM64.HasATOMICS = isSet(HWCap, hwcap_ATOMICS) && os != "android"
+
+       // Check to see if executing on a NeoverseN1 and in order to do that,
+       // check the AUXV for the CPUID bit. The getMIDR function executes an
+       // instruction which would normally be an illegal instruction, but it's
+       // trapped by the kernel, the value sanitized and then returned. Without
+       // the CPUID bit the kernel will not trap the instruction and the process
+       // will be terminated with SIGILL.
+       if ARM64.HasCPUID {
+               midr := getMIDR()
+               part_num := uint16((midr >> 4) & 0xfff)
+               implementor := byte((midr >> 24) & 0xff)
+
+               if implementor == 'A' && part_num == 0xd0c {
+                       ARM64.IsNeoverseN1 = true
+               }
+               if implementor == 'A' && part_num == 0xd40 {
+                       ARM64.IsZeus = true
+               }
+       }
+}
+
+func isSet(hwc uint, value uint) bool {
+       return hwc&value != 0
+}

diff --git a/src/internal/cpu/cpu_other.go b/src/internal/cpu/cpu_arm64_linux.go
similarity index 73%
rename from src/internal/cpu/cpu_other.go
rename to src/internal/cpu/cpu_arm64_linux.go
index 8a15fbe79d81340f82b123883bd6a41d1efcf9a0..2f7411ff1e11993f666242055036797d1cece69e 100644 (file)
--- a/src/internal/cpu/cpu_other.go
+++ b/src/internal/cpu/cpu_arm64_linux.go
@@ -2,10 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !linux
-// +build !freebsd
+// +build arm64
+// +build linux
 // +build !android
 
 package cpu
 
-const GOOS = "other"
+func osInit() {
+       hwcapInit("linux")
+}

diff --git a/src/internal/cpu/cpu_arm64_other.go b/src/internal/cpu/cpu_arm64_other.go
new file mode 100644 (file)
index 0000000..f191db2
--- /dev/null
+++ b/src/internal/cpu/cpu_arm64_other.go
@@ -0,0 +1,17 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build !linux
+// +build !freebsd
+// +build !android
+// +build !darwin ios
+
+package cpu
+
+func osInit() {
+       // Other operating systems do not support reading HWCap from auxiliary vector,
+       // reading privileged aarch64 system registers or sysctl in user space to detect
+       // CPU features at runtime.
+}

diff --git a/src/internal/cpu/cpu_freebsd.go b/src/internal/cpu/cpu_freebsd.go
deleted file mode 100644 (file)
index dc37173..0000000
--- a/src/internal/cpu/cpu_freebsd.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// Copyright 2020 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cpu
-
-const GOOS = "freebsd"

diff --git a/src/internal/cpu/cpu_test.go b/src/internal/cpu/cpu_test.go
index 919bbd5ed72e4196168c987cab48a0ed15dc869e..2de7365732d7482242648068aa78bdd77453ab25 100644 (file)
--- a/src/internal/cpu/cpu_test.go
+++ b/src/internal/cpu/cpu_test.go
@@ -18,7 +18,7 @@ func TestMinimalFeatures(t *testing.T) {
        // TODO: maybe do MustSupportFeatureDectection(t) ?
        if runtime.GOARCH == "arm64" {
                switch runtime.GOOS {
-               case "linux", "android":
+               case "linux", "android", "darwin":
                default:
                        t.Skipf("%s/%s is not supported", runtime.GOOS, runtime.GOARCH)
                }
@@ -38,10 +38,7 @@ func MustHaveDebugOptionsSupport(t *testing.T) {
 }
 
 func MustSupportFeatureDectection(t *testing.T) {
-       if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
-               t.Skipf("CPU feature detection is not supported on %s/%s", runtime.GOOS, runtime.GOARCH)
-       }
-       // TODO: maybe there are other platforms?
+       // TODO: add platforms that do not have CPU feature detection support.
 }
 
 func runDebugOptionsTest(t *testing.T, test string, options string) {

diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go
index 52f3cd1fefb9cc77d066417fce33cec234b26a61..e0a43c28aaa693dce07e1ae1689a77d606f6232e 100644 (file)
--- a/src/runtime/os_darwin.go
+++ b/src/runtime/os_darwin.go
@@ -130,6 +130,18 @@ func osinit() {
        physPageSize = getPageSize()
 }
 
+func sysctlbynameInt32(name []byte) (int32, int32) {
+       out := int32(0)
+       nout := unsafe.Sizeof(out)
+       ret := sysctlbyname(&name[0], (*byte)(unsafe.Pointer(&out)), &nout, nil, 0)
+       return ret, out
+}
+
+//go:linkname internal_cpu_getsysctlbyname internal/cpu.getsysctlbyname
+func internal_cpu_getsysctlbyname(name []byte) (int32, int32) {
+       return sysctlbynameInt32(name)
+}
+
 const (
        _CTL_HW      = 6
        _HW_NCPU     = 3

diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go
index c63ba8c6cd27c4b436afb80037d20628020aa22f..c89ce780124cbeca77f57bad28e735e03731a559 100644 (file)
--- a/src/runtime/sys_darwin.go
+++ b/src/runtime/sys_darwin.go
@@ -360,11 +360,18 @@ func setitimer_trampoline()
 
 //go:nosplit
 //go:cgo_unsafe_args
-func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 {
+func sysctl(mib *uint32, miblen uint32, oldp *byte, oldlenp *uintptr, newp *byte, newlen uintptr) int32 {
        return libcCall(unsafe.Pointer(funcPC(sysctl_trampoline)), unsafe.Pointer(&mib))
 }
 func sysctl_trampoline()
 
+//go:nosplit
+//go:cgo_unsafe_args
+func sysctlbyname(name *byte, oldp *byte, oldlenp *uintptr, newp *byte, newlen uintptr) int32 {
+       return libcCall(unsafe.Pointer(funcPC(sysctlbyname_trampoline)), unsafe.Pointer(&name))
+}
+func sysctlbyname_trampoline()
+
 //go:nosplit
 //go:cgo_unsafe_args
 func fcntl(fd, cmd, arg int32) int32 {
@@ -486,6 +493,7 @@ func setNonblock(fd int32) {
 //go:cgo_import_dynamic libc_kill kill "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_setitimer setitimer "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib"
+//go:cgo_import_dynamic libc_sysctlbyname sysctlbyname "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_fcntl fcntl "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_kqueue kqueue "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_kevent kevent "/usr/lib/libSystem.B.dylib"

diff --git a/src/runtime/sys_darwin_amd64.s b/src/runtime/sys_darwin_amd64.s
index 9b5b23901dcd9e16861279378668d380c4d09037..630fb5df64fd2c73fd0c49f0d630a571a41dc2f5 100644 (file)
--- a/src/runtime/sys_darwin_amd64.s
+++ b/src/runtime/sys_darwin_amd64.s
@@ -371,15 +371,27 @@ TEXT runtime·sysctl_trampoline(SB),NOSPLIT,$0
        PUSHQ   BP
        MOVQ    SP, BP
        MOVL    8(DI), SI               // arg 2 miblen
-       MOVQ    16(DI), DX              // arg 3 out
-       MOVQ    24(DI), CX              // arg 4 size
-       MOVQ    32(DI), R8              // arg 5 dst
-       MOVQ    40(DI), R9              // arg 6 ndst
+       MOVQ    16(DI), DX              // arg 3 oldp
+       MOVQ    24(DI), CX              // arg 4 oldlenp
+       MOVQ    32(DI), R8              // arg 5 newp
+       MOVQ    40(DI), R9              // arg 6 newlen
        MOVQ    0(DI), DI               // arg 1 mib
        CALL    libc_sysctl(SB)
        POPQ    BP
        RET
 
+TEXT runtime·sysctlbyname_trampoline(SB),NOSPLIT,$0
+       PUSHQ   BP
+       MOVQ    SP, BP
+       MOVQ    8(DI), SI               // arg 2 oldp
+       MOVQ    16(DI), DX              // arg 3 oldlenp
+       MOVQ    24(DI), CX              // arg 4 newp
+       MOVQ    32(DI), R8              // arg 5 newlen
+       MOVQ    0(DI), DI               // arg 1 name
+       CALL    libc_sysctlbyname(SB)
+       POPQ    BP
+       RET
+
 TEXT runtime·kqueue_trampoline(SB),NOSPLIT,$0
        PUSHQ   BP
        MOVQ    SP, BP

diff --git a/src/runtime/sys_darwin_arm64.s b/src/runtime/sys_darwin_arm64.s
index 9d4d116c507d091c9a2c4b3fd42eb4acee4b61de..96d2ed10767d85270385ea537bd1ee33d34523a6 100644 (file)
--- a/src/runtime/sys_darwin_arm64.s
+++ b/src/runtime/sys_darwin_arm64.s
@@ -301,14 +301,24 @@ TEXT runtime·usleep_trampoline(SB),NOSPLIT,$0
 
 TEXT runtime·sysctl_trampoline(SB),NOSPLIT,$0
        MOVW    8(R0), R1       // arg 2 miblen
-       MOVD    16(R0), R2      // arg 3 out
-       MOVD    24(R0), R3      // arg 4 size
-       MOVD    32(R0), R4      // arg 5 dst
-       MOVD    40(R0), R5      // arg 6 ndst
+       MOVD    16(R0), R2      // arg 3 oldp
+       MOVD    24(R0), R3      // arg 4 oldlenp
+       MOVD    32(R0), R4      // arg 5 newp
+       MOVD    40(R0), R5      // arg 6 newlen
        MOVD    0(R0), R0       // arg 1 mib
        BL      libc_sysctl(SB)
        RET
 
+TEXT runtime·sysctlbyname_trampoline(SB),NOSPLIT,$0
+       MOVD    8(R0), R1       // arg 2 oldp
+       MOVD    16(R0), R2      // arg 3 oldlenp
+       MOVD    24(R0), R3      // arg 4 newp
+       MOVD    32(R0), R4      // arg 5 newlen
+       MOVD    0(R0), R0       // arg 1 name
+       BL      libc_sysctlbyname(SB)
+       RET
+
+
 TEXT runtime·kqueue_trampoline(SB),NOSPLIT,$0
        BL      libc_kqueue(SB)
        RET