runtime: use internal/cpu variables in assembler code

Using internal/cpu variables has the benefit of avoiding false sharing
(as those are padded) and allows memory and cache usage for these variables
to be shared by multiple packages.

Change-Id: I2bf68d03091bf52b466cf689230d5d25d5950037
Reviewed-on: https://go-review.googlesource.com/126599
Run-TryBot: Martin Möhrmann <moehrmann@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>

diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index a6a81c3f63d0ebd385329a45e9a7c66aff31aaeb..725271eec4debcc24453c26c904bb0a01d51403f 100644 (file)
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -881,7 +881,7 @@ TEXT runtime·stackcheck(SB), NOSPLIT, $0-0
 
 // func cputicks() int64
 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
-       CMPB    runtime·support_sse2(SB), $1
+       CMPB    internal∕cpu·X86+const_offset_x86_HasSSE2(SB), $1
        JNE     done
        CMPB    runtime·lfenceBeforeRdtsc(SB), $1
        JNE     mfence

diff --git a/src/runtime/cpuflags.go b/src/runtime/cpuflags.go
new file mode 100644 (file)
index 0000000..dee6116
--- /dev/null
+++ b/src/runtime/cpuflags.go
@@ -0,0 +1,17 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+import (
+       "internal/cpu"
+       "unsafe"
+)
+
+// Offsets into internal/cpu records for use in assembly.
+const (
+       offset_x86_HasAVX2 = unsafe.Offsetof(cpu.X86.HasAVX2)
+       offset_x86_HasERMS = unsafe.Offsetof(cpu.X86.HasERMS)
+       offset_x86_HasSSE2 = unsafe.Offsetof(cpu.X86.HasSSE2)
+)

diff --git a/src/runtime/cpuflags_amd64.go b/src/runtime/cpuflags_amd64.go
index 10ab5f5b00561403ea0df2b027ad30cc26c18feb..8cca4bca8f0b5d08d8173dab921014d894ce8560 100644 (file)
--- a/src/runtime/cpuflags_amd64.go
+++ b/src/runtime/cpuflags_amd64.go
@@ -6,12 +6,6 @@ package runtime
 
 import (
        "internal/cpu"
-       "unsafe"
-)
-
-// Offsets into internal/cpu records for use in assembly.
-const (
-       offsetX86HasAVX2 = unsafe.Offsetof(cpu.X86.HasAVX2)
 )
 
 var useAVXmemmove bool

diff --git a/src/runtime/memclr_386.s b/src/runtime/memclr_386.s
index a6703b364111447a5483342e86e255a45714edc6..318f8839640e26ee93ad274bc1839c832e32c31b 100644 (file)
--- a/src/runtime/memclr_386.s
+++ b/src/runtime/memclr_386.s
@@ -4,6 +4,7 @@
 
 // +build !plan9
 
+#include "go_asm.h"
 #include "textflag.h"
 
 // NOTE: Windows externalthreadhandler expects memclr to preserve DX.
@@ -28,7 +29,7 @@ tail:
        JBE     _5through8
        CMPL    BX, $16
        JBE     _9through16
-       CMPB    runtime·support_sse2(SB), $1
+       CMPB    internal∕cpu·X86+const_offset_x86_HasSSE2(SB), $1
        JNE     nosse2
        PXOR    X0, X0
        CMPL    BX, $32

diff --git a/src/runtime/memclr_amd64.s b/src/runtime/memclr_amd64.s
index d79078fd00bf5f389837603ad1c40fda72b20c18..b64b1477f9365ba37f03c6c016b5cf798ca3e6fc 100644 (file)
--- a/src/runtime/memclr_amd64.s
+++ b/src/runtime/memclr_amd64.s
@@ -38,7 +38,7 @@ tail:
        JBE     _65through128
        CMPQ    BX, $256
        JBE     _129through256
-       CMPB    internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1
+       CMPB    internal∕cpu·X86+const_offset_x86_HasAVX2(SB), $1
        JE loop_preheader_avx2
        // TODO: for really big clears, use MOVNTDQ, even without AVX2.
 

diff --git a/src/runtime/memmove_386.s b/src/runtime/memmove_386.s
index 172ea40820b30b91741d4e61ac29f28af77fef66..85c622b6b6283a645e08058b5fb4cc43d5336602 100644 (file)
--- a/src/runtime/memmove_386.s
+++ b/src/runtime/memmove_386.s
@@ -25,6 +25,7 @@
 
 // +build !plan9
 
+#include "go_asm.h"
 #include "textflag.h"
 
 // func memmove(to, from unsafe.Pointer, n uintptr)
@@ -51,7 +52,7 @@ tail:
        JBE     move_5through8
        CMPL    BX, $16
        JBE     move_9through16
-       CMPB    runtime·support_sse2(SB), $1
+       CMPB    internal∕cpu·X86+const_offset_x86_HasSSE2(SB), $1
        JNE     nosse2
        CMPL    BX, $32
        JBE     move_17through32
@@ -72,7 +73,7 @@ nosse2:
  */
 forward:
        // If REP MOVSB isn't fast, don't use it
-       CMPB    runtime·support_erms(SB), $1 // enhanced REP MOVSB/STOSB
+       CMPB    internal∕cpu·X86+const_offset_x86_HasERMS(SB), $1 // enhanced REP MOVSB/STOSB
        JNE     fwdBy4
 
        // Check alignment

diff --git a/src/runtime/memmove_amd64.s b/src/runtime/memmove_amd64.s
index cb5cd02e45b9231a90bda4a4a9a96f3f5f28e948..c5385a3d4368a7c2e62eda9f67f53ecb8f1d6552 100644 (file)
--- a/src/runtime/memmove_amd64.s
+++ b/src/runtime/memmove_amd64.s
@@ -25,6 +25,7 @@
 
 // +build !plan9
 
+#include "go_asm.h"
 #include "textflag.h"
 
 // func memmove(to, from unsafe.Pointer, n uintptr)
@@ -83,7 +84,7 @@ forward:
        JLS     move_256through2048
 
        // If REP MOVSB isn't fast, don't use it
-       CMPB    runtime·support_erms(SB), $1 // enhanced REP MOVSB/STOSB
+       CMPB    internal∕cpu·X86+const_offset_x86_HasERMS(SB), $1 // enhanced REP MOVSB/STOSB
        JNE     fwdBy8
 
        // Check alignment

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index c9cc7544b829794a51ddbc659dd098cd70f1d9a7..75d309a9f66d1f1307ace7c2cc91c5350ab88017 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -507,7 +507,8 @@ func cpuinit() {
 
        cpu.Initialize(env)
 
-       support_erms = cpu.X86.HasERMS
+       // Support cpu feature variables are used in code generated by the compiler
+       // to guard execution of instructions that can not be assumed to be always supported.
        support_popcnt = cpu.X86.HasPOPCNT
        support_sse2 = cpu.X86.HasSSE2
        support_sse41 = cpu.X86.HasSSE41

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index bbbe1ee852b94b7884875e74b7d59b0d89328471..93119249426d99f4681d624ff6f1f48a65ce4fb3 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -836,16 +836,15 @@ var (
        newprocs   int32
 
        // Information about what cpu features are available.
-       // Set on startup in runtime.cpuinit.
        // Packages outside the runtime should not use these
        // as they are not an external api.
-       // TODO: deprecate these; use internal/cpu directly.
+       // Set on startup in asm_{386,amd64,amd64p32}.s
        processorVersionInfo uint32
        isIntel              bool
        lfenceBeforeRdtsc    bool
 
        // Set in runtime.cpuinit.
-       support_erms          bool
+       // TODO: deprecate these; use internal/cpu directly.
        support_popcnt        bool
        support_sse2          bool
        support_sse41         bool