runtime: fix cputicks on x86

See the following issue for context:
https://github.com/golang/go/issues/9729#issuecomment-74648287
In short, RDTSC can produce skewed results without preceding LFENCE/MFENCE.
Information on this matter is very scrappy in the internet.
But this is what linux kernel does (see rdtsc_barrier).
It also fixes the test program on my machine.

Update #9729

Change-Id: I3c1ffbf129fdfdd388bd5b7911b392b319248e68
Reviewed-on: https://go-review.googlesource.com/5033
Reviewed-by: Ian Lance Taylor <iant@golang.org>

diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index 58a0d502bde0e9490543a52291329a2110e756d7..1574b3060d74cd9644eed4a11a0ba52af21ab815 100644 (file)
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -30,6 +30,19 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
        CPUID
        CMPL    AX, $0
        JE      nocpuinfo
+
+       // Figure out how to serialize RDTSC.
+       // On Intel processors LFENCE is enough. AMD requires MFENCE.
+       // Don't know about the rest, so let's do MFENCE.
+       CMPL    BX, $0x756E6547  // "Genu"
+       JNE     notintel
+       CMPL    DX, $0x49656E69  // "ineI"
+       JNE     notintel
+       CMPL    CX, $0x6C65746E  // "ntel"
+       JNE     notintel
+       MOVB    $1, runtime·lfenceBeforeRdtsc(SB)
+notintel:
+
        MOVL    $1, AX
        CPUID
        MOVL    CX, runtime·cpuid_ecx(SB)
@@ -868,9 +881,17 @@ TEXT runtime·gogetcallersp(SB),NOSPLIT,$0-8
        MOVL    AX, ret+4(FP)
        RET
 
-// int64 runtime·cputicks(void), so really
-// void runtime·cputicks(int64 *ticks)
+// func cputicks() int64
 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
+       TESTL   $0x4000000, runtime·cpuid_edx(SB) // no sse2, no mfence
+       JEQ     done
+       CMPB    runtime·lfenceBeforeRdtsc(SB), $1
+       JNE     mfence
+       BYTE    $0x0f; BYTE $0xae; BYTE $0xe8 // LFENCE
+       JMP     done
+mfence:
+       BYTE    $0x0f; BYTE $0xae; BYTE $0xf0 // MFENCE
+done:
        RDTSC
        MOVL    AX, ret_lo+0(FP)
        MOVL    DX, ret_hi+4(FP)

diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index f6c1c5f6e6afc7a33ea67ef78120635b02f16ec0..270fdc1823431e8d4d4c21c970320a1fae3aa287 100644 (file)
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -30,6 +30,19 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
        CPUID
        CMPQ    AX, $0
        JE      nocpuinfo
+
+       // Figure out how to serialize RDTSC.
+       // On Intel processors LFENCE is enough. AMD requires MFENCE.
+       // Don't know about the rest, so let's do MFENCE.
+       CMPL    BX, $0x756E6547  // "Genu"
+       JNE     notintel
+       CMPL    DX, $0x49656E69  // "ineI"
+       JNE     notintel
+       CMPL    CX, $0x6C65746E  // "ntel"
+       JNE     notintel
+       MOVB    $1, runtime·lfenceBeforeRdtsc(SB)
+notintel:
+
        MOVQ    $1, AX
        CPUID
        MOVL    CX, runtime·cpuid_ecx(SB)
@@ -865,8 +878,15 @@ TEXT runtime·gogetcallersp(SB),NOSPLIT,$0-16
        MOVQ    AX, ret+8(FP)
        RET
 
-// int64 runtime·cputicks(void)
+// func cputicks() int64
 TEXT runtime·cputicks(SB),NOSPLIT,$0-0
+       CMPB    runtime·lfenceBeforeRdtsc(SB), $1
+       JNE     mfence
+       BYTE    $0x0f; BYTE $0xae; BYTE $0xe8 // LFENCE
+       JMP     done
+mfence:
+       BYTE    $0x0f; BYTE $0xae; BYTE $0xf0 // MFENCE
+done:
        RDTSC
        SHLQ    $32, DX
        ADDQ    DX, AX

diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go
index c056bfcdbfc9b7d28228fe764505208f01f63236..f0d26c8edcea8cb08403d4a583b000274570b421 100644 (file)
--- a/src/runtime/runtime1.go
+++ b/src/runtime/runtime1.go
@@ -58,13 +58,6 @@ var (
        iswindows int32
 )
 
-// Information about what cpu features are available.
-// Set on startup in asm_{x86/amd64}.s.
-var (
-//cpuid_ecx uint32
-//cpuid_edx uint32
-)
-
 func goargs() {
        if GOOS == "windows" {
                return

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index ca3e7d564e7efa2ea0ab295398a324d32ad99ed3..ea2d55dbb609ddfb74ef9d27ee4c7a5653a70223 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -562,12 +562,16 @@ var (
        goos        *int8
        ncpu        int32
        iscgo       bool
-       cpuid_ecx   uint32
-       cpuid_edx   uint32
        signote     note
        forcegc     forcegcstate
        sched       schedt
        newprocs    int32
+
+       // Information about what cpu features are available.
+       // Set on startup in asm_{x86,amd64}.s.
+       cpuid_ecx         uint32
+       cpuid_edx         uint32
+       lfenceBeforeRdtsc bool
 )
 
 /*