[release-branch.go1.17] runtime: on windows, read nanotime with one instruction or...

On 64-bit, this is more efficient, and on ARM64, this prevents the time
from moving backwards due to the weaker memory model. On ARM32 due to
the weaker memory model, we issue a memory barrier.

Updates #48072.
Updates #49369.

Change-Id: If4695716c3039d8af14e14808af217f5c99fc93a
Reviewed-on: https://go-review.googlesource.com/c/go/+/361057
Trust: Jason A. Donenfeld <Jason@zx2c4.com>
Run-TryBot: Jason A. Donenfeld <Jason@zx2c4.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
Reviewed-on: https://go-review.googlesource.com/c/go/+/361475
Reviewed-by: Patrik Nyblom <pnyb@google.com>

diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s
index e7782846b25c7c3b6f97391bd75c90eeb56adcc8..64fa6791f46a576f2b8a334c65fbb97043f37104 100644 (file)
--- a/src/runtime/sys_windows_amd64.s
+++ b/src/runtime/sys_windows_amd64.s
@@ -348,16 +348,9 @@ TEXT runtime·nanotime1(SB),NOSPLIT,$0-8
        CMPB    runtime·useQPCTime(SB), $0
        JNE     useQPC
        MOVQ    $_INTERRUPT_TIME, DI
-loop:
-       MOVL    time_hi1(DI), AX
-       MOVL    time_lo(DI), BX
-       MOVL    time_hi2(DI), CX
-       CMPL    AX, CX
-       JNE     loop
-       SHLQ    $32, CX
-       ORQ     BX, CX
-       IMULQ   $100, CX
-       MOVQ    CX, ret+0(FP)
+       MOVQ    time_lo(DI), AX
+       IMULQ   $100, AX
+       MOVQ    AX, ret+0(FP)
        RET
 useQPC:
        JMP     runtime·nanotimeQPC(SB)

diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s
index 48f8c7dedffd5ec8afa7fc22e7156aafc8de5de8..d7ad244161eb0086f0fe7252e66fb077db2962e1 100644 (file)
--- a/src/runtime/sys_windows_arm.s
+++ b/src/runtime/sys_windows_arm.s
@@ -350,7 +350,9 @@ TEXT runtime·nanotime1(SB),NOSPLIT|NOFRAME,$0-8
        MOVW    $_INTERRUPT_TIME, R3
 loop:
        MOVW    time_hi1(R3), R1
+       DMB     MB_ISH
        MOVW    time_lo(R3), R0
+       DMB     MB_ISH
        MOVW    time_hi2(R3), R2
        CMP     R1, R2
        BNE     loop

diff --git a/src/runtime/sys_windows_arm64.s b/src/runtime/sys_windows_arm64.s
index 7a2e11f5ae29832d6536a1fd11a03a2bdff553b2..183128dd09a90795e7d079eadb20c361aa7e06a1 100644 (file)
--- a/src/runtime/sys_windows_arm64.s
+++ b/src/runtime/sys_windows_arm64.s
@@ -415,15 +415,7 @@ TEXT runtime·nanotime1(SB),NOSPLIT|NOFRAME,$0-8
        CMP     $0, R0
        BNE     useQPC
        MOVD    $_INTERRUPT_TIME, R3
-loop:
-       MOVWU   time_hi1(R3), R1
-       MOVWU   time_lo(R3), R0
-       MOVWU   time_hi2(R3), R2
-       CMP     R1, R2
-       BNE     loop
-
-       // wintime = R1:R0, multiply by 100
-       ORR     R1<<32, R0
+       MOVD    time_lo(R3), R0
        MOVD    $100, R1
        MUL     R1, R0
        MOVD    R0, ret+0(FP)

diff --git a/src/runtime/time_windows.h b/src/runtime/time_windows.h
index cd16fd163bb42963272f07fa512ed95a21fd33b7..7c2e65c328b82a1d8b8468e1271033b2a04eb661 100644 (file)
--- a/src/runtime/time_windows.h
+++ b/src/runtime/time_windows.h
@@ -9,6 +9,7 @@
 // http://web.archive.org/web/20210411000829/https://wrkhpi.wordpress.com/2007/08/09/getting-os-information-the-kuser_shared_data-structure/
 
 // Must read hi1, then lo, then hi2. The snapshot is valid if hi1 == hi2.
+// Or, on 64-bit, just read lo:hi1 all at once atomically.
 #define _INTERRUPT_TIME 0x7ffe0008
 #define _SYSTEM_TIME 0x7ffe0014
 #define time_lo 0

diff --git a/src/runtime/time_windows_amd64.s b/src/runtime/time_windows_amd64.s
index 93ab960b067cd5a810d27effdb31cf84712792bd..045f64eb463fdbd466ee300f50f29b6d9cd293f9 100644 (file)
--- a/src/runtime/time_windows_amd64.s
+++ b/src/runtime/time_windows_amd64.s
@@ -12,27 +12,14 @@
 TEXT time·now(SB),NOSPLIT,$0-24
        CMPB    runtime·useQPCTime(SB), $0
        JNE     useQPC
+
        MOVQ    $_INTERRUPT_TIME, DI
-loop:
-       MOVL    time_hi1(DI), AX
-       MOVL    time_lo(DI), BX
-       MOVL    time_hi2(DI), CX
-       CMPL    AX, CX
-       JNE     loop
-       SHLQ    $32, AX
-       ORQ     BX, AX
+       MOVQ    time_lo(DI), AX
        IMULQ   $100, AX
        MOVQ    AX, mono+16(FP)
 
        MOVQ    $_SYSTEM_TIME, DI
-wall:
-       MOVL    time_hi1(DI), AX
-       MOVL    time_lo(DI), BX
-       MOVL    time_hi2(DI), CX
-       CMPL    AX, CX
-       JNE     wall
-       SHLQ    $32, AX
-       ORQ     BX, AX
+       MOVQ    time_lo(DI), AX
        MOVQ    $116444736000000000, DI
        SUBQ    DI, AX
        IMULQ   $100, AX

diff --git a/src/runtime/time_windows_arm.s b/src/runtime/time_windows_arm.s
index 7c763b66edd2f6b4716f67bc5987a2f5e69ab915..6552d75ff1ce5355bcfb3fab1bacbb2c9fd3cb83 100644 (file)
--- a/src/runtime/time_windows_arm.s
+++ b/src/runtime/time_windows_arm.s
@@ -17,7 +17,9 @@ TEXT time·now(SB),NOSPLIT|NOFRAME,$0-20
        MOVW    $_INTERRUPT_TIME, R3
 loop:
        MOVW    time_hi1(R3), R1
+       DMB     MB_ISH
        MOVW    time_lo(R3), R0
+       DMB     MB_ISH
        MOVW    time_hi2(R3), R2
        CMP     R1, R2
        BNE     loop
@@ -34,7 +36,9 @@ loop:
        MOVW    $_SYSTEM_TIME, R3
 wall:
        MOVW    time_hi1(R3), R1
+       DMB     MB_ISH
        MOVW    time_lo(R3), R0
+       DMB     MB_ISH
        MOVW    time_hi2(R3), R2
        CMP     R1, R2
        BNE     wall

diff --git a/src/runtime/time_windows_arm64.s b/src/runtime/time_windows_arm64.s
index ef52ce4c99441eb02c5c00a933e9d0be44fffc88..e8a0eb2f932f9a1ca335efc71ff5ab4d38da3b23 100644 (file)
--- a/src/runtime/time_windows_arm64.s
+++ b/src/runtime/time_windows_arm64.s
@@ -13,34 +13,18 @@ TEXT time·now(SB),NOSPLIT|NOFRAME,$0-24
        MOVB    runtime·useQPCTime(SB), R0
        CMP     $0, R0
        BNE     useQPC
-       MOVD    $_INTERRUPT_TIME, R3
-loop:
-       MOVWU   time_hi1(R3), R1
-       MOVWU   time_lo(R3), R0
-       MOVWU   time_hi2(R3), R2
-       CMP     R1, R2
-       BNE     loop
 
-       // wintime = R1:R0, multiply by 100
-       ORR     R1<<32, R0
+       MOVD    $_INTERRUPT_TIME, R3
+       MOVD    time_lo(R3), R0
        MOVD    $100, R1
        MUL     R1, R0
        MOVD    R0, mono+16(FP)
 
        MOVD    $_SYSTEM_TIME, R3
-wall:
-       MOVWU   time_hi1(R3), R1
-       MOVWU   time_lo(R3), R0
-       MOVWU   time_hi2(R3), R2
-       CMP     R1, R2
-       BNE     wall
-
-       // w = R1:R0 in 100ns units
+       MOVD    time_lo(R3), R0
        // convert to Unix epoch (but still 100ns units)
        #define delta 116444736000000000
-       ORR     R1<<32, R0
        SUB     $delta, R0
-
        // Convert to nSec
        MOVD    $100, R1
        MUL     R1, R0