runtime: optimize timers.cleanHead

goos: linux
goarch: amd64
pkg: time
cpu: AMD Ryzen 9 7950X 16-Core Processor
                                   │ s7base.txt  │               s7.txt                │
                                   │   sec/op    │   sec/op     vs base                │
AdjustTimers10000-32                 291.4µ ± 5%   263.3µ ± 4%   -9.64% (p=0.000 n=10)
AdjustTimers10000SingleThread-32     1.728m ± 5%   1.742m ± 3%        ~ (p=0.796 n=10)
AdjustTimers10000NoReset-32          253.1µ ± 6%   192.2µ ± 2%  -24.07% (p=0.000 n=10)
AdjustTimers10000NoSleep-32          277.3µ ± 3%   237.0µ ± 2%  -14.54% (p=0.000 n=10)
AdjustTimers10000NoResetNoSleep-32   241.4µ ± 2%   185.2µ ± 1%  -23.30% (p=0.000 n=10)

goos: darwin
goarch: arm64
pkg: time
cpu: Apple M3 Pro
                                   │ m3base.txt  │               m3.txt                │
                                   │   sec/op    │   sec/op     vs base                │
AdjustTimers10000-12                 288.1µ ± 1%   272.6µ ± 3%   -5.38% (p=0.000 n=10)
AdjustTimers10000SingleThread-12     1.195m ± 1%   1.126m ± 1%   -5.74% (p=0.000 n=10)
AdjustTimers10000NoReset-12          280.8µ ± 1%   255.1µ ± 2%   -9.14% (p=0.000 n=10)
AdjustTimers10000NoSleep-12          292.5µ ± 1%   250.2µ ± 2%  -14.47% (p=0.000 n=10)
AdjustTimers10000NoResetNoSleep-12   279.8µ ± 2%   230.3µ ± 1%  -17.69% (p=0.000 n=10)

Change-Id: I36edb40ee2cd11ab44d20bff045fa77609dca648
Reviewed-on: https://go-review.googlesource.com/c/go/+/574739
Reviewed-by: Ian Lance Taylor <iant@google.com>
Auto-Submit: Russ Cox <rsc@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

diff --git a/src/runtime/time.go b/src/runtime/time.go
index 96b7b76b07a680fdb99018d4bc6de4afe03c5d96..42f629d16843587b00f8d63bad5ef34f3e28b11e 100644 (file)
--- a/src/runtime/time.go
+++ b/src/runtime/time.go
@@ -239,12 +239,8 @@ func (t *timer) updateHeap(ts *timers) (updated bool) {
                assertLockHeld(&ts.mu)
        }
        if t.state&timerZombie != 0 {
-               // Take timer out of heap, applying final t.whenHeap update first.
-               t.state &^= timerHeaped | timerZombie
-               if t.state&timerModified != 0 {
-                       t.state &^= timerModified
-                       t.whenHeap = t.when
-               }
+               // Take timer out of heap.
+               t.state &^= timerHeaped | timerZombie | timerModified
                if ts != nil {
                        ts.zombies.Add(-1)
                        ts.deleteMin()
@@ -651,6 +647,24 @@ func (ts *timers) cleanHead() {
                        return
                }
 
+               // Delete zombies from tail of heap. It requires no heap adjustments at all,
+               // and doing so increases the chances that when we swap out a zombie
+               // in heap[0] for the tail of the heap, we'll get a non-zombie timer,
+               // shortening this loop.
+               n := len(ts.heap)
+               if t := ts.heap[n-1]; t.astate.Load()&timerZombie != 0 {
+                       t.lock()
+                       if t.state&timerZombie != 0 {
+                               t.state &^= timerHeaped | timerZombie | timerModified
+                               t.ts = nil
+                               ts.zombies.Add(-1)
+                               ts.heap[n-1] = nil
+                               ts.heap = ts.heap[:n-1]
+                       }
+                       t.unlock()
+                       continue
+               }
+
                t := ts.heap[0]
                if t.ts != ts {
                        throw("bad ts")