// If non-nil, the timers containing t.
ts *timers
- // whenHeap is a (perhaps outdated) copy of t.when for use
- // ordering t within t.ts.heap.
- // When t is in a heap but t.whenHeap is outdated,
- // the timerModified state bit is set.
- // The actual update t.whenHeap = t.when must be
- // delayed until the heap can be reordered at the same time
- // (meaning t's lock must be held for whenHeap,
- // and t.ts's lock must be held for the heap reordering).
- // Since writes to whenHeap are protected by two locks (t.mu and t.ts.mu),
- // it is permitted to read whenHeap when holding either one.
- whenHeap int64
-
// sendLock protects sends on the timer's channel.
// Not used for async (pre-Go 1.23) behavior when debug.asynctimerchan.Load() != 0.
sendLock mutex
// access the timers of another P, so we have to lock.
mu mutex
- // heap is the set of timers, ordered by t.whenHeap.
+ // heap is the set of timers, ordered by heap[i].when.
// Must hold lock to access.
- heap []*timer
+ heap []timerWhen
// len is an atomic copy of len(heap).
len atomic.Uint32
// raceCtx is the race context used while executing timer functions.
raceCtx uintptr
- // minWhenHeap is the minimum heap[i].whenHeap value (= heap[0].whenHeap).
+ // minWhenHeap is the minimum heap[i].when value (= heap[0].when).
// The wakeTime method uses minWhenHeap and minWhenModified
// to determine the next wake time.
// If minWhenHeap = 0, it means there are no timers in the heap.
minWhenModified atomic.Int64
}
+type timerWhen struct {
+ timer *timer
+ when int64
+}
+
func (ts *timers) lock() {
lock(&ts.mu)
}
// timerHeaped is set when the timer is stored in some P's heap.
timerHeaped uint8 = 1 << iota
- // timerModified is set when t.when has been modified but
- // t.whenHeap still needs to be updated as well.
- // The change to t.whenHeap waits until the heap in which
+ // timerModified is set when t.when has been modified
+ // but the heap's heap[i].when entry still needs to be updated.
+ // That change waits until the heap in which
// the timer appears can be locked and rearranged.
// timerModified is only set when timerHeaped is also set.
timerModified
// unlock updates t.astate and unlocks the timer.
func (t *timer) unlock() {
t.trace("unlock")
- // Let heap fast paths know whether t.whenHeap is accurate.
+ // Let heap fast paths know whether heap[i].when is accurate.
// Also let maybeRunChan know whether channel is in heap.
t.astate.Store(t.state)
unlock(&t.mu)
return (*hchan)(efaceOf(&t.arg).data)
}
-// updateHeap updates t.whenHeap as directed by t.state, updating t.state
-// and returning a bool indicating whether the state (and t.whenHeap) changed.
+// updateHeap updates t as directed by t.state, updating t.state
+// and returning a bool indicating whether the state (and ts.heap[0].when) changed.
// The caller must hold t's lock, or the world can be stopped instead.
-// If ts != nil, then ts must be locked, t must be ts.heap[0], and updateHeap
+// The timer set t.ts must be non-nil and locked, t must be t.ts.heap[0], and updateHeap
// takes care of moving t within the timers heap to preserve the heap invariants.
// If ts == nil, then t must not be in a heap (or is in a heap that is
// temporarily not maintaining its invariant, such as during timers.adjust).
-func (t *timer) updateHeap(ts *timers) (updated bool) {
+func (t *timer) updateHeap() (updated bool) {
assertWorldStoppedOrLockHeld(&t.mu)
t.trace("updateHeap")
- if ts != nil {
- if t.ts != ts || t != ts.heap[0] {
- badTimer()
- }
- assertLockHeld(&ts.mu)
+ ts := t.ts
+ if ts == nil || t != ts.heap[0].timer {
+ badTimer()
}
+ assertLockHeld(&ts.mu)
if t.state&timerZombie != 0 {
// Take timer out of heap.
t.state &^= timerHeaped | timerZombie | timerModified
- if ts != nil {
- ts.zombies.Add(-1)
- ts.deleteMin()
- }
+ ts.zombies.Add(-1)
+ ts.deleteMin()
return true
}
if t.state&timerModified != 0 {
- // Apply t.whenHeap update and move within heap.
+ // Update ts.heap[0].when and move within heap.
t.state &^= timerModified
- t.whenHeap = t.when
- // Move t to the right position.
- if ts != nil {
- ts.siftDown(0)
- ts.updateMinWhenHeap()
- }
+ ts.heap[0].when = t.when
+ ts.siftDown(0)
+ ts.updateMinWhenHeap()
return true
}
throw("ts set in timer")
}
t.ts = ts
- t.whenHeap = t.when
- ts.heap = append(ts.heap, t)
+ ts.heap = append(ts.heap, timerWhen{t, t.when})
ts.siftUp(len(ts.heap) - 1)
- if t == ts.heap[0] {
+ if t == ts.heap[0].timer {
ts.updateMinWhenHeap()
}
}
// ts must be locked.
func (ts *timers) deleteMin() {
assertLockHeld(&ts.mu)
- t := ts.heap[0]
+ t := ts.heap[0].timer
if t.ts != ts {
throw("wrong timers")
}
if last > 0 {
ts.heap[0] = ts.heap[last]
}
- ts.heap[last] = nil
+ ts.heap[last] = timerWhen{}
ts.heap = ts.heap[:last]
if last > 0 {
ts.siftDown(0)
t.ts.zombies.Add(-1)
t.state &^= timerZombie
}
- // Cannot modify t.whenHeap until t.ts is locked.
+ // The corresponding heap[i].when is updated later.
// See comment in type timer above and in timers.adjust below.
- if when < t.whenHeap {
+ if min := t.ts.minWhenModified.Load(); min == 0 || when < min {
wake = true
// Force timerModified bit out to t.astate before updating t.minWhenModified,
// to synchronize with t.ts.adjust. See comment in adjust.
// in heap[0] for the tail of the heap, we'll get a non-zombie timer,
// shortening this loop.
n := len(ts.heap)
- if t := ts.heap[n-1]; t.astate.Load()&timerZombie != 0 {
+ if t := ts.heap[n-1].timer; t.astate.Load()&timerZombie != 0 {
t.lock()
if t.state&timerZombie != 0 {
t.state &^= timerHeaped | timerZombie | timerModified
t.ts = nil
ts.zombies.Add(-1)
- ts.heap[n-1] = nil
+ ts.heap[n-1] = timerWhen{}
ts.heap = ts.heap[:n-1]
}
t.unlock()
continue
}
- t := ts.heap[0]
+ t := ts.heap[0].timer
if t.ts != ts {
throw("bad ts")
}
}
t.lock()
- updated := t.updateHeap(ts)
+ updated := t.updateHeap()
t.unlock()
if !updated {
// Head of timers does not need adjustment.
// The world is stopped, so we ignore the locking of ts and src here.
// That would introduce a sched < timers lock ordering,
// which we'd rather avoid in the static ranking.
- ts.move(src.heap)
+ for _, tw := range src.heap {
+ t := tw.timer
+ t.ts = nil
+ if t.state&timerZombie != 0 {
+ t.state &^= timerHeaped | timerZombie | timerModified
+ } else {
+ t.state &^= timerModified
+ ts.addHeap(t)
+ }
+ }
src.heap = nil
src.zombies.Store(0)
src.minWhenHeap.Store(0)
}
}
-// moveTimers moves a slice of timers to pp. The slice has been taken
-// from a different P.
-// The world must be stopped so that ts is safe to modify.
-func (ts *timers) move(timers []*timer) {
- assertWorldStopped()
- for _, t := range timers {
- t.ts = nil
- t.updateHeap(nil)
- if t.state&timerHeaped != 0 {
- ts.addHeap(t)
- }
- }
-}
-
// adjust looks through the timers in ts.heap for
// any timers that have been modified to run earlier, and puts them in
// the correct place in the heap. While looking for those timers,
// 2. Set minWhenModified = 0
// (Other goroutines may modify timers and update minWhenModified now.)
// 3. Scan timers
- // 4. Set minWhenHeap = heap[0].whenHeap
+ // 4. Set minWhenHeap = heap[0].when
//
// That order preserves a correct value of wakeTime throughout the entire
// operation:
changed := false
for i := 0; i < len(ts.heap); i++ {
- t := ts.heap[i]
+ tw := &ts.heap[i]
+ t := tw.timer
if t.ts != ts {
throw("bad ts")
}
}
t.lock()
- if t.state&timerHeaped == 0 {
+ switch {
+ case t.state&timerHeaped == 0:
badTimer()
- }
- if t.state&timerZombie != 0 {
- ts.zombies.Add(-1) // updateHeap will return updated=true and we will delete t
- }
- if t.updateHeap(nil) {
+
+ case t.state&timerZombie != 0:
+ ts.zombies.Add(-1)
+ t.state &^= timerHeaped | timerZombie | timerModified
+ n := len(ts.heap)
+ ts.heap[i] = ts.heap[n-1]
+ ts.heap[n-1] = timerWhen{}
+ ts.heap = ts.heap[:n-1]
+ t.ts = nil
+ i--
+ changed = true
+
+ case t.state&timerModified != 0:
+ tw.when = t.when
+ t.state &^= timerModified
changed = true
- if t.state&timerHeaped == 0 {
- n := len(ts.heap)
- ts.heap[i] = ts.heap[n-1]
- ts.heap[n-1] = nil
- ts.heap = ts.heap[:n-1]
- t.ts = nil
- i--
- }
}
t.unlock()
}
if len(ts.heap) == 0 {
return -1
}
- t := ts.heap[0]
+ tw := ts.heap[0]
+ t := tw.timer
if t.ts != ts {
throw("bad ts")
}
- if t.astate.Load()&(timerModified|timerZombie) == 0 && t.whenHeap > now {
+ if t.astate.Load()&(timerModified|timerZombie) == 0 && tw.when > now {
// Fast path: not ready to run.
- // The access of t.whenHeap is protected by the caller holding
- // ts.lock, even though t itself is unlocked.
- return t.whenHeap
+ return tw.when
}
t.lock()
- if t.updateHeap(ts) {
+ if t.updateHeap() {
t.unlock()
goto Redo
}
} else {
next = 0
}
+ ts := t.ts
+ t.when = next
if t.state&timerHeaped != 0 {
- t.when = next
t.state |= timerModified
if next == 0 {
t.state |= timerZombie
t.ts.zombies.Add(1)
}
- } else {
- t.when = next
+ t.updateHeap()
}
- ts := t.ts
- t.updateHeap(ts)
t.unlock()
if raceenabled {
// The caller must have locked ts.
func (ts *timers) verify() {
assertLockHeld(&ts.mu)
- for i, t := range ts.heap {
+ for i, tw := range ts.heap {
if i == 0 {
// First timer has no parent.
continue
// The heap is 4-ary. See siftupTimer and siftdownTimer.
p := (i - 1) / 4
- if t.whenHeap < ts.heap[p].whenHeap {
- print("bad timer heap at ", i, ": ", p, ": ", ts.heap[p].whenHeap, ", ", i, ": ", t.whenHeap, "\n")
+ if tw.when < ts.heap[p].when {
+ print("bad timer heap at ", i, ": ", p, ": ", ts.heap[p].when, ", ", i, ": ", tw.when, "\n")
throw("bad timer heap")
}
}
}
}
-// updateMinWhenHeap sets ts.minWhenHeap to ts.heap[0].whenHeap.
+// updateMinWhenHeap sets ts.minWhenHeap to ts.heap[0].when.
// The caller must have locked ts or the world must be stopped.
func (ts *timers) updateMinWhenHeap() {
assertWorldStoppedOrLockHeld(&ts.mu)
if len(ts.heap) == 0 {
ts.minWhenHeap.Store(0)
} else {
- ts.minWhenHeap.Store(ts.heap[0].whenHeap)
+ ts.minWhenHeap.Store(ts.heap[0].when)
}
}
// siftUp puts the timer at position i in the right place
// in the heap by moving it up toward the top of the heap.
func (ts *timers) siftUp(i int) {
- t := ts.heap
- if i >= len(t) {
+ heap := ts.heap
+ if i >= len(heap) {
badTimer()
}
- when := t[i].whenHeap
+ tw := heap[i]
+ when := tw.when
if when <= 0 {
badTimer()
}
- tmp := t[i]
for i > 0 {
p := (i - 1) / 4 // parent
- if when >= t[p].whenHeap {
+ if when >= heap[p].when {
break
}
- t[i] = t[p]
+ heap[i] = heap[p]
i = p
}
- if tmp != t[i] {
- t[i] = tmp
+ if heap[i].timer != tw.timer {
+ heap[i] = tw
}
}
// siftDown puts the timer at position i in the right place
// in the heap by moving it down toward the bottom of the heap.
func (ts *timers) siftDown(i int) {
- t := ts.heap
- n := len(t)
+ heap := ts.heap
+ n := len(heap)
if i >= n {
badTimer()
}
- when := t[i].whenHeap
+ tw := heap[i]
+ when := tw.when
if when <= 0 {
badTimer()
}
- tmp := t[i]
for {
c := i*4 + 1 // left child
c3 := c + 2 // mid child
if c >= n {
break
}
- w := t[c].whenHeap
- if c+1 < n && t[c+1].whenHeap < w {
- w = t[c+1].whenHeap
+ w := heap[c].when
+ if c+1 < n && heap[c+1].when < w {
+ w = heap[c+1].when
c++
}
if c3 < n {
- w3 := t[c3].whenHeap
- if c3+1 < n && t[c3+1].whenHeap < w3 {
- w3 = t[c3+1].whenHeap
+ w3 := heap[c3].when
+ if c3+1 < n && heap[c3+1].when < w3 {
+ w3 = heap[c3+1].when
c3++
}
if w3 < w {
if w >= when {
break
}
- t[i] = t[c]
+ heap[i] = heap[c]
i = c
}
- if tmp != t[i] {
- t[i] = tmp
+ if heap[i].timer != tw.timer {
+ heap[i] = tw
}
}