Revert "runtime: use semaphore structure for futex locks"

This reverts commit dfb7073bb8e66630156fc14ae50042acef89a929 (CL 585636).

Reason for revert: This is part of a patch series that changed the
handling of contended lock2/unlock2 calls, reducing the maximum
throughput of contended runtime.mutex values, and causing a performance
regression on applications where that is (or became) the bottleneck.

Updates #66999
Updates #67585

Change-Id: I3483bf0b85ba0b77204032a68b7cbe93f142703e
Reviewed-on: https://go-review.googlesource.com/c/go/+/589098
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Than McIntosh <thanm@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
Auto-Submit: Rhys Hiltner <rhys.hiltner@gmail.com>

diff --git a/src/runtime/lock_futex.go b/src/runtime/lock_futex.go
index 2d00635ba789b27400432c4361aae21359f6e362..58690e45e4d5a87bf2e7ddbd9d0f213ac060fa93 100644 (file)
--- a/src/runtime/lock_futex.go
+++ b/src/runtime/lock_futex.go
@@ -23,20 +23,19 @@ import (
 //             If any procs are sleeping on addr, wake up at most cnt.
 
 const (
-       mutex_locked   = 0x1
-       mutex_sleeping = 0x2 // Ensure futex's low 32 bits won't be all zeros
+       mutex_unlocked = 0
+       mutex_locked   = 1
+       mutex_sleeping = 2
 
        active_spin     = 4
        active_spin_cnt = 30
        passive_spin    = 1
 )
 
-// The mutex.key holds two state flags in its lowest bits: When the mutex_locked
-// bit is set, the mutex is locked. When the mutex_sleeping bit is set, a thread
-// is waiting in futexsleep for the mutex to be available. These flags operate
-// independently: a thread can enter lock2, observe that another thread is
-// already asleep, and immediately try to grab the lock anyway without waiting
-// for its "fair" turn.
+// Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping.
+// mutex_sleeping means that there is presumably at least one sleeping thread.
+// Note that there can be spinning threads during all states - they do not
+// affect mutex's state.
 
 // We use the uintptr mutex.key and note.key as a uint32.
 //
@@ -55,16 +54,27 @@ func lock(l *mutex) {
 
 func lock2(l *mutex) {
        gp := getg()
+
        if gp.m.locks < 0 {
                throw("runtime·lock: lock count")
        }
        gp.m.locks++
 
        // Speculative grab for lock.
-       if atomic.Casuintptr(&l.key, 0, mutex_locked) {
+       v := atomic.Xchg(key32(&l.key), mutex_locked)
+       if v == mutex_unlocked {
                return
        }
 
+       // wait is either MUTEX_LOCKED or MUTEX_SLEEPING
+       // depending on whether there is a thread sleeping
+       // on this mutex. If we ever change l->key from
+       // MUTEX_SLEEPING to some other value, we must be
+       // careful to change it back to MUTEX_SLEEPING before
+       // returning, to ensure that the sleeping thread gets
+       // its wakeup call.
+       wait := v
+
        timer := &lockTimer{lock: l}
        timer.begin()
        // On uniprocessors, no point spinning.
@@ -73,39 +83,37 @@ func lock2(l *mutex) {
        if ncpu > 1 {
                spin = active_spin
        }
-Loop:
-       for i := 0; ; i++ {
-               v := atomic.Loaduintptr(&l.key)
-               if v&mutex_locked == 0 {
-                       // Unlocked. Try to lock.
-                       if atomic.Casuintptr(&l.key, v, v|mutex_locked) {
-                               timer.end()
-                               return
+       for {
+               // Try for lock, spinning.
+               for i := 0; i < spin; i++ {
+                       for l.key == mutex_unlocked {
+                               if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
+                                       timer.end()
+                                       return
+                               }
                        }
-                       i = 0
-               }
-               if i < spin {
                        procyield(active_spin_cnt)
-               } else if i < spin+passive_spin {
-                       osyield()
-               } else {
-                       // Someone else has it.
-                       for {
-                               head := v &^ (mutex_locked | mutex_sleeping)
-                               if atomic.Casuintptr(&l.key, v, head|mutex_locked|mutex_sleeping) {
-                                       break
-                               }
-                               v = atomic.Loaduintptr(&l.key)
-                               if v&mutex_locked == 0 {
-                                       continue Loop
+               }
+
+               // Try for lock, rescheduling.
+               for i := 0; i < passive_spin; i++ {
+                       for l.key == mutex_unlocked {
+                               if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
+                                       timer.end()
+                                       return
                                }
                        }
-                       if v&mutex_locked != 0 {
-                               // Queued. Wait.
-                               futexsleep(key32(&l.key), uint32(v), -1)
-                               i = 0
-                       }
+                       osyield()
                }
+
+               // Sleep.
+               v = atomic.Xchg(key32(&l.key), mutex_sleeping)
+               if v == mutex_unlocked {
+                       timer.end()
+                       return
+               }
+               wait = mutex_sleeping
+               futexsleep(key32(&l.key), mutex_sleeping, -1)
        }
 }
 
@@ -114,21 +122,12 @@ func unlock(l *mutex) {
 }
 
 func unlock2(l *mutex) {
-       for {
-               v := atomic.Loaduintptr(&l.key)
-               if v == mutex_locked {
-                       if atomic.Casuintptr(&l.key, mutex_locked, 0) {
-                               break
-                       }
-               } else if v&mutex_locked == 0 {
-                       throw("unlock of unlocked lock")
-               } else {
-                       // Other M's are waiting for the lock.
-                       if atomic.Casuintptr(&l.key, v, v&^mutex_locked) {
-                               futexwakeup(key32(&l.key), 1)
-                               break
-                       }
-               }
+       v := atomic.Xchg(key32(&l.key), mutex_unlocked)
+       if v == mutex_unlocked {
+               throw("unlock of unlocked lock")
+       }
+       if v == mutex_sleeping {
+               futexwakeup(key32(&l.key), 1)
        }
 
        gp := getg()