--- /dev/null
+// Code generated by mkconsts.go. DO NOT EDIT.
+
+//go:build !goexperiment.spinbitmutex
+
+package goexperiment
+
+const SpinbitMutex = false
+const SpinbitMutexInt = 0
--- /dev/null
+// Code generated by mkconsts.go. DO NOT EDIT.
+
+//go:build goexperiment.spinbitmutex
+
+package goexperiment
+
+const SpinbitMutex = true
+const SpinbitMutexInt = 1
// SwissMap enables the SwissTable-based map implementation.
SwissMap bool
+
+ // SpinbitMutex enables the new "spinbit" mutex implementation on supported
+ // platforms. See https://go.dev/issue/68578.
+ SpinbitMutex bool
}
"unsafe"
)
-// This implementation depends on OS-specific implementations of
-//
-// futexsleep(addr *uint32, val uint32, ns int64)
-// Atomically,
-// if *addr == val { sleep }
-// Might be woken up spuriously; that's allowed.
-// Don't sleep longer than ns; ns < 0 means forever.
-//
-// futexwakeup(addr *uint32, cnt uint32)
-// If any procs are sleeping on addr, wake up at most cnt.
-
-const (
- mutex_unlocked = 0
- mutex_locked = 1
- mutex_sleeping = 2
-
- active_spin = 4
- active_spin_cnt = 30
- passive_spin = 1
-)
-
-// Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping.
-// mutex_sleeping means that there is presumably at least one sleeping thread.
-// Note that there can be spinning threads during all states - they do not
-// affect mutex's state.
-
// We use the uintptr mutex.key and note.key as a uint32.
//
//go:nosplit
return (*uint32)(unsafe.Pointer(p))
}
-func mutexContended(l *mutex) bool {
- return atomic.Load(key32(&l.key)) > mutex_locked
-}
-
-func lock(l *mutex) {
- lockWithRank(l, getLockRank(l))
-}
-
-func lock2(l *mutex) {
- gp := getg()
-
- if gp.m.locks < 0 {
- throw("runtime·lock: lock count")
- }
- gp.m.locks++
-
- // Speculative grab for lock.
- v := atomic.Xchg(key32(&l.key), mutex_locked)
- if v == mutex_unlocked {
- return
- }
-
- // wait is either MUTEX_LOCKED or MUTEX_SLEEPING
- // depending on whether there is a thread sleeping
- // on this mutex. If we ever change l->key from
- // MUTEX_SLEEPING to some other value, we must be
- // careful to change it back to MUTEX_SLEEPING before
- // returning, to ensure that the sleeping thread gets
- // its wakeup call.
- wait := v
-
- timer := &lockTimer{lock: l}
- timer.begin()
- // On uniprocessors, no point spinning.
- // On multiprocessors, spin for ACTIVE_SPIN attempts.
- spin := 0
- if ncpu > 1 {
- spin = active_spin
- }
- for {
- // Try for lock, spinning.
- for i := 0; i < spin; i++ {
- for l.key == mutex_unlocked {
- if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
- timer.end()
- return
- }
- }
- procyield(active_spin_cnt)
- }
-
- // Try for lock, rescheduling.
- for i := 0; i < passive_spin; i++ {
- for l.key == mutex_unlocked {
- if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
- timer.end()
- return
- }
- }
- osyield()
- }
-
- // Sleep.
- v = atomic.Xchg(key32(&l.key), mutex_sleeping)
- if v == mutex_unlocked {
- timer.end()
- return
- }
- wait = mutex_sleeping
- futexsleep(key32(&l.key), mutex_sleeping, -1)
- }
-}
-
-func unlock(l *mutex) {
- unlockWithRank(l)
-}
-
-func unlock2(l *mutex) {
- v := atomic.Xchg(key32(&l.key), mutex_unlocked)
- if v == mutex_unlocked {
- throw("unlock of unlocked lock")
- }
- if v == mutex_sleeping {
- futexwakeup(key32(&l.key), 1)
- }
-
- gp := getg()
- gp.m.mLockProfile.recordUnlock(l)
- gp.m.locks--
- if gp.m.locks < 0 {
- throw("runtime·unlock: lock count")
- }
- if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack
- gp.stackguard0 = stackPreempt
- }
-}
-
// One-time notifications.
func noteclear(n *note) {
n.key = 0
}
func checkTimeouts() {}
+
+//go:nosplit
+func semacreate(mp *m) {}
+
+//go:nosplit
+func semasleep(ns int64) int32 {
+ mp := getg().m
+
+ for v := atomic.Xadd(&mp.waitsema, -1); ; v = atomic.Load(&mp.waitsema) {
+ if int32(v) >= 0 {
+ return 0
+ }
+ futexsleep(&mp.waitsema, v, ns)
+ if ns >= 0 {
+ if int32(v) >= 0 {
+ return 0
+ } else {
+ return -1
+ }
+ }
+ }
+}
+
+//go:nosplit
+func semawakeup(mp *m) {
+ v := atomic.Xadd(&mp.waitsema, 1)
+ if v == 0 {
+ futexwakeup(&mp.waitsema, 1)
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build (dragonfly || freebsd || linux) && !goexperiment.spinbitmutex
+
+package runtime
+
+import (
+ "internal/runtime/atomic"
+)
+
+// This implementation depends on OS-specific implementations of
+//
+// futexsleep(addr *uint32, val uint32, ns int64)
+// Atomically,
+// if *addr == val { sleep }
+// Might be woken up spuriously; that's allowed.
+// Don't sleep longer than ns; ns < 0 means forever.
+//
+// futexwakeup(addr *uint32, cnt uint32)
+// If any procs are sleeping on addr, wake up at most cnt.
+
+const (
+ mutex_unlocked = 0
+ mutex_locked = 1
+ mutex_sleeping = 2
+
+ active_spin = 4
+ active_spin_cnt = 30
+ passive_spin = 1
+)
+
+// Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping.
+// mutex_sleeping means that there is presumably at least one sleeping thread.
+// Note that there can be spinning threads during all states - they do not
+// affect mutex's state.
+
+type mWaitList struct{}
+
+func mutexContended(l *mutex) bool {
+ return atomic.Load(key32(&l.key)) > mutex_locked
+}
+
+func lock(l *mutex) {
+ lockWithRank(l, getLockRank(l))
+}
+
+func lock2(l *mutex) {
+ gp := getg()
+
+ if gp.m.locks < 0 {
+ throw("runtime·lock: lock count")
+ }
+ gp.m.locks++
+
+ // Speculative grab for lock.
+ v := atomic.Xchg(key32(&l.key), mutex_locked)
+ if v == mutex_unlocked {
+ return
+ }
+
+ // wait is either MUTEX_LOCKED or MUTEX_SLEEPING
+ // depending on whether there is a thread sleeping
+ // on this mutex. If we ever change l->key from
+ // MUTEX_SLEEPING to some other value, we must be
+ // careful to change it back to MUTEX_SLEEPING before
+ // returning, to ensure that the sleeping thread gets
+ // its wakeup call.
+ wait := v
+
+ timer := &lockTimer{lock: l}
+ timer.begin()
+ // On uniprocessors, no point spinning.
+ // On multiprocessors, spin for ACTIVE_SPIN attempts.
+ spin := 0
+ if ncpu > 1 {
+ spin = active_spin
+ }
+ for {
+ // Try for lock, spinning.
+ for i := 0; i < spin; i++ {
+ for l.key == mutex_unlocked {
+ if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
+ timer.end()
+ return
+ }
+ }
+ procyield(active_spin_cnt)
+ }
+
+ // Try for lock, rescheduling.
+ for i := 0; i < passive_spin; i++ {
+ for l.key == mutex_unlocked {
+ if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
+ timer.end()
+ return
+ }
+ }
+ osyield()
+ }
+
+ // Sleep.
+ v = atomic.Xchg(key32(&l.key), mutex_sleeping)
+ if v == mutex_unlocked {
+ timer.end()
+ return
+ }
+ wait = mutex_sleeping
+ futexsleep(key32(&l.key), mutex_sleeping, -1)
+ }
+}
+
+func unlock(l *mutex) {
+ unlockWithRank(l)
+}
+
+func unlock2(l *mutex) {
+ v := atomic.Xchg(key32(&l.key), mutex_unlocked)
+ if v == mutex_unlocked {
+ throw("unlock of unlocked lock")
+ }
+ if v == mutex_sleeping {
+ futexwakeup(key32(&l.key), 1)
+ }
+
+ gp := getg()
+ gp.m.mLockProfile.recordUnlock(l)
+ gp.m.locks--
+ if gp.m.locks < 0 {
+ throw("runtime·unlock: lock count")
+ }
+ if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack
+ gp.stackguard0 = stackPreempt
+ }
+}
passive_spin = 1
)
+type mWaitList struct{}
+
func mutexContended(l *mutex) bool {
return false
}
"unsafe"
)
-// This implementation depends on OS-specific implementations of
-//
-// func semacreate(mp *m)
-// Create a semaphore for mp, if it does not already have one.
-//
-// func semasleep(ns int64) int32
-// If ns < 0, acquire m's semaphore and return 0.
-// If ns >= 0, try to acquire m's semaphore for at most ns nanoseconds.
-// Return 0 if the semaphore was acquired, -1 if interrupted or timed out.
-//
-// func semawakeup(mp *m)
-// Wake up mp, which is or will soon be sleeping on its semaphore.
-const (
- locked uintptr = 1
-
- active_spin = 4
- active_spin_cnt = 30
- passive_spin = 1
-)
-
-func mutexContended(l *mutex) bool {
- return atomic.Loaduintptr(&l.key) > locked
-}
-
-func lock(l *mutex) {
- lockWithRank(l, getLockRank(l))
-}
-
-func lock2(l *mutex) {
- gp := getg()
- if gp.m.locks < 0 {
- throw("runtime·lock: lock count")
- }
- gp.m.locks++
-
- // Speculative grab for lock.
- if atomic.Casuintptr(&l.key, 0, locked) {
- return
- }
- semacreate(gp.m)
-
- timer := &lockTimer{lock: l}
- timer.begin()
- // On uniprocessor's, no point spinning.
- // On multiprocessors, spin for ACTIVE_SPIN attempts.
- spin := 0
- if ncpu > 1 {
- spin = active_spin
- }
-Loop:
- for i := 0; ; i++ {
- v := atomic.Loaduintptr(&l.key)
- if v&locked == 0 {
- // Unlocked. Try to lock.
- if atomic.Casuintptr(&l.key, v, v|locked) {
- timer.end()
- return
- }
- i = 0
- }
- if i < spin {
- procyield(active_spin_cnt)
- } else if i < spin+passive_spin {
- osyield()
- } else {
- // Someone else has it.
- // l->waitm points to a linked list of M's waiting
- // for this lock, chained through m->nextwaitm.
- // Queue this M.
- for {
- gp.m.nextwaitm = muintptr(v &^ locked)
- if atomic.Casuintptr(&l.key, v, uintptr(unsafe.Pointer(gp.m))|locked) {
- break
- }
- v = atomic.Loaduintptr(&l.key)
- if v&locked == 0 {
- continue Loop
- }
- }
- if v&locked != 0 {
- // Queued. Wait.
- semasleep(-1)
- i = 0
- }
- }
- }
-}
-
-func unlock(l *mutex) {
- unlockWithRank(l)
-}
-
-// We might not be holding a p in this code.
-//
-//go:nowritebarrier
-func unlock2(l *mutex) {
- gp := getg()
- var mp *m
- for {
- v := atomic.Loaduintptr(&l.key)
- if v == locked {
- if atomic.Casuintptr(&l.key, locked, 0) {
- break
- }
- } else {
- // Other M's are waiting for the lock.
- // Dequeue an M.
- mp = muintptr(v &^ locked).ptr()
- if atomic.Casuintptr(&l.key, v, uintptr(mp.nextwaitm)) {
- // Dequeued an M. Wake it.
- semawakeup(mp)
- break
- }
- }
- }
- gp.m.mLockProfile.recordUnlock(l)
- gp.m.locks--
- if gp.m.locks < 0 {
- throw("runtime·unlock: lock count")
- }
- if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack
- gp.stackguard0 = stackPreempt
- }
-}
-
// One-time notifications.
func noteclear(n *note) {
n.key = 0
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build aix || darwin || netbsd || openbsd || plan9 || solaris || windows || ((dragonfly || freebsd || linux) && goexperiment.spinbitmutex)
+
+package runtime
+
+import (
+ "internal/runtime/atomic"
+ "unsafe"
+)
+
+// This implementation depends on OS-specific implementations of
+//
+// func semacreate(mp *m)
+// Create a semaphore for mp, if it does not already have one.
+//
+// func semasleep(ns int64) int32
+// If ns < 0, acquire m's semaphore and return 0.
+// If ns >= 0, try to acquire m's semaphore for at most ns nanoseconds.
+// Return 0 if the semaphore was acquired, -1 if interrupted or timed out.
+//
+// func semawakeup(mp *m)
+// Wake up mp, which is or will soon be sleeping on its semaphore.
+const (
+ locked uintptr = 1
+
+ active_spin = 4
+ active_spin_cnt = 30
+ passive_spin = 1
+)
+
+// mWaitList is part of the M struct, and holds the list of Ms that are waiting
+// for a particular runtime.mutex.
+//
+// When an M is unable to immediately obtain a lock, it adds itself to the list
+// of Ms waiting for the lock. It does that via this struct's next field,
+// forming a singly-linked list with the mutex's key field pointing to the head
+// of the list.
+type mWaitList struct {
+ next muintptr // next m waiting for lock
+}
+
+func mutexContended(l *mutex) bool {
+ return atomic.Loaduintptr(&l.key) > locked
+}
+
+func lock(l *mutex) {
+ lockWithRank(l, getLockRank(l))
+}
+
+func lock2(l *mutex) {
+ gp := getg()
+ if gp.m.locks < 0 {
+ throw("runtime·lock: lock count")
+ }
+ gp.m.locks++
+
+ // Speculative grab for lock.
+ if atomic.Casuintptr(&l.key, 0, locked) {
+ return
+ }
+ semacreate(gp.m)
+
+ timer := &lockTimer{lock: l}
+ timer.begin()
+ // On uniprocessor's, no point spinning.
+ // On multiprocessors, spin for ACTIVE_SPIN attempts.
+ spin := 0
+ if ncpu > 1 {
+ spin = active_spin
+ }
+Loop:
+ for i := 0; ; i++ {
+ v := atomic.Loaduintptr(&l.key)
+ if v&locked == 0 {
+ // Unlocked. Try to lock.
+ if atomic.Casuintptr(&l.key, v, v|locked) {
+ timer.end()
+ return
+ }
+ i = 0
+ }
+ if i < spin {
+ procyield(active_spin_cnt)
+ } else if i < spin+passive_spin {
+ osyield()
+ } else {
+ // Someone else has it.
+ // l.key points to a linked list of M's waiting
+ // for this lock, chained through m.mWaitList.next.
+ // Queue this M.
+ for {
+ gp.m.mWaitList.next = muintptr(v &^ locked)
+ if atomic.Casuintptr(&l.key, v, uintptr(unsafe.Pointer(gp.m))|locked) {
+ break
+ }
+ v = atomic.Loaduintptr(&l.key)
+ if v&locked == 0 {
+ continue Loop
+ }
+ }
+ if v&locked != 0 {
+ // Queued. Wait.
+ semasleep(-1)
+ i = 0
+ }
+ }
+ }
+}
+
+func unlock(l *mutex) {
+ unlockWithRank(l)
+}
+
+// We might not be holding a p in this code.
+//
+//go:nowritebarrier
+func unlock2(l *mutex) {
+ gp := getg()
+ var mp *m
+ for {
+ v := atomic.Loaduintptr(&l.key)
+ if v == locked {
+ if atomic.Casuintptr(&l.key, locked, 0) {
+ break
+ }
+ } else {
+ // Other M's are waiting for the lock.
+ // Dequeue an M.
+ mp = muintptr(v &^ locked).ptr()
+ if atomic.Casuintptr(&l.key, v, uintptr(mp.mWaitList.next)) {
+ // Dequeued an M. Wake it.
+ semawakeup(mp)
+ break
+ }
+ }
+ }
+ gp.m.mLockProfile.recordUnlock(l)
+ gp.m.locks--
+ if gp.m.locks < 0 {
+ throw("runtime·unlock: lock count")
+ }
+ if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack
+ gp.stackguard0 = stackPreempt
+ }
+}
active_spin_cnt = 30
)
+type mWaitList struct{}
+
func mutexContended(l *mutex) bool {
return false
}
_SIG_SETMASK = 3
)
-type mOS struct{}
+type mOS struct {
+ waitsema uint32 // semaphore for parking on locks
+}
//go:noescape
func lwp_create(param *lwpparams) int32
"unsafe"
)
-type mOS struct{}
+type mOS struct {
+ waitsema uint32 // semaphore for parking on locks
+}
//go:noescape
func thr_new(param *thrparam, size int32) int32
// This is a pointer to a chunk of memory allocated with a special
// mmap invocation in vgetrandomGetState().
vgetrandomState uintptr
+
+ waitsema uint32 // semaphore for parking on locks
}
//go:noescape
createstack [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it.
lockedExt uint32 // tracking for external LockOSThread
lockedInt uint32 // tracking for internal lockOSThread
- nextwaitm muintptr // next m waiting for lock
+ mWaitList mWaitList // list of runtime lock waiters
mLockProfile mLockProfile // fields relating to runtime.lock contention
profStack []uintptr // used for memory/block/mutex stack traces