mutexWMask = (1<<20 - 1) << 43
)
-// Read operations must do RWLock(true)/RWUnlock(true).
-// Write operations must do RWLock(false)/RWUnlock(false).
-// Misc operations must do Incref/Decref. Misc operations include functions like
-// setsockopt and setDeadline. They need to use Incref/Decref to ensure that
-// they operate on the correct fd in presence of a concurrent Close call
+// Read operations must do rwlock(true)/rwunlock(true).
+// Write operations must do rwlock(false)/rwunlock(false).
+// Misc operations must do incref/decref. Misc operations include functions like
+// setsockopt and setDeadline. They need to use incref/decref to ensure that
+// they operate on the correct fd in presence of a concurrent close call
// (otherwise fd can be closed under their feet).
-// Close operation must do IncrefAndClose/Decref.
+// Close operation must do increfAndClose/decref.
-// RWLock/Incref return whether fd is open.
-// RWUnlock/Decref return whether fd is closed and there are no remaining references.
+// rwlock/incref return whether fd is open.
+// rwunlock/decref return whether fd is closed and there are no remaining references.
-func (mu *fdMutex) Incref() bool {
+func (mu *fdMutex) incref() bool {
for {
old := atomic.LoadUint64(&mu.state)
if old&mutexClosed != 0 {
}
}
-func (mu *fdMutex) IncrefAndClose() bool {
+func (mu *fdMutex) increfAndClose() bool {
for {
old := atomic.LoadUint64(&mu.state)
if old&mutexClosed != 0 {
}
}
-func (mu *fdMutex) Decref() bool {
+func (mu *fdMutex) decref() bool {
for {
old := atomic.LoadUint64(&mu.state)
if old&mutexRefMask == 0 {
}
}
-func (mu *fdMutex) RWLock(read bool) bool {
+func (mu *fdMutex) rwlock(read bool) bool {
var mutexBit, mutexWait, mutexMask uint64
var mutexSema *uint32
if read {
}
}
-func (mu *fdMutex) RWUnlock(read bool) bool {
+func (mu *fdMutex) rwunlock(read bool) bool {
var mutexBit, mutexWait, mutexMask uint64
var mutexSema *uint32
if read {
func TestMutexLock(t *testing.T) {
var mu fdMutex
- if !mu.Incref() {
+ if !mu.incref() {
t.Fatal("broken")
}
- if mu.Decref() {
+ if mu.decref() {
t.Fatal("broken")
}
- if !mu.RWLock(true) {
+ if !mu.rwlock(true) {
t.Fatal("broken")
}
- if mu.RWUnlock(true) {
+ if mu.rwunlock(true) {
t.Fatal("broken")
}
- if !mu.RWLock(false) {
+ if !mu.rwlock(false) {
t.Fatal("broken")
}
- if mu.RWUnlock(false) {
+ if mu.rwunlock(false) {
t.Fatal("broken")
}
}
func TestMutexClose(t *testing.T) {
var mu fdMutex
- if !mu.IncrefAndClose() {
+ if !mu.increfAndClose() {
t.Fatal("broken")
}
- if mu.Incref() {
+ if mu.incref() {
t.Fatal("broken")
}
- if mu.RWLock(true) {
+ if mu.rwlock(true) {
t.Fatal("broken")
}
- if mu.RWLock(false) {
+ if mu.rwlock(false) {
t.Fatal("broken")
}
- if mu.IncrefAndClose() {
+ if mu.increfAndClose() {
t.Fatal("broken")
}
}
func TestMutexCloseUnblock(t *testing.T) {
c := make(chan bool)
var mu fdMutex
- mu.RWLock(true)
+ mu.rwlock(true)
for i := 0; i < 4; i++ {
go func() {
- if mu.RWLock(true) {
+ if mu.rwlock(true) {
t.Error("broken")
return
}
t.Fatal("broken")
default:
}
- mu.IncrefAndClose() // Must unblock the readers.
+ mu.increfAndClose() // Must unblock the readers.
for i := 0; i < 4; i++ {
select {
case <-c:
t.Fatal("broken")
}
}
- if mu.Decref() {
+ if mu.decref() {
t.Fatal("broken")
}
- if !mu.RWUnlock(true) {
+ if !mu.rwunlock(true) {
t.Fatal("broken")
}
}
}
var mu fdMutex
- ensurePanics(func() { mu.Decref() })
- ensurePanics(func() { mu.RWUnlock(true) })
- ensurePanics(func() { mu.RWUnlock(false) })
+ ensurePanics(func() { mu.decref() })
+ ensurePanics(func() { mu.rwunlock(true) })
+ ensurePanics(func() { mu.rwunlock(false) })
- ensurePanics(func() { mu.Incref(); mu.Decref(); mu.Decref() })
- ensurePanics(func() { mu.RWLock(true); mu.RWUnlock(true); mu.RWUnlock(true) })
- ensurePanics(func() { mu.RWLock(false); mu.RWUnlock(false); mu.RWUnlock(false) })
+ ensurePanics(func() { mu.incref(); mu.decref(); mu.decref() })
+ ensurePanics(func() { mu.rwlock(true); mu.rwunlock(true); mu.rwunlock(true) })
+ ensurePanics(func() { mu.rwlock(false); mu.rwunlock(false); mu.rwunlock(false) })
// ensure that it's still not broken
- mu.Incref()
- mu.Decref()
- mu.RWLock(true)
- mu.RWUnlock(true)
- mu.RWLock(false)
- mu.RWUnlock(false)
+ mu.incref()
+ mu.decref()
+ mu.rwlock(true)
+ mu.rwunlock(true)
+ mu.rwlock(false)
+ mu.rwunlock(false)
}
func TestMutexStress(t *testing.T) {
for i := 0; i < N; i++ {
switch r.Intn(3) {
case 0:
- if !mu.Incref() {
+ if !mu.incref() {
t.Error("broken")
return
}
- if mu.Decref() {
+ if mu.decref() {
t.Error("broken")
return
}
case 1:
- if !mu.RWLock(true) {
+ if !mu.rwlock(true) {
t.Error("broken")
return
}
}
readState[0]++
readState[1]++
- if mu.RWUnlock(true) {
+ if mu.rwunlock(true) {
t.Error("broken")
return
}
case 2:
- if !mu.RWLock(false) {
+ if !mu.rwlock(false) {
t.Error("broken")
return
}
}
writeState[0]++
writeState[1]++
- if mu.RWUnlock(false) {
+ if mu.rwunlock(false) {
t.Error("broken")
return
}
for p := 0; p < P; p++ {
<-done
}
- if !mu.IncrefAndClose() {
+ if !mu.increfAndClose() {
t.Fatal("broken")
}
- if !mu.Decref() {
+ if !mu.decref() {
t.Fatal("broken")
}
}
// Add a reference to this fd.
// Returns an error if the fd cannot be used.
func (fd *netFD) incref() error {
- if !fd.fdmu.Incref() {
+ if !fd.fdmu.incref() {
return errClosing
}
return nil
// Remove a reference to this FD and close if we've been asked to do so
// (and there are no references left).
func (fd *netFD) decref() {
- if fd.fdmu.Decref() {
+ if fd.fdmu.decref() {
fd.destroy()
}
}
// Add a reference to this fd and lock for reading.
// Returns an error if the fd cannot be used.
func (fd *netFD) readLock() error {
- if !fd.fdmu.RWLock(true) {
+ if !fd.fdmu.rwlock(true) {
return errClosing
}
return nil
// Unlock for reading and remove a reference to this FD.
func (fd *netFD) readUnlock() {
- if fd.fdmu.RWUnlock(true) {
+ if fd.fdmu.rwunlock(true) {
fd.destroy()
}
}
// Add a reference to this fd and lock for writing.
// Returns an error if the fd cannot be used.
func (fd *netFD) writeLock() error {
- if !fd.fdmu.RWLock(false) {
+ if !fd.fdmu.rwlock(false) {
return errClosing
}
return nil
// Unlock for writing and remove a reference to this FD.
func (fd *netFD) writeUnlock() {
- if fd.fdmu.RWUnlock(false) {
+ if fd.fdmu.rwunlock(false) {
fd.destroy()
}
}
}
func (fd *netFD) Close() error {
- if !fd.fdmu.IncrefAndClose() {
+ if !fd.fdmu.increfAndClose() {
return errClosing
}
if !fd.ok() {
closing bool
}
-func (pd *pollDesc) Init(fd *netFD) error { pd.fd = fd; return nil }
+func (pd *pollDesc) init(fd *netFD) error { pd.fd = fd; return nil }
-func (pd *pollDesc) Close() {}
+func (pd *pollDesc) close() {}
-func (pd *pollDesc) Evict() {
+func (pd *pollDesc) evict() {
pd.closing = true
if pd.fd != nil {
syscall.StopIO(pd.fd.sysfd)
}
}
-func (pd *pollDesc) Prepare(mode int) error {
+func (pd *pollDesc) prepare(mode int) error {
if pd.closing {
return errClosing
}
return nil
}
-func (pd *pollDesc) PrepareRead() error { return pd.Prepare('r') }
+func (pd *pollDesc) prepareRead() error { return pd.prepare('r') }
-func (pd *pollDesc) PrepareWrite() error { return pd.Prepare('w') }
+func (pd *pollDesc) prepareWrite() error { return pd.prepare('w') }
-func (pd *pollDesc) Wait(mode int) error {
+func (pd *pollDesc) wait(mode int) error {
if pd.closing {
return errClosing
}
return errTimeout
}
-func (pd *pollDesc) WaitRead() error { return pd.Wait('r') }
+func (pd *pollDesc) waitRead() error { return pd.wait('r') }
-func (pd *pollDesc) WaitWrite() error { return pd.Wait('w') }
+func (pd *pollDesc) waitWrite() error { return pd.wait('w') }
-func (pd *pollDesc) WaitCanceled(mode int) {}
+func (pd *pollDesc) waitCanceled(mode int) {}
-func (pd *pollDesc) WaitCanceledRead() {}
+func (pd *pollDesc) waitCanceledRead() {}
-func (pd *pollDesc) WaitCanceledWrite() {}
+func (pd *pollDesc) waitCanceledWrite() {}
func (fd *netFD) setDeadline(t time.Time) error {
return setDeadlineImpl(fd, t, 'r'+'w')
var serverInit sync.Once
-func (pd *pollDesc) Init(fd *netFD) error {
+func (pd *pollDesc) init(fd *netFD) error {
serverInit.Do(runtime_pollServerInit)
ctx, errno := runtime_pollOpen(uintptr(fd.sysfd))
if errno != 0 {
return nil
}
-func (pd *pollDesc) Close() {
+func (pd *pollDesc) close() {
if pd.runtimeCtx == 0 {
return
}
}
// Evict evicts fd from the pending list, unblocking any I/O running on fd.
-func (pd *pollDesc) Evict() {
+func (pd *pollDesc) evict() {
if pd.runtimeCtx == 0 {
return
}
runtime_pollUnblock(pd.runtimeCtx)
}
-func (pd *pollDesc) Prepare(mode int) error {
+func (pd *pollDesc) prepare(mode int) error {
res := runtime_pollReset(pd.runtimeCtx, mode)
return convertErr(res)
}
-func (pd *pollDesc) PrepareRead() error {
- return pd.Prepare('r')
+func (pd *pollDesc) prepareRead() error {
+ return pd.prepare('r')
}
-func (pd *pollDesc) PrepareWrite() error {
- return pd.Prepare('w')
+func (pd *pollDesc) prepareWrite() error {
+ return pd.prepare('w')
}
-func (pd *pollDesc) Wait(mode int) error {
+func (pd *pollDesc) wait(mode int) error {
res := runtime_pollWait(pd.runtimeCtx, mode)
return convertErr(res)
}
-func (pd *pollDesc) WaitRead() error {
- return pd.Wait('r')
+func (pd *pollDesc) waitRead() error {
+ return pd.wait('r')
}
-func (pd *pollDesc) WaitWrite() error {
- return pd.Wait('w')
+func (pd *pollDesc) waitWrite() error {
+ return pd.wait('w')
}
-func (pd *pollDesc) WaitCanceled(mode int) {
+func (pd *pollDesc) waitCanceled(mode int) {
runtime_pollWaitCanceled(pd.runtimeCtx, mode)
}
-func (pd *pollDesc) WaitCanceledRead() {
- pd.WaitCanceled('r')
+func (pd *pollDesc) waitCanceledRead() {
+ pd.waitCanceled('r')
}
-func (pd *pollDesc) WaitCanceledWrite() {
- pd.WaitCanceled('w')
+func (pd *pollDesc) waitCanceledWrite() {
+ pd.waitCanceled('w')
}
func convertErr(res int) error {
}
func (fd *netFD) init() error {
- if err := fd.pd.Init(fd); err != nil {
+ if err := fd.pd.init(fd); err != nil {
return err
}
return nil
// SO_ERROR socket option to see if the connection
// succeeded or failed. See issue 7474 for further
// details.
- if err := fd.pd.WaitWrite(); err != nil {
+ if err := fd.pd.waitWrite(); err != nil {
select {
case <-cancel:
return errCanceled
func (fd *netFD) destroy() {
// Poller may want to unregister fd in readiness notification mechanism,
// so this must be executed before closeFunc.
- fd.pd.Close()
+ fd.pd.close()
closeFunc(fd.sysfd)
fd.sysfd = -1
runtime.SetFinalizer(fd, nil)
// Add a reference to this fd.
// Returns an error if the fd cannot be used.
func (fd *netFD) incref() error {
- if !fd.fdmu.Incref() {
+ if !fd.fdmu.incref() {
return errClosing
}
return nil
// Remove a reference to this FD and close if we've been asked to do so
// (and there are no references left).
func (fd *netFD) decref() {
- if fd.fdmu.Decref() {
+ if fd.fdmu.decref() {
fd.destroy()
}
}
// Add a reference to this fd and lock for reading.
// Returns an error if the fd cannot be used.
func (fd *netFD) readLock() error {
- if !fd.fdmu.RWLock(true) {
+ if !fd.fdmu.rwlock(true) {
return errClosing
}
return nil
// Unlock for reading and remove a reference to this FD.
func (fd *netFD) readUnlock() {
- if fd.fdmu.RWUnlock(true) {
+ if fd.fdmu.rwunlock(true) {
fd.destroy()
}
}
// Add a reference to this fd and lock for writing.
// Returns an error if the fd cannot be used.
func (fd *netFD) writeLock() error {
- if !fd.fdmu.RWLock(false) {
+ if !fd.fdmu.rwlock(false) {
return errClosing
}
return nil
// Unlock for writing and remove a reference to this FD.
func (fd *netFD) writeUnlock() {
- if fd.fdmu.RWUnlock(false) {
+ if fd.fdmu.rwunlock(false) {
fd.destroy()
}
}
func (fd *netFD) Close() error {
- if !fd.fdmu.IncrefAndClose() {
+ if !fd.fdmu.increfAndClose() {
return errClosing
}
// Unblock any I/O. Once it all unblocks and returns,
// the final decref will close fd.sysfd. This should happen
// fairly quickly, since all the I/O is non-blocking, and any
// attempts to block in the pollDesc will return errClosing.
- fd.pd.Evict()
+ fd.pd.evict()
fd.decref()
return nil
}
return 0, err
}
defer fd.readUnlock()
- if err := fd.pd.PrepareRead(); err != nil {
+ if err := fd.pd.prepareRead(); err != nil {
return 0, err
}
for {
if err != nil {
n = 0
if err == syscall.EAGAIN {
- if err = fd.pd.WaitRead(); err == nil {
+ if err = fd.pd.waitRead(); err == nil {
continue
}
}
return 0, nil, err
}
defer fd.readUnlock()
- if err := fd.pd.PrepareRead(); err != nil {
+ if err := fd.pd.prepareRead(); err != nil {
return 0, nil, err
}
for {
if err != nil {
n = 0
if err == syscall.EAGAIN {
- if err = fd.pd.WaitRead(); err == nil {
+ if err = fd.pd.waitRead(); err == nil {
continue
}
}
return 0, 0, 0, nil, err
}
defer fd.readUnlock()
- if err := fd.pd.PrepareRead(); err != nil {
+ if err := fd.pd.prepareRead(); err != nil {
return 0, 0, 0, nil, err
}
for {
if err != nil {
// TODO(dfc) should n and oobn be set to 0
if err == syscall.EAGAIN {
- if err = fd.pd.WaitRead(); err == nil {
+ if err = fd.pd.waitRead(); err == nil {
continue
}
}
return 0, err
}
defer fd.writeUnlock()
- if err := fd.pd.PrepareWrite(); err != nil {
+ if err := fd.pd.prepareWrite(); err != nil {
return 0, err
}
for {
break
}
if err == syscall.EAGAIN {
- if err = fd.pd.WaitWrite(); err == nil {
+ if err = fd.pd.waitWrite(); err == nil {
continue
}
}
return 0, err
}
defer fd.writeUnlock()
- if err := fd.pd.PrepareWrite(); err != nil {
+ if err := fd.pd.prepareWrite(); err != nil {
return 0, err
}
for {
err = syscall.Sendto(fd.sysfd, p, 0, sa)
if err == syscall.EAGAIN {
- if err = fd.pd.WaitWrite(); err == nil {
+ if err = fd.pd.waitWrite(); err == nil {
continue
}
}
return 0, 0, err
}
defer fd.writeUnlock()
- if err := fd.pd.PrepareWrite(); err != nil {
+ if err := fd.pd.prepareWrite(); err != nil {
return 0, 0, err
}
for {
n, err = syscall.SendmsgN(fd.sysfd, p, oob, sa, 0)
if err == syscall.EAGAIN {
- if err = fd.pd.WaitWrite(); err == nil {
+ if err = fd.pd.waitWrite(); err == nil {
continue
}
}
var s int
var rsa syscall.Sockaddr
- if err = fd.pd.PrepareRead(); err != nil {
+ if err = fd.pd.prepareRead(); err != nil {
return nil, err
}
for {
}
switch nerr.Err {
case syscall.EAGAIN:
- if err = fd.pd.WaitRead(); err == nil {
+ if err = fd.pd.waitRead(); err == nil {
continue
}
case syscall.ECONNABORTED:
func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) error) (int, error) {
fd := o.fd
// Notify runtime netpoll about starting IO.
- err := fd.pd.Prepare(int(o.mode))
+ err := fd.pd.prepare(int(o.mode))
if err != nil {
return 0, err
}
return 0, err
}
// Wait for our request to complete.
- err = fd.pd.Wait(int(o.mode))
+ err = fd.pd.wait(int(o.mode))
if err == nil {
// All is good. Extract our IO results and return.
if o.errno != 0 {
<-o.errc
}
// Wait for cancelation to complete.
- fd.pd.WaitCanceled(int(o.mode))
+ fd.pd.waitCanceled(int(o.mode))
if o.errno != 0 {
err = syscall.Errno(o.errno)
if err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled
}
func (fd *netFD) init() error {
- if err := fd.pd.Init(fd); err != nil {
+ if err := fd.pd.init(fd); err != nil {
return err
}
if hasLoadSetFileCompletionNotificationModes {
}
// Poller may want to unregister fd in readiness notification mechanism,
// so this must be executed before closeFunc.
- fd.pd.Close()
+ fd.pd.close()
closeFunc(fd.sysfd)
fd.sysfd = syscall.InvalidHandle
// no need for a finalizer anymore
// Add a reference to this fd.
// Returns an error if the fd cannot be used.
func (fd *netFD) incref() error {
- if !fd.fdmu.Incref() {
+ if !fd.fdmu.incref() {
return errClosing
}
return nil
// Remove a reference to this FD and close if we've been asked to do so
// (and there are no references left).
func (fd *netFD) decref() {
- if fd.fdmu.Decref() {
+ if fd.fdmu.decref() {
fd.destroy()
}
}
// Add a reference to this fd and lock for reading.
// Returns an error if the fd cannot be used.
func (fd *netFD) readLock() error {
- if !fd.fdmu.RWLock(true) {
+ if !fd.fdmu.rwlock(true) {
return errClosing
}
return nil
// Unlock for reading and remove a reference to this FD.
func (fd *netFD) readUnlock() {
- if fd.fdmu.RWUnlock(true) {
+ if fd.fdmu.rwunlock(true) {
fd.destroy()
}
}
// Add a reference to this fd and lock for writing.
// Returns an error if the fd cannot be used.
func (fd *netFD) writeLock() error {
- if !fd.fdmu.RWLock(false) {
+ if !fd.fdmu.rwlock(false) {
return errClosing
}
return nil
// Unlock for writing and remove a reference to this FD.
func (fd *netFD) writeUnlock() {
- if fd.fdmu.RWUnlock(false) {
+ if fd.fdmu.rwunlock(false) {
fd.destroy()
}
}
func (fd *netFD) Close() error {
- if !fd.fdmu.IncrefAndClose() {
+ if !fd.fdmu.increfAndClose() {
return errClosing
}
// unblock pending reader and writer
- fd.pd.Evict()
+ fd.pd.evict()
fd.decref()
return nil
}
break
}
if err1 == syscall.EAGAIN {
- if err1 = c.pd.WaitWrite(); err1 == nil {
+ if err1 = c.pd.waitWrite(); err1 == nil {
continue
}
}
break
}
if err1 == syscall.EAGAIN {
- if err1 = c.pd.WaitWrite(); err1 == nil {
+ if err1 = c.pd.waitWrite(); err1 == nil {
continue
}
}
break
}
if err1 == syscall.EAGAIN {
- if err1 = c.pd.WaitWrite(); err1 == nil {
+ if err1 = c.pd.waitWrite(); err1 == nil {
continue
}
}
break
}
if err1 == syscall.EAGAIN {
- if err1 = c.pd.WaitWrite(); err1 == nil {
+ if err1 = c.pd.waitWrite(); err1 == nil {
continue
}
}