]> Cypherpunks repositories - gostls13.git/commitdiff
runtime: utilize EVFILT_USER to wake up kevent for kqueue
authorAndy Pan <i@andypan.me>
Wed, 10 Apr 2024 08:48:09 +0000 (16:48 +0800)
committerGopher Robot <gobot@golang.org>
Fri, 12 Apr 2024 21:17:22 +0000 (21:17 +0000)
Fixes #66760

Change-Id: I6ba5bc5b00506b66cb8dc3984a61f32a6358d9bc
Reviewed-on: https://go-review.googlesource.com/c/go/+/577895
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Run-TryBot: Andy Pan <panjf2000@gmail.com>
Auto-Submit: Ian Lance Taylor <iant@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
19 files changed:
src/runtime/defs1_netbsd_386.go
src/runtime/defs1_netbsd_amd64.go
src/runtime/defs1_netbsd_arm.go
src/runtime/defs1_netbsd_arm64.go
src/runtime/defs_darwin.go
src/runtime/defs_darwin_amd64.go
src/runtime/defs_darwin_arm64.go
src/runtime/defs_dragonfly.go
src/runtime/defs_dragonfly_amd64.go
src/runtime/defs_freebsd.go
src/runtime/defs_freebsd_386.go
src/runtime/defs_freebsd_amd64.go
src/runtime/defs_freebsd_arm.go
src/runtime/defs_freebsd_arm64.go
src/runtime/defs_freebsd_riscv64.go
src/runtime/defs_netbsd.go
src/runtime/netpoll_kqueue.go
src/runtime/netpoll_kqueue_event.go [new file with mode: 0644]
src/runtime/netpoll_kqueue_pipe.go [new file with mode: 0644]

index f7fe45b4ab64924ee704bd96dd7200d6c3a2be70..16c55def92693b32758f0586c2682cfd7dde147b 100644 (file)
@@ -84,12 +84,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = 0x0
        _EVFILT_WRITE = 0x1
+       _EVFILT_USER  = 0x8
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type sigset struct {
index 80908cd93185c2d4d598bf87b9e68acf88802fb1..7a035a99c80b2f829407f76ce589da6841f79b82 100644 (file)
@@ -84,12 +84,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = 0x0
        _EVFILT_WRITE = 0x1
+       _EVFILT_USER  = 0x8
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type sigset struct {
index c63e592ff14573c10df284c04ac31103b962c299..77a59d4a05bfbddbb86e59bbbd5895310b5b1429 100644 (file)
@@ -84,12 +84,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = 0x0
        _EVFILT_WRITE = 0x1
+       _EVFILT_USER  = 0x8
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type sigset struct {
index 804b5b0b3f6738360d1288674acdb635b60e37f2..0720461f26cbac4a39a1741ab369375837272adc 100644 (file)
@@ -84,12 +84,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = 0x0
        _EVFILT_WRITE = 0x1
+       _EVFILT_USER  = 0x8
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type sigset struct {
index e37443307f3776d7acfd6edf0e251efe0241456b..9de59be20ba8d84d3e6956d2e73ec6fbf3f47f0e 100644 (file)
@@ -106,12 +106,17 @@ const (
 
        EV_ADD       = C.EV_ADD
        EV_DELETE    = C.EV_DELETE
+       EV_ENABLE    = C.EV_ENABLE
+       EV_DISABLE   = C.EV_DISABLE
        EV_CLEAR     = C.EV_CLEAR
        EV_RECEIPT   = C.EV_RECEIPT
        EV_ERROR     = C.EV_ERROR
        EV_EOF       = C.EV_EOF
        EVFILT_READ  = C.EVFILT_READ
        EVFILT_WRITE = C.EVFILT_WRITE
+       EVFILT_USER  = C.EVFILT_USER
+
+       NOTE_TRIGGER = C.NOTE_TRIGGER
 
        PTHREAD_CREATE_DETACHED = C.PTHREAD_CREATE_DETACHED
 
index f998b0be9118f979e1fa4766298445818ab0c51d..c0ca16d46309a1f8278b5d96af45db2eed16e32c 100644 (file)
@@ -85,12 +85,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0x40
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = -0x1
        _EVFILT_WRITE = -0x2
+       _EVFILT_USER  = -0xa
+
+       _NOTE_TRIGGER = 0x1000000
 
        _PTHREAD_CREATE_DETACHED = 0x2
 
index e07b08e0eec92060e9584b6ad2d9cbd60f724de5..cb534cacda4fc172b271bd30835de3beb400c9b3 100644 (file)
@@ -85,12 +85,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0x40
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = -0x1
        _EVFILT_WRITE = -0x2
+       _EVFILT_USER  = -0xa
+
+       _NOTE_TRIGGER = 0x1000000
 
        _PTHREAD_CREATE_DETACHED = 0x2
 
index 0463f1f116aeac1f4d9530bc43377be59accf015..d94461fe473d8a91e133f53f48a2641472b7ba4c 100644 (file)
@@ -109,11 +109,16 @@ const (
 
        EV_ADD       = C.EV_ADD
        EV_DELETE    = C.EV_DELETE
+       EV_ENABLE    = C.EV_ENABLE
+       EV_DISABLE   = C.EV_DISABLE
        EV_CLEAR     = C.EV_CLEAR
        EV_ERROR     = C.EV_ERROR
        EV_EOF       = C.EV_EOF
        EVFILT_READ  = C.EVFILT_READ
        EVFILT_WRITE = C.EVFILT_WRITE
+       EVFILT_USER  = C.EVFILT_USER
+
+       NOTE_TRIGGER = C.NOTE_TRIGGER
 )
 
 type Rtprio C.struct_rtprio
index 41bfb085d1ac66f985fc82aaf129f57404279997..b142ae1d14d1c8c188f1e9ee4b5009290a3542e8 100644 (file)
@@ -88,11 +88,16 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = -0x1
        _EVFILT_WRITE = -0x2
+       _EVFILT_USER  = -0x9
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type rtprio struct {
index d86ae9133afade6055c44e26f5be2cb880ff8a2e..70d82b90dae8b9e8f44c588d48b791e120c06c3a 100644 (file)
@@ -136,12 +136,17 @@ const (
 
        EV_ADD       = C.EV_ADD
        EV_DELETE    = C.EV_DELETE
+       EV_ENABLE    = C.EV_ENABLE
+       EV_DISABLE   = C.EV_DISABLE
        EV_CLEAR     = C.EV_CLEAR
        EV_RECEIPT   = C.EV_RECEIPT
        EV_ERROR     = C.EV_ERROR
        EV_EOF       = C.EV_EOF
        EVFILT_READ  = C.EVFILT_READ
        EVFILT_WRITE = C.EVFILT_WRITE
+       EVFILT_USER  = C.EVFILT_USER
+
+       NOTE_TRIGGER = C.NOTE_TRIGGER
 )
 
 type Rtprio C.struct_rtprio
index ee8274188ae69e0b74e63f06d4090b186d9b3477..42a0faf74dcc0d7b7b4dbf1c9d5671b7eeb82d48 100644 (file)
@@ -104,12 +104,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0x40
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = -0x1
        _EVFILT_WRITE = -0x2
+       _EVFILT_USER  = -0xb
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type rtprio struct {
index 9003f9201565a14ca9871be08b3406edb557a3e9..8f0b08d48aedd03519efde3dc933068683a3e706 100644 (file)
@@ -104,12 +104,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0x40
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = -0x1
        _EVFILT_WRITE = -0x2
+       _EVFILT_USER  = -0xb
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type rtprio struct {
index 68cc1b9545b7991eba8808c0e28d0a70a9794a22..dbb54da51bd2e5813dc45a5741ad10ad37cf9aaf 100644 (file)
@@ -104,12 +104,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0x40
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = -0x1
        _EVFILT_WRITE = -0x2
+       _EVFILT_USER  = -0xb
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type rtprio struct {
index 1d6723621ae72bb1a09e6842074f7459e794d4db..83b639381be3bfe35d6e58441c2ad8ab912ed7d8 100644 (file)
@@ -104,12 +104,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0x40
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = -0x1
        _EVFILT_WRITE = -0x2
+       _EVFILT_USER  = -0xb
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type rtprio struct {
index b977bde5515e42e9ff18b555bfff1b05a938eddd..5d9a5ac7fda8b9e744d2e00d0cadf48d028af537 100644 (file)
@@ -103,12 +103,17 @@ const (
 
        _EV_ADD       = 0x1
        _EV_DELETE    = 0x2
+       _EV_ENABLE    = 0x4
+       _EV_DISABLE   = 0x8
        _EV_CLEAR     = 0x20
        _EV_RECEIPT   = 0x40
        _EV_ERROR     = 0x4000
        _EV_EOF       = 0x8000
        _EVFILT_READ  = -0x1
        _EVFILT_WRITE = -0x2
+       _EVFILT_USER  = -0xb
+
+       _NOTE_TRIGGER = 0x1000000
 )
 
 type rtprio struct {
index 43923e3075b7aa774ea32a5993b65c5c6d6c2559..ec4d796bd32568136f2528c40fb5fbb5cd5d1ac2 100644 (file)
@@ -110,12 +110,17 @@ const (
 
        EV_ADD       = C.EV_ADD
        EV_DELETE    = C.EV_DELETE
+       EV_ENABLE    = C.EV_ENABLE
+       EV_DISABLE   = C.EV_DISABLE
        EV_CLEAR     = C.EV_CLEAR
        EV_RECEIPT   = 0
        EV_ERROR     = C.EV_ERROR
        EV_EOF       = C.EV_EOF
        EVFILT_READ  = C.EVFILT_READ
        EVFILT_WRITE = C.EVFILT_WRITE
+       EVFILT_USER  = C.EVFILT_USER
+
+       NOTE_TRIGGER = C.NOTE_TRIGGER
 )
 
 type Sigset C.sigset_t
index 32c21a2b2bec0f43250df4b99e22cd0d9a268ae9..6cd80d5c30d16e855c6b69b0814ae3bff1d5f676 100644 (file)
@@ -15,10 +15,7 @@ import (
 )
 
 var (
-       kq int32 = -1
-
-       netpollBreakRd, netpollBreakWr uintptr // for netpollBreak
-
+       kq             int32         = -1
        netpollWakeSig atomic.Uint32 // used to avoid duplicate calls of netpollBreak
 )
 
@@ -29,27 +26,7 @@ func netpollinit() {
                throw("runtime: netpollinit failed")
        }
        closeonexec(kq)
-       r, w, errno := nonblockingPipe()
-       if errno != 0 {
-               println("runtime: pipe failed with", -errno)
-               throw("runtime: pipe failed")
-       }
-       ev := keventt{
-               filter: _EVFILT_READ,
-               flags:  _EV_ADD,
-       }
-       *(*uintptr)(unsafe.Pointer(&ev.ident)) = uintptr(r)
-       n := kevent(kq, &ev, 1, nil, 0, nil)
-       if n < 0 {
-               println("runtime: kevent failed with", -n)
-               throw("runtime: kevent failed")
-       }
-       netpollBreakRd = uintptr(r)
-       netpollBreakWr = uintptr(w)
-}
-
-func netpollIsPollDescriptor(fd uintptr) bool {
-       return fd == uintptr(kq) || fd == netpollBreakRd || fd == netpollBreakWr
+       addWakeupEvent(kq)
 }
 
 func netpollopen(fd uintptr, pd *pollDesc) int32 {
@@ -99,18 +76,7 @@ func netpollBreak() {
                return
        }
 
-       for {
-               var b byte
-               n := write(netpollBreakWr, unsafe.Pointer(&b), 1)
-               if n == 1 || n == -_EAGAIN {
-                       break
-               }
-               if n == -_EINTR {
-                       continue
-               }
-               println("runtime: netpollBreak write failed with", -n)
-               throw("runtime: netpollBreak write failed")
-       }
+       wakeNetpoll(kq)
 }
 
 // netpoll checks for ready network connections.
@@ -159,17 +125,11 @@ retry:
        for i := 0; i < int(n); i++ {
                ev := &events[i]
 
-               if uintptr(ev.ident) == netpollBreakRd {
-                       if ev.filter != _EVFILT_READ {
-                               println("runtime: netpoll: break fd ready for", ev.filter)
-                               throw("runtime: netpoll: break fd ready for something unexpected")
-                       }
+               if isWakeup(ev) {
                        if delay != 0 {
-                               // netpollBreak could be picked up by a
-                               // nonblocking poll. Only read the byte
-                               // if blocking.
-                               var tmp [16]byte
-                               read(int32(netpollBreakRd), noescape(unsafe.Pointer(&tmp[0])), int32(len(tmp)))
+                               // netpollBreak could be picked up by a nonblocking poll.
+                               // Only call drainWakeupEvent and reset the netpollWakeSig if blocking.
+                               drainWakeupEvent(kq)
                                netpollWakeSig.Store(0)
                        }
                        continue
diff --git a/src/runtime/netpoll_kqueue_event.go b/src/runtime/netpoll_kqueue_event.go
new file mode 100644 (file)
index 0000000..6419656
--- /dev/null
@@ -0,0 +1,80 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build darwin || dragonfly || freebsd
+
+package runtime
+
+// Magic number of identifier used for EVFILT_USER.
+// This number had zero Google results when it's created.
+// That way, people will be directed here when this number
+// get printed somehow and they search for it.
+const kqIdent = 0xee1eb9f4
+
+func addWakeupEvent(_ int32) {
+       ev := keventt{
+               ident:  kqIdent,
+               filter: _EVFILT_USER,
+               flags:  _EV_ADD,
+       }
+       for {
+               n := kevent(kq, &ev, 1, nil, 0, nil)
+               if n == 0 {
+                       break
+               }
+               if n == -_EINTR {
+                       // All changes contained in the changelist should have been applied
+                       // before returning EINTR. But let's be skeptical and retry it anyway,
+                       // to make a 100% commitment.
+                       continue
+               }
+               println("runtime: kevent for EVFILT_USER failed with", -n)
+               throw("runtime: kevent failed")
+       }
+}
+
+func wakeNetpoll(kq int32) {
+       ev := keventt{
+               ident:  kqIdent,
+               filter: _EVFILT_USER,
+               flags:  _EV_ENABLE,
+               fflags: _NOTE_TRIGGER,
+       }
+       for {
+               n := kevent(kq, &ev, 1, nil, 0, nil)
+               if n == 0 {
+                       break
+               }
+               if n == -_EINTR {
+                       // Check out the comment in addWakeupEvent.
+                       continue
+               }
+               println("runtime: netpollBreak write failed with", -n)
+               throw("runtime: netpollBreak write failed")
+       }
+}
+
+func isWakeup(ev *keventt) bool {
+       if ev.filter == _EVFILT_USER {
+               if ev.ident == kqIdent {
+                       return true
+               }
+               println("runtime: netpoll: break fd ready for", ev.ident)
+               throw("runtime: netpoll: break fd ready for something unexpected")
+       }
+       return false
+}
+
+func drainWakeupEvent(kq int32) {
+       ev := keventt{
+               ident:  kqIdent,
+               filter: _EVFILT_USER,
+               flags:  _EV_DISABLE,
+       }
+       kevent(kq, &ev, 1, nil, 0, nil)
+}
+
+func netpollIsPollDescriptor(fd uintptr) bool {
+       return fd == uintptr(kq)
+}
diff --git a/src/runtime/netpoll_kqueue_pipe.go b/src/runtime/netpoll_kqueue_pipe.go
new file mode 100644 (file)
index 0000000..98f73e8
--- /dev/null
@@ -0,0 +1,73 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build netbsd || openbsd
+
+package runtime
+
+import "unsafe"
+
+// TODO(panjf2000): NetBSD didn't implement EVFILT_USER for user-established events
+// until NetBSD 10.0, check out https://www.netbsd.org/releases/formal-10/NetBSD-10.0.html
+// Therefore we use the pipe to wake up the kevent on NetBSD at this point. Get back here
+// and switch to EVFILT_USER when we bump up the minimal requirement of NetBSD to 10.0.
+// Alternatively, maybe we can use EVFILT_USER on the NetBSD by checking the kernel version
+// via uname(3) and fall back to the pipe if the kernel version is older than 10.0.
+
+var netpollBreakRd, netpollBreakWr uintptr // for netpollBreak
+
+func addWakeupEvent(kq int32) {
+       r, w, errno := nonblockingPipe()
+       if errno != 0 {
+               println("runtime: pipe failed with", -errno)
+               throw("runtime: pipe failed")
+       }
+       ev := keventt{
+               filter: _EVFILT_READ,
+               flags:  _EV_ADD,
+       }
+       *(*uintptr)(unsafe.Pointer(&ev.ident)) = uintptr(r)
+       n := kevent(kq, &ev, 1, nil, 0, nil)
+       if n < 0 {
+               println("runtime: kevent failed with", -n)
+               throw("runtime: kevent failed")
+       }
+       netpollBreakRd = uintptr(r)
+       netpollBreakWr = uintptr(w)
+}
+
+func wakeNetpoll(_ int32) {
+       for {
+               var b byte
+               n := write(netpollBreakWr, unsafe.Pointer(&b), 1)
+               if n == 1 || n == -_EAGAIN {
+                       break
+               }
+               if n == -_EINTR {
+                       continue
+               }
+               println("runtime: netpollBreak write failed with", -n)
+               throw("runtime: netpollBreak write failed")
+       }
+}
+
+func isWakeup(ev *keventt) bool {
+       if uintptr(ev.ident) == netpollBreakRd {
+               if ev.filter == _EVFILT_READ {
+                       return true
+               }
+               println("runtime: netpoll: break fd ready for", ev.filter)
+               throw("runtime: netpoll: break fd ready for something unexpected")
+       }
+       return false
+}
+
+func drainWakeupEvent(_ int32) {
+       var buf [16]byte
+       read(int32(netpollBreakRd), noescape(unsafe.Pointer(&buf[0])), int32(len(buf)))
+}
+
+func netpollIsPollDescriptor(fd uintptr) bool {
+       return fd == uintptr(kq) || fd == netpollBreakRd || fd == netpollBreakWr
+}