]> Cypherpunks repositories - gostls13.git/commitdiff
runtime: use eventfd as the event wait/notify mechanism for epoll
authorAndy Pan <panjf2000@gmail.com>
Fri, 2 Feb 2024 03:22:57 +0000 (11:22 +0800)
committerGopher Robot <gobot@golang.org>
Tue, 20 Feb 2024 14:59:49 +0000 (14:59 +0000)
Fixes #65443

Change-Id: I9ad4689b36e87ee930d35a38322a8797896483b4
Reviewed-on: https://go-review.googlesource.com/c/go/+/560615
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Than McIntosh <thanm@google.com>
Auto-Submit: Ian Lance Taylor <iant@google.com>
Reviewed-by: Jorropo <jorropo.pgm@gmail.com>
Reviewed-by: Ian Lance Taylor <iant@google.com>
13 files changed:
src/runtime/internal/syscall/defs_linux.go [new file with mode: 0644]
src/runtime/internal/syscall/defs_linux_386.go
src/runtime/internal/syscall/defs_linux_amd64.go
src/runtime/internal/syscall/defs_linux_arm.go
src/runtime/internal/syscall/defs_linux_arm64.go
src/runtime/internal/syscall/defs_linux_loong64.go
src/runtime/internal/syscall/defs_linux_mips64x.go
src/runtime/internal/syscall/defs_linux_mipsx.go
src/runtime/internal/syscall/defs_linux_ppc64x.go
src/runtime/internal/syscall/defs_linux_riscv64.go
src/runtime/internal/syscall/defs_linux_s390x.go
src/runtime/internal/syscall/syscall_linux.go
src/runtime/netpoll_epoll.go

diff --git a/src/runtime/internal/syscall/defs_linux.go b/src/runtime/internal/syscall/defs_linux.go
new file mode 100644 (file)
index 0000000..242a67b
--- /dev/null
@@ -0,0 +1,9 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+       EFD_CLOEXEC = 0x80000
+)
index dc723a60b24482758e6b70ed64697ed3aacf4734..fa0ca5ed0aee95987b1fda229b0f32ca6fe573a4 100644 (file)
@@ -10,6 +10,7 @@ const (
        SYS_EPOLL_PWAIT   = 319
        SYS_EPOLL_CREATE1 = 329
        SYS_EPOLL_PWAIT2  = 441
+       SYS_EVENTFD2      = 328
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -21,6 +22,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x800
 )
 
 type EpollEvent struct {
index 886eb5bda2501a90745357e8a0a7d1f225836f85..71bd906ed45e78368935b13a124fe6c763131a0c 100644 (file)
@@ -10,6 +10,7 @@ const (
        SYS_EPOLL_PWAIT   = 281
        SYS_EPOLL_CREATE1 = 291
        SYS_EPOLL_PWAIT2  = 441
+       SYS_EVENTFD2      = 290
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -21,6 +22,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x800
 )
 
 type EpollEvent struct {
index 8f812a2f68b5c6cb95fbbeeb449056c4fbdccb91..146cda5fcbd998126020a992835a99499620430d 100644 (file)
@@ -10,6 +10,7 @@ const (
        SYS_EPOLL_PWAIT   = 346
        SYS_EPOLL_CREATE1 = 357
        SYS_EPOLL_PWAIT2  = 441
+       SYS_EVENTFD2      = 356
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -21,6 +22,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x800
 )
 
 type EpollEvent struct {
index 48e11b0c512f2b7577fb44af5b4838776b8fdb17..0500b317e9a24544954f608b96a09fc66b2711b4 100644 (file)
@@ -10,6 +10,7 @@ const (
        SYS_EPOLL_PWAIT   = 22
        SYS_FCNTL         = 25
        SYS_EPOLL_PWAIT2  = 441
+       SYS_EVENTFD2      = 19
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -21,6 +22,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x800
 )
 
 type EpollEvent struct {
index b78ef818614cf89dd372974461ba0f997ad45eaa..2c67b7b5689e940d88a7351796e8d3de515cc8fb 100644 (file)
@@ -10,6 +10,7 @@ const (
        SYS_EPOLL_PWAIT   = 22
        SYS_FCNTL         = 25
        SYS_EPOLL_PWAIT2  = 441
+       SYS_EVENTFD2      = 19
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -21,6 +22,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x800
 )
 
 type EpollEvent struct {
index 92b49ca969cb1df99878ed055ed51138f6b42857..3031af241bece67fc7c23d0aa4ee211fdc14b99f 100644 (file)
@@ -12,6 +12,7 @@ const (
        SYS_EPOLL_PWAIT   = 5272
        SYS_EPOLL_CREATE1 = 5285
        SYS_EPOLL_PWAIT2  = 5441
+       SYS_EVENTFD2      = 5284
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -23,6 +24,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x80
 )
 
 type EpollEvent struct {
index e28d09c7f135ae4b184cf8051a297abe90c76d9a..4d51921abd63ef2f7acfa1679175b4c6ea78684e 100644 (file)
@@ -12,6 +12,7 @@ const (
        SYS_EPOLL_PWAIT   = 4313
        SYS_EPOLL_CREATE1 = 4326
        SYS_EPOLL_PWAIT2  = 4441
+       SYS_EVENTFD2      = 4325
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -23,6 +24,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x80
 )
 
 type EpollEvent struct {
index a74483eb6d713b09e6508648766a608a6af840d2..fcd15e60a2b1c3789c3334a0710ad668e7efbd5d 100644 (file)
@@ -12,6 +12,7 @@ const (
        SYS_EPOLL_PWAIT   = 303
        SYS_EPOLL_CREATE1 = 315
        SYS_EPOLL_PWAIT2  = 441
+       SYS_EVENTFD2      = 314
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -23,6 +24,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x800
 )
 
 type EpollEvent struct {
index b78ef818614cf89dd372974461ba0f997ad45eaa..2c67b7b5689e940d88a7351796e8d3de515cc8fb 100644 (file)
@@ -10,6 +10,7 @@ const (
        SYS_EPOLL_PWAIT   = 22
        SYS_FCNTL         = 25
        SYS_EPOLL_PWAIT2  = 441
+       SYS_EVENTFD2      = 19
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -21,6 +22,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x800
 )
 
 type EpollEvent struct {
index a7bb1ba66d666798fe3d889add8eceafe4c4a637..c1deed6c7410d1892445636ad6d2f05126395bd8 100644 (file)
@@ -10,6 +10,7 @@ const (
        SYS_EPOLL_PWAIT   = 312
        SYS_EPOLL_CREATE1 = 327
        SYS_EPOLL_PWAIT2  = 441
+       SYS_EVENTFD2      = 323
 
        EPOLLIN       = 0x1
        EPOLLOUT      = 0x4
@@ -21,6 +22,7 @@ const (
        EPOLL_CTL_ADD = 0x1
        EPOLL_CTL_DEL = 0x2
        EPOLL_CTL_MOD = 0x3
+       EFD_NONBLOCK  = 0x800
 )
 
 type EpollEvent struct {
index 7209634edb3f81380ee9613f0df59c4af62caeb3..cb88659394fd924c4ae55baddce796b0d298891c 100644 (file)
@@ -60,3 +60,8 @@ func EpollCtl(epfd, op, fd int32, event *EpollEvent) (errno uintptr) {
        _, _, e := Syscall6(SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)
        return e
 }
+
+func Eventfd(initval, flags int32) (fd int32, errno uintptr) {
+       r1, _, e := Syscall6(SYS_EVENTFD2, uintptr(initval), uintptr(flags), 0, 0, 0, 0)
+       return int32(r1), e
+}
index cda19fbc278790de815aad1616f33a76a65ec401..a8062b7dc389d67820dc05de67ce393eb4d4d3a6 100644 (file)
@@ -13,11 +13,9 @@ import (
 )
 
 var (
-       epfd int32 = -1 // epoll descriptor
-
-       netpollBreakRd, netpollBreakWr uintptr // for netpollBreak
-
-       netpollWakeSig atomic.Uint32 // used to avoid duplicate calls of netpollBreak
+       epfd           int32         = -1 // epoll descriptor
+       netpollEventFd uintptr            // eventfd for netpollBreak
+       netpollWakeSig atomic.Uint32      // used to avoid duplicate calls of netpollBreak
 )
 
 func netpollinit() {
@@ -27,26 +25,25 @@ func netpollinit() {
                println("runtime: epollcreate failed with", errno)
                throw("runtime: netpollinit failed")
        }
-       r, w, errpipe := nonblockingPipe()
-       if errpipe != 0 {
-               println("runtime: pipe failed with", -errpipe)
-               throw("runtime: pipe failed")
+       efd, errno := syscall.Eventfd(0, syscall.EFD_CLOEXEC|syscall.EFD_NONBLOCK)
+       if errno != 0 {
+               println("runtime: eventfd failed with", -errno)
+               throw("runtime: eventfd failed")
        }
        ev := syscall.EpollEvent{
                Events: syscall.EPOLLIN,
        }
-       *(**uintptr)(unsafe.Pointer(&ev.Data)) = &netpollBreakRd
-       errno = syscall.EpollCtl(epfd, syscall.EPOLL_CTL_ADD, r, &ev)
+       *(**uintptr)(unsafe.Pointer(&ev.Data)) = &netpollEventFd
+       errno = syscall.EpollCtl(epfd, syscall.EPOLL_CTL_ADD, efd, &ev)
        if errno != 0 {
                println("runtime: epollctl failed with", errno)
                throw("runtime: epollctl failed")
        }
-       netpollBreakRd = uintptr(r)
-       netpollBreakWr = uintptr(w)
+       netpollEventFd = uintptr(efd)
 }
 
 func netpollIsPollDescriptor(fd uintptr) bool {
-       return fd == uintptr(epfd) || fd == netpollBreakRd || fd == netpollBreakWr
+       return fd == uintptr(epfd) || fd == netpollEventFd
 }
 
 func netpollopen(fd uintptr, pd *pollDesc) uintptr {
@@ -73,10 +70,11 @@ func netpollBreak() {
                return
        }
 
+       var one uint64 = 1
+       oneSize := int32(unsafe.Sizeof(one))
        for {
-               var b byte
-               n := write(netpollBreakWr, unsafe.Pointer(&b), 1)
-               if n == 1 {
+               n := write(netpollEventFd, noescape(unsafe.Pointer(&one)), oneSize)
+               if n == oneSize {
                        break
                }
                if n == -_EINTR {
@@ -136,17 +134,19 @@ retry:
                        continue
                }
 
-               if *(**uintptr)(unsafe.Pointer(&ev.Data)) == &netpollBreakRd {
+               if *(**uintptr)(unsafe.Pointer(&ev.Data)) == &netpollEventFd {
                        if ev.Events != syscall.EPOLLIN {
-                               println("runtime: netpoll: break fd ready for", ev.Events)
-                               throw("runtime: netpoll: break fd ready for something unexpected")
+                               println("runtime: netpoll: eventfd ready for", ev.Events)
+                               throw("runtime: netpoll: eventfd ready for something unexpected")
                        }
                        if delay != 0 {
                                // netpollBreak could be picked up by a
-                               // nonblocking poll. Only read the byte
-                               // if blocking.
-                               var tmp [16]byte
-                               read(int32(netpollBreakRd), noescape(unsafe.Pointer(&tmp[0])), int32(len(tmp)))
+                               // nonblocking poll. Only read the 8-byte
+                               // integer if blocking.
+                               // Since EFD_SEMAPHORE was not specified,
+                               // the eventfd counter will be reset to 0.
+                               var one uint64
+                               read(int32(netpollEventFd), noescape(unsafe.Pointer(&one)), int32(unsafe.Sizeof(one)))
                                netpollWakeSig.Store(0)
                        }
                        continue