From: Shenghou Ma Date: Fri, 15 Mar 2013 20:01:56 +0000 (+0800) Subject: net, runtime: enable runtime-integrated pollster on Linux/ARM. X-Git-Tag: go1.1rc2~479 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=1d64d04da5f54f2fc0c9801c908380b633ea67d9;p=gostls13.git net, runtime: enable runtime-integrated pollster on Linux/ARM. Results from linux/arm on a Samsung Chromebook (from dfc): localhost(~/go/src/pkg/net) % ~/go/misc/benchcmp {old,new}.txt benchmark old ns/op new ns/op delta BenchmarkTCP4OneShot 568840 350526 -38.38% BenchmarkTCP4OneShot-2 359054 206708 -42.43% BenchmarkTCP4OneShotTimeout 637464 363550 -42.97% BenchmarkTCP4OneShotTimeout-2 374255 216695 -42.10% BenchmarkTCP4Persistent 184974 64984 -64.87% BenchmarkTCP4Persistent-2 109902 47195 -57.06% BenchmarkTCP4PersistentTimeout 210039 64789 -69.15% BenchmarkTCP4PersistentTimeout-2 124284 43374 -65.10% BenchmarkTCP6OneShot 672278 362116 -46.14% BenchmarkTCP6OneShot-2 383631 216400 -43.59% BenchmarkTCP6OneShotTimeout 680740 378306 -44.43% BenchmarkTCP6OneShotTimeout-2 397524 230152 -42.10% BenchmarkTCP6Persistent 172346 65292 -62.12% BenchmarkTCP6Persistent-2 106229 42096 -60.37% BenchmarkTCP6PersistentTimeout 161149 65138 -59.58% BenchmarkTCP6PersistentTimeout-2 152276 44548 -70.75% R=golang-dev, dave, bradfitz, dvyukov, rsc CC=golang-dev https://golang.org/cl/7820045 --- diff --git a/src/pkg/net/fd_linux.go b/src/pkg/net/fd_linux.go deleted file mode 100644 index b83d0348c1..0000000000 --- a/src/pkg/net/fd_linux.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build arm - -// Waiting for FDs via epoll(7). - -package net - -import ( - "os" - "syscall" -) - -const ( - readFlags = syscall.EPOLLIN | syscall.EPOLLRDHUP - writeFlags = syscall.EPOLLOUT -) - -type pollster struct { - epfd int - - // Events we're already waiting for - // Must hold pollServer lock - events map[int]uint32 - - // An event buffer for EpollWait. - // Used without a lock, may only be used by WaitFD. - waitEventBuf [10]syscall.EpollEvent - waitEvents []syscall.EpollEvent - - // An event buffer for EpollCtl, to avoid a malloc. - // Must hold pollServer lock. - ctlEvent syscall.EpollEvent -} - -func newpollster() (p *pollster, err error) { - p = new(pollster) - if p.epfd, err = syscall.EpollCreate1(syscall.EPOLL_CLOEXEC); err != nil { - if err != syscall.ENOSYS { - return nil, os.NewSyscallError("epoll_create1", err) - } - // The arg to epoll_create is a hint to the kernel - // about the number of FDs we will care about. - // We don't know, and since 2.6.8 the kernel ignores it anyhow. - if p.epfd, err = syscall.EpollCreate(16); err != nil { - return nil, os.NewSyscallError("epoll_create", err) - } - syscall.CloseOnExec(p.epfd) - } - p.events = make(map[int]uint32) - return p, nil -} - -// First return value is whether the pollServer should be woken up. -// This version always returns false. -func (p *pollster) AddFD(fd int, mode int, repeat bool) (bool, error) { - // pollServer is locked. - - var already bool - p.ctlEvent.Fd = int32(fd) - p.ctlEvent.Events, already = p.events[fd] - if !repeat { - p.ctlEvent.Events |= syscall.EPOLLONESHOT - } - if mode == 'r' { - p.ctlEvent.Events |= readFlags - } else { - p.ctlEvent.Events |= writeFlags - } - - var op int - if already { - op = syscall.EPOLL_CTL_MOD - } else { - op = syscall.EPOLL_CTL_ADD - } - if err := syscall.EpollCtl(p.epfd, op, fd, &p.ctlEvent); err != nil { - return false, os.NewSyscallError("epoll_ctl", err) - } - p.events[fd] = p.ctlEvent.Events - return false, nil -} - -func (p *pollster) StopWaiting(fd int, bits uint) { - // pollServer is locked. - - events, already := p.events[fd] - if !already { - // The fd returned by the kernel may have been - // cancelled already; return silently. - return - } - - // If syscall.EPOLLONESHOT is not set, the wait - // is a repeating wait, so don't change it. - if events&syscall.EPOLLONESHOT == 0 { - return - } - - // Disable the given bits. - // If we're still waiting for other events, modify the fd - // event in the kernel. Otherwise, delete it. - events &= ^uint32(bits) - if int32(events)&^syscall.EPOLLONESHOT != 0 { - p.ctlEvent.Fd = int32(fd) - p.ctlEvent.Events = events - if err := syscall.EpollCtl(p.epfd, syscall.EPOLL_CTL_MOD, fd, &p.ctlEvent); err != nil { - print("Epoll modify fd=", fd, ": ", err.Error(), "\n") - } - p.events[fd] = events - } else { - if err := syscall.EpollCtl(p.epfd, syscall.EPOLL_CTL_DEL, fd, nil); err != nil { - print("Epoll delete fd=", fd, ": ", err.Error(), "\n") - } - delete(p.events, fd) - } -} - -// Return value is whether the pollServer should be woken up. -// This version always returns false. -func (p *pollster) DelFD(fd int, mode int) bool { - // pollServer is locked. - - if mode == 'r' { - p.StopWaiting(fd, readFlags) - } else { - p.StopWaiting(fd, writeFlags) - } - - // Discard any queued up events. - i := 0 - for i < len(p.waitEvents) { - if fd == int(p.waitEvents[i].Fd) { - copy(p.waitEvents[i:], p.waitEvents[i+1:]) - p.waitEvents = p.waitEvents[:len(p.waitEvents)-1] - } else { - i++ - } - } - return false -} - -func (p *pollster) WaitFD(s *pollServer, nsec int64) (fd int, mode int, err error) { - for len(p.waitEvents) == 0 { - var msec int = -1 - if nsec > 0 { - msec = int((nsec + 1e6 - 1) / 1e6) - } - - s.Unlock() - n, err := syscall.EpollWait(p.epfd, p.waitEventBuf[0:], msec) - s.Lock() - - if err != nil { - if err == syscall.EAGAIN || err == syscall.EINTR { - continue - } - return -1, 0, os.NewSyscallError("epoll_wait", err) - } - if n == 0 { - return -1, 0, nil - } - p.waitEvents = p.waitEventBuf[0:n] - } - - ev := &p.waitEvents[0] - p.waitEvents = p.waitEvents[1:] - - fd = int(ev.Fd) - - if ev.Events&writeFlags != 0 { - p.StopWaiting(fd, writeFlags) - return fd, 'w', nil - } - if ev.Events&readFlags != 0 { - p.StopWaiting(fd, readFlags) - return fd, 'r', nil - } - - // Other events are error conditions - wake whoever is waiting. - events, _ := p.events[fd] - if events&writeFlags != 0 { - p.StopWaiting(fd, writeFlags) - return fd, 'w', nil - } - p.StopWaiting(fd, readFlags) - return fd, 'r', nil -} - -func (p *pollster) Close() error { - return os.NewSyscallError("close", syscall.Close(p.epfd)) -} diff --git a/src/pkg/net/fd_poll_runtime.go b/src/pkg/net/fd_poll_runtime.go index 999d0215b9..e3b4f7e464 100644 --- a/src/pkg/net/fd_poll_runtime.go +++ b/src/pkg/net/fd_poll_runtime.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin linux,386 linux,amd64 +// +build darwin linux package net diff --git a/src/pkg/net/fd_poll_unix.go b/src/pkg/net/fd_poll_unix.go index cacdba25eb..307e577e99 100644 --- a/src/pkg/net/fd_poll_unix.go +++ b/src/pkg/net/fd_poll_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build freebsd linux,arm netbsd openbsd +// +build freebsd netbsd openbsd package net diff --git a/src/pkg/runtime/defs_linux_arm.h b/src/pkg/runtime/defs_linux_arm.h index 82442b23e0..92160966e1 100644 --- a/src/pkg/runtime/defs_linux_arm.h +++ b/src/pkg/runtime/defs_linux_arm.h @@ -2,6 +2,7 @@ // Constants enum { + EINTR = 0x4, ENOMEM = 0xc, EAGAIN = 0xb, @@ -65,6 +66,17 @@ enum { ITIMER_VIRTUAL = 0x1, O_RDONLY = 0, O_CLOEXEC = 02000000, + + EPOLLIN = 0x1, + EPOLLOUT = 0x4, + EPOLLERR = 0x8, + EPOLLHUP = 0x10, + EPOLLRDHUP = 0x2000, + EPOLLET = -0x80000000, + EPOLL_CLOEXEC = 0x80000, + EPOLL_CTL_ADD = 0x1, + EPOLL_CTL_DEL = 0x2, + EPOLL_CTL_MOD = 0x3, }; // Types @@ -146,4 +158,11 @@ struct Sigaction { void *sa_restorer; uint64 sa_mask; }; + +typedef struct EpollEvent EpollEvent; +struct EpollEvent { + uint32 events; + uint32 _pad; + uint64 data; +}; #pragma pack off diff --git a/src/pkg/runtime/netpoll.goc b/src/pkg/runtime/netpoll.goc index 152fa174e9..b314c65338 100644 --- a/src/pkg/runtime/netpoll.goc +++ b/src/pkg/runtime/netpoll.goc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin linux,386 linux,amd64 +// +build darwin linux package net diff --git a/src/pkg/runtime/netpoll_epoll.c b/src/pkg/runtime/netpoll_epoll.c index 53916e1d2e..34ed78addb 100644 --- a/src/pkg/runtime/netpoll_epoll.c +++ b/src/pkg/runtime/netpoll_epoll.c @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build linux,386 linux,amd64 +// +build linux #include "runtime.h" #include "defs_GOOS_GOARCH.h" diff --git a/src/pkg/runtime/netpoll_stub.c b/src/pkg/runtime/netpoll_stub.c index a70b2f772f..39d19a4cea 100644 --- a/src/pkg/runtime/netpoll_stub.c +++ b/src/pkg/runtime/netpoll_stub.c @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build freebsd linux,arm netbsd openbsd plan9 windows +// +build freebsd netbsd openbsd plan9 windows #include "runtime.h" diff --git a/src/pkg/runtime/sys_linux_arm.s b/src/pkg/runtime/sys_linux_arm.s index e3994bccaa..7f813482d1 100644 --- a/src/pkg/runtime/sys_linux_arm.s +++ b/src/pkg/runtime/sys_linux_arm.s @@ -36,6 +36,11 @@ #define SYS_ugetrlimit (SYS_BASE + 191) #define SYS_sched_getaffinity (SYS_BASE + 242) #define SYS_clock_gettime (SYS_BASE + 263) +#define SYS_epoll_create (SYS_BASE + 250) +#define SYS_epoll_ctl (SYS_BASE + 251) +#define SYS_epoll_wait (SYS_BASE + 252) +#define SYS_epoll_create1 (SYS_BASE + 357) +#define SYS_fcntl (SYS_BASE + 55) #define ARM_BASE (SYS_BASE + 0x0f0000) @@ -371,7 +376,6 @@ cascheck: MOVW $0, R0 RET - TEXT runtime·casp(SB),7,$0 B runtime·cas(SB) @@ -387,3 +391,46 @@ TEXT runtime·sched_getaffinity(SB),7,$0 MOVW $SYS_sched_getaffinity, R7 SWI $0 RET + +// int32 runtime·epollcreate(int32 size) +TEXT runtime·epollcreate(SB),7,$0 + MOVW 0(FP), R0 + MOVW $SYS_epoll_create, R7 + SWI $0 + RET + +// int32 runtime·epollcreate1(int32 flags) +TEXT runtime·epollcreate1(SB),7,$0 + MOVW 0(FP), R0 + MOVW $SYS_epoll_create1, R7 + SWI $0 + RET + +// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev) +TEXT runtime·epollctl(SB),7,$0 + MOVW 0(FP), R0 + MOVW 4(FP), R1 + MOVW 8(FP), R2 + MOVW 12(FP), R3 + MOVW $SYS_epoll_ctl, R7 + SWI $0 + RET + +// int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout) +TEXT runtime·epollwait(SB),7,$0 + MOVW 0(FP), R0 + MOVW 4(FP), R1 + MOVW 8(FP), R2 + MOVW 12(FP), R3 + MOVW $SYS_epoll_wait, R7 + SWI $0 + RET + +// void runtime·closeonexec(int32 fd) +TEXT runtime·closeonexec(SB),7,$0 + MOVW 0(FP), R0 // fd + MOVW $2, R1 // F_SETFD + MOVW $1, R2 // FD_CLOEXEC + MOVW $SYS_fcntl, R7 + SWI $0 + RET