From b054c7dc1738c810e74756ae0ac4797ce5d31cf6 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 14 Mar 2022 11:27:46 -0400 Subject: [PATCH] os: raise open file rlimit at startup Some systems set an artificially low soft limit on open file count, for compatibility with code that uses select and its hard-coded maximum file descriptor (limited by the size of fd_set). Go does not use select, so it should not be subject to these limits. On some systems the limit is 256, which is very easy to run into, even in simple programs like gofmt when they parallelize walking a file tree. After a long discussion on go.dev/issue/46279, we decided the best approach was for Go to raise the limit unconditionally for itself, and then leave old software to set the limit back as needed. Code that really wants Go to leave the limit alone can set the hard limit, which Go of course has no choice but to respect. Fixes #46279. Change-Id: Id6107503437d47a870a41be25e822fc79cea08b2 Reviewed-on: https://go-review.googlesource.com/c/go/+/392415 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor --- src/os/rlimit.go | 31 +++++++++++++++++++++++++++++++ src/os/rlimit_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 src/os/rlimit.go create mode 100644 src/os/rlimit_test.go diff --git a/src/os/rlimit.go b/src/os/rlimit.go new file mode 100644 index 0000000000..3e29db9562 --- /dev/null +++ b/src/os/rlimit.go @@ -0,0 +1,31 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package os + +import "syscall" + +// Some systems set an artificially low soft limit on open file count, for compatibility +// with code that uses select and its hard-coded maximum file descriptor +// (limited by the size of fd_set). +// +// Go does not use select, so it should not be subject to these limits. +// On some systems the limit is 256, which is very easy to run into, +// even in simple programs like gofmt when they parallelize walking +// a file tree. +// +// After a long discussion on go.dev/issue/46279, we decided the +// best approach was for Go to raise the limit unconditionally for itself, +// and then leave old software to set the limit back as needed. +// Code that really wants Go to leave the limit alone can set the hard limit, +// which Go of course has no choice but to respect. +func init() { + var lim syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err == nil && lim.Cur != lim.Max { + lim.Cur = lim.Max + syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim) + } +} diff --git a/src/os/rlimit_test.go b/src/os/rlimit_test.go new file mode 100644 index 0000000000..9bb6858a86 --- /dev/null +++ b/src/os/rlimit_test.go @@ -0,0 +1,32 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + . "os" + "testing" +) + +func TestOpenFileLimit(t *testing.T) { + // For open file count, + // macOS sets the default soft limit to 256 and no hard limit. + // CentOS and Fedora set the default soft limit to 1024, + // with hard limits of 4096 and 524288, respectively. + // Check that we can open 1200 files, which proves + // that the rlimit is being raised appropriately on those systems. + var files []*File + for i := 0; i < 1200; i++ { + f, err := Open("rlimit.go") + if err != nil { + t.Error(err) + break + } + files = append(files, f) + } + + for _, f := range files { + f.Close() + } +} -- 2.50.0