net: limit concurrent threads to limit on file descriptors

At least on Darwin, if getaddrinfo can't open a file descriptor it
returns EAI_NONAME ("no such host") rather than a meaningful error.
Limit the number of concurrent getaddrinfo calls to the number of file
descriptors we can open, to make that meaningless error less likely.

We don't apply the same limit to Go lookups, because for that we will
return a meaningful "too many open files" error.

Fixes #25694

Change-Id: I601857190aeb64f11e22b4a834c1c6a722a0788d
Reviewed-on: https://go-review.googlesource.com/121176
Reviewed-by: Bryan C. Mills <bcmills@google.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

diff --git a/src/net/lookup_fake.go b/src/net/lookup_fake.go
index 90c6d4718315357a7fbbd8ff38a86490027773b2..d3d1dbc90032f743641eaad1dc56966a8c30a9ba 100644 (file)
--- a/src/net/lookup_fake.go
+++ b/src/net/lookup_fake.go
@@ -50,3 +50,9 @@ func (*Resolver) lookupTXT(ctx context.Context, name string) (txts []string, err
 func (*Resolver) lookupAddr(ctx context.Context, addr string) (ptrs []string, err error) {
        return nil, syscall.ENOPROTOOPT
 }
+
+// concurrentThreadsLimit returns the number of threads we permit to
+// run concurrently doing DNS lookups.
+func concurrentThreadsLimit() int {
+       return 500
+}

diff --git a/src/net/lookup_plan9.go b/src/net/lookup_plan9.go
index e0b38c69b99ddd0d0c050e566748a6f8b881313a..5547f0b0eeb3a19b06d0f76dc76057505c1e731f 100644 (file)
--- a/src/net/lookup_plan9.go
+++ b/src/net/lookup_plan9.go
@@ -329,3 +329,9 @@ func (*Resolver) lookupAddr(ctx context.Context, addr string) (name []string, er
        }
        return
 }
+
+// concurrentThreadsLimit returns the number of threads we permit to
+// run concurrently doing DNS lookups.
+func concurrentThreadsLimit() int {
+       return 500
+}

diff --git a/src/net/lookup_unix.go b/src/net/lookup_unix.go
index 0cf4c99e0c7c4c5c2920a8c1324ddc75159f0152..2c3191aca8a66801c51acab72f18479e7d799dc9 100644 (file)
--- a/src/net/lookup_unix.go
+++ b/src/net/lookup_unix.go
@@ -9,6 +9,7 @@ package net
 import (
        "context"
        "sync"
+       "syscall"
 
        "golang_org/x/net/dns/dnsmessage"
 )
@@ -315,3 +316,27 @@ func (r *Resolver) lookupAddr(ctx context.Context, addr string) ([]string, error
        }
        return r.goLookupPTR(ctx, addr)
 }
+
+// concurrentThreadsLimit returns the number of threads we permit to
+// run concurrently doing DNS lookups via cgo. A DNS lookup may use a
+// file descriptor so we limit this to less than the number of
+// permitted open files. On some systems, notably Darwin, if
+// getaddrinfo is unable to open a file descriptor it simply returns
+// EAI_NONAME rather than a useful error. Limiting the number of
+// concurrent getaddrinfo calls to less than the permitted number of
+// file descriptors makes that error less likely. We don't bother to
+// apply the same limit to DNS lookups run directly from Go, because
+// there we will return a meaningful "too many open files" error.
+func concurrentThreadsLimit() int {
+       var rlim syscall.Rlimit
+       if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil {
+               return 500
+       }
+       r := int(rlim.Cur)
+       if r > 500 {
+               r = 500
+       } else if r > 30 {
+               r -= 30
+       }
+       return r
+}

diff --git a/src/net/lookup_windows.go b/src/net/lookup_windows.go
index e1a811ce397d49748244b167524827fcaf31f8a6..f76e0af400dcc87ffeba13678f2b3deed3a4d637 100644 (file)
--- a/src/net/lookup_windows.go
+++ b/src/net/lookup_windows.go
@@ -364,3 +364,9 @@ Cname:
        }
        return name
 }
+
+// concurrentThreadsLimit returns the number of threads we permit to
+// run concurrently doing DNS lookups.
+func concurrentThreadsLimit() int {
+       return 500
+}

diff --git a/src/net/net.go b/src/net/net.go
index 48c500167052c238c32ca4f9484e4fcc858248a3..c9099862699068a1c5b9c11ddec8b9aa36b75470 100644 (file)
--- a/src/net/net.go
+++ b/src/net/net.go
@@ -84,6 +84,7 @@ import (
        "internal/poll"
        "io"
        "os"
+       "sync"
        "syscall"
        "time"
 )
@@ -610,9 +611,14 @@ func genericReadFrom(w io.Writer, r io.Reader) (n int64, err error) {
 // server is not responding. Then the many lookups each use a different
 // thread, and the system or the program runs out of threads.
 
-var threadLimit = make(chan struct{}, 500)
+var threadLimit chan struct{}
+
+var threadOnce sync.Once
 
 func acquireThread() {
+       threadOnce.Do(func() {
+               threadLimit = make(chan struct{}, concurrentThreadsLimit())
+       })
        threadLimit <- struct{}{}
 }