"io"
"io/fs"
"log"
+ "math"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
+ "cmd/go/internal/base"
"cmd/go/internal/cache"
"cmd/go/internal/cfg"
"cmd/go/internal/robustio"
cgoEnabled string // raw value from 'go env CGO_ENABLED'
)
+// netTestSem is a semaphore limiting the number of tests that may use the
+// external network in parallel. If non-nil, it contains one buffer slot per
+// test (send to acquire), with a low enough limit that the overall number of
+// connections (summed across subprocesses) stays at or below base.NetLimit.
+var netTestSem chan struct{}
+
var exeSuffix string = func() string {
if runtime.GOOS == "windows" {
return ".exe"
}
}
+ if n, limited := base.NetLimit(); limited && n > 0 {
+ // Split the network limit into chunks, so that each parallel script can
+ // have one chunk. We want to run as many parallel scripts as possible, but
+ // also want to give each script as high a limit as possible.
+ // We arbitrarily split by sqrt(n) to try to balance those two goals.
+ netTestLimit := int(math.Sqrt(float64(n)))
+ netTestSem = make(chan struct{}, netTestLimit)
+ reducedLimit := fmt.Sprintf(",%s=%d", base.NetLimitGodebug.Name(), n/netTestLimit)
+ os.Setenv("GODEBUG", os.Getenv("GODEBUG")+reducedLimit)
+ }
+
// Don't let these environment variables confuse the test.
os.Setenv("GOENV", "off")
os.Unsetenv("GOFLAGS")
tempdir string
ran bool
inParallel bool
+ hasNet bool
stdout, stderr bytes.Buffer
execDir string // dir for tg.run
}
if tg.ran {
tg.t.Fatal("internal testsuite error: call to parallel after run")
}
+ if tg.hasNet {
+ tg.t.Fatal("internal testsuite error: call to parallel after acquireNet")
+ }
for _, e := range tg.env {
if strings.HasPrefix(e, "GOROOT=") || strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
val := e[strings.Index(e, "=")+1:]
tg.t.Parallel()
}
+// acquireNet skips t if the network is unavailable, and otherwise acquires a
+// netTestSem token for t to be released at the end of the test.
+//
+// t.Parallel must not be called after acquireNet.
+func (tg *testgoData) acquireNet() {
+ tg.t.Helper()
+ if tg.hasNet {
+ return
+ }
+
+ testenv.MustHaveExternalNetwork(tg.t)
+ if netTestSem != nil {
+ netTestSem <- struct{}{}
+ tg.t.Cleanup(func() { <-netTestSem })
+ }
+ tg.setenv("TESTGONETWORK", "")
+ tg.hasNet = true
+}
+
// pwd returns the current directory.
func (tg *testgoData) pwd() string {
tg.t.Helper()
// command.
func (tg *testgoData) setenv(name, val string) {
tg.t.Helper()
- if tg.inParallel && (name == "GOROOT" || name == "GOPATH" || name == "GOBIN") && (strings.HasPrefix(val, "testdata") || strings.HasPrefix(val, "./testdata")) {
- tg.t.Fatalf("internal testsuite error: call to setenv with testdata (%s=%s) after parallel", name, val)
- }
tg.unsetenv(name)
tg.env = append(tg.env, name+"="+val)
}
func (tg *testgoData) unsetenv(name string) {
if tg.env == nil {
tg.env = append([]string(nil), os.Environ()...)
- tg.env = append(tg.env, "GO111MODULE=off")
+ tg.env = append(tg.env, "GO111MODULE=off", "TESTGONETWORK=panic")
+ if testing.Short() {
+ tg.env = append(tg.env, "TESTGOVCS=panic")
+ }
}
for i, v := range tg.env {
if strings.HasPrefix(v, name+"=") {
// cmd/go: custom import path checking should not apply to Go packages without import comment.
func TestIssue10952(t *testing.T) {
- testenv.MustHaveExternalNetwork(t)
testenv.MustHaveExecPath(t, "git")
tg := testgo(t)
defer tg.cleanup()
tg.parallel()
+ tg.acquireNet()
+
tg.tempDir("src")
tg.setenv("GOPATH", tg.path("."))
const importPath = "github.com/zombiezen/go-get-issue-10952"
// Test git clone URL that uses SCP-like syntax and custom import path checking.
func TestIssue11457(t *testing.T) {
- testenv.MustHaveExternalNetwork(t)
testenv.MustHaveExecPath(t, "git")
tg := testgo(t)
defer tg.cleanup()
tg.parallel()
+ tg.acquireNet()
+
tg.tempDir("src")
tg.setenv("GOPATH", tg.path("."))
const importPath = "rsc.io/go-get-issue-11457"
}
func TestGetGitDefaultBranch(t *testing.T) {
- testenv.MustHaveExternalNetwork(t)
testenv.MustHaveExecPath(t, "git")
tg := testgo(t)
defer tg.cleanup()
tg.parallel()
+ tg.acquireNet()
+
tg.tempDir("src")
tg.setenv("GOPATH", tg.path("."))
}
func TestDefaultGOPATHGet(t *testing.T) {
- testenv.MustHaveExternalNetwork(t)
testenv.MustHaveExecPath(t, "git")
tg := testgo(t)
defer tg.cleanup()
tg.parallel()
+ tg.acquireNet()
+
tg.setenv("GOPATH", "")
tg.tempDir("home")
tg.setenv(homeEnvName(), tg.path("home"))
--- /dev/null
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import (
+ "fmt"
+ "internal/godebug"
+ "runtime"
+ "strconv"
+ "sync"
+)
+
+var NetLimitGodebug = godebug.New("#cmdgonetlimit")
+
+// NetLimit returns the limit on concurrent network operations
+// configured by GODEBUG=cmdgonetlimit, if any.
+//
+// A limit of 0 (indicated by 0, true) means that network operations should not
+// be allowed.
+func NetLimit() (int, bool) {
+ netLimitOnce.Do(func() {
+ s := NetLimitGodebug.Value()
+ if s == "" {
+ return
+ }
+
+ n, err := strconv.Atoi(s)
+ if err != nil {
+ Fatalf("invalid %s: %v", NetLimitGodebug.Name(), err)
+ }
+ if n < 0 {
+ // Treat negative values as unlimited.
+ return
+ }
+ netLimitSem = make(chan struct{}, n)
+ })
+
+ return cap(netLimitSem), netLimitSem != nil
+}
+
+// AcquireNet acquires a semaphore token for a network operation.
+func AcquireNet() (release func(), err error) {
+ hasToken := false
+ if n, ok := NetLimit(); ok {
+ if n == 0 {
+ return nil, fmt.Errorf("network disabled by %v=%v", NetLimitGodebug.Name(), NetLimitGodebug.Value())
+ }
+ netLimitSem <- struct{}{}
+ hasToken = true
+ }
+
+ checker := new(netTokenChecker)
+ runtime.SetFinalizer(checker, (*netTokenChecker).panicUnreleased)
+
+ return func() {
+ if checker.released {
+ panic("internal error: net token released twice")
+ }
+ checker.released = true
+ if hasToken {
+ <-netLimitSem
+ }
+ runtime.SetFinalizer(checker, nil)
+ }, nil
+}
+
+var (
+ netLimitOnce sync.Once
+ netLimitSem chan struct{}
+)
+
+type netTokenChecker struct {
+ released bool
+ // We want to use a finalizer to check that all acquired tokens are returned,
+ // so we arbitrarily pad the tokens with a string to defeat the runtime's
+ // “tiny allocator”.
+ unusedAvoidTinyAllocator string
+}
+
+func (c *netTokenChecker) panicUnreleased() {
+ panic("internal error: net token acquired but not released")
+}
"sync"
"time"
+ "cmd/go/internal/base"
"cmd/go/internal/lockedfile"
"cmd/go/internal/par"
"cmd/go/internal/web"
// The git protocol sends all known refs and ls-remote filters them on the client side,
// so we might as well record both heads and tags in one shot.
// Most of the time we only care about tags but sometimes we care about heads too.
+ release, err := base.AcquireNet()
+ if err != nil {
+ r.refsErr = err
+ return
+ }
out, gitErr := Run(ctx, r.dir, "git", "ls-remote", "-q", r.remote)
+ release()
+
if gitErr != nil {
if rerr, ok := gitErr.(*RunError); ok {
if bytes.Contains(rerr.Stderr, []byte("fatal: could not read Username")) {
ref = hash
refspec = hash + ":refs/dummy"
}
- _, err := Run(ctx, r.dir, "git", "fetch", "-f", "--depth=1", r.remote, refspec)
+
+ release, err := base.AcquireNet()
+ if err != nil {
+ return nil, err
+ }
+ _, err = Run(ctx, r.dir, "git", "fetch", "-f", "--depth=1", r.remote, refspec)
+ release()
+
if err == nil {
return r.statLocal(ctx, rev, ref)
}
// golang.org/issue/34266 and
// https://github.com/git/git/blob/4c86140027f4a0d2caaa3ab4bd8bfc5ce3c11c8a/transport.c#L1303-L1309.)
+ release, err := base.AcquireNet()
+ if err != nil {
+ return err
+ }
+ defer release()
+
if _, err := Run(ctx, r.dir, "git", "fetch", "-f", r.remote, "refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"); err != nil {
return err
}
"path/filepath"
"strconv"
"time"
+
+ "cmd/go/internal/base"
)
func svnParseStat(rev, out string) (*RevInfo, error) {
remotePath += "/" + subdir
}
+ release, err := base.AcquireNet()
+ if err != nil {
+ return err
+ }
out, err := Run(ctx, workDir, []string{
"svn", "list",
"--non-interactive",
"--revision", rev,
"--", remotePath,
})
+ release()
if err != nil {
return err
}
}
defer os.RemoveAll(exportDir) // best-effort
+ release, err = base.AcquireNet()
+ if err != nil {
+ return err
+ }
_, err = Run(ctx, workDir, []string{
"svn", "export",
"--non-interactive",
"--", remotePath,
exportDir,
})
+ release()
if err != nil {
return err
}
"sync"
"time"
+ "cmd/go/internal/base"
"cmd/go/internal/lockedfile"
"cmd/go/internal/par"
"cmd/go/internal/str"
defer unlock()
if _, err := os.Stat(filepath.Join(r.dir, "."+vcs)); err != nil {
- if _, err := Run(ctx, r.dir, cmd.init(r.remote)); err != nil {
+ release, err := base.AcquireNet()
+ if err != nil {
+ return nil, err
+ }
+ _, err = Run(ctx, r.dir, cmd.init(r.remote))
+ release()
+
+ if err != nil {
os.RemoveAll(r.dir)
return nil, err
}
func (r *vcsRepo) fetch(ctx context.Context) {
if len(r.cmd.fetch) > 0 {
+ release, err := base.AcquireNet()
+ if err != nil {
+ r.fetchErr = err
+ return
+ }
_, r.fetchErr = Run(ctx, r.dir, r.cmd.fetch)
+ release()
}
}
"sync"
"time"
+ "cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/search"
"cmd/go/internal/str"
}
os.MkdirAll(dir, 0777) // Ignore errors — if unsuccessful, the command will likely fail.
+ release, err := base.AcquireNet()
+ if err != nil {
+ return err
+ }
+ defer release()
+
return v.runVerboseOnly(dir, v.PingCmd, "scheme", scheme, "repo", repo)
}
// Create creates a new copy of repo in dir.
// The parent of dir must exist; dir must not.
func (v *Cmd) Create(dir, repo string) error {
+ release, err := base.AcquireNet()
+ if err != nil {
+ return err
+ }
+ defer release()
+
for _, cmd := range v.CreateCmd {
if err := v.run(filepath.Dir(dir), cmd, "dir", dir, "repo", repo); err != nil {
return err
// Download downloads any new changes for the repo in dir.
func (v *Cmd) Download(dir string) error {
+ release, err := base.AcquireNet()
+ if err != nil {
+ return err
+ }
+ defer release()
+
for _, cmd := range v.DownloadCmd {
if err := v.run(dir, cmd); err != nil {
return err
}
}
+ release, err := base.AcquireNet()
+ if err != nil {
+ return err
+ }
+ defer release()
+
if tag == "" && v.TagSyncDefault != nil {
for _, cmd := range v.TagSyncDefault {
if err := v.run(dir, cmd); err != nil {
"crypto/tls"
"errors"
"fmt"
+ "io"
"mime"
"net"
"net/http"
"time"
"cmd/go/internal/auth"
+ "cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/internal/browser"
)
req.URL.Host = t.ToHost
}
+ release, err := base.AcquireNet()
+ if err != nil {
+ return nil, nil, err
+ }
+
var res *http.Response
if security == Insecure && url.Scheme == "https" { // fail earlier
res, err = impatientInsecureHTTPClient.Do(req)
res, err = securityPreservingDefaultClient.Do(req)
}
}
+
+ if res == nil || res.Body == nil {
+ release()
+ } else {
+ body := res.Body
+ res.Body = hookCloser{
+ ReadCloser: body,
+ afterClose: release,
+ }
+ }
+
return url, res, err
}
}
return false
}
+
+type hookCloser struct {
+ io.ReadCloser
+ afterClose func()
+}
+
+func (c hookCloser) Close() error {
+ err := c.ReadCloser.Close()
+ c.afterClose()
+ return err
+}
defer removeAll(workdir)
}
- s, err := script.NewState(ctx, workdir, env)
+ s, err := script.NewState(tbContext(ctx, t), workdir, env)
if err != nil {
t.Fatal(err)
}
}
}
+// testingTBKey is the Context key for a testing.TB.
+type testingTBKey struct{}
+
+// tbContext returns a Context derived from ctx and associated with t.
+func tbContext(ctx context.Context, t testing.TB) context.Context {
+ return context.WithValue(ctx, testingTBKey{}, t)
+}
+
+// tbFromContext returns the testing.TB associated with ctx, if any.
+func tbFromContext(ctx context.Context) (testing.TB, bool) {
+ t := ctx.Value(testingTBKey{})
+ if t == nil {
+ return nil, false
+ }
+ return t.(testing.TB), true
+}
+
// initScriptState creates the initial directory structure in s for unpacking a
// cmd/go script.
func initScriptDirs(t testing.TB, s *script.State) {
"TESTGO_VCSTEST_HOST=" + httpURL.Host,
"TESTGO_VCSTEST_TLS_HOST=" + httpsURL.Host,
"TESTGO_VCSTEST_CERT=" + srvCertFile,
+ "TESTGONETWORK=panic", // cleared by the [net] condition
"GOSUMDB=" + testSumDBVerifierKey,
"GONOPROXY=",
"GONOSUMDB=",
// Require all tests that use VCS commands to be skipped in short mode.
env = append(env, "TESTGOVCS=panic")
}
+
if os.Getenv("CGO_ENABLED") != "" || runtime.GOOS != goHostOS || runtime.GOARCH != goHostArch {
// If the actual CGO_ENABLED might not match the cmd/go default, set it
// explicitly in the environment. Otherwise, leave it unset so that we also
"runtime"
"runtime/debug"
"strings"
+ "sync"
)
func scriptConditions() map[string]script.Cond {
return platform.BuildModeSupported(runtime.Compiler, mode, GOOS, GOARCH), nil
}
+var scriptNetEnabled sync.Map // testing.TB → already enabled
+
func hasNet(s *script.State, host string) (bool, error) {
if !testenv.HasExternalNetwork() {
return false, nil
// TODO(bcmills): Add a flag or environment variable to allow skipping tests
// for specific hosts and/or skipping all net tests except for specific hosts.
+ t, ok := tbFromContext(s.Context())
+ if !ok {
+ return false, errors.New("script Context unexpectedly missing testing.TB key")
+ }
+
+ if netTestSem != nil {
+ // When the number of external network connections is limited, we limit the
+ // number of net tests that can run concurrently so that the overall number
+ // of network connections won't exceed the limit.
+ _, dup := scriptNetEnabled.LoadOrStore(t, true)
+ if !dup {
+ // Acquire a net token for this test until the test completes.
+ netTestSem <- struct{}{}
+ t.Cleanup(func() {
+ <-netTestSem
+ scriptNetEnabled.Delete(t)
+ })
+ }
+ }
+
// Since we have confirmed that the network is available,
// allow cmd/go to use it.
s.Setenv("TESTGONETWORK", "")