]> Cypherpunks repositories - gostls13.git/commitdiff
cmd/dist: run various one-off tests in parallel
authorRuss Cox <rsc@golang.org>
Mon, 21 Dec 2015 21:08:57 +0000 (16:08 -0500)
committerRuss Cox <rsc@golang.org>
Tue, 29 Dec 2015 15:50:04 +0000 (15:50 +0000)
Takes 15% off my all.bash run time
(after this and earlier CLs, now down to 3½ from 5½ minutes).

For #10571.

Change-Id: Iac316ffb730c9ff0a0faa7cc3b82ed4f7e6d4361
Reviewed-on: https://go-review.googlesource.com/18088
Reviewed-by: Ian Lance Taylor <iant@golang.org>
src/cmd/dist/test.go

index 9ac01616535a0dec7c396af6b5bf902e415fb121..3f07c8a54a9db7dc98cc94229cf95ee098318641 100644 (file)
@@ -41,15 +41,17 @@ func cmdtest() {
 
 // tester executes cmdtest.
 type tester struct {
-       race      bool
-       listMode  bool
-       rebuild   bool
-       keepGoing bool
-       runRxStr  string
-       runRx     *regexp.Regexp
-       runRxWant bool     // want runRx to match (true) or not match (false)
-       runNames  []string // tests to run, exclusive with runRx; empty means all
-       banner    string   // prefix, or "" for none
+       race        bool
+       listMode    bool
+       rebuild     bool
+       failed      bool
+       keepGoing   bool
+       runRxStr    string
+       runRx       *regexp.Regexp
+       runRxWant   bool     // want runRx to match (true) or not match (false)
+       runNames    []string // tests to run, exclusive with runRx; empty means all
+       banner      string   // prefix, or "" for none
+       lastHeading string   // last dir heading printed
 
        goroot     string
        goarch     string
@@ -62,6 +64,17 @@ type tester struct {
 
        tests        []distTest
        timeoutScale int
+
+       worklist []*work
+}
+
+type work struct {
+       dt    *distTest
+       cmd   *exec.Cmd
+       start chan bool
+       out   []byte
+       err   error
+       end   chan bool
 }
 
 // A distTest is a test run by dist test.
@@ -69,7 +82,7 @@ type tester struct {
 type distTest struct {
        name    string // unique test name; may be filtered with -run flag
        heading string // group section; this header is printed before the test is run.
-       fn      func() error
+       fn      func(*distTest) error
 }
 
 func mustEnv(k string) string {
@@ -175,22 +188,14 @@ func (t *tester) run() {
                }
        }
 
-       var lastHeading string
-       ok := true
        for _, dt := range t.tests {
                if !t.shouldRunTest(dt.name) {
                        t.partial = true
                        continue
                }
-               if dt.heading != "" && lastHeading != dt.heading {
-                       lastHeading = dt.heading
-                       t.out(dt.heading)
-               }
-               if vflag > 0 {
-                       fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
-               }
-               if err := dt.fn(); err != nil {
-                       ok = false
+               dt := dt // dt used in background after this iteration
+               if err := dt.fn(&dt); err != nil {
+                       t.failed = true
                        if t.keepGoing {
                                log.Printf("Failed: %v", err)
                        } else {
@@ -198,7 +203,8 @@ func (t *tester) run() {
                        }
                }
        }
-       if !ok {
+       t.runPending(nil)
+       if t.failed {
                fmt.Println("\nFAILED")
                os.Exit(1)
        } else if t.partial {
@@ -256,10 +262,11 @@ func (t *tester) registerStdTest(pkg string) {
        t.tests = append(t.tests, distTest{
                name:    testName,
                heading: "Testing packages.",
-               fn: func() error {
+               fn: func(dt *distTest) error {
                        if ranGoTest {
                                return nil
                        }
+                       t.runPending(dt)
                        ranGoTest = true
                        args := []string{
                                "test",
@@ -288,10 +295,11 @@ func (t *tester) registerRaceBenchTest(pkg string) {
        t.tests = append(t.tests, distTest{
                name:    testName,
                heading: "Running benchmarks briefly.",
-               fn: func() error {
+               fn: func(dt *distTest) error {
                        if ranGoBench {
                                return nil
                        }
+                       t.runPending(dt)
                        ranGoBench = true
                        args := []string{
                                "test",
@@ -355,12 +363,12 @@ func (t *tester) registerTests() {
        t.tests = append(t.tests, distTest{
                name:    testName,
                heading: "GOMAXPROCS=2 runtime -cpu=1,2,4",
-               fn: func() error {
-                       cmd := t.dirCmd("src", "go", "test", "-short", t.timeout(300), t.tags(), "runtime", "-cpu=1,2,4")
+               fn: func(dt *distTest) error {
+                       cmd := t.addCmd(dt, "src", "go", "test", "-short", t.timeout(300), t.tags(), "runtime", "-cpu=1,2,4")
                        // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code,
                        // creation of first goroutines and first garbage collections in the parallel setting.
                        cmd.Env = mergeEnvLists([]string{"GOMAXPROCS=2"}, os.Environ())
-                       return cmd.Run()
+                       return nil
                },
        })
 
@@ -393,8 +401,9 @@ func (t *tester) registerTests() {
                t.tests = append(t.tests, distTest{
                        name:    "nolibgcc:" + pkg,
                        heading: "Testing without libgcc.",
-                       fn: func() error {
-                               return t.dirCmd("src", "go", "test", "-short", "-ldflags=-linkmode=internal -libgcc=none", t.tags(), pkg, "-run="+run).Run()
+                       fn: func(dt *distTest) error {
+                               t.addCmd(dt, "src", "go", "test", "-short", "-ldflags=-linkmode=internal -libgcc=none", t.tags(), pkg, "-run="+run)
+                               return nil
                        },
                })
        }
@@ -403,8 +412,9 @@ func (t *tester) registerTests() {
        t.tests = append(t.tests, distTest{
                name:    "sync_cpu",
                heading: "sync -cpu=10",
-               fn: func() error {
-                       return t.dirCmd("src", "go", "test", "sync", "-short", t.timeout(120), t.tags(), "-cpu=10").Run()
+               fn: func(dt *distTest) error {
+                       t.addCmd(dt, "src", "go", "test", "sync", "-short", t.timeout(120), t.tags(), "-cpu=10")
+                       return nil
                },
        })
 
@@ -413,17 +423,17 @@ func (t *tester) registerTests() {
                t.tests = append(t.tests, distTest{
                        name:    "cgo_stdio",
                        heading: "../misc/cgo/stdio",
-                       fn: func() error {
-                               return t.dirCmd("misc/cgo/stdio",
-                                       "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".").Run()
+                       fn: func(dt *distTest) error {
+                               t.addCmd(dt, "misc/cgo/stdio", "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".")
+                               return nil
                        },
                })
                t.tests = append(t.tests, distTest{
                        name:    "cgo_life",
                        heading: "../misc/cgo/life",
-                       fn: func() error {
-                               return t.dirCmd("misc/cgo/life",
-                                       "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".").Run()
+                       fn: func(dt *distTest) error {
+                               t.addCmd(dt, "misc/cgo/life", "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".")
+                               return nil
                        },
                })
        }
@@ -455,15 +465,15 @@ func (t *tester) registerTests() {
                        t.tests = append(t.tests, distTest{
                                name:    "testso",
                                heading: "../misc/cgo/testso",
-                               fn: func() error {
-                                       return t.cgoTestSO("misc/cgo/testso")
+                               fn: func(dt *distTest) error {
+                                       return t.cgoTestSO(dt, "misc/cgo/testso")
                                },
                        })
                        t.tests = append(t.tests, distTest{
                                name:    "testsovar",
                                heading: "../misc/cgo/testsovar",
-                               fn: func() error {
-                                       return t.cgoTestSO("misc/cgo/testsovar")
+                               fn: func(dt *distTest) error {
+                                       return t.cgoTestSO(dt, "misc/cgo/testsovar")
                                },
                        })
                }
@@ -504,7 +514,7 @@ func (t *tester) registerTests() {
                                        continue
                                }
                        }
-                       t.registerTest("shootout:"+name, "../test/bench/shootout", "time", "./timing.sh", "-test", name)
+                       t.registerSeqTest("shootout:"+name, "../test/bench/shootout", "time", "./timing.sh", "-test", name)
                }
        }
        if t.goos != "android" && !t.iOS() {
@@ -517,7 +527,7 @@ func (t *tester) registerTests() {
                        t.tests = append(t.tests, distTest{
                                name:    fmt.Sprintf("test:%d_%d", shard, nShards),
                                heading: "../test",
-                               fn:      func() error { return t.testDirTest(shard, nShards) },
+                               fn:      func(dt *distTest) error { return t.testDirTest(dt, shard, nShards) },
                        })
                }
        }
@@ -525,8 +535,9 @@ func (t *tester) registerTests() {
                t.tests = append(t.tests, distTest{
                        name:    "api",
                        heading: "API check",
-                       fn: func() error {
-                               return t.dirCmd("src", "go", "run", filepath.Join(t.goroot, "src/cmd/api/run.go")).Run()
+                       fn: func(dt *distTest) error {
+                               t.addCmd(dt, "src", "go", "run", filepath.Join(t.goroot, "src/cmd/api/run.go"))
+                               return nil
                        },
                })
        }
@@ -543,7 +554,7 @@ func (t *tester) isRegisteredTestName(testName string) bool {
        return false
 }
 
-func (t *tester) registerTest(name, dirBanner, bin string, args ...string) {
+func (t *tester) registerTest1(seq bool, name, dirBanner, bin string, args ...string) {
        if bin == "time" && !t.haveTime {
                bin, args = args[0], args[1:]
        }
@@ -553,19 +564,37 @@ func (t *tester) registerTest(name, dirBanner, bin string, args ...string) {
        t.tests = append(t.tests, distTest{
                name:    name,
                heading: dirBanner,
-               fn: func() error {
-                       return t.dirCmd(filepath.Join(t.goroot, "src", dirBanner), bin, args...).Run()
+               fn: func(dt *distTest) error {
+                       if seq {
+                               t.runPending(dt)
+                               return t.dirCmd(filepath.Join(t.goroot, "src", dirBanner), bin, args...).Run()
+                       }
+                       t.addCmd(dt, filepath.Join(t.goroot, "src", dirBanner), bin, args...)
+                       return nil
                },
        })
 }
 
-func (t *tester) dirCmd(dir string, bin string, args ...string) *exec.Cmd {
+func (t *tester) registerTest(name, dirBanner, bin string, args ...string) {
+       t.registerTest1(false, name, dirBanner, bin, args...)
+}
+
+func (t *tester) registerSeqTest(name, dirBanner, bin string, args ...string) {
+       t.registerTest1(true, name, dirBanner, bin, args...)
+}
+
+func (t *tester) bgDirCmd(dir, bin string, args ...string) *exec.Cmd {
        cmd := exec.Command(bin, args...)
        if filepath.IsAbs(dir) {
                cmd.Dir = dir
        } else {
                cmd.Dir = filepath.Join(t.goroot, dir)
        }
+       return cmd
+}
+
+func (t *tester) dirCmd(dir, bin string, args ...string) *exec.Cmd {
+       cmd := t.bgDirCmd(dir, bin, args...)
        cmd.Stdout = os.Stdout
        cmd.Stderr = os.Stderr
        if vflag > 1 {
@@ -574,6 +603,15 @@ func (t *tester) dirCmd(dir string, bin string, args ...string) *exec.Cmd {
        return cmd
 }
 
+func (t *tester) addCmd(dt *distTest, dir, bin string, args ...string) *exec.Cmd {
+       w := &work{
+               dt:  dt,
+               cmd: t.bgDirCmd(dir, bin, args...),
+       }
+       t.worklist = append(t.worklist, w)
+       return w.cmd
+}
+
 func (t *tester) iOS() bool {
        return t.goos == "darwin" && (t.goarch == "arm" || t.goarch == "arm64")
 }
@@ -643,7 +681,7 @@ func (t *tester) supportedBuildmode(mode string) bool {
        }
 }
 
-func (t *tester) cgoTest() error {
+func (t *tester) cgoTest(dt *distTest) error {
        env := mergeEnvLists([]string{"GOTRACEBACK=2"}, os.Environ())
 
        if t.goos == "android" || t.iOS() {
@@ -652,19 +690,13 @@ func (t *tester) cgoTest() error {
                return cmd.Run()
        }
 
-       cmd := t.dirCmd("misc/cgo/test", "go", "test", t.tags(), "-ldflags", "-linkmode=auto")
+       cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", t.tags(), "-ldflags", "-linkmode=auto")
        cmd.Env = env
-       if err := cmd.Run(); err != nil {
-               return err
-       }
 
        if t.gohostos != "dragonfly" {
                // linkmode=internal fails on dragonfly since errno is a TLS relocation.
-               cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=internal")
+               cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=internal")
                cmd.Env = env
-               if err := cmd.Run(); err != nil {
-                       return err
-               }
        }
 
        pair := t.gohostos + "-" + t.goarch
@@ -676,37 +708,24 @@ func (t *tester) cgoTest() error {
                if !t.extLink() {
                        break
                }
-               cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external")
+               cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external")
                cmd.Env = env
-               if err := cmd.Run(); err != nil {
-                       return err
-               }
-               cmd = t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external -s")
+               cmd = t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external -s")
                cmd.Env = env
-               if err := cmd.Run(); err != nil {
-                       return err
-               }
        case "android-arm",
                "dragonfly-386", "dragonfly-amd64",
                "freebsd-386", "freebsd-amd64", "freebsd-arm",
                "linux-386", "linux-amd64", "linux-arm",
                "netbsd-386", "netbsd-amd64":
 
-               cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external")
+               cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external")
                cmd.Env = env
-               if err := cmd.Run(); err != nil {
-                       return err
-               }
-               cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=auto")
+
+               cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=auto")
                cmd.Env = env
-               if err := cmd.Run(); err != nil {
-                       return err
-               }
-               cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=external")
+
+               cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=external")
                cmd.Env = env
-               if err := cmd.Run(); err != nil {
-                       return err
-               }
 
                switch pair {
                case "netbsd-386", "netbsd-amd64":
@@ -726,29 +745,17 @@ func (t *tester) cgoTest() error {
                        if err := cmd.Run(); err != nil {
                                fmt.Println("No support for static linking found (lacks libc.a?), skip cgo static linking test.")
                        } else {
-                               cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`)
+                               cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`)
                                cmd.Env = env
-                               if err := cmd.Run(); err != nil {
-                                       return err
-                               }
 
-                               cmd = t.dirCmd("misc/cgo/nocgo", "go", "test")
+                               cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test")
                                cmd.Env = env
-                               if err := cmd.Run(); err != nil {
-                                       return err
-                               }
 
-                               cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external`)
+                               cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external`)
                                cmd.Env = env
-                               if err := cmd.Run(); err != nil {
-                                       return err
-                               }
 
-                               cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`)
+                               cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`)
                                cmd.Env = env
-                               if err := cmd.Run(); err != nil {
-                                       return err
-                               }
                        }
 
                        if pair != "freebsd-amd64" { // clang -pie fails to link misc/cgo/test
@@ -759,21 +766,15 @@ func (t *tester) cgoTest() error {
                                if err := cmd.Run(); err != nil {
                                        fmt.Println("No support for -pie found, skip cgo PIE test.")
                                } else {
-                                       cmd = t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
+                                       cmd = t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
                                        cmd.Env = env
-                                       if err := cmd.Run(); err != nil {
-                                               return fmt.Errorf("pie cgo/test: %v", err)
-                                       }
-                                       cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
+
+                                       cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
                                        cmd.Env = env
-                                       if err := cmd.Run(); err != nil {
-                                               return fmt.Errorf("pie cgo/testtls: %v", err)
-                                       }
-                                       cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
+
+                                       cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
                                        cmd.Env = env
-                                       if err := cmd.Run(); err != nil {
-                                               return fmt.Errorf("pie cgo/nocgo: %v", err)
-                                       }
+
                                }
                        }
                }
@@ -782,6 +783,78 @@ func (t *tester) cgoTest() error {
        return nil
 }
 
+// run pending test commands, in parallel, emitting headers as appropriate.
+// When finished, emit header for dt, which is going to run after the
+// pending commands are done (and runPending returns).
+func (t *tester) runPending(nextTest *distTest) {
+       worklist := t.worklist
+       t.worklist = nil
+       for _, w := range worklist {
+               w.start = make(chan bool)
+               w.end = make(chan bool)
+               go func(w *work) {
+                       if !<-w.start {
+                               w.out = []byte(fmt.Sprintf("skipped due to earlier error\n"))
+                       } else {
+                               w.out, w.err = w.cmd.CombinedOutput()
+                       }
+                       w.end <- true
+               }(w)
+       }
+
+       started := 0
+       ended := 0
+       var last *distTest
+       for ended < len(worklist) {
+               for started < len(worklist) && started-ended < maxbg {
+                       //println("start", started)
+                       w := worklist[started]
+                       started++
+                       w.start <- !t.failed || t.keepGoing
+               }
+               w := worklist[ended]
+               dt := w.dt
+               if dt.heading != "" && t.lastHeading != dt.heading {
+                       t.lastHeading = dt.heading
+                       t.out(dt.heading)
+               }
+               if dt != last {
+                       // Assumes all the entries for a single dt are in one worklist.
+                       last = w.dt
+                       if vflag > 0 {
+                               fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
+                       }
+               }
+               if vflag > 1 {
+                       errprintf("%s\n", strings.Join(w.cmd.Args, " "))
+               }
+               //println("wait", ended)
+               ended++
+               <-w.end
+               os.Stdout.Write(w.out)
+               if w.err != nil {
+                       log.Printf("Failed: %v", w.err)
+                       t.failed = true
+                       if !t.keepGoing {
+                               break
+                       }
+               }
+       }
+       if t.failed && !t.keepGoing {
+               log.Fatal("FAILED")
+       }
+
+       if dt := nextTest; dt != nil {
+               if dt.heading != "" && t.lastHeading != dt.heading {
+                       t.lastHeading = dt.heading
+                       t.out(dt.heading)
+               }
+               if vflag > 0 {
+                       fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
+               }
+       }
+}
+
 func (t *tester) cgoTestSOSupported() bool {
        if t.goos == "android" || t.iOS() {
                // No exec facility on Android or iOS.
@@ -798,7 +871,9 @@ func (t *tester) cgoTestSOSupported() bool {
        return true
 }
 
-func (t *tester) cgoTestSO(testpath string) error {
+func (t *tester) cgoTestSO(dt *distTest, testpath string) error {
+       t.runPending(dt)
+
        dir := filepath.Join(t.goroot, testpath)
 
        // build shared object
@@ -866,34 +941,24 @@ func (t *tester) raceDetectorSupported() bool {
        return false
 }
 
-func (t *tester) raceTest() error {
-       if err := t.dirCmd("src", "go", "test", "-race", "-i", "runtime/race", "flag", "os/exec").Run(); err != nil {
-               return err
-       }
-       if err := t.dirCmd("src", "go", "test", "-race", "-run=Output", "runtime/race").Run(); err != nil {
-               return err
-       }
-       if err := t.dirCmd("src", "go", "test", "-race", "-short", "-run=TestParse|TestEcho", "flag", "os/exec").Run(); err != nil {
-               return err
-       }
+func (t *tester) raceTest(dt *distTest) error {
+       t.addCmd(dt, "src", "go", "test", "-race", "-i", "runtime/race", "flag", "os/exec")
+       t.addCmd(dt, "src", "go", "test", "-race", "-run=Output", "runtime/race")
+       t.addCmd(dt, "src", "go", "test", "-race", "-short", "-run=TestParse|TestEcho", "flag", "os/exec")
        if t.cgoEnabled {
                env := mergeEnvLists([]string{"GOTRACEBACK=2"}, os.Environ())
-               cmd := t.dirCmd("misc/cgo/test", "go", "test", "-race", "-short")
+               cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-race", "-short")
                cmd.Env = env
-               if err := cmd.Run(); err != nil {
-                       return err
-               }
        }
        if t.extLink() {
                // Test with external linking; see issue 9133.
-               if err := t.dirCmd("src", "go", "test", "-race", "-short", "-ldflags=-linkmode=external", "-run=TestParse|TestEcho", "flag", "os/exec").Run(); err != nil {
-                       return err
-               }
+               t.addCmd(dt, "src", "go", "test", "-race", "-short", "-ldflags=-linkmode=external", "-run=TestParse|TestEcho", "flag", "os/exec")
        }
        return nil
 }
 
-func (t *tester) testDirTest(shard, shards int) error {
+func (t *tester) testDirTest(dt *distTest, shard, shards int) error {
+       t.runPending(dt)
        const runExe = "runtest.exe" // named exe for Windows, but harmless elsewhere
        cmd := t.dirCmd("test", "go", "build", "-o", runExe, "run.go")
        cmd.Env = mergeEnvLists([]string{"GOOS=" + t.gohostos, "GOARCH=" + t.gohostarch, "GOMAXPROCS="}, os.Environ())