From: Josh Bleecher Snyder Date: Thu, 20 Apr 2017 23:41:42 +0000 (-0700) Subject: cmd/go: add support for concurrent backend compilation X-Git-Tag: go1.9beta1~304 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=f4e5bd483b1c6f731c9925d3d1b66d2aba88980e;p=gostls13.git cmd/go: add support for concurrent backend compilation It is disabled by default. It can be enabled by setting the environment variable GO19CONCURRENTCOMPILATION=1. Benchmarking results are presented in a grid. Columns are different values of c (compiler backend concurrency); rows are different values of p (process concurrency). 'go build -a std cmd', a 4 core raspberry pi 3: c=1 c=2 c=4 StdCmd/p=1 504s ± 2% 413s ± 4% 367s ± 3% StdCmd/p=2 314s ± 3% 266s ± 4% 267s ± 4% StdCmd/p=4 254s ± 5% 241s ± 5% 238s ± 6% 'go build -a std cmd', an 8 core darwin/amd64 laptop: c=1 c=2 c=4 c=6 c=8 StdCmd/p=1 40.4s ± 7% 31.0s ± 1% 27.3s ± 1% 27.8s ± 0% 27.7s ± 0% StdCmd/p=2 21.9s ± 1% 17.9s ± 1% 16.9s ± 1% 17.0s ± 1% 17.2s ± 0% StdCmd/p=4 17.4s ± 2% 14.5s ± 2% 13.3s ± 2% 13.5s ± 2% 13.6s ± 2% StdCmd/p=6 16.9s ± 1% 14.2s ± 2% 13.1s ± 2% 13.2s ± 2% 13.3s ± 2% StdCmd/p=8 16.7s ± 2% 14.2s ± 2% 13.2s ± 3% 13.2s ± 2% 13.4s ± 2% 'go build -a std cmd', a 96 core arm64 server: c=1 c=2 c=4 c=6 c=8 c=16 c=32 c=64 c=96 StdCmd/p=1 173s ± 1% 133s ± 1% 114s ± 1% 109s ± 1% 106s ± 0% 106s ± 1% 107s ± 1% 110s ± 1% 113s ± 1% StdCmd/p=2 94.2s ± 2% 71.5s ± 1% 61.7s ± 1% 58.7s ± 1% 57.5s ± 2% 56.9s ± 1% 58.0s ± 1% 59.6s ± 1% 61.0s ± 1% StdCmd/p=4 74.1s ± 2% 53.5s ± 1% 43.7s ± 2% 40.5s ± 1% 39.2s ± 2% 38.9s ± 2% 39.5s ± 3% 40.3s ± 2% 40.8s ± 1% StdCmd/p=6 69.3s ± 1% 50.2s ± 2% 40.3s ± 2% 37.3s ± 3% 36.0s ± 3% 35.3s ± 2% 36.0s ± 2% 36.8s ± 2% 37.5s ± 2% StdCmd/p=8 66.1s ± 2% 47.7s ± 2% 38.6s ± 2% 35.7s ± 2% 34.4s ± 1% 33.6s ± 2% 34.2s ± 2% 34.6s ± 1% 35.0s ± 1% StdCmd/p=16 63.4s ± 2% 45.3s ± 2% 36.3s ± 2% 33.3s ± 2% 32.0s ± 3% 31.6s ± 2% 32.1s ± 2% 32.5s ± 2% 32.7s ± 2% StdCmd/p=32 62.2s ± 1% 44.2s ± 2% 35.3s ± 2% 32.4s ± 2% 31.2s ± 2% 30.9s ± 2% 31.1s ± 2% 31.7s ± 2% 32.0s ± 2% StdCmd/p=64 62.2s ± 1% 44.3s ± 2% 35.4s ± 2% 32.4s ± 2% 31.2s ± 2% 30.9s ± 2% 31.2s ± 2% 31.8s ± 3% 32.2s ± 3% StdCmd/p=96 62.2s ± 2% 44.4s ± 2% 35.3s ± 2% 32.3s ± 2% 31.1s ± 2% 30.9s ± 3% 31.3s ± 2% 31.7s ± 1% 32.1s ± 2% benchjuju, an 8 core darwin/amd64 laptop: c=1 c=2 c=4 c=6 c=8 BuildJuju/p=1 55.3s ± 0% 46.3s ± 0% 41.9s ± 0% 41.4s ± 1% 41.3s ± 0% BuildJuju/p=2 33.7s ± 1% 28.4s ± 1% 26.7s ± 1% 26.6s ± 1% 26.8s ± 1% BuildJuju/p=4 24.7s ± 1% 22.3s ± 1% 21.4s ± 1% 21.7s ± 1% 21.8s ± 1% BuildJuju/p=6 20.6s ± 1% 19.3s ± 2% 19.4s ± 1% 19.7s ± 1% 19.9s ± 1% BuildJuju/p=8 20.6s ± 2% 19.5s ± 2% 19.3s ± 2% 19.6s ± 1% 19.8s ± 2% Updates #15756 Change-Id: I8a56e88953071a05eee764002024c54cd888a56c Reviewed-on: https://go-review.googlesource.com/41819 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 0b304f97fd..0419eb29c2 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -144,6 +144,8 @@ See also: go install, go get, go clean. `, } +const concurrentGCBackendCompilationEnabledByDefault = false + func init() { // break init cycle CmdBuild.Run = runBuild @@ -2252,6 +2254,12 @@ func (gcToolchain) gc(b *Builder, p *load.Package, archive, obj string, asmhdr b if asmhdr { args = append(args, "-asmhdr", obj+"go_asm.h") } + + // Add -c=N to use concurrent backend compilation, if possible. + if c := gcBackendConcurrency(gcflags); c > 1 { + args = append(args, fmt.Sprintf("-c=%d", c)) + } + for _, f := range gofiles { args = append(args, mkAbs(p.Dir, f)) } @@ -2260,6 +2268,77 @@ func (gcToolchain) gc(b *Builder, p *load.Package, archive, obj string, asmhdr b return ofile, output, err } +// gcBackendConcurrency returns the backend compiler concurrency level for a package compilation. +func gcBackendConcurrency(gcflags []string) int { + // First, check whether we can use -c at all for this compilation. + canDashC := concurrentGCBackendCompilationEnabledByDefault + + switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e { + case "0": + canDashC = false + case "1": + canDashC = true + case "": + // Not set. Use default. + default: + log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e) + } + + if os.Getenv("GOEXPERIMENT") != "" { + // Concurrent compilation is presumed incompatible with GOEXPERIMENTs. + canDashC = false + } + +CheckFlags: + for _, flag := range gcflags { + // Concurrent compilation is presumed incompatible with any gcflags, + // except for a small whitelist of commonly used flags. + // If the user knows better, they can manually add their own -c to the gcflags. + switch flag { + case "-N", "-l", "-S", "-B", "-C", "-I": + // OK + default: + canDashC = false + break CheckFlags + } + } + + if !canDashC { + return 1 + } + + // Decide how many concurrent backend compilations to allow. + // + // If we allow too many, in theory we might end up with p concurrent processes, + // each with c concurrent backend compiles, all fighting over the same resources. + // However, in practice, that seems not to happen too much. + // Most build graphs are surprisingly serial, so p==1 for much of the build. + // Furthermore, concurrent backend compilation is only enabled for a part + // of the overall compiler execution, so c==1 for much of the build. + // So don't worry too much about that interaction for now. + // + // However, in practice, setting c above 4 tends not to help very much. + // See the analysis in CL 41192. + // + // TODO(josharian): attempt to detect whether this particular compilation + // is likely to be a bottleneck, e.g. when: + // - it has no successor packages to compile (usually package main) + // - all paths through the build graph pass through it + // - critical path scheduling says it is high priority + // and in such a case, set c to runtime.NumCPU. + // We do this now when p==1. + if cfg.BuildP == 1 { + // No process parallelism. Max out c. + return runtime.NumCPU() + } + // Some process parallelism. Set c to min(4, numcpu). + c := 4 + if ncpu := runtime.NumCPU(); ncpu < c { + c = ncpu + } + return c +} + func (gcToolchain) asm(b *Builder, p *load.Package, obj string, sfiles []string) ([]string, error) { // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. inc := filepath.Join(cfg.GOROOT, "pkg", "include")