From: Russ Cox Date: Fri, 11 Feb 2022 22:17:54 +0000 (-0500) Subject: cmd/go: cache compiler flag info X-Git-Tag: go1.20rc1~340 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=db259cdd80eff527e8f344d678031c516167d258;p=gostls13.git cmd/go: cache compiler flag info When you run 'go env' or any command that needs to consider what the default gcc flags are (such as 'go list net' or 'go list '), the go command runs gcc (or clang) a few times to see what flags are available. These runs can be quite expensive on some systems, particularly Macs that seem to need to occasionally cache something before gcc/clang can execute quickly. To fix this, cache the derived information about gcc under a cache key derived from the size and modification time of the compiler binary. This is not foolproof, but it should be good enough. % go install cmd/go % time go env >/dev/null 0.22 real 0.01 user 0.01 sys % time go env >/dev/null 0.03 real 0.01 user 0.01 sys % Fixes #50982. Change-Id: Iba7955dd10f610f2793e1accbd2d06922f928faa Reviewed-on: https://go-review.googlesource.com/c/go/+/392454 TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Run-TryBot: Russ Cox Auto-Submit: Russ Cox --- diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 10499c2d3e..66ef5ceee3 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -58,6 +58,7 @@ For more about environment variables, see 'go help environment'. func init() { CmdEnv.Run = runEnv // break init cycle base.AddChdirFlag(&CmdEnv.Flag) + base.AddBuildFlagsNX(&CmdEnv.Flag) } var ( diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go index fc46d19bc4..8beb1345d0 100644 --- a/src/cmd/go/internal/work/action.go +++ b/src/cmd/go/internal/work/action.go @@ -33,11 +33,12 @@ import ( // It does not hold per-package state, because we // build packages in parallel, and the builder is shared. type Builder struct { - WorkDir string // the temporary work directory (ends in filepath.Separator) - actionCache map[cacheKey]*Action // a cache of already-constructed actions - mkdirCache map[string]bool // a cache of created directories - flagCache map[[2]string]bool // a cache of supported compiler flags - Print func(args ...any) (int, error) + WorkDir string // the temporary work directory (ends in filepath.Separator) + actionCache map[cacheKey]*Action // a cache of already-constructed actions + mkdirCache map[string]bool // a cache of created directories + flagCache map[[2]string]bool // a cache of supported compiler flags + gccCompilerIDCache map[string]cache.ActionID // cache for gccCompilerID + Print func(args ...any) (int, error) IsCmdList bool // running as part of go list; set p.Stale and additional fields below NeedError bool // list needs p.Error diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go index f0b12e1036..db56714788 100644 --- a/src/cmd/go/internal/work/buildid.go +++ b/src/cmd/go/internal/work/buildid.go @@ -17,6 +17,7 @@ import ( "cmd/go/internal/fsys" "cmd/go/internal/str" "cmd/internal/buildid" + "cmd/internal/quoted" ) // Build IDs @@ -206,14 +207,20 @@ func (b *Builder) toolID(name string) string { // In order to get reproducible builds for released compilers, we // detect a released compiler by the absence of "experimental" in the // --version output, and in that case we just use the version string. -func (b *Builder) gccToolID(name, language string) (string, error) { +// +// gccToolID also returns the underlying executable for the compiler. +// The caller assumes that stat of the exe can be used, combined with the id, +// to detect changes in the underlying compiler. The returned exe can be empty, +// which means to rely only on the id. +func (b *Builder) gccToolID(name, language string) (id, exe string, err error) { key := name + "." + language b.id.Lock() - id := b.toolIDCache[key] + id = b.toolIDCache[key] + exe = b.toolIDCache[key+".exe"] b.id.Unlock() if id != "" { - return id, nil + return id, exe, nil } // Invoke the driver with -### to see the subcommands and the @@ -225,19 +232,19 @@ func (b *Builder) gccToolID(name, language string) (string, error) { cmd.Env = append(os.Environ(), "LC_ALL=C") out, err := cmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("%s: %v; output: %q", name, err, out) + return "", "", fmt.Errorf("%s: %v; output: %q", name, err, out) } version := "" lines := strings.Split(string(out), "\n") for _, line := range lines { - if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" { + if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" || len(fields) > 2 && fields[2] == "version" { version = line break } } if version == "" { - return "", fmt.Errorf("%s: can not find version number in %q", name, out) + return "", "", fmt.Errorf("%s: can not find version number in %q", name, out) } if !strings.Contains(version, "experimental") { @@ -248,20 +255,20 @@ func (b *Builder) gccToolID(name, language string) (string, error) { // a leading space is the compiler proper. compiler := "" for _, line := range lines { - if len(line) > 1 && line[0] == ' ' { + if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " (in-process)") { compiler = line break } } if compiler == "" { - return "", fmt.Errorf("%s: can not find compilation command in %q", name, out) + return "", "", fmt.Errorf("%s: can not find compilation command in %q", name, out) } - fields := strings.Fields(compiler) + fields, _ := quoted.Split(compiler) if len(fields) == 0 { - return "", fmt.Errorf("%s: compilation command confusion %q", name, out) + return "", "", fmt.Errorf("%s: compilation command confusion %q", name, out) } - exe := fields[0] + exe = fields[0] if !strings.ContainsAny(exe, `/\`) { if lp, err := exec.LookPath(exe); err == nil { exe = lp @@ -269,7 +276,7 @@ func (b *Builder) gccToolID(name, language string) (string, error) { } id, err = buildid.ReadFile(exe) if err != nil { - return "", err + return "", "", err } // If we can't find a build ID, use a hash. @@ -280,9 +287,10 @@ func (b *Builder) gccToolID(name, language string) (string, error) { b.id.Lock() b.toolIDCache[key] = id + b.toolIDCache[key+".exe"] = exe b.id.Unlock() - return id, nil + return id, exe, nil } // Check if assembler used by gccgo is GNU as. diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 90d96400b8..344f409199 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -282,21 +282,21 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { // so that the prebuilt .a files from a Go binary install // don't need to be rebuilt with the local compiler. if !p.Standard { - if ccID, err := b.gccToolID(ccExe[0], "c"); err == nil { + if ccID, _, err := b.gccToolID(ccExe[0], "c"); err == nil { fmt.Fprintf(h, "CC ID=%q\n", ccID) } } if len(p.CXXFiles)+len(p.SwigCXXFiles) > 0 { cxxExe := b.cxxExe() fmt.Fprintf(h, "CXX=%q %q\n", cxxExe, cxxflags) - if cxxID, err := b.gccToolID(cxxExe[0], "c++"); err == nil { + if cxxID, _, err := b.gccToolID(cxxExe[0], "c++"); err == nil { fmt.Fprintf(h, "CXX ID=%q\n", cxxID) } } if len(p.FFiles) > 0 { fcExe := b.fcExe() fmt.Fprintf(h, "FC=%q %q\n", fcExe, fflags) - if fcID, err := b.gccToolID(fcExe[0], "f95"); err == nil { + if fcID, _, err := b.gccToolID(fcExe[0], "f95"); err == nil { fmt.Fprintf(h, "FC ID=%q\n", fcID) } } @@ -350,7 +350,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { } case "gccgo": - id, err := b.gccToolID(BuildToolchain.compiler(), "go") + id, _, err := b.gccToolID(BuildToolchain.compiler(), "go") if err != nil { base.Fatalf("%v", err) } @@ -358,7 +358,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { fmt.Fprintf(h, "pkgpath %s\n", gccgoPkgpath(p)) fmt.Fprintf(h, "ar %q\n", BuildToolchain.(gccgoToolchain).ar()) if len(p.SFiles) > 0 { - id, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp") + id, _, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp") // Ignore error; different assembler versions // are unlikely to make any difference anyhow. fmt.Fprintf(h, "asm %q\n", id) @@ -1359,7 +1359,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { // Or external linker settings and flags? case "gccgo": - id, err := b.gccToolID(BuildToolchain.linker(), "go") + id, _, err := b.gccToolID(BuildToolchain.linker(), "go") if err != nil { base.Fatalf("%v", err) } @@ -2689,21 +2689,23 @@ func (b *Builder) gccNoPie(linker []string) string { func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { key := [2]string{compiler[0], flag} - b.exec.Lock() - defer b.exec.Unlock() - if b, ok := b.flagCache[key]; ok { - return b - } - if b.flagCache == nil { - b.flagCache = make(map[[2]string]bool) - } - - tmp := os.DevNull + // We used to write an empty C file, but that gets complicated with go + // build -n. We tried using a file that does not exist, but that fails on + // systems with GCC version 4.2.1; that is the last GPLv2 version of GCC, + // so some systems have frozen on it. Now we pass an empty file on stdin, + // which should work at least for GCC and clang. + // + // If the argument is "-Wl,", then it is testing the linker. In that case, + // skip "-c". If it's not "-Wl,", then we are testing the compiler and can + // omit the linking step with "-c". + // + // Using the same CFLAGS/LDFLAGS here and for building the program. // On the iOS builder the command // $CC -Wl,--no-gc-sections -x c - -o /dev/null < /dev/null // is failing with: // Unable to remove existing file: Invalid argument + tmp := os.DevNull if runtime.GOOS == "windows" || runtime.GOOS == "ios" { f, err := os.CreateTemp(b.WorkDir, "") if err != nil { @@ -2714,17 +2716,6 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { defer os.Remove(tmp) } - // We used to write an empty C file, but that gets complicated with go - // build -n. We tried using a file that does not exist, but that fails on - // systems with GCC version 4.2.1; that is the last GPLv2 version of GCC, - // so some systems have frozen on it. Now we pass an empty file on stdin, - // which should work at least for GCC and clang. - // - // If the argument is "-Wl,", then it is testing the linker. In that case, - // skip "-c". If it's not "-Wl,", then we are testing the compiler and can - // omit the linking step with "-c". - // - // Using the same CFLAGS/LDFLAGS here and for building the program. cmdArgs := str.StringList(compiler, flag) if strings.HasPrefix(flag, "-Wl,") /* linker flag */ { ldflags, err := buildFlags("LDFLAGS", defaultCFlags, nil, checkLinkerFlags) @@ -2743,12 +2734,37 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { cmdArgs = append(cmdArgs, "-x", "c", "-", "-o", tmp) - if cfg.BuildN || cfg.BuildX { + if cfg.BuildN { b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs)) - if cfg.BuildN { - return false + return false + } + + // gccCompilerID acquires b.exec, so do before acquiring lock. + compilerID, cacheOK := b.gccCompilerID(compiler[0]) + + b.exec.Lock() + defer b.exec.Unlock() + if b, ok := b.flagCache[key]; ok { + return b + } + if b.flagCache == nil { + b.flagCache = make(map[[2]string]bool) + } + + // Look in build cache. + var flagID cache.ActionID + if cacheOK { + flagID = cache.Subkey(compilerID, "gccSupportsFlag "+flag) + if data, _, err := cache.Default().GetBytes(flagID); err == nil { + supported := string(data) == "true" + b.flagCache[key] = supported + return supported } } + + if cfg.BuildX { + b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs)) + } cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) cmd.Dir = b.WorkDir cmd.Env = append(cmd.Environ(), "LC_ALL=C") @@ -2765,10 +2781,120 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { !bytes.Contains(out, []byte("is not supported")) && !bytes.Contains(out, []byte("not recognized")) && !bytes.Contains(out, []byte("unsupported")) + + if cacheOK { + s := "false" + if supported { + s = "true" + } + cache.Default().PutBytes(flagID, []byte(s)) + } + b.flagCache[key] = supported return supported } +// statString returns a string form of an os.FileInfo, for serializing and comparison. +func statString(info os.FileInfo) string { + return fmt.Sprintf("stat %d %x %v %v\n", info.Size(), uint64(info.Mode()), info.ModTime(), info.IsDir()) +} + +// gccCompilerID returns a build cache key for the current gcc, +// as identified by running 'compiler'. +// The caller can use subkeys of the key. +// Other parts of cmd/go can use the id as a hash +// of the installed compiler version. +func (b *Builder) gccCompilerID(compiler string) (id cache.ActionID, ok bool) { + if cfg.BuildN { + b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously([]string{compiler, "--version"})) + return cache.ActionID{}, false + } + + b.exec.Lock() + defer b.exec.Unlock() + + if id, ok := b.gccCompilerIDCache[compiler]; ok { + return id, ok + } + + // We hash the compiler's full path to get a cache entry key. + // That cache entry holds a validation description, + // which is of the form: + // + // filename \x00 statinfo \x00 + // ... + // compiler id + // + // If os.Stat of each filename matches statinfo, + // then the entry is still valid, and we can use the + // compiler id without any further expense. + // + // Otherwise, we compute a new validation description + // and compiler id (below). + exe, err := exec.LookPath(compiler) + if err != nil { + return cache.ActionID{}, false + } + + h := cache.NewHash("gccCompilerID") + fmt.Fprintf(h, "gccCompilerID %q", exe) + key := h.Sum() + data, _, err := cache.Default().GetBytes(key) + if err == nil && len(data) > len(id) { + stats := strings.Split(string(data[:len(data)-len(id)]), "\x00") + if len(stats)%2 != 0 { + goto Miss + } + for i := 0; i+2 <= len(stats); i++ { + info, err := os.Stat(stats[i]) + if err != nil || statString(info) != stats[i+1] { + goto Miss + } + } + copy(id[:], data[len(data)-len(id):]) + return id, true + Miss: + } + + // Validation failed. Compute a new description (in buf) and compiler ID (in h). + // For now, there are only at most two filenames in the stat information. + // The first one is the compiler executable we invoke. + // The second is the underlying compiler as reported by -v -### + // (see b.gccToolID implementation in buildid.go). + toolID, exe2, err := b.gccToolID(compiler, "c") + if err != nil { + return cache.ActionID{}, false + } + + exes := []string{exe, exe2} + str.Uniq(&exes) + fmt.Fprintf(h, "gccCompilerID %q %q\n", exes, toolID) + id = h.Sum() + + var buf bytes.Buffer + for _, exe := range exes { + if exe == "" { + continue + } + info, err := os.Stat(exe) + if err != nil { + return cache.ActionID{}, false + } + buf.WriteString(exe) + buf.WriteString("\x00") + buf.WriteString(statString(info)) + buf.WriteString("\x00") + } + buf.Write(id[:]) + + cache.Default().PutBytes(key, buf.Bytes()) + if b.gccCompilerIDCache == nil { + b.gccCompilerIDCache = make(map[string]cache.ActionID) + } + b.gccCompilerIDCache[compiler] = id + return id, true +} + // gccArchArgs returns arguments to pass to gcc based on the architecture. func (b *Builder) gccArchArgs() []string { switch cfg.Goarch { diff --git a/src/cmd/go/testdata/script/env_cache.txt b/src/cmd/go/testdata/script/env_cache.txt new file mode 100644 index 0000000000..f2af7ee623 --- /dev/null +++ b/src/cmd/go/testdata/script/env_cache.txt @@ -0,0 +1,5 @@ +# go env should caches compiler results +go env +go env -x +! stdout '\|\| true' +