From 08362246b6a1dd7bffeb68f1d5116b9b2fe6209a Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 13 Oct 2017 09:28:15 -0400 Subject: [PATCH] cmd/go/internal/work: factor build.go into multiple files build.go - commands and misc helpers action.go - action graph construction exec.go - action graph execution gc.go - gc toolchain gccgo.go - gccgo toolchain Change-Id: I39b6e2490ac05334c2321e9ad88df694a6efa82f Reviewed-on: https://go-review.googlesource.com/70671 Run-TryBot: Russ Cox Reviewed-by: David Crawshaw --- src/cmd/go/internal/work/action.go | 691 ++++++ src/cmd/go/internal/work/build.go | 3485 ---------------------------- src/cmd/go/internal/work/exec.go | 1904 +++++++++++++++ src/cmd/go/internal/work/gc.go | 481 ++++ src/cmd/go/internal/work/gccgo.go | 497 ++++ 5 files changed, 3573 insertions(+), 3485 deletions(-) create mode 100644 src/cmd/go/internal/work/action.go create mode 100644 src/cmd/go/internal/work/exec.go create mode 100644 src/cmd/go/internal/work/gc.go create mode 100644 src/cmd/go/internal/work/gccgo.go diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go new file mode 100644 index 0000000000..7501ae862d --- /dev/null +++ b/src/cmd/go/internal/work/action.go @@ -0,0 +1,691 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Action graph creation (planning). + +package work + +import ( + "bufio" + "bytes" + "container/heap" + "debug/elf" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/internal/buildid" +) + +// A Builder holds global state about a build. +// It does not hold per-package state, because we +// build packages in parallel, and the builder is shared. +type Builder struct { + WorkDir string // the temporary work directory (ends in filepath.Separator) + actionCache map[cacheKey]*Action // a cache of already-constructed actions + mkdirCache map[string]bool // a cache of created directories + flagCache map[[2]string]bool // a cache of supported compiler flags + Print func(args ...interface{}) (int, error) + + objdirSeq int // counter for NewObjdir + pkgSeq int + + output sync.Mutex + scriptDir string // current directory in printed script + + exec sync.Mutex + readySema chan bool + ready actionQueue +} + +// NOTE: Much of Action would not need to be exported if not for test. +// Maybe test functionality should move into this package too? + +// An Action represents a single action in the action graph. +type Action struct { + Mode string // description of action operation + Package *load.Package // the package this action works on + Deps []*Action // actions that must happen before this one + Func func(*Builder, *Action) error // the action itself (nil = no-op) + IgnoreFail bool // whether to run f even if dependencies fail + TestOutput *bytes.Buffer // test output buffer + Args []string // additional args for runProgram + + triggers []*Action // inverse of deps + buildID string + + // Generated files, directories. + Objdir string // directory for intermediate objects + Target string // goal of the action: the created package or executable + built string // the actual created package or executable + + // Execution state. + pending int // number of deps yet to complete + priority int // relative execution priority + Failed bool // whether the action failed +} + +// An actionQueue is a priority queue of actions. +type actionQueue []*Action + +// Implement heap.Interface +func (q *actionQueue) Len() int { return len(*q) } +func (q *actionQueue) Swap(i, j int) { (*q)[i], (*q)[j] = (*q)[j], (*q)[i] } +func (q *actionQueue) Less(i, j int) bool { return (*q)[i].priority < (*q)[j].priority } +func (q *actionQueue) Push(x interface{}) { *q = append(*q, x.(*Action)) } +func (q *actionQueue) Pop() interface{} { + n := len(*q) - 1 + x := (*q)[n] + *q = (*q)[:n] + return x +} + +func (q *actionQueue) push(a *Action) { + heap.Push(q, a) +} + +func (q *actionQueue) pop() *Action { + return heap.Pop(q).(*Action) +} + +type actionJSON struct { + ID int + Mode string + Package string + Deps []int `json:",omitempty"` + IgnoreFail bool `json:",omitempty"` + Args []string `json:",omitempty"` + Link bool `json:",omitempty"` + Objdir string `json:",omitempty"` + Target string `json:",omitempty"` + Priority int `json:",omitempty"` + Failed bool `json:",omitempty"` + Built string `json:",omitempty"` +} + +// cacheKey is the key for the action cache. +type cacheKey struct { + mode string + p *load.Package +} + +func actionGraphJSON(a *Action) string { + var workq []*Action + var inWorkq = make(map[*Action]int) + + add := func(a *Action) { + if _, ok := inWorkq[a]; ok { + return + } + inWorkq[a] = len(workq) + workq = append(workq, a) + } + add(a) + + for i := 0; i < len(workq); i++ { + for _, dep := range workq[i].Deps { + add(dep) + } + } + + var list []*actionJSON + for id, a := range workq { + aj := &actionJSON{ + Mode: a.Mode, + ID: id, + IgnoreFail: a.IgnoreFail, + Args: a.Args, + Objdir: a.Objdir, + Target: a.Target, + Failed: a.Failed, + Priority: a.priority, + Built: a.built, + } + if a.Package != nil { + // TODO(rsc): Make this a unique key for a.Package somehow. + aj.Package = a.Package.ImportPath + } + for _, a1 := range a.Deps { + aj.Deps = append(aj.Deps, inWorkq[a1]) + } + list = append(list, aj) + } + + js, err := json.MarshalIndent(list, "", "\t") + if err != nil { + fmt.Fprintf(os.Stderr, "go: writing debug action graph: %v\n", err) + return "" + } + return string(js) +} + +// BuildMode specifies the build mode: +// are we just building things or also installing the results? +type BuildMode int + +const ( + ModeBuild BuildMode = iota + ModeInstall +) + +func (b *Builder) Init() { + var err error + b.Print = func(a ...interface{}) (int, error) { + return fmt.Fprint(os.Stderr, a...) + } + b.actionCache = make(map[cacheKey]*Action) + b.mkdirCache = make(map[string]bool) + + if cfg.BuildN { + b.WorkDir = "$WORK" + } else { + b.WorkDir, err = ioutil.TempDir("", "go-build") + if err != nil { + base.Fatalf("%s", err) + } + if cfg.BuildX || cfg.BuildWork { + fmt.Fprintf(os.Stderr, "WORK=%s\n", b.WorkDir) + } + if !cfg.BuildWork { + workdir := b.WorkDir + base.AtExit(func() { os.RemoveAll(workdir) }) + } + } + + if _, ok := cfg.OSArchSupportsCgo[cfg.Goos+"/"+cfg.Goarch]; !ok && cfg.BuildContext.Compiler == "gc" { + fmt.Fprintf(os.Stderr, "cmd/go: unsupported GOOS/GOARCH pair %s/%s\n", cfg.Goos, cfg.Goarch) + os.Exit(2) + } + for _, tag := range cfg.BuildContext.BuildTags { + if strings.Contains(tag, ",") { + fmt.Fprintf(os.Stderr, "cmd/go: -tags space-separated list contains comma\n") + os.Exit(2) + } + } +} + +// NewObjdir returns the name of a fresh object directory under b.WorkDir. +// It is up to the caller to call b.Mkdir on the result at an appropriate time. +// The result ends in a slash, so that file names in that directory +// can be constructed with direct string addition. +// +// NewObjdir must be called only from a single goroutine at a time, +// so it is safe to call during action graph construction, but it must not +// be called during action graph execution. +func (b *Builder) NewObjdir() string { + b.objdirSeq++ + return filepath.Join(b.WorkDir, fmt.Sprintf("b%03d", b.objdirSeq)) + string(filepath.Separator) +} + +// readpkglist returns the list of packages that were built into the shared library +// at shlibpath. For the native toolchain this list is stored, newline separated, in +// an ELF note with name "Go\x00\x00" and type 1. For GCCGO it is extracted from the +// .go_export section. +func readpkglist(shlibpath string) (pkgs []*load.Package) { + var stk load.ImportStack + if cfg.BuildToolchainName == "gccgo" { + f, _ := elf.Open(shlibpath) + sect := f.Section(".go_export") + data, _ := sect.Data() + scanner := bufio.NewScanner(bytes.NewBuffer(data)) + for scanner.Scan() { + t := scanner.Text() + if strings.HasPrefix(t, "pkgpath ") { + t = strings.TrimPrefix(t, "pkgpath ") + t = strings.TrimSuffix(t, ";") + pkgs = append(pkgs, load.LoadPackage(t, &stk)) + } + } + } else { + pkglistbytes, err := buildid.ReadELFNote(shlibpath, "Go\x00\x00", 1) + if err != nil { + base.Fatalf("readELFNote failed: %v", err) + } + scanner := bufio.NewScanner(bytes.NewBuffer(pkglistbytes)) + for scanner.Scan() { + t := scanner.Text() + pkgs = append(pkgs, load.LoadPackage(t, &stk)) + } + } + return +} + +// cacheAction looks up {mode, p} in the cache and returns the resulting action. +// If the cache has no such action, f() is recorded and returned. +func (b *Builder) cacheAction(mode string, p *load.Package, f func() *Action) *Action { + a := b.actionCache[cacheKey{mode, p}] + if a == nil { + a = f() + b.actionCache[cacheKey{mode, p}] = a + } + return a +} + +// AutoAction returns the "right" action for go build or go install of p. +func (b *Builder) AutoAction(mode, depMode BuildMode, p *load.Package) *Action { + if p.Name == "main" { + return b.LinkAction(mode, depMode, p) + } + return b.CompileAction(mode, depMode, p) +} + +// CompileAction returns the action for compiling and possibly installing +// (according to mode) the given package. The resulting action is only +// for building packages (archives), never for linking executables. +// depMode is the action (build or install) to use when building dependencies. +// To turn package main into an executable, call b.Link instead. +func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Action { + if mode == ModeInstall && p.Internal.Local && p.Target == "" { + // Imported via local path. No permanent target. + mode = ModeBuild + } + if mode == ModeInstall && p.Name == "main" { + // We never install the .a file for a main package. + mode = ModeBuild + } + + // Construct package build action. + a := b.cacheAction("build", p, func() *Action { + a := &Action{ + Mode: "build", + Package: p, + Func: (*Builder).build, + Objdir: b.NewObjdir(), + } + a.Target = a.Objdir + "_pkg_.a" + a.built = a.Target + + for _, p1 := range p.Internal.Imports { + a.Deps = append(a.Deps, b.CompileAction(depMode, depMode, p1)) + } + + if p.Standard { + switch p.ImportPath { + case "builtin", "unsafe": + // Fake packages - nothing to build. + a.Mode = "built-in package" + a.Func = nil + return a + } + + // gccgo standard library is "fake" too. + if cfg.BuildToolchainName == "gccgo" { + // the target name is needed for cgo. + a.Mode = "gccgo stdlib" + a.Target = p.Target + a.Func = nil + return a + } + } + + if !p.Stale && p.Target != "" && p.Name != "main" { + // p.Stale==false implies that p.Target is up-to-date. + // Record target name for use by actions depending on this one. + a.Mode = "use installed" + a.Target = p.Target + a.Func = nil + a.built = a.Target + return a + } + return a + }) + + // Construct install action. + if mode == ModeInstall { + a = b.installAction(a) + } + + return a +} + +// LinkAction returns the action for linking p into an executable +// and possibly installing the result (according to mode). +// depMode is the action (build or install) to use when compiling dependencies. +func (b *Builder) LinkAction(mode, depMode BuildMode, p *load.Package) *Action { + // Construct link action. + a := b.cacheAction("link", p, func() *Action { + a := &Action{ + Mode: "link", + Package: p, + } + + if !p.Stale && p.Target != "" { + // p.Stale==false implies that p.Target is up-to-date. + // Record target name for use by actions depending on this one. + a.Mode = "use installed" + a.Func = nil + a.Target = p.Target + a.built = a.Target + return a + } + + a1 := b.CompileAction(ModeBuild, depMode, p) + a.Func = (*Builder).link + a.Deps = []*Action{a1} + a.Objdir = a1.Objdir + + // An executable file. (This is the name of a temporary file.) + // Because we run the temporary file in 'go run' and 'go test', + // the name will show up in ps listings. If the caller has specified + // a name, use that instead of a.out. The binary is generated + // in an otherwise empty subdirectory named exe to avoid + // naming conflicts. The only possible conflict is if we were + // to create a top-level package named exe. + name := "a.out" + if p.Internal.ExeName != "" { + name = p.Internal.ExeName + } else if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" && p.Target != "" { + // On OS X, the linker output name gets recorded in the + // shared library's LC_ID_DYLIB load command. + // The code invoking the linker knows to pass only the final + // path element. Arrange that the path element matches what + // we'll install it as; otherwise the library is only loadable as "a.out". + // On Windows, DLL file name is recorded in PE file + // export section, so do like on OS X. + _, name = filepath.Split(p.Target) + } + a.Target = a.Objdir + filepath.Join("exe", name) + cfg.ExeSuffix + a.built = a.Target + b.addTransitiveLinkDeps(a, a1, "") + return a + }) + + if mode == ModeInstall { + a = b.installAction(a) + } + + return a +} + +// installAction returns the action for installing the result of a1. +func (b *Builder) installAction(a1 *Action) *Action { + // If there's no actual action to build a1, + // there's nothing to install either. + // This happens if a1 corresponds to reusing an already-built object. + if a1.Func == nil { + return a1 + } + + p := a1.Package + return b.cacheAction(a1.Mode+"-install", p, func() *Action { + a := &Action{ + Mode: a1.Mode + "-install", + Func: BuildInstallFunc, + Package: p, + Objdir: a1.Objdir, + Deps: []*Action{a1}, + Target: p.Target, + built: p.Target, + } + b.addInstallHeaderAction(a) + return a + }) +} + +// addTransitiveLinkDeps adds to the link action a all packages +// that are transitive dependencies of a1.Deps. +// That is, if a is a link of package main, a1 is the compile of package main +// and a1.Deps is the actions for building packages directly imported by +// package main (what the compiler needs). The linker needs all packages +// transitively imported by the whole program; addTransitiveLinkDeps +// makes sure those are present in a.Deps. +// If shlib is non-empty, then a corresponds to the build and installation of shlib, +// so any rebuild of shlib should not be added as a dependency. +func (b *Builder) addTransitiveLinkDeps(a, a1 *Action, shlib string) { + // Expand Deps to include all built packages, for the linker. + // Use breadth-first search to find rebuilt-for-test packages + // before the standard ones. + // TODO(rsc): Eliminate the standard ones from the action graph, + // which will require doing a little bit more rebuilding. + workq := []*Action{a1} + haveDep := map[string]bool{} + if a1.Package != nil { + haveDep[a1.Package.ImportPath] = true + } + for i := 0; i < len(workq); i++ { + a1 := workq[i] + for _, a2 := range a1.Deps { + // TODO(rsc): Find a better discriminator than the Mode strings, once the dust settles. + if a2.Package == nil || (a2.Mode != "build-install" && a2.Mode != "build" && a2.Mode != "use installed") || haveDep[a2.Package.ImportPath] { + continue + } + haveDep[a2.Package.ImportPath] = true + a.Deps = append(a.Deps, a2) + if a2.Mode == "build-install" { + a2 = a2.Deps[0] // walk children of "build" action + } + workq = append(workq, a2) + } + } + + // If this is go build -linkshared, then the link depends on the shared libraries + // in addition to the packages themselves. (The compile steps do not.) + if cfg.BuildLinkshared { + haveShlib := map[string]bool{shlib: true} + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.Shlib == "" || haveShlib[filepath.Base(p1.Shlib)] { + continue + } + haveShlib[filepath.Base(p1.Shlib)] = true + // TODO(rsc): The use of ModeInstall here is suspect, but if we only do ModeBuild, + // we'll end up building an overall library or executable that depends at runtime + // on other libraries that are out-of-date, which is clearly not good either. + a.Deps = append(a.Deps, b.linkSharedAction(ModeInstall, ModeInstall, p1.Shlib, nil)) + } + } +} + +// addInstallHeaderAction adds an install header action to a, if needed. +// The action a should be an install action as generated by either +// b.CompileAction or b.LinkAction with mode=ModeInstall, +// and so a.Deps[0] is the corresponding build action. +func (b *Builder) addInstallHeaderAction(a *Action) { + // Install header for cgo in c-archive and c-shared modes. + p := a.Package + if p.UsesCgo() && (cfg.BuildBuildmode == "c-archive" || cfg.BuildBuildmode == "c-shared") { + hdrTarget := a.Target[:len(a.Target)-len(filepath.Ext(a.Target))] + ".h" + if cfg.BuildContext.Compiler == "gccgo" { + // For the header file, remove the "lib" + // added by go/build, so we generate pkg.h + // rather than libpkg.h. + dir, file := filepath.Split(hdrTarget) + file = strings.TrimPrefix(file, "lib") + hdrTarget = filepath.Join(dir, file) + } + ah := &Action{ + Mode: "install header", + Package: a.Package, + Deps: []*Action{a.Deps[0]}, + Func: (*Builder).installHeader, + Objdir: a.Deps[0].Objdir, + Target: hdrTarget, + } + a.Deps = append(a.Deps, ah) + } +} + +// buildmodeShared takes the "go build" action a1 into the building of a shared library of a1.Deps. +// That is, the input a1 represents "go build pkgs" and the result represents "go build -buidmode=shared pkgs". +func (b *Builder) buildmodeShared(mode, depMode BuildMode, args []string, pkgs []*load.Package, a1 *Action) *Action { + name, err := libname(args, pkgs) + if err != nil { + base.Fatalf("%v", err) + } + return b.linkSharedAction(mode, depMode, name, a1) +} + +// linkSharedAction takes a grouping action a1 corresponding to a list of built packages +// and returns an action that links them together into a shared library with the name shlib. +// If a1 is nil, shlib should be an absolute path to an existing shared library, +// and then linkSharedAction reads that library to find out the package list. +func (b *Builder) linkSharedAction(mode, depMode BuildMode, shlib string, a1 *Action) *Action { + fullShlib := shlib + shlib = filepath.Base(shlib) + a := b.cacheAction("build-shlib "+shlib, nil, func() *Action { + if a1 == nil { + // TODO(rsc): Need to find some other place to store config, + // not in pkg directory. See golang.org/issue/22196. + pkgs := readpkglist(fullShlib) + a1 = &Action{ + Mode: "shlib packages", + } + for _, p := range pkgs { + a1.Deps = append(a1.Deps, b.CompileAction(mode, depMode, p)) + } + } + + // Add implicit dependencies to pkgs list. + // Currently buildmode=shared forces external linking mode, and + // external linking mode forces an import of runtime/cgo (and + // math on arm). So if it was not passed on the command line and + // it is not present in another shared library, add it here. + // TODO(rsc): Maybe this should only happen if "runtime" is in the original package set. + // TODO(rsc): This should probably be changed to use load.LinkerDeps(p). + // TODO(rsc): Find out and explain here why gccgo is excluded. + // If the answer is that gccgo is different in implicit linker deps, maybe + // load.LinkerDeps should be used and updated. + if cfg.BuildToolchainName != "gccgo" { + add := func(pkg string) { + for _, a2 := range a1.Deps { + if a2.Package.ImportPath == pkg { + return + } + } + var stk load.ImportStack + p := load.LoadPackage(pkg, &stk) + if p.Error != nil { + base.Fatalf("load %s: %v", pkg, p.Error) + } + load.ComputeStale(p) + // Assume that if pkg (runtime/cgo or math) + // is already accounted for in a different shared library, + // then that shared library also contains runtime, + // so that anything we do will depend on that library, + // so we don't need to include pkg in our shared library. + if p.Shlib == "" || filepath.Base(p.Shlib) == pkg { + a1.Deps = append(a1.Deps, b.CompileAction(depMode, depMode, p)) + } + } + add("runtime/cgo") + if cfg.Goarch == "arm" { + add("math") + } + } + + // Determine the eventual install target and compute staleness. + // TODO(rsc): This doesn't belong here and should be with the + // other staleness code. When we move to content-based staleness + // determination, that will happen for us. + + // The install target is root/pkg/shlib, where root is the source root + // in which all the packages lie. + // TODO(rsc): Perhaps this cross-root check should apply to the full + // transitive package dependency list, not just the ones named + // on the command line? + pkgDir := a1.Deps[0].Package.Internal.Build.PkgTargetRoot + for _, a2 := range a1.Deps { + if dir := a2.Package.Internal.Build.PkgTargetRoot; dir != pkgDir { + // TODO(rsc): Misuse of base.Fatalf? + base.Fatalf("installing shared library: cannot use packages %s and %s from different roots %s and %s", + a1.Deps[0].Package.ImportPath, + a2.Package.ImportPath, + pkgDir, + dir) + } + } + // TODO(rsc): Find out and explain here why gccgo is different. + if cfg.BuildToolchainName == "gccgo" { + pkgDir = filepath.Join(pkgDir, "shlibs") + } + target := filepath.Join(pkgDir, shlib) + + // The install target is stale if it doesn't exist or if it is older than + // any of the .a files that are written into it. + // TODO(rsc): This computation does not detect packages that + // have been removed from a wildcard used to construct the package list + // but are still present in the installed list. + // It would be possible to detect this by reading the pkg list + // out of any installed target, but content-based staleness + // determination should discover that too. + var built time.Time + if fi, err := os.Stat(target); err == nil { + built = fi.ModTime() + } + stale := cfg.BuildA + if !stale { + for _, a2 := range a1.Deps { + if a2.Target == "" { + continue + } + if a2.Func != nil { + // a2 is going to be rebuilt (reuse of existing target would have Func==nil). + stale = true + break + } + info, err := os.Stat(a2.Target) + if err != nil || info.ModTime().After(built) { + stale = true + break + } + } + } + if !stale { + return &Action{ + Mode: "use installed buildmode=shared", + Target: target, + Deps: []*Action{a1}, + } + } + // Link packages into a shared library. + a := &Action{ + Mode: "go build -buildmode=shared", + Objdir: b.NewObjdir(), + Func: (*Builder).linkShared, + Deps: []*Action{a1}, + Args: []string{target}, // awful side-channel for install action + } + a.Target = filepath.Join(a.Objdir, shlib) + b.addTransitiveLinkDeps(a, a1, shlib) + return a + }) + + // Install result. + if mode == ModeInstall && a.Func != nil { + buildAction := a + a = b.cacheAction("install-shlib "+shlib, nil, func() *Action { + a := &Action{ + Mode: "go install -buildmode=shared", + Objdir: buildAction.Objdir, + Func: BuildInstallFunc, + Deps: []*Action{buildAction}, + Target: buildAction.Args[0], + } + for _, a2 := range buildAction.Deps[0].Deps { + p := a2.Package + if p.Target == "" { + continue + } + a.Deps = append(a.Deps, &Action{ + Mode: "shlibname", + Package: p, + Func: (*Builder).installShlibname, + Target: strings.TrimSuffix(p.Target, ".a") + ".shlibname", + Deps: []*Action{a.Deps[0]}, + }) + } + return a + }) + } + + return a +} diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 38aa84934d..478ebc680d 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -5,36 +5,20 @@ package work import ( - "bufio" - "bytes" - "container/heap" - "crypto/sha1" - "crypto/sha256" - "debug/elf" - "encoding/json" "errors" "flag" "fmt" "go/build" - "io" - "io/ioutil" - "log" "os" "os/exec" "path" "path/filepath" - "regexp" "runtime" - "strconv" "strings" - "sync" - "time" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/load" - "cmd/go/internal/str" - "cmd/internal/buildid" ) var CmdBuild = &base.Command{ @@ -651,3475 +635,6 @@ func InstallPackages(args []string, forGet bool) { } } -// A Builder holds global state about a build. -// It does not hold per-package state, because we -// build packages in parallel, and the builder is shared. -type Builder struct { - WorkDir string // the temporary work directory (ends in filepath.Separator) - actionCache map[cacheKey]*Action // a cache of already-constructed actions - mkdirCache map[string]bool // a cache of created directories - flagCache map[[2]string]bool // a cache of supported compiler flags - Print func(args ...interface{}) (int, error) - - objdirSeq int // counter for NewObjdir - pkgSeq int - - output sync.Mutex - scriptDir string // current directory in printed script - - exec sync.Mutex - readySema chan bool - ready actionQueue -} - -// NOTE: Much of Action would not need to be exported if not for test. -// Maybe test functionality should move into this package too? - -// An Action represents a single action in the action graph. -type Action struct { - Mode string // description of action operation - Package *load.Package // the package this action works on - Deps []*Action // actions that must happen before this one - Func func(*Builder, *Action) error // the action itself (nil = no-op) - IgnoreFail bool // whether to run f even if dependencies fail - TestOutput *bytes.Buffer // test output buffer - Args []string // additional args for runProgram - - triggers []*Action // inverse of deps - buildID string - - // Generated files, directories. - Objdir string // directory for intermediate objects - Target string // goal of the action: the created package or executable - built string // the actual created package or executable - - // Execution state. - pending int // number of deps yet to complete - priority int // relative execution priority - Failed bool // whether the action failed -} - -type actionJSON struct { - ID int - Mode string - Package string - Deps []int `json:",omitempty"` - IgnoreFail bool `json:",omitempty"` - Args []string `json:",omitempty"` - Link bool `json:",omitempty"` - Objdir string `json:",omitempty"` - Target string `json:",omitempty"` - Priority int `json:",omitempty"` - Failed bool `json:",omitempty"` - Built string `json:",omitempty"` -} - -// cacheKey is the key for the action cache. -type cacheKey struct { - mode string - p *load.Package -} - -func actionGraphJSON(a *Action) string { - var workq []*Action - var inWorkq = make(map[*Action]int) - - add := func(a *Action) { - if _, ok := inWorkq[a]; ok { - return - } - inWorkq[a] = len(workq) - workq = append(workq, a) - } - add(a) - - for i := 0; i < len(workq); i++ { - for _, dep := range workq[i].Deps { - add(dep) - } - } - - var list []*actionJSON - for id, a := range workq { - aj := &actionJSON{ - Mode: a.Mode, - ID: id, - IgnoreFail: a.IgnoreFail, - Args: a.Args, - Objdir: a.Objdir, - Target: a.Target, - Failed: a.Failed, - Priority: a.priority, - Built: a.built, - } - if a.Package != nil { - // TODO(rsc): Make this a unique key for a.Package somehow. - aj.Package = a.Package.ImportPath - } - for _, a1 := range a.Deps { - aj.Deps = append(aj.Deps, inWorkq[a1]) - } - list = append(list, aj) - } - - js, err := json.MarshalIndent(list, "", "\t") - if err != nil { - fmt.Fprintf(os.Stderr, "go: writing debug action graph: %v\n", err) - return "" - } - return string(js) -} - -// BuildMode specifies the build mode: -// are we just building things or also installing the results? -type BuildMode int - -const ( - ModeBuild BuildMode = iota - ModeInstall -) - -func (b *Builder) Init() { - var err error - b.Print = func(a ...interface{}) (int, error) { - return fmt.Fprint(os.Stderr, a...) - } - b.actionCache = make(map[cacheKey]*Action) - b.mkdirCache = make(map[string]bool) - - if cfg.BuildN { - b.WorkDir = "$WORK" - } else { - b.WorkDir, err = ioutil.TempDir("", "go-build") - if err != nil { - base.Fatalf("%s", err) - } - if cfg.BuildX || cfg.BuildWork { - fmt.Fprintf(os.Stderr, "WORK=%s\n", b.WorkDir) - } - if !cfg.BuildWork { - workdir := b.WorkDir - base.AtExit(func() { os.RemoveAll(workdir) }) - } - } - - if _, ok := cfg.OSArchSupportsCgo[cfg.Goos+"/"+cfg.Goarch]; !ok && cfg.BuildContext.Compiler == "gc" { - fmt.Fprintf(os.Stderr, "cmd/go: unsupported GOOS/GOARCH pair %s/%s\n", cfg.Goos, cfg.Goarch) - os.Exit(2) - } - for _, tag := range cfg.BuildContext.BuildTags { - if strings.Contains(tag, ",") { - fmt.Fprintf(os.Stderr, "cmd/go: -tags space-separated list contains comma\n") - os.Exit(2) - } - } -} - -// NewObjdir returns the name of a fresh object directory under b.WorkDir. -// It is up to the caller to call b.Mkdir on the result at an appropriate time. -// The result ends in a slash, so that file names in that directory -// can be constructed with direct string addition. -// -// NewObjdir must be called only from a single goroutine at a time, -// so it is safe to call during action graph construction, but it must not -// be called during action graph execution. -func (b *Builder) NewObjdir() string { - b.objdirSeq++ - return filepath.Join(b.WorkDir, fmt.Sprintf("b%03d", b.objdirSeq)) + string(filepath.Separator) -} - -// readpkglist returns the list of packages that were built into the shared library -// at shlibpath. For the native toolchain this list is stored, newline separated, in -// an ELF note with name "Go\x00\x00" and type 1. For GCCGO it is extracted from the -// .go_export section. -func readpkglist(shlibpath string) (pkgs []*load.Package) { - var stk load.ImportStack - if cfg.BuildToolchainName == "gccgo" { - f, _ := elf.Open(shlibpath) - sect := f.Section(".go_export") - data, _ := sect.Data() - scanner := bufio.NewScanner(bytes.NewBuffer(data)) - for scanner.Scan() { - t := scanner.Text() - if strings.HasPrefix(t, "pkgpath ") { - t = strings.TrimPrefix(t, "pkgpath ") - t = strings.TrimSuffix(t, ";") - pkgs = append(pkgs, load.LoadPackage(t, &stk)) - } - } - } else { - pkglistbytes, err := buildid.ReadELFNote(shlibpath, "Go\x00\x00", 1) - if err != nil { - base.Fatalf("readELFNote failed: %v", err) - } - scanner := bufio.NewScanner(bytes.NewBuffer(pkglistbytes)) - for scanner.Scan() { - t := scanner.Text() - pkgs = append(pkgs, load.LoadPackage(t, &stk)) - } - } - return -} - -// cacheAction looks up {mode, p} in the cache and returns the resulting action. -// If the cache has no such action, f() is recorded and returned. -func (b *Builder) cacheAction(mode string, p *load.Package, f func() *Action) *Action { - a := b.actionCache[cacheKey{mode, p}] - if a == nil { - a = f() - b.actionCache[cacheKey{mode, p}] = a - } - return a -} - -// AutoAction returns the "right" action for go build or go install of p. -func (b *Builder) AutoAction(mode, depMode BuildMode, p *load.Package) *Action { - if p.Name == "main" { - return b.LinkAction(mode, depMode, p) - } - return b.CompileAction(mode, depMode, p) -} - -// CompileAction returns the action for compiling and possibly installing -// (according to mode) the given package. The resulting action is only -// for building packages (archives), never for linking executables. -// depMode is the action (build or install) to use when building dependencies. -// To turn package main into an executable, call b.Link instead. -func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Action { - if mode == ModeInstall && p.Internal.Local && p.Target == "" { - // Imported via local path. No permanent target. - mode = ModeBuild - } - if mode == ModeInstall && p.Name == "main" { - // We never install the .a file for a main package. - mode = ModeBuild - } - - // Construct package build action. - a := b.cacheAction("build", p, func() *Action { - a := &Action{ - Mode: "build", - Package: p, - Func: (*Builder).build, - Objdir: b.NewObjdir(), - } - a.Target = a.Objdir + "_pkg_.a" - a.built = a.Target - - for _, p1 := range p.Internal.Imports { - a.Deps = append(a.Deps, b.CompileAction(depMode, depMode, p1)) - } - - if p.Standard { - switch p.ImportPath { - case "builtin", "unsafe": - // Fake packages - nothing to build. - a.Mode = "built-in package" - a.Func = nil - return a - } - - // gccgo standard library is "fake" too. - if cfg.BuildToolchainName == "gccgo" { - // the target name is needed for cgo. - a.Mode = "gccgo stdlib" - a.Target = p.Target - a.Func = nil - return a - } - } - - if !p.Stale && p.Target != "" && p.Name != "main" { - // p.Stale==false implies that p.Target is up-to-date. - // Record target name for use by actions depending on this one. - a.Mode = "use installed" - a.Target = p.Target - a.Func = nil - a.built = a.Target - return a - } - return a - }) - - // Construct install action. - if mode == ModeInstall { - a = b.installAction(a) - } - - return a -} - -// LinkAction returns the action for linking p into an executable -// and possibly installing the result (according to mode). -// depMode is the action (build or install) to use when compiling dependencies. -func (b *Builder) LinkAction(mode, depMode BuildMode, p *load.Package) *Action { - // Construct link action. - a := b.cacheAction("link", p, func() *Action { - a := &Action{ - Mode: "link", - Package: p, - } - - if !p.Stale && p.Target != "" { - // p.Stale==false implies that p.Target is up-to-date. - // Record target name for use by actions depending on this one. - a.Mode = "use installed" - a.Func = nil - a.Target = p.Target - a.built = a.Target - return a - } - - a1 := b.CompileAction(ModeBuild, depMode, p) - a.Func = (*Builder).link - a.Deps = []*Action{a1} - a.Objdir = a1.Objdir - - // An executable file. (This is the name of a temporary file.) - // Because we run the temporary file in 'go run' and 'go test', - // the name will show up in ps listings. If the caller has specified - // a name, use that instead of a.out. The binary is generated - // in an otherwise empty subdirectory named exe to avoid - // naming conflicts. The only possible conflict is if we were - // to create a top-level package named exe. - name := "a.out" - if p.Internal.ExeName != "" { - name = p.Internal.ExeName - } else if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" && p.Target != "" { - // On OS X, the linker output name gets recorded in the - // shared library's LC_ID_DYLIB load command. - // The code invoking the linker knows to pass only the final - // path element. Arrange that the path element matches what - // we'll install it as; otherwise the library is only loadable as "a.out". - // On Windows, DLL file name is recorded in PE file - // export section, so do like on OS X. - _, name = filepath.Split(p.Target) - } - a.Target = a.Objdir + filepath.Join("exe", name) + cfg.ExeSuffix - a.built = a.Target - b.addTransitiveLinkDeps(a, a1, "") - return a - }) - - if mode == ModeInstall { - a = b.installAction(a) - } - - return a -} - -// installAction returns the action for installing the result of a1. -func (b *Builder) installAction(a1 *Action) *Action { - // If there's no actual action to build a1, - // there's nothing to install either. - // This happens if a1 corresponds to reusing an already-built object. - if a1.Func == nil { - return a1 - } - - p := a1.Package - return b.cacheAction(a1.Mode+"-install", p, func() *Action { - a := &Action{ - Mode: a1.Mode + "-install", - Func: BuildInstallFunc, - Package: p, - Objdir: a1.Objdir, - Deps: []*Action{a1}, - Target: p.Target, - built: p.Target, - } - b.addInstallHeaderAction(a) - return a - }) -} - -// addTransitiveLinkDeps adds to the link action a all packages -// that are transitive dependencies of a1.Deps. -// That is, if a is a link of package main, a1 is the compile of package main -// and a1.Deps is the actions for building packages directly imported by -// package main (what the compiler needs). The linker needs all packages -// transitively imported by the whole program; addTransitiveLinkDeps -// makes sure those are present in a.Deps. -// If shlib is non-empty, then a corresponds to the build and installation of shlib, -// so any rebuild of shlib should not be added as a dependency. -func (b *Builder) addTransitiveLinkDeps(a, a1 *Action, shlib string) { - // Expand Deps to include all built packages, for the linker. - // Use breadth-first search to find rebuilt-for-test packages - // before the standard ones. - // TODO(rsc): Eliminate the standard ones from the action graph, - // which will require doing a little bit more rebuilding. - workq := []*Action{a1} - haveDep := map[string]bool{} - if a1.Package != nil { - haveDep[a1.Package.ImportPath] = true - } - for i := 0; i < len(workq); i++ { - a1 := workq[i] - for _, a2 := range a1.Deps { - // TODO(rsc): Find a better discriminator than the Mode strings, once the dust settles. - if a2.Package == nil || (a2.Mode != "build-install" && a2.Mode != "build" && a2.Mode != "use installed") || haveDep[a2.Package.ImportPath] { - continue - } - haveDep[a2.Package.ImportPath] = true - a.Deps = append(a.Deps, a2) - if a2.Mode == "build-install" { - a2 = a2.Deps[0] // walk children of "build" action - } - workq = append(workq, a2) - } - } - - // If this is go build -linkshared, then the link depends on the shared libraries - // in addition to the packages themselves. (The compile steps do not.) - if cfg.BuildLinkshared { - haveShlib := map[string]bool{shlib: true} - for _, a1 := range a.Deps { - p1 := a1.Package - if p1 == nil || p1.Shlib == "" || haveShlib[filepath.Base(p1.Shlib)] { - continue - } - haveShlib[filepath.Base(p1.Shlib)] = true - // TODO(rsc): The use of ModeInstall here is suspect, but if we only do ModeBuild, - // we'll end up building an overall library or executable that depends at runtime - // on other libraries that are out-of-date, which is clearly not good either. - a.Deps = append(a.Deps, b.linkSharedAction(ModeInstall, ModeInstall, p1.Shlib, nil)) - } - } -} - -// addInstallHeaderAction adds an install header action to a, if needed. -// The action a should be an install action as generated by either -// b.CompileAction or b.LinkAction with mode=ModeInstall, -// and so a.Deps[0] is the corresponding build action. -func (b *Builder) addInstallHeaderAction(a *Action) { - // Install header for cgo in c-archive and c-shared modes. - p := a.Package - if p.UsesCgo() && (cfg.BuildBuildmode == "c-archive" || cfg.BuildBuildmode == "c-shared") { - hdrTarget := a.Target[:len(a.Target)-len(filepath.Ext(a.Target))] + ".h" - if cfg.BuildContext.Compiler == "gccgo" { - // For the header file, remove the "lib" - // added by go/build, so we generate pkg.h - // rather than libpkg.h. - dir, file := filepath.Split(hdrTarget) - file = strings.TrimPrefix(file, "lib") - hdrTarget = filepath.Join(dir, file) - } - ah := &Action{ - Mode: "install header", - Package: a.Package, - Deps: []*Action{a.Deps[0]}, - Func: (*Builder).installHeader, - Objdir: a.Deps[0].Objdir, - Target: hdrTarget, - } - a.Deps = append(a.Deps, ah) - } -} - -// buildmodeShared takes the "go build" action a1 into the building of a shared library of a1.Deps. -// That is, the input a1 represents "go build pkgs" and the result represents "go build -buidmode=shared pkgs". -func (b *Builder) buildmodeShared(mode, depMode BuildMode, args []string, pkgs []*load.Package, a1 *Action) *Action { - name, err := libname(args, pkgs) - if err != nil { - base.Fatalf("%v", err) - } - return b.linkSharedAction(mode, depMode, name, a1) -} - -// linkSharedAction takes a grouping action a1 corresponding to a list of built packages -// and returns an action that links them together into a shared library with the name shlib. -// If a1 is nil, shlib should be an absolute path to an existing shared library, -// and then linkSharedAction reads that library to find out the package list. -func (b *Builder) linkSharedAction(mode, depMode BuildMode, shlib string, a1 *Action) *Action { - fullShlib := shlib - shlib = filepath.Base(shlib) - a := b.cacheAction("build-shlib "+shlib, nil, func() *Action { - if a1 == nil { - // TODO(rsc): Need to find some other place to store config, - // not in pkg directory. See golang.org/issue/22196. - pkgs := readpkglist(fullShlib) - a1 = &Action{ - Mode: "shlib packages", - } - for _, p := range pkgs { - a1.Deps = append(a1.Deps, b.CompileAction(mode, depMode, p)) - } - } - - // Add implicit dependencies to pkgs list. - // Currently buildmode=shared forces external linking mode, and - // external linking mode forces an import of runtime/cgo (and - // math on arm). So if it was not passed on the command line and - // it is not present in another shared library, add it here. - // TODO(rsc): Maybe this should only happen if "runtime" is in the original package set. - // TODO(rsc): This should probably be changed to use load.LinkerDeps(p). - // TODO(rsc): Find out and explain here why gccgo is excluded. - // If the answer is that gccgo is different in implicit linker deps, maybe - // load.LinkerDeps should be used and updated. - if cfg.BuildToolchainName != "gccgo" { - add := func(pkg string) { - for _, a2 := range a1.Deps { - if a2.Package.ImportPath == pkg { - return - } - } - var stk load.ImportStack - p := load.LoadPackage(pkg, &stk) - if p.Error != nil { - base.Fatalf("load %s: %v", pkg, p.Error) - } - load.ComputeStale(p) - // Assume that if pkg (runtime/cgo or math) - // is already accounted for in a different shared library, - // then that shared library also contains runtime, - // so that anything we do will depend on that library, - // so we don't need to include pkg in our shared library. - if p.Shlib == "" || filepath.Base(p.Shlib) == pkg { - a1.Deps = append(a1.Deps, b.CompileAction(depMode, depMode, p)) - } - } - add("runtime/cgo") - if cfg.Goarch == "arm" { - add("math") - } - } - - // Determine the eventual install target and compute staleness. - // TODO(rsc): This doesn't belong here and should be with the - // other staleness code. When we move to content-based staleness - // determination, that will happen for us. - - // The install target is root/pkg/shlib, where root is the source root - // in which all the packages lie. - // TODO(rsc): Perhaps this cross-root check should apply to the full - // transitive package dependency list, not just the ones named - // on the command line? - pkgDir := a1.Deps[0].Package.Internal.Build.PkgTargetRoot - for _, a2 := range a1.Deps { - if dir := a2.Package.Internal.Build.PkgTargetRoot; dir != pkgDir { - // TODO(rsc): Misuse of base.Fatalf? - base.Fatalf("installing shared library: cannot use packages %s and %s from different roots %s and %s", - a1.Deps[0].Package.ImportPath, - a2.Package.ImportPath, - pkgDir, - dir) - } - } - // TODO(rsc): Find out and explain here why gccgo is different. - if cfg.BuildToolchainName == "gccgo" { - pkgDir = filepath.Join(pkgDir, "shlibs") - } - target := filepath.Join(pkgDir, shlib) - - // The install target is stale if it doesn't exist or if it is older than - // any of the .a files that are written into it. - // TODO(rsc): This computation does not detect packages that - // have been removed from a wildcard used to construct the package list - // but are still present in the installed list. - // It would be possible to detect this by reading the pkg list - // out of any installed target, but content-based staleness - // determination should discover that too. - var built time.Time - if fi, err := os.Stat(target); err == nil { - built = fi.ModTime() - } - stale := cfg.BuildA - if !stale { - for _, a2 := range a1.Deps { - if a2.Target == "" { - continue - } - if a2.Func != nil { - // a2 is going to be rebuilt (reuse of existing target would have Func==nil). - stale = true - break - } - info, err := os.Stat(a2.Target) - if err != nil || info.ModTime().After(built) { - stale = true - break - } - } - } - if !stale { - return &Action{ - Mode: "use installed buildmode=shared", - Target: target, - Deps: []*Action{a1}, - } - } - // Link packages into a shared library. - a := &Action{ - Mode: "go build -buildmode=shared", - Objdir: b.NewObjdir(), - Func: (*Builder).linkShared, - Deps: []*Action{a1}, - Args: []string{target}, // awful side-channel for install action - } - a.Target = filepath.Join(a.Objdir, shlib) - b.addTransitiveLinkDeps(a, a1, shlib) - return a - }) - - // Install result. - if mode == ModeInstall && a.Func != nil { - buildAction := a - a = b.cacheAction("install-shlib "+shlib, nil, func() *Action { - a := &Action{ - Mode: "go install -buildmode=shared", - Objdir: buildAction.Objdir, - Func: BuildInstallFunc, - Deps: []*Action{buildAction}, - Target: buildAction.Args[0], - } - for _, a2 := range buildAction.Deps[0].Deps { - p := a2.Package - if p.Target == "" { - continue - } - a.Deps = append(a.Deps, &Action{ - Mode: "shlibname", - Package: p, - Func: (*Builder).installShlibname, - Target: strings.TrimSuffix(p.Target, ".a") + ".shlibname", - Deps: []*Action{a.Deps[0]}, - }) - } - return a - }) - } - - return a -} - -// actionList returns the list of actions in the dag rooted at root -// as visited in a depth-first post-order traversal. -func actionList(root *Action) []*Action { - seen := map[*Action]bool{} - all := []*Action{} - var walk func(*Action) - walk = func(a *Action) { - if seen[a] { - return - } - seen[a] = true - for _, a1 := range a.Deps { - walk(a1) - } - all = append(all, a) - } - walk(root) - return all -} - -// do runs the action graph rooted at root. -func (b *Builder) Do(root *Action) { - // Build list of all actions, assigning depth-first post-order priority. - // The original implementation here was a true queue - // (using a channel) but it had the effect of getting - // distracted by low-level leaf actions to the detriment - // of completing higher-level actions. The order of - // work does not matter much to overall execution time, - // but when running "go test std" it is nice to see each test - // results as soon as possible. The priorities assigned - // ensure that, all else being equal, the execution prefers - // to do what it would have done first in a simple depth-first - // dependency order traversal. - all := actionList(root) - for i, a := range all { - a.priority = i - } - - if cfg.DebugActiongraph != "" { - js := actionGraphJSON(root) - if err := ioutil.WriteFile(cfg.DebugActiongraph, []byte(js), 0666); err != nil { - fmt.Fprintf(os.Stderr, "go: writing action graph: %v\n", err) - base.SetExitStatus(1) - } - } - - b.readySema = make(chan bool, len(all)) - - // Initialize per-action execution state. - for _, a := range all { - for _, a1 := range a.Deps { - a1.triggers = append(a1.triggers, a) - } - a.pending = len(a.Deps) - if a.pending == 0 { - b.ready.push(a) - b.readySema <- true - } - } - - // Handle runs a single action and takes care of triggering - // any actions that are runnable as a result. - handle := func(a *Action) { - var err error - - if a.Func != nil && (!a.Failed || a.IgnoreFail) { - if a.Objdir != "" { - err = b.Mkdir(a.Objdir) - } - if err == nil { - err = a.Func(b, a) - } - } - - // The actions run in parallel but all the updates to the - // shared work state are serialized through b.exec. - b.exec.Lock() - defer b.exec.Unlock() - - if err != nil { - if err == errPrintedOutput { - base.SetExitStatus(2) - } else { - base.Errorf("%s", err) - } - a.Failed = true - } - - for _, a0 := range a.triggers { - if a.Failed { - a0.Failed = true - } - if a0.pending--; a0.pending == 0 { - b.ready.push(a0) - b.readySema <- true - } - } - - if a == root { - close(b.readySema) - } - } - - var wg sync.WaitGroup - - // Kick off goroutines according to parallelism. - // If we are using the -n flag (just printing commands) - // drop the parallelism to 1, both to make the output - // deterministic and because there is no real work anyway. - par := cfg.BuildP - if cfg.BuildN { - par = 1 - } - for i := 0; i < par; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case _, ok := <-b.readySema: - if !ok { - return - } - // Receiving a value from b.readySema entitles - // us to take from the ready queue. - b.exec.Lock() - a := b.ready.pop() - b.exec.Unlock() - handle(a) - case <-base.Interrupted: - base.SetExitStatus(1) - return - } - } - }() - } - - wg.Wait() -} - -// build is the action for building a single package or command. -func (b *Builder) build(a *Action) (err error) { - // Return an error for binary-only package. - // We only reach this if isStale believes the binary form is - // either not present or not usable. - if a.Package.BinaryOnly { - return fmt.Errorf("missing or invalid package binary for binary-only package %s", a.Package.ImportPath) - } - - defer func() { - if err != nil && err != errPrintedOutput { - err = fmt.Errorf("go build %s: %v", a.Package.ImportPath, err) - } - }() - if cfg.BuildN { - // In -n mode, print a banner between packages. - // The banner is five lines so that when changes to - // different sections of the bootstrap script have to - // be merged, the banners give patch something - // to use to find its context. - b.Print("\n#\n# " + a.Package.ImportPath + "\n#\n\n") - } - - if cfg.BuildV { - b.Print(a.Package.ImportPath + "\n") - } - - objdir := a.Objdir - - // make target directory - dir, _ := filepath.Split(a.Target) - if dir != "" { - if err := b.Mkdir(dir); err != nil { - return err - } - } - - // We want to keep the action ID available for consultation later, - // but we'll append to it the SHA256 of the file (without this ID included). - // We don't know the SHA256 yet, so make one up to find and replace - // later. Becuase the action ID is a hash of the inputs to this built, - // the chance of SHA256(actionID) occurring elsewhere in the result - // of the build is essentially zero, at least in 2017. - actionID := a.Package.Internal.BuildID - if actionID == "" { - return fmt.Errorf("missing action ID") - } - a.buildID = actionID + "." + fmt.Sprintf("%x", sha256.Sum256([]byte(actionID))) - - var gofiles, cgofiles, objdirCgofiles, cfiles, sfiles, cxxfiles, objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string - - gofiles = append(gofiles, a.Package.GoFiles...) - cgofiles = append(cgofiles, a.Package.CgoFiles...) - cfiles = append(cfiles, a.Package.CFiles...) - sfiles = append(sfiles, a.Package.SFiles...) - cxxfiles = append(cxxfiles, a.Package.CXXFiles...) - - if a.Package.UsesCgo() || a.Package.UsesSwig() { - if pcCFLAGS, pcLDFLAGS, err = b.getPkgConfigFlags(a.Package); err != nil { - return - } - } - - // Run SWIG on each .swig and .swigcxx file. - // Each run will generate two files, a .go file and a .c or .cxx file. - // The .go file will use import "C" and is to be processed by cgo. - if a.Package.UsesSwig() { - outGo, outC, outCXX, err := b.swig(a.Package, objdir, pcCFLAGS) - if err != nil { - return err - } - objdirCgofiles = append(objdirCgofiles, outGo...) - cfiles = append(cfiles, outC...) - cxxfiles = append(cxxfiles, outCXX...) - } - - // Run cgo. - if a.Package.UsesCgo() || a.Package.UsesSwig() { - // In a package using cgo, cgo compiles the C, C++ and assembly files with gcc. - // There is one exception: runtime/cgo's job is to bridge the - // cgo and non-cgo worlds, so it necessarily has files in both. - // In that case gcc only gets the gcc_* files. - var gccfiles []string - gccfiles = append(gccfiles, cfiles...) - cfiles = nil - if a.Package.Standard && a.Package.ImportPath == "runtime/cgo" { - filter := func(files, nongcc, gcc []string) ([]string, []string) { - for _, f := range files { - if strings.HasPrefix(f, "gcc_") { - gcc = append(gcc, f) - } else { - nongcc = append(nongcc, f) - } - } - return nongcc, gcc - } - sfiles, gccfiles = filter(sfiles, sfiles[:0], gccfiles) - } else { - for _, sfile := range sfiles { - data, err := ioutil.ReadFile(filepath.Join(a.Package.Dir, sfile)) - if err == nil { - if bytes.HasPrefix(data, []byte("TEXT")) || bytes.Contains(data, []byte("\nTEXT")) || - bytes.HasPrefix(data, []byte("DATA")) || bytes.Contains(data, []byte("\nDATA")) || - bytes.HasPrefix(data, []byte("GLOBL")) || bytes.Contains(data, []byte("\nGLOBL")) { - return fmt.Errorf("package using cgo has Go assembly file %s", sfile) - } - } - } - gccfiles = append(gccfiles, sfiles...) - sfiles = nil - } - - outGo, outObj, err := b.cgo(a, base.Tool("cgo"), objdir, pcCFLAGS, pcLDFLAGS, cgofiles, objdirCgofiles, gccfiles, cxxfiles, a.Package.MFiles, a.Package.FFiles) - if err != nil { - return err - } - if cfg.BuildToolchainName == "gccgo" { - cgoObjects = append(cgoObjects, a.Objdir+"_cgo_flags") - } - cgoObjects = append(cgoObjects, outObj...) - gofiles = append(gofiles, outGo...) - } - - // Sanity check only, since Package.load already checked as well. - if len(gofiles) == 0 { - return &load.NoGoError{Package: a.Package} - } - - // If we're doing coverage, preprocess the .go files and put them in the work directory - if a.Package.Internal.CoverMode != "" { - for i, file := range gofiles { - var sourceFile string - var coverFile string - var key string - if strings.HasSuffix(file, ".cgo1.go") { - // cgo files have absolute paths - base := filepath.Base(file) - sourceFile = file - coverFile = objdir + base - key = strings.TrimSuffix(base, ".cgo1.go") + ".go" - } else { - sourceFile = filepath.Join(a.Package.Dir, file) - coverFile = objdir + file - key = file - } - cover := a.Package.Internal.CoverVars[key] - if cover == nil || base.IsTestFile(file) { - // Not covering this file. - continue - } - if err := b.cover(a, coverFile, sourceFile, 0666, cover.Var); err != nil { - return err - } - gofiles[i] = coverFile - } - } - - // Prepare Go import config. - var icfg bytes.Buffer - for _, a1 := range a.Deps { - p1 := a1.Package - if p1 == nil || p1.ImportPath == "" { - continue - } - path := p1.ImportPath - i := strings.LastIndex(path, "/vendor/") - if i >= 0 { - i += len("/vendor/") - } else if strings.HasPrefix(path, "vendor/") { - i = len("vendor/") - } else { - continue - } - fmt.Fprintf(&icfg, "importmap %s=%s\n", path[i:], path) - } - for _, a1 := range a.Deps { - p1 := a1.Package - if p1 == nil || p1.ImportPath == "" || a1.built == "" { - continue - } - fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built) - } - - // Compile Go. - objpkg := objdir + "_pkg_.a" - ofile, out, err := BuildToolchain.gc(b, a, objpkg, icfg.Bytes(), len(sfiles) > 0, gofiles) - if len(out) > 0 { - b.showOutput(a.Package.Dir, a.Package.ImportPath, b.processOutput(out)) - if err != nil { - return errPrintedOutput - } - } - if err != nil { - return err - } - if ofile != objpkg { - objects = append(objects, ofile) - } - - // Copy .h files named for goos or goarch or goos_goarch - // to names using GOOS and GOARCH. - // For example, defs_linux_amd64.h becomes defs_GOOS_GOARCH.h. - _goos_goarch := "_" + cfg.Goos + "_" + cfg.Goarch - _goos := "_" + cfg.Goos - _goarch := "_" + cfg.Goarch - for _, file := range a.Package.HFiles { - name, ext := fileExtSplit(file) - switch { - case strings.HasSuffix(name, _goos_goarch): - targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext - if err := b.copyFile(a, objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { - return err - } - case strings.HasSuffix(name, _goarch): - targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext - if err := b.copyFile(a, objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { - return err - } - case strings.HasSuffix(name, _goos): - targ := file[:len(name)-len(_goos)] + "_GOOS." + ext - if err := b.copyFile(a, objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { - return err - } - } - } - - for _, file := range cfiles { - out := file[:len(file)-len(".c")] + ".o" - if err := BuildToolchain.cc(b, a, objdir+out, file); err != nil { - return err - } - objects = append(objects, out) - } - - // Assemble .s files. - if len(sfiles) > 0 { - ofiles, err := BuildToolchain.asm(b, a, sfiles) - if err != nil { - return err - } - objects = append(objects, ofiles...) - } - - // NOTE(rsc): On Windows, it is critically important that the - // gcc-compiled objects (cgoObjects) be listed after the ordinary - // objects in the archive. I do not know why this is. - // https://golang.org/issue/2601 - objects = append(objects, cgoObjects...) - - // Add system object files. - for _, syso := range a.Package.SysoFiles { - objects = append(objects, filepath.Join(a.Package.Dir, syso)) - } - - // Pack into archive in objdir directory. - // If the Go compiler wrote an archive, we only need to add the - // object files for non-Go sources to the archive. - // If the Go compiler wrote an archive and the package is entirely - // Go sources, there is no pack to execute at all. - if len(objects) > 0 { - if err := BuildToolchain.pack(b, a, objpkg, objects); err != nil { - return err - } - } - - // Update the binary with the final build ID. - // But if OmitDebug is set, don't, because we set OmitDebug - // on binaries that we are going to run and then delete. - // There's no point in doing work on such a binary. - // Worse, opening the binary for write here makes it - // essentially impossible to safely fork+exec due to a fundamental - // incompatibility between ETXTBSY and threads on modern Unix systems. - // See golang.org/issue/22220. - if !a.Package.Internal.OmitDebug { - if err := b.updateBuildID(a, actionID, objpkg); err != nil { - return err - } - } - - return nil -} - -func (b *Builder) link(a *Action) (err error) { - importcfg := a.Objdir + "importcfg.link" - if err := b.writeLinkImportcfg(a, importcfg); err != nil { - return err - } - - // make target directory - dir, _ := filepath.Split(a.Target) - if dir != "" { - if err := b.Mkdir(dir); err != nil { - return err - } - } - - actionID := a.Package.Internal.BuildID - if actionID == "" { - return fmt.Errorf("missing action ID") - } - a.buildID = actionID + "." + fmt.Sprintf("%x", sha256.Sum256([]byte(actionID))) - - objpkg := a.Objdir + "_pkg_.a" - if err := BuildToolchain.ld(b, a, a.Target, importcfg, objpkg); err != nil { - return err - } - - if err := b.updateBuildID(a, actionID, a.Target); err != nil { - return err - } - - return nil -} - -func (b *Builder) updateBuildID(a *Action, actionID, target string) error { - if cfg.BuildX || cfg.BuildN { - b.Showcmd("", "%s # internal", joinUnambiguously(str.StringList(base.Tool("buildid"), "-w", target))) - if cfg.BuildN { - return nil - } - } - - // Find occurrences of old ID and compute new content-based ID. - r, err := os.Open(target) - if err != nil { - return err - } - matches, hash, err := buildid.FindAndHash(r, a.buildID, 0) - r.Close() - if err != nil { - return err - } - newID := fmt.Sprintf("%s.%x", actionID, hash) - if len(newID) != len(a.buildID) { - return fmt.Errorf("internal error: build ID length mismatch %d+1+%d != %d", len(actionID), len(hash)*2, len(a.buildID)) - } - - // Replace with new content-based ID. - a.buildID = newID - if len(matches) == 0 { - // Assume the user specified -buildid= to override what we were going to choose. - return nil - } - w, err := os.OpenFile(target, os.O_WRONLY, 0) - if err != nil { - return err - } - err = buildid.Rewrite(w, matches, newID) - if err != nil { - w.Close() - return err - } - if err := w.Close(); err != nil { - return err - } - return nil -} - -func (b *Builder) writeLinkImportcfg(a *Action, file string) error { - // Prepare Go import cfg. - var icfg bytes.Buffer - for _, a1 := range a.Deps { - p1 := a1.Package - if p1 == nil { - continue - } - fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built) - if p1.Shlib != "" { - fmt.Fprintf(&icfg, "packageshlib %s=%s\n", p1.ImportPath, p1.Shlib) - } - } - return b.writeFile(file, icfg.Bytes()) -} - -// PkgconfigCmd returns a pkg-config binary name -// defaultPkgConfig is defined in zdefaultcc.go, written by cmd/dist. -func (b *Builder) PkgconfigCmd() string { - return envList("PKG_CONFIG", cfg.DefaultPkgConfig)[0] -} - -// splitPkgConfigOutput parses the pkg-config output into a slice of -// flags. pkg-config always uses \ to escape special characters. -func splitPkgConfigOutput(out []byte) []string { - if len(out) == 0 { - return nil - } - var flags []string - flag := make([]byte, len(out)) - r, w := 0, 0 - for r < len(out) { - switch out[r] { - case ' ', '\t', '\r', '\n': - if w > 0 { - flags = append(flags, string(flag[:w])) - } - w = 0 - case '\\': - r++ - fallthrough - default: - if r < len(out) { - flag[w] = out[r] - w++ - } - } - r++ - } - if w > 0 { - flags = append(flags, string(flag[:w])) - } - return flags -} - -// Calls pkg-config if needed and returns the cflags/ldflags needed to build the package. -func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, err error) { - if pkgs := p.CgoPkgConfig; len(pkgs) > 0 { - var out []byte - out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--cflags", pkgs) - if err != nil { - b.showOutput(p.Dir, b.PkgconfigCmd()+" --cflags "+strings.Join(pkgs, " "), string(out)) - b.Print(err.Error() + "\n") - err = errPrintedOutput - return - } - if len(out) > 0 { - cflags = splitPkgConfigOutput(out) - } - out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--libs", pkgs) - if err != nil { - b.showOutput(p.Dir, b.PkgconfigCmd()+" --libs "+strings.Join(pkgs, " "), string(out)) - b.Print(err.Error() + "\n") - err = errPrintedOutput - return - } - if len(out) > 0 { - ldflags = strings.Fields(string(out)) - } - } - return -} - -func (b *Builder) installShlibname(a *Action) error { - a1 := a.Deps[0] - err := ioutil.WriteFile(a.Target, []byte(filepath.Base(a1.Target)+"\n"), 0666) - if err != nil { - return err - } - if cfg.BuildX { - b.Showcmd("", "echo '%s' > %s # internal", filepath.Base(a1.Target), a.Target) - } - return nil -} - -func (b *Builder) linkShared(a *Action) (err error) { - importcfg := a.Objdir + "importcfg.link" - if err := b.writeLinkImportcfg(a, importcfg); err != nil { - return err - } - return BuildToolchain.ldShared(b, a.Deps[0].Deps, a.Target, importcfg, a.Deps) -} - -// BuildInstallFunc is the action for installing a single package or executable. -func BuildInstallFunc(b *Builder, a *Action) (err error) { - defer func() { - if err != nil && err != errPrintedOutput { - // a.Package == nil is possible for the go install -buildmode=shared - // action that installs libmangledname.so, which corresponds to - // a list of packages, not just one. - sep, path := "", "" - if a.Package != nil { - sep, path = " ", a.Package.ImportPath - } - err = fmt.Errorf("go install%s%s: %v", sep, path, err) - } - }() - a1 := a.Deps[0] - perm := os.FileMode(0666) - if a1.Mode == "link" { - switch cfg.BuildBuildmode { - case "c-archive", "c-shared", "plugin": - default: - perm = 0777 - } - } - - // make target directory - dir, _ := filepath.Split(a.Target) - if dir != "" { - if err := b.Mkdir(dir); err != nil { - return err - } - } - - // remove object dir to keep the amount of - // garbage down in a large build. On an operating system - // with aggressive buffering, cleaning incrementally like - // this keeps the intermediate objects from hitting the disk. - if !cfg.BuildWork { - defer func() { - if cfg.BuildX { - b.Showcmd("", "rm -r %s", a1.Objdir) - } - os.RemoveAll(a1.Objdir) - if _, err := os.Stat(a1.Target); err == nil { - if cfg.BuildX { - b.Showcmd("", "rm %s", a1.Target) - } - os.Remove(a1.Target) - } - }() - } - - return b.moveOrCopyFile(a, a.Target, a1.Target, perm, false) -} - -// moveOrCopyFile is like 'mv src dst' or 'cp src dst'. -func (b *Builder) moveOrCopyFile(a *Action, dst, src string, perm os.FileMode, force bool) error { - if cfg.BuildN { - b.Showcmd("", "mv %s %s", src, dst) - return nil - } - - // If we can update the mode and rename to the dst, do it. - // Otherwise fall back to standard copy. - - // If the destination directory has the group sticky bit set, - // we have to copy the file to retain the correct permissions. - // https://golang.org/issue/18878 - if fi, err := os.Stat(filepath.Dir(dst)); err == nil { - if fi.IsDir() && (fi.Mode()&os.ModeSetgid) != 0 { - return b.copyFile(a, dst, src, perm, force) - } - } - - // The perm argument is meant to be adjusted according to umask, - // but we don't know what the umask is. - // Create a dummy file to find out. - // This avoids build tags and works even on systems like Plan 9 - // where the file mask computation incorporates other information. - mode := perm - f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) - if err == nil { - fi, err := f.Stat() - if err == nil { - mode = fi.Mode() & 0777 - } - name := f.Name() - f.Close() - os.Remove(name) - } - - if err := os.Chmod(src, mode); err == nil { - if err := os.Rename(src, dst); err == nil { - if cfg.BuildX { - b.Showcmd("", "mv %s %s", src, dst) - } - return nil - } - } - - return b.copyFile(a, dst, src, perm, force) -} - -// copyFile is like 'cp src dst'. -func (b *Builder) copyFile(a *Action, dst, src string, perm os.FileMode, force bool) error { - if cfg.BuildN || cfg.BuildX { - b.Showcmd("", "cp %s %s", src, dst) - if cfg.BuildN { - return nil - } - } - - sf, err := os.Open(src) - if err != nil { - return err - } - defer sf.Close() - - // Be careful about removing/overwriting dst. - // Do not remove/overwrite if dst exists and is a directory - // or a non-object file. - if fi, err := os.Stat(dst); err == nil { - if fi.IsDir() { - return fmt.Errorf("build output %q already exists and is a directory", dst) - } - if !force && fi.Mode().IsRegular() && !isObject(dst) { - return fmt.Errorf("build output %q already exists and is not an object file", dst) - } - } - - // On Windows, remove lingering ~ file from last attempt. - if base.ToolIsWindows { - if _, err := os.Stat(dst + "~"); err == nil { - os.Remove(dst + "~") - } - } - - mayberemovefile(dst) - df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - if err != nil && base.ToolIsWindows { - // Windows does not allow deletion of a binary file - // while it is executing. Try to move it out of the way. - // If the move fails, which is likely, we'll try again the - // next time we do an install of this binary. - if err := os.Rename(dst, dst+"~"); err == nil { - os.Remove(dst + "~") - } - df, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - } - if err != nil { - return err - } - - _, err = io.Copy(df, sf) - df.Close() - if err != nil { - mayberemovefile(dst) - return fmt.Errorf("copying %s to %s: %v", src, dst, err) - } - return nil -} - -// writeFile writes the text to file. -func (b *Builder) writeFile(file string, text []byte) error { - if cfg.BuildN || cfg.BuildX { - b.Showcmd("", "cat >%s << 'EOF' # internal\n%sEOF", file, text) - } - if cfg.BuildN { - return nil - } - return ioutil.WriteFile(file, text, 0666) -} - -// Install the cgo export header file, if there is one. -func (b *Builder) installHeader(a *Action) error { - src := a.Objdir + "_cgo_install.h" - if _, err := os.Stat(src); os.IsNotExist(err) { - // If the file does not exist, there are no exported - // functions, and we do not install anything. - if cfg.BuildX { - b.Showcmd("", "# %s not created", src) - } - return nil - } - - dir, _ := filepath.Split(a.Target) - if dir != "" { - if err := b.Mkdir(dir); err != nil { - return err - } - } - - return b.moveOrCopyFile(a, a.Target, src, 0666, true) -} - -// cover runs, in effect, -// go tool cover -mode=b.coverMode -var="varName" -o dst.go src.go -func (b *Builder) cover(a *Action, dst, src string, perm os.FileMode, varName string) error { - return b.run(a.Objdir, "cover "+a.Package.ImportPath, nil, - cfg.BuildToolexec, - base.Tool("cover"), - "-mode", a.Package.Internal.CoverMode, - "-var", varName, - "-o", dst, - src) -} - -var objectMagic = [][]byte{ - {'!', '<', 'a', 'r', 'c', 'h', '>', '\n'}, // Package archive - {'\x7F', 'E', 'L', 'F'}, // ELF - {0xFE, 0xED, 0xFA, 0xCE}, // Mach-O big-endian 32-bit - {0xFE, 0xED, 0xFA, 0xCF}, // Mach-O big-endian 64-bit - {0xCE, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 32-bit - {0xCF, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 64-bit - {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc - {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 - {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 - {0x00, 0x00, 0x06, 0x47}, // Plan 9 arm -} - -func isObject(s string) bool { - f, err := os.Open(s) - if err != nil { - return false - } - defer f.Close() - buf := make([]byte, 64) - io.ReadFull(f, buf) - for _, magic := range objectMagic { - if bytes.HasPrefix(buf, magic) { - return true - } - } - return false -} - -// mayberemovefile removes a file only if it is a regular file -// When running as a user with sufficient privileges, we may delete -// even device files, for example, which is not intended. -func mayberemovefile(s string) { - if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() { - return - } - os.Remove(s) -} - -// fmtcmd formats a command in the manner of fmt.Sprintf but also: -// -// If dir is non-empty and the script is not in dir right now, -// fmtcmd inserts "cd dir\n" before the command. -// -// fmtcmd replaces the value of b.WorkDir with $WORK. -// fmtcmd replaces the value of goroot with $GOROOT. -// fmtcmd replaces the value of b.gobin with $GOBIN. -// -// fmtcmd replaces the name of the current directory with dot (.) -// but only when it is at the beginning of a space-separated token. -// -func (b *Builder) fmtcmd(dir string, format string, args ...interface{}) string { - cmd := fmt.Sprintf(format, args...) - if dir != "" && dir != "/" { - cmd = strings.Replace(" "+cmd, " "+dir, " .", -1)[1:] - if b.scriptDir != dir { - b.scriptDir = dir - cmd = "cd " + dir + "\n" + cmd - } - } - if b.WorkDir != "" { - cmd = strings.Replace(cmd, b.WorkDir, "$WORK", -1) - } - return cmd -} - -// showcmd prints the given command to standard output -// for the implementation of -n or -x. -func (b *Builder) Showcmd(dir string, format string, args ...interface{}) { - b.output.Lock() - defer b.output.Unlock() - b.Print(b.fmtcmd(dir, format, args...) + "\n") -} - -// showOutput prints "# desc" followed by the given output. -// The output is expected to contain references to 'dir', usually -// the source directory for the package that has failed to build. -// showOutput rewrites mentions of dir with a relative path to dir -// when the relative path is shorter. This is usually more pleasant. -// For example, if fmt doesn't compile and we are in src/html, -// the output is -// -// $ go build -// # fmt -// ../fmt/print.go:1090: undefined: asdf -// $ -// -// instead of -// -// $ go build -// # fmt -// /usr/gopher/go/src/fmt/print.go:1090: undefined: asdf -// $ -// -// showOutput also replaces references to the work directory with $WORK. -// -func (b *Builder) showOutput(dir, desc, out string) { - prefix := "# " + desc - suffix := "\n" + out - if reldir := base.ShortPath(dir); reldir != dir { - suffix = strings.Replace(suffix, " "+dir, " "+reldir, -1) - suffix = strings.Replace(suffix, "\n"+dir, "\n"+reldir, -1) - } - suffix = strings.Replace(suffix, " "+b.WorkDir, " $WORK", -1) - - b.output.Lock() - defer b.output.Unlock() - b.Print(prefix, suffix) -} - -// errPrintedOutput is a special error indicating that a command failed -// but that it generated output as well, and that output has already -// been printed, so there's no point showing 'exit status 1' or whatever -// the wait status was. The main executor, builder.do, knows not to -// print this error. -var errPrintedOutput = errors.New("already printed output - no need to show error") - -var cgoLine = regexp.MustCompile(`\[[^\[\]]+\.cgo1\.go:[0-9]+(:[0-9]+)?\]`) -var cgoTypeSigRe = regexp.MustCompile(`\b_Ctype_\B`) - -// run runs the command given by cmdline in the directory dir. -// If the command fails, run prints information about the failure -// and returns a non-nil error. -func (b *Builder) run(dir string, desc string, env []string, cmdargs ...interface{}) error { - out, err := b.runOut(dir, desc, env, cmdargs...) - if len(out) > 0 { - if desc == "" { - desc = b.fmtcmd(dir, "%s", strings.Join(str.StringList(cmdargs...), " ")) - } - b.showOutput(dir, desc, b.processOutput(out)) - if err != nil { - err = errPrintedOutput - } - } - return err -} - -// processOutput prepares the output of runOut to be output to the console. -func (b *Builder) processOutput(out []byte) string { - if out[len(out)-1] != '\n' { - out = append(out, '\n') - } - messages := string(out) - // Fix up output referring to cgo-generated code to be more readable. - // Replace x.go:19[/tmp/.../x.cgo1.go:18] with x.go:19. - // Replace *[100]_Ctype_foo with *[100]C.foo. - // If we're using -x, assume we're debugging and want the full dump, so disable the rewrite. - if !cfg.BuildX && cgoLine.MatchString(messages) { - messages = cgoLine.ReplaceAllString(messages, "") - messages = cgoTypeSigRe.ReplaceAllString(messages, "C.") - } - return messages -} - -// runOut runs the command given by cmdline in the directory dir. -// It returns the command output and any errors that occurred. -func (b *Builder) runOut(dir string, desc string, env []string, cmdargs ...interface{}) ([]byte, error) { - cmdline := str.StringList(cmdargs...) - if cfg.BuildN || cfg.BuildX { - var envcmdline string - for _, e := range env { - if j := strings.IndexByte(e, '='); j != -1 { - if strings.ContainsRune(e[j+1:], '\'') { - envcmdline += fmt.Sprintf("%s=%q", e[:j], e[j+1:]) - } else { - envcmdline += fmt.Sprintf("%s='%s'", e[:j], e[j+1:]) - } - envcmdline += " " - } - } - envcmdline += joinUnambiguously(cmdline) - b.Showcmd(dir, "%s", envcmdline) - if cfg.BuildN { - return nil, nil - } - } - - nbusy := 0 - for { - var buf bytes.Buffer - cmd := exec.Command(cmdline[0], cmdline[1:]...) - cmd.Stdout = &buf - cmd.Stderr = &buf - cmd.Dir = dir - cmd.Env = base.MergeEnvLists(env, base.EnvForDir(cmd.Dir, os.Environ())) - err := cmd.Run() - - // cmd.Run will fail on Unix if some other process has the binary - // we want to run open for writing. This can happen here because - // we build and install the cgo command and then run it. - // If another command was kicked off while we were writing the - // cgo binary, the child process for that command may be holding - // a reference to the fd, keeping us from running exec. - // - // But, you might reasonably wonder, how can this happen? - // The cgo fd, like all our fds, is close-on-exec, so that we need - // not worry about other processes inheriting the fd accidentally. - // The answer is that running a command is fork and exec. - // A child forked while the cgo fd is open inherits that fd. - // Until the child has called exec, it holds the fd open and the - // kernel will not let us run cgo. Even if the child were to close - // the fd explicitly, it would still be open from the time of the fork - // until the time of the explicit close, and the race would remain. - // - // On Unix systems, this results in ETXTBSY, which formats - // as "text file busy". Rather than hard-code specific error cases, - // we just look for that string. If this happens, sleep a little - // and try again. We let this happen three times, with increasing - // sleep lengths: 100+200+400 ms = 0.7 seconds. - // - // An alternate solution might be to split the cmd.Run into - // separate cmd.Start and cmd.Wait, and then use an RWLock - // to make sure that copyFile only executes when no cmd.Start - // call is in progress. However, cmd.Start (really syscall.forkExec) - // only guarantees that when it returns, the exec is committed to - // happen and succeed. It uses a close-on-exec file descriptor - // itself to determine this, so we know that when cmd.Start returns, - // at least one close-on-exec file descriptor has been closed. - // However, we cannot be sure that all of them have been closed, - // so the program might still encounter ETXTBSY even with such - // an RWLock. The race window would be smaller, perhaps, but not - // guaranteed to be gone. - // - // Sleeping when we observe the race seems to be the most reliable - // option we have. - // - // https://golang.org/issue/3001 - // - if err != nil && nbusy < 3 && strings.Contains(err.Error(), "text file busy") { - time.Sleep(100 * time.Millisecond << uint(nbusy)) - nbusy++ - continue - } - - // err can be something like 'exit status 1'. - // Add information about what program was running. - // Note that if buf.Bytes() is non-empty, the caller usually - // shows buf.Bytes() and does not print err at all, so the - // prefix here does not make most output any more verbose. - if err != nil { - err = errors.New(cmdline[0] + ": " + err.Error()) - } - return buf.Bytes(), err - } -} - -// joinUnambiguously prints the slice, quoting where necessary to make the -// output unambiguous. -// TODO: See issue 5279. The printing of commands needs a complete redo. -func joinUnambiguously(a []string) string { - var buf bytes.Buffer - for i, s := range a { - if i > 0 { - buf.WriteByte(' ') - } - q := strconv.Quote(s) - if s == "" || strings.Contains(s, " ") || len(q) > len(s)+2 { - buf.WriteString(q) - } else { - buf.WriteString(s) - } - } - return buf.String() -} - -// mkdir makes the named directory. -func (b *Builder) Mkdir(dir string) error { - b.exec.Lock() - defer b.exec.Unlock() - // We can be a little aggressive about being - // sure directories exist. Skip repeated calls. - if b.mkdirCache[dir] { - return nil - } - b.mkdirCache[dir] = true - - if cfg.BuildN || cfg.BuildX { - b.Showcmd("", "mkdir -p %s", dir) - if cfg.BuildN { - return nil - } - } - - if err := os.MkdirAll(dir, 0777); err != nil { - return err - } - return nil -} - -// symlink creates a symlink newname -> oldname. -func (b *Builder) Symlink(oldname, newname string) error { - if cfg.BuildN || cfg.BuildX { - b.Showcmd("", "ln -s %s %s", oldname, newname) - if cfg.BuildN { - return nil - } - } - return os.Symlink(oldname, newname) -} - -// mkAbs returns an absolute path corresponding to -// evaluating f in the directory dir. -// We always pass absolute paths of source files so that -// the error messages will include the full path to a file -// in need of attention. -func mkAbs(dir, f string) string { - // Leave absolute paths alone. - // Also, during -n mode we use the pseudo-directory $WORK - // instead of creating an actual work directory that won't be used. - // Leave paths beginning with $WORK alone too. - if filepath.IsAbs(f) || strings.HasPrefix(f, "$WORK") { - return f - } - return filepath.Join(dir, f) -} - -type toolchain interface { - // gc runs the compiler in a specific directory on a set of files - // and returns the name of the generated output file. - gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) - // cc runs the toolchain's C compiler in a directory on a C file - // to produce an output file. - cc(b *Builder, a *Action, ofile, cfile string) error - // asm runs the assembler in a specific directory on specific files - // and returns a list of named output files. - asm(b *Builder, a *Action, sfiles []string) ([]string, error) - // pack runs the archive packer in a specific directory to create - // an archive from a set of object files. - // typically it is run in the object directory. - pack(b *Builder, a *Action, afile string, ofiles []string) error - // ld runs the linker to create an executable starting at mainpkg. - ld(b *Builder, root *Action, out, importcfg, mainpkg string) error - // ldShared runs the linker to create a shared library containing the pkgs built by toplevelactions - ldShared(b *Builder, toplevelactions []*Action, out, importcfg string, allactions []*Action) error - - compiler() string - linker() string -} - -type noToolchain struct{} - -func noCompiler() error { - log.Fatalf("unknown compiler %q", cfg.BuildContext.Compiler) - return nil -} - -func (noToolchain) compiler() string { - noCompiler() - return "" -} - -func (noToolchain) linker() string { - noCompiler() - return "" -} - -func (noToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) { - return "", nil, noCompiler() -} - -func (noToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { - return nil, noCompiler() -} - -func (noToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { - return noCompiler() -} - -func (noToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { - return noCompiler() -} - -func (noToolchain) ldShared(b *Builder, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { - return noCompiler() -} - -func (noToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { - return noCompiler() -} - -// The Go toolchain. -type gcToolchain struct{} - -func (gcToolchain) compiler() string { - return base.Tool("compile") -} - -func (gcToolchain) linker() string { - return base.Tool("link") -} - -func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { - p := a.Package - objdir := a.Objdir - if archive != "" { - ofile = archive - } else { - out := "_go_.o" - ofile = objdir + out - } - - pkgpath := p.ImportPath - if cfg.BuildBuildmode == "plugin" { - pkgpath = pluginPath(p) - } else if p.Name == "main" { - pkgpath = "main" - } - gcargs := []string{"-p", pkgpath} - if p.Standard { - gcargs = append(gcargs, "-std") - } - compilingRuntime := p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) - if compilingRuntime { - // runtime compiles with a special gc flag to emit - // additional reflect type data. - gcargs = append(gcargs, "-+") - } - - // If we're giving the compiler the entire package (no C etc files), tell it that, - // so that it can give good error messages about forward declarations. - // Exceptions: a few standard packages have forward declarations for - // pieces supplied behind-the-scenes by package runtime. - extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) - if p.Standard { - switch p.ImportPath { - case "bytes", "internal/poll", "net", "os", "runtime/pprof", "sync", "syscall", "time": - extFiles++ - } - } - if extFiles == 0 { - gcargs = append(gcargs, "-complete") - } - if cfg.BuildContext.InstallSuffix != "" { - gcargs = append(gcargs, "-installsuffix", cfg.BuildContext.InstallSuffix) - } - if a.buildID != "" { - gcargs = append(gcargs, "-buildid", a.buildID) - } - platform := cfg.Goos + "/" + cfg.Goarch - if p.Internal.OmitDebug || platform == "nacl/amd64p32" || platform == "darwin/arm" || platform == "darwin/arm64" || cfg.Goos == "plan9" { - gcargs = append(gcargs, "-dwarf=false") - } - - gcflags := buildGcflags - if compilingRuntime { - // Remove -N, if present. - // It is not possible to build the runtime with no optimizations, - // because the compiler cannot eliminate enough write barriers. - gcflags = make([]string, len(buildGcflags)) - copy(gcflags, buildGcflags) - for i := 0; i < len(gcflags); i++ { - if gcflags[i] == "-N" { - copy(gcflags[i:], gcflags[i+1:]) - gcflags = gcflags[:len(gcflags)-1] - i-- - } - } - } - args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", b.WorkDir, gcflags, gcargs, "-D", p.Internal.LocalPrefix} - if importcfg != nil { - if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { - return "", nil, err - } - args = append(args, "-importcfg", objdir+"importcfg") - } - if ofile == archive { - args = append(args, "-pack") - } - if asmhdr { - args = append(args, "-asmhdr", objdir+"go_asm.h") - } - - // Add -c=N to use concurrent backend compilation, if possible. - if c := gcBackendConcurrency(gcflags); c > 1 { - args = append(args, fmt.Sprintf("-c=%d", c)) - } - - for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) - } - - output, err = b.runOut(p.Dir, p.ImportPath, nil, args...) - return ofile, output, err -} - -// gcBackendConcurrency returns the backend compiler concurrency level for a package compilation. -func gcBackendConcurrency(gcflags []string) int { - // First, check whether we can use -c at all for this compilation. - canDashC := concurrentGCBackendCompilationEnabledByDefault - - switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e { - case "0": - canDashC = false - case "1": - canDashC = true - case "": - // Not set. Use default. - default: - log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e) - } - - if os.Getenv("GOEXPERIMENT") != "" { - // Concurrent compilation is presumed incompatible with GOEXPERIMENTs. - canDashC = false - } - -CheckFlags: - for _, flag := range gcflags { - // Concurrent compilation is presumed incompatible with any gcflags, - // except for a small whitelist of commonly used flags. - // If the user knows better, they can manually add their own -c to the gcflags. - switch flag { - case "-N", "-l", "-S", "-B", "-C", "-I": - // OK - default: - canDashC = false - break CheckFlags - } - } - - if !canDashC { - return 1 - } - - // Decide how many concurrent backend compilations to allow. - // - // If we allow too many, in theory we might end up with p concurrent processes, - // each with c concurrent backend compiles, all fighting over the same resources. - // However, in practice, that seems not to happen too much. - // Most build graphs are surprisingly serial, so p==1 for much of the build. - // Furthermore, concurrent backend compilation is only enabled for a part - // of the overall compiler execution, so c==1 for much of the build. - // So don't worry too much about that interaction for now. - // - // However, in practice, setting c above 4 tends not to help very much. - // See the analysis in CL 41192. - // - // TODO(josharian): attempt to detect whether this particular compilation - // is likely to be a bottleneck, e.g. when: - // - it has no successor packages to compile (usually package main) - // - all paths through the build graph pass through it - // - critical path scheduling says it is high priority - // and in such a case, set c to runtime.NumCPU. - // We do this now when p==1. - if cfg.BuildP == 1 { - // No process parallelism. Max out c. - return runtime.NumCPU() - } - // Some process parallelism. Set c to min(4, numcpu). - c := 4 - if ncpu := runtime.NumCPU(); ncpu < c { - c = ncpu - } - return c -} - -func (gcToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { - p := a.Package - // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. - inc := filepath.Join(cfg.GOROOT, "pkg", "include") - args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-trimpath", b.WorkDir, "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, buildAsmflags} - if p.ImportPath == "runtime" && cfg.Goarch == "386" { - for _, arg := range buildAsmflags { - if arg == "-dynlink" { - args = append(args, "-D=GOBUILDMODE_shared=1") - } - } - } - var ofiles []string - for _, sfile := range sfiles { - ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" - ofiles = append(ofiles, ofile) - a := append(args, "-o", ofile, mkAbs(p.Dir, sfile)) - if err := b.run(p.Dir, p.ImportPath, nil, a...); err != nil { - return nil, err - } - } - return ofiles, nil -} - -// toolVerify checks that the command line args writes the same output file -// if run using newTool instead. -// Unused now but kept around for future use. -func toolVerify(b *Builder, p *load.Package, newTool string, ofile string, args []interface{}) error { - newArgs := make([]interface{}, len(args)) - copy(newArgs, args) - newArgs[1] = base.Tool(newTool) - newArgs[3] = ofile + ".new" // x.6 becomes x.6.new - if err := b.run(p.Dir, p.ImportPath, nil, newArgs...); err != nil { - return err - } - data1, err := ioutil.ReadFile(ofile) - if err != nil { - return err - } - data2, err := ioutil.ReadFile(ofile + ".new") - if err != nil { - return err - } - if !bytes.Equal(data1, data2) { - return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(str.StringList(args...), " "), strings.Join(str.StringList(newArgs...), " ")) - } - os.Remove(ofile + ".new") - return nil -} - -func (gcToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { - var absOfiles []string - for _, f := range ofiles { - absOfiles = append(absOfiles, mkAbs(a.Objdir, f)) - } - absAfile := mkAbs(a.Objdir, afile) - - // The archive file should have been created by the compiler. - // Since it used to not work that way, verify. - if !cfg.BuildN { - if _, err := os.Stat(absAfile); err != nil { - base.Fatalf("os.Stat of archive file failed: %v", err) - } - } - - p := a.Package - if cfg.BuildN || cfg.BuildX { - cmdline := str.StringList(base.Tool("pack"), "r", absAfile, absOfiles) - b.Showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) - } - if cfg.BuildN { - return nil - } - if err := packInternal(b, absAfile, absOfiles); err != nil { - b.showOutput(p.Dir, p.ImportPath, err.Error()+"\n") - return errPrintedOutput - } - return nil -} - -func packInternal(b *Builder, afile string, ofiles []string) error { - dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0) - if err != nil { - return err - } - defer dst.Close() // only for error returns or panics - w := bufio.NewWriter(dst) - - for _, ofile := range ofiles { - src, err := os.Open(ofile) - if err != nil { - return err - } - fi, err := src.Stat() - if err != nil { - src.Close() - return err - } - // Note: Not using %-16.16s format because we care - // about bytes, not runes. - name := fi.Name() - if len(name) > 16 { - name = name[:16] - } else { - name += strings.Repeat(" ", 16-len(name)) - } - size := fi.Size() - fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n", - name, 0, 0, 0, 0644, size) - n, err := io.Copy(w, src) - src.Close() - if err == nil && n < size { - err = io.ErrUnexpectedEOF - } else if err == nil && n > size { - err = fmt.Errorf("file larger than size reported by stat") - } - if err != nil { - return fmt.Errorf("copying %s to %s: %v", ofile, afile, err) - } - if size&1 != 0 { - w.WriteByte(0) - } - } - - if err := w.Flush(); err != nil { - return err - } - return dst.Close() -} - -// setextld sets the appropriate linker flags for the specified compiler. -func setextld(ldflags []string, compiler []string) []string { - for _, f := range ldflags { - if f == "-extld" || strings.HasPrefix(f, "-extld=") { - // don't override -extld if supplied - return ldflags - } - } - ldflags = append(ldflags, "-extld="+compiler[0]) - if len(compiler) > 1 { - extldflags := false - add := strings.Join(compiler[1:], " ") - for i, f := range ldflags { - if f == "-extldflags" && i+1 < len(ldflags) { - ldflags[i+1] = add + " " + ldflags[i+1] - extldflags = true - break - } else if strings.HasPrefix(f, "-extldflags=") { - ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] - extldflags = true - break - } - } - if !extldflags { - ldflags = append(ldflags, "-extldflags="+add) - } - } - return ldflags -} - -// pluginPath computes the package path for a plugin main package. -// -// This is typically the import path of the main package p, unless the -// plugin is being built directly from source files. In that case we -// combine the package build ID with the contents of the main package -// source files. This allows us to identify two different plugins -// built from two source files with the same name. -func pluginPath(p *load.Package) string { - if p.ImportPath != "command-line-arguments" { - return p.ImportPath - } - h := sha1.New() - fmt.Fprintf(h, "build ID: %s\n", p.Internal.BuildID) - for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) { - data, err := ioutil.ReadFile(filepath.Join(p.Dir, file)) - if err != nil { - base.Fatalf("go: %s", err) - } - h.Write(data) - } - return fmt.Sprintf("plugin/unnamed-%x", h.Sum(nil)) -} - -func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { - cxx := len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 - for _, a := range root.Deps { - if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { - cxx = true - } - } - var ldflags []string - if cfg.BuildContext.InstallSuffix != "" { - ldflags = append(ldflags, "-installsuffix", cfg.BuildContext.InstallSuffix) - } - if root.Package.Internal.OmitDebug { - ldflags = append(ldflags, "-s", "-w") - } - if cfg.BuildBuildmode == "plugin" { - ldflags = append(ldflags, "-pluginpath", pluginPath(root.Package)) - } - - // TODO(rsc): This is probably wrong - see golang.org/issue/22155. - if cfg.GOROOT != runtime.GOROOT() { - ldflags = append(ldflags, "-X=runtime/internal/sys.DefaultGoroot="+cfg.GOROOT) - } - - // Store BuildID inside toolchain binaries as a unique identifier of the - // tool being run, for use by content-based staleness determination. - if root.Package.Goroot && strings.HasPrefix(root.Package.ImportPath, "cmd/") { - ldflags = append(ldflags, "-X=cmd/internal/objabi.buildID="+root.buildID) - } - - // If the user has not specified the -extld option, then specify the - // appropriate linker. In case of C++ code, use the compiler named - // by the CXX environment variable or defaultCXX if CXX is not set. - // Else, use the CC environment variable and defaultCC as fallback. - var compiler []string - if cxx { - compiler = envList("CXX", cfg.DefaultCXX) - } else { - compiler = envList("CC", cfg.DefaultCC) - } - ldflags = append(ldflags, "-buildmode="+ldBuildmode) - if root.buildID != "" { - ldflags = append(ldflags, "-buildid="+root.buildID) - } - ldflags = append(ldflags, cfg.BuildLdflags...) - ldflags = setextld(ldflags, compiler) - - // On OS X when using external linking to build a shared library, - // the argument passed here to -o ends up recorded in the final - // shared library in the LC_ID_DYLIB load command. - // To avoid putting the temporary output directory name there - // (and making the resulting shared library useless), - // run the link in the output directory so that -o can name - // just the final path element. - // On Windows, DLL file name is recorded in PE file - // export section, so do like on OS X. - dir := "." - if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" { - dir, out = filepath.Split(out) - } - - return b.run(dir, root.Package.ImportPath, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) -} - -func (gcToolchain) ldShared(b *Builder, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { - ldflags := []string{"-installsuffix", cfg.BuildContext.InstallSuffix} - ldflags = append(ldflags, "-buildmode=shared") - ldflags = append(ldflags, cfg.BuildLdflags...) - cxx := false - for _, a := range allactions { - if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { - cxx = true - } - } - // If the user has not specified the -extld option, then specify the - // appropriate linker. In case of C++ code, use the compiler named - // by the CXX environment variable or defaultCXX if CXX is not set. - // Else, use the CC environment variable and defaultCC as fallback. - var compiler []string - if cxx { - compiler = envList("CXX", cfg.DefaultCXX) - } else { - compiler = envList("CC", cfg.DefaultCC) - } - ldflags = setextld(ldflags, compiler) - for _, d := range toplevelactions { - if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries - continue - } - ldflags = append(ldflags, d.Package.ImportPath+"="+d.Target) - } - return b.run(".", out, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags) -} - -func (gcToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { - return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(a.Package.Dir, cfile)) -} - -// The Gccgo toolchain. -type gccgoToolchain struct{} - -var GccgoName, GccgoBin string -var gccgoErr error - -func init() { - GccgoName = os.Getenv("GCCGO") - if GccgoName == "" { - GccgoName = "gccgo" - } - GccgoBin, gccgoErr = exec.LookPath(GccgoName) -} - -func (gccgoToolchain) compiler() string { - checkGccgoBin() - return GccgoBin -} - -func (gccgoToolchain) linker() string { - checkGccgoBin() - return GccgoBin -} - -func checkGccgoBin() { - if gccgoErr == nil { - return - } - fmt.Fprintf(os.Stderr, "cmd/go: gccgo: %s\n", gccgoErr) - os.Exit(2) -} - -func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { - p := a.Package - objdir := a.Objdir - out := "_go_.o" - ofile = objdir + out - gcargs := []string{"-g"} - gcargs = append(gcargs, b.gccArchArgs()...) - if pkgpath := gccgoPkgpath(p); pkgpath != "" { - gcargs = append(gcargs, "-fgo-pkgpath="+pkgpath) - } - if p.Internal.LocalPrefix != "" { - gcargs = append(gcargs, "-fgo-relative-import-path="+p.Internal.LocalPrefix) - } - - args := str.StringList(tools.compiler(), "-c", gcargs, "-o", ofile) - if importcfg != nil { - if b.gccSupportsFlag(args[:1], "-fgo-importcfg=/dev/null") { - if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { - return "", nil, err - } - args = append(args, "-fgo-importcfg="+objdir+"importcfg") - } else { - root := objdir + "_importcfgroot_" - if err := buildImportcfgSymlinks(b, root, importcfg); err != nil { - return "", nil, err - } - args = append(args, "-I", root) - } - } - args = append(args, buildGccgoflags...) - for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) - } - - output, err = b.runOut(p.Dir, p.ImportPath, nil, args) - return ofile, output, err -} - -// buildImportcfgSymlinks builds in root a tree of symlinks -// implementing the directives from importcfg. -// This serves as a temporary transition mechanism until -// we can depend on gccgo reading an importcfg directly. -// (The Go 1.9 and later gc compilers already do.) -func buildImportcfgSymlinks(b *Builder, root string, importcfg []byte) error { - for lineNum, line := range strings.Split(string(importcfg), "\n") { - lineNum++ // 1-based - line = strings.TrimSpace(line) - if line == "" { - continue - } - if line == "" || strings.HasPrefix(line, "#") { - continue - } - var verb, args string - if i := strings.Index(line, " "); i < 0 { - verb = line - } else { - verb, args = line[:i], strings.TrimSpace(line[i+1:]) - } - var before, after string - if i := strings.Index(args, "="); i >= 0 { - before, after = args[:i], args[i+1:] - } - switch verb { - default: - base.Fatalf("importcfg:%d: unknown directive %q", lineNum, verb) - case "packagefile": - if before == "" || after == "" { - return fmt.Errorf(`importcfg:%d: invalid packagefile: syntax is "packagefile path=filename": %s`, lineNum, line) - } - archive := gccgoArchive(root, before) - if err := b.Mkdir(filepath.Dir(archive)); err != nil { - return err - } - if err := b.Symlink(after, archive); err != nil { - return err - } - case "importmap": - if before == "" || after == "" { - return fmt.Errorf(`importcfg:%d: invalid importmap: syntax is "importmap old=new": %s`, lineNum, line) - } - beforeA := gccgoArchive(root, before) - afterA := gccgoArchive(root, after) - if err := b.Mkdir(filepath.Dir(beforeA)); err != nil { - return err - } - if err := b.Mkdir(filepath.Dir(afterA)); err != nil { - return err - } - if err := b.Symlink(afterA, beforeA); err != nil { - return err - } - case "packageshlib": - return fmt.Errorf("gccgo -importcfg does not support shared libraries") - } - } - return nil -} - -func (tools gccgoToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { - p := a.Package - var ofiles []string - for _, sfile := range sfiles { - ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" - ofiles = append(ofiles, ofile) - sfile = mkAbs(p.Dir, sfile) - defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} - if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { - defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) - } - defs = tools.maybePIC(defs) - defs = append(defs, b.gccArchArgs()...) - err := b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-xassembler-with-cpp", "-I", a.Objdir, "-c", "-o", ofile, defs, sfile) - if err != nil { - return nil, err - } - } - return ofiles, nil -} - -func gccgoArchive(basedir, imp string) string { - end := filepath.FromSlash(imp + ".a") - afile := filepath.Join(basedir, end) - // add "lib" to the final element - return filepath.Join(filepath.Dir(afile), "lib"+filepath.Base(afile)) -} - -func (gccgoToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { - p := a.Package - objdir := a.Objdir - var absOfiles []string - for _, f := range ofiles { - absOfiles = append(absOfiles, mkAbs(objdir, f)) - } - return b.run(p.Dir, p.ImportPath, nil, "ar", "rc", mkAbs(objdir, afile), absOfiles) -} - -func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string, allactions []*Action, buildmode, desc string) error { - // gccgo needs explicit linking with all package dependencies, - // and all LDFLAGS from cgo dependencies. - apackagePathsSeen := make(map[string]bool) - afiles := []string{} - shlibs := []string{} - ldflags := b.gccArchArgs() - cgoldflags := []string{} - usesCgo := false - cxx := false - objc := false - fortran := false - if root.Package != nil { - cxx = len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 - objc = len(root.Package.MFiles) > 0 - fortran = len(root.Package.FFiles) > 0 - } - - readCgoFlags := func(flagsFile string) error { - flags, err := ioutil.ReadFile(flagsFile) - if err != nil { - return err - } - const ldflagsPrefix = "_CGO_LDFLAGS=" - for _, line := range strings.Split(string(flags), "\n") { - if strings.HasPrefix(line, ldflagsPrefix) { - newFlags := strings.Fields(line[len(ldflagsPrefix):]) - for _, flag := range newFlags { - // Every _cgo_flags file has -g and -O2 in _CGO_LDFLAGS - // but they don't mean anything to the linker so filter - // them out. - if flag != "-g" && !strings.HasPrefix(flag, "-O") { - cgoldflags = append(cgoldflags, flag) - } - } - } - } - return nil - } - - newID := 0 - readAndRemoveCgoFlags := func(archive string) (string, error) { - newID++ - newArchive := root.Objdir + fmt.Sprintf("_pkg%d_.a", newID) - if err := b.copyFile(root, newArchive, archive, 0666, false); err != nil { - return "", err - } - if cfg.BuildN || cfg.BuildX { - b.Showcmd("", "ar d %s _cgo_flags", newArchive) - if cfg.BuildN { - // TODO(rsc): We could do better about showing the right _cgo_flags even in -n mode. - // Either the archive is already built and we can read them out, - // or we're printing commands to build the archive and can - // forward the _cgo_flags directly to this step. - return "", nil - } - } - err := b.run(root.Objdir, desc, nil, "ar", "x", newArchive, "_cgo_flags") - if err != nil { - return "", err - } - err = b.run(".", desc, nil, "ar", "d", newArchive, "_cgo_flags") - if err != nil { - return "", err - } - err = readCgoFlags(filepath.Join(root.Objdir, "_cgo_flags")) - if err != nil { - return "", err - } - return newArchive, nil - } - - actionsSeen := make(map[*Action]bool) - // Make a pre-order depth-first traversal of the action graph, taking note of - // whether a shared library action has been seen on the way to an action (the - // construction of the graph means that if any path to a node passes through - // a shared library action, they all do). - var walk func(a *Action, seenShlib bool) - var err error - walk = func(a *Action, seenShlib bool) { - if actionsSeen[a] { - return - } - actionsSeen[a] = true - if a.Package != nil && !seenShlib { - if a.Package.Standard { - return - } - // We record the target of the first time we see a .a file - // for a package to make sure that we prefer the 'install' - // rather than the 'build' location (which may not exist any - // more). We still need to traverse the dependencies of the - // build action though so saying - // if apackagePathsSeen[a.Package.ImportPath] { return } - // doesn't work. - if !apackagePathsSeen[a.Package.ImportPath] { - apackagePathsSeen[a.Package.ImportPath] = true - target := a.Target - if len(a.Package.CgoFiles) > 0 || a.Package.UsesSwig() { - target, err = readAndRemoveCgoFlags(target) - if err != nil { - return - } - } - afiles = append(afiles, target) - } - } - if strings.HasSuffix(a.Target, ".so") { - shlibs = append(shlibs, a.Target) - seenShlib = true - } - for _, a1 := range a.Deps { - walk(a1, seenShlib) - if err != nil { - return - } - } - } - for _, a1 := range root.Deps { - walk(a1, false) - if err != nil { - return err - } - } - - for _, a := range allactions { - // Gather CgoLDFLAGS, but not from standard packages. - // The go tool can dig up runtime/cgo from GOROOT and - // think that it should use its CgoLDFLAGS, but gccgo - // doesn't use runtime/cgo. - if a.Package == nil { - continue - } - if !a.Package.Standard { - cgoldflags = append(cgoldflags, a.Package.CgoLDFLAGS...) - } - if len(a.Package.CgoFiles) > 0 { - usesCgo = true - } - if a.Package.UsesSwig() { - usesCgo = true - } - if len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0 { - cxx = true - } - if len(a.Package.MFiles) > 0 { - objc = true - } - if len(a.Package.FFiles) > 0 { - fortran = true - } - } - - ldflags = append(ldflags, "-Wl,--whole-archive") - ldflags = append(ldflags, afiles...) - ldflags = append(ldflags, "-Wl,--no-whole-archive") - - ldflags = append(ldflags, cgoldflags...) - ldflags = append(ldflags, envList("CGO_LDFLAGS", "")...) - if root.Package != nil { - ldflags = append(ldflags, root.Package.CgoLDFLAGS...) - } - - ldflags = str.StringList("-Wl,-(", ldflags, "-Wl,-)") - - for _, shlib := range shlibs { - ldflags = append( - ldflags, - "-L"+filepath.Dir(shlib), - "-Wl,-rpath="+filepath.Dir(shlib), - "-l"+strings.TrimSuffix( - strings.TrimPrefix(filepath.Base(shlib), "lib"), - ".so")) - } - - var realOut string - switch buildmode { - case "exe": - if usesCgo && cfg.Goos == "linux" { - ldflags = append(ldflags, "-Wl,-E") - } - - case "c-archive": - // Link the Go files into a single .o, and also link - // in -lgolibbegin. - // - // We need to use --whole-archive with -lgolibbegin - // because it doesn't define any symbols that will - // cause the contents to be pulled in; it's just - // initialization code. - // - // The user remains responsible for linking against - // -lgo -lpthread -lm in the final link. We can't use - // -r to pick them up because we can't combine - // split-stack and non-split-stack code in a single -r - // link, and libgo picks up non-split-stack code from - // libffi. - ldflags = append(ldflags, "-Wl,-r", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive") - - if nopie := b.gccNoPie([]string{tools.linker()}); nopie != "" { - ldflags = append(ldflags, nopie) - } - - // We are creating an object file, so we don't want a build ID. - ldflags = b.disableBuildID(ldflags) - - realOut = out - out = out + ".o" - - case "c-shared": - ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") - case "shared": - ldflags = append(ldflags, "-zdefs", "-shared", "-nostdlib", "-lgo", "-lgcc_s", "-lgcc", "-lc") - - default: - base.Fatalf("-buildmode=%s not supported for gccgo", buildmode) - } - - switch buildmode { - case "exe", "c-shared": - if cxx { - ldflags = append(ldflags, "-lstdc++") - } - if objc { - ldflags = append(ldflags, "-lobjc") - } - if fortran { - fc := os.Getenv("FC") - if fc == "" { - fc = "gfortran" - } - // support gfortran out of the box and let others pass the correct link options - // via CGO_LDFLAGS - if strings.Contains(fc, "gfortran") { - ldflags = append(ldflags, "-lgfortran") - } - } - } - - if err := b.run(".", desc, nil, tools.linker(), "-o", out, ldflags, buildGccgoflags); err != nil { - return err - } - - switch buildmode { - case "c-archive": - if err := b.run(".", desc, nil, "ar", "rc", realOut, out); err != nil { - return err - } - } - return nil -} - -func (tools gccgoToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { - return tools.link(b, root, out, importcfg, root.Deps, ldBuildmode, root.Package.ImportPath) -} - -func (tools gccgoToolchain) ldShared(b *Builder, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { - fakeRoot := &Action{Mode: "gccgo ldshared"} - fakeRoot.Deps = toplevelactions - return tools.link(b, fakeRoot, out, importcfg, allactions, "shared", out) -} - -func (tools gccgoToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { - p := a.Package - inc := filepath.Join(cfg.GOROOT, "pkg", "include") - cfile = mkAbs(p.Dir, cfile) - defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} - defs = append(defs, b.gccArchArgs()...) - if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { - defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) - } - switch cfg.Goarch { - case "386", "amd64": - defs = append(defs, "-fsplit-stack") - } - defs = tools.maybePIC(defs) - return b.run(p.Dir, p.ImportPath, nil, envList("CC", cfg.DefaultCC), "-Wall", "-g", - "-I", a.Objdir, "-I", inc, "-o", ofile, defs, "-c", cfile) -} - -// maybePIC adds -fPIC to the list of arguments if needed. -func (tools gccgoToolchain) maybePIC(args []string) []string { - switch cfg.BuildBuildmode { - case "c-shared", "shared", "plugin": - args = append(args, "-fPIC") - } - return args -} - -func gccgoPkgpath(p *load.Package) string { - if p.Internal.Build.IsCommand() && !p.Internal.ForceLibrary { - return "" - } - return p.ImportPath -} - -func gccgoCleanPkgpath(p *load.Package) string { - clean := func(r rune) rune { - switch { - case 'A' <= r && r <= 'Z', 'a' <= r && r <= 'z', - '0' <= r && r <= '9': - return r - } - return '_' - } - return strings.Map(clean, gccgoPkgpath(p)) -} - -// gcc runs the gcc C compiler to create an object from a single C file. -func (b *Builder) gcc(p *load.Package, workdir, out string, flags []string, cfile string) error { - return b.ccompile(p, out, flags, cfile, b.GccCmd(p.Dir, workdir)) -} - -// gxx runs the g++ C++ compiler to create an object from a single C++ file. -func (b *Builder) gxx(p *load.Package, workdir, out string, flags []string, cxxfile string) error { - return b.ccompile(p, out, flags, cxxfile, b.GxxCmd(p.Dir, workdir)) -} - -// gfortran runs the gfortran Fortran compiler to create an object from a single Fortran file. -func (b *Builder) gfortran(p *load.Package, workdir, out string, flags []string, ffile string) error { - return b.ccompile(p, out, flags, ffile, b.gfortranCmd(p.Dir, workdir)) -} - -// ccompile runs the given C or C++ compiler and creates an object from a single source file. -func (b *Builder) ccompile(p *load.Package, outfile string, flags []string, file string, compiler []string) error { - file = mkAbs(p.Dir, file) - desc := p.ImportPath - if !filepath.IsAbs(outfile) { - outfile = filepath.Join(p.Dir, outfile) - } - output, err := b.runOut(filepath.Dir(file), desc, nil, compiler, flags, "-o", outfile, "-c", filepath.Base(file)) - if len(output) > 0 { - // On FreeBSD 11, when we pass -g to clang 3.8 it - // invokes its internal assembler with -dwarf-version=2. - // When it sees .section .note.GNU-stack, it warns - // "DWARF2 only supports one section per compilation unit". - // This warning makes no sense, since the section is empty, - // but it confuses people. - // We work around the problem by detecting the warning - // and dropping -g and trying again. - if bytes.Contains(output, []byte("DWARF2 only supports one section per compilation unit")) { - newFlags := make([]string, 0, len(flags)) - for _, f := range flags { - if !strings.HasPrefix(f, "-g") { - newFlags = append(newFlags, f) - } - } - if len(newFlags) < len(flags) { - return b.ccompile(p, outfile, newFlags, file, compiler) - } - } - - b.showOutput(p.Dir, desc, b.processOutput(output)) - if err != nil { - err = errPrintedOutput - } else if os.Getenv("GO_BUILDER_NAME") != "" { - return errors.New("C compiler warning promoted to error on Go builders") - } - } - return err -} - -// gccld runs the gcc linker to create an executable from a set of object files. -func (b *Builder) gccld(p *load.Package, objdir, out string, flags []string, objs []string) error { - var cmd []string - if len(p.CXXFiles) > 0 || len(p.SwigCXXFiles) > 0 { - cmd = b.GxxCmd(p.Dir, objdir) - } else { - cmd = b.GccCmd(p.Dir, objdir) - } - return b.run(p.Dir, p.ImportPath, nil, cmd, "-o", out, objs, flags) -} - -// gccCmd returns a gcc command line prefix -// defaultCC is defined in zdefaultcc.go, written by cmd/dist. -func (b *Builder) GccCmd(incdir, workdir string) []string { - return b.compilerCmd("CC", cfg.DefaultCC, incdir, workdir) -} - -// gxxCmd returns a g++ command line prefix -// defaultCXX is defined in zdefaultcc.go, written by cmd/dist. -func (b *Builder) GxxCmd(incdir, workdir string) []string { - return b.compilerCmd("CXX", cfg.DefaultCXX, incdir, workdir) -} - -// gfortranCmd returns a gfortran command line prefix. -func (b *Builder) gfortranCmd(incdir, workdir string) []string { - return b.compilerCmd("FC", "gfortran", incdir, workdir) -} - -// compilerCmd returns a command line prefix for the given environment -// variable and using the default command when the variable is empty. -func (b *Builder) compilerCmd(envvar, defcmd, incdir, workdir string) []string { - // NOTE: env.go's mkEnv knows that the first three - // strings returned are "gcc", "-I", incdir (and cuts them off). - - compiler := envList(envvar, defcmd) - a := []string{compiler[0], "-I", incdir} - a = append(a, compiler[1:]...) - - // Definitely want -fPIC but on Windows gcc complains - // "-fPIC ignored for target (all code is position independent)" - if cfg.Goos != "windows" { - a = append(a, "-fPIC") - } - a = append(a, b.gccArchArgs()...) - // gcc-4.5 and beyond require explicit "-pthread" flag - // for multithreading with pthread library. - if cfg.BuildContext.CgoEnabled { - switch cfg.Goos { - case "windows": - a = append(a, "-mthreads") - default: - a = append(a, "-pthread") - } - } - - // disable ASCII art in clang errors, if possible - if b.gccSupportsFlag(compiler, "-fno-caret-diagnostics") { - a = append(a, "-fno-caret-diagnostics") - } - // clang is too smart about command-line arguments - if b.gccSupportsFlag(compiler, "-Qunused-arguments") { - a = append(a, "-Qunused-arguments") - } - - // disable word wrapping in error messages - a = append(a, "-fmessage-length=0") - - // Tell gcc not to include the work directory in object files. - if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { - if workdir == "" { - workdir = b.WorkDir - } - workdir = strings.TrimSuffix(workdir, string(filepath.Separator)) - a = append(a, "-fdebug-prefix-map="+workdir+"=/tmp/go-build") - } - - // Tell gcc not to include flags in object files, which defeats the - // point of -fdebug-prefix-map above. - if b.gccSupportsFlag(compiler, "-gno-record-gcc-switches") { - a = append(a, "-gno-record-gcc-switches") - } - - // On OS X, some of the compilers behave as if -fno-common - // is always set, and the Mach-O linker in 6l/8l assumes this. - // See https://golang.org/issue/3253. - if cfg.Goos == "darwin" { - a = append(a, "-fno-common") - } - - return a -} - -// gccNoPie returns the flag to use to request non-PIE. On systems -// with PIE (position independent executables) enabled by default, -// -no-pie must be passed when doing a partial link with -Wl,-r. -// But -no-pie is not supported by all compilers, and clang spells it -nopie. -func (b *Builder) gccNoPie(linker []string) string { - if b.gccSupportsFlag(linker, "-no-pie") { - return "-no-pie" - } - if b.gccSupportsFlag(linker, "-nopie") { - return "-nopie" - } - return "" -} - -// gccSupportsFlag checks to see if the compiler supports a flag. -func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { - key := [2]string{compiler[0], flag} - - b.exec.Lock() - defer b.exec.Unlock() - if b, ok := b.flagCache[key]; ok { - return b - } - if b.flagCache == nil { - if cfg.BuildN || cfg.BuildX { - b.Showcmd(b.WorkDir, "touch trivial.c") - } - if !cfg.BuildN { - src := filepath.Join(b.WorkDir, "trivial.c") - if err := ioutil.WriteFile(src, []byte{}, 0666); err != nil { - return false - } - } - b.flagCache = make(map[[2]string]bool) - } - cmdArgs := append([]string(nil), compiler...) - cmdArgs = append(cmdArgs, flag, "-c", "trivial.c") - if cfg.BuildN || cfg.BuildX { - b.Showcmd(b.WorkDir, "%s", joinUnambiguously(cmdArgs)) - if cfg.BuildN { - return false - } - } - cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) - cmd.Dir = b.WorkDir - cmd.Env = base.MergeEnvLists([]string{"LC_ALL=C"}, base.EnvForDir(cmd.Dir, os.Environ())) - out, err := cmd.CombinedOutput() - // GCC says "unrecognized command line option". - // clang says "unknown argument". - supported := err == nil && !bytes.Contains(out, []byte("unrecognized")) && !bytes.Contains(out, []byte("unknown")) - b.flagCache[key] = supported - return supported -} - -// gccArchArgs returns arguments to pass to gcc based on the architecture. -func (b *Builder) gccArchArgs() []string { - switch cfg.Goarch { - case "386": - return []string{"-m32"} - case "amd64", "amd64p32": - return []string{"-m64"} - case "arm": - return []string{"-marm"} // not thumb - case "s390x": - return []string{"-m64", "-march=z196"} - case "mips64", "mips64le": - return []string{"-mabi=64"} - case "mips", "mipsle": - return []string{"-mabi=32", "-march=mips32"} - } - return nil -} - -// envList returns the value of the given environment variable broken -// into fields, using the default value when the variable is empty. -func envList(key, def string) []string { - v := os.Getenv(key) - if v == "" { - v = def - } - return strings.Fields(v) -} - -// CFlags returns the flags to use when invoking the C, C++ or Fortran compilers, or cgo. -func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, ldflags []string) { - defaults := "-g -O2" - - cppflags = str.StringList(envList("CGO_CPPFLAGS", ""), p.CgoCPPFLAGS) - cflags = str.StringList(envList("CGO_CFLAGS", defaults), p.CgoCFLAGS) - cxxflags = str.StringList(envList("CGO_CXXFLAGS", defaults), p.CgoCXXFLAGS) - fflags = str.StringList(envList("CGO_FFLAGS", defaults), p.CgoFFLAGS) - ldflags = str.StringList(envList("CGO_LDFLAGS", defaults), p.CgoLDFLAGS) - return -} - -var cgoRe = regexp.MustCompile(`[/\\:]`) - -func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgofiles, objdirCgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { - p := a.Package - cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoFFLAGS, cgoLDFLAGS := b.CFlags(p) - cgoCPPFLAGS = append(cgoCPPFLAGS, pcCFLAGS...) - cgoLDFLAGS = append(cgoLDFLAGS, pcLDFLAGS...) - // If we are compiling Objective-C code, then we need to link against libobjc - if len(mfiles) > 0 { - cgoLDFLAGS = append(cgoLDFLAGS, "-lobjc") - } - - // Likewise for Fortran, except there are many Fortran compilers. - // Support gfortran out of the box and let others pass the correct link options - // via CGO_LDFLAGS - if len(ffiles) > 0 { - fc := os.Getenv("FC") - if fc == "" { - fc = "gfortran" - } - if strings.Contains(fc, "gfortran") { - cgoLDFLAGS = append(cgoLDFLAGS, "-lgfortran") - } - } - - if cfg.BuildMSan { - cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...) - cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) - } - - // Allows including _cgo_export.h from .[ch] files in the package. - cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", objdir) - - // If we have cgo files in the object directory, then copy any - // other cgo files into the object directory, and pass a - // -srcdir option to cgo. - var srcdirarg []string - if len(objdirCgofiles) > 0 { - for _, fn := range cgofiles { - if err := b.copyFile(a, objdir+filepath.Base(fn), filepath.Join(p.Dir, fn), 0666, false); err != nil { - return nil, nil, err - } - } - cgofiles = append(cgofiles, objdirCgofiles...) - srcdirarg = []string{"-srcdir", objdir} - } - - // cgo - // TODO: CGO_FLAGS? - gofiles := []string{objdir + "_cgo_gotypes.go"} - cfiles := []string{"_cgo_export.c"} - for _, fn := range cgofiles { - f := cgoRe.ReplaceAllString(fn[:len(fn)-2], "_") - gofiles = append(gofiles, objdir+f+"cgo1.go") - cfiles = append(cfiles, f+"cgo2.c") - } - - // TODO: make cgo not depend on $GOARCH? - - cgoflags := []string{} - if p.Standard && p.ImportPath == "runtime/cgo" { - cgoflags = append(cgoflags, "-import_runtime_cgo=false") - } - if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo") { - cgoflags = append(cgoflags, "-import_syscall=false") - } - - // Update $CGO_LDFLAGS with p.CgoLDFLAGS. - var cgoenv []string - if len(cgoLDFLAGS) > 0 { - flags := make([]string, len(cgoLDFLAGS)) - for i, f := range cgoLDFLAGS { - flags[i] = strconv.Quote(f) - } - cgoenv = []string{"CGO_LDFLAGS=" + strings.Join(flags, " ")} - } - - if cfg.BuildToolchainName == "gccgo" { - switch cfg.Goarch { - case "386", "amd64": - cgoCFLAGS = append(cgoCFLAGS, "-fsplit-stack") - } - cgoflags = append(cgoflags, "-gccgo") - if pkgpath := gccgoPkgpath(p); pkgpath != "" { - cgoflags = append(cgoflags, "-gccgopkgpath="+pkgpath) - } - } - - switch cfg.BuildBuildmode { - case "c-archive", "c-shared": - // Tell cgo that if there are any exported functions - // it should generate a header file that C code can - // #include. - cgoflags = append(cgoflags, "-exportheader="+objdir+"_cgo_install.h") - } - - if err := b.run(p.Dir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, srcdirarg, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { - return nil, nil, err - } - outGo = append(outGo, gofiles...) - - // Use sequential object file names to keep them distinct - // and short enough to fit in the .a header file name slots. - // We no longer collect them all into _all.o, and we'd like - // tools to see both the .o suffix and unique names, so - // we need to make them short enough not to be truncated - // in the final archive. - oseq := 0 - nextOfile := func() string { - oseq++ - return objdir + fmt.Sprintf("_x%03d.o", oseq) - } - - // gcc - cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS) - for _, cfile := range cfiles { - ofile := nextOfile() - if err := b.gcc(p, a.Objdir, ofile, cflags, objdir+cfile); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - for _, file := range gccfiles { - ofile := nextOfile() - if err := b.gcc(p, a.Objdir, ofile, cflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS) - for _, file := range gxxfiles { - ofile := nextOfile() - if err := b.gxx(p, a.Objdir, ofile, cxxflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - for _, file := range mfiles { - ofile := nextOfile() - if err := b.gcc(p, a.Objdir, ofile, cflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS) - for _, file := range ffiles { - ofile := nextOfile() - if err := b.gfortran(p, a.Objdir, ofile, fflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - switch cfg.BuildToolchainName { - case "gc": - importGo := objdir + "_cgo_import.go" - if err := b.dynimport(p, objdir, importGo, cgoExe, cflags, cgoLDFLAGS, outObj); err != nil { - return nil, nil, err - } - outGo = append(outGo, importGo) - - case "gccgo": - defunC := objdir + "_cgo_defun.c" - defunObj := objdir + "_cgo_defun.o" - if err := BuildToolchain.cc(b, a, defunObj, defunC); err != nil { - return nil, nil, err - } - outObj = append(outObj, defunObj) - - default: - noCompiler() - } - - return outGo, outObj, nil -} - -// dynimport creates a Go source file named importGo containing -// //go:cgo_import_dynamic directives for each symbol or library -// dynamically imported by the object files outObj. -func (b *Builder) dynimport(p *load.Package, objdir, importGo, cgoExe string, cflags, cgoLDFLAGS, outObj []string) error { - cfile := objdir + "_cgo_main.c" - ofile := objdir + "_cgo_main.o" - if err := b.gcc(p, objdir, ofile, cflags, cfile); err != nil { - return err - } - - linkobj := str.StringList(ofile, outObj, p.SysoFiles) - dynobj := objdir + "_cgo_.o" - - // we need to use -pie for Linux/ARM to get accurate imported sym - ldflags := cgoLDFLAGS - if (cfg.Goarch == "arm" && cfg.Goos == "linux") || cfg.Goos == "android" { - ldflags = append(ldflags, "-pie") - } - if err := b.gccld(p, objdir, dynobj, ldflags, linkobj); err != nil { - return err - } - - // cgo -dynimport - var cgoflags []string - if p.Standard && p.ImportPath == "runtime/cgo" { - cgoflags = []string{"-dynlinker"} // record path to dynamic linker - } - return b.run(p.Dir, p.ImportPath, nil, cfg.BuildToolexec, cgoExe, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags) -} - -// Run SWIG on all SWIG input files. -// TODO: Don't build a shared library, once SWIG emits the necessary -// pragmas for external linking. -func (b *Builder) swig(p *load.Package, objdir string, pcCFLAGS []string) (outGo, outC, outCXX []string, err error) { - if err := b.swigVersionCheck(); err != nil { - return nil, nil, nil, err - } - - intgosize, err := b.swigIntSize(objdir) - if err != nil { - return nil, nil, nil, err - } - - for _, f := range p.SwigFiles { - goFile, cFile, err := b.swigOne(p, f, objdir, pcCFLAGS, false, intgosize) - if err != nil { - return nil, nil, nil, err - } - if goFile != "" { - outGo = append(outGo, goFile) - } - if cFile != "" { - outC = append(outC, cFile) - } - } - for _, f := range p.SwigCXXFiles { - goFile, cxxFile, err := b.swigOne(p, f, objdir, pcCFLAGS, true, intgosize) - if err != nil { - return nil, nil, nil, err - } - if goFile != "" { - outGo = append(outGo, goFile) - } - if cxxFile != "" { - outCXX = append(outCXX, cxxFile) - } - } - return outGo, outC, outCXX, nil -} - -// Make sure SWIG is new enough. -var ( - swigCheckOnce sync.Once - swigCheck error -) - -func (b *Builder) swigDoVersionCheck() error { - out, err := b.runOut("", "", nil, "swig", "-version") - if err != nil { - return err - } - re := regexp.MustCompile(`[vV]ersion +([\d]+)([.][\d]+)?([.][\d]+)?`) - matches := re.FindSubmatch(out) - if matches == nil { - // Can't find version number; hope for the best. - return nil - } - - major, err := strconv.Atoi(string(matches[1])) - if err != nil { - // Can't find version number; hope for the best. - return nil - } - const errmsg = "must have SWIG version >= 3.0.6" - if major < 3 { - return errors.New(errmsg) - } - if major > 3 { - // 4.0 or later - return nil - } - - // We have SWIG version 3.x. - if len(matches[2]) > 0 { - minor, err := strconv.Atoi(string(matches[2][1:])) - if err != nil { - return nil - } - if minor > 0 { - // 3.1 or later - return nil - } - } - - // We have SWIG version 3.0.x. - if len(matches[3]) > 0 { - patch, err := strconv.Atoi(string(matches[3][1:])) - if err != nil { - return nil - } - if patch < 6 { - // Before 3.0.6. - return errors.New(errmsg) - } - } - - return nil -} - -func (b *Builder) swigVersionCheck() error { - swigCheckOnce.Do(func() { - swigCheck = b.swigDoVersionCheck() - }) - return swigCheck -} - -// Find the value to pass for the -intgosize option to swig. -var ( - swigIntSizeOnce sync.Once - swigIntSize string - swigIntSizeError error -) - -// This code fails to build if sizeof(int) <= 32 -const swigIntSizeCode = ` -package main -const i int = 1 << 32 -` - -// Determine the size of int on the target system for the -intgosize option -// of swig >= 2.0.9. Run only once. -func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { - if cfg.BuildN { - return "$INTBITS", nil - } - src := filepath.Join(b.WorkDir, "swig_intsize.go") - if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { - return - } - srcs := []string{src} - - p := load.GoFilesPackage(srcs) - - if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, false, srcs); e != nil { - return "32", nil - } - return "64", nil -} - -// Determine the size of int on the target system for the -intgosize option -// of swig >= 2.0.9. -func (b *Builder) swigIntSize(objdir string) (intsize string, err error) { - swigIntSizeOnce.Do(func() { - swigIntSize, swigIntSizeError = b.swigDoIntSize(objdir) - }) - return swigIntSize, swigIntSizeError -} - -// Run SWIG on one SWIG input file. -func (b *Builder) swigOne(p *load.Package, file, objdir string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) { - cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, _, _ := b.CFlags(p) - var cflags []string - if cxx { - cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCXXFLAGS) - } else { - cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCFLAGS) - } - - n := 5 // length of ".swig" - if cxx { - n = 8 // length of ".swigcxx" - } - base := file[:len(file)-n] - goFile := base + ".go" - gccBase := base + "_wrap." - gccExt := "c" - if cxx { - gccExt = "cxx" - } - - gccgo := cfg.BuildToolchainName == "gccgo" - - // swig - args := []string{ - "-go", - "-cgo", - "-intgosize", intgosize, - "-module", base, - "-o", objdir + gccBase + gccExt, - "-outdir", objdir, - } - - for _, f := range cflags { - if len(f) > 3 && f[:2] == "-I" { - args = append(args, f) - } - } - - if gccgo { - args = append(args, "-gccgo") - if pkgpath := gccgoPkgpath(p); pkgpath != "" { - args = append(args, "-go-pkgpath", pkgpath) - } - } - if cxx { - args = append(args, "-c++") - } - - out, err := b.runOut(p.Dir, p.ImportPath, nil, "swig", args, file) - if err != nil { - if len(out) > 0 { - if bytes.Contains(out, []byte("-intgosize")) || bytes.Contains(out, []byte("-cgo")) { - return "", "", errors.New("must have SWIG version >= 3.0.6") - } - b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig error - return "", "", errPrintedOutput - } - return "", "", err - } - if len(out) > 0 { - b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig warning - } - - return goFile, objdir + gccBase + gccExt, nil -} - -// disableBuildID adjusts a linker command line to avoid creating a -// build ID when creating an object file rather than an executable or -// shared library. Some systems, such as Ubuntu, always add -// --build-id to every link, but we don't want a build ID when we are -// producing an object file. On some of those system a plain -r (not -// -Wl,-r) will turn off --build-id, but clang 3.0 doesn't support a -// plain -r. I don't know how to turn off --build-id when using clang -// other than passing a trailing --build-id=none. So that is what we -// do, but only on systems likely to support it, which is to say, -// systems that normally use gold or the GNU linker. -func (b *Builder) disableBuildID(ldflags []string) []string { - switch cfg.Goos { - case "android", "dragonfly", "linux", "netbsd": - ldflags = append(ldflags, "-Wl,--build-id=none") - } - return ldflags -} - -// An actionQueue is a priority queue of actions. -type actionQueue []*Action - -// Implement heap.Interface -func (q *actionQueue) Len() int { return len(*q) } -func (q *actionQueue) Swap(i, j int) { (*q)[i], (*q)[j] = (*q)[j], (*q)[i] } -func (q *actionQueue) Less(i, j int) bool { return (*q)[i].priority < (*q)[j].priority } -func (q *actionQueue) Push(x interface{}) { *q = append(*q, x.(*Action)) } -func (q *actionQueue) Pop() interface{} { - n := len(*q) - 1 - x := (*q)[n] - *q = (*q)[:n] - return x -} - -func (q *actionQueue) push(a *Action) { - heap.Push(q, a) -} - -func (q *actionQueue) pop() *Action { - return heap.Pop(q).(*Action) -} - func InstrumentInit() { if !cfg.BuildRace && !cfg.BuildMSan { return diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go new file mode 100644 index 0000000000..360afbdaf1 --- /dev/null +++ b/src/cmd/go/internal/work/exec.go @@ -0,0 +1,1904 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Action graph execution. + +package work + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" + "cmd/internal/buildid" +) + +// actionList returns the list of actions in the dag rooted at root +// as visited in a depth-first post-order traversal. +func actionList(root *Action) []*Action { + seen := map[*Action]bool{} + all := []*Action{} + var walk func(*Action) + walk = func(a *Action) { + if seen[a] { + return + } + seen[a] = true + for _, a1 := range a.Deps { + walk(a1) + } + all = append(all, a) + } + walk(root) + return all +} + +// do runs the action graph rooted at root. +func (b *Builder) Do(root *Action) { + // Build list of all actions, assigning depth-first post-order priority. + // The original implementation here was a true queue + // (using a channel) but it had the effect of getting + // distracted by low-level leaf actions to the detriment + // of completing higher-level actions. The order of + // work does not matter much to overall execution time, + // but when running "go test std" it is nice to see each test + // results as soon as possible. The priorities assigned + // ensure that, all else being equal, the execution prefers + // to do what it would have done first in a simple depth-first + // dependency order traversal. + all := actionList(root) + for i, a := range all { + a.priority = i + } + + if cfg.DebugActiongraph != "" { + js := actionGraphJSON(root) + if err := ioutil.WriteFile(cfg.DebugActiongraph, []byte(js), 0666); err != nil { + fmt.Fprintf(os.Stderr, "go: writing action graph: %v\n", err) + base.SetExitStatus(1) + } + } + + b.readySema = make(chan bool, len(all)) + + // Initialize per-action execution state. + for _, a := range all { + for _, a1 := range a.Deps { + a1.triggers = append(a1.triggers, a) + } + a.pending = len(a.Deps) + if a.pending == 0 { + b.ready.push(a) + b.readySema <- true + } + } + + // Handle runs a single action and takes care of triggering + // any actions that are runnable as a result. + handle := func(a *Action) { + var err error + + if a.Func != nil && (!a.Failed || a.IgnoreFail) { + if a.Objdir != "" { + err = b.Mkdir(a.Objdir) + } + if err == nil { + err = a.Func(b, a) + } + } + + // The actions run in parallel but all the updates to the + // shared work state are serialized through b.exec. + b.exec.Lock() + defer b.exec.Unlock() + + if err != nil { + if err == errPrintedOutput { + base.SetExitStatus(2) + } else { + base.Errorf("%s", err) + } + a.Failed = true + } + + for _, a0 := range a.triggers { + if a.Failed { + a0.Failed = true + } + if a0.pending--; a0.pending == 0 { + b.ready.push(a0) + b.readySema <- true + } + } + + if a == root { + close(b.readySema) + } + } + + var wg sync.WaitGroup + + // Kick off goroutines according to parallelism. + // If we are using the -n flag (just printing commands) + // drop the parallelism to 1, both to make the output + // deterministic and because there is no real work anyway. + par := cfg.BuildP + if cfg.BuildN { + par = 1 + } + for i := 0; i < par; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case _, ok := <-b.readySema: + if !ok { + return + } + // Receiving a value from b.readySema entitles + // us to take from the ready queue. + b.exec.Lock() + a := b.ready.pop() + b.exec.Unlock() + handle(a) + case <-base.Interrupted: + base.SetExitStatus(1) + return + } + } + }() + } + + wg.Wait() +} + +// build is the action for building a single package or command. +func (b *Builder) build(a *Action) (err error) { + // Return an error for binary-only package. + // We only reach this if isStale believes the binary form is + // either not present or not usable. + if a.Package.BinaryOnly { + return fmt.Errorf("missing or invalid package binary for binary-only package %s", a.Package.ImportPath) + } + + defer func() { + if err != nil && err != errPrintedOutput { + err = fmt.Errorf("go build %s: %v", a.Package.ImportPath, err) + } + }() + if cfg.BuildN { + // In -n mode, print a banner between packages. + // The banner is five lines so that when changes to + // different sections of the bootstrap script have to + // be merged, the banners give patch something + // to use to find its context. + b.Print("\n#\n# " + a.Package.ImportPath + "\n#\n\n") + } + + if cfg.BuildV { + b.Print(a.Package.ImportPath + "\n") + } + + objdir := a.Objdir + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + // We want to keep the action ID available for consultation later, + // but we'll append to it the SHA256 of the file (without this ID included). + // We don't know the SHA256 yet, so make one up to find and replace + // later. Becuase the action ID is a hash of the inputs to this built, + // the chance of SHA256(actionID) occurring elsewhere in the result + // of the build is essentially zero, at least in 2017. + actionID := a.Package.Internal.BuildID + if actionID == "" { + return fmt.Errorf("missing action ID") + } + a.buildID = actionID + "." + fmt.Sprintf("%x", sha256.Sum256([]byte(actionID))) + + var gofiles, cgofiles, objdirCgofiles, cfiles, sfiles, cxxfiles, objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string + + gofiles = append(gofiles, a.Package.GoFiles...) + cgofiles = append(cgofiles, a.Package.CgoFiles...) + cfiles = append(cfiles, a.Package.CFiles...) + sfiles = append(sfiles, a.Package.SFiles...) + cxxfiles = append(cxxfiles, a.Package.CXXFiles...) + + if a.Package.UsesCgo() || a.Package.UsesSwig() { + if pcCFLAGS, pcLDFLAGS, err = b.getPkgConfigFlags(a.Package); err != nil { + return + } + } + + // Run SWIG on each .swig and .swigcxx file. + // Each run will generate two files, a .go file and a .c or .cxx file. + // The .go file will use import "C" and is to be processed by cgo. + if a.Package.UsesSwig() { + outGo, outC, outCXX, err := b.swig(a.Package, objdir, pcCFLAGS) + if err != nil { + return err + } + objdirCgofiles = append(objdirCgofiles, outGo...) + cfiles = append(cfiles, outC...) + cxxfiles = append(cxxfiles, outCXX...) + } + + // Run cgo. + if a.Package.UsesCgo() || a.Package.UsesSwig() { + // In a package using cgo, cgo compiles the C, C++ and assembly files with gcc. + // There is one exception: runtime/cgo's job is to bridge the + // cgo and non-cgo worlds, so it necessarily has files in both. + // In that case gcc only gets the gcc_* files. + var gccfiles []string + gccfiles = append(gccfiles, cfiles...) + cfiles = nil + if a.Package.Standard && a.Package.ImportPath == "runtime/cgo" { + filter := func(files, nongcc, gcc []string) ([]string, []string) { + for _, f := range files { + if strings.HasPrefix(f, "gcc_") { + gcc = append(gcc, f) + } else { + nongcc = append(nongcc, f) + } + } + return nongcc, gcc + } + sfiles, gccfiles = filter(sfiles, sfiles[:0], gccfiles) + } else { + for _, sfile := range sfiles { + data, err := ioutil.ReadFile(filepath.Join(a.Package.Dir, sfile)) + if err == nil { + if bytes.HasPrefix(data, []byte("TEXT")) || bytes.Contains(data, []byte("\nTEXT")) || + bytes.HasPrefix(data, []byte("DATA")) || bytes.Contains(data, []byte("\nDATA")) || + bytes.HasPrefix(data, []byte("GLOBL")) || bytes.Contains(data, []byte("\nGLOBL")) { + return fmt.Errorf("package using cgo has Go assembly file %s", sfile) + } + } + } + gccfiles = append(gccfiles, sfiles...) + sfiles = nil + } + + outGo, outObj, err := b.cgo(a, base.Tool("cgo"), objdir, pcCFLAGS, pcLDFLAGS, cgofiles, objdirCgofiles, gccfiles, cxxfiles, a.Package.MFiles, a.Package.FFiles) + if err != nil { + return err + } + if cfg.BuildToolchainName == "gccgo" { + cgoObjects = append(cgoObjects, a.Objdir+"_cgo_flags") + } + cgoObjects = append(cgoObjects, outObj...) + gofiles = append(gofiles, outGo...) + } + + // Sanity check only, since Package.load already checked as well. + if len(gofiles) == 0 { + return &load.NoGoError{Package: a.Package} + } + + // If we're doing coverage, preprocess the .go files and put them in the work directory + if a.Package.Internal.CoverMode != "" { + for i, file := range gofiles { + var sourceFile string + var coverFile string + var key string + if strings.HasSuffix(file, ".cgo1.go") { + // cgo files have absolute paths + base := filepath.Base(file) + sourceFile = file + coverFile = objdir + base + key = strings.TrimSuffix(base, ".cgo1.go") + ".go" + } else { + sourceFile = filepath.Join(a.Package.Dir, file) + coverFile = objdir + file + key = file + } + cover := a.Package.Internal.CoverVars[key] + if cover == nil || base.IsTestFile(file) { + // Not covering this file. + continue + } + if err := b.cover(a, coverFile, sourceFile, 0666, cover.Var); err != nil { + return err + } + gofiles[i] = coverFile + } + } + + // Prepare Go import config. + var icfg bytes.Buffer + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.ImportPath == "" { + continue + } + path := p1.ImportPath + i := strings.LastIndex(path, "/vendor/") + if i >= 0 { + i += len("/vendor/") + } else if strings.HasPrefix(path, "vendor/") { + i = len("vendor/") + } else { + continue + } + fmt.Fprintf(&icfg, "importmap %s=%s\n", path[i:], path) + } + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil || p1.ImportPath == "" || a1.built == "" { + continue + } + fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built) + } + + // Compile Go. + objpkg := objdir + "_pkg_.a" + ofile, out, err := BuildToolchain.gc(b, a, objpkg, icfg.Bytes(), len(sfiles) > 0, gofiles) + if len(out) > 0 { + b.showOutput(a.Package.Dir, a.Package.ImportPath, b.processOutput(out)) + if err != nil { + return errPrintedOutput + } + } + if err != nil { + return err + } + if ofile != objpkg { + objects = append(objects, ofile) + } + + // Copy .h files named for goos or goarch or goos_goarch + // to names using GOOS and GOARCH. + // For example, defs_linux_amd64.h becomes defs_GOOS_GOARCH.h. + _goos_goarch := "_" + cfg.Goos + "_" + cfg.Goarch + _goos := "_" + cfg.Goos + _goarch := "_" + cfg.Goarch + for _, file := range a.Package.HFiles { + name, ext := fileExtSplit(file) + switch { + case strings.HasSuffix(name, _goos_goarch): + targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext + if err := b.copyFile(a, objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + case strings.HasSuffix(name, _goarch): + targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext + if err := b.copyFile(a, objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + case strings.HasSuffix(name, _goos): + targ := file[:len(name)-len(_goos)] + "_GOOS." + ext + if err := b.copyFile(a, objdir+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + } + } + + for _, file := range cfiles { + out := file[:len(file)-len(".c")] + ".o" + if err := BuildToolchain.cc(b, a, objdir+out, file); err != nil { + return err + } + objects = append(objects, out) + } + + // Assemble .s files. + if len(sfiles) > 0 { + ofiles, err := BuildToolchain.asm(b, a, sfiles) + if err != nil { + return err + } + objects = append(objects, ofiles...) + } + + // NOTE(rsc): On Windows, it is critically important that the + // gcc-compiled objects (cgoObjects) be listed after the ordinary + // objects in the archive. I do not know why this is. + // https://golang.org/issue/2601 + objects = append(objects, cgoObjects...) + + // Add system object files. + for _, syso := range a.Package.SysoFiles { + objects = append(objects, filepath.Join(a.Package.Dir, syso)) + } + + // Pack into archive in objdir directory. + // If the Go compiler wrote an archive, we only need to add the + // object files for non-Go sources to the archive. + // If the Go compiler wrote an archive and the package is entirely + // Go sources, there is no pack to execute at all. + if len(objects) > 0 { + if err := BuildToolchain.pack(b, a, objpkg, objects); err != nil { + return err + } + } + + if err := b.updateBuildID(a, actionID, objpkg); err != nil { + return err + } + + return nil +} + +func (b *Builder) link(a *Action) (err error) { + importcfg := a.Objdir + "importcfg.link" + if err := b.writeLinkImportcfg(a, importcfg); err != nil { + return err + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + actionID := a.Package.Internal.BuildID + if actionID == "" { + return fmt.Errorf("missing action ID") + } + a.buildID = actionID + "." + fmt.Sprintf("%x", sha256.Sum256([]byte(actionID))) + + objpkg := a.Objdir + "_pkg_.a" + if err := BuildToolchain.ld(b, a, a.Target, importcfg, objpkg); err != nil { + return err + } + + // Update the binary with the final build ID. + // But if OmitDebug is set, don't, because we set OmitDebug + // on binaries that we are going to run and then delete. + // There's no point in doing work on such a binary. + // Worse, opening the binary for write here makes it + // essentially impossible to safely fork+exec due to a fundamental + // incompatibility between ETXTBSY and threads on modern Unix systems. + // See golang.org/issue/22220. + if !a.Package.Internal.OmitDebug { + if err := b.updateBuildID(a, actionID, a.Target); err != nil { + return err + } + } + + return nil +} + +func (b *Builder) updateBuildID(a *Action, actionID, target string) error { + if cfg.BuildX || cfg.BuildN { + b.Showcmd("", "%s # internal", joinUnambiguously(str.StringList(base.Tool("buildid"), "-w", target))) + if cfg.BuildN { + return nil + } + } + + // Find occurrences of old ID and compute new content-based ID. + r, err := os.Open(target) + if err != nil { + return err + } + matches, hash, err := buildid.FindAndHash(r, a.buildID, 0) + r.Close() + if err != nil { + return err + } + newID := fmt.Sprintf("%s.%x", actionID, hash) + if len(newID) != len(a.buildID) { + return fmt.Errorf("internal error: build ID length mismatch %d+1+%d != %d", len(actionID), len(hash)*2, len(a.buildID)) + } + + // Replace with new content-based ID. + a.buildID = newID + if len(matches) == 0 { + // Assume the user specified -buildid= to override what we were going to choose. + return nil + } + w, err := os.OpenFile(target, os.O_WRONLY, 0) + if err != nil { + return err + } + err = buildid.Rewrite(w, matches, newID) + if err != nil { + w.Close() + return err + } + if err := w.Close(); err != nil { + return err + } + return nil +} + +func (b *Builder) writeLinkImportcfg(a *Action, file string) error { + // Prepare Go import cfg. + var icfg bytes.Buffer + for _, a1 := range a.Deps { + p1 := a1.Package + if p1 == nil { + continue + } + fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built) + if p1.Shlib != "" { + fmt.Fprintf(&icfg, "packageshlib %s=%s\n", p1.ImportPath, p1.Shlib) + } + } + return b.writeFile(file, icfg.Bytes()) +} + +// PkgconfigCmd returns a pkg-config binary name +// defaultPkgConfig is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) PkgconfigCmd() string { + return envList("PKG_CONFIG", cfg.DefaultPkgConfig)[0] +} + +// splitPkgConfigOutput parses the pkg-config output into a slice of +// flags. pkg-config always uses \ to escape special characters. +func splitPkgConfigOutput(out []byte) []string { + if len(out) == 0 { + return nil + } + var flags []string + flag := make([]byte, len(out)) + r, w := 0, 0 + for r < len(out) { + switch out[r] { + case ' ', '\t', '\r', '\n': + if w > 0 { + flags = append(flags, string(flag[:w])) + } + w = 0 + case '\\': + r++ + fallthrough + default: + if r < len(out) { + flag[w] = out[r] + w++ + } + } + r++ + } + if w > 0 { + flags = append(flags, string(flag[:w])) + } + return flags +} + +// Calls pkg-config if needed and returns the cflags/ldflags needed to build the package. +func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, err error) { + if pkgs := p.CgoPkgConfig; len(pkgs) > 0 { + var out []byte + out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--cflags", pkgs) + if err != nil { + b.showOutput(p.Dir, b.PkgconfigCmd()+" --cflags "+strings.Join(pkgs, " "), string(out)) + b.Print(err.Error() + "\n") + err = errPrintedOutput + return + } + if len(out) > 0 { + cflags = splitPkgConfigOutput(out) + } + out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--libs", pkgs) + if err != nil { + b.showOutput(p.Dir, b.PkgconfigCmd()+" --libs "+strings.Join(pkgs, " "), string(out)) + b.Print(err.Error() + "\n") + err = errPrintedOutput + return + } + if len(out) > 0 { + ldflags = strings.Fields(string(out)) + } + } + return +} + +func (b *Builder) installShlibname(a *Action) error { + // TODO: BuildN + a1 := a.Deps[0] + err := ioutil.WriteFile(a.Target, []byte(filepath.Base(a1.Target)+"\n"), 0666) + if err != nil { + return err + } + if cfg.BuildX { + b.Showcmd("", "echo '%s' > %s # internal", filepath.Base(a1.Target), a.Target) + } + return nil +} + +func (b *Builder) linkShared(a *Action) (err error) { + importcfg := a.Objdir + "importcfg.link" + if err := b.writeLinkImportcfg(a, importcfg); err != nil { + return err + } + return BuildToolchain.ldShared(b, a.Deps[0].Deps, a.Target, importcfg, a.Deps) +} + +// BuildInstallFunc is the action for installing a single package or executable. +func BuildInstallFunc(b *Builder, a *Action) (err error) { + defer func() { + if err != nil && err != errPrintedOutput { + // a.Package == nil is possible for the go install -buildmode=shared + // action that installs libmangledname.so, which corresponds to + // a list of packages, not just one. + sep, path := "", "" + if a.Package != nil { + sep, path = " ", a.Package.ImportPath + } + err = fmt.Errorf("go install%s%s: %v", sep, path, err) + } + }() + a1 := a.Deps[0] + perm := os.FileMode(0666) + if a1.Mode == "link" { + switch cfg.BuildBuildmode { + case "c-archive", "c-shared", "plugin": + default: + perm = 0777 + } + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + // remove object dir to keep the amount of + // garbage down in a large build. On an operating system + // with aggressive buffering, cleaning incrementally like + // this keeps the intermediate objects from hitting the disk. + if !cfg.BuildWork { + defer func() { + if cfg.BuildX { + b.Showcmd("", "rm -r %s", a1.Objdir) + } + os.RemoveAll(a1.Objdir) + if _, err := os.Stat(a1.Target); err == nil { + if cfg.BuildX { + b.Showcmd("", "rm %s", a1.Target) + } + os.Remove(a1.Target) + } + }() + } + + return b.moveOrCopyFile(a, a.Target, a1.Target, perm, false) +} + +// moveOrCopyFile is like 'mv src dst' or 'cp src dst'. +func (b *Builder) moveOrCopyFile(a *Action, dst, src string, perm os.FileMode, force bool) error { + if cfg.BuildN { + b.Showcmd("", "mv %s %s", src, dst) + return nil + } + + // If we can update the mode and rename to the dst, do it. + // Otherwise fall back to standard copy. + + // If the destination directory has the group sticky bit set, + // we have to copy the file to retain the correct permissions. + // https://golang.org/issue/18878 + if fi, err := os.Stat(filepath.Dir(dst)); err == nil { + if fi.IsDir() && (fi.Mode()&os.ModeSetgid) != 0 { + return b.copyFile(a, dst, src, perm, force) + } + } + + // The perm argument is meant to be adjusted according to umask, + // but we don't know what the umask is. + // Create a dummy file to find out. + // This avoids build tags and works even on systems like Plan 9 + // where the file mask computation incorporates other information. + mode := perm + f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err == nil { + fi, err := f.Stat() + if err == nil { + mode = fi.Mode() & 0777 + } + name := f.Name() + f.Close() + os.Remove(name) + } + + if err := os.Chmod(src, mode); err == nil { + if err := os.Rename(src, dst); err == nil { + if cfg.BuildX { + b.Showcmd("", "mv %s %s", src, dst) + } + return nil + } + } + + return b.copyFile(a, dst, src, perm, force) +} + +// copyFile is like 'cp src dst'. +func (b *Builder) copyFile(a *Action, dst, src string, perm os.FileMode, force bool) error { + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "cp %s %s", src, dst) + if cfg.BuildN { + return nil + } + } + + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + + // Be careful about removing/overwriting dst. + // Do not remove/overwrite if dst exists and is a directory + // or a non-object file. + if fi, err := os.Stat(dst); err == nil { + if fi.IsDir() { + return fmt.Errorf("build output %q already exists and is a directory", dst) + } + if !force && fi.Mode().IsRegular() && !isObject(dst) { + return fmt.Errorf("build output %q already exists and is not an object file", dst) + } + } + + // On Windows, remove lingering ~ file from last attempt. + if base.ToolIsWindows { + if _, err := os.Stat(dst + "~"); err == nil { + os.Remove(dst + "~") + } + } + + mayberemovefile(dst) + df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil && base.ToolIsWindows { + // Windows does not allow deletion of a binary file + // while it is executing. Try to move it out of the way. + // If the move fails, which is likely, we'll try again the + // next time we do an install of this binary. + if err := os.Rename(dst, dst+"~"); err == nil { + os.Remove(dst + "~") + } + df, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + } + if err != nil { + return err + } + + _, err = io.Copy(df, sf) + df.Close() + if err != nil { + mayberemovefile(dst) + return fmt.Errorf("copying %s to %s: %v", src, dst, err) + } + return nil +} + +// writeFile writes the text to file. +func (b *Builder) writeFile(file string, text []byte) error { + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "cat >%s << 'EOF' # internal\n%sEOF", file, text) + } + if cfg.BuildN { + return nil + } + return ioutil.WriteFile(file, text, 0666) +} + +// Install the cgo export header file, if there is one. +func (b *Builder) installHeader(a *Action) error { + src := a.Objdir + "_cgo_install.h" + if _, err := os.Stat(src); os.IsNotExist(err) { + // If the file does not exist, there are no exported + // functions, and we do not install anything. + if cfg.BuildX { + b.Showcmd("", "# %s not created", src) + } + return nil + } + + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + return b.moveOrCopyFile(a, a.Target, src, 0666, true) +} + +// cover runs, in effect, +// go tool cover -mode=b.coverMode -var="varName" -o dst.go src.go +func (b *Builder) cover(a *Action, dst, src string, perm os.FileMode, varName string) error { + return b.run(a.Objdir, "cover "+a.Package.ImportPath, nil, + cfg.BuildToolexec, + base.Tool("cover"), + "-mode", a.Package.Internal.CoverMode, + "-var", varName, + "-o", dst, + src) +} + +var objectMagic = [][]byte{ + {'!', '<', 'a', 'r', 'c', 'h', '>', '\n'}, // Package archive + {'\x7F', 'E', 'L', 'F'}, // ELF + {0xFE, 0xED, 0xFA, 0xCE}, // Mach-O big-endian 32-bit + {0xFE, 0xED, 0xFA, 0xCF}, // Mach-O big-endian 64-bit + {0xCE, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 32-bit + {0xCF, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 64-bit + {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc + {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 + {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 + {0x00, 0x00, 0x06, 0x47}, // Plan 9 arm +} + +func isObject(s string) bool { + f, err := os.Open(s) + if err != nil { + return false + } + defer f.Close() + buf := make([]byte, 64) + io.ReadFull(f, buf) + for _, magic := range objectMagic { + if bytes.HasPrefix(buf, magic) { + return true + } + } + return false +} + +// mayberemovefile removes a file only if it is a regular file +// When running as a user with sufficient privileges, we may delete +// even device files, for example, which is not intended. +func mayberemovefile(s string) { + if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() { + return + } + os.Remove(s) +} + +// fmtcmd formats a command in the manner of fmt.Sprintf but also: +// +// If dir is non-empty and the script is not in dir right now, +// fmtcmd inserts "cd dir\n" before the command. +// +// fmtcmd replaces the value of b.WorkDir with $WORK. +// fmtcmd replaces the value of goroot with $GOROOT. +// fmtcmd replaces the value of b.gobin with $GOBIN. +// +// fmtcmd replaces the name of the current directory with dot (.) +// but only when it is at the beginning of a space-separated token. +// +func (b *Builder) fmtcmd(dir string, format string, args ...interface{}) string { + cmd := fmt.Sprintf(format, args...) + if dir != "" && dir != "/" { + cmd = strings.Replace(" "+cmd, " "+dir, " .", -1)[1:] + if b.scriptDir != dir { + b.scriptDir = dir + cmd = "cd " + dir + "\n" + cmd + } + } + if b.WorkDir != "" { + cmd = strings.Replace(cmd, b.WorkDir, "$WORK", -1) + } + return cmd +} + +// showcmd prints the given command to standard output +// for the implementation of -n or -x. +func (b *Builder) Showcmd(dir string, format string, args ...interface{}) { + b.output.Lock() + defer b.output.Unlock() + b.Print(b.fmtcmd(dir, format, args...) + "\n") +} + +// showOutput prints "# desc" followed by the given output. +// The output is expected to contain references to 'dir', usually +// the source directory for the package that has failed to build. +// showOutput rewrites mentions of dir with a relative path to dir +// when the relative path is shorter. This is usually more pleasant. +// For example, if fmt doesn't compile and we are in src/html, +// the output is +// +// $ go build +// # fmt +// ../fmt/print.go:1090: undefined: asdf +// $ +// +// instead of +// +// $ go build +// # fmt +// /usr/gopher/go/src/fmt/print.go:1090: undefined: asdf +// $ +// +// showOutput also replaces references to the work directory with $WORK. +// +func (b *Builder) showOutput(dir, desc, out string) { + prefix := "# " + desc + suffix := "\n" + out + if reldir := base.ShortPath(dir); reldir != dir { + suffix = strings.Replace(suffix, " "+dir, " "+reldir, -1) + suffix = strings.Replace(suffix, "\n"+dir, "\n"+reldir, -1) + } + suffix = strings.Replace(suffix, " "+b.WorkDir, " $WORK", -1) + + b.output.Lock() + defer b.output.Unlock() + b.Print(prefix, suffix) +} + +// errPrintedOutput is a special error indicating that a command failed +// but that it generated output as well, and that output has already +// been printed, so there's no point showing 'exit status 1' or whatever +// the wait status was. The main executor, builder.do, knows not to +// print this error. +var errPrintedOutput = errors.New("already printed output - no need to show error") + +var cgoLine = regexp.MustCompile(`\[[^\[\]]+\.cgo1\.go:[0-9]+(:[0-9]+)?\]`) +var cgoTypeSigRe = regexp.MustCompile(`\b_Ctype_\B`) + +// run runs the command given by cmdline in the directory dir. +// If the command fails, run prints information about the failure +// and returns a non-nil error. +func (b *Builder) run(dir string, desc string, env []string, cmdargs ...interface{}) error { + out, err := b.runOut(dir, desc, env, cmdargs...) + if len(out) > 0 { + if desc == "" { + desc = b.fmtcmd(dir, "%s", strings.Join(str.StringList(cmdargs...), " ")) + } + b.showOutput(dir, desc, b.processOutput(out)) + if err != nil { + err = errPrintedOutput + } + } + return err +} + +// processOutput prepares the output of runOut to be output to the console. +func (b *Builder) processOutput(out []byte) string { + if out[len(out)-1] != '\n' { + out = append(out, '\n') + } + messages := string(out) + // Fix up output referring to cgo-generated code to be more readable. + // Replace x.go:19[/tmp/.../x.cgo1.go:18] with x.go:19. + // Replace *[100]_Ctype_foo with *[100]C.foo. + // If we're using -x, assume we're debugging and want the full dump, so disable the rewrite. + if !cfg.BuildX && cgoLine.MatchString(messages) { + messages = cgoLine.ReplaceAllString(messages, "") + messages = cgoTypeSigRe.ReplaceAllString(messages, "C.") + } + return messages +} + +// runOut runs the command given by cmdline in the directory dir. +// It returns the command output and any errors that occurred. +func (b *Builder) runOut(dir string, desc string, env []string, cmdargs ...interface{}) ([]byte, error) { + cmdline := str.StringList(cmdargs...) + if cfg.BuildN || cfg.BuildX { + var envcmdline string + for _, e := range env { + if j := strings.IndexByte(e, '='); j != -1 { + if strings.ContainsRune(e[j+1:], '\'') { + envcmdline += fmt.Sprintf("%s=%q", e[:j], e[j+1:]) + } else { + envcmdline += fmt.Sprintf("%s='%s'", e[:j], e[j+1:]) + } + envcmdline += " " + } + } + envcmdline += joinUnambiguously(cmdline) + b.Showcmd(dir, "%s", envcmdline) + if cfg.BuildN { + return nil, nil + } + } + + nbusy := 0 + for { + var buf bytes.Buffer + cmd := exec.Command(cmdline[0], cmdline[1:]...) + cmd.Stdout = &buf + cmd.Stderr = &buf + cmd.Dir = dir + cmd.Env = base.MergeEnvLists(env, base.EnvForDir(cmd.Dir, os.Environ())) + err := cmd.Run() + + // cmd.Run will fail on Unix if some other process has the binary + // we want to run open for writing. This can happen here because + // we build and install the cgo command and then run it. + // If another command was kicked off while we were writing the + // cgo binary, the child process for that command may be holding + // a reference to the fd, keeping us from running exec. + // + // But, you might reasonably wonder, how can this happen? + // The cgo fd, like all our fds, is close-on-exec, so that we need + // not worry about other processes inheriting the fd accidentally. + // The answer is that running a command is fork and exec. + // A child forked while the cgo fd is open inherits that fd. + // Until the child has called exec, it holds the fd open and the + // kernel will not let us run cgo. Even if the child were to close + // the fd explicitly, it would still be open from the time of the fork + // until the time of the explicit close, and the race would remain. + // + // On Unix systems, this results in ETXTBSY, which formats + // as "text file busy". Rather than hard-code specific error cases, + // we just look for that string. If this happens, sleep a little + // and try again. We let this happen three times, with increasing + // sleep lengths: 100+200+400 ms = 0.7 seconds. + // + // An alternate solution might be to split the cmd.Run into + // separate cmd.Start and cmd.Wait, and then use an RWLock + // to make sure that copyFile only executes when no cmd.Start + // call is in progress. However, cmd.Start (really syscall.forkExec) + // only guarantees that when it returns, the exec is committed to + // happen and succeed. It uses a close-on-exec file descriptor + // itself to determine this, so we know that when cmd.Start returns, + // at least one close-on-exec file descriptor has been closed. + // However, we cannot be sure that all of them have been closed, + // so the program might still encounter ETXTBSY even with such + // an RWLock. The race window would be smaller, perhaps, but not + // guaranteed to be gone. + // + // Sleeping when we observe the race seems to be the most reliable + // option we have. + // + // https://golang.org/issue/3001 + // + if err != nil && nbusy < 3 && strings.Contains(err.Error(), "text file busy") { + time.Sleep(100 * time.Millisecond << uint(nbusy)) + nbusy++ + continue + } + + // err can be something like 'exit status 1'. + // Add information about what program was running. + // Note that if buf.Bytes() is non-empty, the caller usually + // shows buf.Bytes() and does not print err at all, so the + // prefix here does not make most output any more verbose. + if err != nil { + err = errors.New(cmdline[0] + ": " + err.Error()) + } + return buf.Bytes(), err + } +} + +// joinUnambiguously prints the slice, quoting where necessary to make the +// output unambiguous. +// TODO: See issue 5279. The printing of commands needs a complete redo. +func joinUnambiguously(a []string) string { + var buf bytes.Buffer + for i, s := range a { + if i > 0 { + buf.WriteByte(' ') + } + q := strconv.Quote(s) + if s == "" || strings.Contains(s, " ") || len(q) > len(s)+2 { + buf.WriteString(q) + } else { + buf.WriteString(s) + } + } + return buf.String() +} + +// mkdir makes the named directory. +func (b *Builder) Mkdir(dir string) error { + b.exec.Lock() + defer b.exec.Unlock() + // We can be a little aggressive about being + // sure directories exist. Skip repeated calls. + if b.mkdirCache[dir] { + return nil + } + b.mkdirCache[dir] = true + + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "mkdir -p %s", dir) + if cfg.BuildN { + return nil + } + } + + if err := os.MkdirAll(dir, 0777); err != nil { + return err + } + return nil +} + +// symlink creates a symlink newname -> oldname. +func (b *Builder) Symlink(oldname, newname string) error { + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "ln -s %s %s", oldname, newname) + if cfg.BuildN { + return nil + } + } + return os.Symlink(oldname, newname) +} + +// mkAbs returns an absolute path corresponding to +// evaluating f in the directory dir. +// We always pass absolute paths of source files so that +// the error messages will include the full path to a file +// in need of attention. +func mkAbs(dir, f string) string { + // Leave absolute paths alone. + // Also, during -n mode we use the pseudo-directory $WORK + // instead of creating an actual work directory that won't be used. + // Leave paths beginning with $WORK alone too. + if filepath.IsAbs(f) || strings.HasPrefix(f, "$WORK") { + return f + } + return filepath.Join(dir, f) +} + +type toolchain interface { + // gc runs the compiler in a specific directory on a set of files + // and returns the name of the generated output file. + gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) + // cc runs the toolchain's C compiler in a directory on a C file + // to produce an output file. + cc(b *Builder, a *Action, ofile, cfile string) error + // asm runs the assembler in a specific directory on specific files + // and returns a list of named output files. + asm(b *Builder, a *Action, sfiles []string) ([]string, error) + // pack runs the archive packer in a specific directory to create + // an archive from a set of object files. + // typically it is run in the object directory. + pack(b *Builder, a *Action, afile string, ofiles []string) error + // ld runs the linker to create an executable starting at mainpkg. + ld(b *Builder, root *Action, out, importcfg, mainpkg string) error + // ldShared runs the linker to create a shared library containing the pkgs built by toplevelactions + ldShared(b *Builder, toplevelactions []*Action, out, importcfg string, allactions []*Action) error + + compiler() string + linker() string +} + +type noToolchain struct{} + +func noCompiler() error { + log.Fatalf("unknown compiler %q", cfg.BuildContext.Compiler) + return nil +} + +func (noToolchain) compiler() string { + noCompiler() + return "" +} + +func (noToolchain) linker() string { + noCompiler() + return "" +} + +func (noToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, out []byte, err error) { + return "", nil, noCompiler() +} + +func (noToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + return nil, noCompiler() +} + +func (noToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + return noCompiler() +} + +func (noToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { + return noCompiler() +} + +func (noToolchain) ldShared(b *Builder, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { + return noCompiler() +} + +func (noToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + return noCompiler() +} + +// gcc runs the gcc C compiler to create an object from a single C file. +func (b *Builder) gcc(p *load.Package, workdir, out string, flags []string, cfile string) error { + return b.ccompile(p, out, flags, cfile, b.GccCmd(p.Dir, workdir)) +} + +// gxx runs the g++ C++ compiler to create an object from a single C++ file. +func (b *Builder) gxx(p *load.Package, workdir, out string, flags []string, cxxfile string) error { + return b.ccompile(p, out, flags, cxxfile, b.GxxCmd(p.Dir, workdir)) +} + +// gfortran runs the gfortran Fortran compiler to create an object from a single Fortran file. +func (b *Builder) gfortran(p *load.Package, workdir, out string, flags []string, ffile string) error { + return b.ccompile(p, out, flags, ffile, b.gfortranCmd(p.Dir, workdir)) +} + +// ccompile runs the given C or C++ compiler and creates an object from a single source file. +func (b *Builder) ccompile(p *load.Package, outfile string, flags []string, file string, compiler []string) error { + file = mkAbs(p.Dir, file) + desc := p.ImportPath + if !filepath.IsAbs(outfile) { + outfile = filepath.Join(p.Dir, outfile) + } + output, err := b.runOut(filepath.Dir(file), desc, nil, compiler, flags, "-o", outfile, "-c", filepath.Base(file)) + if len(output) > 0 { + // On FreeBSD 11, when we pass -g to clang 3.8 it + // invokes its internal assembler with -dwarf-version=2. + // When it sees .section .note.GNU-stack, it warns + // "DWARF2 only supports one section per compilation unit". + // This warning makes no sense, since the section is empty, + // but it confuses people. + // We work around the problem by detecting the warning + // and dropping -g and trying again. + if bytes.Contains(output, []byte("DWARF2 only supports one section per compilation unit")) { + newFlags := make([]string, 0, len(flags)) + for _, f := range flags { + if !strings.HasPrefix(f, "-g") { + newFlags = append(newFlags, f) + } + } + if len(newFlags) < len(flags) { + return b.ccompile(p, outfile, newFlags, file, compiler) + } + } + + b.showOutput(p.Dir, desc, b.processOutput(output)) + if err != nil { + err = errPrintedOutput + } else if os.Getenv("GO_BUILDER_NAME") != "" { + return errors.New("C compiler warning promoted to error on Go builders") + } + } + return err +} + +// gccld runs the gcc linker to create an executable from a set of object files. +func (b *Builder) gccld(p *load.Package, objdir, out string, flags []string, objs []string) error { + var cmd []string + if len(p.CXXFiles) > 0 || len(p.SwigCXXFiles) > 0 { + cmd = b.GxxCmd(p.Dir, objdir) + } else { + cmd = b.GccCmd(p.Dir, objdir) + } + return b.run(p.Dir, p.ImportPath, nil, cmd, "-o", out, objs, flags) +} + +// gccCmd returns a gcc command line prefix +// defaultCC is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) GccCmd(incdir, workdir string) []string { + return b.compilerCmd("CC", cfg.DefaultCC, incdir, workdir) +} + +// gxxCmd returns a g++ command line prefix +// defaultCXX is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) GxxCmd(incdir, workdir string) []string { + return b.compilerCmd("CXX", cfg.DefaultCXX, incdir, workdir) +} + +// gfortranCmd returns a gfortran command line prefix. +func (b *Builder) gfortranCmd(incdir, workdir string) []string { + return b.compilerCmd("FC", "gfortran", incdir, workdir) +} + +// compilerCmd returns a command line prefix for the given environment +// variable and using the default command when the variable is empty. +func (b *Builder) compilerCmd(envvar, defcmd, incdir, workdir string) []string { + // NOTE: env.go's mkEnv knows that the first three + // strings returned are "gcc", "-I", incdir (and cuts them off). + + compiler := envList(envvar, defcmd) + a := []string{compiler[0], "-I", incdir} + a = append(a, compiler[1:]...) + + // Definitely want -fPIC but on Windows gcc complains + // "-fPIC ignored for target (all code is position independent)" + if cfg.Goos != "windows" { + a = append(a, "-fPIC") + } + a = append(a, b.gccArchArgs()...) + // gcc-4.5 and beyond require explicit "-pthread" flag + // for multithreading with pthread library. + if cfg.BuildContext.CgoEnabled { + switch cfg.Goos { + case "windows": + a = append(a, "-mthreads") + default: + a = append(a, "-pthread") + } + } + + // disable ASCII art in clang errors, if possible + if b.gccSupportsFlag(compiler, "-fno-caret-diagnostics") { + a = append(a, "-fno-caret-diagnostics") + } + // clang is too smart about command-line arguments + if b.gccSupportsFlag(compiler, "-Qunused-arguments") { + a = append(a, "-Qunused-arguments") + } + + // disable word wrapping in error messages + a = append(a, "-fmessage-length=0") + + // Tell gcc not to include the work directory in object files. + if b.gccSupportsFlag(compiler, "-fdebug-prefix-map=a=b") { + if workdir == "" { + workdir = b.WorkDir + } + workdir = strings.TrimSuffix(workdir, string(filepath.Separator)) + a = append(a, "-fdebug-prefix-map="+workdir+"=/tmp/go-build") + } + + // Tell gcc not to include flags in object files, which defeats the + // point of -fdebug-prefix-map above. + if b.gccSupportsFlag(compiler, "-gno-record-gcc-switches") { + a = append(a, "-gno-record-gcc-switches") + } + + // On OS X, some of the compilers behave as if -fno-common + // is always set, and the Mach-O linker in 6l/8l assumes this. + // See https://golang.org/issue/3253. + if cfg.Goos == "darwin" { + a = append(a, "-fno-common") + } + + return a +} + +// gccNoPie returns the flag to use to request non-PIE. On systems +// with PIE (position independent executables) enabled by default, +// -no-pie must be passed when doing a partial link with -Wl,-r. +// But -no-pie is not supported by all compilers, and clang spells it -nopie. +func (b *Builder) gccNoPie(linker []string) string { + if b.gccSupportsFlag(linker, "-no-pie") { + return "-no-pie" + } + if b.gccSupportsFlag(linker, "-nopie") { + return "-nopie" + } + return "" +} + +// gccSupportsFlag checks to see if the compiler supports a flag. +func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { + key := [2]string{compiler[0], flag} + + b.exec.Lock() + defer b.exec.Unlock() + if b, ok := b.flagCache[key]; ok { + return b + } + if b.flagCache == nil { + if cfg.BuildN || cfg.BuildX { + b.Showcmd(b.WorkDir, "touch trivial.c") + } + if !cfg.BuildN { + src := filepath.Join(b.WorkDir, "trivial.c") + if err := ioutil.WriteFile(src, []byte{}, 0666); err != nil { + return false + } + } + b.flagCache = make(map[[2]string]bool) + } + cmdArgs := append([]string(nil), compiler...) + cmdArgs = append(cmdArgs, flag, "-c", "trivial.c") + if cfg.BuildN || cfg.BuildX { + b.Showcmd(b.WorkDir, "%s", joinUnambiguously(cmdArgs)) + if cfg.BuildN { + return false + } + } + cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) + cmd.Dir = b.WorkDir + cmd.Env = base.MergeEnvLists([]string{"LC_ALL=C"}, base.EnvForDir(cmd.Dir, os.Environ())) + out, err := cmd.CombinedOutput() + // GCC says "unrecognized command line option". + // clang says "unknown argument". + supported := err == nil && !bytes.Contains(out, []byte("unrecognized")) && !bytes.Contains(out, []byte("unknown")) + b.flagCache[key] = supported + return supported +} + +// gccArchArgs returns arguments to pass to gcc based on the architecture. +func (b *Builder) gccArchArgs() []string { + switch cfg.Goarch { + case "386": + return []string{"-m32"} + case "amd64", "amd64p32": + return []string{"-m64"} + case "arm": + return []string{"-marm"} // not thumb + case "s390x": + return []string{"-m64", "-march=z196"} + case "mips64", "mips64le": + return []string{"-mabi=64"} + case "mips", "mipsle": + return []string{"-mabi=32", "-march=mips32"} + } + return nil +} + +// envList returns the value of the given environment variable broken +// into fields, using the default value when the variable is empty. +func envList(key, def string) []string { + v := os.Getenv(key) + if v == "" { + v = def + } + return strings.Fields(v) +} + +// CFlags returns the flags to use when invoking the C, C++ or Fortran compilers, or cgo. +func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, ldflags []string) { + defaults := "-g -O2" + + cppflags = str.StringList(envList("CGO_CPPFLAGS", ""), p.CgoCPPFLAGS) + cflags = str.StringList(envList("CGO_CFLAGS", defaults), p.CgoCFLAGS) + cxxflags = str.StringList(envList("CGO_CXXFLAGS", defaults), p.CgoCXXFLAGS) + fflags = str.StringList(envList("CGO_FFLAGS", defaults), p.CgoFFLAGS) + ldflags = str.StringList(envList("CGO_LDFLAGS", defaults), p.CgoLDFLAGS) + return +} + +var cgoRe = regexp.MustCompile(`[/\\:]`) + +func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgofiles, objdirCgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { + p := a.Package + cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoFFLAGS, cgoLDFLAGS := b.CFlags(p) + cgoCPPFLAGS = append(cgoCPPFLAGS, pcCFLAGS...) + cgoLDFLAGS = append(cgoLDFLAGS, pcLDFLAGS...) + // If we are compiling Objective-C code, then we need to link against libobjc + if len(mfiles) > 0 { + cgoLDFLAGS = append(cgoLDFLAGS, "-lobjc") + } + + // Likewise for Fortran, except there are many Fortran compilers. + // Support gfortran out of the box and let others pass the correct link options + // via CGO_LDFLAGS + if len(ffiles) > 0 { + fc := os.Getenv("FC") + if fc == "" { + fc = "gfortran" + } + if strings.Contains(fc, "gfortran") { + cgoLDFLAGS = append(cgoLDFLAGS, "-lgfortran") + } + } + + if cfg.BuildMSan { + cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...) + cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) + } + + // Allows including _cgo_export.h from .[ch] files in the package. + cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", objdir) + + // If we have cgo files in the object directory, then copy any + // other cgo files into the object directory, and pass a + // -srcdir option to cgo. + var srcdirarg []string + if len(objdirCgofiles) > 0 { + for _, fn := range cgofiles { + if err := b.copyFile(a, objdir+filepath.Base(fn), filepath.Join(p.Dir, fn), 0666, false); err != nil { + return nil, nil, err + } + } + cgofiles = append(cgofiles, objdirCgofiles...) + srcdirarg = []string{"-srcdir", objdir} + } + + // cgo + // TODO: CGO_FLAGS? + gofiles := []string{objdir + "_cgo_gotypes.go"} + cfiles := []string{"_cgo_export.c"} + for _, fn := range cgofiles { + f := cgoRe.ReplaceAllString(fn[:len(fn)-2], "_") + gofiles = append(gofiles, objdir+f+"cgo1.go") + cfiles = append(cfiles, f+"cgo2.c") + } + + // TODO: make cgo not depend on $GOARCH? + + cgoflags := []string{} + if p.Standard && p.ImportPath == "runtime/cgo" { + cgoflags = append(cgoflags, "-import_runtime_cgo=false") + } + if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo") { + cgoflags = append(cgoflags, "-import_syscall=false") + } + + // Update $CGO_LDFLAGS with p.CgoLDFLAGS. + var cgoenv []string + if len(cgoLDFLAGS) > 0 { + flags := make([]string, len(cgoLDFLAGS)) + for i, f := range cgoLDFLAGS { + flags[i] = strconv.Quote(f) + } + cgoenv = []string{"CGO_LDFLAGS=" + strings.Join(flags, " ")} + } + + if cfg.BuildToolchainName == "gccgo" { + switch cfg.Goarch { + case "386", "amd64": + cgoCFLAGS = append(cgoCFLAGS, "-fsplit-stack") + } + cgoflags = append(cgoflags, "-gccgo") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + cgoflags = append(cgoflags, "-gccgopkgpath="+pkgpath) + } + } + + switch cfg.BuildBuildmode { + case "c-archive", "c-shared": + // Tell cgo that if there are any exported functions + // it should generate a header file that C code can + // #include. + cgoflags = append(cgoflags, "-exportheader="+objdir+"_cgo_install.h") + } + + if err := b.run(p.Dir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, srcdirarg, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { + return nil, nil, err + } + outGo = append(outGo, gofiles...) + + // Use sequential object file names to keep them distinct + // and short enough to fit in the .a header file name slots. + // We no longer collect them all into _all.o, and we'd like + // tools to see both the .o suffix and unique names, so + // we need to make them short enough not to be truncated + // in the final archive. + oseq := 0 + nextOfile := func() string { + oseq++ + return objdir + fmt.Sprintf("_x%03d.o", oseq) + } + + // gcc + cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS) + for _, cfile := range cfiles { + ofile := nextOfile() + if err := b.gcc(p, a.Objdir, ofile, cflags, objdir+cfile); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + for _, file := range gccfiles { + ofile := nextOfile() + if err := b.gcc(p, a.Objdir, ofile, cflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS) + for _, file := range gxxfiles { + ofile := nextOfile() + if err := b.gxx(p, a.Objdir, ofile, cxxflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + for _, file := range mfiles { + ofile := nextOfile() + if err := b.gcc(p, a.Objdir, ofile, cflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS) + for _, file := range ffiles { + ofile := nextOfile() + if err := b.gfortran(p, a.Objdir, ofile, fflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + switch cfg.BuildToolchainName { + case "gc": + importGo := objdir + "_cgo_import.go" + if err := b.dynimport(p, objdir, importGo, cgoExe, cflags, cgoLDFLAGS, outObj); err != nil { + return nil, nil, err + } + outGo = append(outGo, importGo) + + case "gccgo": + defunC := objdir + "_cgo_defun.c" + defunObj := objdir + "_cgo_defun.o" + if err := BuildToolchain.cc(b, a, defunObj, defunC); err != nil { + return nil, nil, err + } + outObj = append(outObj, defunObj) + + default: + noCompiler() + } + + return outGo, outObj, nil +} + +// dynimport creates a Go source file named importGo containing +// //go:cgo_import_dynamic directives for each symbol or library +// dynamically imported by the object files outObj. +func (b *Builder) dynimport(p *load.Package, objdir, importGo, cgoExe string, cflags, cgoLDFLAGS, outObj []string) error { + cfile := objdir + "_cgo_main.c" + ofile := objdir + "_cgo_main.o" + if err := b.gcc(p, objdir, ofile, cflags, cfile); err != nil { + return err + } + + linkobj := str.StringList(ofile, outObj, p.SysoFiles) + dynobj := objdir + "_cgo_.o" + + // we need to use -pie for Linux/ARM to get accurate imported sym + ldflags := cgoLDFLAGS + if (cfg.Goarch == "arm" && cfg.Goos == "linux") || cfg.Goos == "android" { + ldflags = append(ldflags, "-pie") + } + if err := b.gccld(p, objdir, dynobj, ldflags, linkobj); err != nil { + return err + } + + // cgo -dynimport + var cgoflags []string + if p.Standard && p.ImportPath == "runtime/cgo" { + cgoflags = []string{"-dynlinker"} // record path to dynamic linker + } + return b.run(p.Dir, p.ImportPath, nil, cfg.BuildToolexec, cgoExe, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags) +} + +// Run SWIG on all SWIG input files. +// TODO: Don't build a shared library, once SWIG emits the necessary +// pragmas for external linking. +func (b *Builder) swig(p *load.Package, objdir string, pcCFLAGS []string) (outGo, outC, outCXX []string, err error) { + if err := b.swigVersionCheck(); err != nil { + return nil, nil, nil, err + } + + intgosize, err := b.swigIntSize(objdir) + if err != nil { + return nil, nil, nil, err + } + + for _, f := range p.SwigFiles { + goFile, cFile, err := b.swigOne(p, f, objdir, pcCFLAGS, false, intgosize) + if err != nil { + return nil, nil, nil, err + } + if goFile != "" { + outGo = append(outGo, goFile) + } + if cFile != "" { + outC = append(outC, cFile) + } + } + for _, f := range p.SwigCXXFiles { + goFile, cxxFile, err := b.swigOne(p, f, objdir, pcCFLAGS, true, intgosize) + if err != nil { + return nil, nil, nil, err + } + if goFile != "" { + outGo = append(outGo, goFile) + } + if cxxFile != "" { + outCXX = append(outCXX, cxxFile) + } + } + return outGo, outC, outCXX, nil +} + +// Make sure SWIG is new enough. +var ( + swigCheckOnce sync.Once + swigCheck error +) + +func (b *Builder) swigDoVersionCheck() error { + out, err := b.runOut("", "", nil, "swig", "-version") + if err != nil { + return err + } + re := regexp.MustCompile(`[vV]ersion +([\d]+)([.][\d]+)?([.][\d]+)?`) + matches := re.FindSubmatch(out) + if matches == nil { + // Can't find version number; hope for the best. + return nil + } + + major, err := strconv.Atoi(string(matches[1])) + if err != nil { + // Can't find version number; hope for the best. + return nil + } + const errmsg = "must have SWIG version >= 3.0.6" + if major < 3 { + return errors.New(errmsg) + } + if major > 3 { + // 4.0 or later + return nil + } + + // We have SWIG version 3.x. + if len(matches[2]) > 0 { + minor, err := strconv.Atoi(string(matches[2][1:])) + if err != nil { + return nil + } + if minor > 0 { + // 3.1 or later + return nil + } + } + + // We have SWIG version 3.0.x. + if len(matches[3]) > 0 { + patch, err := strconv.Atoi(string(matches[3][1:])) + if err != nil { + return nil + } + if patch < 6 { + // Before 3.0.6. + return errors.New(errmsg) + } + } + + return nil +} + +func (b *Builder) swigVersionCheck() error { + swigCheckOnce.Do(func() { + swigCheck = b.swigDoVersionCheck() + }) + return swigCheck +} + +// Find the value to pass for the -intgosize option to swig. +var ( + swigIntSizeOnce sync.Once + swigIntSize string + swigIntSizeError error +) + +// This code fails to build if sizeof(int) <= 32 +const swigIntSizeCode = ` +package main +const i int = 1 << 32 +` + +// Determine the size of int on the target system for the -intgosize option +// of swig >= 2.0.9. Run only once. +func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { + if cfg.BuildN { + return "$INTBITS", nil + } + src := filepath.Join(b.WorkDir, "swig_intsize.go") + if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { + return + } + srcs := []string{src} + + p := load.GoFilesPackage(srcs) + + if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, false, srcs); e != nil { + return "32", nil + } + return "64", nil +} + +// Determine the size of int on the target system for the -intgosize option +// of swig >= 2.0.9. +func (b *Builder) swigIntSize(objdir string) (intsize string, err error) { + swigIntSizeOnce.Do(func() { + swigIntSize, swigIntSizeError = b.swigDoIntSize(objdir) + }) + return swigIntSize, swigIntSizeError +} + +// Run SWIG on one SWIG input file. +func (b *Builder) swigOne(p *load.Package, file, objdir string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) { + cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, _, _ := b.CFlags(p) + var cflags []string + if cxx { + cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCXXFLAGS) + } else { + cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCFLAGS) + } + + n := 5 // length of ".swig" + if cxx { + n = 8 // length of ".swigcxx" + } + base := file[:len(file)-n] + goFile := base + ".go" + gccBase := base + "_wrap." + gccExt := "c" + if cxx { + gccExt = "cxx" + } + + gccgo := cfg.BuildToolchainName == "gccgo" + + // swig + args := []string{ + "-go", + "-cgo", + "-intgosize", intgosize, + "-module", base, + "-o", objdir + gccBase + gccExt, + "-outdir", objdir, + } + + for _, f := range cflags { + if len(f) > 3 && f[:2] == "-I" { + args = append(args, f) + } + } + + if gccgo { + args = append(args, "-gccgo") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + args = append(args, "-go-pkgpath", pkgpath) + } + } + if cxx { + args = append(args, "-c++") + } + + out, err := b.runOut(p.Dir, p.ImportPath, nil, "swig", args, file) + if err != nil { + if len(out) > 0 { + if bytes.Contains(out, []byte("-intgosize")) || bytes.Contains(out, []byte("-cgo")) { + return "", "", errors.New("must have SWIG version >= 3.0.6") + } + b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig error + return "", "", errPrintedOutput + } + return "", "", err + } + if len(out) > 0 { + b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig warning + } + + return goFile, objdir + gccBase + gccExt, nil +} + +// disableBuildID adjusts a linker command line to avoid creating a +// build ID when creating an object file rather than an executable or +// shared library. Some systems, such as Ubuntu, always add +// --build-id to every link, but we don't want a build ID when we are +// producing an object file. On some of those system a plain -r (not +// -Wl,-r) will turn off --build-id, but clang 3.0 doesn't support a +// plain -r. I don't know how to turn off --build-id when using clang +// other than passing a trailing --build-id=none. So that is what we +// do, but only on systems likely to support it, which is to say, +// systems that normally use gold or the GNU linker. +func (b *Builder) disableBuildID(ldflags []string) []string { + switch cfg.Goos { + case "android", "dragonfly", "linux", "netbsd": + ldflags = append(ldflags, "-Wl,--build-id=none") + } + return ldflags +} diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go new file mode 100644 index 0000000000..514d5beef9 --- /dev/null +++ b/src/cmd/go/internal/work/gc.go @@ -0,0 +1,481 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" + "crypto/sha1" +) + +// The Go toolchain. + +type gcToolchain struct{} + +func (gcToolchain) compiler() string { + return base.Tool("compile") +} + +func (gcToolchain) linker() string { + return base.Tool("link") +} + +func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { + p := a.Package + objdir := a.Objdir + if archive != "" { + ofile = archive + } else { + out := "_go_.o" + ofile = objdir + out + } + + pkgpath := p.ImportPath + if cfg.BuildBuildmode == "plugin" { + pkgpath = pluginPath(p) + } else if p.Name == "main" { + pkgpath = "main" + } + gcargs := []string{"-p", pkgpath} + if p.Standard { + gcargs = append(gcargs, "-std") + } + compilingRuntime := p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) + if compilingRuntime { + // runtime compiles with a special gc flag to emit + // additional reflect type data. + gcargs = append(gcargs, "-+") + } + + // If we're giving the compiler the entire package (no C etc files), tell it that, + // so that it can give good error messages about forward declarations. + // Exceptions: a few standard packages have forward declarations for + // pieces supplied behind-the-scenes by package runtime. + extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) + if p.Standard { + switch p.ImportPath { + case "bytes", "internal/poll", "net", "os", "runtime/pprof", "sync", "syscall", "time": + extFiles++ + } + } + if extFiles == 0 { + gcargs = append(gcargs, "-complete") + } + if cfg.BuildContext.InstallSuffix != "" { + gcargs = append(gcargs, "-installsuffix", cfg.BuildContext.InstallSuffix) + } + if a.buildID != "" { + gcargs = append(gcargs, "-buildid", a.buildID) + } + platform := cfg.Goos + "/" + cfg.Goarch + if p.Internal.OmitDebug || platform == "nacl/amd64p32" || platform == "darwin/arm" || platform == "darwin/arm64" || cfg.Goos == "plan9" { + gcargs = append(gcargs, "-dwarf=false") + } + + gcflags := buildGcflags + if compilingRuntime { + // Remove -N, if present. + // It is not possible to build the runtime with no optimizations, + // because the compiler cannot eliminate enough write barriers. + gcflags = make([]string, len(buildGcflags)) + copy(gcflags, buildGcflags) + for i := 0; i < len(gcflags); i++ { + if gcflags[i] == "-N" { + copy(gcflags[i:], gcflags[i+1:]) + gcflags = gcflags[:len(gcflags)-1] + i-- + } + } + } + args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", b.WorkDir, gcflags, gcargs, "-D", p.Internal.LocalPrefix} + if importcfg != nil { + if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { + return "", nil, err + } + args = append(args, "-importcfg", objdir+"importcfg") + } + if ofile == archive { + args = append(args, "-pack") + } + if asmhdr { + args = append(args, "-asmhdr", objdir+"go_asm.h") + } + + // Add -c=N to use concurrent backend compilation, if possible. + if c := gcBackendConcurrency(gcflags); c > 1 { + args = append(args, fmt.Sprintf("-c=%d", c)) + } + + for _, f := range gofiles { + args = append(args, mkAbs(p.Dir, f)) + } + + output, err = b.runOut(p.Dir, p.ImportPath, nil, args...) + return ofile, output, err +} + +// gcBackendConcurrency returns the backend compiler concurrency level for a package compilation. +func gcBackendConcurrency(gcflags []string) int { + // First, check whether we can use -c at all for this compilation. + canDashC := concurrentGCBackendCompilationEnabledByDefault + + switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e { + case "0": + canDashC = false + case "1": + canDashC = true + case "": + // Not set. Use default. + default: + log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e) + } + + if os.Getenv("GOEXPERIMENT") != "" { + // Concurrent compilation is presumed incompatible with GOEXPERIMENTs. + canDashC = false + } + +CheckFlags: + for _, flag := range gcflags { + // Concurrent compilation is presumed incompatible with any gcflags, + // except for a small whitelist of commonly used flags. + // If the user knows better, they can manually add their own -c to the gcflags. + switch flag { + case "-N", "-l", "-S", "-B", "-C", "-I": + // OK + default: + canDashC = false + break CheckFlags + } + } + + if !canDashC { + return 1 + } + + // Decide how many concurrent backend compilations to allow. + // + // If we allow too many, in theory we might end up with p concurrent processes, + // each with c concurrent backend compiles, all fighting over the same resources. + // However, in practice, that seems not to happen too much. + // Most build graphs are surprisingly serial, so p==1 for much of the build. + // Furthermore, concurrent backend compilation is only enabled for a part + // of the overall compiler execution, so c==1 for much of the build. + // So don't worry too much about that interaction for now. + // + // However, in practice, setting c above 4 tends not to help very much. + // See the analysis in CL 41192. + // + // TODO(josharian): attempt to detect whether this particular compilation + // is likely to be a bottleneck, e.g. when: + // - it has no successor packages to compile (usually package main) + // - all paths through the build graph pass through it + // - critical path scheduling says it is high priority + // and in such a case, set c to runtime.NumCPU. + // We do this now when p==1. + if cfg.BuildP == 1 { + // No process parallelism. Max out c. + return runtime.NumCPU() + } + // Some process parallelism. Set c to min(4, numcpu). + c := 4 + if ncpu := runtime.NumCPU(); ncpu < c { + c = ncpu + } + return c +} + +func (gcToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + p := a.Package + // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. + inc := filepath.Join(cfg.GOROOT, "pkg", "include") + args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-trimpath", b.WorkDir, "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, buildAsmflags} + if p.ImportPath == "runtime" && cfg.Goarch == "386" { + for _, arg := range buildAsmflags { + if arg == "-dynlink" { + args = append(args, "-D=GOBUILDMODE_shared=1") + } + } + } + var ofiles []string + for _, sfile := range sfiles { + ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + a := append(args, "-o", ofile, mkAbs(p.Dir, sfile)) + if err := b.run(p.Dir, p.ImportPath, nil, a...); err != nil { + return nil, err + } + } + return ofiles, nil +} + +// toolVerify checks that the command line args writes the same output file +// if run using newTool instead. +// Unused now but kept around for future use. +func toolVerify(b *Builder, p *load.Package, newTool string, ofile string, args []interface{}) error { + newArgs := make([]interface{}, len(args)) + copy(newArgs, args) + newArgs[1] = base.Tool(newTool) + newArgs[3] = ofile + ".new" // x.6 becomes x.6.new + if err := b.run(p.Dir, p.ImportPath, nil, newArgs...); err != nil { + return err + } + data1, err := ioutil.ReadFile(ofile) + if err != nil { + return err + } + data2, err := ioutil.ReadFile(ofile + ".new") + if err != nil { + return err + } + if !bytes.Equal(data1, data2) { + return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(str.StringList(args...), " "), strings.Join(str.StringList(newArgs...), " ")) + } + os.Remove(ofile + ".new") + return nil +} + +func (gcToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + var absOfiles []string + for _, f := range ofiles { + absOfiles = append(absOfiles, mkAbs(a.Objdir, f)) + } + absAfile := mkAbs(a.Objdir, afile) + + // The archive file should have been created by the compiler. + // Since it used to not work that way, verify. + if !cfg.BuildN { + if _, err := os.Stat(absAfile); err != nil { + base.Fatalf("os.Stat of archive file failed: %v", err) + } + } + + p := a.Package + if cfg.BuildN || cfg.BuildX { + cmdline := str.StringList(base.Tool("pack"), "r", absAfile, absOfiles) + b.Showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) + } + if cfg.BuildN { + return nil + } + if err := packInternal(b, absAfile, absOfiles); err != nil { + b.showOutput(p.Dir, p.ImportPath, err.Error()+"\n") + return errPrintedOutput + } + return nil +} + +func packInternal(b *Builder, afile string, ofiles []string) error { + dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return err + } + defer dst.Close() // only for error returns or panics + w := bufio.NewWriter(dst) + + for _, ofile := range ofiles { + src, err := os.Open(ofile) + if err != nil { + return err + } + fi, err := src.Stat() + if err != nil { + src.Close() + return err + } + // Note: Not using %-16.16s format because we care + // about bytes, not runes. + name := fi.Name() + if len(name) > 16 { + name = name[:16] + } else { + name += strings.Repeat(" ", 16-len(name)) + } + size := fi.Size() + fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n", + name, 0, 0, 0, 0644, size) + n, err := io.Copy(w, src) + src.Close() + if err == nil && n < size { + err = io.ErrUnexpectedEOF + } else if err == nil && n > size { + err = fmt.Errorf("file larger than size reported by stat") + } + if err != nil { + return fmt.Errorf("copying %s to %s: %v", ofile, afile, err) + } + if size&1 != 0 { + w.WriteByte(0) + } + } + + if err := w.Flush(); err != nil { + return err + } + return dst.Close() +} + +// setextld sets the appropriate linker flags for the specified compiler. +func setextld(ldflags []string, compiler []string) []string { + for _, f := range ldflags { + if f == "-extld" || strings.HasPrefix(f, "-extld=") { + // don't override -extld if supplied + return ldflags + } + } + ldflags = append(ldflags, "-extld="+compiler[0]) + if len(compiler) > 1 { + extldflags := false + add := strings.Join(compiler[1:], " ") + for i, f := range ldflags { + if f == "-extldflags" && i+1 < len(ldflags) { + ldflags[i+1] = add + " " + ldflags[i+1] + extldflags = true + break + } else if strings.HasPrefix(f, "-extldflags=") { + ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] + extldflags = true + break + } + } + if !extldflags { + ldflags = append(ldflags, "-extldflags="+add) + } + } + return ldflags +} + +// pluginPath computes the package path for a plugin main package. +// +// This is typically the import path of the main package p, unless the +// plugin is being built directly from source files. In that case we +// combine the package build ID with the contents of the main package +// source files. This allows us to identify two different plugins +// built from two source files with the same name. +func pluginPath(p *load.Package) string { + if p.ImportPath != "command-line-arguments" { + return p.ImportPath + } + h := sha1.New() + fmt.Fprintf(h, "build ID: %s\n", p.Internal.BuildID) + for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) { + data, err := ioutil.ReadFile(filepath.Join(p.Dir, file)) + if err != nil { + base.Fatalf("go: %s", err) + } + h.Write(data) + } + return fmt.Sprintf("plugin/unnamed-%x", h.Sum(nil)) +} + +func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { + cxx := len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 + for _, a := range root.Deps { + if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { + cxx = true + } + } + var ldflags []string + if cfg.BuildContext.InstallSuffix != "" { + ldflags = append(ldflags, "-installsuffix", cfg.BuildContext.InstallSuffix) + } + if root.Package.Internal.OmitDebug { + ldflags = append(ldflags, "-s", "-w") + } + if cfg.BuildBuildmode == "plugin" { + ldflags = append(ldflags, "-pluginpath", pluginPath(root.Package)) + } + + // TODO(rsc): This is probably wrong - see golang.org/issue/22155. + if cfg.GOROOT != runtime.GOROOT() { + ldflags = append(ldflags, "-X=runtime/internal/sys.DefaultGoroot="+cfg.GOROOT) + } + + // Store BuildID inside toolchain binaries as a unique identifier of the + // tool being run, for use by content-based staleness determination. + if root.Package.Goroot && strings.HasPrefix(root.Package.ImportPath, "cmd/") { + ldflags = append(ldflags, "-X=cmd/internal/objabi.buildID="+root.buildID) + } + + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", cfg.DefaultCXX) + } else { + compiler = envList("CC", cfg.DefaultCC) + } + ldflags = append(ldflags, "-buildmode="+ldBuildmode) + if root.buildID != "" { + ldflags = append(ldflags, "-buildid="+root.buildID) + } + ldflags = append(ldflags, cfg.BuildLdflags...) + ldflags = setextld(ldflags, compiler) + + // On OS X when using external linking to build a shared library, + // the argument passed here to -o ends up recorded in the final + // shared library in the LC_ID_DYLIB load command. + // To avoid putting the temporary output directory name there + // (and making the resulting shared library useless), + // run the link in the output directory so that -o can name + // just the final path element. + // On Windows, DLL file name is recorded in PE file + // export section, so do like on OS X. + dir := "." + if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" { + dir, out = filepath.Split(out) + } + + return b.run(dir, root.Package.ImportPath, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) +} + +func (gcToolchain) ldShared(b *Builder, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { + ldflags := []string{"-installsuffix", cfg.BuildContext.InstallSuffix} + ldflags = append(ldflags, "-buildmode=shared") + ldflags = append(ldflags, cfg.BuildLdflags...) + cxx := false + for _, a := range allactions { + if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { + cxx = true + } + } + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", cfg.DefaultCXX) + } else { + compiler = envList("CC", cfg.DefaultCC) + } + ldflags = setextld(ldflags, compiler) + for _, d := range toplevelactions { + if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries + continue + } + ldflags = append(ldflags, d.Package.ImportPath+"="+d.Target) + } + return b.run(".", out, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags) +} + +func (gcToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(a.Package.Dir, cfile)) +} diff --git a/src/cmd/go/internal/work/gccgo.go b/src/cmd/go/internal/work/gccgo.go new file mode 100644 index 0000000000..1d7155c977 --- /dev/null +++ b/src/cmd/go/internal/work/gccgo.go @@ -0,0 +1,497 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" +) + +// The Gccgo toolchain. + +type gccgoToolchain struct{} + +var GccgoName, GccgoBin string +var gccgoErr error + +func init() { + GccgoName = os.Getenv("GCCGO") + if GccgoName == "" { + GccgoName = "gccgo" + } + GccgoBin, gccgoErr = exec.LookPath(GccgoName) +} + +func (gccgoToolchain) compiler() string { + checkGccgoBin() + return GccgoBin +} + +func (gccgoToolchain) linker() string { + checkGccgoBin() + return GccgoBin +} + +func checkGccgoBin() { + if gccgoErr == nil { + return + } + fmt.Fprintf(os.Stderr, "cmd/go: gccgo: %s\n", gccgoErr) + os.Exit(2) +} + +func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { + p := a.Package + objdir := a.Objdir + out := "_go_.o" + ofile = objdir + out + gcargs := []string{"-g"} + gcargs = append(gcargs, b.gccArchArgs()...) + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + gcargs = append(gcargs, "-fgo-pkgpath="+pkgpath) + } + if p.Internal.LocalPrefix != "" { + gcargs = append(gcargs, "-fgo-relative-import-path="+p.Internal.LocalPrefix) + } + + args := str.StringList(tools.compiler(), "-c", gcargs, "-o", ofile) + if importcfg != nil { + if b.gccSupportsFlag(args[:1], "-fgo-importcfg=/dev/null") { + if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { + return "", nil, err + } + args = append(args, "-fgo-importcfg="+objdir+"importcfg") + } else { + root := objdir + "_importcfgroot_" + if err := buildImportcfgSymlinks(b, root, importcfg); err != nil { + return "", nil, err + } + args = append(args, "-I", root) + } + } + args = append(args, buildGccgoflags...) + for _, f := range gofiles { + args = append(args, mkAbs(p.Dir, f)) + } + + output, err = b.runOut(p.Dir, p.ImportPath, nil, args) + return ofile, output, err +} + +// buildImportcfgSymlinks builds in root a tree of symlinks +// implementing the directives from importcfg. +// This serves as a temporary transition mechanism until +// we can depend on gccgo reading an importcfg directly. +// (The Go 1.9 and later gc compilers already do.) +func buildImportcfgSymlinks(b *Builder, root string, importcfg []byte) error { + for lineNum, line := range strings.Split(string(importcfg), "\n") { + lineNum++ // 1-based + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line == "" || strings.HasPrefix(line, "#") { + continue + } + var verb, args string + if i := strings.Index(line, " "); i < 0 { + verb = line + } else { + verb, args = line[:i], strings.TrimSpace(line[i+1:]) + } + var before, after string + if i := strings.Index(args, "="); i >= 0 { + before, after = args[:i], args[i+1:] + } + switch verb { + default: + base.Fatalf("importcfg:%d: unknown directive %q", lineNum, verb) + case "packagefile": + if before == "" || after == "" { + return fmt.Errorf(`importcfg:%d: invalid packagefile: syntax is "packagefile path=filename": %s`, lineNum, line) + } + archive := gccgoArchive(root, before) + if err := b.Mkdir(filepath.Dir(archive)); err != nil { + return err + } + if err := b.Symlink(after, archive); err != nil { + return err + } + case "importmap": + if before == "" || after == "" { + return fmt.Errorf(`importcfg:%d: invalid importmap: syntax is "importmap old=new": %s`, lineNum, line) + } + beforeA := gccgoArchive(root, before) + afterA := gccgoArchive(root, after) + if err := b.Mkdir(filepath.Dir(beforeA)); err != nil { + return err + } + if err := b.Mkdir(filepath.Dir(afterA)); err != nil { + return err + } + if err := b.Symlink(afterA, beforeA); err != nil { + return err + } + case "packageshlib": + return fmt.Errorf("gccgo -importcfg does not support shared libraries") + } + } + return nil +} + +func (tools gccgoToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { + p := a.Package + var ofiles []string + for _, sfile := range sfiles { + ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + sfile = mkAbs(p.Dir, sfile) + defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} + if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { + defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) + } + defs = tools.maybePIC(defs) + defs = append(defs, b.gccArchArgs()...) + err := b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-xassembler-with-cpp", "-I", a.Objdir, "-c", "-o", ofile, defs, sfile) + if err != nil { + return nil, err + } + } + return ofiles, nil +} + +func gccgoArchive(basedir, imp string) string { + end := filepath.FromSlash(imp + ".a") + afile := filepath.Join(basedir, end) + // add "lib" to the final element + return filepath.Join(filepath.Dir(afile), "lib"+filepath.Base(afile)) +} + +func (gccgoToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { + p := a.Package + objdir := a.Objdir + var absOfiles []string + for _, f := range ofiles { + absOfiles = append(absOfiles, mkAbs(objdir, f)) + } + return b.run(p.Dir, p.ImportPath, nil, "ar", "rc", mkAbs(objdir, afile), absOfiles) +} + +func (tools gccgoToolchain) link(b *Builder, root *Action, out, importcfg string, allactions []*Action, buildmode, desc string) error { + // gccgo needs explicit linking with all package dependencies, + // and all LDFLAGS from cgo dependencies. + apackagePathsSeen := make(map[string]bool) + afiles := []string{} + shlibs := []string{} + ldflags := b.gccArchArgs() + cgoldflags := []string{} + usesCgo := false + cxx := false + objc := false + fortran := false + if root.Package != nil { + cxx = len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 + objc = len(root.Package.MFiles) > 0 + fortran = len(root.Package.FFiles) > 0 + } + + readCgoFlags := func(flagsFile string) error { + flags, err := ioutil.ReadFile(flagsFile) + if err != nil { + return err + } + const ldflagsPrefix = "_CGO_LDFLAGS=" + for _, line := range strings.Split(string(flags), "\n") { + if strings.HasPrefix(line, ldflagsPrefix) { + newFlags := strings.Fields(line[len(ldflagsPrefix):]) + for _, flag := range newFlags { + // Every _cgo_flags file has -g and -O2 in _CGO_LDFLAGS + // but they don't mean anything to the linker so filter + // them out. + if flag != "-g" && !strings.HasPrefix(flag, "-O") { + cgoldflags = append(cgoldflags, flag) + } + } + } + } + return nil + } + + newID := 0 + readAndRemoveCgoFlags := func(archive string) (string, error) { + newID++ + newArchive := root.Objdir + fmt.Sprintf("_pkg%d_.a", newID) + if err := b.copyFile(root, newArchive, archive, 0666, false); err != nil { + return "", err + } + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "ar d %s _cgo_flags", newArchive) + if cfg.BuildN { + // TODO(rsc): We could do better about showing the right _cgo_flags even in -n mode. + // Either the archive is already built and we can read them out, + // or we're printing commands to build the archive and can + // forward the _cgo_flags directly to this step. + return "", nil + } + } + err := b.run(root.Objdir, desc, nil, "ar", "x", newArchive, "_cgo_flags") + if err != nil { + return "", err + } + err = b.run(".", desc, nil, "ar", "d", newArchive, "_cgo_flags") + if err != nil { + return "", err + } + err = readCgoFlags(filepath.Join(root.Objdir, "_cgo_flags")) + if err != nil { + return "", err + } + return newArchive, nil + } + + actionsSeen := make(map[*Action]bool) + // Make a pre-order depth-first traversal of the action graph, taking note of + // whether a shared library action has been seen on the way to an action (the + // construction of the graph means that if any path to a node passes through + // a shared library action, they all do). + var walk func(a *Action, seenShlib bool) + var err error + walk = func(a *Action, seenShlib bool) { + if actionsSeen[a] { + return + } + actionsSeen[a] = true + if a.Package != nil && !seenShlib { + if a.Package.Standard { + return + } + // We record the target of the first time we see a .a file + // for a package to make sure that we prefer the 'install' + // rather than the 'build' location (which may not exist any + // more). We still need to traverse the dependencies of the + // build action though so saying + // if apackagePathsSeen[a.Package.ImportPath] { return } + // doesn't work. + if !apackagePathsSeen[a.Package.ImportPath] { + apackagePathsSeen[a.Package.ImportPath] = true + target := a.Target + if len(a.Package.CgoFiles) > 0 || a.Package.UsesSwig() { + target, err = readAndRemoveCgoFlags(target) + if err != nil { + return + } + } + afiles = append(afiles, target) + } + } + if strings.HasSuffix(a.Target, ".so") { + shlibs = append(shlibs, a.Target) + seenShlib = true + } + for _, a1 := range a.Deps { + walk(a1, seenShlib) + if err != nil { + return + } + } + } + for _, a1 := range root.Deps { + walk(a1, false) + if err != nil { + return err + } + } + + for _, a := range allactions { + // Gather CgoLDFLAGS, but not from standard packages. + // The go tool can dig up runtime/cgo from GOROOT and + // think that it should use its CgoLDFLAGS, but gccgo + // doesn't use runtime/cgo. + if a.Package == nil { + continue + } + if !a.Package.Standard { + cgoldflags = append(cgoldflags, a.Package.CgoLDFLAGS...) + } + if len(a.Package.CgoFiles) > 0 { + usesCgo = true + } + if a.Package.UsesSwig() { + usesCgo = true + } + if len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0 { + cxx = true + } + if len(a.Package.MFiles) > 0 { + objc = true + } + if len(a.Package.FFiles) > 0 { + fortran = true + } + } + + ldflags = append(ldflags, "-Wl,--whole-archive") + ldflags = append(ldflags, afiles...) + ldflags = append(ldflags, "-Wl,--no-whole-archive") + + ldflags = append(ldflags, cgoldflags...) + ldflags = append(ldflags, envList("CGO_LDFLAGS", "")...) + if root.Package != nil { + ldflags = append(ldflags, root.Package.CgoLDFLAGS...) + } + + ldflags = str.StringList("-Wl,-(", ldflags, "-Wl,-)") + + for _, shlib := range shlibs { + ldflags = append( + ldflags, + "-L"+filepath.Dir(shlib), + "-Wl,-rpath="+filepath.Dir(shlib), + "-l"+strings.TrimSuffix( + strings.TrimPrefix(filepath.Base(shlib), "lib"), + ".so")) + } + + var realOut string + switch buildmode { + case "exe": + if usesCgo && cfg.Goos == "linux" { + ldflags = append(ldflags, "-Wl,-E") + } + + case "c-archive": + // Link the Go files into a single .o, and also link + // in -lgolibbegin. + // + // We need to use --whole-archive with -lgolibbegin + // because it doesn't define any symbols that will + // cause the contents to be pulled in; it's just + // initialization code. + // + // The user remains responsible for linking against + // -lgo -lpthread -lm in the final link. We can't use + // -r to pick them up because we can't combine + // split-stack and non-split-stack code in a single -r + // link, and libgo picks up non-split-stack code from + // libffi. + ldflags = append(ldflags, "-Wl,-r", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive") + + if nopie := b.gccNoPie([]string{tools.linker()}); nopie != "" { + ldflags = append(ldflags, nopie) + } + + // We are creating an object file, so we don't want a build ID. + ldflags = b.disableBuildID(ldflags) + + realOut = out + out = out + ".o" + + case "c-shared": + ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") + case "shared": + ldflags = append(ldflags, "-zdefs", "-shared", "-nostdlib", "-lgo", "-lgcc_s", "-lgcc", "-lc") + + default: + base.Fatalf("-buildmode=%s not supported for gccgo", buildmode) + } + + switch buildmode { + case "exe", "c-shared": + if cxx { + ldflags = append(ldflags, "-lstdc++") + } + if objc { + ldflags = append(ldflags, "-lobjc") + } + if fortran { + fc := os.Getenv("FC") + if fc == "" { + fc = "gfortran" + } + // support gfortran out of the box and let others pass the correct link options + // via CGO_LDFLAGS + if strings.Contains(fc, "gfortran") { + ldflags = append(ldflags, "-lgfortran") + } + } + } + + if err := b.run(".", desc, nil, tools.linker(), "-o", out, ldflags, buildGccgoflags); err != nil { + return err + } + + switch buildmode { + case "c-archive": + if err := b.run(".", desc, nil, "ar", "rc", realOut, out); err != nil { + return err + } + } + return nil +} + +func (tools gccgoToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { + return tools.link(b, root, out, importcfg, root.Deps, ldBuildmode, root.Package.ImportPath) +} + +func (tools gccgoToolchain) ldShared(b *Builder, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { + fakeRoot := &Action{Mode: "gccgo ldshared"} + fakeRoot.Deps = toplevelactions + return tools.link(b, fakeRoot, out, importcfg, allactions, "shared", out) +} + +func (tools gccgoToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { + p := a.Package + inc := filepath.Join(cfg.GOROOT, "pkg", "include") + cfile = mkAbs(p.Dir, cfile) + defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} + defs = append(defs, b.gccArchArgs()...) + if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { + defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) + } + switch cfg.Goarch { + case "386", "amd64": + defs = append(defs, "-fsplit-stack") + } + defs = tools.maybePIC(defs) + return b.run(p.Dir, p.ImportPath, nil, envList("CC", cfg.DefaultCC), "-Wall", "-g", + "-I", a.Objdir, "-I", inc, "-o", ofile, defs, "-c", cfile) +} + +// maybePIC adds -fPIC to the list of arguments if needed. +func (tools gccgoToolchain) maybePIC(args []string) []string { + switch cfg.BuildBuildmode { + case "c-shared", "shared", "plugin": + args = append(args, "-fPIC") + } + return args +} + +func gccgoPkgpath(p *load.Package) string { + if p.Internal.Build.IsCommand() && !p.Internal.ForceLibrary { + return "" + } + return p.ImportPath +} + +func gccgoCleanPkgpath(p *load.Package) string { + clean := func(r rune) rune { + switch { + case 'A' <= r && r <= 'Z', 'a' <= r && r <= 'z', + '0' <= r && r <= '9': + return r + } + return '_' + } + return strings.Map(clean, gccgoPkgpath(p)) +} -- 2.48.1