From: Jay Conrod Date: Wed, 23 Oct 2019 21:14:09 +0000 (-0400) Subject: cmd/dist: support GOROOT vendoring X-Git-Tag: go1.14beta1~532 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=60f34f739de41d8000661baac5dc1cebba80e892;p=gostls13.git cmd/dist: support GOROOT vendoring In the second step of make.bash, cmd/dist builds cmd/go by invoking the compiler, linker, and other tools directly on transitive dependencies of cmd/go. Essentially, cmd/dist acts as a minimal version of 'go install' when building go_toolchain. Until now, cmd/go has had no transitive dependencies in vendor directories. This changes in CL 202698, where several packages are deleted and equivalent versions in golang.org/x/mod are used instead. So this CL adds support to cmd/dist for vendor directories. Updates #31761 Change-Id: Iab4cdc7e505069a8df296287d16fbaa871944955 Reviewed-on: https://go-review.googlesource.com/c/go/+/203537 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index 8d29eb98a7..9eb9e8f241 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -605,26 +605,26 @@ func startInstall(dir string) chan struct{} { // runInstall installs the library, package, or binary associated with dir, // which is relative to $GOROOT/src. -func runInstall(dir string, ch chan struct{}) { - if dir == "net" || dir == "os/user" || dir == "crypto/x509" { - fatalf("go_bootstrap cannot depend on cgo package %s", dir) +func runInstall(pkg string, ch chan struct{}) { + if pkg == "net" || pkg == "os/user" || pkg == "crypto/x509" { + fatalf("go_bootstrap cannot depend on cgo package %s", pkg) } defer close(ch) - if dir == "unsafe" { + if pkg == "unsafe" { return } if vflag > 0 { if goos != gohostos || goarch != gohostarch { - errprintf("%s (%s/%s)\n", dir, goos, goarch) + errprintf("%s (%s/%s)\n", pkg, goos, goarch) } else { - errprintf("%s\n", dir) + errprintf("%s\n", pkg) } } - workdir := pathf("%s/%s", workdir, dir) + workdir := pathf("%s/%s", workdir, pkg) xmkdirall(workdir) var clean []string @@ -634,11 +634,14 @@ func runInstall(dir string, ch chan struct{}) { } }() - // path = full path to dir. - path := pathf("%s/src/%s", goroot, dir) + // dir = full path to pkg. + dir := pathf("%s/src/%s", goroot, pkg) name := filepath.Base(dir) - ispkg := !strings.HasPrefix(dir, "cmd/") || strings.Contains(dir, "/internal/") + // ispkg predicts whether the package should be linked as a binary, based + // on the name. There should be no "main" packages in vendor, since + // 'go mod vendor' will only copy imported packages there. + ispkg := !strings.HasPrefix(pkg, "cmd/") || strings.Contains(pkg, "/internal/") || strings.Contains(pkg, "/vendor/") // Start final link command line. // Note: code below knows that link.p[targ] is the target. @@ -650,7 +653,7 @@ func runInstall(dir string, ch chan struct{}) { if ispkg { // Go library (package). ispackcmd = true - link = []string{"pack", pathf("%s/pkg/%s_%s/%s.a", goroot, goos, goarch, dir)} + link = []string{"pack", packagefile(pkg)} targ = len(link) - 1 xmkdirall(filepath.Dir(link[targ])) } else { @@ -675,7 +678,7 @@ func runInstall(dir string, ch chan struct{}) { // Gather files that are sources for this target. // Everything in that directory, and any target-specific // additions. - files := xreaddir(path) + files := xreaddir(dir) // Remove files beginning with . or _, // which are likely to be editor temporary files. @@ -687,7 +690,7 @@ func runInstall(dir string, ch chan struct{}) { }) for _, dt := range deptab { - if dir == dt.prefix || strings.HasSuffix(dt.prefix, "/") && strings.HasPrefix(dir, dt.prefix) { + if pkg == dt.prefix || strings.HasSuffix(dt.prefix, "/") && strings.HasPrefix(pkg, dt.prefix) { for _, p := range dt.dep { p = os.ExpandEnv(p) files = append(files, p) @@ -699,7 +702,7 @@ func runInstall(dir string, ch chan struct{}) { // Convert to absolute paths. for i, p := range files { if !filepath.IsAbs(p) { - files[i] = pathf("%s/%s", path, p) + files[i] = pathf("%s/%s", dir, p) } } @@ -715,7 +718,7 @@ func runInstall(dir string, ch chan struct{}) { return false ok: t := mtime(p) - if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, dir) { + if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, pkg) { return false } if strings.HasSuffix(p, ".go") { @@ -742,7 +745,7 @@ func runInstall(dir string, ch chan struct{}) { } // For package runtime, copy some files into the work space. - if dir == "runtime" { + if pkg == "runtime" { xmkdirall(pathf("%s/pkg/include", goroot)) // For use by assembly and C files. copyfile(pathf("%s/pkg/include/textflag.h", goroot), @@ -764,7 +767,7 @@ func runInstall(dir string, ch chan struct{}) { if vflag > 1 { errprintf("generate %s\n", p) } - gt.gen(path, p) + gt.gen(dir, p) // Do not add generated file to clean list. // In runtime, we want to be able to // build the package with the go tool, @@ -782,22 +785,31 @@ func runInstall(dir string, ch chan struct{}) { built: } - // Make sure dependencies are installed. - var deps []string + // Resolve imported packages to actual package paths. + // Make sure they're installed. + importMap := make(map[string]string) for _, p := range gofiles { - deps = append(deps, readimports(p)...) + for _, imp := range readimports(p) { + importMap[imp] = resolveVendor(imp, dir) + } + } + sortedImports := make([]string, 0, len(importMap)) + for imp := range importMap { + sortedImports = append(sortedImports, imp) } - for _, dir1 := range deps { - startInstall(dir1) + sort.Strings(sortedImports) + + for _, dep := range importMap { + startInstall(dep) } - for _, dir1 := range deps { - install(dir1) + for _, dep := range importMap { + install(dep) } if goos != gohostos || goarch != gohostarch { // We've generated the right files; the go command can do the build. if vflag > 1 { - errprintf("skip build for cross-compile %s\n", dir) + errprintf("skip build for cross-compile %s\n", pkg) } return } @@ -830,18 +842,35 @@ func runInstall(dir string, ch chan struct{}) { if err := ioutil.WriteFile(goasmh, nil, 0666); err != nil { fatalf("cannot write empty go_asm.h: %s", err) } - bgrun(&wg, path, asmabis...) + bgrun(&wg, dir, asmabis...) bgwait(&wg) } + // Build an importcfg file for the compiler. + buf := &bytes.Buffer{} + for _, imp := range sortedImports { + if imp == "unsafe" { + continue + } + dep := importMap[imp] + if imp != dep { + fmt.Fprintf(buf, "importmap %s=%s\n", imp, dep) + } + fmt.Fprintf(buf, "packagefile %s=%s\n", dep, packagefile(dep)) + } + importcfg := pathf("%s/importcfg", workdir) + if err := ioutil.WriteFile(importcfg, buf.Bytes(), 0666); err != nil { + fatalf("cannot write importcfg file: %v", err) + } + var archive string // The next loop will compile individual non-Go files. // Hand the Go files to the compiler en masse. // For packages containing assembly, this writes go_asm.h, which // the assembly files will need. - pkg := dir - if strings.HasPrefix(dir, "cmd/") && strings.Count(dir, "/") == 1 { - pkg = "main" + pkgName := pkg + if strings.HasPrefix(pkg, "cmd/") && strings.Count(pkg, "/") == 1 { + pkgName = "main" } b := pathf("%s/_go_.a", workdir) clean = append(clean, b) @@ -852,11 +881,11 @@ func runInstall(dir string, ch chan struct{}) { } // Compile Go code. - compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkg} + compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkgName, "-importcfg", importcfg} if gogcflags != "" { compile = append(compile, strings.Fields(gogcflags)...) } - if dir == "runtime" { + if pkg == "runtime" { compile = append(compile, "-+") } if len(sfiles) > 0 { @@ -874,7 +903,7 @@ func runInstall(dir string, ch chan struct{}) { // We use bgrun and immediately wait for it instead of calling run() synchronously. // This executes all jobs through the bgwork channel and allows the process // to exit cleanly in case an error occurs. - bgrun(&wg, path, compile...) + bgrun(&wg, dir, compile...) bgwait(&wg) // Compile the files. @@ -888,7 +917,7 @@ func runInstall(dir string, ch chan struct{}) { // Change the last character of the output file (which was c or s). b = b[:len(b)-1] + "o" compile = append(compile, "-o", b, p) - bgrun(&wg, path, compile...) + bgrun(&wg, dir, compile...) link = append(link, b) if doclean { @@ -909,6 +938,12 @@ func runInstall(dir string, ch chan struct{}) { bgwait(&wg) } +// packagefile returns the path to a compiled .a file for the given package +// path. Paths may need to be resolved with resolveVendor first. +func packagefile(pkg string) string { + return pathf("%s/pkg/%s_%s/%s.a", goroot, goos, goarch, pkg) +} + // matchfield reports whether the field (x,y,z) matches this build. // all the elements in the field must be satisfied. func matchfield(f string) bool { @@ -940,7 +975,7 @@ func matchtag(tag string) bool { // of GOOS and GOARCH. // We also allow the special tag cmd_go_bootstrap. // See ../go/bootstrap.go and package go/build. -func shouldbuild(file, dir string) bool { +func shouldbuild(file, pkg string) bool { // Check file name for GOOS or GOARCH. name := filepath.Base(file) excluded := func(list []string, ok string) bool { @@ -982,7 +1017,7 @@ func shouldbuild(file, dir string) bool { if code == "package documentation" { return false } - if code == "package main" && dir != "cmd/go" && dir != "cmd/cgo" { + if code == "package main" && pkg != "cmd/go" && pkg != "cmd/cgo" { return false } if !strings.HasPrefix(p, "//") { diff --git a/src/cmd/dist/imports.go b/src/cmd/dist/imports.go index bf64d6668a..05dd84d0f1 100644 --- a/src/cmd/dist/imports.go +++ b/src/cmd/dist/imports.go @@ -11,7 +11,10 @@ package main import ( "bufio" "errors" + "fmt" "io" + "path" + "path/filepath" "strconv" "strings" "unicode/utf8" @@ -243,3 +246,31 @@ func readimports(file string) []string { return imports } + +// resolveVendor returns a unique package path imported with the given import +// path from srcDir. +// +// resolveVendor assumes that a package is vendored if and only if its first +// path component contains a dot. If a package is vendored, its import path +// is returned with a "vendor" or "cmd/vendor" prefix, depending on srcDir. +// Otherwise, the import path is returned verbatim. +func resolveVendor(imp, srcDir string) string { + var first string + if i := strings.Index(imp, "/"); i < 0 { + first = imp + } else { + first = imp[:i] + } + isStandard := !strings.Contains(first, ".") + if isStandard { + return imp + } + + if strings.HasPrefix(srcDir, filepath.Join(goroot, "src", "cmd")) { + return path.Join("cmd", "vendor", imp) + } else if strings.HasPrefix(srcDir, filepath.Join(goroot, "src")) { + return path.Join("vendor", imp) + } else { + panic(fmt.Sprintf("srcDir %q not in GOOROT/src", srcDir)) + } +}