From 1635205a72d26721af54f01ccbab8e0b51ded3a9 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 19 Jan 2023 17:04:07 -0500 Subject: [PATCH] cmd/dist: add -distpack flag to build distribution archives We want to enable others to reproduce the exact distribution archives we are serving on go.dev/dl. Today the code for building those archives lives in golang.org/x/build, which is fundamentally tied to running on the Go team build infrastructure and not easy for others to run. This CL adds a new flag -distpack to cmd/dist, usually invoked as make.bash -distpack, to build the distribution archives using code in the main repository that anyone can run. Starting in Go 1.21, the Go team build infrastructure will run this instead of its current custom code to build those archives. The current builds are not reproducible even given identical infrastructure, because the archives are stamped with the current time. It is helpful to have a timestamp in the archives indicating when the code is from, but that time needs to be reproducible. To ensure this, the new -distpack flag extends the VERSION file to include a time stamp, which it uses as the modification time for all files in the archive. The new -distpack flag is implemented by a separate program, cmd/distpack, instead of being in cmd/dist, so that it can be compiled by the toolchain being distributed and not the bootstrap toolchain. Otherwise details like the exact compression algorithms might vary from one bootstrap toolchain to another and produce non-reproducible builds. So there is a new 'go tool distpack', but it's omitted from the distributions themselves, just as 'go tool dist' is. make.bash already accepts any flags for cmd/dist, including -distpack. make.bat is less sophisticated and looks for each known flag, so this CL adds an update to look for -distpack. The CL also changes make.bat to accept the idiomatic Go -flagname in addition to the non-idiomatic (for Go) --flagname. Previously it insisted on the --flag form. I have confirmed that using make.bash -distpack produces the identical distribution archives for windows/amd64, linux/amd64, darwin/amd64, and darwin/arm64 whether it is run on windows/amd64, linux/amd64, or darwin/amd64 hosts. For #24904. Change-Id: Ie6d69365ee3d7294d05b4f96ffb9159b41918074 Reviewed-on: https://go-review.googlesource.com/c/go/+/470676 TryBot-Result: Gopher Robot Reviewed-by: Heschi Kreinick Run-TryBot: Russ Cox Reviewed-by: Carlos Amedee --- src/cmd/dist/build.go | 40 +++- src/cmd/distpack/archive.go | 197 ++++++++++++++++ src/cmd/distpack/archive_test.go | 39 ++++ src/cmd/distpack/pack.go | 376 +++++++++++++++++++++++++++++++ src/cmd/distpack/test.go | 166 ++++++++++++++ src/make.bat | 40 +++- 6 files changed, 849 insertions(+), 9 deletions(-) create mode 100644 src/cmd/distpack/archive.go create mode 100644 src/cmd/distpack/archive_test.go create mode 100644 src/cmd/distpack/pack.go create mode 100644 src/cmd/distpack/test.go diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index d82aaa3be2..a76c312709 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -351,12 +351,40 @@ func chomp(s string) string { } // findgoversion determines the Go version to use in the version string. +// It also parses any other metadata found in the version file. func findgoversion() string { // The $GOROOT/VERSION file takes priority, for distributions // without the source repo. path := pathf("%s/VERSION", goroot) if isfile(path) { b := chomp(readfile(path)) + + // Starting in Go 1.21 the VERSION file starts with the + // version on a line by itself but then can contain other + // metadata about the release, one item per line. + if i := strings.Index(b, "\n"); i >= 0 { + rest := b[i+1:] + b = chomp(b[:i]) + for _, line := range strings.Split(rest, "\n") { + f := strings.Fields(line) + if len(f) == 0 { + continue + } + switch f[0] { + default: + fatalf("VERSION: unexpected line: %s", line) + case "time": + if len(f) != 2 { + fatalf("VERSION: unexpected time line: %s", line) + } + _, err := time.Parse(time.RFC3339, f[1]) + if err != nil { + fatalf("VERSION: bad time: %s", err) + } + } + } + } + // Commands such as "dist version > VERSION" will cause // the shell to create an empty VERSION file and set dist's // stdout to its fd. dist in turn looks at VERSION and uses @@ -591,6 +619,7 @@ func mustLinkExternal(goos, goarch string, cgoEnabled bool) bool { // exclude files with that prefix. // Note that this table applies only to the build of cmd/go, // after the main compiler bootstrap. +// Files listed here should also be listed in ../distpack/pack.go's srcArch.Remove list. var deptab = []struct { prefix string // prefix of target dep []string // dependency tweaks for targets with that prefix @@ -1206,6 +1235,9 @@ func clean() { // Remove cached version info. xremove(pathf("%s/VERSION.cache", goroot)) + + // Remove distribution packages. + xremoveall(pathf("%s/pkg/distpack", goroot)) } } @@ -1347,9 +1379,10 @@ func cmdbootstrap() { timelog("start", "dist bootstrap") defer timelog("end", "dist bootstrap") - var debug, force, noBanner, noClean bool + var debug, distpack, force, noBanner, noClean bool flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all") flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process") + flag.BoolVar(&distpack, "distpack", distpack, "write distribution files to pkg/distpack") flag.BoolVar(&force, "force", force, "build even if the port is marked as broken") flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner") flag.BoolVar(&noClean, "no-clean", noClean, "print deprecation warning") @@ -1592,6 +1625,11 @@ func cmdbootstrap() { os.Setenv("CC", oldcc) } + if distpack { + xprintf("Packaging archives for %s/%s.\n", goos, goarch) + run("", ShowOutput|CheckExit, pathf("%s/distpack", tooldir)) + } + // Print trailing banner unless instructed otherwise. if !noBanner { banner() diff --git a/src/cmd/distpack/archive.go b/src/cmd/distpack/archive.go new file mode 100644 index 0000000000..2fdc006b55 --- /dev/null +++ b/src/cmd/distpack/archive.go @@ -0,0 +1,197 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "io/fs" + "log" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" +) + +// An Archive describes an archive to write: a collection of files. +// Directories are implied by the files and not explicitly listed. +type Archive struct { + Files []File +} + +// A File describes a single file to write to an archive. +type File struct { + Name string // name in archive + Time time.Time // modification time + Mode fs.FileMode + Size int64 + Src string // source file in OS file system +} + +// Info returns a FileInfo about the file, for use with tar.FileInfoHeader +// and zip.FileInfoHeader. +func (f *File) Info() fs.FileInfo { + return fileInfo{f} +} + +// A fileInfo is an implementation of fs.FileInfo describing a File. +type fileInfo struct { + f *File +} + +func (i fileInfo) Name() string { return path.Base(i.f.Name) } +func (i fileInfo) ModTime() time.Time { return i.f.Time } +func (i fileInfo) Mode() fs.FileMode { return i.f.Mode } +func (i fileInfo) IsDir() bool { return false } +func (i fileInfo) Size() int64 { return i.f.Size } +func (i fileInfo) Sys() any { return nil } + +// NewArchive returns a new Archive containing all the files in the directory dir. +// The archive can be amended afterward using methods like Add and Filter. +func NewArchive(dir string) (*Archive, error) { + a := new(Archive) + err := fs.WalkDir(os.DirFS(dir), ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + a.Add(name, filepath.Join(dir, name), info) + return nil + }) + if err != nil { + return nil, err + } + a.Sort() + return a, nil +} + +// Add adds a file with the given name and info to the archive. +// The content of the file comes from the operating system file src. +// After a sequence of one or more calls to Add, +// the caller should invoke Sort to re-sort the archive's files. +func (a *Archive) Add(name, src string, info fs.FileInfo) { + a.Files = append(a.Files, File{ + Name: name, + Time: info.ModTime(), + Mode: info.Mode(), + Size: info.Size(), + Src: src, + }) +} + +// Sort sorts the files in the archive. +// It is only necessary to call Sort after calling Add. +// ArchiveDir returns a sorted archive, and the other methods +// preserve the sorting of the archive. +func (a *Archive) Sort() { + sort.Slice(a.Files, func(i, j int) bool { + return a.Files[i].Name < a.Files[j].Name + }) +} + +// Clone returns a copy of the Archive. +// Method calls like Add and Filter invoked on the copy do not affect the original, +// nor do calls on the original affect the copy. +func (a *Archive) Clone() *Archive { + b := &Archive{ + Files: make([]File, len(a.Files)), + } + copy(b.Files, a.Files) + return b +} + +// AddPrefix adds a prefix to all file names in the archive. +func (a *Archive) AddPrefix(prefix string) { + for i := range a.Files { + a.Files[i].Name = path.Join(prefix, a.Files[i].Name) + } +} + +// Filter removes files from the archive for which keep(name) returns false. +func (a *Archive) Filter(keep func(name string) bool) { + files := a.Files[:0] + for _, f := range a.Files { + if keep(f.Name) { + files = append(files, f) + } + } + a.Files = files +} + +// SetMode changes the mode of every file in the archive +// to be mode(name, m), where m is the file's current mode. +func (a *Archive) SetMode(mode func(name string, m fs.FileMode) fs.FileMode) { + for i := range a.Files { + a.Files[i].Mode = mode(a.Files[i].Name, a.Files[i].Mode) + } +} + +// Remove removes files matching any of the patterns from the archive. +// The patterns use the syntax of path.Match, with an extension of allowing +// a leading **/ or trailing /**, which match any number of path elements +// (including no path elements) before or after the main match. +func (a *Archive) Remove(patterns ...string) { + a.Filter(func(name string) bool { + for _, pattern := range patterns { + match, err := amatch(pattern, name) + if err != nil { + log.Fatalf("archive remove: %v", err) + } + if match { + return false + } + } + return true + }) +} + +// SetTime sets the modification time of all files in the archive to t. +func (a *Archive) SetTime(t time.Time) { + for i := range a.Files { + a.Files[i].Time = t + } +} + +func amatch(pattern, name string) (bool, error) { + // firstN returns the prefix of name corresponding to the first n path elements. + // If n <= 0, firstN returns the entire name. + firstN := func(name string, n int) string { + for i := 0; i < len(name); i++ { + if name[i] == '/' { + if n--; n == 0 { + return name[:i] + } + } + } + return name + } + + // lastN returns the suffix of name corresponding to the last n path elements. + // If n <= 0, lastN returns the entire name. + lastN := func(name string, n int) string { + for i := len(name) - 1; i >= 0; i-- { + if name[i] == '/' { + if n--; n == 0 { + return name[i+1:] + } + } + } + return name + } + + if p, ok := strings.CutPrefix(pattern, "**/"); ok { + return path.Match(p, lastN(name, 1+strings.Count(p, "/"))) + } + if p, ok := strings.CutSuffix(pattern, "/**"); ok { + return path.Match(p, firstN(name, 1+strings.Count(p, "/"))) + } + return path.Match(pattern, name) +} diff --git a/src/cmd/distpack/archive_test.go b/src/cmd/distpack/archive_test.go new file mode 100644 index 0000000000..620b970aeb --- /dev/null +++ b/src/cmd/distpack/archive_test.go @@ -0,0 +1,39 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "testing" + +var amatchTests = []struct { + pattern string + name string + ok bool +}{ + {"a", "a", true}, + {"a", "b", false}, + {"a/**", "a", true}, + {"a/**", "b", false}, + {"a/**", "a/b", true}, + {"a/**", "b/b", false}, + {"a/**", "a/b/c/d/e/f", true}, + {"a/**", "z/a/b/c/d/e/f", false}, + {"**/a", "a", true}, + {"**/a", "b", false}, + {"**/a", "x/a", true}, + {"**/a", "x/a/b", false}, + {"**/a", "x/y/z/a", true}, + {"**/a", "x/y/z/a/b", false}, + + {"go/pkg/tool/*/compile", "go/pkg/tool/darwin_amd64/compile", true}, +} + +func TestAmatch(t *testing.T) { + for _, tt := range amatchTests { + ok, err := amatch(tt.pattern, tt.name) + if ok != tt.ok || err != nil { + t.Errorf("amatch(%q, %q) = %v, %v, want %v, nil", tt.pattern, tt.name, ok, err, tt.ok) + } + } +} diff --git a/src/cmd/distpack/pack.go b/src/cmd/distpack/pack.go new file mode 100644 index 0000000000..ffeb4a1611 --- /dev/null +++ b/src/cmd/distpack/pack.go @@ -0,0 +1,376 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Distpack creates the tgz and zip files for a Go distribution. +// It writes into GOROOT/pkg/distpack: +// +// - a binary distribution (tgz or zip) for the current GOOS and GOARCH +// - a source distribution that is independent of GOOS/GOARCH +// - the module mod, info, and zip files for a distribution in module form +// (as used by GOTOOLCHAIN support in the go command). +// +// Distpack is typically invoked by the -distpack flag to make.bash. +// A cross-compiled distribution for goos/goarch can be built using: +// +// GOOS=goos GOARCH=goarch ./make.bash -distpack +// +package main + +import ( + "archive/tar" + "archive/zip" + "compress/flate" + "compress/gzip" + "crypto/sha256" + "flag" + "fmt" + "io" + "io/fs" + "log" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "time" +) + +func usage() { + fmt.Fprintf(os.Stderr, "usage: distpack\n") + os.Exit(2) +} + +const ( + modPath = "golang.org/toolchain" + modVersionPrefix = "v0.0.1" +) + +var ( + goroot string + gohostos string + gohostarch string + goos string + goarch string +) + +func main() { + log.SetPrefix("distpack: ") + log.SetFlags(0) + flag.Usage = usage + flag.Parse() + if flag.NArg() != 0 { + usage() + } + + // Load context. + goroot = runtime.GOROOT() + if goroot == "" { + log.Fatalf("missing $GOROOT") + } + gohostos = runtime.GOOS + gohostarch = runtime.GOARCH + goos = os.Getenv("GOOS") + if goos == "" { + goos = gohostos + } + goarch = os.Getenv("GOARCH") + if goarch == "" { + goarch = gohostarch + } + goosUnderGoarch := goos + "_" + goarch + goosDashGoarch := goos + "-" + goarch + exe := "" + if goos == "windows" { + exe = ".exe" + } + version, versionTime := readVERSION(goroot) + + // Start with files from GOROOT, filtering out non-distribution files. + base, err := NewArchive(goroot) + if err != nil { + log.Fatal(err) + } + base.SetTime(versionTime) + base.SetMode(mode) + base.Remove( + ".git/**", + ".gitattributes", + ".github/**", + ".gitignore", + "VERSION.cache", + "misc/cgo/*/_obj/**", + "**/.DS_Store", + "**/*.exe~", // go.dev/issue/23894 + // Generated during make.bat/make.bash. + "src/cmd/dist/dist", + "src/cmd/dist/dist.exe", + ) + + // The source distribution removes files generated during the release build. + // See ../dist/build.go's deptab. + srcArch := base.Clone() + srcArch.Remove( + "bin/**", + "pkg/**", + // Generated during cmd/dist. See ../dist/build.go:/deptab. + "src/cmd/cgo/zdefaultcc.go", + "src/cmd/go/internal/cfg/zdefaultcc.go", + "src/cmd/go/internal/cfg/zosarch.go", + "src/cmd/internal/objabi/zbootstrap.go", + "src/go/build/zcgo.go", + "src/internal/buildcfg/zbootstrap.go", + "src/runtime/internal/sys/zversion.go", + "src/time/tzdata/zzipdata.go", + ) + srcArch.AddPrefix("go") + testSrc(srcArch) + + // The binary distribution includes only a subset of bin and pkg. + binArch := base.Clone() + binArch.Filter(func(name string) bool { + // Discard bin/ for now, will add back later. + if strings.HasPrefix(name, "bin/") { + return false + } + // Discard most of pkg. + if strings.HasPrefix(name, "pkg/") { + // Keep pkg/include. + if strings.HasPrefix(name, "pkg/include/") { + return true + } + // Discard other pkg except pkg/tool. + if !strings.HasPrefix(name, "pkg/tool/") { + return false + } + // Inside pkg/tool, keep only $GOOS_$GOARCH. + if !strings.HasPrefix(name, "pkg/tool/"+goosUnderGoarch+"/") { + return false + } + // Inside pkg/tool/$GOOS_$GOARCH, discard helper tools. + switch strings.TrimSuffix(path.Base(name), ".exe") { + case "api", "dist", "distpack", "metadata": + return false + } + } + return true + }) + + // Add go and gofmt to bin, using cross-compiled binaries + // if this is a cross-compiled distribution. + binExes := []string{ + "go", + "gofmt", + } + crossBin := "bin" + if goos != gohostos || goarch != gohostarch { + crossBin = "bin/" + goosUnderGoarch + } + for _, b := range binExes { + name := "bin/" + b + exe + src := filepath.Join(goroot, crossBin, b+exe) + info, err := os.Stat(src) + if err != nil { + log.Fatal(err) + } + binArch.Add(name, src, info) + } + binArch.Sort() + binArch.SetTime(versionTime) // fix added files + binArch.SetMode(mode) // fix added files + + zipArch := binArch.Clone() + zipArch.AddPrefix("go") + testZip(zipArch) + + // The module distribution is the binary distribution with unnecessary files removed + // and file names using the necessary prefix for the module. + modArch := binArch.Clone() + modArch.Remove( + "api/**", + "doc/**", + "misc/**", + "test/**", + ) + modVers := modVersionPrefix + "-" + version + "." + goosDashGoarch + modArch.AddPrefix(modPath + "@" + modVers) + testMod(modArch) + + // distpack returns the full path to name in the distpack directory. + distpack := func(name string) string { + return filepath.Join(goroot, "pkg/distpack", name) + } + if err := os.MkdirAll(filepath.Join(goroot, "pkg/distpack"), 0777); err != nil { + log.Fatal(err) + } + + writeTgz(distpack(version+".src.tar.gz"), srcArch) + + if goos == "windows" { + writeZip(distpack(version+"."+goos+"-"+goarch+".zip"), zipArch) + } else { + writeTgz(distpack(version+"."+goos+"-"+goarch+".tar.gz"), zipArch) + } + + writeZip(distpack(modVers+".zip"), modArch) + writeFile(distpack(modVers+".mod"), + []byte(fmt.Sprintf("module %s\n", modPath))) + writeFile(distpack(modVers+".info"), + []byte(fmt.Sprintf("{%q:%q, %q:%q}\n", + "Version", modVers, + "Time", versionTime.Format(time.RFC3339)))) +} + +// mode computes the mode for the given file name. +func mode(name string, _ fs.FileMode) fs.FileMode { + if strings.HasPrefix(name, "bin/") || + strings.HasPrefix(name, "pkg/tool/") || + strings.HasSuffix(name, ".bash") || + strings.HasSuffix(name, ".sh") || + strings.HasSuffix(name, ".pl") || + strings.HasSuffix(name, ".rc") { + return 0o755 + } + return 0o644 +} + +// readVERSION reads the VERSION file. +// The first line of the file is the Go version. +// Additional lines are 'key value' pairs setting other data. +// The only valid key at the moment is 'time', which sets the modification time for file archives. +func readVERSION(goroot string) (version string, t time.Time) { + data, err := os.ReadFile(filepath.Join(goroot, "VERSION")) + if err != nil { + log.Fatal(err) + } + version, rest, _ := strings.Cut(string(data), "\n") + for _, line := range strings.Split(rest, "\n") { + f := strings.Fields(line) + if len(f) == 0 { + continue + } + switch f[0] { + default: + log.Fatalf("VERSION: unexpected line: %s", line) + case "time": + if len(f) != 2 { + log.Fatalf("VERSION: unexpected time line: %s", line) + } + t, err = time.ParseInLocation(time.RFC3339, f[1], time.UTC) + if err != nil { + log.Fatalf("VERSION: bad time: %s", err) + } + } + } + return version, t +} + +// writeFile writes a file with the given name and data or fatals. +func writeFile(name string, data []byte) { + if err := os.WriteFile(name, data, 0666); err != nil { + log.Fatal(err) + } + reportHash(name) +} + +// check panics if err is not nil. Otherwise it returns x. +// It is only meant to be used in a function that has deferred +// a function to recover appropriately from the panic. +func check[T any](x T, err error) T { + check1(err) + return x +} + +// check1 panics if err is not nil. +// It is only meant to be used in a function that has deferred +// a function to recover appropriately from the panic. +func check1(err error) { + if err != nil { + panic(err) + } +} + +// writeTgz writes the archive in tgz form to the file named name. +func writeTgz(name string, a *Archive) { + out, err := os.Create(name) + if err != nil { + log.Fatal(err) + } + + var f File + defer func() { + if err := recover(); err != nil { + extra := "" + if f.Name != "" { + extra = " " + f.Name + } + log.Fatalf("writing %s%s: %v", name, extra, err) + } + }() + + zw := check(gzip.NewWriterLevel(out, gzip.BestCompression)) + tw := tar.NewWriter(zw) + for _, f = range a.Files { + h := check(tar.FileInfoHeader(f.Info(), "")) + h.Name = f.Name + if err := tw.WriteHeader(h); err != nil { + panic(err) + } + r := check(os.Open(f.Src)) + check(io.Copy(tw, r)) + check1(r.Close()) + } + f.Name = "" + check1(tw.Close()) + check1(zw.Close()) + check1(out.Close()) + reportHash(name) +} + +// writeZip writes the archive in zip form to the file named name. +func writeZip(name string, a *Archive) { + out, err := os.Create(name) + if err != nil { + log.Fatal(err) + } + + var f File + defer func() { + if err := recover(); err != nil { + extra := "" + if f.Name != "" { + extra = " " + f.Name + } + log.Fatalf("writing %s%s: %v", name, extra, err) + } + }() + + zw := zip.NewWriter(out) + zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(out, flate.BestCompression) + }) + for _, f = range a.Files { + h := check(zip.FileInfoHeader(f.Info())) + h.Name = f.Name + h.Method = zip.Deflate + w := check(zw.CreateHeader(h)) + r := check(os.Open(f.Src)) + check(io.Copy(w, r)) + check1(r.Close()) + } + f.Name = "" + check1(zw.Close()) + check1(out.Close()) + reportHash(name) +} + +func reportHash(name string) { + f, err := os.Open(name) + if err != nil { + log.Fatal(err) + } + h := sha256.New() + io.Copy(h, f) + f.Close() + fmt.Printf("distpack: %x %s\n", h.Sum(nil)[:8], filepath.Base(name)) +} diff --git a/src/cmd/distpack/test.go b/src/cmd/distpack/test.go new file mode 100644 index 0000000000..93c6564594 --- /dev/null +++ b/src/cmd/distpack/test.go @@ -0,0 +1,166 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains tests applied to the archives before they are written. + +package main + +import ( + "bytes" + "fmt" + "log" + "os" + "path" + "path/filepath" + "strings" +) + +type testRule struct { + name string + goos string + exclude bool +} + +var srcRules = []testRule{ + {name: "go/VERSION"}, + {name: "go/src/cmd/go/main.go"}, + {name: "go/src/bytes/bytes.go"}, + {name: "go/.DS_Store", exclude: true}, + {name: "go/.git", exclude: true}, + {name: "go/.gitattributes", exclude: true}, + {name: "go/.github", exclude: true}, + {name: "go/VERSION.cache", exclude: true}, + {name: "go/bin/**", exclude: true}, + {name: "go/pkg/**", exclude: true}, + {name: "go/src/cmd/dist/dist", exclude: true}, + {name: "go/src/cmd/dist/dist.exe", exclude: true}, + {name: "go/src/runtime/internal/sys/zversion.go", exclude: true}, + {name: "go/src/time/tzdata/zzipdata.go", exclude: true}, +} + +var zipRules = []testRule{ + {name: "go/VERSION"}, + {name: "go/src/cmd/go/main.go"}, + {name: "go/src/bytes/bytes.go"}, + + {name: "go/.DS_Store", exclude: true}, + {name: "go/.git", exclude: true}, + {name: "go/.gitattributes", exclude: true}, + {name: "go/.github", exclude: true}, + {name: "go/VERSION.cache", exclude: true}, + {name: "go/bin", exclude: true}, + {name: "go/pkg", exclude: true}, + {name: "go/src/cmd/dist/dist", exclude: true}, + {name: "go/src/cmd/dist/dist.exe", exclude: true}, + + {name: "go/bin/go", goos: "linux"}, + {name: "go/bin/go", goos: "darwin"}, + {name: "go/bin/go", goos: "windows", exclude: true}, + {name: "go/bin/go.exe", goos: "windows"}, + {name: "go/bin/gofmt", goos: "linux"}, + {name: "go/bin/gofmt", goos: "darwin"}, + {name: "go/bin/gofmt", goos: "windows", exclude: true}, + {name: "go/bin/gofmt.exe", goos: "windows"}, + {name: "go/pkg/tool/*/compile", goos: "linux"}, + {name: "go/pkg/tool/*/compile", goos: "darwin"}, + {name: "go/pkg/tool/*/compile", goos: "windows", exclude: true}, + {name: "go/pkg/tool/*/compile.exe", goos: "windows"}, +} + +var modRules = []testRule{ + {name: "golang.org/toolchain@*/VERSION"}, + {name: "golang.org/toolchain@*/src/cmd/go/main.go"}, + {name: "golang.org/toolchain@*/src/bytes/bytes.go"}, + + {name: "golang.org/toolchain@*/.DS_Store", exclude: true}, + {name: "golang.org/toolchain@*/.git", exclude: true}, + {name: "golang.org/toolchain@*/.gitattributes", exclude: true}, + {name: "golang.org/toolchain@*/.github", exclude: true}, + {name: "golang.org/toolchain@*/VERSION.cache", exclude: true}, + {name: "golang.org/toolchain@*/bin", exclude: true}, + {name: "golang.org/toolchain@*/pkg", exclude: true}, + {name: "golang.org/toolchain@*/src/cmd/dist/dist", exclude: true}, + {name: "golang.org/toolchain@*/src/cmd/dist/dist.exe", exclude: true}, + + {name: "golang.org/toolchain@*/bin/go", goos: "linux"}, + {name: "golang.org/toolchain@*/bin/go", goos: "darwin"}, + {name: "golang.org/toolchain@*/bin/go", goos: "windows", exclude: true}, + {name: "golang.org/toolchain@*/bin/go.exe", goos: "windows"}, + {name: "golang.org/toolchain@*/bin/gofmt", goos: "linux"}, + {name: "golang.org/toolchain@*/bin/gofmt", goos: "darwin"}, + {name: "golang.org/toolchain@*/bin/gofmt", goos: "windows", exclude: true}, + {name: "golang.org/toolchain@*/bin/gofmt.exe", goos: "windows"}, + {name: "golang.org/toolchain@*/pkg/tool/*/compile", goos: "linux"}, + {name: "golang.org/toolchain@*/pkg/tool/*/compile", goos: "darwin"}, + {name: "golang.org/toolchain@*/pkg/tool/*/compile", goos: "windows", exclude: true}, + {name: "golang.org/toolchain@*/pkg/tool/*/compile.exe", goos: "windows"}, +} + +func testSrc(a *Archive) { + test("source", a, srcRules) + + // Check that no generated files slip in, even if new ones are added. + for _, f := range a.Files { + if strings.HasPrefix(path.Base(f.Name), "z") { + data, err := os.ReadFile(filepath.Join(goroot, strings.TrimPrefix(f.Name, "go/"))) + if err != nil { + log.Fatalf("checking source archive: %v", err) + } + if strings.Contains(string(data), "generated by go tool dist; DO NOT EDIT") { + log.Fatalf("unexpected source archive file: %s (generated by dist)", f.Name) + } + } + } +} + +func testZip(a *Archive) { test("binary", a, zipRules) } +func testMod(a *Archive) { test("module", a, modRules) } + +func test(kind string, a *Archive, rules []testRule) { + ok := true + have := make([]bool, len(rules)) + for _, f := range a.Files { + for i, r := range rules { + if r.goos != "" && r.goos != goos { + continue + } + match, err := amatch(r.name, f.Name) + if err != nil { + log.Fatal(err) + } + if match { + if r.exclude { + ok = false + if !have[i] { + log.Printf("unexpected %s archive file: %s", kind, f.Name) + have[i] = true // silence future prints for excluded directory + } + } else { + have[i] = true + } + } + } + } + missing := false + for i, r := range rules { + if r.goos != "" && r.goos != goos { + continue + } + if !r.exclude && !have[i] { + missing = true + log.Printf("missing %s archive file: %s", kind, r.name) + } + } + if missing { + ok = false + var buf bytes.Buffer + for _, f := range a.Files { + fmt.Fprintf(&buf, "\n\t%s", f.Name) + } + log.Printf("archive contents: %d files%s", len(a.Files), buf.Bytes()) + } + if !ok { + log.Fatalf("bad archive file") + } +} diff --git a/src/make.bat b/src/make.bat index 814d12c300..3b861cb91d 100644 --- a/src/make.bat +++ b/src/make.bat @@ -39,6 +39,10 @@ :: Keep environment variables within this script :: unless invoked with --no-local. +if x%1==x-no-local goto nolocal +if x%2==x-no-local goto nolocal +if x%3==x-no-local goto nolocal +if x%4==x-no-local goto nolocal if x%1==x--no-local goto nolocal if x%2==x--no-local goto nolocal if x%3==x--no-local goto nolocal @@ -113,20 +117,40 @@ call .\env.bat del env.bat if x%vflag==x-v echo. +if x%1==x-dist-tool goto copydist +if x%2==x-dist-tool goto copydist +if x%3==x-dist-tool goto copydist +if x%4==x-dist-tool goto copydist if x%1==x--dist-tool goto copydist if x%2==x--dist-tool goto copydist if x%3==x--dist-tool goto copydist if x%4==x--dist-tool goto copydist set bootstrapflags= -if x%1==x--no-clean set bootstrapflags=--no-clean -if x%2==x--no-clean set bootstrapflags=--no-clean -if x%3==x--no-clean set bootstrapflags=--no-clean -if x%4==x--no-clean set bootstrapflags=--no-clean -if x%1==x--no-banner set bootstrapflags=%bootstrapflags% --no-banner -if x%2==x--no-banner set bootstrapflags=%bootstrapflags% --no-banner -if x%3==x--no-banner set bootstrapflags=%bootstrapflags% --no-banner -if x%4==x--no-banner set bootstrapflags=%bootstrapflags% --no-banner +if x%1==x-no-clean set bootstrapflags=-no-clean +if x%2==x-no-clean set bootstrapflags=-no-clean +if x%3==x-no-clean set bootstrapflags=-no-clean +if x%4==x-no-clean set bootstrapflags=-no-clean +if x%1==x--no-clean set bootstrapflags=-no-clean +if x%2==x--no-clean set bootstrapflags=-no-clean +if x%3==x--no-clean set bootstrapflags=-no-clean +if x%4==x--no-clean set bootstrapflags=-no-clean +if x%1==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner +if x%2==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner +if x%3==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner +if x%4==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner +if x%1==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner +if x%2==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner +if x%3==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner +if x%4==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner +if x%1==x-distpack set bootstrapflags=%bootstrapflags% -distpack +if x%2==x-distpack set bootstrapflags=%bootstrapflags% -distpack +if x%3==x-distpack set bootstrapflags=%bootstrapflags% -distpack +if x%4==x-distpack set bootstrapflags=%bootstrapflags% -distpack +if x%1==x--distpack set bootstrapflags=%bootstrapflags% -distpack +if x%2==x--distpack set bootstrapflags=%bootstrapflags% -distpack +if x%3==x--distpack set bootstrapflags=%bootstrapflags% -distpack +if x%4==x--distpack set bootstrapflags=%bootstrapflags% -distpack :: Run dist bootstrap to complete make.bash. :: Bootstrap installs a proper cmd/dist, built with the new toolchain. -- 2.48.1