"strings"
"cmd/go/internal/base"
+ "cmd/go/internal/lockedfile"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/module"
"cmd/go/internal/par"
+ "cmd/go/internal/renameio"
"cmd/go/internal/semver"
)
return filepath.Join(PkgMod, enc+"@"+encVer), nil
}
+// lockVersion locks a file within the module cache that guards the downloading
+// and extraction of the zipfile for the given module version.
+func lockVersion(mod module.Version) (unlock func(), err error) {
+ path, err := CachePath(mod, "lock")
+ if err != nil {
+ return nil, err
+ }
+ if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+ return nil, err
+ }
+ return lockedfile.MutexAt(path).Lock()
+}
+
+// SideLock locks a file within the module cache that that guards edits to files
+// outside the cache, such as go.sum and go.mod files in the user's working
+// directory. It returns a function that must be called to unlock the file.
+func SideLock() (unlock func()) {
+ if PkgMod == "" {
+ base.Fatalf("go: internal error: modfetch.PkgMod not set")
+ }
+ path := filepath.Join(PkgMod, "cache", "lock")
+ if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+ base.Fatalf("go: failed to create cache directory %s: %v", filepath.Dir(path), err)
+ }
+ unlock, err := lockedfile.MutexAt(path).Lock()
+ if err != nil {
+ base.Fatalf("go: failed to lock file at %v", path)
+ }
+ return unlock
+}
+
// A cachingRepo is a cache around an underlying Repo,
// avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not Zip).
// It is also safe for simultaneous use by multiple goroutines
// and should ignore it.
var oldVgoPrefix = []byte("//vgo 0.0.")
-// readDiskGoMod reads a cached stat result from disk,
+// readDiskGoMod reads a cached go.mod file from disk,
// returning the name of the cache file and the result.
// If the read fails, the caller can use
// writeDiskGoMod(file, data) to write a new cache entry.
if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil {
return err
}
- // Write data to temp file next to target file.
- f, err := ioutil.TempFile(filepath.Dir(file), filepath.Base(file)+".tmp-")
- if err != nil {
- return err
- }
- defer os.Remove(f.Name())
- defer f.Close()
- if _, err := f.Write(data); err != nil {
- return err
- }
- if err := f.Close(); err != nil {
- return err
- }
- // Rename temp file onto cache file,
- // so that the cache file is always a complete file.
- if err := os.Rename(f.Name(), file); err != nil {
+
+ if err := renameio.WriteFile(file, data); err != nil {
return err
}
base.Fatalf("go: internal error: misuse of rewriteVersionList")
}
- // TODO(rsc): We should do some kind of directory locking here,
- // to avoid lost updates.
+ listFile := filepath.Join(dir, "list")
+
+ // We use a separate lockfile here instead of locking listFile itself because
+ // we want to use Rename to write the file atomically. The list may be read by
+ // a GOPROXY HTTP server, and if we crash midway through a rewrite (or if the
+ // HTTP server ignores our locking and serves the file midway through a
+ // rewrite) it's better to serve a stale list than a truncated one.
+ unlock, err := lockedfile.MutexAt(listFile + ".lock").Lock()
+ if err != nil {
+ base.Fatalf("go: can't lock version list lockfile: %v", err)
+ }
+ defer unlock()
infos, err := ioutil.ReadDir(dir)
if err != nil {
buf.WriteString(v)
buf.WriteString("\n")
}
- listFile := filepath.Join(dir, "list")
old, _ := ioutil.ReadFile(listFile)
if bytes.Equal(buf.Bytes(), old) {
return
}
- // TODO: Use rename to install file,
- // so that readers never see an incomplete file.
- ioutil.WriteFile(listFile, buf.Bytes(), 0666)
+
+ if err := renameio.WriteFile(listFile, buf.Bytes()); err != nil {
+ base.Fatalf("go: failed to write version list: %v", err)
+ }
}
"cmd/go/internal/dirhash"
"cmd/go/internal/module"
"cmd/go/internal/par"
+ "cmd/go/internal/renameio"
)
var downloadCache par.Cache
return "", fmt.Errorf("missing modfetch.PkgMod")
}
- // The par.Cache here avoids duplicate work but also
- // avoids conflicts from simultaneous calls by multiple goroutines
- // for the same version.
+ // The par.Cache here avoids duplicate work.
type cached struct {
dir string
err error
if err != nil {
return cached{"", err}
}
- if files, _ := ioutil.ReadDir(dir); len(files) == 0 {
- zipfile, err := DownloadZip(mod)
- if err != nil {
- return cached{"", err}
- }
- modpath := mod.Path + "@" + mod.Version
- if err := Unzip(dir, zipfile, modpath, 0); err != nil {
- fmt.Fprintf(os.Stderr, "-> %s\n", err)
- return cached{"", err}
- }
+ if err := download(mod, dir); err != nil {
+ return cached{"", err}
}
checkSum(mod)
return cached{dir, nil}
return c.dir, c.err
}
+func download(mod module.Version, dir string) (err error) {
+ // If the directory exists, the module has already been extracted.
+ fi, err := os.Stat(dir)
+ if err == nil && fi.IsDir() {
+ return nil
+ }
+
+ // To avoid cluttering the cache with extraneous files,
+ // DownloadZip uses the same lockfile as Download.
+ // Invoke DownloadZip before locking the file.
+ zipfile, err := DownloadZip(mod)
+ if err != nil {
+ return err
+ }
+
+ if cfg.CmdName != "mod download" {
+ fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version)
+ }
+
+ unlock, err := lockVersion(mod)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
+ // Check whether the directory was populated while we were waiting on the lock.
+ fi, err = os.Stat(dir)
+ if err == nil && fi.IsDir() {
+ return nil
+ }
+
+ // Clean up any remaining temporary directories from previous runs.
+ // This is only safe to do because the lock file ensures that their writers
+ // are no longer active.
+ parentDir := filepath.Dir(dir)
+ tmpPrefix := filepath.Base(dir) + ".tmp-"
+ if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil {
+ for _, path := range old {
+ RemoveAll(path) // best effort
+ }
+ }
+
+ // Extract the zip file to a temporary directory, then rename it to the
+ // final path. That way, we can use the existence of the source directory to
+ // signal that it has been extracted successfully, and if someone deletes
+ // the entire directory (e.g. as an attempt to prune out file corruption)
+ // the module cache will still be left in a recoverable state.
+ if err := os.MkdirAll(parentDir, 0777); err != nil {
+ return err
+ }
+ tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if err != nil {
+ RemoveAll(tmpDir)
+ }
+ }()
+
+ modpath := mod.Path + "@" + mod.Version
+ if err := Unzip(tmpDir, zipfile, modpath, 0); err != nil {
+ fmt.Fprintf(os.Stderr, "-> %s\n", err)
+ return err
+ }
+
+ return os.Rename(tmpDir, dir)
+}
+
var downloadZipCache par.Cache
// DownloadZip downloads the specific module version to the
// local zip cache and returns the name of the zip file.
func DownloadZip(mod module.Version) (zipfile string, err error) {
- // The par.Cache here avoids duplicate work but also
- // avoids conflicts from simultaneous calls by multiple goroutines
- // for the same version.
+ // The par.Cache here avoids duplicate work.
type cached struct {
zipfile string
err error
if err != nil {
return cached{"", err}
}
+
+ // Skip locking if the zipfile already exists.
if _, err := os.Stat(zipfile); err == nil {
- // Use it.
- // This should only happen if the mod/cache directory is preinitialized
- // or if pkg/mod/path was removed but not pkg/mod/cache/download.
- if cfg.CmdName != "mod download" {
- fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version)
- }
- } else {
- if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
- return cached{"", err}
- }
- if cfg.CmdName != "mod download" {
- fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
- }
- if err := downloadZip(mod, zipfile); err != nil {
- return cached{"", err}
- }
+ return cached{zipfile, nil}
+ }
+
+ // The zip file does not exist. Acquire the lock and create it.
+ if cfg.CmdName != "mod download" {
+ fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
+ }
+ unlock, err := lockVersion(mod)
+ if err != nil {
+ return cached{"", err}
+ }
+ defer unlock()
+
+ // Double-check that the zipfile was not created while we were waiting for
+ // the lock.
+ if _, err := os.Stat(zipfile); err == nil {
+ return cached{zipfile, nil}
+ }
+ if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
+ return cached{"", err}
+ }
+ if err := downloadZip(mod, zipfile); err != nil {
+ return cached{"", err}
}
return cached{zipfile, nil}
}).(cached)
return c.zipfile, c.err
}
-func downloadZip(mod module.Version, target string) error {
- repo, err := Lookup(mod.Path)
- if err != nil {
- return err
+func downloadZip(mod module.Version, zipfile string) (err error) {
+ // Clean up any remaining tempfiles from previous runs.
+ // This is only safe to do because the lock file ensures that their
+ // writers are no longer active.
+ for _, base := range []string{zipfile, zipfile + "hash"} {
+ if old, err := filepath.Glob(renameio.Pattern(base)); err == nil {
+ for _, path := range old {
+ os.Remove(path) // best effort
+ }
+ }
}
- tmpfile, err := ioutil.TempFile("", "go-codezip-")
+
+ // From here to the os.Rename call below is functionally almost equivalent to
+ // renameio.WriteToFile, with one key difference: we want to validate the
+ // contents of the file (by hashing it) before we commit it. Because the file
+ // is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
+ // validate it: we can't just tee the stream as we write it.
+ f, err := ioutil.TempFile(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile)))
if err != nil {
return err
}
defer func() {
- tmpfile.Close()
- os.Remove(tmpfile.Name())
+ if err != nil {
+ f.Close()
+ os.Remove(f.Name())
+ }
}()
- if err := repo.Zip(tmpfile, mod.Version); err != nil {
+
+ repo, err := Lookup(mod.Path)
+ if err != nil {
+ return err
+ }
+ if err := repo.Zip(f, mod.Version); err != nil {
return err
}
- // Double-check zip file looks OK.
- fi, err := tmpfile.Stat()
+ // Double-check that the paths within the zip file are well-formed.
+ //
+ // TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
+ fi, err := f.Stat()
if err != nil {
return err
}
- z, err := zip.NewReader(tmpfile, fi.Size())
+ z, err := zip.NewReader(f, fi.Size())
if err != nil {
return err
}
}
}
- hash, err := dirhash.HashZip(tmpfile.Name(), dirhash.DefaultHash)
- if err != nil {
+ // Sync the file before renaming it: otherwise, after a crash the reader may
+ // observe a 0-length file instead of the actual contents.
+ // See https://golang.org/issue/22397#issuecomment-380831736.
+ if err := f.Sync(); err != nil {
return err
}
- checkOneSum(mod, hash) // check before installing the zip file
- if _, err := tmpfile.Seek(0, io.SeekStart); err != nil {
+ if err := f.Close(); err != nil {
return err
}
- w, err := os.Create(target)
+
+ // Hash the zip file and check the sum before renaming to the final location.
+ hash, err := dirhash.HashZip(f.Name(), dirhash.DefaultHash)
if err != nil {
return err
}
- if _, err := io.Copy(w, tmpfile); err != nil {
- w.Close()
- return fmt.Errorf("copying: %v", err)
+ checkOneSum(mod, hash)
+
+ if err := renameio.WriteFile(zipfile+"hash", []byte(hash)); err != nil {
+ return err
}
- if err := w.Close(); err != nil {
+ if err := os.Rename(f.Name(), zipfile); err != nil {
return err
}
- return ioutil.WriteFile(target+"hash", []byte(hash), 0666)
+
+ // TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
+
+ return nil
}
var GoSumFile string // path to go.sum; set by package modload
+type modSum struct {
+ mod module.Version
+ sum string
+}
+
var goSum struct {
mu sync.Mutex
m map[module.Version][]string // content of go.sum file (+ go.modverify if present)
+ checked map[modSum]bool // sums actually checked during execution
+ dirty bool // whether we added any new sums to m
+ overwrite bool // if true, overwrite go.sum without incorporating its contents
enabled bool // whether to use go.sum at all
modverify string // path to go.modverify, to be deleted
}
}
goSum.m = make(map[module.Version][]string)
+ goSum.checked = make(map[modSum]bool)
data, err := ioutil.ReadFile(GoSumFile)
if err != nil && !os.IsNotExist(err) {
base.Fatalf("go: %v", err)
}
goSum.enabled = true
- readGoSum(GoSumFile, data)
+ readGoSum(goSum.m, GoSumFile, data)
// Add old go.modverify file.
// We'll delete go.modverify in WriteGoSum.
alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify"
if data, err := ioutil.ReadFile(alt); err == nil {
- readGoSum(alt, data)
+ migrate := make(map[module.Version][]string)
+ readGoSum(migrate, alt, data)
+ for mod, sums := range migrate {
+ for _, sum := range sums {
+ checkOneSumLocked(mod, sum)
+ }
+ }
goSum.modverify = alt
}
return true
// readGoSum parses data, which is the content of file,
// and adds it to goSum.m. The goSum lock must be held.
-func readGoSum(file string, data []byte) {
+func readGoSum(dst map[module.Version][]string, file string, data []byte) {
lineno := 0
for len(data) > 0 {
var line []byte
continue
}
mod := module.Version{Path: f[0], Version: f[1]}
- goSum.m[mod] = append(goSum.m[mod], f[2])
+ dst[mod] = append(dst[mod], f[2])
}
}
// Do the file I/O before acquiring the go.sum lock.
ziphash, err := CachePath(mod, "ziphash")
if err != nil {
- base.Fatalf("go: verifying %s@%s: %v", mod.Path, mod.Version, err)
+ base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err)
}
data, err := ioutil.ReadFile(ziphash)
if err != nil {
// This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
return
}
- base.Fatalf("go: verifying %s@%s: %v", mod.Path, mod.Version, err)
+ base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err)
}
h := strings.TrimSpace(string(data))
if !strings.HasPrefix(h, "h1:") {
- base.Fatalf("go: verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h)
+ base.Fatalf("verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h)
}
checkOneSum(mod, h)
func checkGoMod(path, version string, data []byte) {
h, err := goModSum(data)
if err != nil {
- base.Fatalf("go: verifying %s %s go.mod: %v", path, version, err)
+ base.Fatalf("verifying %s %s go.mod: %v", path, version, err)
}
checkOneSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
func checkOneSum(mod module.Version, h string) {
goSum.mu.Lock()
defer goSum.mu.Unlock()
- if !initGoSum() {
- return
+ if initGoSum() {
+ checkOneSumLocked(mod, h)
}
+}
+
+func checkOneSumLocked(mod module.Version, h string) {
+ goSum.checked[modSum{mod, h}] = true
for _, vh := range goSum.m[mod] {
if h == vh {
return
}
if strings.HasPrefix(vh, "h1:") {
- base.Fatalf("go: verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v", mod.Path, mod.Version, h, vh)
+ base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v", mod.Path, mod.Version, h, vh)
}
}
if len(goSum.m[mod]) > 0 {
fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v", mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
}
goSum.m[mod] = append(goSum.m[mod], h)
+ goSum.dirty = true
}
// Sum returns the checksum for the downloaded copy of the given module,
func WriteGoSum() {
goSum.mu.Lock()
defer goSum.mu.Unlock()
- if !initGoSum() {
+
+ if !goSum.enabled {
+ // If we haven't read the go.sum file yet, don't bother writing it: at best,
+ // we could rename the go.modverify file if it isn't empty, but we haven't
+ // needed to touch it so far — how important could it be?
+ return
+ }
+ if !goSum.dirty {
+ // Don't bother opening the go.sum file if we don't have anything to add.
return
}
+ // We want to avoid races between creating the lockfile and deleting it, but
+ // we also don't want to leave a permanent lockfile in the user's repository.
+ //
+ // On top of that, if we crash while writing go.sum, we don't want to lose the
+ // sums that were already present in the file, so it's important that we write
+ // the file by renaming rather than truncating — which means that we can't
+ // lock the go.sum file itself.
+ //
+ // Instead, we'll lock a distinguished file in the cache directory: that will
+ // only race if the user runs `go clean -modcache` concurrently with a command
+ // that updates go.sum, and that's already racy to begin with.
+ //
+ // We'll end up slightly over-synchronizing go.sum writes if the user runs a
+ // bunch of go commands that update sums in separate modules simultaneously,
+ // but that's unlikely to matter in practice.
+
+ unlock := SideLock()
+ defer unlock()
+
+ if !goSum.overwrite {
+ // Re-read the go.sum file to incorporate any sums added by other processes
+ // in the meantime.
+ data, err := ioutil.ReadFile(GoSumFile)
+ if err != nil && !os.IsNotExist(err) {
+ base.Fatalf("go: re-reading go.sum: %v", err)
+ }
+
+ // Add only the sums that we actually checked: the user may have edited or
+ // truncated the file to remove erroneous hashes, and we shouldn't restore
+ // them without good reason.
+ goSum.m = make(map[module.Version][]string, len(goSum.m))
+ readGoSum(goSum.m, GoSumFile, data)
+ for ms := range goSum.checked {
+ checkOneSumLocked(ms.mod, ms.sum)
+ }
+ }
+
var mods []module.Version
for m := range goSum.m {
mods = append(mods, m)
}
}
- data, _ := ioutil.ReadFile(GoSumFile)
- if !bytes.Equal(data, buf.Bytes()) {
- if err := ioutil.WriteFile(GoSumFile, buf.Bytes(), 0666); err != nil {
- base.Fatalf("go: writing go.sum: %v", err)
- }
+ if err := renameio.WriteFile(GoSumFile, buf.Bytes()); err != nil {
+ base.Fatalf("go: writing go.sum: %v", err)
}
+ goSum.checked = make(map[modSum]bool)
+ goSum.dirty = false
+ goSum.overwrite = false
+
if goSum.modverify != "" {
- os.Remove(goSum.modverify)
+ os.Remove(goSum.modverify) // best effort
}
}
noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")}
if !keep[m] && !keep[noGoMod] {
delete(goSum.m, m)
+ goSum.dirty = true
+ goSum.overwrite = true
}
}
}