Over time there may exist two modules with names that differ only in case.
On systems with case-insensitive file systems, we need to make sure those
modules do not collide in the download cache.
Do this by using the new "safe encoding" for file system paths as well as
proxy paths.
Fixes #25992.
Change-Id: I717a9987a87ad5c6927d063bf30d10d9229498c9
Reviewed-on: https://go-review.googlesource.com/124379
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
"fmt"
"io/ioutil"
"os"
- "path/filepath"
"cmd/go/internal/base"
"cmd/go/internal/dirhash"
func verifyMod(mod module.Version) bool {
ok := true
- zip := filepath.Join(modfetch.SrcMod, "cache/download", mod.Path, "/@v/", mod.Version+".zip")
- _, zipErr := os.Stat(zip)
- dir := filepath.Join(modfetch.SrcMod, mod.Path+"@"+mod.Version)
- _, dirErr := os.Stat(dir)
+ zip, zipErr := modfetch.CachePath(mod, "zip")
+ if zipErr == nil {
+ _, zipErr = os.Stat(zip)
+ }
+ dir, dirErr := modfetch.DownloadDir(mod)
+ if dirErr == nil {
+ _, dirErr = os.Stat(dir)
+ }
data, err := ioutil.ReadFile(zip + "hash")
if err != nil {
if zipErr != nil && os.IsNotExist(zipErr) && dirErr != nil && os.IsNotExist(dirErr) {
"cmd/go/internal/base"
"cmd/go/internal/modfetch/codehost"
+ "cmd/go/internal/module"
"cmd/go/internal/par"
"cmd/go/internal/semver"
)
var SrcMod string // $GOPATH/src/mod; set by package modload
+func cacheDir(path string) (string, error) {
+ if SrcMod == "" {
+ return "", fmt.Errorf("internal error: modfetch.SrcMod not set")
+ }
+ enc, err := module.EncodePath(path)
+ if err != nil {
+ return "", err
+ }
+ return filepath.Join(SrcMod, "cache/download", enc, "/@v"), nil
+}
+
+func CachePath(m module.Version, suffix string) (string, error) {
+ dir, err := cacheDir(m.Path)
+ if err != nil {
+ return "", err
+ }
+ if !semver.IsValid(m.Version) {
+ return "", fmt.Errorf("non-semver module version %q", m.Version)
+ }
+ if semver.Canonical(m.Version) != m.Version {
+ return "", fmt.Errorf("non-canonical module version %q", m.Version)
+ }
+ return filepath.Join(dir, m.Version+"."+suffix), nil
+}
+
+func DownloadDir(m module.Version) (string, error) {
+ if SrcMod == "" {
+ return "", fmt.Errorf("internal error: modfetch.SrcMod not set")
+ }
+ enc, err := module.EncodePath(m.Path)
+ if err != nil {
+ return "", err
+ }
+ if !semver.IsValid(m.Version) {
+ return "", fmt.Errorf("non-semver module version %q", m.Version)
+ }
+ if semver.Canonical(m.Version) != m.Version {
+ return "", fmt.Errorf("non-canonical module version %q", m.Version)
+ }
+ return filepath.Join(SrcMod, enc+"@"+m.Version), nil
+}
+
// A cachingRepo is a cache around an underlying Repo,
// avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not Zip).
// It is also safe for simultaneous use by multiple goroutines
return "", nil, errNotCached
}
rev = rev[:12]
- dir, err := os.Open(filepath.Join(SrcMod, "cache/download", path, "@v"))
+ cdir, err := cacheDir(path)
+ if err != nil {
+ return "", nil, errNotCached
+ }
+ dir, err := os.Open(cdir)
if err != nil {
return "", nil, errNotCached
}
// If the read fails, the caller can use
// writeDiskCache(file, data) to write a new cache entry.
func readDiskCache(path, rev, suffix string) (file string, data []byte, err error) {
- if !semver.IsValid(rev) || SrcMod == "" {
+ file, err = CachePath(module.Version{Path: path, Version: rev}, suffix)
+ if err != nil {
return "", nil, errNotCached
}
- file = filepath.Join(SrcMod, "cache/download", path, "@v", rev+"."+suffix)
data, err = ioutil.ReadFile(file)
if err != nil {
return file, nil, errNotCached
err error
}
c := downloadCache.Do(mod, func() interface{} {
- modpath := mod.Path + "@" + mod.Version
- dir = filepath.Join(SrcMod, modpath)
+ dir, err := DownloadDir(mod)
+ if err != nil {
+ return cached{"", err}
+ }
if files, _ := ioutil.ReadDir(dir); len(files) == 0 {
- zipfile := filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".zip")
+ zipfile, err := CachePath(mod, "zip")
+ if err != nil {
+ return cached{"", err}
+ }
if _, err := os.Stat(zipfile); err == nil {
// Use it.
// This should only happen if the mod/cache directory is preinitialized
// or if src/mod/path was removed but not src/mod/cache/download.
fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version)
} else {
- if err := os.MkdirAll(filepath.Join(SrcMod, "cache/download", mod.Path, "@v"), 0777); err != nil {
+ if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
return cached{"", err}
}
fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
return cached{"", err}
}
}
+ modpath := mod.Path + "@" + mod.Version
if err := Unzip(dir, zipfile, modpath, 0); err != nil {
fmt.Fprintf(os.Stderr, "-> %s\n", err)
return cached{"", err}
}
// Do the file I/O before acquiring the go.sum lock.
- data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".ziphash"))
+ ziphash, err := CachePath(mod, "ziphash")
+ if err != nil {
+ base.Fatalf("go: verifying %s@%s: %v", mod.Path, mod.Version, err)
+ }
+ data, err := ioutil.ReadFile(ziphash)
if err != nil {
if os.IsNotExist(err) {
// This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
return ""
}
- data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".ziphash"))
+ ziphash, err := CachePath(mod, "ziphash")
+ if err != nil {
+ return ""
+ }
+ data, err := ioutil.ReadFile(ziphash)
if err != nil {
return ""
}
"time"
"cmd/go/internal/modfetch/codehost"
+ "cmd/go/internal/module"
"cmd/go/internal/semver"
)
// Don't echo $GOPROXY back in case it has user:password in it (sigh).
return nil, fmt.Errorf("invalid $GOPROXY setting")
}
- return newProxyRepo(u.String(), path), nil
+ return newProxyRepo(u.String(), path)
}
type proxyRepo struct {
path string
}
-func newProxyRepo(baseURL, path string) Repo {
- return &proxyRepo{strings.TrimSuffix(baseURL, "/") + "/" + pathEscape(path), path}
+func newProxyRepo(baseURL, path string) (Repo, error) {
+ enc, err := module.EncodePath(path)
+ if err != nil {
+ return nil, err
+ }
+ return &proxyRepo{strings.TrimSuffix(baseURL, "/") + "/" + pathEscape(enc), path}, nil
}
func (p *proxyRepo) ModulePath() string {
if rr.VCS == "mod" {
// Fetch module from proxy with base URL rr.Repo.
- return newProxyRepo(rr.Repo, path), nil
+ return newProxyRepo(rr.Repo, path)
}
code, err := lookupCodeRepo(rr)
"cmd/go/internal/modinfo"
"cmd/go/internal/module"
"cmd/go/internal/search"
- "cmd/go/internal/semver"
"encoding/hex"
"fmt"
"os"
m.Version = q.Version
m.Time = &q.Time
}
-
- if semver.IsValid(m.Version) {
- dir := filepath.Join(modfetch.SrcMod, m.Path+"@"+m.Version)
- if stat, err := os.Stat(dir); err == nil && stat.IsDir() {
+ dir, err := modfetch.DownloadDir(module.Version{Path: m.Path, Version: m.Version})
+ if err == nil {
+ if info, err := os.Stat(dir); err == nil && info.IsDir() {
m.Dir = dir
}
}
tg.grepStderr(`go get: disabled by -getmode=vendor`, "expected disabled")
}
+func TestModPathCase(t *testing.T) {
+ tg := testGoModules(t)
+ defer tg.cleanup()
+
+ tg.run("get", "rsc.io/QUOTE")
+
+ tg.run("list", "-m", "all")
+ tg.grepStdout(`^rsc.io/quote v1.5.2`, "want lower-case quote v1.5.2")
+ tg.grepStdout(`^rsc.io/QUOTE v1.5.2`, "want upper-case quote v1.5.2")
+
+ // Note: the package is rsc.io/QUOTE/QUOTE to avoid
+ // a case-sensitive import collision error in load/pkg.go.
+ // Once the module code is checking imports within a module,
+ // that error should probably e relaxed, so that it's allowed to have
+ // both x.com/FOO/bar and x.com/foo/bar in the same program
+ // provided the module paths are x.com/FOO and x.com/foo.
+ tg.run("list", "-f=DEPS {{.Deps}}\nDIR {{.Dir}}", "rsc.io/QUOTE/QUOTE")
+ tg.grepStdout(`DEPS.*rsc.io/quote`, "want quote as dep")
+ tg.grepStdout(`DIR.*!q!u!o!t!e`, "want !q!u!o!t!e in directory name")
+}
+
func TestModBadDomain(t *testing.T) {
tg := testGoModules(t)
defer tg.cleanup()
"path/filepath"
"strings"
"sync"
+ "testing"
"cmd/go/internal/modfetch"
"cmd/go/internal/modfetch/codehost"
if i < 0 {
continue
}
- path := strings.Replace(name[:i], "_", "/", -1)
+ enc := strings.Replace(name[:i], "_", "/", -1)
+ path, err := module.DecodePath(enc)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "go proxy_test: %v", err)
+ continue
+ }
vers := name[i+1:]
modList = append(modList, module.Version{Path: path, Version: vers})
}
http.NotFound(w, r)
return
}
- path, file := path[:i], path[i+len("/@v/"):]
+ enc, file := path[:i], path[i+len("/@v/"):]
+ path, err := module.DecodePath(enc)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
+ http.NotFound(w, r)
+ return
+ }
if file == "list" {
n := 0
for _, m := range modList {
var archiveCache par.Cache
func readArchive(path, vers string) *txtar.Archive {
- prefix := strings.Replace(path, "/", "_", -1)
+ enc, err := module.EncodePath(path)
+ if err != nil {
+ return nil
+ }
+
+ prefix := strings.Replace(enc, "/", "_", -1)
name := filepath.Join(cmdGoDir, "testdata/mod", prefix+"_"+vers+".txt")
a := archiveCache.Do(name, func() interface{} {
a, err := txtar.ParseFile(name)
if err != nil {
- if !os.IsNotExist(err) {
+ if testing.Verbose() || !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "go proxy: %v\n", err)
}
a = nil
--- /dev/null
+rsc.io/quote@v2.0.0 && cp mod/rsc.io_quote_v0.0.0-20180709153244-fd906ed3b100.txt mod/rsc.io_quote_v2.0.0.txt
+
+-- .mod --
+module rsc.io/QUOTE
+
+require rsc.io/quote v1.5.2
+-- .info --
+{"Version":"v1.5.2","Name":"","Short":"","Time":"2018-07-15T16:25:34Z"}
+-- go.mod --
+module rsc.io/QUOTE
+
+require rsc.io/quote v1.5.2
+-- QUOTE/quote.go --
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// PACKAGE QUOTE COLLECTS LOUD SAYINGS.
+package QUOTE
+
+import (
+ "strings"
+
+ "rsc.io/quote"
+)
+
+// HELLO RETURNS A GREETING.
+func HELLO() string {
+ return strings.ToUpper(quote.Hello())
+}
+
+// GLASS RETURNS A USEFUL PHRASE FOR WORLD TRAVELERS.
+func GLASS() string {
+ return strings.ToUpper(quote.GLASS())
+}
+
+// GO RETURNS A GO PROVERB.
+func GO() string {
+ return strings.ToUpper(quote.GO())
+}
+
+// OPT RETURNS AN OPTIMIZATION TRUTH.
+func OPT() string {
+ return strings.ToUpper(quote.OPT())
+}
+-- QUOTE/quote_test.go --
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package QUOTE
+
+import (
+ "os"
+ "testing"
+)
+
+func init() {
+ os.Setenv("LC_ALL", "en")
+}
+
+func TestHELLO(t *testing.T) {
+ hello := "HELLO, WORLD"
+ if out := HELLO(); out != hello {
+ t.Errorf("HELLO() = %q, want %q", out, hello)
+ }
+}
+
+func TestGLASS(t *testing.T) {
+ glass := "I CAN EAT GLASS AND IT DOESN'T HURT ME."
+ if out := GLASS(); out != glass {
+ t.Errorf("GLASS() = %q, want %q", out, glass)
+ }
+}
+
+func TestGO(t *testing.T) {
+ go1 := "DON'T COMMUNICATE BY SHARING MEMORY, SHARE MEMORY BY COMMUNICATING."
+ if out := GO(); out != go1 {
+ t.Errorf("GO() = %q, want %q", out, go1)
+ }
+}
+
+func TestOPT(t *testing.T) {
+ opt := "IF A PROGRAM IS TOO SLOW, IT MUST HAVE A LOOP."
+ if out := OPT(); out != opt {
+ t.Errorf("OPT() = %q, want %q", out, opt)
+ }
+}