From: Michael Matloob Date: Thu, 28 Oct 2021 18:54:30 +0000 (-0400) Subject: cmd/go: use workspace modules' go.sum files to check sums X-Git-Tag: go1.18beta1~349 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=1cd600301ea2a0b13d5e158282200114dc9de3fd;p=gostls13.git cmd/go: use workspace modules' go.sum files to check sums By default, use workspace modules' go.sum files to check sums. Any missing sums will still be written to go.work.sum For #45713 Change-Id: I0f537602523dfec44d423c3c80c7ef396e1397b1 Reviewed-on: https://go-review.googlesource.com/c/go/+/359478 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 408b2860ad..e246c1a04d 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -384,7 +384,8 @@ func RemoveAll(dir string) error { return robustio.RemoveAll(dir) } -var GoSumFile string // path to go.sum; set by package modload +var GoSumFile string // path to go.sum; set by package modload +var WorkspaceGoSumFiles []string // path to module go.sums in workspace; set by package modload type modSum struct { mod module.Version @@ -393,10 +394,11 @@ type modSum struct { var goSum struct { mu sync.Mutex - m map[module.Version][]string // content of go.sum file - status map[modSum]modSumStatus // state of sums in m - overwrite bool // if true, overwrite go.sum without incorporating its contents - enabled bool // whether to use go.sum at all + m map[module.Version][]string // content of go.sum file + w map[string]map[module.Version][]string // sum file in workspace -> content of that sum file + status map[modSum]modSumStatus // state of sums in m + overwrite bool // if true, overwrite go.sum without incorporating its contents + enabled bool // whether to use go.sum at all } type modSumStatus struct { @@ -417,23 +419,38 @@ func initGoSum() (bool, error) { goSum.m = make(map[module.Version][]string) goSum.status = make(map[modSum]modSumStatus) + goSum.w = make(map[string]map[module.Version][]string) + + for _, f := range WorkspaceGoSumFiles { + goSum.w[f] = make(map[module.Version][]string) + _, err := readGoSumFile(goSum.w[f], f) + if err != nil { + return false, err + } + } + + enabled, err := readGoSumFile(goSum.m, GoSumFile) + goSum.enabled = enabled + return enabled, err +} + +func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) { var ( data []byte err error ) - if actualSumFile, ok := fsys.OverlayPath(GoSumFile); ok { + if actualSumFile, ok := fsys.OverlayPath(file); ok { // Don't lock go.sum if it's part of the overlay. // On Plan 9, locking requires chmod, and we don't want to modify any file // in the overlay. See #44700. data, err = os.ReadFile(actualSumFile) } else { - data, err = lockedfile.Read(GoSumFile) + data, err = lockedfile.Read(file) } if err != nil && !os.IsNotExist(err) { return false, err } - goSum.enabled = true - readGoSum(goSum.m, GoSumFile, data) + readGoSum(dst, file, data) return true, nil } @@ -485,6 +502,16 @@ func HaveSum(mod module.Version) bool { if err != nil || !inited { return false } + for _, goSums := range goSum.w { + for _, h := range goSums[mod] { + if !strings.HasPrefix(h, "h1:") { + continue + } + if !goSum.status[modSum{mod, h}].dirty { + return true + } + } + } for _, h := range goSum.m[mod] { if !strings.HasPrefix(h, "h1:") { continue @@ -602,15 +629,32 @@ func checkModSum(mod module.Version, h string) error { // If it finds a conflicting pair instead, it calls base.Fatalf. // goSum.mu must be locked. func haveModSumLocked(mod module.Version, h string) bool { + sumFileName := "go.sum" + if strings.HasSuffix(GoSumFile, "go.work.sum") { + sumFileName = "go.work.sum" + } for _, vh := range goSum.m[mod] { if h == vh { return true } if strings.HasPrefix(vh, "h1:") { - base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v"+goSumMismatch, mod.Path, mod.Version, h, vh) + base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh) + } + } + // Also check workspace sums. + foundMatch := false + // Check sums from all files in case there are conflicts between + // the files. + for goSumFile, goSums := range goSum.w { + for _, vh := range goSums[mod] { + if h == vh { + foundMatch = true + } else if strings.HasPrefix(vh, "h1:") { + base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh) + } } } - return false + return foundMatch } // addModSumLocked adds the pair mod,h to go.sum. @@ -749,7 +793,7 @@ Outer: goSum.m = make(map[module.Version][]string, len(goSum.m)) readGoSum(goSum.m, GoSumFile, data) for ms, st := range goSum.status { - if st.used { + if st.used && !sumInWorkspaceModulesLocked(ms.mod) { addModSumLocked(ms.mod, ms.sum) } } @@ -767,7 +811,7 @@ Outer: sort.Strings(list) for _, h := range list { st := goSum.status[modSum{m, h}] - if !st.dirty || (st.used && keep[m]) { + if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(m) { fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) } } @@ -784,6 +828,15 @@ Outer: return nil } +func sumInWorkspaceModulesLocked(m module.Version) bool { + for _, goSums := range goSum.w { + if _, ok := goSums[m]; ok { + return true + } + } + return false +} + // TrimGoSum trims go.sum to contain only the modules needed for reproducible // builds. // diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index a6e49c6c71..fcf6ce2620 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -624,8 +624,10 @@ func LoadModFile(ctx context.Context) *Requirements { if err != nil { base.Fatalf("reading go.work: %v", err) } - _ = TODOWorkspaces("Support falling back to individual module go.sum " + - "files for sums not in the workspace sum file.") + for _, modRoot := range modRoots { + sumFile := strings.TrimSuffix(modFilePath(modRoot), ".mod") + ".sum" + modfetch.WorkspaceGoSumFiles = append(modfetch.WorkspaceGoSumFiles, sumFile) + } modfetch.GoSumFile = workFilePath + ".sum" } else if modRoots == nil { // We're in module mode, but not inside a module. diff --git a/src/cmd/go/testdata/script/work_sum.txt b/src/cmd/go/testdata/script/work_sum.txt index 99f66a4003..20261e7cbd 100644 --- a/src/cmd/go/testdata/script/work_sum.txt +++ b/src/cmd/go/testdata/script/work_sum.txt @@ -8,8 +8,6 @@ golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:pvCbr/wm8HzDD3fVywevekuf golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= rsc.io/quote v1.5.2 h1:3fEykkD9k7lYzXqCYrwGAf7iNhbk4yCjHmKBN9td4L0= rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= -rsc.io/sampler v1.3.0 h1:HLGR/BgEtI3r0uymSP/nl2uPLsUnNJX8toRyhfpBTII= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -- go.work -- go 1.18 @@ -20,6 +18,9 @@ go 1.18 module example.com/hi require "rsc.io/quote" v1.5.2 +-- go.sum -- +rsc.io/sampler v1.3.0 h1:HLGR/BgEtI3r0uymSP/nl2uPLsUnNJX8toRyhfpBTII= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -- main.go -- package main diff --git a/src/cmd/go/testdata/script/work_sum_mismatch.txt b/src/cmd/go/testdata/script/work_sum_mismatch.txt new file mode 100644 index 0000000000..42994ea5d5 --- /dev/null +++ b/src/cmd/go/testdata/script/work_sum_mismatch.txt @@ -0,0 +1,61 @@ +# Test mismatched sums in go.sum files + +! go run ./a +cmpenv stderr want-error + +-- want-error -- +verifying rsc.io/sampler@v1.3.0/go.mod: checksum mismatch + downloaded: h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= + $WORK${/}gopath${/}src${/}a${/}go.sum: h1:U1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= + +SECURITY ERROR +This download does NOT match an earlier download recorded in go.sum. +The bits may have been replaced on the origin server, or an attacker may +have intercepted the download attempt. + +For more information, see 'go help module-auth'. +-- go.work -- +go 1.18 + +directory ./a +directory ./b +-- a/go.mod -- +go 1.18 + +module example.com/hi + +require "rsc.io/quote" v1.5.2 +-- a/go.sum -- +rsc.io/sampler v1.3.0 h1:HLGR/BgEtI3r0uymSP/nl2uPLsUnNJX8toRyhfpBTII= +rsc.io/sampler v1.3.0/go.mod h1:U1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +-- a/main.go -- +package main + +import ( + "fmt" + "rsc.io/quote" +) + +func main() { + fmt.Println(quote.Hello()) +} +-- b/go.mod -- +go 1.18 + +module example.com/hi + +require "rsc.io/quote" v1.5.2 +-- b/go.sum -- +rsc.io/sampler v1.3.0 h1:HLGR/BgEtI3r0uymSP/nl2uPLsUnNJX8toRyhfpBTII= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +-- b/main.go -- +package main + +import ( + "fmt" + "rsc.io/quote" +) + +func main() { + fmt.Println(quote.Hello()) +} \ No newline at end of file