cmd/go: work out VCS information once per repository

We need VCS status information for each main package we load.
If two main packages are under the same VCS repository,
we can reuse that information to avoid duplicating work.

For instance, the kubernetes holds 51 main packages in its root module,
meaning that "go list ./..." repeated the same git calls 51 times.
Instead, use a global par.Cache to deduplicate that work.

Below are the numbers on kubernetes 5eb584d1cb6917,
via "benchcmd -n 8 KubernetesListPackages go list ./...":

	name                    old time/op         new time/op         delta
	KubernetesListPackages          8.91s ± 0%          3.33s ± 1%  -62.61%  (p=0.000 n=7+8)

	name                    old user-time/op    new user-time/op    delta
	KubernetesListPackages          11.2s ± 1%           8.1s ± 2%  -27.50%  (p=0.000 n=7+8)

	name                    old sys-time/op     new sys-time/op     delta
	KubernetesListPackages          8.02s ± 0%          1.67s ± 6%  -79.21%  (p=0.001 n=6+8)

	name                    old peak-RSS-bytes  new peak-RSS-bytes  delta
	KubernetesListPackages          127MB ± 2%          123MB ± 7%     ~     (p=0.328 n=8+8)

Fixes #49582.

Change-Id: Ib7ef5dc7a35c83a11e209441f5d6f3b8da068259
Reviewed-on: https://go-review.googlesource.com/c/go/+/365394
Trust: Daniel Martí <mvdan@mvdan.cc>
Trust: Dominik Honnef <dominik@honnef.co>
Run-TryBot: Bryan C. Mills <bcmills@google.com>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Go Bot <gobot@golang.org>

diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go
index 360d265de6718b49f571b91891658a95e5caf0e5..41afa42f0f1b75cc2ec990ee277ff6f021dc9414 100644 (file)
--- a/src/cmd/go/internal/load/pkg.go
+++ b/src/cmd/go/internal/load/pkg.go
@@ -2203,6 +2203,10 @@ func (p *Package) collectDeps() {
        }
 }
 
+// vcsStatusCache maps repository directories (string)
+// to their VCS information (vcsStatusError).
+var vcsStatusCache par.Cache
+
 // setBuildInfo gathers build information, formats it as a string to be
 // embedded in the binary, then sets p.Internal.BuildInfo to that string.
 // setBuildInfo should only be called on a main package with no errors.
@@ -2365,11 +2369,20 @@ func (p *Package) setBuildInfo() {
                        return
                }
 
-               st, err := vcsCmd.Status(vcsCmd, repoDir)
-               if err != nil {
+               type vcsStatusError struct {
+                       Status vcs.Status
+                       Err    error
+               }
+               cached := vcsStatusCache.Do(repoDir, func() interface{} {
+                       st, err := vcsCmd.Status(vcsCmd, repoDir)
+                       return vcsStatusError{st, err}
+               }).(vcsStatusError)
+               if err := cached.Err; err != nil {
                        setVCSError(err)
                        return
                }
+               st := cached.Status
+
                if st.Revision != "" {
                        appendSetting(vcsCmd.Cmd+"revision", st.Revision)
                }

diff --git a/src/cmd/go/testdata/script/version_buildvcs_git.txt b/src/cmd/go/testdata/script/version_buildvcs_git.txt
index 3d56c6d8b465a87310ada4fe54d8e00fb010576a..72cbe282857573b94d190c621d37f838fbf3d36e 100644 (file)
--- a/src/cmd/go/testdata/script/version_buildvcs_git.txt
+++ b/src/cmd/go/testdata/script/version_buildvcs_git.txt
@@ -16,6 +16,7 @@ rm $GOBIN/a$GOEXE
 
 # If there is a repository, but it can't be used for some reason,
 # there should be an error. It should hint about -buildvcs=false.
+# Also ensure that multiple errors are collected by "go list -e".
 cd ..
 mkdir .git
 env PATH=$WORK${/}fakebin${:}$oldpath
@@ -24,6 +25,10 @@ chmod 0755 $WORK/fakebin/git
 cd a
 ! go install
 stderr '^error obtaining VCS status: exit status 1\n\tUse -buildvcs=false to disable VCS stamping.$'
+go list -e -f '{{.ImportPath}}: {{.Error}}' ./...
+stdout -count=1 '^example\.com/a: error obtaining VCS status'
+stdout -count=1 '^example\.com/a/library: <nil>'
+stdout -count=1 '^example\.com/a/othermain: error obtaining VCS status'
 cd ..
 env PATH=$oldpath
 rm .git
@@ -99,6 +104,14 @@ go version -m $GOBIN/d$GOEXE
 exec git checkout go.mod
 rm $GOBIN/d$GOEXE
 
+# If we're loading multiple main packages,
+# but they share the same VCS repository,
+# we only need to execute VCS status commands once.
+go list -x ./...
+stdout -count=3 '^example.com'
+stderr -count=1 '^git status'
+stderr -count=1 '^git show'
+
 -- $WORK/fakebin/git --
 #!/bin/sh
 exit 1
@@ -114,6 +127,12 @@ go 1.18
 -- repo/a/a.go --
 package main
 
+func main() {}
+-- repo/a/library/f.go --
+package library
+-- repo/a/othermain/f.go --
+package main
+
 func main() {}
 -- repo/b/go.mod --
 module example.com/b