From: Russ Cox Date: Tue, 11 Jun 2019 17:53:12 +0000 (-0400) Subject: cmd/go: add GOPRIVATE environment variable X-Git-Tag: go1.13beta1~86 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=82cf8bca9cf20297bc0edf481cc530c9b3f4bf1e;p=gostls13.git cmd/go: add GOPRIVATE environment variable It is too confusing to have to set GONOSUMDB and GONOPROXY in common use cases, but one cannot be guaranteed to be a subset of the other. This CL adds GOPRIVATE, which takes the same kind of pattern list but is defined as "these patterns are private (non-public) modules". Today the implication is that GOPRIVATE is the default setting for GONOSUMDB and GONOPROXY. If there are other accommodations to make for private packages in the future or in other tools, having this clear statement of intent will let us do that. (For example maybe an IDE integration would hyperlink an import path to godoc.org; consulting GOPRIVATE would be a reasonable way to decide not to do that for certain imports. In contrast, consulting GONOPROXY or GONOSUMDB clearly would not.) Fixes #32184. Change-Id: If54c12d353c7a0a5c0e0273764140cce3c154a02 Reviewed-on: https://go-review.googlesource.com/c/go/+/181719 Run-TryBot: Russ Cox Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod TryBot-Result: Gobot Gobot --- diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 43f62e822d..9f1b4880b9 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -47,8 +47,9 @@ // importpath import path syntax // modules modules, module versions, and more // module-get module-aware go get -// packages package lists and patterns // module-auth module authentication using go.sum +// module-private module configuration for non-public modules +// packages package lists and patterns // testflag testing flags // testfunc testing functions // @@ -1568,19 +1569,16 @@ // For more details see: 'go help gopath'. // GOPROXY // URL of Go module proxy. See 'go help modules'. -// GONOPROXY +// GOPRIVATE, GONOPROXY, GONOSUMDB // Comma-separated list of glob patterns (in the syntax of Go's path.Match) -// of module path prefixes that should always be fetched directly, ignoring -// the GOPROXY setting. See 'go help modules'. +// of module path prefixes that should always be fetched directly +// or that should not be compared against the checksum database. +// See 'go help module-private'. +// GOROOT +// The root of the go tree. // GOSUMDB // The name of checksum database to use and optionally its public key and // URL. See 'go help module-auth'. -// GONOSUMDB -// Comma-separated list of glob patterns (in the syntax of Go's path.Match) -// of module path prefixes that should not be compared against the checksum -// database. See 'go help module-auth'. -// GOROOT -// The root of the go tree. // GOTMPDIR // The directory where the go command will write // temporary source files, packages, and binaries. @@ -2598,19 +2596,8 @@ // to cause a direct connection to be attempted at that point in the search. // Any proxies listed after "direct" are never consulted. // -// The GONOPROXY environment variable is a comma-separated list of -// glob patterns (in the syntax of Go's path.Match) of module path prefixes -// that should always be fetched directly, ignoring the GOPROXY setting. -// For example, -// -// GONOPROXY=*.corp.example.com,rsc.io/private -// -// forces a direct connection to download modules with path prefixes matching -// either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private", -// and "rsc.io/private/quux". -// -// The 'go env -w' command (see 'go help env') can be used to set these variables -// for future go command invocations. +// The GOPRIVATE and GONOPROXY environment variables allow bypassing +// the proxy for selected modules. See 'go help module-private' for details. // // No matter the source of the modules, the go command checks downloads against // known checksums, to detect unexpected changes in the content of any specific @@ -2642,90 +2629,6 @@ // are still ignored. // // -// Package lists and patterns -// -// Many commands apply to a set of packages: -// -// go action [packages] -// -// Usually, [packages] is a list of import paths. -// -// An import path that is a rooted path or that begins with -// a . or .. element is interpreted as a file system path and -// denotes the package in that directory. -// -// Otherwise, the import path P denotes the package found in -// the directory DIR/src/P for some DIR listed in the GOPATH -// environment variable (For more details see: 'go help gopath'). -// -// If no import paths are given, the action applies to the -// package in the current directory. -// -// There are four reserved names for paths that should not be used -// for packages to be built with the go tool: -// -// - "main" denotes the top-level package in a stand-alone executable. -// -// - "all" expands to all packages found in all the GOPATH -// trees. For example, 'go list all' lists all the packages on the local -// system. When using modules, "all" expands to all packages in -// the main module and their dependencies, including dependencies -// needed by tests of any of those. -// -// - "std" is like all but expands to just the packages in the standard -// Go library. -// -// - "cmd" expands to the Go repository's commands and their -// internal libraries. -// -// Import paths beginning with "cmd/" only match source code in -// the Go repository. -// -// An import path is a pattern if it includes one or more "..." wildcards, -// each of which can match any string, including the empty string and -// strings containing slashes. Such a pattern expands to all package -// directories found in the GOPATH trees with names matching the -// patterns. -// -// To make common patterns more convenient, there are two special cases. -// First, /... at the end of the pattern can match an empty string, -// so that net/... matches both net and packages in its subdirectories, like net/http. -// Second, any slash-separated pattern element containing a wildcard never -// participates in a match of the "vendor" element in the path of a vendored -// package, so that ./... does not match packages in subdirectories of -// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. -// Note, however, that a directory named vendor that itself contains code -// is not a vendored package: cmd/vendor would be a command named vendor, -// and the pattern cmd/... matches it. -// See golang.org/s/go15vendor for more about vendoring. -// -// An import path can also name a package to be downloaded from -// a remote repository. Run 'go help importpath' for details. -// -// Every package in a program must have a unique import path. -// By convention, this is arranged by starting each path with a -// unique prefix that belongs to you. For example, paths used -// internally at Google all begin with 'google', and paths -// denoting remote repositories begin with the path to the code, -// such as 'github.com/user/repo'. -// -// Packages in a program need not have unique package names, -// but there are two reserved package names with special meaning. -// The name main indicates a command, not a library. -// Commands are built into binaries and cannot be imported. -// The name documentation indicates documentation for -// a non-Go program in the directory. Files in package documentation -// are ignored by the go command. -// -// As a special case, if the package list is a list of .go files from a -// single directory, the command is applied to a single synthesized -// package made up of exactly those files, ignoring any build constraints -// in those files and ignoring any other files in the directory. -// -// Directory and file names that begin with "." or "_" are ignored -// by the go tool, as are directories named "testdata". -// -// // Module authentication using go.sum // // The go command tries to authenticate every downloaded module, @@ -2805,23 +2708,142 @@ // the checksum database is not consulted, and all unrecognized modules are // accepted, at the cost of giving up the security guarantee of verified repeatable // downloads for all modules. A better way to bypass the checksum database -// for specific modules is to use the GONOSUMDB environment variable. +// for specific modules is to use the GOPRIVATE or GONOSUMDB environment +// variables. See 'go help module-private' for details. +// +// The 'go env -w' command (see 'go help env') can be used to set these variables +// for future go command invocations. +// +// +// Module configuration for non-public modules // -// The GONOSUMDB environment variable is a comma-separated list of -// glob patterns (in the syntax of Go's path.Match) of module path prefixes -// that should not be compared against the checksum database. +// The go command defaults to downloading modules from the public Go module +// mirror at proxy.golang.org. It also defaults to validating downloaded modules, +// regardless of source, against the public Go checksum database at sum.golang.org. +// These defaults work well for publicly available source code. +// +// The GOPRIVATE environment variable controls which modules the go command +// considers to be private (not available publicly) and should therefore not use the +// proxy or checksum database. The variable is a comma-separated list of +// glob patterns (in the syntax of Go's path.Match) of module path prefixes. // For example, // -// GONOSUMDB=*.corp.example.com,rsc.io/private +// GOPRIVATE=*.corp.example.com,rsc.io/private +// +// causes the go command to treat as private any module with a path prefix +// matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, +// and rsc.io/private/quux. +// +// The GOPRIVATE environment variable may be used by other tools as well to +// identify non-public modules. For example, an editor could use GOPRIVATE +// to decide whether to hyperlink a package import to a godoc.org page. +// +// For fine-grained control over module download and validation, the GONOPROXY +// and GONOSUMDB environment variables accept the same kind of glob list +// and override GOPRIVATE for the specific decision of whether to use the proxy +// and checksum database, respectively. +// +// For example, if a company ran a module proxy serving private modules, +// users would configure go using: +// +// GOPRIVATE=*.corp.example.com +// GOPROXY=proxy.example.com +// GONOPROXY=none // -// disables checksum database lookups for modules with path prefixes matching -// either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private", -// and "rsc.io/private/quux". +// This would tell the go comamnd and other tools that modules beginning with +// a corp.example.com subdomain are private but that the company proxy should +// be used for downloading both public and private modules, because +// GONOPROXY has been set to a pattern that won't match any modules, +// overriding GOPRIVATE. // // The 'go env -w' command (see 'go help env') can be used to set these variables // for future go command invocations. // // +// Package lists and patterns +// +// Many commands apply to a set of packages: +// +// go action [packages] +// +// Usually, [packages] is a list of import paths. +// +// An import path that is a rooted path or that begins with +// a . or .. element is interpreted as a file system path and +// denotes the package in that directory. +// +// Otherwise, the import path P denotes the package found in +// the directory DIR/src/P for some DIR listed in the GOPATH +// environment variable (For more details see: 'go help gopath'). +// +// If no import paths are given, the action applies to the +// package in the current directory. +// +// There are four reserved names for paths that should not be used +// for packages to be built with the go tool: +// +// - "main" denotes the top-level package in a stand-alone executable. +// +// - "all" expands to all packages found in all the GOPATH +// trees. For example, 'go list all' lists all the packages on the local +// system. When using modules, "all" expands to all packages in +// the main module and their dependencies, including dependencies +// needed by tests of any of those. +// +// - "std" is like all but expands to just the packages in the standard +// Go library. +// +// - "cmd" expands to the Go repository's commands and their +// internal libraries. +// +// Import paths beginning with "cmd/" only match source code in +// the Go repository. +// +// An import path is a pattern if it includes one or more "..." wildcards, +// each of which can match any string, including the empty string and +// strings containing slashes. Such a pattern expands to all package +// directories found in the GOPATH trees with names matching the +// patterns. +// +// To make common patterns more convenient, there are two special cases. +// First, /... at the end of the pattern can match an empty string, +// so that net/... matches both net and packages in its subdirectories, like net/http. +// Second, any slash-separated pattern element containing a wildcard never +// participates in a match of the "vendor" element in the path of a vendored +// package, so that ./... does not match packages in subdirectories of +// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. +// Note, however, that a directory named vendor that itself contains code +// is not a vendored package: cmd/vendor would be a command named vendor, +// and the pattern cmd/... matches it. +// See golang.org/s/go15vendor for more about vendoring. +// +// An import path can also name a package to be downloaded from +// a remote repository. Run 'go help importpath' for details. +// +// Every package in a program must have a unique import path. +// By convention, this is arranged by starting each path with a +// unique prefix that belongs to you. For example, paths used +// internally at Google all begin with 'google', and paths +// denoting remote repositories begin with the path to the code, +// such as 'github.com/user/repo'. +// +// Packages in a program need not have unique package names, +// but there are two reserved package names with special meaning. +// The name main indicates a command, not a library. +// Commands are built into binaries and cannot be imported. +// The name documentation indicates documentation for +// a non-Go program in the directory. Files in package documentation +// are ignored by the go command. +// +// As a special case, if the package list is a list of .go files from a +// single directory, the command is applied to a single synthesized +// package made up of exactly those files, ignoring any build constraints +// in those files and ignoring any other files in the directory. +// +// Directory and file names that begin with "." or "_" are ignored +// by the go tool, as are directories named "testdata". +// +// // Testing flags // // The 'go test' command takes both flags that apply to 'go test' itself diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index c3c9c978a5..a0b51a72c3 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -265,6 +265,7 @@ var knownEnv = ` GOOS GOPATH GOPPC64 + GOPRIVATE GOPROXY GOROOT GOSUMDB @@ -291,30 +292,13 @@ var ( GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64)) GOWASM = envOr("GOWASM", fmt.Sprint(objabi.GOWASM)) - GOPROXY = goproxy() - GOSUMDB = gosumdb() - GONOPROXY = Getenv("GONOPROXY") - GONOSUMDB = Getenv("GONOSUMDB") + GOPROXY = envOr("GOPROXY", "https://proxy.golang.org,direct") + GOSUMDB = envOr("GOSUMDB", "sum.golang.org") + GOPRIVATE = Getenv("GOPRIVATE") + GONOPROXY = envOr("GONOPROXY", GOPRIVATE) + GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE) ) -func goproxy() string { - v := Getenv("GOPROXY") - if v != "" { - return v - } - - return "https://proxy.golang.org,direct" -} - -func gosumdb() string { - v := Getenv("GOSUMDB") - if v != "" { - return v - } - - return "sum.golang.org" -} - // GetArchEnv returns the name and setting of the // GOARCH-specific architecture environment variable. // If the current architecture has no GOARCH-specific variable, diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index b3d12dd681..17852deed1 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -79,6 +79,7 @@ func MkEnv() []cfg.EnvVar { {Name: "GONOSUMDB", Value: cfg.GONOSUMDB}, {Name: "GOOS", Value: cfg.Goos}, {Name: "GOPATH", Value: cfg.BuildContext.GOPATH}, + {Name: "GOPRIVATE", Value: cfg.GOPRIVATE}, {Name: "GOPROXY", Value: cfg.GOPROXY}, {Name: "GOROOT", Value: cfg.GOROOT}, {Name: "GOSUMDB", Value: cfg.GOSUMDB}, diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index 2cdd76c6cd..c2b5fb4b83 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -510,19 +510,16 @@ General-purpose environment variables: For more details see: 'go help gopath'. GOPROXY URL of Go module proxy. See 'go help modules'. - GONOPROXY + GOPRIVATE, GONOPROXY, GONOSUMDB Comma-separated list of glob patterns (in the syntax of Go's path.Match) - of module path prefixes that should always be fetched directly, ignoring - the GOPROXY setting. See 'go help modules'. + of module path prefixes that should always be fetched directly + or that should not be compared against the checksum database. + See 'go help module-private'. + GOROOT + The root of the go tree. GOSUMDB The name of checksum database to use and optionally its public key and URL. See 'go help module-auth'. - GONOSUMDB - Comma-separated list of glob patterns (in the syntax of Go's path.Match) - of module path prefixes that should not be compared against the checksum - database. See 'go help module-auth'. - GOROOT - The root of the go tree. GOTMPDIR The directory where the go command will write temporary source files, packages, and binaries. diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 94cb0d3a19..bc1d35e690 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -631,7 +631,7 @@ For more information, see 'go help module-auth'. ` -var HelpSum = &base.Command{ +var HelpModuleAuth = &base.Command{ UsageLine: "module-auth", Short: "module authentication using go.sum", Long: ` @@ -712,18 +712,56 @@ If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag, the checksum database is not consulted, and all unrecognized modules are accepted, at the cost of giving up the security guarantee of verified repeatable downloads for all modules. A better way to bypass the checksum database -for specific modules is to use the GONOSUMDB environment variable. +for specific modules is to use the GOPRIVATE or GONOSUMDB environment +variables. See 'go help module-private' for details. -The GONOSUMDB environment variable is a comma-separated list of -glob patterns (in the syntax of Go's path.Match) of module path prefixes -that should not be compared against the checksum database. +The 'go env -w' command (see 'go help env') can be used to set these variables +for future go command invocations. +`, +} + +var HelpModulePrivate = &base.Command{ + UsageLine: "module-private", + Short: "module configuration for non-public modules", + Long: ` +The go command defaults to downloading modules from the public Go module +mirror at proxy.golang.org. It also defaults to validating downloaded modules, +regardless of source, against the public Go checksum database at sum.golang.org. +These defaults work well for publicly available source code. + +The GOPRIVATE environment variable controls which modules the go command +considers to be private (not available publicly) and should therefore not use the +proxy or checksum database. The variable is a comma-separated list of +glob patterns (in the syntax of Go's path.Match) of module path prefixes. For example, - GONOSUMDB=*.corp.example.com,rsc.io/private + GOPRIVATE=*.corp.example.com,rsc.io/private + +causes the go command to treat as private any module with a path prefix +matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, +and rsc.io/private/quux. + +The GOPRIVATE environment variable may be used by other tools as well to +identify non-public modules. For example, an editor could use GOPRIVATE +to decide whether to hyperlink a package import to a godoc.org page. + +For fine-grained control over module download and validation, the GONOPROXY +and GONOSUMDB environment variables accept the same kind of glob list +and override GOPRIVATE for the specific decision of whether to use the proxy +and checksum database, respectively. + +For example, if a company ran a module proxy serving private modules, +users would configure go using: + + GOPRIVATE=*.corp.example.com + GOPROXY=proxy.example.com + GONOPROXY=none -disables checksum database lookups for modules with path prefixes matching -either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private", -and "rsc.io/private/quux". +This would tell the go comamnd and other tools that modules beginning with +a corp.example.com subdomain are private but that the company proxy should +be used for downloading both public and private modules, because +GONOPROXY has been set to a pattern that won't match any modules, +overriding GOPRIVATE. The 'go env -w' command (see 'go help env') can be used to set these variables for future go command invocations. diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index 92e977646f..95351269db 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -241,8 +241,8 @@ func lookup(proxy, path string) (r Repo, err error) { var ( errModVendor = errors.New("module lookup disabled by -mod=vendor") errProxyOff = errors.New("module lookup disabled by GOPROXY=off") - errNoproxy error = notExistError("disabled by GONOPROXY") - errUseProxy error = notExistError("path does not match GONOPROXY") + errNoproxy error = notExistError("disabled by GOPRIVATE/GONOPROXY") + errUseProxy error = notExistError("path does not match GOPRIVATE/GONOPROXY") ) func lookupDirect(path string) (Repo, error) { diff --git a/src/cmd/go/internal/modload/help.go b/src/cmd/go/internal/modload/help.go index 5630524ca2..9ce8dfb33b 100644 --- a/src/cmd/go/internal/modload/help.go +++ b/src/cmd/go/internal/modload/help.go @@ -348,19 +348,8 @@ HTTP response. The string "direct" may appear in the proxy list, to cause a direct connection to be attempted at that point in the search. Any proxies listed after "direct" are never consulted. -The GONOPROXY environment variable is a comma-separated list of -glob patterns (in the syntax of Go's path.Match) of module path prefixes -that should always be fetched directly, ignoring the GOPROXY setting. -For example, - - GONOPROXY=*.corp.example.com,rsc.io/private - -forces a direct connection to download modules with path prefixes matching -either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private", -and "rsc.io/private/quux". - -The 'go env -w' command (see 'go help env') can be used to set these variables -for future go command invocations. +The GOPRIVATE and GONOPROXY environment variables allow bypassing +the proxy for selected modules. See 'go help module-private' for details. No matter the source of the modules, the go command checks downloads against known checksums, to detect unexpected changes in the content of any specific diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 0207862d0b..73da736882 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -71,8 +71,9 @@ func init() { help.HelpImportPath, modload.HelpModules, modget.HelpModuleGet, + modfetch.HelpModuleAuth, + modfetch.HelpModulePrivate, help.HelpPackages, - modfetch.HelpSum, test.HelpTestflag, test.HelpTestfunc, } diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index c169a4ceec..87331599f6 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -111,6 +111,7 @@ func (ts *testScript) setup() { "GOOS=" + runtime.GOOS, "GOPATH=" + filepath.Join(ts.workdir, "gopath"), "GOPROXY=" + proxyURL, + "GOPRIVATE=", "GOROOT=" + testGOROOT, "GOSUMDB=" + testSumDBVerifierKey, "GONOPROXY=", diff --git a/src/cmd/go/testdata/script/mod_gonoproxy.txt b/src/cmd/go/testdata/script/mod_gonoproxy.txt index f2eb4efb6b..2bd94cdee0 100644 --- a/src/cmd/go/testdata/script/mod_gonoproxy.txt +++ b/src/cmd/go/testdata/script/mod_gonoproxy.txt @@ -1,7 +1,7 @@ env GO111MODULE=on env sumdb=$GOSUMDB env proxy=$GOPROXY -env GOPROXY GONOPROXY GOSUMDB GONOSUMDB +env GOPRIVATE GOPROXY GONOPROXY GOSUMDB GONOSUMDB env dbname=localhost.localdev/sumdb # disagree with sumdb fails @@ -13,20 +13,25 @@ stderr 'SECURITY ERROR' # but GONOSUMDB bypasses sumdb, for rsc.io/quote, rsc.io/sampler, golang.org/x/text env GONOSUMDB='*/quote,*/*mple*,golang.org/x' go get rsc.io/quote +rm go.sum +env GOPRIVATE='*/quote,*/*mple*,golang.org/x' +env GONOPROXY=none # that is, proxy all despite GOPRIVATE +go get rsc.io/quote # and GONOPROXY bypasses proxy [!net] skip [!exec:git] skip +env GOPRIVATE=none env GONOPROXY='*/fortune' ! go get rsc.io/fortune # does not exist in real world, only on test proxy stderr 'git ls-remote' env GOSUMDB= -env GONOPROXY='*/x' +env GONOPROXY= +env GOPRIVATE='*/x' go get golang.org/x/text go list -m all ! stdout 'text.*v0.0.0-2017' # should not have the version from the proxy - -- go.mod.orig -- module m