From 7020759859a3b7c2f033e497eca750b15d3aa38d Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 24 Aug 2023 13:29:39 -0400 Subject: [PATCH] cmd/go: implement "go build -json" This adds support for a "-json" flag in all build-related go subcommands. This causes build output and build failures to be reported to stdout in a machine-readable way. For #62067. Fixes #23037. Change-Id: Id045c5bd5dde9d16cc09dde6248a4b9637896a30 Reviewed-on: https://go-review.googlesource.com/c/go/+/536397 LUCI-TryBot-Result: Go LUCI Reviewed-by: Russ Cox --- src/cmd/go/alldocs.go | 37 ++++++++++++++++ src/cmd/go/internal/cfg/cfg.go | 53 +++++++++++----------- src/cmd/go/internal/clean/clean.go | 2 +- src/cmd/go/internal/fix/fix.go | 2 +- src/cmd/go/internal/generate/generate.go | 2 +- src/cmd/go/internal/help/helpdoc.go | 37 ++++++++++++++++ src/cmd/go/internal/list/list.go | 3 +- src/cmd/go/internal/load/printer.go | 54 ++++++++++++++++++++++- src/cmd/go/internal/test/testflag.go | 3 +- src/cmd/go/internal/vet/vetflag.go | 5 ++- src/cmd/go/internal/work/action.go | 9 ++-- src/cmd/go/internal/work/build.go | 12 +++++ src/cmd/go/main.go | 1 + src/cmd/go/testdata/script/build_json.txt | 48 ++++++++++++++++++++ 14 files changed, 233 insertions(+), 35 deletions(-) create mode 100644 src/cmd/go/testdata/script/build_json.txt diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 7621bb86af..c81fad7738 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -38,6 +38,7 @@ // Additional help topics: // // buildconstraint build constraints +// buildjson build -json encoding // buildmode build modes // c calling between Go and C // cache build and test caching @@ -184,6 +185,9 @@ // or, if set explicitly, has _race appended to it. Likewise for the -msan // and -asan flags. Using a -buildmode option that requires non-default compile // flags has a similar effect. +// -json +// Emit build output in JSON suitable for automated processing. +// See 'go help buildjson' for the encoding details. // -ldflags '[pattern=]arg list' // arguments to pass on each go tool link invocation. // -linkshared @@ -2139,6 +2143,39 @@ // has a term for a Go major release, the language version used when compiling // the file will be the minimum version implied by the build constraint. // +// # Build -json encoding +// +// The 'go build' and 'go install' commands take a -json flag that reports +// build output and failures as structured JSON output on standard output. +// +// The JSON stream is a newline-separated sequence of BuildEvent objects +// corresponding to the Go struct: +// +// type BuildEvent struct { +// ImportPath string +// Action string +// Output string +// } +// +// The ImportPath field gives the package ID of the package being built. +// This matches the Package.ImportPath field of go list -json. +// +// The Action field is one of the following: +// +// build-output - The toolchain printed output +// build-fail - The build failed +// +// The Output field is set for Action == "build-output" and is a portion of +// the build's output. The concatenation of the Output fields of all output +// events is the exact output of the build. A single event may contain one +// or more lines of output and there may be more than one output event for +// a given ImportPath. This matches the definition of the TestEvent.Output +// field produced by go test -json. +// +// Note that there may also be non-JSON error text on stdnard error, even +// with the -json flag. Typically, this indicates an early, serious error. +// Consumers should be robust to this. +// // # Build modes // // The 'go build' and 'go install' commands take a -buildmode argument which diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index a81219d396..11b3893810 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -65,31 +65,34 @@ func ToolExeSuffix() string { // These are general "build flags" used by build and other commands. var ( - BuildA bool // -a flag - BuildBuildmode string // -buildmode flag - BuildBuildvcs = "auto" // -buildvcs flag: "true", "false", or "auto" - BuildContext = defaultContext() - BuildMod string // -mod flag - BuildModExplicit bool // whether -mod was set explicitly - BuildModReason string // reason -mod was set, if set by default - BuildLinkshared bool // -linkshared flag - BuildMSan bool // -msan flag - BuildASan bool // -asan flag - BuildCover bool // -cover flag - BuildCoverMode string // -covermode flag - BuildCoverPkg []string // -coverpkg flag - BuildN bool // -n flag - BuildO string // -o flag - BuildP = runtime.GOMAXPROCS(0) // -p flag - BuildPGO string // -pgo flag - BuildPkgdir string // -pkgdir flag - BuildRace bool // -race flag - BuildToolexec []string // -toolexec flag - BuildToolchainName string - BuildTrimpath bool // -trimpath flag - BuildV bool // -v flag - BuildWork bool // -work flag - BuildX bool // -x flag + BuildA bool // -a flag + BuildBuildmode string // -buildmode flag + BuildBuildvcs = "auto" // -buildvcs flag: "true", "false", or "auto" + BuildContext = defaultContext() + BuildMod string // -mod flag + BuildModExplicit bool // whether -mod was set explicitly + BuildModReason string // reason -mod was set, if set by default + BuildLinkshared bool // -linkshared flag + BuildMSan bool // -msan flag + BuildASan bool // -asan flag + BuildCover bool // -cover flag + BuildCoverMode string // -covermode flag + BuildCoverPkg []string // -coverpkg flag + BuildJSON bool // -json flag + BuildN bool // -n flag + BuildO string // -o flag + BuildP = runtime.GOMAXPROCS(0) // -p flag + BuildPGO string // -pgo flag + BuildPkgdir string // -pkgdir flag + BuildRace bool // -race flag + BuildToolexec []string // -toolexec flag + BuildToolchainName string + BuildToolchainCompiler func() string + BuildToolchainLinker func() string + BuildTrimpath bool // -trimpath flag + BuildV bool // -v flag + BuildWork bool // -work flag + BuildX bool // -x flag ModCacheRW bool // -modcacherw flag ModFile string // -modfile flag diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 291ac8e5e9..37566025ce 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -114,7 +114,7 @@ func init() { // mentioned explicitly in the docs but they // are part of the build flags. - work.AddBuildFlags(CmdClean, work.DefaultBuildFlags) + work.AddBuildFlags(CmdClean, work.OmitBuildOnlyFlags) } func runClean(ctx context.Context, cmd *base.Command, args []string) { diff --git a/src/cmd/go/internal/fix/fix.go b/src/cmd/go/internal/fix/fix.go index 3705b30ef9..28ad58daf5 100644 --- a/src/cmd/go/internal/fix/fix.go +++ b/src/cmd/go/internal/fix/fix.go @@ -40,7 +40,7 @@ See also: go fmt, go vet. var fixes = CmdFix.Flag.String("fix", "", "comma-separated list of fixes to apply") func init() { - work.AddBuildFlags(CmdFix, work.DefaultBuildFlags) + work.AddBuildFlags(CmdFix, work.OmitBuildOnlyFlags) CmdFix.Run = runFix // fix cycle } diff --git a/src/cmd/go/internal/generate/generate.go b/src/cmd/go/internal/generate/generate.go index 3a3b95786a..0f4b4a972e 100644 --- a/src/cmd/go/internal/generate/generate.go +++ b/src/cmd/go/internal/generate/generate.go @@ -176,7 +176,7 @@ var ( ) func init() { - work.AddBuildFlags(CmdGenerate, work.DefaultBuildFlags) + work.AddBuildFlags(CmdGenerate, work.OmitBuildOnlyFlags) CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "") CmdGenerate.Flag.StringVar(&generateSkipFlag, "skip", "", "") } diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index 2bf3680c35..e4324cefe7 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -1044,3 +1044,40 @@ If the server responds with an error again, the fetch fails: a URL-specific GOAUTH will only be attempted once per fetch. `, } + +var HelpBuildJSON = &base.Command{ + UsageLine: "buildjson", + Short: "build -json encoding", + Long: ` +The 'go build' and 'go install' commands take a -json flag that reports +build output and failures as structured JSON output on standard output. + +The JSON stream is a newline-separated sequence of BuildEvent objects +corresponding to the Go struct: + + type BuildEvent struct { + ImportPath string + Action string + Output string + } + +The ImportPath field gives the package ID of the package being built. +This matches the Package.ImportPath field of go list -json. + +The Action field is one of the following: + + build-output - The toolchain printed output + build-fail - The build failed + +The Output field is set for Action == "build-output" and is a portion of +the build's output. The concatenation of the Output fields of all output +events is the exact output of the build. A single event may contain one +or more lines of output and there may be more than one output event for +a given ImportPath. This matches the definition of the TestEvent.Output +field produced by go test -json. + +Note that there may also be non-JSON error text on stdnard error, even +with the -json flag. Typically, this indicates an early, serious error. +Consumers should be robust to this. + `, +} diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index ffcf531fec..ee5f28fd7d 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -345,7 +345,8 @@ For more about modules, see https://golang.org/ref/mod. func init() { CmdList.Run = runList // break init cycle - work.AddBuildFlags(CmdList, work.DefaultBuildFlags) + // Omit build -json because list has its own -json + work.AddBuildFlags(CmdList, work.OmitJSONFlag) if cfg.Experiment != nil && cfg.Experiment.CoverageRedesign { work.AddCoverFlags(CmdList, nil) } diff --git a/src/cmd/go/internal/load/printer.go b/src/cmd/go/internal/load/printer.go index 7eee2b06c2..81954c0f34 100644 --- a/src/cmd/go/internal/load/printer.go +++ b/src/cmd/go/internal/load/printer.go @@ -6,6 +6,8 @@ package load import ( "cmd/go/internal/base" + "cmd/go/internal/cfg" + "encoding/json" "fmt" "io" "os" @@ -45,7 +47,9 @@ func DefaultPrinter() Printer { } var defaultPrinter = sync.OnceValue(func() Printer { - // TODO: This will return a JSON printer once that's an option. + if cfg.BuildJSON { + return NewJSONPrinter(os.Stdout) + } return &TextPrinter{os.Stderr} }) @@ -72,3 +76,51 @@ func (p *TextPrinter) Errorf(_ *Package, format string, args ...any) { fmt.Fprint(p.Writer, ensureNewline(fmt.Sprintf(format, args...))) base.SetExitStatus(1) } + +// A JSONPrinter emits output about a build in JSON format. +type JSONPrinter struct { + enc *json.Encoder +} + +func NewJSONPrinter(w io.Writer) *JSONPrinter { + return &JSONPrinter{json.NewEncoder(w)} +} + +type jsonBuildEvent struct { + ImportPath string + Action string + Output string `json:",omitempty"` // Non-empty if Action == “build-output” +} + +func (p *JSONPrinter) Output(pkg *Package, args ...any) { + ev := &jsonBuildEvent{ + Action: "build-output", + Output: fmt.Sprint(args...), + } + if ev.Output == "" { + // There's no point in emitting a completely empty output event. + return + } + if pkg != nil { + ev.ImportPath = pkg.Desc() + } + p.enc.Encode(ev) +} + +func (p *JSONPrinter) Errorf(pkg *Package, format string, args ...any) { + s := ensureNewline(fmt.Sprintf(format, args...)) + // For clarity, emit each line as a separate output event. + for len(s) > 0 { + i := strings.IndexByte(s, '\n') + p.Output(pkg, s[:i+1]) + s = s[i+1:] + } + ev := &jsonBuildEvent{ + Action: "build-fail", + } + if pkg != nil { + ev.ImportPath = pkg.Desc() + } + p.enc.Encode(ev) + base.SetExitStatus(1) +} diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index 4686e550fd..22fc2b4c16 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -26,13 +26,14 @@ import ( // some are for both. func init() { - work.AddBuildFlags(CmdTest, work.OmitVFlag) + work.AddBuildFlags(CmdTest, work.OmitVFlag|work.OmitJSONFlag) cf := CmdTest.Flag cf.BoolVar(&testC, "c", false, "") cf.StringVar(&testO, "o", "", "") work.AddCoverFlags(CmdTest, &testCoverProfile) cf.Var((*base.StringsFlag)(&work.ExecCmd), "exec", "") + // TODO(austin): Make test -json imply build -json. cf.BoolVar(&testJSON, "json", false, "") cf.Var(&testVet, "vet", "") diff --git a/src/cmd/go/internal/vet/vetflag.go b/src/cmd/go/internal/vet/vetflag.go index eb7af6508d..d0bdb58a50 100644 --- a/src/cmd/go/internal/vet/vetflag.go +++ b/src/cmd/go/internal/vet/vetflag.go @@ -38,7 +38,10 @@ import ( var vetTool string // -vettool func init() { - work.AddBuildFlags(CmdVet, work.DefaultBuildFlags) + // For now, we omit the -json flag for vet because we could plausibly + // support -json specific to the vet command in the future (perhaps using + // the same format as build -json). + work.AddBuildFlags(CmdVet, work.OmitJSONFlag) CmdVet.Flag.StringVar(&vetTool, "vettool", "", "") } diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go index ec384b6d9b..7567284d70 100644 --- a/src/cmd/go/internal/work/action.go +++ b/src/cmd/go/internal/work/action.go @@ -269,6 +269,7 @@ func NewBuilder(workDir string) *Builder { b.toolIDCache = make(map[string]string) b.buildIDCache = make(map[string]string) + printWorkDir := false if workDir != "" { b.WorkDir = workDir } else if cfg.BuildN { @@ -291,13 +292,15 @@ func NewBuilder(workDir string) *Builder { } b.WorkDir = tmp builderWorkDirs.Store(b, b.WorkDir) - if cfg.BuildX || cfg.BuildWork { - fmt.Fprintf(os.Stderr, "WORK=%s\n", b.WorkDir) - } + printWorkDir = cfg.BuildX || cfg.BuildWork } b.backgroundSh = NewShell(b.WorkDir, nil) + if printWorkDir { + b.BackgroundShell().Print("WORK=", b.WorkDir, "\n") + } + if err := CheckGOOSARCHPair(cfg.Goos, cfg.Goarch); err != nil { fmt.Fprintf(os.Stderr, "go: %v\n", err) base.SetExitStatus(2) diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 4d05d79722..3508d51fbb 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -141,6 +141,9 @@ and test commands: or, if set explicitly, has _race appended to it. Likewise for the -msan and -asan flags. Using a -buildmode option that requires non-default compile flags has a similar effect. + -json + Emit build output in JSON suitable for automated processing. + See 'go help buildjson' for the encoding details. -ldflags '[pattern=]arg list' arguments to pass on each go tool link invocation. -linkshared @@ -300,6 +303,8 @@ const ( OmitModFlag BuildFlagMask = 1 << iota OmitModCommonFlags OmitVFlag + OmitBuildOnlyFlags // Omit flags that only affect building packages + OmitJSONFlag ) // AddBuildFlags adds the flags common to the build, clean, get, @@ -332,6 +337,13 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.StringVar(&fsys.OverlayFile, "overlay", "", "") } cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "") + if mask&(OmitBuildOnlyFlags|OmitJSONFlag) == 0 { + // TODO(#62250): OmitBuildOnlyFlags should apply to many more flags + // here, but we let a bunch of flags slip in before we realized that + // many of them don't make sense for most subcommands. We might even + // want to separate "AddBuildFlags" and "AddSelectionFlags". + cmd.Flag.BoolVar(&cfg.BuildJSON, "json", false, "") + } cmd.Flag.Var(&load.BuildLdflags, "ldflags", "") cmd.Flag.BoolVar(&cfg.BuildLinkshared, "linkshared", false, "") cmd.Flag.BoolVar(&cfg.BuildMSan, "msan", false, "") diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index eedec2b962..4faeb8a172 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -71,6 +71,7 @@ func init() { vet.CmdVet, help.HelpBuildConstraint, + help.HelpBuildJSON, help.HelpBuildmode, help.HelpC, help.HelpCache, diff --git a/src/cmd/go/testdata/script/build_json.txt b/src/cmd/go/testdata/script/build_json.txt new file mode 100644 index 0000000000..4506660fe6 --- /dev/null +++ b/src/cmd/go/testdata/script/build_json.txt @@ -0,0 +1,48 @@ +[short] skip + +# Basic build error. This test also checks that the output is fully-formed JSON. +! go build -json -o=$devnull ./compileerror +stdout '^\{"ImportPath":"m/compileerror","Action":"build-output","Output":"# m/compileerror\\n"\}$' +stdout '^\{"ImportPath":"m/compileerror","Action":"build-output","Output":"compileerror/main.go:3:11: undefined: y\\n"}$' +stdout '^\{"ImportPath":"m/compileerror","Action":"build-fail"\}$' +! stderr '.' + +# Check that a build failure in an imported package is attributed correctly. +! go build -json -o=$devnull ./importerror +stdout '"ImportPath":"m/compileerror","Action":"build-fail"' +! stderr '.' + +# TODO(#65335): Attributing this to "x" doesn't make much sense, +# especially since the reported line is the import statement. +! go build -json -o=$devnull ./loaderror +stdout '"ImportPath":"x","Action":"build-output","Output":".*package x is not in std.*\\n"' +stdout '"ImportPath":"x","Action":"build-fail"' +! stderr '.' + +# Check that a load error in an imported package is attributed correctly. +! go build -json -o=$devnull ./loadimporterror +stdout '"ImportPath":"x","Action":"build-output","Output":".*package x is not in std.*\\n"' +stdout '"ImportPath":"x","Action":"build-fail"' +! stderr '.' + +-- go.mod -- +module m +go 1.21 +-- compileerror/main.go -- +package compileerror + +const x = y +-- importerror/main.go -- +package main + +import _ "m/compileerror" +-- loaderror/main.go -- +// A bad import causes a failure directly in cmd/go during import processing. + +package loaderror + +import _ "x" +-- loadimporterror/main.go -- +package loadimporterror + +import _ "m/loaderror" -- 2.48.1