From 41fc05d02357b8681b7a5adfbaedf315328d309e Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Sun, 24 Aug 2014 11:34:12 -0700 Subject: [PATCH] cmd/go: add go generate First cut. Works well enough to support yacc via https://golang.org/cl/125620044. LGTM=alex.brainman, rsc R=rsc, alex.brainman CC=golang-codereviews https://golang.org/cl/125580044 --- src/cmd/go/doc.go | 93 ++++++- src/cmd/go/generate.go | 340 ++++++++++++++++++++++++++ src/cmd/go/main.go | 1 + src/cmd/go/test.bash | 27 ++ src/cmd/go/testdata/generate/test1.go | 13 + src/cmd/go/testdata/generate/test2.go | 10 + src/cmd/go/testdata/generate/test3.go | 9 + 7 files changed, 491 insertions(+), 2 deletions(-) create mode 100644 src/cmd/go/generate.go create mode 100644 src/cmd/go/testdata/generate/test1.go create mode 100644 src/cmd/go/testdata/generate/test2.go create mode 100644 src/cmd/go/testdata/generate/test3.go diff --git a/src/cmd/go/doc.go b/src/cmd/go/doc.go index 4778048b52..19fe5cd3ef 100644 --- a/src/cmd/go/doc.go +++ b/src/cmd/go/doc.go @@ -19,6 +19,7 @@ The commands are: env print Go environment information fix run go tool fix on packages fmt run gofmt on package sources + generate generate Go files by processing source get download and install packages and dependencies install compile and install packages and dependencies list list packages @@ -219,6 +220,94 @@ To run gofmt with specific options, run gofmt itself. See also: go fix, go vet. +Generate Go files by processing source + +Usage: + + go generate [-run regexp] [file.go... | packages] + +Generate runs commands described by directives within existing +files. Those commands can run any process but the intent is to +create or update Go source files, for instance by running yacc. + +Go generate is never run automatically by go build, go get, go test, +and so on. It must be run explicitly. + +Directives are written as a whole-line comment of the form + + //go:generate command argument... + +(note: no space in "//go") where command is the generator to be +run, corresponding to an executable file that can be run locally. +It must either be in the shell path (gofmt), a fully qualified path +(/usr/you/bin/mytool), or a command alias, described below. + +The arguments are space-separated tokens or double-quoted strings +passed to the generator as individual arguments when it is run. + +Quoted strings use Go syntax and are evaluated before execution; a +quoted string appears a single argument to the generator. + +Go generate sets several variables when it runs the generator: + + $GOFILE + The base name of the file. + $GOPACKAGE + The name of the package of the file containing the directive. + +Other than variable substition and quoted-string evaluation, no +special processing such as "globbing" is performed on the command +line. + +As a last step before running the command, any invocations of any +environment variables with alphanumeric names, such as $GOFILE or +$HOME, are expanded throughout the command line. The syntax for +variable expansion is $NAME on all operating systems. Due to the +order of evaluation, variables are expanded even inside quoted +strings. If the variable NAME is not set, $NAME expands to the +empty string. + +A directive of the form, + + //go:generate -command xxx args... + +specifies, for the remainder of this source file only, that the +string xxx represents the command identified by the arguments. This +can be used to create aliases or to handle multiword generators. +For example, + + //go:generate -command yacc go tool yacc + +specifies that the command "yacc" represents the generator +"go tool yacc". + +Generate processes packages in the order given on the command line, +one at a time. If the command line lists .go files, they are treated +as a single package. Within a package, generate processes the +source files in a package in file name order, one at a time. Within +a source file, generate runs generators in the order they appear +in the file, one at a time. + +If any generator returns an error exit status, "go generate" skips +all further processing for that package. + +The generator is run in the package's source directory. + +Go generate accepts one specific flag: + + -run="" + if non-empty, specifies a regular expression to + select directives whose command matches the expression. + +It also accepts the standard build flags -v, -n, and -x. +The -v flag prints the names of packages and files as they are +processed. +The -n flag prints commands that would be executed. +The -x flag prints commands as they are executed. + +For more about specifying packages, see 'go help packages'. + + Download and install packages and dependencies Usage: @@ -750,10 +839,10 @@ will result in the following request(s): If that page contains the meta tag - + the go tool will verify that https://example.org/?go-get=1 contains the -same meta tag and then git clone https://code.example/r/p/exproj into +same meta tag and then git clone https://code.org/r/p/exproj into GOPATH/src/example.org. New downloaded packages are written to the first directory diff --git a/src/cmd/go/generate.go b/src/cmd/go/generate.go new file mode 100644 index 0000000000..34b10314d2 --- /dev/null +++ b/src/cmd/go/generate.go @@ -0,0 +1,340 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +var cmdGenerate = &Command{ + Run: runGenerate, + UsageLine: "generate [-run regexp] [file.go... | packages]", + Short: "generate Go files by processing source", + Long: ` +Generate runs commands described by directives within existing +files. Those commands can run any process but the intent is to +create or update Go source files, for instance by running yacc. + +Go generate is never run automatically by go build, go get, go test, +and so on. It must be run explicitly. + +Directives are written as a whole-line comment of the form + + //go:generate command argument... + +(note: no space in "//go") where command is the generator to be +run, corresponding to an executable file that can be run locally. +It must either be in the shell path (gofmt), a fully qualified path +(/usr/you/bin/mytool), or a command alias, described below. + +The arguments are space-separated tokens or double-quoted strings +passed to the generator as individual arguments when it is run. + +Quoted strings use Go syntax and are evaluated before execution; a +quoted string appears a single argument to the generator. + +Go generate sets several variables when it runs the generator: + + $GOFILE + The base name of the file. + $GOPACKAGE + The name of the package of the file containing the directive. + +Other than variable substition and quoted-string evaluation, no +special processing such as "globbing" is performed on the command +line. + +As a last step before running the command, any invocations of any +environment variables with alphanumeric names, such as $GOFILE or +$HOME, are expanded throughout the command line. The syntax for +variable expansion is $NAME on all operating systems. Due to the +order of evaluation, variables are expanded even inside quoted +strings. If the variable NAME is not set, $NAME expands to the +empty string. + +A directive of the form, + + //go:generate -command xxx args... + +specifies, for the remainder of this source file only, that the +string xxx represents the command identified by the arguments. This +can be used to create aliases or to handle multiword generators. +For example, + + //go:generate -command yacc go tool yacc + +specifies that the command "yacc" represents the generator +"go tool yacc". + +Generate processes packages in the order given on the command line, +one at a time. If the command line lists .go files, they are treated +as a single package. Within a package, generate processes the +source files in a package in file name order, one at a time. Within +a source file, generate runs generators in the order they appear +in the file, one at a time. + +If any generator returns an error exit status, "go generate" skips +all further processing for that package. + +The generator is run in the package's source directory. + +Go generate accepts one specific flag: + + -run="" + if non-empty, specifies a regular expression to + select directives whose command matches the expression. + +It also accepts the standard build flags -v, -n, and -x. +The -v flag prints the names of packages and files as they are +processed. +The -n flag prints commands that would be executed. +The -x flag prints commands as they are executed. + +For more about specifying packages, see 'go help packages'. + `, +} + +var generateRunFlag string // generate -run flag + +func init() { + addBuildFlags(cmdGenerate) + cmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "") +} + +func runGenerate(cmd *Command, args []string) { + // Even if the arguments are .go files, this loop suffices. + for _, pkg := range packages(args) { + for _, file := range pkg.gofiles { + if !generate(pkg.Name, file) { + break + } + } + } +} + +// generate runs the generation directives for a single file. +func generate(pkg, absFile string) bool { + fd, err := os.Open(absFile) + if err != nil { + log.Fatalf("generate: %s", err) + } + defer fd.Close() + g := &Generator{ + r: fd, + path: absFile, + pkg: pkg, + commands: make(map[string][]string), + } + return g.run() +} + +// A Generator represents the state of a single Go source file +// being scanned for generator commands. +type Generator struct { + r io.Reader + path string // full rooted path name. + dir string // full rooted directory of file. + file string // base name of file. + pkg string + commands map[string][]string + lineNum int +} + +// run runs the generators in the current file. +func (g *Generator) run() (ok bool) { + // Processing below here calls g.errorf on failure, which does panic(stop). + // If we encouter an error, we abort the package. + defer func() { + e := recover() + if e != nil { + ok = false + if e != stop { + panic(e) + } + } + }() + g.dir, g.file = filepath.Split(g.path) + g.dir = filepath.Clean(g.dir) // No final separator please. + if buildV { + fmt.Fprintf(os.Stderr, "%s\n", shortPath(g.path)) + } + + s := bufio.NewScanner(g.r) + for s.Scan() { + g.lineNum++ + if !bytes.HasPrefix(s.Bytes(), []byte("//go:generate ")) && !bytes.HasPrefix(s.Bytes(), []byte("//go:generate\t")) { + continue + } + words := g.split(s.Text()) + if len(words) == 0 { + g.errorf("no arguments to directive") + } + if words[0] == "-command" { + g.setShorthand(words) + continue + } + // Run the command line. + if buildN || buildX { + fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " ")) + } + if buildN { + continue + } + g.exec(words) + } + if s.Err() != nil { + g.errorf("error reading %s: %s", shortPath(g.path), s.Err()) + } + return true +} + +// split breaks the line into words, evaluating quoted +// strings and evaluating environment variables. +// The initial //go:generate element is dropped. +func (g *Generator) split(line string) []string { + // Parse line, obeying quoted strings. + var words []string + line = line[len("//go:generate "):] + // One (possibly quoted) word per iteration. +Words: + for { + line = strings.TrimLeft(line, " \t") + if len(line) == 0 { + break + } + if line[0] == '"' { + for i := 1; i < len(line); i++ { + c := line[i] // Only looking for ASCII so this is OK. + switch c { + case '\\': + if i+1 == len(line) { + g.errorf("bad backslash") + } + i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote). + case '"': + word, err := strconv.Unquote(line[0 : i+1]) + if err != nil { + g.errorf("bad quoted string") + } + words = append(words, word) + line = line[i+1:] + // Check the next character is space or end of line. + if len(line) > 0 && line[0] != ' ' && line[0] != '\t' { + g.errorf("expect space after quoted argument") + } + continue Words + } + } + g.errorf("mismatched quoted string") + } + i := strings.IndexAny(line, " \t") + if i < 0 { + i = len(line) + } + words = append(words, line[0:i]) + line = line[i:] + } + // Substitute command if required. + if len(words) > 0 && g.commands[words[0]] != nil { + // Replace 0th word by command substitution. + words = append(g.commands[words[0]], words[1:]...) + } + // Substitute environment variables. + for i, word := range words { + words[i] = g.expandEnv(word) + } + return words +} + +var stop = fmt.Errorf("error in generation") + +// errorf logs an error message prefixed with the file and line number. +// It then exits the program because generation stops at the first error. +func (g *Generator) errorf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "%s:%d: %s\n", shortPath(g.path), g.lineNum, + fmt.Sprintf(format, args...)) + panic(stop) +} + +// expandEnv expands any $XXX invocations in word. +func (g *Generator) expandEnv(word string) string { + if !strings.ContainsRune(word, '$') { + return word + } + var buf bytes.Buffer + var w int + var r rune + for i := 0; i < len(word); i += w { + r, w = utf8.DecodeRuneInString(word[i:]) + if r != '$' { + buf.WriteRune(r) + continue + } + w += g.identLength(word[i+w:]) + envVar := word[i+1 : i+w] + var sub string + switch envVar { + case "GOFILE": + sub = g.file + case "GOPACKAGE": + sub = g.pkg + default: + sub = os.Getenv(envVar) + } + buf.WriteString(sub) + } + return buf.String() +} + +// identLength returns the length of the identifier beginning the string. +func (g *Generator) identLength(word string) int { + for i, r := range word { + if r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) { + continue + } + return i + } + return len(word) +} + +// setShorthand installs a new shorthand as defined by a -command directive. +func (g *Generator) setShorthand(words []string) { + // Create command shorthand. + if len(words) == 1 { + g.errorf("no command specified for -command") + } + command := words[1] + if g.commands[command] != nil { + g.errorf("command %q defined multiply defined", command) + } + g.commands[command] = words[2:len(words):len(words)] // force later append to make copy +} + +// exec runs the command specified by the argument. The first word is +// the command name itself. +func (g *Generator) exec(words []string) { + cmd := exec.Command(words[0], words[1:]...) + // Standard in and out of generator should be the usual. + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + // Run the command in the package directory. + cmd.Dir = g.dir + cmd.Env = mergeEnvLists([]string{"GOFILE=" + g.file, "GOPACKAGE=" + g.pkg}, os.Environ()) + err := cmd.Run() + if err != nil { + g.errorf("running %q: %s", words[0], err) + } +} diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 5b1194aaa3..eb69606def 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -79,6 +79,7 @@ var commands = []*Command{ cmdEnv, cmdFix, cmdFmt, + cmdGenerate, cmdGet, cmdInstall, cmdList, diff --git a/src/cmd/go/test.bash b/src/cmd/go/test.bash index 2bb929fb03..24640e2723 100755 --- a/src/cmd/go/test.bash +++ b/src/cmd/go/test.bash @@ -878,6 +878,33 @@ elif ! grep 'File with non-runnable example was built.' testdata/std.out > /dev/ ok=false fi +TEST 'go generate handles simple command' +if ! ./testgo generate ./testdata/generate/test1.go > testdata/std.out; then + echo "go test ./testdata/generate/test1.go failed to run" + ok=false +elif ! grep 'Success' testdata/std.out > /dev/null; then + echo "go test ./testdata/generate/test1.go generated wrong output" + ok=false +fi + +TEST 'go generate handles command alias' +if ! ./testgo generate ./testdata/generate/test2.go > testdata/std.out; then + echo "go test ./testdata/generate/test2.go failed to run" + ok=false +elif ! grep 'Now is the time for all good men' testdata/std.out > /dev/null; then + echo "go test ./testdata/generate/test2.go generated wrong output" + ok=false +fi + +TEST 'go generate variable substitution' +if ! ./testgo generate ./testdata/generate/test3.go > testdata/std.out; then + echo "go test ./testdata/generate/test3.go failed to run" + ok=false +elif ! grep "$GOARCH test3.go p xyzp/test3.go/123" testdata/std.out > /dev/null; then + echo "go test ./testdata/generate/test3.go generated wrong output" + ok=false +fi + # clean up if $started; then stop; fi rm -rf testdata/bin testdata/bin1 diff --git a/src/cmd/go/testdata/generate/test1.go b/src/cmd/go/testdata/generate/test1.go new file mode 100644 index 0000000000..1f05734f04 --- /dev/null +++ b/src/cmd/go/testdata/generate/test1.go @@ -0,0 +1,13 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Simple test for go generate. + +// We include a build tag that go generate should ignore. + +// +build ignore + +//go:generate echo Success + +package p diff --git a/src/cmd/go/testdata/generate/test2.go b/src/cmd/go/testdata/generate/test2.go new file mode 100644 index 0000000000..ef1a3d9515 --- /dev/null +++ b/src/cmd/go/testdata/generate/test2.go @@ -0,0 +1,10 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test that go generate handles command aliases. + +//go:generate -command run echo Now is the time +//go:generate run for all good men + +package p diff --git a/src/cmd/go/testdata/generate/test3.go b/src/cmd/go/testdata/generate/test3.go new file mode 100644 index 0000000000..41ffb7ea87 --- /dev/null +++ b/src/cmd/go/testdata/generate/test3.go @@ -0,0 +1,9 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test go generate variable substitution. + +//go:generate echo $GOARCH $GOFILE $GOPACKAGE xyz$GOPACKAGE/$GOFILE/123 + +package p -- 2.48.1