From 8246171baee397165318638390c37e07b5b3ebc7 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Tue, 30 Jul 2024 14:20:53 +0000 Subject: [PATCH] cmd: add README generation for compiler + linker script tests Add in automatic README generation and README consistency checking for the cmd/compile and cmd/link script tests. This code is adapted from the similar facility in cmd/go (e.g. scriptreadme_test.go); the README helps folks writing new tests understand the mechanics. Updates #68606. Change-Id: I8ff7ff8e814abd4385bd670440511b2c60a4cef6 Reviewed-on: https://go-review.googlesource.com/c/go/+/601756 Reviewed-by: Cherry Mui LUCI-TryBot-Result: Go LUCI --- src/cmd/compile/script_test.go | 7 +- src/cmd/compile/testdata/script/README | 286 +++++++++++++++++++ src/cmd/internal/script/scripttest/readme.go | 142 +++++++++ src/cmd/internal/script/scripttest/run.go | 23 +- src/cmd/link/script_test.go | 7 +- src/cmd/link/testdata/script/README | 286 +++++++++++++++++++ 6 files changed, 746 insertions(+), 5 deletions(-) create mode 100644 src/cmd/compile/testdata/script/README create mode 100644 src/cmd/internal/script/scripttest/readme.go create mode 100644 src/cmd/link/testdata/script/README diff --git a/src/cmd/compile/script_test.go b/src/cmd/compile/script_test.go index 962e4bb754..0e32e0769e 100644 --- a/src/cmd/compile/script_test.go +++ b/src/cmd/compile/script_test.go @@ -6,12 +6,17 @@ package main import ( "cmd/internal/script/scripttest" + "flag" "internal/testenv" "os" "runtime" "testing" ) +//go:generate go test cmd/compile -v -run=TestScript/README --fixreadme + +var fixReadme = flag.Bool("fixreadme", false, "if true, update README for script tests") + var testCompiler string // TestMain allows this test binary to run as the compiler @@ -58,5 +63,5 @@ func TestScript(t *testing.T) { }, } } - scripttest.RunToolScriptTest(t, repls, "testdata/script/*.txt") + scripttest.RunToolScriptTest(t, repls, "testdata/script", *fixReadme) } diff --git a/src/cmd/compile/testdata/script/README b/src/cmd/compile/testdata/script/README new file mode 100644 index 0000000000..9ec997a138 --- /dev/null +++ b/src/cmd/compile/testdata/script/README @@ -0,0 +1,286 @@ +This file is generated by 'go generate'. DO NOT EDIT. + +This directory holds test scripts *.txt run during 'go test cmd/'. +To run a specific script foo.txt + + go test cmd/ -run=Script/^foo$ + +In general script files should have short names: a few words, + not whole sentences. +The first word should be the general category of behavior being tested, +often the name of a go subcommand (build, link, compile, ...) or concept (vendor, pattern). + +Each script is a text archive (go doc internal/txtar). +The script begins with an actual command script to run +followed by the content of zero or more supporting files to +create in the script's temporary file system before it starts executing. + +As an example, run_hello.txt says: + + # hello world + go run hello.go + stderr 'hello world' + ! stdout . + + -- hello.go -- + package main + func main() { println("hello world") } + +Each script runs in a fresh temporary work directory tree, available to scripts as $WORK. +Scripts also have access to other environment variables, including: + + GOARCH= + GOOS= + TMPDIR=$WORK/tmp + devnull= + goversion= + +On Plan 9, the variables $path and $home are set instead of $PATH and $HOME. +On Windows, the variables $USERPROFILE and $TMP are set instead of +$HOME and $TMPDIR. + +The lines at the top of the script are a sequence of commands to be executed by +a small script engine configured in .../cmd/internal/script/scripttest/run.go (not the system shell). + +Each line of a script is parsed into a sequence of space-separated command +words, with environment variable expansion within each word and # marking +an end-of-line comment. Additional variables named ':' and '/' are expanded +within script arguments (expanding to the value of os.PathListSeparator and +os.PathSeparator respectively) but are not inherited in subprocess environments. + +Adding single quotes around text keeps spaces in that text from being treated +as word separators and also disables environment variable expansion. Inside a +single-quoted block of text, a repeated single quote indicates a literal single +quote, as in: + + 'Don''t communicate by sharing memory.' + +A line beginning with # is a comment and conventionally explains what is being +done or tested at the start of a new section of the script. + +Commands are executed one at a time, and errors are checked for each command; +if any command fails unexpectedly, no subsequent commands in the script are +executed. The command prefix ! indicates that the command on the rest of the +line (typically go or a matching predicate) must fail instead of succeeding. +The command prefix ? indicates that the command may or may not succeed, but the +script should continue regardless. + +The command prefix [cond] indicates that the command on the rest of the line +should only run when the condition is satisfied. + +A condition can be negated: [!root] means to run the rest of the line only if +the user is not root. Multiple conditions may be given for a single command, +for example, '[linux] [amd64] skip'. The command will run if all conditions are +satisfied. + +When TestScript runs a script and the script fails, by default TestScript shows +the execution of the most recent phase of the script (since the last # comment) +and only shows the # comments for earlier phases. + +Note also that in reported output, the actual name of the per-script temporary directory +has been consistently replaced with the literal string $WORK. + +The available commands are: +cat files... + concatenate files and print to the script's stdout buffer + + +cc args... + run the platform C compiler + + +cd dir + change the working directory + + +chmod perm paths... + change file mode bits + + Changes the permissions of the named files or directories to + be equal to perm. + Only numerical permissions are supported. + +cmp [-q] file1 file2 + compare files for differences + + By convention, file1 is the actual data and file2 is the + expected data. + The command succeeds if the file contents are identical. + File1 can be 'stdout' or 'stderr' to compare the stdout or + stderr buffer from the most recent command. + +cmpenv [-q] file1 file2 + compare files for differences, with environment expansion + + By convention, file1 is the actual data and file2 is the + expected data. + The command succeeds if the file contents are identical + after substituting variables from the script environment. + File1 can be 'stdout' or 'stderr' to compare the script's + stdout or stderr buffer. + +cp src... dst + copy files to a target file or directory + + src can include 'stdout' or 'stderr' to copy from the + script's stdout or stderr buffer. + +echo string... + display a line of text + + +env [key[=value]...] + set or log the values of environment variables + + With no arguments, print the script environment to the log. + Otherwise, add the listed key=value pairs to the environment + or print the listed keys. + +exec program [args...] [&] + run an executable program with arguments + + Note that 'exec' does not terminate the script (unlike Unix + shells). + +exists [-readonly] [-exec] file... + check that files exist + + +go [args...] [&] + run the 'go' program provided by the script host + + +grep [-count=N] [-q] 'pattern' file + find lines in a file that match a pattern + + The command succeeds if at least one match (or the exact + count, if given) is found. + The -q flag suppresses printing of matches. + +help [-v] name... + log help text for commands and conditions + + To display help for a specific condition, enclose it in + brackets: 'help [amd64]'. + To display complete documentation when listing all commands, + pass the -v flag. + +mkdir path... + create directories, if they do not already exist + + Unlike Unix mkdir, parent directories are always created if + needed. + +mv old new + rename a file or directory to a new path + + OS-specific restrictions may apply when old and new are in + different directories. + +replace [old new]... file + replace strings in a file + + The 'old' and 'new' arguments are unquoted as if in quoted + Go strings. + +rm path... + remove a file or directory + + If the path is a directory, its contents are removed + recursively. + +skip [msg] + skip the current test + + +sleep duration [&] + sleep for a specified duration + + The duration must be given as a Go time.Duration string. + +stderr [-count=N] [-q] 'pattern' file + find lines in the stderr buffer that match a pattern + + The command succeeds if at least one match (or the exact + count, if given) is found. + The -q flag suppresses printing of matches. + +stdout [-count=N] [-q] 'pattern' file + find lines in the stdout buffer that match a pattern + + The command succeeds if at least one match (or the exact + count, if given) is found. + The -q flag suppresses printing of matches. + +stop [msg] + stop execution of the script + + The message is written to the script log, but no error is + reported from the script engine. + +symlink path -> target + create a symlink + + Creates path as a symlink to target. + The '->' token (like in 'ls -l' output on Unix) is required. + +wait + wait for completion of background commands + + Waits for all background commands to complete. + The output (and any error) from each command is printed to + the log in the order in which the commands were started. + After the call to 'wait', the script's stdout and stderr + buffers contain the concatenation of the background + commands' outputs. + + + +The available conditions are: +[GOARCH:*] + runtime.GOARCH == +[GODEBUG:*] + GODEBUG contains +[GOEXPERIMENT:*] + GOEXPERIMENT is enabled +[GOOS:*] + runtime.GOOS == +[asan] + GOOS/GOARCH supports -asan +[buildmode:*] + go supports -buildmode= +[cgo] + host CGO_ENABLED +[cgolinkext] + platform requires external linking for cgo +[compiler:*] + runtime.Compiler == +[cross] + cmd/go GOOS/GOARCH != GOHOSTOS/GOHOSTARCH +[exec:*] + names an executable in the test binary's PATH +[fuzz] + GOOS/GOARCH supports -fuzz +[fuzz-instrumented] + GOOS/GOARCH supports -fuzz with instrumentation +[go-builder] + GO_BUILDER_NAME is non-empty +[link] + testenv.HasLink() +[msan] + GOOS/GOARCH supports -msan +[mustlinkext] + platform always requires external linking +[pielinkext] + platform requires external linking for PIE +[race] + GOOS/GOARCH supports -race +[root] + os.Geteuid() == 0 +[short] + testing.Short() +[symlink] + testenv.HasSymlink() +[verbose] + testing.Verbose() + diff --git a/src/cmd/internal/script/scripttest/readme.go b/src/cmd/internal/script/scripttest/readme.go new file mode 100644 index 0000000000..af7397223f --- /dev/null +++ b/src/cmd/internal/script/scripttest/readme.go @@ -0,0 +1,142 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scripttest + +import ( + "bytes" + "cmd/internal/script" + "internal/diff" + "internal/testenv" + "os" + "path/filepath" + "strings" + "testing" + "text/template" +) + +func checkScriptReadme(t *testing.T, engine *script.Engine, env []string, scriptspath, gotool string, fixReadme bool) { + var args struct { + Language string + Commands string + Conditions string + } + + cmds := new(strings.Builder) + if err := engine.ListCmds(cmds, true); err != nil { + t.Fatal(err) + } + args.Commands = cmds.String() + + conds := new(strings.Builder) + if err := engine.ListConds(conds, nil); err != nil { + t.Fatal(err) + } + args.Conditions = conds.String() + + doc := new(strings.Builder) + cmd := testenv.Command(t, gotool, "doc", "cmd/internal/script") + cmd.Env = env + cmd.Stdout = doc + if err := cmd.Run(); err != nil { + t.Fatal(cmd, ":", err) + } + _, lang, ok := strings.Cut(doc.String(), "# Script Language\n\n") + if !ok { + t.Fatalf("%q did not include Script Language section", cmd) + } + lang, _, ok = strings.Cut(lang, "\n\nvar ") + if !ok { + t.Fatalf("%q did not include vars after Script Language section", cmd) + } + args.Language = lang + + tmpl := template.Must(template.New("README").Parse(readmeTmpl[1:])) + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, args); err != nil { + t.Fatal(err) + } + + readmePath := filepath.Join(scriptspath, "README") + old, err := os.ReadFile(readmePath) + if err != nil { + t.Fatal(err) + } + diff := diff.Diff(readmePath, old, "readmeTmpl", buf.Bytes()) + if diff == nil { + t.Logf("%s is up to date.", readmePath) + return + } + + if fixReadme { + if err := os.WriteFile(readmePath, buf.Bytes(), 0666); err != nil { + t.Fatal(err) + } + t.Logf("wrote %d bytes to %s", buf.Len(), readmePath) + } else { + t.Logf("\n%s", diff) + t.Errorf("%s is stale. To update, run 'go generate cmd/go'.", readmePath) + } +} + +const readmeTmpl = ` +This file is generated by 'go generate'. DO NOT EDIT. + +This directory holds test scripts *.txt run during 'go test cmd/'. +To run a specific script foo.txt + + go test cmd/ -run=Script/^foo$ + +In general script files should have short names: a few words, + not whole sentences. +The first word should be the general category of behavior being tested, +often the name of a go subcommand (build, link, compile, ...) or concept (vendor, pattern). + +Each script is a text archive (go doc internal/txtar). +The script begins with an actual command script to run +followed by the content of zero or more supporting files to +create in the script's temporary file system before it starts executing. + +As an example, run_hello.txt says: + + # hello world + go run hello.go + stderr 'hello world' + ! stdout . + + -- hello.go -- + package main + func main() { println("hello world") } + +Each script runs in a fresh temporary work directory tree, available to scripts as $WORK. +Scripts also have access to other environment variables, including: + + GOARCH= + GOOS= + TMPDIR=$WORK/tmp + devnull= + goversion= + +On Plan 9, the variables $path and $home are set instead of $PATH and $HOME. +On Windows, the variables $USERPROFILE and $TMP are set instead of +$HOME and $TMPDIR. + +The lines at the top of the script are a sequence of commands to be executed by +a small script engine configured in .../cmd/internal/script/scripttest/run.go (not the system shell). + +{{.Language}} + +When TestScript runs a script and the script fails, by default TestScript shows +the execution of the most recent phase of the script (since the last # comment) +and only shows the # comments for earlier phases. + +Note also that in reported output, the actual name of the per-script temporary directory +has been consistently replaced with the literal string $WORK. + +The available commands are: +{{.Commands}} + +The available conditions are: +{{.Conditions}} +` diff --git a/src/cmd/internal/script/scripttest/run.go b/src/cmd/internal/script/scripttest/run.go index 29eb6f88f2..8dff13e22e 100644 --- a/src/cmd/internal/script/scripttest/run.go +++ b/src/cmd/internal/script/scripttest/run.go @@ -34,7 +34,7 @@ type ToolReplacement struct { // is that we'll be called from the top level cmd/X dir for tool X, // and that instead of executing the install tool X we'll use the // test binary instead. -func RunToolScriptTest(t *testing.T, repls []ToolReplacement, pattern string) { +func RunToolScriptTest(t *testing.T, repls []ToolReplacement, scriptsdir string, fixReadme bool) { // Nearly all script tests involve doing builds, so don't // bother here if we don't have "go build". testenv.MustHaveGoBuild(t) @@ -125,9 +125,9 @@ func RunToolScriptTest(t *testing.T, repls []ToolReplacement, pattern string) { // Add in commands for "go" and "cc". testgo := filepath.Join(tgr, "bin", "go") gocmd := script.Program(testgo, interrupt, gracePeriod) - cccmd := script.Program(goEnv("CC"), interrupt, gracePeriod) addcmd("go", gocmd) - addcmd("cc", cccmd) + cmdExec := cmds["exec"] + addcmd("cc", scriptCC(cmdExec, goEnv("CC"))) // Add various helpful conditions related to builds and toolchain use. goHostOS, goHostArch := goEnv("GOHOSTOS"), goEnv("GOHOSTARCH") @@ -153,8 +153,13 @@ func RunToolScriptTest(t *testing.T, repls []ToolReplacement, pattern string) { Quiet: !testing.Verbose(), } + t.Run("README", func(t *testing.T) { + checkScriptReadme(t, engine, env, scriptsdir, gotool, fixReadme) + }) + // ... and kick off tests. ctx := context.Background() + pattern := filepath.Join(scriptsdir, "*.txt") RunTests(t, ctx, engine, env, pattern) } @@ -253,3 +258,15 @@ func tempEnvName() string { return "TMPDIR" } } + +// scriptCC runs the platform C compiler. +func scriptCC(cmdExec script.Cmd, ccexe string) script.Cmd { + return script.Command( + script.CmdUsage{ + Summary: "run the platform C compiler", + Args: "args...", + }, + func(s *script.State, args ...string) (script.WaitFunc, error) { + return cmdExec.Run(s, append([]string{ccexe}, args...)...) + }) +} diff --git a/src/cmd/link/script_test.go b/src/cmd/link/script_test.go index 379d47593e..670a939b74 100644 --- a/src/cmd/link/script_test.go +++ b/src/cmd/link/script_test.go @@ -6,11 +6,16 @@ package main import ( "cmd/internal/script/scripttest" + "flag" "internal/testenv" "runtime" "testing" ) +//go:generate go test cmd/link -v -run=TestScript/README --fixreadme + +var fixReadme = flag.Bool("fixreadme", false, "if true, update README for script tests") + func TestScript(t *testing.T) { testenv.MustHaveGoBuild(t) doReplacement := true @@ -33,5 +38,5 @@ func TestScript(t *testing.T) { }, } } - scripttest.RunToolScriptTest(t, repls, "testdata/script/*.txt") + scripttest.RunToolScriptTest(t, repls, "testdata/script", *fixReadme) } diff --git a/src/cmd/link/testdata/script/README b/src/cmd/link/testdata/script/README new file mode 100644 index 0000000000..9ec997a138 --- /dev/null +++ b/src/cmd/link/testdata/script/README @@ -0,0 +1,286 @@ +This file is generated by 'go generate'. DO NOT EDIT. + +This directory holds test scripts *.txt run during 'go test cmd/'. +To run a specific script foo.txt + + go test cmd/ -run=Script/^foo$ + +In general script files should have short names: a few words, + not whole sentences. +The first word should be the general category of behavior being tested, +often the name of a go subcommand (build, link, compile, ...) or concept (vendor, pattern). + +Each script is a text archive (go doc internal/txtar). +The script begins with an actual command script to run +followed by the content of zero or more supporting files to +create in the script's temporary file system before it starts executing. + +As an example, run_hello.txt says: + + # hello world + go run hello.go + stderr 'hello world' + ! stdout . + + -- hello.go -- + package main + func main() { println("hello world") } + +Each script runs in a fresh temporary work directory tree, available to scripts as $WORK. +Scripts also have access to other environment variables, including: + + GOARCH= + GOOS= + TMPDIR=$WORK/tmp + devnull= + goversion= + +On Plan 9, the variables $path and $home are set instead of $PATH and $HOME. +On Windows, the variables $USERPROFILE and $TMP are set instead of +$HOME and $TMPDIR. + +The lines at the top of the script are a sequence of commands to be executed by +a small script engine configured in .../cmd/internal/script/scripttest/run.go (not the system shell). + +Each line of a script is parsed into a sequence of space-separated command +words, with environment variable expansion within each word and # marking +an end-of-line comment. Additional variables named ':' and '/' are expanded +within script arguments (expanding to the value of os.PathListSeparator and +os.PathSeparator respectively) but are not inherited in subprocess environments. + +Adding single quotes around text keeps spaces in that text from being treated +as word separators and also disables environment variable expansion. Inside a +single-quoted block of text, a repeated single quote indicates a literal single +quote, as in: + + 'Don''t communicate by sharing memory.' + +A line beginning with # is a comment and conventionally explains what is being +done or tested at the start of a new section of the script. + +Commands are executed one at a time, and errors are checked for each command; +if any command fails unexpectedly, no subsequent commands in the script are +executed. The command prefix ! indicates that the command on the rest of the +line (typically go or a matching predicate) must fail instead of succeeding. +The command prefix ? indicates that the command may or may not succeed, but the +script should continue regardless. + +The command prefix [cond] indicates that the command on the rest of the line +should only run when the condition is satisfied. + +A condition can be negated: [!root] means to run the rest of the line only if +the user is not root. Multiple conditions may be given for a single command, +for example, '[linux] [amd64] skip'. The command will run if all conditions are +satisfied. + +When TestScript runs a script and the script fails, by default TestScript shows +the execution of the most recent phase of the script (since the last # comment) +and only shows the # comments for earlier phases. + +Note also that in reported output, the actual name of the per-script temporary directory +has been consistently replaced with the literal string $WORK. + +The available commands are: +cat files... + concatenate files and print to the script's stdout buffer + + +cc args... + run the platform C compiler + + +cd dir + change the working directory + + +chmod perm paths... + change file mode bits + + Changes the permissions of the named files or directories to + be equal to perm. + Only numerical permissions are supported. + +cmp [-q] file1 file2 + compare files for differences + + By convention, file1 is the actual data and file2 is the + expected data. + The command succeeds if the file contents are identical. + File1 can be 'stdout' or 'stderr' to compare the stdout or + stderr buffer from the most recent command. + +cmpenv [-q] file1 file2 + compare files for differences, with environment expansion + + By convention, file1 is the actual data and file2 is the + expected data. + The command succeeds if the file contents are identical + after substituting variables from the script environment. + File1 can be 'stdout' or 'stderr' to compare the script's + stdout or stderr buffer. + +cp src... dst + copy files to a target file or directory + + src can include 'stdout' or 'stderr' to copy from the + script's stdout or stderr buffer. + +echo string... + display a line of text + + +env [key[=value]...] + set or log the values of environment variables + + With no arguments, print the script environment to the log. + Otherwise, add the listed key=value pairs to the environment + or print the listed keys. + +exec program [args...] [&] + run an executable program with arguments + + Note that 'exec' does not terminate the script (unlike Unix + shells). + +exists [-readonly] [-exec] file... + check that files exist + + +go [args...] [&] + run the 'go' program provided by the script host + + +grep [-count=N] [-q] 'pattern' file + find lines in a file that match a pattern + + The command succeeds if at least one match (or the exact + count, if given) is found. + The -q flag suppresses printing of matches. + +help [-v] name... + log help text for commands and conditions + + To display help for a specific condition, enclose it in + brackets: 'help [amd64]'. + To display complete documentation when listing all commands, + pass the -v flag. + +mkdir path... + create directories, if they do not already exist + + Unlike Unix mkdir, parent directories are always created if + needed. + +mv old new + rename a file or directory to a new path + + OS-specific restrictions may apply when old and new are in + different directories. + +replace [old new]... file + replace strings in a file + + The 'old' and 'new' arguments are unquoted as if in quoted + Go strings. + +rm path... + remove a file or directory + + If the path is a directory, its contents are removed + recursively. + +skip [msg] + skip the current test + + +sleep duration [&] + sleep for a specified duration + + The duration must be given as a Go time.Duration string. + +stderr [-count=N] [-q] 'pattern' file + find lines in the stderr buffer that match a pattern + + The command succeeds if at least one match (or the exact + count, if given) is found. + The -q flag suppresses printing of matches. + +stdout [-count=N] [-q] 'pattern' file + find lines in the stdout buffer that match a pattern + + The command succeeds if at least one match (or the exact + count, if given) is found. + The -q flag suppresses printing of matches. + +stop [msg] + stop execution of the script + + The message is written to the script log, but no error is + reported from the script engine. + +symlink path -> target + create a symlink + + Creates path as a symlink to target. + The '->' token (like in 'ls -l' output on Unix) is required. + +wait + wait for completion of background commands + + Waits for all background commands to complete. + The output (and any error) from each command is printed to + the log in the order in which the commands were started. + After the call to 'wait', the script's stdout and stderr + buffers contain the concatenation of the background + commands' outputs. + + + +The available conditions are: +[GOARCH:*] + runtime.GOARCH == +[GODEBUG:*] + GODEBUG contains +[GOEXPERIMENT:*] + GOEXPERIMENT is enabled +[GOOS:*] + runtime.GOOS == +[asan] + GOOS/GOARCH supports -asan +[buildmode:*] + go supports -buildmode= +[cgo] + host CGO_ENABLED +[cgolinkext] + platform requires external linking for cgo +[compiler:*] + runtime.Compiler == +[cross] + cmd/go GOOS/GOARCH != GOHOSTOS/GOHOSTARCH +[exec:*] + names an executable in the test binary's PATH +[fuzz] + GOOS/GOARCH supports -fuzz +[fuzz-instrumented] + GOOS/GOARCH supports -fuzz with instrumentation +[go-builder] + GO_BUILDER_NAME is non-empty +[link] + testenv.HasLink() +[msan] + GOOS/GOARCH supports -msan +[mustlinkext] + platform always requires external linking +[pielinkext] + platform requires external linking for PIE +[race] + GOOS/GOARCH supports -race +[root] + os.Geteuid() == 0 +[short] + testing.Short() +[symlink] + testenv.HasSymlink() +[verbose] + testing.Verbose() + -- 2.48.1