From: Jay Conrod Date: Tue, 1 Jun 2021 16:49:50 +0000 (-0400) Subject: [dev.fuzz] cmd/go: split test_fuzz_minimize from test_fuzz_mutator X-Git-Tag: go1.18beta1~1282^2~48 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=a3b5084204a69e441fa1cc4162a14d93c3507d54;p=gostls13.git [dev.fuzz] cmd/go: split test_fuzz_minimize from test_fuzz_mutator This CL moves test cases for minimization into a new script. The tests are changed to be more deterministic. Any non-zero byte now triggers a failure. Change-Id: Id45a721071dbfc3aec46be369966b0a67a745c55 Reviewed-on: https://go-review.googlesource.com/c/go/+/323851 Trust: Jay Conrod Trust: Katie Hockman Run-TryBot: Jay Conrod Reviewed-by: Katie Hockman TryBot-Result: Go Bot --- diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize.txt b/src/cmd/go/testdata/script/test_fuzz_minimize.txt new file mode 100644 index 0000000000..7652759668 --- /dev/null +++ b/src/cmd/go/testdata/script/test_fuzz_minimize.txt @@ -0,0 +1,159 @@ +# TODO(jayconrod): support shared memory on more platforms. +[!darwin] [!linux] [!windows] skip + +[short] skip + +# We clean the fuzz cache during this test. Don't clean the user's cache. +env GOCACHE=$WORK/gocache + +# TODO(b/181800488): remove -parallel=1, here and below. For now, when a +# crash is found, all workers keep running, wasting resources and reducing +# the number of executions available to the minimizer, increasing flakiness. + +# Test that minimization is working for recoverable errors. +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go +! stdout '^ok' +stdout 'got the minimum size!' +stdout 'contains a non-zero byte' +stdout FAIL + +# Check that the bytes written to testdata are of length 50 (the minimum size) +go run check_testdata.go FuzzMinimizerRecoverable 50 + +# Test that re-running the minimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable minimizer_test.go +rm testdata + +# Test that minimization is working for non-recoverable errors. +! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go +! stdout '^ok' +# TODO(jayconrod): fix in CL 321835. +stdout 'found a crash, minimizing' +# stdout 'fuzzing process terminated unexpectedly while minimizing: exit status 99' +stdout FAIL + +# Check that re-running the value causes a crash. +! go test -run=FuzzMinimizerNonrecoverable minimizer_test.go +rm testdata + +# Clear the fuzzing cache. There may already be minimized inputs that would +# interfere with the next stage of the test. +go clean -fuzzcache + +# Test that minimization can be cancelled by fuzzminimizetime and the latest +# crash will still be logged and written to testdata. +! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -parallel=1 -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go +! stdout '^ok' +stdout 'testdata[/\\]corpus[/\\]FuzzMinimizerRecoverable[/\\]' +# TODO(jayconrod): implement -fuzzminimizetime in 321835. +# ! stdout 'got the minimum size!' # it shouldn't have had enough time to minimize it +stdout FAIL + +# Test that re-running the unminimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable minimizer_test.go + +# TODO(jayconrod,katiehockman): add a test which verifies that the right bytes +# are written to testdata in the case of an interrupt during minimization. + +-- go.mod -- +module m + +go 1.16 +-- minimizer_test.go -- +package fuzz_test + +import ( + "os" + "testing" +) + +func FuzzMinimizerRecoverable(f *testing.F) { + f.Add(make([]byte, 100)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) < 50 { + // Make sure that b is large enough that it can be minimized + return + } + // Given the randomness of the mutations, this should allow the + // minimizer to trim down the value a bit. + for _, n := range b { + if n != 0 { + if len(b) == 50 { + t.Log("got the minimum size!") + } + t.Fatal("contains a non-zero byte") + } + } + }) +} + +func FuzzMinimizerNonrecoverable(f *testing.F) { + f.Add(make([]byte, 100)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) < 50 { + // Make sure that b is large enough that it can be minimized + return + } + // Given the randomness of the mutations, this should allow the + // minimizer to trim down the value a bit. + for _, n := range b { + if n != 0 { + t.Log("contains a non-zero byte") + os.Exit(99) + } + } + }) +} +-- check_testdata.go -- +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" +) + +func main() { + target := os.Args[1] + numBytes, err := strconv.Atoi(os.Args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Open the file in testdata (there should only be one) + dir := fmt.Sprintf("testdata/corpus/%s", target) + files, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(files) != 1 { + fmt.Fprintf(os.Stderr, "expected one file, got %d", len(files)) + os.Exit(1) + } + got, err := ioutil.ReadFile(filepath.Join(dir, files[0].Name())) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Make sure that there were exactly 100 bytes written to the corpus entry + prefix := []byte("[]byte(") + i := bytes.Index(got, prefix) + gotBytes := got[i+len(prefix) : len(got)-1] + s, err := strconv.Unquote(string(gotBytes)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if want, got := numBytes, len(s); want != got { + fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got) + os.Exit(1) + } +} diff --git a/src/cmd/go/testdata/script/test_fuzz_mutator.txt b/src/cmd/go/testdata/script/test_fuzz_mutator.txt index 1d0c399a6c..fb7984c662 100644 --- a/src/cmd/go/testdata/script/test_fuzz_mutator.txt +++ b/src/cmd/go/testdata/script/test_fuzz_mutator.txt @@ -10,11 +10,12 @@ [short] skip +go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz +go run check_logs.go fuzz fuzz.worker + # TODO(b/181800488): remove -parallel=1, here and below. For now, when a # crash is found, all workers keep running, wasting resources and reducing # the number of executions available to the minimizer, increasing flakiness. -go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz -go run check_logs.go fuzz fuzz.worker # Test that the mutator is good enough to find several unique mutations. ! go test -fuzz=FuzzMutator -parallel=1 -fuzztime=100x mutator_test.go @@ -22,41 +23,6 @@ go run check_logs.go fuzz fuzz.worker stdout FAIL stdout 'mutator found enough unique mutations' -# Test that minimization is working for recoverable errors. -! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go -! stdout '^ok' -stdout 'got the minimum size!' -stdout 'contains a letter' -stdout FAIL - -# Check that the bytes written to testdata are of length 50 (the minimum size) -go run check_testdata.go FuzzMinimizerRecoverable 50 - -# Test that re-running the minimized value causes a crash. -! go test -run=FuzzMinimizerRecoverable minimizer_test.go - -# Test that minimization is working for non-recoverable errors. -! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go -! stdout '^ok' -stdout 'got the minimum size!' -stdout 'contains a letter' -stdout FAIL - -# Check that the bytes written to testdata are of length 50 (the minimum size) -go run check_testdata.go FuzzMinimizerNonrecoverable 50 - -# Test that minimization can be cancelled by fuzzminimizetime and the latest -# crash will still be logged and written to testdata. -! go test -fuzz=FuzzNonMinimizable -run=FuzzNonMinimizable -parallel=1 -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go -! stdout '^ok' -stdout 'testdata[/\\]corpus[/\\]FuzzNonMinimizable[/\\]' -! stdout 'got the minimum size!' # it shouldn't have had enough time to minimize it -stdout 'at least 20 bytes' -stdout FAIL - -# TODO(jayconrod,katiehockman): add a test which verifies that the right bytes -# are written to testdata in the case of an interrupt during minimization. - -- go.mod -- module m @@ -105,63 +71,6 @@ func FuzzB(f *testing.F) { }) } --- minimizer_test.go -- -package fuzz_test - -import ( - "bytes" - "fmt" - "os" - "testing" -) - -func FuzzMinimizerRecoverable(f *testing.F) { - f.Fuzz(func(t *testing.T, b []byte) { - if len(b) < 50 { - // Make sure that b is large enough that it can be minimized - return - } - // Given the randomness of the mutations, this should allow the - // minimizer to trim down the value a bit. - if bytes.ContainsAny(b, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") { - if len(b) == 50 { - fmt.Fprint(os.Stderr, "got the minimum size!\n") - } - t.Error("contains a letter") - } - }) -} - -func FuzzMinimizerNonrecoverable(f *testing.F) { - f.Fuzz(func(t *testing.T, b []byte) { - if len(b) < 50 { - // Make sure that b is large enough that it can be minimized - return - } - // Given the randomness of the mutations, this should allow the - // minimizer to trim down the value quite a bit. - if bytes.ContainsAny(b, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") { - if len(b) == 50 { - fmt.Fprint(os.Stderr, "got the minimum size!\n") - } - panic("contains a letter") - } - }) -} - -func FuzzNonMinimizable(f *testing.F) { - f.Fuzz(func(t *testing.T, b []byte) { - if len(b) < 20 { - // Make sure that b is large enough that minimization will try to run. - return - } - panic("at least 20 bytes") - if len(b) == 20 { - fmt.Fprint(os.Stderr, "got the minimum size!\n") - } - }) -} - -- check_logs.go -- // +build ignore @@ -255,59 +164,6 @@ func checkWorkerLog(r io.Reader) error { } return nil } --- check_testdata.go -- -// +build ignore - -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" -) - -func main() { - target := os.Args[1] - numBytes, err := strconv.Atoi(os.Args[2]) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - - // Open the file in testdata (there should only be one) - dir := fmt.Sprintf("testdata/corpus/%s", target) - files, err := ioutil.ReadDir(dir) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - if len(files) != 1 { - fmt.Fprintf(os.Stderr, "expected one file, got %d", len(files)) - os.Exit(1) - } - got, err := ioutil.ReadFile(filepath.Join(dir, files[0].Name())) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - - // Make sure that there were exactly 100 bytes written to the corpus entry - prefix := []byte("[]byte(") - i := bytes.Index(got, prefix) - gotBytes := got[i+len(prefix) : len(got)-1] - s, err := strconv.Unquote(string(gotBytes)) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - if want, got := numBytes, len(s); want != got { - fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got) - os.Exit(1) - } -} -- mutator_test.go -- package fuzz_test