From: Katie Hockman Date: Tue, 7 Dec 2021 18:50:51 +0000 (-0500) Subject: cmd/go: fix flaky test X-Git-Tag: go1.18beta1~63 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=daf901810553b0ccdd9562523ecfad7d11e9b001;p=gostls13.git cmd/go: fix flaky test Change-Id: I641c7b8bcf8b9a8f0637995b26eea0fbe2900ef9 Reviewed-on: https://go-review.googlesource.com/c/go/+/369978 Reviewed-by: Roland Shoemaker Reviewed-by: Bryan Mills Trust: Katie Hockman Run-TryBot: Katie Hockman TryBot-Result: Gopher Robot --- diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt index c9b04d02ea..5d0de17f6b 100644 --- a/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt +++ b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt @@ -1,3 +1,4 @@ +[short] skip [!fuzz-instrumented] skip # Test that when an interesting value is discovered (one that expands coverage), @@ -15,7 +16,7 @@ go test -c -fuzz=. # Build using shared build cache for speed. env GOCACHE=$WORK/gocache exec ./fuzz.test$GOEXE -test.fuzzcachedir=$GOCACHE/fuzz -test.fuzz=FuzzMinCache -test.fuzztime=1000x -go run check_cache.go $GOCACHE/fuzz/FuzzMinCache +go run check_cache/check_cache.go $GOCACHE/fuzz/FuzzMinCache go test -c -fuzz=. # Build using shared build cache for speed. env GOCACHE=$WORK/gocache @@ -25,12 +26,12 @@ env GOCACHE=$WORK/gocache # be flaky like we want. ! exec ./fuzz.test$GOEXE -test.fuzzcachedir=$GOCACHE/fuzz -test.fuzz=FuzzMinimizerCrashInMinimization -test.run=FuzzMinimizerCrashInMinimization -test.fuzztime=10000x -test.parallel=1 ! stdout '^ok' -stdout 'got the minimum size!' +stdout -count=1 'got the minimum size!' stdout -count=1 'flaky failure' stdout FAIL - -# Make sure the crash that was written will fail when run with go test -! go test -run=FuzzMinimizerCrashInMinimization . +# Check that the input written to testdata will reproduce the error, and is the +# smallest possible. +go run check_testdata/check_testdata.go FuzzMinimizerCrashInMinimization 50 # Test that a nonrecoverable error that occurs while minimizing an interesting # input is reported correctly. @@ -39,9 +40,8 @@ stdout FAIL stdout -count=1 'fuzzing process hung or terminated unexpectedly while minimizing' stdout -count=1 'EOF' stdout FAIL - -# Make sure the crash that was written will fail when run with go test -! go test -run=FuzzMinimizerNonrecoverableCrashInMinimization . +# Check that the input written to testdata will reproduce the error. +go run check_testdata/check_testdata.go FuzzMinimizerNonrecoverableCrashInMinimization 100 -- go.mod -- module fuzz @@ -55,8 +55,8 @@ import ( "io" ) -func Y(w io.Writer, b []byte) { - if !bytes.Equal(b, []byte("y")) { +func Y(w io.Writer, s string) { + if !bytes.Equal([]byte(s), []byte("y")) { w.Write([]byte("not equal")) } } @@ -67,45 +67,54 @@ import ( "bytes" "io" "os" + "strings" "testing" + "unicode/utf8" ) func FuzzMinimizerCrashInMinimization(f *testing.F) { - seed := make([]byte, 1000) + seed := strings.Repeat("A", 1000) f.Add(seed) - f.Fuzz(func(t *testing.T, b []byte) { - if len(b) < 50 || len(b) > 1100 { + i := 3 + f.Fuzz(func(t *testing.T, s string) { + if len(s) < 50 || len(s) > 1100 { // Make sure that b is large enough that it can be minimized return } - if !bytes.Equal(b, seed) { - // This should have hit a new edge, and the interesting input - // should be attempting minimization - Y(io.Discard, b) + if s != seed { + // This should hit a new edge, and the interesting input + // should attempt minimization + Y(io.Discard, s) } - if len(b) < 55 { + if i > 0 { + // Don't let it fail right away. + i-- + } else if utf8.RuneCountInString(s) == len(s) && len(s) <= 100 { + // Make sure this only fails if the number of bytes in the + // marshaled string is the same as the unmarshaled string, + // so that we can check the length of the testdata file. t.Error("flaky failure") - } - if len(b) == 50 { - t.Log("got the minimum size!") + if len(s) == 50 { + t.Error("got the minimum size!") + } } }) } func FuzzMinimizerNonrecoverableCrashInMinimization(f *testing.F) { - seed := make([]byte, 1000) + seed := strings.Repeat("A", 1000) f.Add(seed) - f.Fuzz(func(t *testing.T, b []byte) { - if len(b) < 50 || len(b) > 1100 { - // Make sure that b is large enough that it can be minimized + i := 3 + f.Fuzz(func(t *testing.T, s string) { + if len(s) < 50 || len(s) > 1100 { return } - if !bytes.Equal(b, seed) { - // This should have hit a new edge, and the interesting input - // should be attempting minimization - Y(io.Discard, b) + if s != seed { + Y(io.Discard, s) } - if len(b) < 55 { + if i > 0 { + i-- + } else if utf8.RuneCountInString(s) == len(s) && len(s) <= 100 { os.Exit(19) } }) @@ -131,7 +140,59 @@ func sum(buf []byte) int { } return n } --- check_cache.go -- +-- check_testdata/check_testdata.go -- +//go:build ignore +// +build ignore + +// check_testdata.go checks that the string written +// is not longer than the provided length. +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" +) + +func main() { + wantLen, err := strconv.Atoi(os.Args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + testName := os.Args[1] + dir := filepath.Join("testdata/fuzz", testName) + + files, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + if len(files) == 0 { + fmt.Fprintf(os.Stderr, "expect at least one failure to be written to testdata\n") + os.Exit(1) + } + + fname := files[0].Name() + contents, err := ioutil.ReadFile(filepath.Join(dir, fname)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + contentsLen := len(contents) - len(`go test fuzz v1 +string("") +`) + if got, want := contentsLen, wantLen; got > want { + fmt.Fprintf(os.Stderr, "expect length <= %d, got %d\n", want, got) + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "%s\n", contents) +} + +-- check_cache/check_cache.go -- //go:build ignore // +build ignore