From aea29a9016cb5c3e160f94e6a95b197407de8c2c Mon Sep 17 00:00:00 2001 From: Katie Hockman Date: Fri, 18 Sep 2020 10:13:23 -0400 Subject: [PATCH] [dev.fuzz] testing: implement F.Fuzz to run seed corpus Change-Id: Ibd204a5d0596c4f8acf598289055c17a836d9023 Reviewed-on: https://go-review.googlesource.com/c/go/+/255957 Run-TryBot: Katie Hockman TryBot-Result: Go Bot Trust: Katie Hockman Trust: Jay Conrod Reviewed-by: Jay Conrod --- src/cmd/go/testdata/script/test_fuzz.txt | 122 +++++++++++++++++++++++ src/testing/fuzz.go | 86 +++++++++++++--- 2 files changed, 192 insertions(+), 16 deletions(-) diff --git a/src/cmd/go/testdata/script/test_fuzz.txt b/src/cmd/go/testdata/script/test_fuzz.txt index 24350ee450..ab93f5cc2e 100644 --- a/src/cmd/go/testdata/script/test_fuzz.txt +++ b/src/cmd/go/testdata/script/test_fuzz.txt @@ -18,6 +18,12 @@ go test -fuzz Fuzz success_fuzz_test.go stdout ok ! stdout FAIL +# Test error with seed corpus in f.Fuzz +! go test -run FuzzError fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'error here' + [short] stop # Test that calling panic(nil) in a fuzz target causes a non-zero exit status. @@ -30,6 +36,39 @@ go test skipped_fuzz_test.go stdout ok ! stdout FAIL +# Test that multiple calls to f.Fuzz causes a non-zero exit status. +! go test multiple_fuzz_calls_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test that missing *T in f.Fuzz causes a non-zero exit status. +! go test incomplete_fuzz_call_fuzz_test.go +! stdout ^ok +stdout FAIL + +# Test success with seed corpus in f.Fuzz +go test -run FuzzPass fuzz_add_test.go +stdout ok +! stdout FAIL +! stdout 'off by one error' + +# Test fatal with seed corpus in f.Fuzz +! go test -run FuzzFatal fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'fatal here' + +# Test panic with seed corpus in f.Fuzz +! go test -run FuzzPanic fuzz_add_test.go +! stdout ^ok +stdout FAIL +stdout 'off by one error' + +# Test panic(nil) with seed corpus in f.Fuzz +! go test -run FuzzNilPanic fuzz_add_test.go +! stdout ^ok +stdout FAIL + -- fail_fuzz_test.go -- package fail_fuzz @@ -64,3 +103,86 @@ import "testing" func Fuzz(f *testing.F) { f.Skip() } + +-- multiple_fuzz_calls_fuzz_test.go -- +package multiple_fuzz_calls_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + // no-op + }) + + f.Fuzz(func(t *testing.T, b []byte) { + // this second call should panic + }) +} + +-- incomplete_fuzz_call_fuzz_test.go -- +package incomplete_fuzz_call_fuzz + +import "testing" + +func Fuzz(f *testing.F) { + f.Fuzz(func(b []byte) { + // this is missing *testing.T as the first param, so should panic + }) +} + +-- fuzz_add_test.go -- +package fuzz_add + +import "testing" + +func add(f *testing.F) { + f.Helper() + f.Add([]byte("123")) + f.Add([]byte("12345")) + f.Add([]byte("")) +} + +func FuzzPass(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == -1 { + t.Fatal("fatal here") // will not be executed + } + }) +} + +func FuzzError(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 3 { + t.Error("error here") + } + }) +} + +func FuzzFatal(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 0 { + t.Fatal("fatal here") + } + }) +} + +func FuzzPanic(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 5 { + panic("off by one error") + } + }) +} + +func FuzzNilPanic(f *testing.F) { + add(f) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) == 3 { + panic(nil) + } + }) +} \ No newline at end of file diff --git a/src/testing/fuzz.go b/src/testing/fuzz.go index d159f2e425..6f985c7c38 100644 --- a/src/testing/fuzz.go +++ b/src/testing/fuzz.go @@ -5,6 +5,7 @@ package testing import ( + "errors" "flag" "fmt" "os" @@ -27,11 +28,12 @@ type InternalFuzzTarget struct { // F is a type passed to fuzz targets for fuzz testing. type F struct { common - context *fuzzContext - corpus []corpusEntry // corpus is the in-memory corpus - result FuzzResult // result is the result of running the fuzz target - fuzzFunc func(f *F) // fuzzFunc is the function which makes up the fuzz target - fuzz bool // fuzz indicates whether or not the fuzzing engine should run + context *fuzzContext + corpus []corpusEntry // corpus is the in-memory corpus + result FuzzResult // result is the result of running the fuzz target + fuzzFunc func(f *F) // fuzzFunc is the function which makes up the fuzz target + fuzz bool // fuzz indicates whether the fuzzing engine should run + fuzzCalled bool // fuzzCalled indicates whether f.Fuzz has been called for this target } // corpus corpusEntry @@ -60,21 +62,73 @@ func (f *F) Add(args ...interface{}) { } // Fuzz runs the fuzz function, ff, for fuzz testing. It runs ff in a separate -// goroutine. Only one call to Fuzz is allowed per fuzz target, and any -// subsequent calls will panic. If ff fails for a set of arguments, those -// arguments will be added to the seed corpus. +// goroutine. Only the first call to Fuzz will be executed, and any subsequent +// calls will panic. If ff fails for a set of arguments, those arguments will be +// added to the seed corpus. func (f *F) Fuzz(ff interface{}) { - return + if f.fuzzCalled { + panic("testing: found more than one call to Fuzz, will skip") + } + f.fuzzCalled = true + + fn, ok := ff.(func(*T, []byte)) + if !ok { + panic("testing: Fuzz function must have type func(*testing.T, []byte)") + } + + var errStr string + run := func(t *T, b []byte) { + defer func() { + err := recover() + // If the function has recovered but the test hasn't finished, + // it is due to a nil panic or runtime.GoExit. + if !t.finished && err == nil { + err = errNilPanicOrGoexit + } + if err != nil { + t.Fail() + t.output = []byte(fmt.Sprintf(" panic: %s\n", err)) + } + f.setRan() + t.signal <- true // signal that the test has finished + }() + fn(t, b) + t.finished = true + } + + // Run the seed corpus first + for _, c := range f.corpus { + t := &T{ + common: common{ + signal: make(chan bool), + w: f.w, + chatty: f.chatty, + }, + context: newTestContext(1, nil), + } + go run(t, c.b) + <-t.signal + if t.Failed() { + f.Fail() + errStr += string(t.output) + } + } + if f.Failed() { + f.result = FuzzResult{Error: errors.New(errStr)} + return + } + + // TODO: if f.fuzz is set, run fuzzing engine } -func (f *F) report(name string) { +func (f *F) report() { if f.Failed() { - fmt.Fprintf(f.w, "--- FAIL: %s\n%s\n", name, f.result.String()) + fmt.Fprintf(f.w, "--- FAIL: %s\n%s\n", f.name, f.result.String()) } else if f.chatty != nil { if f.Skipped() { - f.chatty.Updatef(name, "SKIP\n") + f.chatty.Updatef(f.name, "SKIP\n") } else { - f.chatty.Updatef(name, "PASS\n") + f.chatty.Updatef(f.name, "PASS\n") } } } @@ -100,7 +154,7 @@ func (f *F) run(name string, fn func(f *F)) (ran, ok bool) { // runTarget runs the given target, handling panics and exits // within the test, and reporting errors. -func (f *F) runTarget(fn func(f *F)) { +func (f *F) runTarget(fn func(*F)) { defer func() { err := recover() // If the function has recovered but the test hasn't finished, @@ -112,7 +166,7 @@ func (f *F) runTarget(fn func(f *F)) { f.Fail() f.result = FuzzResult{Error: fmt.Errorf("%s", err)} } - f.report(f.name) + f.report() f.setRan() f.signal <- true // signal that the test has finished }() @@ -133,7 +187,7 @@ func (r FuzzResult) String() string { if r.Error == nil { return s } - s = fmt.Sprintf("error: %s", r.Error.Error()) + s = fmt.Sprintf("%s", r.Error.Error()) if r.Crasher != nil { s += fmt.Sprintf("\ncrasher: %b", r.Crasher) } -- 2.48.1