From 37f5885f29068d869e161c4ee8ee17c3e1195cc8 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 16 Aug 2021 16:48:27 -0700 Subject: [PATCH] [dev.fuzz] internal/fuzz: fail minimization on non-reproducible crash workerServer.minimize now returns a response with Success = false when the fuzz function run with the original input does not produce an error. This may indicate flakiness. The coordinator still records a crash, but it will use the unminimized input with its original error message. When minimization of interesting inputs is supported, Success = false indicates that new coverage couldn't be reproduced, and the input will be discarded. Change-Id: I72c0e9808f0b0e5390dc7b64141cd0d653ee0af3 Reviewed-on: https://go-review.googlesource.com/c/go/+/342996 Trust: Jay Conrod Trust: Katie Hockman Reviewed-by: Katie Hockman --- .../go/testdata/script/test_fuzz_minimize.txt | 2 +- src/internal/fuzz/minimize_test.go | 51 +++++++++++++------ src/internal/fuzz/worker.go | 49 +++++++++--------- 3 files changed, 62 insertions(+), 40 deletions(-) diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize.txt b/src/cmd/go/testdata/script/test_fuzz_minimize.txt index 337059da3f..473d63ebfa 100644 --- a/src/cmd/go/testdata/script/test_fuzz_minimize.txt +++ b/src/cmd/go/testdata/script/test_fuzz_minimize.txt @@ -24,7 +24,7 @@ rm testdata ! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=10000x minimizer_test.go ! stdout '^ok' stdout 'found a crash, minimizing' -stdout 'fuzzing process terminated unexpectedly while minimizing: exit status 99' +stdout 'fuzzing process terminated unexpectedly: exit status 99' stdout FAIL # Check that re-running the value causes a crash. diff --git a/src/internal/fuzz/minimize_test.go b/src/internal/fuzz/minimize_test.go index d786cf809e..bcb0572d19 100644 --- a/src/internal/fuzz/minimize_test.go +++ b/src/internal/fuzz/minimize_test.go @@ -16,12 +16,14 @@ import ( func TestMinimizeInput(t *testing.T) { type testcase struct { + name string fn func(CorpusEntry) error input []interface{} expected []interface{} } cases := []testcase{ { + name: "ones_byte", fn: func(e CorpusEntry) error { b := e.Values[0].([]byte) ones := 0 @@ -39,6 +41,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{[]byte{1, 1, 1}}, }, { + name: "ones_string", fn: func(e CorpusEntry) error { b := e.Values[0].(string) ones := 0 @@ -56,6 +59,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{"111"}, }, { + name: "int", fn: func(e CorpusEntry) error { i := e.Values[0].(int) if i > 100 { @@ -67,6 +71,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{123}, }, { + name: "int8", fn: func(e CorpusEntry) error { i := e.Values[0].(int8) if i > 10 { @@ -78,6 +83,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{int8(12)}, }, { + name: "int16", fn: func(e CorpusEntry) error { i := e.Values[0].(int16) if i > 10 { @@ -100,6 +106,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{int32(21)}, }, { + name: "int32", fn: func(e CorpusEntry) error { i := e.Values[0].(uint) if i > 10 { @@ -111,6 +118,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{uint(12)}, }, { + name: "uint8", fn: func(e CorpusEntry) error { i := e.Values[0].(uint8) if i > 10 { @@ -122,6 +130,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{uint8(25)}, }, { + name: "uint16", fn: func(e CorpusEntry) error { i := e.Values[0].(uint16) if i > 10 { @@ -133,6 +142,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{uint16(65)}, }, { + name: "uint32", fn: func(e CorpusEntry) error { i := e.Values[0].(uint32) if i > 10 { @@ -144,6 +154,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{uint32(42)}, }, { + name: "float32", fn: func(e CorpusEntry) error { if i := e.Values[0].(float32); i == 1.23 { return nil @@ -154,6 +165,7 @@ func TestMinimizeInput(t *testing.T) { expected: []interface{}{float32(1.2)}, }, { + name: "float64", fn: func(e CorpusEntry) error { if i := e.Values[0].(float64); i == 1.23 { return nil @@ -168,6 +180,7 @@ func TestMinimizeInput(t *testing.T) { // If we are on a 64 bit platform add int64 and uint64 tests if v := int64(1<<63 - 1); int64(int(v)) == v { cases = append(cases, testcase{ + name: "int64", fn: func(e CorpusEntry) error { i := e.Values[0].(int64) if i > 10 { @@ -178,6 +191,7 @@ func TestMinimizeInput(t *testing.T) { input: []interface{}{int64(1<<63 - 1)}, expected: []interface{}{int64(92)}, }, testcase{ + name: "uint64", fn: func(e CorpusEntry) error { i := e.Values[0].(uint64) if i > 10 { @@ -191,20 +205,27 @@ func TestMinimizeInput(t *testing.T) { } for _, tc := range cases { - ws := &workerServer{ - fuzzFn: tc.fn, - } - count := int64(0) - vals := tc.input - err := ws.minimizeInput(context.Background(), vals, &count, 0) - if err == nil { - t.Error("minimizeInput didn't fail") - } - if expected := fmt.Sprintf("bad %v", tc.input[0]); err.Error() != expected { - t.Errorf("unexpected error: got %s, want %s", err, expected) - } - if !reflect.DeepEqual(vals, tc.expected) { - t.Errorf("unexpected results: got %v, want %v", vals, tc.expected) - } + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ws := &workerServer{ + fuzzFn: tc.fn, + } + count := int64(0) + vals := tc.input + success, err := ws.minimizeInput(context.Background(), vals, &count, 0) + if !success { + t.Errorf("minimizeInput did not succeed") + } + if err == nil { + t.Error("minimizeInput didn't fail") + } + if expected := fmt.Sprintf("bad %v", tc.input[0]); err.Error() != expected { + t.Errorf("unexpected error: got %s, want %s", err, expected) + } + if !reflect.DeepEqual(vals, tc.expected) { + t.Errorf("unexpected results: got %v, want %v", vals, tc.expected) + } + }) } } diff --git a/src/internal/fuzz/worker.go b/src/internal/fuzz/worker.go index 8a5c8696de..81c5157dab 100644 --- a/src/internal/fuzz/worker.go +++ b/src/internal/fuzz/worker.go @@ -196,11 +196,8 @@ func (w *worker) coordinate(ctx context.Context) error { totalDuration: resp.TotalDuration, entryDuration: resp.InterestingDuration, entry: entry, - } - if resp.Err != "" { - result.crasherMsg = resp.Err - } else if resp.CoverageData != nil { - result.coverageData = resp.CoverageData + crasherMsg: resp.Err, + coverageData: resp.CoverageData, } w.coordinator.resultC <- result @@ -208,14 +205,19 @@ func (w *worker) coordinate(ctx context.Context) error { // Received input to minimize from coordinator. result, err := w.minimize(ctx, input) if err != nil { - // Failed to minimize. Send back the original crash. - fmt.Fprintln(w.coordinator.opts.Log, err) + // Error minimizing. Send back the original input. If it didn't cause + // an error before, report it as causing an error now. + // TODO(fuzz): double-check this is handled correctly when + // implementing -keepfuzzing. result = fuzzResult{ entry: input.entry, crasherMsg: input.crasherMsg, minimizeAttempted: true, limit: input.limit, } + if result.crasherMsg == "" { + result.crasherMsg = err.Error() + } } w.coordinator.resultC <- result } @@ -223,11 +225,9 @@ func (w *worker) coordinate(ctx context.Context) error { } // minimize tells a worker process to attempt to find a smaller value that -// causes an error. minimize may restart the worker repeatedly if the error -// causes (or already caused) the worker process to terminate. -// -// TODO: support minimizing inputs that expand coverage in a specific way, -// for example, by ensuring that an input activates a specific set of counters. +// either causes an error (if we started minimizing because we found an input +// that causes an error) or preserves new coverage (if we started minimizing +// because we found an input that expands coverage). func (w *worker) minimize(ctx context.Context, input fuzzMinimizeInput) (min fuzzResult, err error) { if w.coordinator.opts.MinimizeTimeout != 0 { var cancel func() @@ -261,10 +261,10 @@ func (w *worker) minimize(ctx context.Context, input fuzzMinimizeInput) (min fuz return fuzzResult{}, fmt.Errorf("fuzzing process terminated unexpectedly while minimizing: %w", w.waitErr) } - if resp.Err == "" { - // Minimization did not find a smaller input that caused a crash. - return min, nil + if input.crasherMsg != "" && resp.Err == "" && !resp.Success { + return fuzzResult{}, fmt.Errorf("attempted to minimize but could not reproduce") } + min.crasherMsg = resp.Err min.count = resp.Count min.totalDuration = resp.Duration @@ -498,9 +498,11 @@ type minimizeArgs struct { // minimizeResponse contains results from workerServer.minimize. type minimizeResponse struct { - // Err is the error string caused by the value in shared memory. - // If Err is empty, minimize was unable to find any shorter values that - // caused errors, and the value in shared memory is the original value. + // Success is true if the worker found a smaller input, stored in shared + // memory, that was "interesting" for the same reason as the original input. + Success bool + + // Err is the error string caused by the value in shared memory, if any. Err string // Duration is the time spent minimizing, not including starting or cleaning up. @@ -734,7 +736,7 @@ func (ws *workerServer) minimize(ctx context.Context, args minimizeArgs) (resp m // Minimize the values in vals, then write to shared memory. We only write // to shared memory after completing minimization. If the worker terminates // unexpectedly before then, the coordinator will use the original input. - err = ws.minimizeInput(ctx, vals, &mem.header().count, args.Limit) + resp.Success, err = ws.minimizeInput(ctx, vals, &mem.header().count, args.Limit) writeToMem(vals, mem) if err != nil { resp.Err = err.Error() @@ -748,16 +750,15 @@ func (ws *workerServer) minimize(ctx context.Context, args minimizeArgs) (resp m // mem just in case an unrecoverable error occurs. It uses the context to // determine how long to run, stopping once closed. It returns the last error it // found. -func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, count *int64, limit int64) error { +func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, count *int64, limit int64) (success bool, retErr error) { shouldStop := func() bool { return ctx.Err() != nil || (limit > 0 && *count >= limit) } if shouldStop() { - return nil + return false, nil } var valI int - var retErr error tryMinimized := func(candidate interface{}) bool { prev := vals[valI] // Set vals[valI] to the candidate after it has been @@ -822,7 +823,7 @@ func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, c for valI = range vals { if shouldStop() { - return retErr + break } switch v := vals[valI].(type) { case bool: @@ -869,7 +870,7 @@ func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, c panic("unreachable") } } - return retErr + return retErr != nil, retErr } func writeToMem(vals []interface{}, mem *sharedMem) { -- 2.50.0