]> Cypherpunks repositories - gostls13.git/commitdiff
[dev.fuzz] internal/fuzz: fail minimization on non-reproducible crash
authorJay Conrod <jayconrod@google.com>
Mon, 16 Aug 2021 23:48:27 +0000 (16:48 -0700)
committerJay Conrod <jayconrod@google.com>
Tue, 31 Aug 2021 20:39:19 +0000 (20:39 +0000)
workerServer.minimize now returns a response with Success = false when
the fuzz function run with the original input does not produce an
error. This may indicate flakiness.

The coordinator still records a crash, but it will use the unminimized
input with its original error message.

When minimization of interesting inputs is supported, Success = false
indicates that new coverage couldn't be reproduced, and the input will
be discarded.

Change-Id: I72c0e9808f0b0e5390dc7b64141cd0d653ee0af3
Reviewed-on: https://go-review.googlesource.com/c/go/+/342996
Trust: Jay Conrod <jayconrod@google.com>
Trust: Katie Hockman <katie@golang.org>
Reviewed-by: Katie Hockman <katie@golang.org>
src/cmd/go/testdata/script/test_fuzz_minimize.txt
src/internal/fuzz/minimize_test.go
src/internal/fuzz/worker.go

index 337059da3fbad8a747e28c6466d44a5d4d13dd7e..473d63ebfa9b1092b104f6530c7f75372afceb81 100644 (file)
@@ -24,7 +24,7 @@ rm testdata
 ! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=10000x minimizer_test.go
 ! stdout '^ok'
 stdout 'found a crash, minimizing'
-stdout 'fuzzing process terminated unexpectedly while minimizing: exit status 99'
+stdout 'fuzzing process terminated unexpectedly: exit status 99'
 stdout FAIL
 
 # Check that re-running the value causes a crash.
index d786cf809e5eff446d5f51a99f1f379ecefd112f..bcb0572d1978d3ba64a5df96f0df3b045ba88147 100644 (file)
@@ -16,12 +16,14 @@ import (
 
 func TestMinimizeInput(t *testing.T) {
        type testcase struct {
+               name     string
                fn       func(CorpusEntry) error
                input    []interface{}
                expected []interface{}
        }
        cases := []testcase{
                {
+                       name: "ones_byte",
                        fn: func(e CorpusEntry) error {
                                b := e.Values[0].([]byte)
                                ones := 0
@@ -39,6 +41,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{[]byte{1, 1, 1}},
                },
                {
+                       name: "ones_string",
                        fn: func(e CorpusEntry) error {
                                b := e.Values[0].(string)
                                ones := 0
@@ -56,6 +59,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{"111"},
                },
                {
+                       name: "int",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(int)
                                if i > 100 {
@@ -67,6 +71,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{123},
                },
                {
+                       name: "int8",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(int8)
                                if i > 10 {
@@ -78,6 +83,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{int8(12)},
                },
                {
+                       name: "int16",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(int16)
                                if i > 10 {
@@ -100,6 +106,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{int32(21)},
                },
                {
+                       name: "int32",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(uint)
                                if i > 10 {
@@ -111,6 +118,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{uint(12)},
                },
                {
+                       name: "uint8",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(uint8)
                                if i > 10 {
@@ -122,6 +130,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{uint8(25)},
                },
                {
+                       name: "uint16",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(uint16)
                                if i > 10 {
@@ -133,6 +142,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{uint16(65)},
                },
                {
+                       name: "uint32",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(uint32)
                                if i > 10 {
@@ -144,6 +154,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{uint32(42)},
                },
                {
+                       name: "float32",
                        fn: func(e CorpusEntry) error {
                                if i := e.Values[0].(float32); i == 1.23 {
                                        return nil
@@ -154,6 +165,7 @@ func TestMinimizeInput(t *testing.T) {
                        expected: []interface{}{float32(1.2)},
                },
                {
+                       name: "float64",
                        fn: func(e CorpusEntry) error {
                                if i := e.Values[0].(float64); i == 1.23 {
                                        return nil
@@ -168,6 +180,7 @@ func TestMinimizeInput(t *testing.T) {
        // If we are on a 64 bit platform add int64 and uint64 tests
        if v := int64(1<<63 - 1); int64(int(v)) == v {
                cases = append(cases, testcase{
+                       name: "int64",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(int64)
                                if i > 10 {
@@ -178,6 +191,7 @@ func TestMinimizeInput(t *testing.T) {
                        input:    []interface{}{int64(1<<63 - 1)},
                        expected: []interface{}{int64(92)},
                }, testcase{
+                       name: "uint64",
                        fn: func(e CorpusEntry) error {
                                i := e.Values[0].(uint64)
                                if i > 10 {
@@ -191,20 +205,27 @@ func TestMinimizeInput(t *testing.T) {
        }
 
        for _, tc := range cases {
-               ws := &workerServer{
-                       fuzzFn: tc.fn,
-               }
-               count := int64(0)
-               vals := tc.input
-               err := ws.minimizeInput(context.Background(), vals, &count, 0)
-               if err == nil {
-                       t.Error("minimizeInput didn't fail")
-               }
-               if expected := fmt.Sprintf("bad %v", tc.input[0]); err.Error() != expected {
-                       t.Errorf("unexpected error: got %s, want %s", err, expected)
-               }
-               if !reflect.DeepEqual(vals, tc.expected) {
-                       t.Errorf("unexpected results: got %v, want %v", vals, tc.expected)
-               }
+               tc := tc
+               t.Run(tc.name, func(t *testing.T) {
+                       t.Parallel()
+                       ws := &workerServer{
+                               fuzzFn: tc.fn,
+                       }
+                       count := int64(0)
+                       vals := tc.input
+                       success, err := ws.minimizeInput(context.Background(), vals, &count, 0)
+                       if !success {
+                               t.Errorf("minimizeInput did not succeed")
+                       }
+                       if err == nil {
+                               t.Error("minimizeInput didn't fail")
+                       }
+                       if expected := fmt.Sprintf("bad %v", tc.input[0]); err.Error() != expected {
+                               t.Errorf("unexpected error: got %s, want %s", err, expected)
+                       }
+                       if !reflect.DeepEqual(vals, tc.expected) {
+                               t.Errorf("unexpected results: got %v, want %v", vals, tc.expected)
+                       }
+               })
        }
 }
index 8a5c8696de42003732989c59827125ba4daba62d..81c5157dab6bfeaa71d90a448b5c897eb6a5a538 100644 (file)
@@ -196,11 +196,8 @@ func (w *worker) coordinate(ctx context.Context) error {
                                totalDuration: resp.TotalDuration,
                                entryDuration: resp.InterestingDuration,
                                entry:         entry,
-                       }
-                       if resp.Err != "" {
-                               result.crasherMsg = resp.Err
-                       } else if resp.CoverageData != nil {
-                               result.coverageData = resp.CoverageData
+                               crasherMsg:    resp.Err,
+                               coverageData:  resp.CoverageData,
                        }
                        w.coordinator.resultC <- result
 
@@ -208,14 +205,19 @@ func (w *worker) coordinate(ctx context.Context) error {
                        // Received input to minimize from coordinator.
                        result, err := w.minimize(ctx, input)
                        if err != nil {
-                               // Failed to minimize. Send back the original crash.
-                               fmt.Fprintln(w.coordinator.opts.Log, err)
+                               // Error minimizing. Send back the original input. If it didn't cause
+                               // an error before, report it as causing an error now.
+                               // TODO(fuzz): double-check this is handled correctly when
+                               // implementing -keepfuzzing.
                                result = fuzzResult{
                                        entry:             input.entry,
                                        crasherMsg:        input.crasherMsg,
                                        minimizeAttempted: true,
                                        limit:             input.limit,
                                }
+                               if result.crasherMsg == "" {
+                                       result.crasherMsg = err.Error()
+                               }
                        }
                        w.coordinator.resultC <- result
                }
@@ -223,11 +225,9 @@ func (w *worker) coordinate(ctx context.Context) error {
 }
 
 // minimize tells a worker process to attempt to find a smaller value that
-// causes an error. minimize may restart the worker repeatedly if the error
-// causes (or already caused) the worker process to terminate.
-//
-// TODO: support minimizing inputs that expand coverage in a specific way,
-// for example, by ensuring that an input activates a specific set of counters.
+// either causes an error (if we started minimizing because we found an input
+// that causes an error) or preserves new coverage (if we started minimizing
+// because we found an input that expands coverage).
 func (w *worker) minimize(ctx context.Context, input fuzzMinimizeInput) (min fuzzResult, err error) {
        if w.coordinator.opts.MinimizeTimeout != 0 {
                var cancel func()
@@ -261,10 +261,10 @@ func (w *worker) minimize(ctx context.Context, input fuzzMinimizeInput) (min fuz
                return fuzzResult{}, fmt.Errorf("fuzzing process terminated unexpectedly while minimizing: %w", w.waitErr)
        }
 
-       if resp.Err == "" {
-               // Minimization did not find a smaller input that caused a crash.
-               return min, nil
+       if input.crasherMsg != "" && resp.Err == "" && !resp.Success {
+               return fuzzResult{}, fmt.Errorf("attempted to minimize but could not reproduce")
        }
+
        min.crasherMsg = resp.Err
        min.count = resp.Count
        min.totalDuration = resp.Duration
@@ -498,9 +498,11 @@ type minimizeArgs struct {
 
 // minimizeResponse contains results from workerServer.minimize.
 type minimizeResponse struct {
-       // Err is the error string caused by the value in shared memory.
-       // If Err is empty, minimize was unable to find any shorter values that
-       // caused errors, and the value in shared memory is the original value.
+       // Success is true if the worker found a smaller input, stored in shared
+       // memory, that was "interesting" for the same reason as the original input.
+       Success bool
+
+       // Err is the error string caused by the value in shared memory, if any.
        Err string
 
        // Duration is the time spent minimizing, not including starting or cleaning up.
@@ -734,7 +736,7 @@ func (ws *workerServer) minimize(ctx context.Context, args minimizeArgs) (resp m
        // Minimize the values in vals, then write to shared memory. We only write
        // to shared memory after completing minimization. If the worker terminates
        // unexpectedly before then, the coordinator will use the original input.
-       err = ws.minimizeInput(ctx, vals, &mem.header().count, args.Limit)
+       resp.Success, err = ws.minimizeInput(ctx, vals, &mem.header().count, args.Limit)
        writeToMem(vals, mem)
        if err != nil {
                resp.Err = err.Error()
@@ -748,16 +750,15 @@ func (ws *workerServer) minimize(ctx context.Context, args minimizeArgs) (resp m
 // mem just in case an unrecoverable error occurs. It uses the context to
 // determine how long to run, stopping once closed. It returns the last error it
 // found.
-func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, count *int64, limit int64) error {
+func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, count *int64, limit int64) (success bool, retErr error) {
        shouldStop := func() bool {
                return ctx.Err() != nil || (limit > 0 && *count >= limit)
        }
        if shouldStop() {
-               return nil
+               return false, nil
        }
 
        var valI int
-       var retErr error
        tryMinimized := func(candidate interface{}) bool {
                prev := vals[valI]
                // Set vals[valI] to the candidate after it has been
@@ -822,7 +823,7 @@ func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, c
 
        for valI = range vals {
                if shouldStop() {
-                       return retErr
+                       break
                }
                switch v := vals[valI].(type) {
                case bool:
@@ -869,7 +870,7 @@ func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, c
                        panic("unreachable")
                }
        }
-       return retErr
+       return retErr != nil, retErr
 }
 
 func writeToMem(vals []interface{}, mem *sharedMem) {