"cpuprofile": true,
"failfast": true,
"fuzz": true,
+ "fuzzminimizetime": true,
"fuzztime": true,
"list": true,
"memprofile": true,
cf.Bool("short", false, "")
cf.DurationVar(&testTimeout, "timeout", 10*time.Minute, "")
cf.String("fuzztime", "", "")
+ cf.String("fuzzminimizetime", "", "")
cf.StringVar(&testTrace, "trace", "", "")
cf.BoolVar(&testV, "v", false, "")
cf.Var(&testShuffle, "shuffle", "")
}
func FuzzInt(f *testing.F) {
+ f.Add(0)
f.Fuzz(func(t *testing.T, a int) {
- if a > 200 && a < 250 {
+ if a != 0 {
panic("this input caused a crash!")
}
})
}
func FuzzUint(f *testing.F) {
+ f.Add(uint(0))
f.Fuzz(func(t *testing.T, a uint) {
- if a > 200 && a < 250 {
+ if a != 0 {
panic("this input caused a crash!")
}
})
}
func FuzzBool(f *testing.F) {
+ f.Add(false)
f.Fuzz(func(t *testing.T, a bool) {
if a {
panic("this input caused a crash!")
}
func FuzzByte(f *testing.F) {
+ f.Add(byte(0))
f.Fuzz(func(t *testing.T, a byte) {
- if a > 50 {
+ if a != 0 {
panic("this input caused a crash!")
}
})
}
func FuzzRune(f *testing.F) {
+ f.Add(rune(0))
f.Fuzz(func(t *testing.T, a rune) {
- if a > 50 {
+ if a != 0 {
panic("this input caused a crash!")
}
})
}
func FuzzString(f *testing.F) {
+ f.Add("")
f.Fuzz(func(t *testing.T, a string) {
if a != "" {
panic("this input caused a crash!")
[short] skip
+# TODO(b/181800488): remove -parallel=1, here and below. For now, when a
+# crash is found, all workers keep running, wasting resources and reducing
+# the number of executions available to the minimizer, increasing flakiness.
go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz
go run check_logs.go fuzz fuzz.worker
stdout 'mutator found enough unique mutations'
# Test that minimization is working for recoverable errors.
-! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=1000x minimizer_test.go
+! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go
! stdout '^ok'
stdout 'got the minimum size!'
stdout 'contains a letter'
! go test -run=FuzzMinimizerRecoverable minimizer_test.go
# Test that minimization is working for non-recoverable errors.
-! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=1000x minimizer_test.go
+! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go
! stdout '^ok'
stdout 'got the minimum size!'
stdout 'contains a letter'
# Check that the bytes written to testdata are of length 50 (the minimum size)
go run check_testdata.go FuzzMinimizerNonrecoverable 50
-# Test that minimization can be cancelled by fuzztime and the latest crash will
-# still be logged and written to testdata.
-! go test -fuzz=FuzzNonMinimizable -run=FuzzNonMinimizable -parallel=1 -fuzztime=5s minimizer_test.go
+# Test that minimization can be cancelled by fuzzminimizetime and the latest
+# crash will still be logged and written to testdata.
+! go test -fuzz=FuzzNonMinimizable -run=FuzzNonMinimizable -parallel=1 -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go
! stdout '^ok'
stdout 'testdata[/\\]corpus[/\\]FuzzNonMinimizable[/\\]'
! stdout 'got the minimum size!' # it shouldn't have had enough time to minimize it
import (
"bytes"
"testing"
- "time"
)
func FuzzMinimizerRecoverable(f *testing.F) {
if len(b) == 20 {
t.Log("got the minimum size!")
}
- time.Sleep(4 * time.Second)
})
}
os.Exit(1)
}
if want, got := numBytes, len(s); want != got {
- fmt.Fprintf(os.Stderr, "want %d bytes, got %d", want, got)
+ fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got)
os.Exit(1)
}
}
// has loaded. If zero, there will be no time limit.
Timeout time.Duration
- // Count is the number of random values to generate and test. If zero,
+ // Limit is the number of random values to generate and test. If zero,
// there will be no limit on the number of generated values.
- Count int64
+ Limit int64
+
+ // MinimizeTimeout is the amount of wall clock time to spend minimizing
+ // after discovering a crasher. If zero, there will be no time limit.
+ MinimizeTimeout time.Duration
+
+ // MinimizeLimit is the maximum number of calls to the fuzz function to be
+ // made while minimizing after finding a crash. If zero, there will be
+ // no limit.
+ MinimizeLimit int64
// parallel is the number of worker processes to run in parallel. If zero,
// CoordinateFuzzing will run GOMAXPROCS workers.
if opts.Parallel == 0 {
opts.Parallel = runtime.GOMAXPROCS(0)
}
- if opts.Count > 0 && int64(opts.Parallel) > opts.Count {
+ if opts.Limit > 0 && int64(opts.Parallel) > opts.Limit {
// Don't start more workers than we need.
- opts.Parallel = int(opts.Count)
+ opts.Parallel = int(opts.Limit)
}
c, err := newCoordinator(opts)
case result := <-c.resultC:
// Received response from worker.
c.updateStats(result)
- if c.opts.Count > 0 && c.count >= c.opts.Count {
+ if c.opts.Limit > 0 && c.count >= c.opts.Limit {
stop(nil)
}
// a limit for one worker. If there are no executions left, nextInput returns
// a zero value and false.
func (c *coordinator) nextInput() (fuzzInput, bool) {
- if c.opts.Count > 0 && c.count+c.countWaiting >= c.opts.Count {
+ if c.opts.Limit > 0 && c.count+c.countWaiting >= c.opts.Limit {
// Workers already testing all requested inputs.
return fuzzInput{}, false
}
return input, true
}
- if c.opts.Count > 0 {
- input.countRequested = c.opts.Count / int64(c.opts.Parallel)
- if c.opts.Count%int64(c.opts.Parallel) > 0 {
+ if c.opts.Limit > 0 {
+ input.countRequested = c.opts.Limit / int64(c.opts.Parallel)
+ if c.opts.Limit%int64(c.opts.Parallel) > 0 {
input.countRequested++
}
- remaining := c.opts.Count - c.count - c.countWaiting
+ remaining := c.opts.Limit - c.count - c.countWaiting
if input.countRequested > remaining {
input.countRequested = remaining
}
// sharedMemHeader stores metadata in shared memory.
type sharedMemHeader struct {
- length int
+ // count is the number of times the worker has called the fuzz function.
+ // May be reset by coordinator.
+ count int64
+
+ // valueLen is the length of the value that was last fuzzed.
+ valueLen int
}
// sharedMemSize returns the size needed for a shared memory buffer that can
// valueRef returns the value currently stored in shared memory. The returned
// slice points to shared memory; it is not a copy.
func (m *sharedMem) valueRef() []byte {
- length := m.header().length
+ length := m.header().valueLen
valueOffset := int(unsafe.Sizeof(sharedMemHeader{}))
return m.region[valueOffset : valueOffset+length]
}
if len(b) > cap(v) {
panic(fmt.Sprintf("value length %d larger than shared memory capacity %d", len(b), cap(v)))
}
- m.header().length = len(b)
+ m.header().valueLen = len(b)
copy(v[:cap(v)], b)
}
if n > cap(v) {
panic(fmt.Sprintf("length %d larger than shared memory capacity %d", n, cap(v)))
}
- m.header().length = n
+ m.header().valueLen = n
}
// TODO(jayconrod): add method to resize the buffer. We'll need that when the
case input := <-w.coordinator.inputC:
// Received input from coordinator.
- args := fuzzArgs{Count: input.countRequested, Duration: workerFuzzDuration, CoverageOnly: input.coverageOnly}
+ args := fuzzArgs{Limit: input.countRequested, Timeout: workerFuzzDuration, CoverageOnly: input.coverageOnly}
if interestingCount < input.interestingCount {
// The coordinator's coverage data has changed, so send the data
// to the client.
// the other workers from receiving more inputs.
message := fmt.Sprintf("fuzzing process terminated unexpectedly: %v", w.waitErr)
err = w.waitErr
- res, minimized, minErr := w.minimize(ctx)
+ var result fuzzResult
+ var minimized bool
+ if !input.coverageOnly {
+ var minErr error
+ result, minimized, minErr = w.minimize(ctx)
+ if minErr != nil {
+ err = minErr
+ }
+ }
if !minimized {
// Minimization did not find a smaller crashing value, so
// return the one we already found.
- res = fuzzResult{
- entry: CorpusEntry{Data: value},
- crasherMsg: message,
- }
- }
- if minErr != nil {
- err = minErr
+ result.entry = CorpusEntry{Data: value}
+ result.crasherMsg = message
}
- w.coordinator.resultC <- res
+ w.coordinator.resultC <- result
return err
}
+
result := fuzzResult{
countRequested: input.countRequested,
count: resp.Count,
}
}
-// minimize asks a workerServer to attempt to minimize what is currently in
-// shared memory. It runs for a maxium of 1 minute. The worker must be stopped
-// when minimize is called.
+// minimize asks a workerServer to attempt to minimize a value that caused an
+// unexpected termination of the worker process. The value must be in shared
+// memory, and the worker must be stopped. The execution count in shared memory
+// is reset once before restarting the worker.
func (w *worker) minimize(ctx context.Context) (res fuzzResult, minimized bool, retErr error) {
- fmt.Fprint(w.coordinator.opts.Log, "found a crash, currently minimizing for up to 1 minute\n")
+ if w.coordinator.opts.MinimizeTimeout != 0 {
+ fmt.Fprintf(w.coordinator.opts.Log, "found a crash, minimizing for up to %v\n", w.coordinator.opts.MinimizeTimeout)
+ } else if w.coordinator.opts.MinimizeLimit != 0 {
+ fmt.Fprintf(w.coordinator.opts.Log, "found a crash, minimizing for up to %d execs\n", w.coordinator.opts.MinimizeLimit)
+ } else {
+ fmt.Fprintf(w.coordinator.opts.Log, "found a crash, minimizing...\n")
+ }
+ start := time.Now()
+ if w.coordinator.opts.MinimizeTimeout != 0 {
+ var cancel func()
+ ctx, cancel = context.WithTimeout(ctx, w.coordinator.opts.MinimizeTimeout)
+ defer cancel()
+ }
defer func() {
w.stop()
if retErr == nil {
retErr = w.waitErr
}
}()
- // In case we can't minimize it at all, save the last crash value that we
- // found to send to the coordinator once the time is up.
- minimizeDeadline := time.Now().Add(time.Minute)
- for rem := time.Until(minimizeDeadline); rem > 0; {
+
+ mem := <-w.memMu
+ mem.header().count = 0
+ w.memMu <- mem
+
+ for {
+ if ctx.Err() != nil {
+ return res, minimized, retErr
+ }
// Restart the worker.
if err := w.start(); err != nil {
return res, minimized, err
}
- args := minimizeArgs{Duration: rem}
+ if err := w.client.ping(ctx); err != nil {
+ return res, minimized, err
+ }
+ args := minimizeArgs{Limit: w.coordinator.opts.MinimizeLimit}
+ if w.coordinator.opts.MinimizeTimeout != 0 {
+ elapsed := time.Now().Sub(start)
+ args.Timeout = w.coordinator.opts.MinimizeTimeout - elapsed
+ if args.Timeout < 0 {
+ return res, minimized, retErr
+ }
+ }
value, err := w.client.minimize(ctx, args)
if err == nil {
// Minimization finished successfully, meaning that it
// so stop trying.
return res, minimized, nil
}
+ w.stop()
// Minimization will return an error for a non-recoverable problem, so
// a non-nil error is expected. However, make sure it didn't fail for
// some other reason which should cause us to stop minimizing.
// The bytes in memory caused a legitimate crash, so stop the worker and
// save this value and error message.
- w.stop()
message := fmt.Sprintf("fuzzing process terminated unexpectedly: %v", w.waitErr)
res = fuzzResult{
entry: CorpusEntry{Data: value},
}
minimized = true
}
- return res, minimized, nil
// TODO(jayconrod,katiehockman): while minimizing, every panic message is
// logged to STDOUT. We should probably suppress all but the last one to
// lower the noise.
// minimizeArgs contains arguments to workerServer.minimize. The value to
// minimize is already in shared memory.
type minimizeArgs struct {
- Duration time.Duration
+ // Timeout is the time to spend minimizing. This may include time to start up,
+ // especially if the input causes the worker process to terminated, requiring
+ // repeated restarts.
+ Timeout time.Duration
+
+ // Limit is the maximum number of values to test, without spending more time
+ // than Duration. 0 indicates no limit.
+ Limit int64
}
// minimizeResponse contains results from workerServer.minimize.
// fuzzArgs contains arguments to workerServer.fuzz. The value to fuzz is
// passed in shared memory.
type fuzzArgs struct {
- // Duration is the time to spend fuzzing, not including starting or
+ // Timeout is the time to spend fuzzing, not including starting or
// cleaning up.
- Duration time.Duration
+ Timeout time.Duration
- // Count is the number of values to test, without spending more time
- // than Duration.
- Count int64
+ // Limit is the maximum number of values to test, without spending more time
+ // than Duration. 0 indicates no limit.
+ Limit int64
// CoverageOnly indicates whether this is a coverage-only run (ie. fuzzing
// should not occur).
start := time.Now()
defer func() { resp.Duration = time.Since(start) }()
- fuzzCtx, cancel := context.WithTimeout(ctx, args.Duration)
+ fuzzCtx, cancel := context.WithTimeout(ctx, args.Timeout)
defer cancel()
mem := <-ws.memMu
- defer func() { ws.memMu <- mem }()
+ defer func() {
+ ws.memMu <- mem
+ resp.Count = mem.header().count
+ }()
vals, err := unmarshalCorpusFile(mem.valueCopy())
if err != nil {
return resp
default:
- resp.Count++
+ mem.header().count++
ws.m.mutate(vals, cap(mem.valueRef()))
writeToMem(vals, mem)
if err := ws.fuzzFn(CorpusEntry{Values: vals}); err != nil {
- // TODO(jayconrod,katiehockman): consider making the maximum
- // minimization time customizable with a go command flag.
+ // TODO(jayconrod,katiehockman): report unminimized input to coordinator
+ // immediately so it can stop other workers.
+ // TODO(jayconrod,katiehockman): use -fuzzminimizetime to limit time or
+ // iterations spent on minimization.
minCtx, minCancel := context.WithTimeout(ctx, time.Minute)
defer minCancel()
- if minErr := ws.minimizeInput(minCtx, vals, mem); minErr != nil {
+ if minErr := ws.minimizeInput(minCtx, vals, mem, &mem.header().count, args.Limit); minErr != nil {
// Minimization found a different error, so use that one.
err = minErr
}
return resp
}
}
- if args.Count > 0 && resp.Count == args.Count {
+ if args.Limit > 0 && mem.header().count == args.Limit {
return resp
}
}
}
}
-func (ws *workerServer) minimize(ctx context.Context, args minimizeArgs) minimizeResponse {
+func (ws *workerServer) minimize(ctx context.Context, args minimizeArgs) (resp minimizeResponse) {
mem := <-ws.memMu
defer func() { ws.memMu <- mem }()
vals, err := unmarshalCorpusFile(mem.valueCopy())
if err != nil {
panic(err)
}
- ctx, cancel := context.WithTimeout(ctx, args.Duration)
- defer cancel()
- ws.minimizeInput(ctx, vals, mem)
- return minimizeResponse{}
+ if args.Timeout != 0 {
+ var cancel func()
+ ctx, cancel = context.WithTimeout(ctx, args.Timeout)
+ defer cancel()
+ }
+ ws.minimizeInput(ctx, vals, mem, &mem.header().count, args.Limit)
+ return resp
}
// minimizeInput applies a series of minimizing transformations on the provided
// mem just in case an unrecoverable error occurs. It uses the context to
// determine how long to run, stopping once closed. It returns the last error it
// found.
-func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, mem *sharedMem) (retErr error) {
+func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, mem *sharedMem, count *int64, limit int64) (retErr error) {
// Make sure the last crashing value is written to mem.
defer writeToMem(vals, mem)
+ shouldStop := func() bool {
+ return ctx.Err() != nil || (limit > 0 && *count >= limit)
+ }
+ if shouldStop() {
+ return nil
+ }
+
// tryMinimized will run the fuzz function for the values in vals at the
// time the function is called. If err is nil, then the minimization was
// unsuccessful, since we expect an error to still occur.
// The fuzz function failed, so save the most recent error.
retErr = err
}
+ *count++
return err
}
for valI := range vals {
// First, try to cut the tail.
for n := 1024; n != 0; n /= 2 {
for len(v) > n {
- if ctx.Err() != nil {
+ if shouldStop() {
return retErr
}
vals[valI] = v[:len(v)-n]
// Then, try to remove each individual byte.
tmp := make([]byte, len(v))
for i := 0; i < len(v)-1; i++ {
- if ctx.Err() != nil {
+ if shouldStop() {
return retErr
}
candidate := tmp[:len(v)-1]
for i := 0; i < len(v)-1; i++ {
copy(tmp, v[:i])
for j := len(v); j > i+1; j-- {
- if ctx.Err() != nil {
+ if shouldStop() {
return retErr
}
candidate := tmp[:len(v)-j+i]
if !ok {
return nil, fuzzResponse{}, errSharedMemClosed
}
+ mem.header().count = 0
mem.setValue(valueIn)
wc.memMu <- mem
func initFuzzFlags() {
matchFuzz = flag.String("test.fuzz", "", "run the fuzz target matching `regexp`")
- flag.Var(&fuzzDuration, "test.fuzztime", "time to spend fuzzing default is to run indefinitely")
+ flag.Var(&fuzzDuration, "test.fuzztime", "time to spend fuzzing; default is to run indefinitely")
+ flag.Var(&minimizeDuration, "test.fuzzminimizetime", "time to spend minimizing a value after finding a crash; default is to minimize for 60s")
fuzzCacheDir = flag.String("test.fuzzcachedir", "", "directory where interesting fuzzing inputs are stored")
isFuzzWorker = flag.Bool("test.fuzzworker", false, "coordinate with the parent process to fuzz random values")
}
var (
- matchFuzz *string
- fuzzDuration durationOrCountFlag
- fuzzCacheDir *string
- isFuzzWorker *bool
+ matchFuzz *string
+ fuzzDuration durationOrCountFlag
+ minimizeDuration = durationOrCountFlag{d: 60 * time.Second}
+ fuzzCacheDir *string
+ isFuzzWorker *bool
// corpusDir is the parent directory of the target's seed corpus within
// the package.
// actual fuzzing.
corpusTargetDir := filepath.Join(corpusDir, f.name)
cacheTargetDir := filepath.Join(*fuzzCacheDir, f.name)
- err := f.fuzzContext.coordinateFuzzing(fuzzDuration.d, int64(fuzzDuration.n), *parallel, f.corpus, types, corpusTargetDir, cacheTargetDir)
+ err := f.fuzzContext.coordinateFuzzing(
+ fuzzDuration.d,
+ int64(fuzzDuration.n),
+ minimizeDuration.d,
+ int64(minimizeDuration.n),
+ *parallel,
+ f.corpus,
+ types,
+ corpusTargetDir,
+ cacheTargetDir)
if err != nil {
f.result = FuzzResult{Error: err}
f.Fail()
// fuzzContext holds all fields that are common to all fuzz targets.
type fuzzContext struct {
importPath func() string
- coordinateFuzzing func(time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error
+ coordinateFuzzing func(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error
runFuzzWorker func(func(corpusEntry) error) error
readCorpus func(string, []reflect.Type) ([]corpusEntry, error)
resetCoverage func()
testlog.SetPanicOnExit0(v)
}
-func (TestDeps) CoordinateFuzzing(timeout time.Duration, count int64, parallel int, seed []fuzz.CorpusEntry, types []reflect.Type, corpusDir, cacheDir string) (err error) {
+func (TestDeps) CoordinateFuzzing(
+ timeout time.Duration,
+ limit int64,
+ minimizeTimeout time.Duration,
+ minimizeLimit int64,
+ parallel int,
+ seed []fuzz.CorpusEntry,
+ types []reflect.Type,
+ corpusDir,
+ cacheDir string) (err error) {
// Fuzzing may be interrupted with a timeout or if the user presses ^C.
// In either case, we'll stop worker processes gracefully and save
// crashers and interesting values.
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
defer cancel()
err = fuzz.CoordinateFuzzing(ctx, fuzz.CoordinateFuzzingOpts{
- Log: os.Stderr,
- Timeout: timeout,
- Count: count,
- Parallel: parallel,
- Seed: seed,
- Types: types,
- CorpusDir: corpusDir,
- CacheDir: cacheDir,
+ Log: os.Stderr,
+ Timeout: timeout,
+ Limit: limit,
+ MinimizeTimeout: minimizeTimeout,
+ MinimizeLimit: minimizeLimit,
+ Parallel: parallel,
+ Seed: seed,
+ Types: types,
+ CorpusDir: corpusDir,
+ CacheDir: cacheDir,
})
if err == ctx.Err() {
return nil
func (f matchStringOnly) StartTestLog(io.Writer) {}
func (f matchStringOnly) StopTestLog() error { return errMain }
func (f matchStringOnly) SetPanicOnExit0(bool) {}
-func (f matchStringOnly) CoordinateFuzzing(time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error {
+func (f matchStringOnly) CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error {
return errMain
}
func (f matchStringOnly) RunFuzzWorker(func(corpusEntry) error) error { return errMain }
StartTestLog(io.Writer)
StopTestLog() error
WriteProfileTo(string, io.Writer, int) error
- CoordinateFuzzing(time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error
+ CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error
RunFuzzWorker(func(corpusEntry) error) error
ReadCorpus(string, []reflect.Type) ([]corpusEntry, error)
ResetCoverage()