previousN int // number of iterations in the previous run
previousDuration time.Duration // total duration of the previous run
benchFunc func(b *B)
+ benchTime time.Duration
bytes int64
+ missingBytes bool // one of the subbenchmarks does not have bytes set.
timerOn bool
showAllocResult bool
+ hasSub bool
result BenchmarkResult
parallelism int // RunParallel creates parallelism*GOMAXPROCS goroutines
// The initial states of memStats.Mallocs and memStats.TotalAlloc.
}
}
-// run times the benchmark function in a separate goroutine.
+// probe runs benchFunc to examine if it has any subbenchmarks.
+func (b *B) probe() {
+ if ctx := b.context; ctx != nil {
+ // Extend maxLen, if needed.
+ if n := len(b.name) + ctx.extLen + 1; n > ctx.maxLen {
+ ctx.maxLen = n + 8 // Add additional slack to avoid too many jumps in size.
+ }
+ }
+ go func() {
+ // Signal that we're done whether we return normally
+ // or by FailNow's runtime.Goexit.
+ defer func() {
+ b.signal <- true
+ }()
+
+ benchmarkLock.Lock()
+ defer benchmarkLock.Unlock()
+
+ b.N = 0
+ b.benchFunc(b)
+ }()
+ <-b.signal
+}
+
+// run executes the benchmark in a separate goroutine, including all of its
+// subbenchmarks.
func (b *B) run() BenchmarkResult {
+ if b.context != nil {
+ // Running go test --test.bench
+ b.context.processBench(b) // Must call doBench.
+ } else {
+ // Running func Benchmark.
+ b.doBench()
+ }
+ return b.result
+}
+
+func (b *B) doBench() BenchmarkResult {
go b.launch()
<-b.signal
return b.result
// launch launches the benchmark function. It gradually increases the number
// of benchmark iterations until the benchmark runs for the requested benchtime.
-// It prints timing information in this form
-// testing.BenchmarkHello 100000 19 ns/op
-// launch is run by the run function as a separate goroutine.
+// launch is run by the doBench function as a separate goroutine.
func (b *B) launch() {
// Run the benchmark for a single iteration in case it's expensive.
n := 1
b.runN(n)
// Run the benchmark for at least the specified amount of time.
- d := *benchTime
+ d := b.benchTime
for !b.failed && b.duration < d && n < 1e9 {
last := n
// Predict required iterations.
type benchContext struct {
maxLen int // The largest recorded benchmark name.
+ extLen int // Maximum extension length.
}
// An internal function but exported because it is cross-package; part of the implementation
maxprocs = procs
}
}
- maxlen := 0
+ ctx := &benchContext{
+ extLen: len(benchmarkName("", maxprocs)),
+ }
var bs []InternalBenchmark
for _, Benchmark := range benchmarks {
matched, err := matchString(*matchBenchmarks, Benchmark.Name)
if matched {
bs = append(bs, Benchmark)
benchName := benchmarkName(Benchmark.Name, maxprocs)
- if l := len(benchName); l > maxlen {
- maxlen = l
+ if l := len(benchName) + ctx.extLen + 1; l > ctx.maxLen {
+ ctx.maxLen = l
}
}
}
- ok := true
main := &B{
common: common{name: "Main"},
- context: &benchContext{
- maxLen: maxlen,
+ benchFunc: func(b *B) {
+ for _, Benchmark := range bs {
+ b.runBench(Benchmark.Name, Benchmark.F)
+ }
},
+ benchTime: *benchTime,
+ context: ctx,
}
- for _, Benchmark := range bs {
- ok = ok && expandCPU(main, Benchmark)
- }
- return ok
+ main.runN(1)
+ return !main.failed
}
-func expandCPU(parent *B, Benchmark InternalBenchmark) bool {
- ok := true
+// processBench runs bench b for the configured CPU counts and prints the results.
+func (ctx *benchContext) processBench(b *B) {
for _, procs := range cpuList {
runtime.GOMAXPROCS(procs)
- benchName := benchmarkName(Benchmark.Name, procs)
- fmt.Printf("%-*s\t", parent.context.maxLen, benchName)
- b := parent.runBench(Benchmark.Name, Benchmark.F)
- r := b.result
+ benchName := benchmarkName(b.name, procs)
+ b := &B{
+ common: common{
+ signal: make(chan bool),
+ name: benchName,
+ },
+ benchFunc: b.benchFunc,
+ benchTime: b.benchTime,
+ }
+ fmt.Printf("%-*s\t", ctx.maxLen, benchName)
+ r := b.doBench()
if b.failed {
- ok = false
// The output could be very long here, but probably isn't.
// We print it all, regardless, because we don't want to trim the reason
// the benchmark failed.
fmt.Fprintf(os.Stderr, "testing: %s left GOMAXPROCS set to %d\n", benchName, p)
}
}
- return ok
}
// runBench benchmarks f as a subbenchmark with the given name. It reports
// whether there were any failures.
//
// A subbenchmark is like any other benchmark. A benchmark that calls Run at
-// least once will not be measured itself and will only run for one iteration.
-func (b *B) runBench(name string, f func(b *B)) *B {
+// least once will not be measured itself.
+func (b *B) runBench(name string, f func(b *B)) bool {
+ // Since b has subbenchmarks, we will no longer run it as a benchmark itself.
+ // Release the lock and acquire it on exit to ensure locks stay paired.
+ b.hasSub = true
+ benchmarkLock.Unlock()
+ defer benchmarkLock.Lock()
+
+ if b.level > 0 {
+ name = b.name + "/" + name
+ }
sub := &B{
common: common{
signal: make(chan bool),
level: b.level + 1,
},
benchFunc: f,
+ benchTime: b.benchTime,
context: b.context,
}
- sub.run()
- return sub
+ if sub.probe(); !sub.hasSub {
+ b.add(sub.run())
+ }
+ return !sub.failed
+}
+
+// add simulates running benchmarks in sequence in a single iteration. It is
+// used to give some meaningful results in case func Benchmark is used in
+// combination with Run.
+func (b *B) add(other BenchmarkResult) {
+ r := &b.result
+ // The aggregated BenchmarkResults resemble running all subbenchmarks as
+ // in sequence in a single benchmark.
+ r.N = 1
+ r.T += time.Duration(other.NsPerOp())
+ if other.Bytes == 0 {
+ // Summing Bytes is meaningless in aggregate if not all subbenchmarks
+ // set it.
+ b.missingBytes = true
+ r.Bytes = 0
+ }
+ if !b.missingBytes {
+ r.Bytes += other.Bytes
+ }
+ r.MemAllocs += uint64(other.AllocsPerOp())
+ r.MemBytes += uint64(other.AllocedBytesPerOp())
}
// trimOutput shortens the output from a benchmark, which can be very long.
signal: make(chan bool),
},
benchFunc: f,
+ benchTime: *benchTime,
}
return b.run()
}
package testing
+import (
+ "time"
+)
+
func TestTestContext(t *T) {
const (
add1 = 0
}
}
}
+
+// TODO: remove this stub when API is exposed
+func (b *B) Run(name string, f func(b *B)) bool { return b.runBench(name, f) }
+
+func TestBRun(t *T) {
+ work := func(b *B) {
+ for i := 0; i < b.N; i++ {
+ time.Sleep(time.Nanosecond)
+ }
+ }
+ testCases := []struct {
+ desc string
+ failed bool
+ f func(*B)
+ }{{
+ desc: "simulate sequential run of subbenchmarks.",
+ f: func(b *B) {
+ b.Run("", func(b *B) { work(b) })
+ time1 := b.result.NsPerOp()
+ b.Run("", func(b *B) { work(b) })
+ time2 := b.result.NsPerOp()
+ if time1 >= time2 {
+ t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2)
+ }
+ },
+ }, {
+ desc: "bytes set by all benchmarks",
+ f: func(b *B) {
+ b.Run("", func(b *B) { b.SetBytes(10); work(b) })
+ b.Run("", func(b *B) { b.SetBytes(10); work(b) })
+ if b.result.Bytes != 20 {
+ t.Errorf("bytes: got: %d; want 20", b.result.Bytes)
+ }
+ },
+ }, {
+ desc: "bytes set by some benchmarks",
+ // In this case the bytes result is meaningless, so it must be 0.
+ f: func(b *B) {
+ b.Run("", func(b *B) { b.SetBytes(10); work(b) })
+ b.Run("", func(b *B) { work(b) })
+ b.Run("", func(b *B) { b.SetBytes(10); work(b) })
+ if b.result.Bytes != 0 {
+ t.Errorf("bytes: got: %d; want 0", b.result.Bytes)
+ }
+ },
+ }, {
+ desc: "failure carried over to root",
+ failed: true,
+ f: func(b *B) { b.Fail() },
+ }, {
+ desc: "memory allocation",
+ f: func(b *B) {
+ const bufSize = 256
+ alloc := func(b *B) {
+ var buf [bufSize]byte
+ for i := 0; i < b.N; i++ {
+ _ = append([]byte(nil), buf[:]...)
+ }
+ }
+ b.Run("", func(b *B) { alloc(b) })
+ b.Run("", func(b *B) { alloc(b) })
+ if got := b.result.MemAllocs; got != 2 {
+ t.Errorf("MemAllocs was %v; want 2", got)
+ }
+ if got := b.result.MemBytes; got != 2*bufSize {
+ t.Errorf("MemBytes was %v; want %v", got, 2*bufSize)
+ }
+ },
+ }}
+ for _, tc := range testCases {
+ var ok bool
+ // This is almost like the Benchmark function, except that we override
+ // the benchtime and catch the failure result of the subbenchmark.
+ root := &B{
+ common: common{
+ signal: make(chan bool),
+ },
+ benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
+ benchTime: time.Microsecond,
+ }
+ root.run()
+ if ok != !tc.failed {
+ t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed)
+ }
+ if !ok != root.Failed() {
+ t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
+ }
+ // All tests are run as subtests
+ if root.result.N != 1 {
+ t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
+ }
+ }
+}