testing: improve documentation, examples, release notes for

testing.b.Loop.

This CL added documentation of the no-inlining semantic of b.Loop, with
a concrete example. This CL also tries to improve the release note to be
more descriptive.

Fixes #61515

Change-Id: I1e13cc92d5d6bdbf40fb44f44475e249747b807f
Reviewed-on: https://go-review.googlesource.com/c/go/+/633536
Reviewed-by: Michael Knyszek <mknyszek@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Commit-Queue: Junyang Shao <shaojunyang@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Reviewed-by: David Chase <drchase@google.com>

diff --git a/doc/next/6-stdlib/6-testing-bloop.md b/doc/next/6-stdlib/6-testing-bloop.md
new file mode 100644 (file)
index 0000000..37fb293
--- /dev/null
+++ b/doc/next/6-stdlib/6-testing-bloop.md
@@ -0,0 +1,5 @@
+### New benchmark function
+
+Benchmarks may now use the faster and less error-prone [testing.B.Loop] method to perform benchmark iterations like `for b.Loop() { ... }` in place of the typical loop structures involving `b.N` like `for i := n; i < b.N; i++ { ... }` or `for range b.N`. This offers two significant advantages:
+ - The benchmark function will execute exactly once per -count, so expensive setup and cleanup steps execute only once.
+ - Function call parameters and results are kept alive, preventing the compiler from fully optimizing away the loop body.

diff --git a/doc/next/6-stdlib/99-minor/testing/61515.md b/doc/next/6-stdlib/99-minor/testing/61515.md
index f724eb1b9025feee9ea76b820b021919f30dfc92..696633a3304b83d6e9091ee39d1444f93dcb60c9 100644 (file)
--- a/doc/next/6-stdlib/99-minor/testing/61515.md
+++ b/doc/next/6-stdlib/99-minor/testing/61515.md
@@ -1 +1 @@
-Benchmarks can use the new [B.Loop] method in `for b.Loop() { ... }` loops to determine if iteration should continue.
+<!-- testing.B.Loop mentioned in 6-stdlib/6-testing-bloop.md. -->
\ No newline at end of file

diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go
index 67a5abccb4d82e6ebe6d91efefd281698852c1e0..dbc0814884dafe48c91bae57915fd71c8b319894 100644 (file)
--- a/src/testing/benchmark.go
+++ b/src/testing/benchmark.go
@@ -405,6 +405,22 @@ func (b *B) loopSlowPath() bool {
 // A benchmark should either use Loop or contain an explicit loop from 0 to b.N, but not both.
 // After the benchmark finishes, b.N will contain the total number of calls to op, so the benchmark
 // may use b.N to compute other average metrics.
+//
+// The parameters and results of function calls inside the body of "for b.Loop() {...}" are guaranteed
+// not to be optimized away.
+// Also, the local loop scaling for b.Loop ensures the benchmark function containing the loop will only
+// be executed once, i.e. for such construct:
+//
+//     testing.Benchmark(func(b *testing.B) {
+//                     ...(setup)
+//                     for b.Loop() {
+//                             ...(benchmark logic)
+//                     }
+//                     ...(clean-up)
+//     }
+//
+// The ...(setup) and ...(clean-up) logic will only be executed once.
+// Also benchtime=Nx (N>1) will result in exactly N executions instead of N+1 for b.N style loops.
 func (b *B) Loop() bool {
        if b.loopN != 0 && b.loopN < b.N {
                b.loopN++

diff --git a/src/testing/benchmark_test.go b/src/testing/benchmark_test.go
index 1f55fa5060ed031350916eb97d6b0d133e977091..259b70ed4c95e0dee7f311fccd8c5c3f6d16b892 100644 (file)
--- a/src/testing/benchmark_test.go
+++ b/src/testing/benchmark_test.go
@@ -149,6 +149,36 @@ func TestBLoopHasResults(t *testing.T) {
        }
 }
 
+func ExampleB_Loop() {
+       simpleFunc := func(i int) int {
+               return i + 1
+       }
+       n := 0
+       testing.Benchmark(func(b *testing.B) {
+               // Unlike "for i := range N {...}" style loops, this
+               // setup logic will only be executed once, so simpleFunc
+               // will always get argument 1.
+               n++
+               // It behaves just like "for i := range N {...}", except with keeping
+               // function call parameters and results alive.
+               for b.Loop() {
+                       // This function call, if was in a normal loop, will be optimized away
+                       // completely, first by inlining, then by dead code elimination.
+                       // In a b.Loop loop, the compiler ensures that this function is not optimized away.
+                       simpleFunc(n)
+               }
+               // This clean-up will only be executed once, so after the benchmark, the user
+               // will see n == 2.
+               n++
+               // Use b.ReportMetric as usual just like what a user may do after
+               // b.N loop.
+       })
+       // We can expect n == 2 here.
+
+       // The return value of the above Benchmark could be used just like
+       // a b.N loop benchmark as well.
+}
+
 func ExampleB_RunParallel() {
        // Parallel benchmark for text/template.Template.Execute on a single object.
        testing.Benchmark(func(b *testing.B) {