go/token: benchmark FileSet.{Position,AddExistingFiles}

This CL adds a benchmark of FileSet.Position, the lookup
operation, and the new AddExistingFiles. It is evident
that its behavior is quadratic in important cases:

(Apple M1)
BenchmarkFileSet_AddExistingFiles/sequence-8         	       3	 362768139 ns/op

Change-Id: I256fdc776135e1924666d127afb37dacbefc860f
Reviewed-on: https://go-review.googlesource.com/c/go/+/675875
Reviewed-by: Robert Findley <rfindley@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Alan Donovan <adonovan@google.com>

diff --git a/src/go/token/export_test.go b/src/go/token/export_test.go
new file mode 100644 (file)
index 0000000..b1bd269
--- /dev/null
+++ b/src/go/token/export_test.go
@@ -0,0 +1,9 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package token
+
+// exports for tests
+
+func SearchInts(a []int, x int) int { return searchInts(a, x) }

diff --git a/src/go/token/position_bench_test.go b/src/go/token/position_bench_test.go
index 41be7285b7d2ca384e1fcc5926a1b4348952d317..7bb9de8946901d451fd78df82804425cca5df93f 100644 (file)
--- a/src/go/token/position_bench_test.go
+++ b/src/go/token/position_bench_test.go
@@ -2,9 +2,14 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package token
+package token_test
 
 import (
+       "go/build"
+       "go/token"
+       "math/rand/v2"
+       "os"
+       "path/filepath"
        "testing"
 )
 
@@ -14,11 +19,103 @@ func BenchmarkSearchInts(b *testing.B) {
                data[i] = i
        }
        const x = 8
-       if r := searchInts(data, x); r != x {
+       if r := token.SearchInts(data, x); r != x {
                b.Errorf("got index = %d; want %d", r, x)
        }
        b.ResetTimer()
        for i := 0; i < b.N; i++ {
-               searchInts(data, x)
+               token.SearchInts(data, x)
        }
 }
+
+func BenchmarkFileSet_Position(b *testing.B) {
+       rng := rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64()))
+
+       // Create a FileSet based on the files of net/http,
+       // a single large package.
+       netHTTPFset := token.NewFileSet()
+       pkg, err := build.Default.Import("net/http", "", 0)
+       if err != nil {
+               b.Fatal(err)
+       }
+       for _, filename := range pkg.GoFiles {
+               filename = filepath.Join(pkg.Dir, filename)
+               fi, err := os.Stat(filename)
+               if err != nil {
+                       b.Fatal(err)
+               }
+               netHTTPFset.AddFile(filename, -1, int(fi.Size()))
+       }
+
+       // Measure randomly distributed Pos values across net/http.
+       b.Run("random", func(b *testing.B) {
+               base := netHTTPFset.Base()
+               for b.Loop() {
+                       pos := token.Pos(rng.IntN(base))
+                       _ = netHTTPFset.Position(pos)
+               }
+       })
+
+       // Measure random lookups within the same file of net/http.
+       // (It's faster because of the "last file" cache.)
+       b.Run("file", func(b *testing.B) {
+               var file *token.File
+               for file = range netHTTPFset.Iterate {
+                       break
+               }
+               base, size := file.Base(), file.Size()
+               for b.Loop() {
+                       _ = netHTTPFset.Position(token.Pos(base + rng.IntN(size)))
+               }
+       })
+
+       // Measure random lookups on a FileSet with a great many files.
+       b.Run("manyfiles", func(b *testing.B) {
+               fset := token.NewFileSet()
+               for range 25000 {
+                       fset.AddFile("", -1, 10000)
+               }
+               base := fset.Base()
+               for b.Loop() {
+                       pos := token.Pos(rng.IntN(base))
+                       _ = fset.Position(pos)
+               }
+       })
+}
+
+func BenchmarkFileSet_AddExistingFiles(b *testing.B) {
+       // Create the "universe" of files.
+       fset := token.NewFileSet()
+       var files []*token.File
+       for range 25000 {
+               files = append(files, fset.AddFile("", -1, 10000))
+       }
+       rand.Shuffle(len(files), func(i, j int) {
+               files[i], files[j] = files[j], files[i]
+       })
+
+       // choose returns n random files.
+       choose := func(n int) []*token.File {
+               res := make([]*token.File, n)
+               for i := range res {
+                       res[i] = files[rand.IntN(n)]
+               }
+               return files[:n]
+       }
+
+       // Measure the cost of  creating a FileSet with a large number
+       // of files added in small handfuls, with some overlap.
+       // This case is critical to gopls.
+       b.Run("sequence", func(b *testing.B) {
+               for b.Loop() {
+                       b.StopTimer()
+                       fset2 := token.NewFileSet()
+                       fset2.AddExistingFiles(files[:10000]...)
+                       b.StartTimer()
+
+                       for range 1000 {
+                               fset2.AddExistingFiles(choose(10)...) // about one package
+                       }
+               }
+       })
+}