From 881a16542e357fd85ac492424021ff380175675a Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 5 Oct 2022 12:54:00 -0400 Subject: [PATCH] cmd/go/internal/modindex: ignore non-source files for index We were saving non-go file information in the module index files, leading in an unnecessary increase in memory usage in modules containing many non-go files. This was a bug because this information is never used. Don't save that information. For #54226 Change-Id: I0644064f83f96e3a9f43b7e66ca94d69d9603376 Reviewed-on: https://go-review.googlesource.com/c/go/+/439118 TryBot-Result: Gopher Robot Reviewed-by: Michael Matloob Run-TryBot: Michael Matloob Reviewed-by: Bryan Mills --- src/cmd/go/internal/modindex/build.go | 7 ++++++- src/cmd/go/internal/modindex/index_test.go | 17 +++++++++++++++++ src/cmd/go/internal/modindex/scan.go | 5 ++++- .../modindex/testdata/ignore_non_source/a.syso | 1 + .../modindex/testdata/ignore_non_source/b.go | 0 .../testdata/ignore_non_source/bar.json | 0 .../modindex/testdata/ignore_non_source/baz.log | 0 .../modindex/testdata/ignore_non_source/c.c | 0 8 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 src/cmd/go/internal/modindex/testdata/ignore_non_source/a.syso create mode 100644 src/cmd/go/internal/modindex/testdata/ignore_non_source/b.go create mode 100644 src/cmd/go/internal/modindex/testdata/ignore_non_source/bar.json create mode 100644 src/cmd/go/internal/modindex/testdata/ignore_non_source/baz.log create mode 100644 src/cmd/go/internal/modindex/testdata/ignore_non_source/c.c diff --git a/src/cmd/go/internal/modindex/build.go b/src/cmd/go/internal/modindex/build.go index 8903c156bb..e4380973e0 100644 --- a/src/cmd/go/internal/modindex/build.go +++ b/src/cmd/go/internal/modindex/build.go @@ -477,6 +477,8 @@ type fileEmbed struct { pos token.Position } +var errNonSource = errors.New("non source file") + // getFileInfo extracts the information needed from each go file for the module // index. // @@ -484,6 +486,9 @@ type fileEmbed struct { // Imports and returns that section of the file in the FileInfo's Header field, // even though it only considers text until the first non-comment // for +build lines. +// +// getFileInfo will return errNonSource if the file is not a source or object +// file and shouldn't even be added to IgnoredFiles. func getFileInfo(dir, name string, fset *token.FileSet) (*fileInfo, error) { if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") { @@ -498,7 +503,7 @@ func getFileInfo(dir, name string, fset *token.FileSet) (*fileInfo, error) { if ext != ".go" && fileListForExt(&dummyPkg, ext) == nil { // skip - return nil, nil + return nil, errNonSource } info := &fileInfo{name: filepath.Join(dir, name), fset: fset} diff --git a/src/cmd/go/internal/modindex/index_test.go b/src/cmd/go/internal/modindex/index_test.go index 2c072f909d..1c32973d39 100644 --- a/src/cmd/go/internal/modindex/index_test.go +++ b/src/cmd/go/internal/modindex/index_test.go @@ -85,3 +85,20 @@ func TestIndex(t *testing.T) { } }) } + +func TestImportRaw_IgnoreNonGo(t *testing.T) { + path := filepath.Join("testdata", "ignore_non_source") + p := importRaw(path, ".") + + wantFiles := []string{"a.syso", "b.go", "c.c"} + + var gotFiles []string + for i := range p.sourceFiles { + gotFiles = append(gotFiles, p.sourceFiles[i].name) + } + + if !reflect.DeepEqual(gotFiles, wantFiles) { + t.Errorf("names of files in importRaw(testdata/ignore_non_source): got %v; want %v", + gotFiles, wantFiles) + } +} diff --git a/src/cmd/go/internal/modindex/scan.go b/src/cmd/go/internal/modindex/scan.go index 56ba9e86c8..7207e1e523 100644 --- a/src/cmd/go/internal/modindex/scan.go +++ b/src/cmd/go/internal/modindex/scan.go @@ -210,7 +210,10 @@ func importRaw(modroot, reldir string) *rawPackage { continue } info, err := getFileInfo(absdir, name, fset) - if err != nil { + if err == errNonSource { + // not a source or object file. completely ignore in the index + continue + } else if err != nil { p.sourceFiles = append(p.sourceFiles, &rawFile{name: name, error: err.Error()}) continue } else if info == nil { diff --git a/src/cmd/go/internal/modindex/testdata/ignore_non_source/a.syso b/src/cmd/go/internal/modindex/testdata/ignore_non_source/a.syso new file mode 100644 index 0000000000..9527d05936 --- /dev/null +++ b/src/cmd/go/internal/modindex/testdata/ignore_non_source/a.syso @@ -0,0 +1 @@ +package ignore_non_source diff --git a/src/cmd/go/internal/modindex/testdata/ignore_non_source/b.go b/src/cmd/go/internal/modindex/testdata/ignore_non_source/b.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/cmd/go/internal/modindex/testdata/ignore_non_source/bar.json b/src/cmd/go/internal/modindex/testdata/ignore_non_source/bar.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/cmd/go/internal/modindex/testdata/ignore_non_source/baz.log b/src/cmd/go/internal/modindex/testdata/ignore_non_source/baz.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/cmd/go/internal/modindex/testdata/ignore_non_source/c.c b/src/cmd/go/internal/modindex/testdata/ignore_non_source/c.c new file mode 100644 index 0000000000..e69de29bb2 -- 2.50.0