From: Michael Matloob Date: Tue, 5 Apr 2022 22:47:23 +0000 (-0400) Subject: cmd/go: changes to use modindex X-Git-Tag: go1.19beta1~79 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=ee87cd1dd95fe46099e393143d9a6dcccf0a6db0;p=gostls13.git cmd/go: changes to use modindex This CL makes the changes to actually use the module index when loading packages and instead of scanning their directories to see if they contain go files or to extract imports. Change-Id: I70106181cf64d6fd5a416644ba518b6b90030e0a Reviewed-on: https://go-review.googlesource.com/c/go/+/403778 Reviewed-by: Michael Matloob Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot Run-TryBot: Michael Matloob --- diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 511bdc1734..8ceacec326 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -35,6 +35,7 @@ import ( "cmd/go/internal/fsys" "cmd/go/internal/imports" "cmd/go/internal/modfetch" + "cmd/go/internal/modindex" "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/par" @@ -871,7 +872,16 @@ func loadPackageData(ctx context.Context, path, parentPath, parentDir, parentRoo if !cfg.ModulesEnabled { buildMode = build.ImportComment } + if modroot := modload.PackageModRoot(ctx, r.dir); modroot != "" { + if mi, err := modindex.Get(modroot); err == nil { + data.p, data.err = mi.Import(cfg.BuildContext, mi.RelPath(r.dir), buildMode) + goto Happy + } else if !errors.Is(err, modindex.ErrNotIndexed) { + base.Fatalf("go: %v", err) + } + } data.p, data.err = cfg.BuildContext.ImportDir(r.dir, buildMode) + Happy: if cfg.ModulesEnabled { // Override data.p.Root, since ImportDir sets it to $GOPATH, if // the module is inside $GOPATH/src. diff --git a/src/cmd/go/internal/modindex/read.go b/src/cmd/go/internal/modindex/read.go index 2579c516d6..e180ca5450 100644 --- a/src/cmd/go/internal/modindex/read.go +++ b/src/cmd/go/internal/modindex/read.go @@ -134,7 +134,7 @@ func openIndex(modroot string, ismodcache bool) (*ModuleIndex, error) { if err != nil { return result{nil, err} } - return mi + return result{mi, nil} }).(result) return r.mi, r.err } diff --git a/src/cmd/go/internal/modindex/scan.go b/src/cmd/go/internal/modindex/scan.go index 6e42e4ecac..e40d3e0f53 100644 --- a/src/cmd/go/internal/modindex/scan.go +++ b/src/cmd/go/internal/modindex/scan.go @@ -20,7 +20,7 @@ import ( // if the module shouldn't be indexed, and nil otherwise. func moduleWalkErr(modroot string, path string, info fs.FileInfo, err error) error { if err != nil { - return err + return ErrNotIndexed } // stop at module boundaries if info.IsDir() && path != modroot { diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index bfc73cc2f9..7b1bc732fc 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -63,6 +63,26 @@ func PackageModuleInfo(ctx context.Context, pkgpath string) *modinfo.ModulePubli return moduleInfo(ctx, rs, m, 0) } +// PackageModRoot returns the module root directory for the module that provides +// a given package. If modules are not enabled or if the package is in the +// standard library or if the package was not successfully loaded with +// LoadPackages or ImportFromFiles, the empty string is returned. +func PackageModRoot(ctx context.Context, pkgpath string) string { + if isStandardImportPath(pkgpath) || !Enabled() || cfg.BuildMod == "vendor" { + return "" + } + m, ok := findModule(loaded, pkgpath) + if !ok { + return "" + } + const needSum = true + root, _, err := fetch(ctx, m, needSum) + if err != nil { + return "" + } + return root +} + func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { if !Enabled() { return nil diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 4862f625b4..22286e5e2d 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -20,8 +20,10 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/fsys" "cmd/go/internal/modfetch" + "cmd/go/internal/modindex" "cmd/go/internal/par" "cmd/go/internal/search" + "cmd/go/internal/str" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -247,9 +249,9 @@ func (e *invalidImportError) Unwrap() error { // If the package is present in exactly one module, importFromModules will // return the module, its root directory, and a list of other modules that // lexically could have provided the package but did not. -func importFromModules(ctx context.Context, path string, rs *Requirements, mg *ModuleGraph) (m module.Version, dir string, altMods []module.Version, err error) { - invalidf := func(format string, args ...interface{}) (module.Version, string, []module.Version, error) { - return module.Version{}, "", nil, &invalidImportError{ +func importFromModules(ctx context.Context, path string, rs *Requirements, mg *ModuleGraph) (m module.Version, modroot, dir string, altMods []module.Version, err error) { + invalidf := func(format string, args ...interface{}) (module.Version, string, string, []module.Version, error) { + return module.Version{}, "", "", nil, &invalidImportError{ importPath: path, err: fmt.Errorf(format, args...), } @@ -270,11 +272,11 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M if path == "C" { // There's no directory for import "C". - return module.Version{}, "", nil, nil + return module.Version{}, "", "", nil, nil } // Before any further lookup, check that the path is valid. if err := module.CheckImportPath(path); err != nil { - return module.Version{}, "", nil, &invalidImportError{importPath: path, err: err} + return module.Version{}, "", "", nil, &invalidImportError{importPath: path, err: err} } // Is the package in the standard library? @@ -283,14 +285,18 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M for _, mainModule := range MainModules.Versions() { if MainModules.InGorootSrc(mainModule) { if dir, ok, err := dirInModule(path, MainModules.PathPrefix(mainModule), MainModules.ModRoot(mainModule), true); err != nil { - return module.Version{}, dir, nil, err + return module.Version{}, MainModules.ModRoot(mainModule), dir, nil, err } else if ok { - return mainModule, dir, nil, nil + return mainModule, MainModules.ModRoot(mainModule), dir, nil, nil } } } - dir := filepath.Join(cfg.GOROOT, "src", path) - return module.Version{}, dir, nil, nil + dir := filepath.Join(cfg.GOROOTsrc, path) + modroot = cfg.GOROOTsrc + if str.HasPathPrefix(path, "cmd") { + modroot = filepath.Join(cfg.GOROOTsrc, "cmd") + } + return module.Version{}, modroot, dir, nil, nil } // -mod=vendor is special. @@ -301,23 +307,23 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M mainDir, mainOK, mainErr := dirInModule(path, MainModules.PathPrefix(mainModule), modRoot, true) vendorDir, vendorOK, _ := dirInModule(path, "", filepath.Join(modRoot, "vendor"), false) if mainOK && vendorOK { - return module.Version{}, "", nil, &AmbiguousImportError{importPath: path, Dirs: []string{mainDir, vendorDir}} + return module.Version{}, modRoot, "", nil, &AmbiguousImportError{importPath: path, Dirs: []string{mainDir, vendorDir}} } // Prefer to return main directory if there is one, // Note that we're not checking that the package exists. // We'll leave that for load. if !vendorOK && mainDir != "" { - return mainModule, mainDir, nil, nil + return mainModule, modRoot, mainDir, nil, nil } if mainErr != nil { - return module.Version{}, "", nil, mainErr + return module.Version{}, "", "", nil, mainErr } readVendorList(mainModule) - return vendorPkgModule[path], vendorDir, nil, nil + return vendorPkgModule[path], modRoot, vendorDir, nil, nil } // Check each module on the build list. - var dirs []string + var dirs, roots []string var mods []module.Version // Iterate over possible modules for the path, not all selected modules. @@ -368,12 +374,13 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M // continue the loop and find the package in some other module, // we need to look at this module to make sure the import is // not ambiguous. - return module.Version{}, "", nil, err + return module.Version{}, "", "", nil, err } if dir, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { - return module.Version{}, "", nil, err + return module.Version{}, "", "", nil, err } else if ok { mods = append(mods, m) + roots = append(roots, root) dirs = append(dirs, dir) } else { altMods = append(altMods, m) @@ -387,9 +394,10 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M for i := 0; i < len(mods)/2; i++ { j := len(mods) - 1 - i mods[i], mods[j] = mods[j], mods[i] + roots[i], roots[j] = roots[j], roots[i] dirs[i], dirs[j] = dirs[j], dirs[i] } - return module.Version{}, "", nil, &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} + return module.Version{}, "", "", nil, &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } if len(sumErrMods) > 0 { @@ -397,7 +405,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M j := len(sumErrMods) - 1 - i sumErrMods[i], sumErrMods[j] = sumErrMods[j], sumErrMods[i] } - return module.Version{}, "", nil, &ImportMissingSumError{ + return module.Version{}, "", "", nil, &ImportMissingSumError{ importPath: path, mods: sumErrMods, found: len(mods) > 0, @@ -405,7 +413,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M } if len(mods) == 1 { - return mods[0], dirs[0], altMods, nil + return mods[0], roots[0], dirs[0], altMods, nil } if mg != nil { @@ -415,7 +423,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M if !HasModRoot() { queryErr = ErrNoModRoot } - return module.Version{}, "", nil, &ImportMissingError{Path: path, QueryErr: queryErr, isStd: pathIsStd} + return module.Version{}, "", "", nil, &ImportMissingError{Path: path, QueryErr: queryErr, isStd: pathIsStd} } // So far we've checked the root dependencies. @@ -426,7 +434,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M // the module graph, so we can't return an ImportMissingError here — one // of the missing modules might actually contain the package in question, // in which case we shouldn't go looking for it in some new dependency. - return module.Version{}, "", nil, err + return module.Version{}, "", "", nil, err } } } @@ -650,6 +658,15 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // We don't care about build tags, not even "+build ignore". // We're just looking for a plausible directory. res := haveGoFilesCache.Do(dir, func() any { + // modindex.Get will return ErrNotIndexed for any directories which + // are reached through a symlink, so that they will be handled by + // fsys.IsDirWithGoFiles below. + if mi, err := modindex.Get(mdir); err == nil { + isDirWithGoFiles, err := mi.IsDirWithGoFiles(mi.RelPath(dir)) + return goFilesEntry{isDirWithGoFiles, err} + } else if !errors.Is(err, modindex.ErrNotIndexed) { + return goFilesEntry{err: err} + } ok, err := fsys.IsDirWithGoFiles(dir) return goFilesEntry{haveGoFiles: ok, err: err} }).(goFilesEntry) diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 5214a9e2d1..b2c3ba2633 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -116,6 +116,7 @@ import ( "cmd/go/internal/fsys" "cmd/go/internal/imports" "cmd/go/internal/modfetch" + "cmd/go/internal/modindex" "cmd/go/internal/mvs" "cmd/go/internal/par" "cmd/go/internal/search" @@ -1401,7 +1402,7 @@ func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err err // // In some sense, we can think of this as ‘upgraded the module providing // pkg.path from "none" to a version higher than "none"’. - if _, _, _, err = importFromModules(ctx, pkg.path, rs, nil); err == nil { + if _, _, _, _, err = importFromModules(ctx, pkg.path, rs, nil); err == nil { changed = true break } @@ -1612,7 +1613,7 @@ func (ld *loader) preloadRootModules(ctx context.Context, rootPkgs []string) (ch // If the main module is tidy and the package is in "all" — or if we're // lucky — we can identify all of its imports without actually loading the // full module graph. - m, _, _, err := importFromModules(ctx, path, ld.requirements, nil) + m, _, _, _, err := importFromModules(ctx, path, ld.requirements, nil) if err != nil { var missing *ImportMissingError if errors.As(err, &missing) && ld.ResolveMissingImports { @@ -1699,7 +1700,8 @@ func (ld *loader) load(ctx context.Context, pkg *loadPkg) { } } - pkg.mod, pkg.dir, pkg.altMods, pkg.err = importFromModules(ctx, pkg.path, ld.requirements, mg) + var modroot string + pkg.mod, modroot, pkg.dir, pkg.altMods, pkg.err = importFromModules(ctx, pkg.path, ld.requirements, mg) if pkg.dir == "" { return } @@ -1729,7 +1731,7 @@ func (ld *loader) load(ctx context.Context, pkg *loadPkg) { // We can't scan standard packages for gccgo. } else { var err error - imports, testImports, err = scanDir(pkg.dir, ld.Tags) + imports, testImports, err = scanDir(modroot, pkg.dir, ld.Tags) if err != nil { pkg.err = err return @@ -1958,7 +1960,7 @@ func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) pkg := pkg ld.work.Add(func() { - mod, _, _, err := importFromModules(ctx, pkg.path, rs, mg) + mod, _, _, _, err := importFromModules(ctx, pkg.path, rs, mg) if mod != pkg.mod { mismatches := <-mismatchMu mismatches[pkg] = mismatch{mod: mod, err: err} @@ -2099,8 +2101,16 @@ func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) // during "go vendor", we look into "// +build appengine" files and // may see these legacy imports. We drop them so that the module // search does not look for modules to try to satisfy them. -func scanDir(dir string, tags map[string]bool) (imports_, testImports []string, err error) { +func scanDir(modroot string, dir string, tags map[string]bool) (imports_, testImports []string, err error) { + if mi, mierr := modindex.Get(modroot); mierr == nil { + imports_, testImports, err = mi.ScanDir(mi.RelPath(dir), tags) + goto Happy + } else if !errors.Is(mierr, modindex.ErrNotIndexed) { + return nil, nil, mierr + } + imports_, testImports, err = imports.ScanDir(dir, tags) +Happy: filter := func(x []string) []string { w := 0 diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index 799c48e50a..cddb9f8067 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -107,7 +107,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f if !have[name] { have[name] = true if isMatch(name) { - if _, _, err := scanDir(path, tags); err != imports.ErrNoGo { + if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo { m.Pkgs = append(m.Pkgs, name) } } @@ -208,7 +208,7 @@ func MatchInModule(ctx context.Context, pattern string, m module.Version, tags m return match } if haveGoFiles { - if _, _, err := scanDir(dir, tags); err != imports.ErrNoGo { + if _, _, err := scanDir(root, dir, tags); err != imports.ErrNoGo { // ErrNoGo indicates that the directory is not actually a Go package, // perhaps due to the tags in use. Any other non-nil error indicates a // problem with one or more of the Go source files, but such an error does diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index 04bc8d581a..d1fe36ec21 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -170,6 +170,7 @@ func (ts *testScript) setup() { "GOCACHE=" + testGOCACHE, "GODEBUG=" + os.Getenv("GODEBUG"), "GOEXE=" + cfg.ExeSuffix, + "GOINDEX=true", "GOOS=" + runtime.GOOS, "GOPATH=" + filepath.Join(ts.workdir, "gopath"), "GOPROXY=" + proxyURL,