// Show vendor-expanded paths in listing
p.TestImports = p.Resolve(p.TestImports)
p.XTestImports = p.Resolve(p.XTestImports)
- p.TestEmbedFiles = p.ResolveEmbed(p.TestEmbedPatterns)
- p.XTestEmbedFiles = p.ResolveEmbed(p.XTestEmbedPatterns)
p.DepOnly = !cmdline[p]
if *listCompiled {
stk.Push(path)
defer stk.Pop()
- p.EmbedFiles, p.Internal.Embed, err = p.resolveEmbed(p.EmbedPatterns)
+ p.EmbedFiles, p.Internal.Embed, err = resolveEmbed(p.Dir, p.EmbedPatterns)
if err != nil {
setError(err)
embedErr := err.(*EmbedError)
}
// ResolveEmbed resolves //go:embed patterns and returns only the file list.
-// For use by go list to compute p.TestEmbedFiles and p.XTestEmbedFiles.
-func (p *Package) ResolveEmbed(patterns []string) []string {
- files, _, _ := p.resolveEmbed(patterns)
- return files
+// For use by go mod vendor to find embedded files it should copy into the
+// vendor directory.
+// TODO(#42504): Once go mod vendor uses load.PackagesAndErrors, just
+// call (*Package).ResolveEmbed
+func ResolveEmbed(dir string, patterns []string) ([]string, error) {
+ files, _, err := resolveEmbed(dir, patterns)
+ return files, err
}
// resolveEmbed resolves //go:embed patterns to precise file lists.
// It sets files to the list of unique files matched (for go list),
// and it sets pmap to the more precise mapping from
// patterns to files.
-func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[string][]string, err error) {
+func resolveEmbed(pkgdir string, patterns []string) (files []string, pmap map[string][]string, err error) {
var pattern string
defer func() {
if err != nil {
}
}()
+ // TODO(rsc): All these messages need position information for better error reports.
pmap = make(map[string][]string)
have := make(map[string]int)
dirOK := make(map[string]bool)
}
// Glob to find matches.
- match, err := fsys.Glob(p.Dir + string(filepath.Separator) + filepath.FromSlash(pattern))
+ match, err := fsys.Glob(pkgdir + string(filepath.Separator) + filepath.FromSlash(pattern))
if err != nil {
return nil, nil, err
}
// then there may be other things lying around, like symbolic links or .git directories.)
var list []string
for _, file := range match {
- rel := filepath.ToSlash(file[len(p.Dir)+1:]) // file, relative to p.Dir
+ rel := filepath.ToSlash(file[len(pkgdir)+1:]) // file, relative to p.Dir
what := "file"
info, err := fsys.Lstat(file)
// Check that directories along path do not begin a new module
// (do not contain a go.mod).
- for dir := file; len(dir) > len(p.Dir)+1 && !dirOK[dir]; dir = filepath.Dir(dir) {
+ for dir := file; len(dir) > len(pkgdir)+1 && !dirOK[dir]; dir = filepath.Dir(dir) {
if _, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil {
return nil, nil, fmt.Errorf("cannot embed %s %s: in different module", what, rel)
}
if dir != file {
if info, err := fsys.Lstat(dir); err == nil && !info.IsDir() {
- return nil, nil, fmt.Errorf("cannot embed %s %s: in non-directory %s", what, rel, dir[len(p.Dir)+1:])
+ return nil, nil, fmt.Errorf("cannot embed %s %s: in non-directory %s", what, rel, dir[len(pkgdir)+1:])
}
}
dirOK[dir] = true
if err != nil {
return err
}
- rel := filepath.ToSlash(path[len(p.Dir)+1:])
+ rel := filepath.ToSlash(path[len(pkgdir)+1:])
name := info.Name()
if path != file && (isBadEmbedName(name) || name[0] == '.' || name[0] == '_') {
// Ignore bad names, assuming they won't go into modules.
imports = append(imports, p1)
}
var err error
- p.TestEmbedFiles, testEmbed, err = p.resolveEmbed(p.TestEmbedPatterns)
+ p.TestEmbedFiles, testEmbed, err = resolveEmbed(p.Dir, p.TestEmbedPatterns)
if err != nil && ptestErr == nil {
ptestErr = &PackageError{
ImportStack: stk.Copy(),
}
p.XTestImports[i] = p1.ImportPath
}
- p.XTestEmbedFiles, xtestEmbed, err = p.resolveEmbed(p.XTestEmbedPatterns)
+ p.XTestEmbedFiles, xtestEmbed, err = resolveEmbed(p.Dir, p.XTestEmbedPatterns)
if err != nil && pxtestErr == nil {
pxtestErr = &PackageError{
ImportStack: stk.Copy(),
import (
"bytes"
"context"
+ "errors"
"fmt"
+ "go/build"
"io"
"io/fs"
"os"
"cmd/go/internal/cfg"
"cmd/go/internal/fsys"
"cmd/go/internal/imports"
+ "cmd/go/internal/load"
"cmd/go/internal/modload"
+ "cmd/go/internal/str"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
}
func vendorPkg(vdir, pkg string) {
+ // TODO(#42504): Instead of calling modload.ImportMap then build.ImportDir,
+ // just call load.PackagesAndErrors. To do that, we need to add a good way
+ // to ignore build constraints.
realPath := modload.ImportMap(pkg)
if realPath != pkg && modload.ImportMap(realPath) != "" {
fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
}
+ copiedFiles := make(map[string]bool)
dst := filepath.Join(vdir, pkg)
src := modload.PackageDir(realPath)
if src == "" {
fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath)
}
- copyDir(dst, src, matchPotentialSourceFile)
+ copyDir(dst, src, matchPotentialSourceFile, copiedFiles)
if m := modload.PackageModule(realPath); m.Path != "" {
- copyMetadata(m.Path, realPath, dst, src)
+ copyMetadata(m.Path, realPath, dst, src, copiedFiles)
+ }
+
+ ctx := build.Default
+ ctx.UseAllFiles = true
+ bp, err := ctx.ImportDir(src, build.IgnoreVendor)
+ // Because UseAllFiles is set on the build.Context, it's possible ta get
+ // a MultiplePackageError on an otherwise valid package: the package could
+ // have different names for GOOS=windows and GOOS=mac for example. On the
+ // other hand if there's a NoGoError, the package might have source files
+ // specifying "// +build ignore" those packages should be skipped because
+ // embeds from ignored files can't be used.
+ // TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
+ // need to figure this out when we switch to PackagesAndErrors as per the
+ // TODO above.
+ var multiplePackageError *build.MultiplePackageError
+ var noGoError *build.NoGoError
+ if err != nil {
+ if errors.As(err, &noGoError) {
+ return // No source files in this package are built. Skip embeds in ignored files.
+ } else if !errors.As(err, &multiplePackageError) { // multiplePackgeErrors are okay, but others are not.
+ base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
+ }
+ }
+ embedPatterns := str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
+ embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
+ if err != nil {
+ base.Fatalf("go mod vendor: %v", err)
+ }
+ for _, embed := range embeds {
+ embedDst := filepath.Join(dst, embed)
+ if copiedFiles[embedDst] {
+ continue
+ }
+
+ // Copy the file as is done by copyDir below.
+ r, err := os.Open(filepath.Join(src, embed))
+ if err != nil {
+ base.Fatalf("go mod vendor: %v", err)
+ }
+ if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
+ base.Fatalf("go mod vendor: %v", err)
+ }
+ w, err := os.Create(embedDst)
+ if err != nil {
+ base.Fatalf("go mod vendor: %v", err)
+ }
+ if _, err := io.Copy(w, r); err != nil {
+ base.Fatalf("go mod vendor: %v", err)
+ }
+ r.Close()
+ if err := w.Close(); err != nil {
+ base.Fatalf("go mod vendor: %v", err)
+ }
}
}
// copyMetadata copies metadata files from parents of src to parents of dst,
// stopping after processing the src parent for modPath.
-func copyMetadata(modPath, pkg, dst, src string) {
+func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
for parent := 0; ; parent++ {
if copiedMetadata[metakey{modPath, dst}] {
break
}
copiedMetadata[metakey{modPath, dst}] = true
if parent > 0 {
- copyDir(dst, src, matchMetadata)
+ copyDir(dst, src, matchMetadata, copiedFiles)
}
if modPath == pkg {
break
}
// copyDir copies all regular files satisfying match(info) from src to dst.
-func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool) {
+func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
files, err := os.ReadDir(src)
if err != nil {
base.Fatalf("go mod vendor: %v", err)
if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
continue
}
+ copiedFiles[file.Name()] = true
r, err := os.Open(filepath.Join(src, file.Name()))
if err != nil {
base.Fatalf("go mod vendor: %v", err)
}
- w, err := os.Create(filepath.Join(dst, file.Name()))
+ dstPath := filepath.Join(dst, file.Name())
+ copiedFiles[dstPath] = true
+ w, err := os.Create(dstPath)
if err != nil {
base.Fatalf("go mod vendor: %v", err)
}
stdout '\[x\*t\*t\]'
go list -f '{{.EmbedFiles}}'
stdout '\[x.txt\]'
+go list -test -f '{{.TestEmbedPatterns}}'
+stdout '\[y\*t\*t\]'
+go list -test -f '{{.TestEmbedFiles}}'
+stdout '\[y.txt\]'
+go list -test -f '{{.XTestEmbedPatterns}}'
+stdout '\[z\*t\*t\]'
+go list -test -f '{{.XTestEmbedFiles}}'
+stdout '\[z.txt\]'
# build embeds x.txt
go build -x
//go:embed x*t*t
var X embed.FS
+-- x_test.go --
+package p
+
+import "embed"
+
+//go:embed y*t*t
+var Y string
+
+-- x_x_test.go --
+package p_test
+
+import "embed"
+
+//go:embed z*t*t
+var Z string
+
-- x.go2 --
package p
-- x.txt --
hello
+-- y.txt --
+-- z.txt --
-- x.txt2 --
not hello
--- /dev/null
+go mod vendor
+cmp vendor/example.com/a/samedir_embed.txt a/samedir_embed.txt
+cmp vendor/example.com/a/subdir/embed.txt a/subdir/embed.txt
+cmp vendor/example.com/a/subdir/test/embed.txt a/subdir/test/embed.txt
+cmp vendor/example.com/a/subdir/test/xtest/embed.txt a/subdir/test/xtest/embed.txt
+
+cd broken_no_matching_files
+! go mod vendor
+stderr 'go mod vendor: pattern foo.txt: no matching files found'
+
+cd ../broken_bad_pattern
+! go mod vendor
+stderr 'go mod vendor: pattern ../foo.txt: invalid pattern syntax'
+
+# matchPotentialSourceFile prunes out tests and unbuilt code.
+# Make sure that they are vendored if they are embedded files.
+cd ../embed_unbuilt
+go mod vendor
+cmp vendor/example.com/dep/unbuilt.go dep/unbuilt.go
+cmp vendor/example.com/dep/dep_test.go dep/dep_test.go
+! exists vendor/example.com/dep/not_embedded_unbuilt.go
+! exists vendor/example.com/dep/not_embedded_dep_test.go
+-- go.mod --
+module example.com/foo
+go 1.16
+
+require (
+ example.com/a v0.1.0
+)
+
+replace (
+ example.com/a v0.1.0 => ./a
+)
+-- foo.go --
+package main
+
+import (
+ "fmt"
+
+ "example.com/a"
+)
+
+func main() {
+ fmt.Println(a.Str())
+}
+-- a/go.mod --
+module example.com/a
+-- a/a.go --
+package a
+
+import _ "embed"
+
+//go:embed samedir_embed.txt
+var sameDir string
+
+//go:embed subdir/embed.txt
+var subDir string
+
+func Str() string {
+ return sameDir + subDir
+}
+-- a/a_test.go --
+package a
+
+import _ "embed"
+
+//go:embed subdir/test/embed.txt
+var subderTest string
+-- a/a_x_test.go --
+package a_test
+
+import _ "embed"
+
+//go:embed subdir/test/xtest/embed.txt
+var subdirXtest string
+-- a/samedir_embed.txt --
+embedded file in same directory as package
+-- a/subdir/embed.txt --
+embedded file in subdirectory of package
+-- a/subdir/test/embed.txt --
+embedded file of test in subdirectory of package
+-- a/subdir/test/xtest/embed.txt --
+embedded file of xtest in subdirectory of package
+-- broken_no_matching_files/go.mod --
+module example.com/broken
+go 1.16
+
+require (
+ example.com/brokendep v0.1.0
+)
+
+replace (
+ example.com/brokendep v0.1.0 => ./brokendep
+)
+-- broken_no_matching_files/f.go --
+package broken
+
+import _ "example.com/brokendep"
+
+func F() {}
+-- broken_no_matching_files/brokendep/go.mod --
+module example.com/brokendep
+go 1.16
+-- broken_no_matching_files/brokendep/f.go --
+package brokendep
+
+import _ "embed"
+
+//go:embed foo.txt
+var foo string
+-- broken_bad_pattern/go.mod --
+module example.com/broken
+go 1.16
+
+require (
+ example.com/brokendep v0.1.0
+)
+
+replace (
+ example.com/brokendep v0.1.0 => ./brokendep
+)
+-- broken_bad_pattern/f.go --
+package broken
+
+import _ "example.com/brokendep"
+
+func F() {}
+-- broken_bad_pattern/brokendep/go.mod --
+module example.com/brokendep
+go 1.16
+-- broken_bad_pattern/brokendep/f.go --
+package brokendep
+
+import _ "embed"
+
+//go:embed ../foo.txt
+var foo string
+-- embed_unbuilt/go.mod --
+module example.com/foo
+go 1.16
+
+require (
+ example.com/dep v0.1.0
+)
+
+replace (
+ example.com/dep v0.1.0 => ./dep
+)
+-- embed_unbuilt/foo.go --
+package a
+
+import _ "example.com/dep"
+
+func F() {}
+-- embed_unbuilt/dep/go.mod --
+module example.com/dep
+go 1.16
+-- embed_unbuilt/dep/dep.go --
+package dep
+
+import _ "embed"
+
+//go:embed unbuilt.go
+var unbuilt string
+
+//go:embed dep_test.go
+var depTest string
+-- embed_unbuilt/dep/unbuilt.go --
+// +build ignore
+
+package dep
+-- embed_unbuilt/dep/not_embedded_unbuilt.go --
+// +build ignore
+
+package dep
+-- embed_unbuilt/dep/dep_test.go --
+package dep
+-- embed_unbuilt/dep/not_embedded_dep_test.go --
+package dep