]> Cypherpunks repositories - gostls13.git/commitdiff
cmd/go: support walking through overlay directories
authorMichael Matloob <matloob@golang.org>
Tue, 6 Oct 2020 17:16:46 +0000 (13:16 -0400)
committerMichael Matloob <matloob@golang.org>
Mon, 12 Oct 2020 21:13:04 +0000 (21:13 +0000)
Change-Id: I7d9d75aa1dbc34fec5073ca36091c626b9dd4920
Reviewed-on: https://go-review.googlesource.com/c/go/+/261537
Trust: Michael Matloob <matloob@golang.org>
Trust: Jay Conrod <jayconrod@google.com>
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
src/cmd/go/internal/fsys/fsys.go
src/cmd/go/internal/fsys/fsys_test.go
src/cmd/go/internal/modload/search.go
src/cmd/go/internal/search/search.go
src/cmd/go/testdata/script/list_overlay.txt

index d64ce0aba1cec19848f25df22aa7d2e47f429e84..489af93496f1ca718f1de586345d71c8d52ba7eb 100644 (file)
@@ -208,8 +208,8 @@ func IsDir(path string) (bool, error) {
 }
 
 // parentIsOverlayFile returns whether name or any of
-// its parents are directories in the overlay, and the first parent found,
-// including name itself, that's a directory in the overlay.
+// its parents are files in the overlay, and the first parent found,
+// including name itself, that's a file in the overlay.
 func parentIsOverlayFile(name string) (string, bool) {
        if overlay != nil {
                // Check if name can't possibly be a directory because
@@ -385,6 +385,80 @@ func IsDirWithGoFiles(dir string) (bool, error) {
        return false, firstErr
 }
 
+// walk recursively descends path, calling walkFn. Copied, with some
+// modifications from path/filepath.walk.
+func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
+       if !info.IsDir() {
+               return walkFn(path, info, nil)
+       }
+
+       fis, readErr := ReadDir(path)
+       walkErr := walkFn(path, info, readErr)
+       // If readErr != nil, walk can't walk into this directory.
+       // walkErr != nil means walkFn want walk to skip this directory or stop walking.
+       // Therefore, if one of readErr and walkErr isn't nil, walk will return.
+       if readErr != nil || walkErr != nil {
+               // The caller's behavior is controlled by the return value, which is decided
+               // by walkFn. walkFn may ignore readErr and return nil.
+               // If walkFn returns SkipDir, it will be handled by the caller.
+               // So walk should return whatever walkFn returns.
+               return walkErr
+       }
+
+       for _, fi := range fis {
+               filename := filepath.Join(path, fi.Name())
+               if walkErr = walk(filename, fi, walkFn); walkErr != nil {
+                       if !fi.IsDir() || walkErr != filepath.SkipDir {
+                               return walkErr
+                       }
+               }
+       }
+       return nil
+}
+
+// Walk walks the file tree rooted at root, calling walkFn for each file or
+// directory in the tree, including root.
+func Walk(root string, walkFn filepath.WalkFunc) error {
+       info, err := lstat(root)
+       if err != nil {
+               err = walkFn(root, nil, err)
+       } else {
+               err = walk(root, info, walkFn)
+       }
+       if err == filepath.SkipDir {
+               return nil
+       }
+       return err
+}
+
+// lstat implements a version of os.Lstat that operates on the overlay filesystem.
+func lstat(path string) (os.FileInfo, error) {
+       cpath := canonicalize(path)
+
+       if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok {
+               return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist}
+       }
+
+       node, ok := overlay[cpath]
+       if !ok {
+               // The file or directory is not overlaid.
+               return os.Lstat(cpath)
+       }
+
+       switch {
+       case node.isDeleted():
+               return nil, &os.PathError{Op: "lstat", Path: cpath, Err: os.ErrNotExist}
+       case node.isDir():
+               return fakeDir(filepath.Base(cpath)), nil
+       default:
+               fi, err := os.Lstat(node.actualFilePath)
+               if err != nil {
+                       return nil, err
+               }
+               return fakeFile{name: filepath.Base(cpath), real: fi}, nil
+       }
+}
+
 // fakeFile provides an os.FileInfo implementation for an overlaid file,
 // so that the file has the name of the overlaid file, but takes all
 // other characteristics of the replacement file.
index 4b530594271cc383c0101eb002e15fddeef75e37..0c3069a6a2277b45987bb45a5b06d6fc8faf9870 100644 (file)
@@ -3,10 +3,12 @@ package fsys
 import (
        "cmd/go/internal/txtar"
        "encoding/json"
+       "errors"
        "fmt"
        "io/ioutil"
        "os"
        "path/filepath"
+       "reflect"
        "testing"
 )
 
@@ -22,7 +24,10 @@ func initOverlay(t *testing.T, config string) {
        if err != nil {
                t.Fatal(err)
        }
-       cwd = t.TempDir()
+       cwd = filepath.Join(t.TempDir(), "root")
+       if err := os.Mkdir(cwd, 0777); err != nil {
+               t.Fatal(err)
+       }
        if err := os.Chdir(cwd); err != nil {
                t.Fatal(err)
        }
@@ -477,3 +482,334 @@ contents don't matter for this test
                }
        }
 }
+
+func TestWalk(t *testing.T) {
+       type file struct {
+               path  string
+               name  string
+               size  int64
+               mode  os.FileMode
+               isDir bool
+       }
+       testCases := []struct {
+               name      string
+               overlay   string
+               root      string
+               wantFiles []file
+       }{
+               {"no overlay", `
+{}
+-- file.txt --
+`,
+                       ".",
+                       []file{
+                               {".", "root", 0, os.ModeDir | 0700, true},
+                               {"file.txt", "file.txt", 0, 0600, false},
+                       },
+               },
+               {"overlay with different file", `
+{
+       "Replace": {
+               "file.txt": "other.txt"
+       }
+}
+-- file.txt --
+-- other.txt --
+contents of other file
+`,
+                       ".",
+                       []file{
+                               {".", "root", 0, os.ModeDir | 0500, true},
+                               {"file.txt", "file.txt", 23, 0600, false},
+                               {"other.txt", "other.txt", 23, 0600, false},
+                       },
+               },
+               {"overlay with new file", `
+{
+       "Replace": {
+               "file.txt": "other.txt"
+       }
+}
+-- other.txt --
+contents of other file
+`,
+                       ".",
+                       []file{
+                               {".", "root", 0, os.ModeDir | 0500, true},
+                               {"file.txt", "file.txt", 23, 0600, false},
+                               {"other.txt", "other.txt", 23, 0600, false},
+                       },
+               },
+               {"overlay with new directory", `
+{
+       "Replace": {
+               "dir/file.txt": "other.txt"
+       }
+}
+-- other.txt --
+contents of other file
+`,
+                       ".",
+                       []file{
+                               {".", "root", 0, os.ModeDir | 0500, true},
+                               {"dir", "dir", 0, os.ModeDir | 0500, true},
+                               {"dir" + string(filepath.Separator) + "file.txt", "file.txt", 23, 0600, false},
+                               {"other.txt", "other.txt", 23, 0600, false},
+                       },
+               },
+       }
+
+       for _, tc := range testCases {
+               t.Run(tc.name, func(t *testing.T) {
+                       initOverlay(t, tc.overlay)
+
+                       var got []file
+                       Walk(tc.root, func(path string, info os.FileInfo, err error) error {
+                               got = append(got, file{path, info.Name(), info.Size(), info.Mode(), info.IsDir()})
+                               return nil
+                       })
+
+                       if len(got) != len(tc.wantFiles) {
+                               t.Errorf("Walk: saw %#v in walk; want %#v", got, tc.wantFiles)
+                       }
+                       for i := 0; i < len(got) && i < len(tc.wantFiles); i++ {
+                               if got[i].path != tc.wantFiles[i].path {
+                                       t.Errorf("path of file #%v in walk, got %q, want %q", i, got[i].path, tc.wantFiles[i].path)
+                               }
+                               if got[i].name != tc.wantFiles[i].name {
+                                       t.Errorf("name of file #%v in walk, got %q, want %q", i, got[i].name, tc.wantFiles[i].name)
+                               }
+                               if got[i].mode&(os.ModeDir|0700) != tc.wantFiles[i].mode {
+                                       t.Errorf("mode&(os.ModeDir|0700) for mode of file #%v in walk, got %v, want %v", i, got[i].mode&(os.ModeDir|0700), tc.wantFiles[i].mode)
+                               }
+                               if got[i].isDir != tc.wantFiles[i].isDir {
+                                       t.Errorf("isDir for file #%v in walk, got %v, want %v", i, got[i].isDir, tc.wantFiles[i].isDir)
+                               }
+                               if tc.wantFiles[i].isDir {
+                                       continue // don't check size for directories
+                               }
+                               if got[i].size != tc.wantFiles[i].size {
+                                       t.Errorf("size of file #%v in walk, got %v, want %v", i, got[i].size, tc.wantFiles[i].size)
+                               }
+                       }
+               })
+       }
+}
+
+func TestWalk_SkipDir(t *testing.T) {
+       initOverlay(t, `
+{
+       "Replace": {
+               "skipthisdir/file.go": "dummy.txt",
+               "dontskip/file.go": "dummy.txt",
+               "dontskip/skip/file.go": "dummy.txt"
+       }
+}
+-- dummy.txt --
+`)
+
+       var seen []string
+       Walk(".", func(path string, info os.FileInfo, err error) error {
+               seen = append(seen, path)
+               if path == "skipthisdir" || path == filepath.Join("dontskip", "skip") {
+                       return filepath.SkipDir
+               }
+               return nil
+       })
+
+       wantSeen := []string{".", "dontskip", filepath.Join("dontskip", "file.go"), filepath.Join("dontskip", "skip"), "dummy.txt", "skipthisdir"}
+
+       if len(seen) != len(wantSeen) {
+               t.Errorf("paths seen in walk: got %v entries; want %v entries", len(seen), len(wantSeen))
+       }
+
+       for i := 0; i < len(seen) && i < len(wantSeen); i++ {
+               if seen[i] != wantSeen[i] {
+                       t.Errorf("path #%v seen walking tree: want %q, got %q", i, seen[i], wantSeen[i])
+               }
+       }
+}
+
+func TestWalk_Error(t *testing.T) {
+       initOverlay(t, "{}")
+
+       alreadyCalled := false
+       err := Walk("foo", func(path string, info os.FileInfo, err error) error {
+               if alreadyCalled {
+                       t.Fatal("expected walk function to be called exactly once, but it was called more than once")
+               }
+               alreadyCalled = true
+               return errors.New("returned from function")
+       })
+       if !alreadyCalled {
+               t.Fatal("expected walk function to be called exactly once, but it was never called")
+
+       }
+       if err == nil {
+               t.Fatalf("Walk: got no error, want error")
+       }
+       if err.Error() != "returned from function" {
+               t.Fatalf("Walk: got error %v, want \"returned from function\" error", err)
+       }
+}
+
+func TestWalk_Symlink(t *testing.T) {
+       initOverlay(t, `{
+       "Replace": {"overlay_symlink": "symlink"}
+}
+-- dir/file --`)
+
+       // Create symlink
+       if err := os.Symlink("dir", "symlink"); err != nil {
+               t.Error(err)
+       }
+
+       testCases := []struct {
+               name      string
+               dir       string
+               wantFiles []string
+       }{
+               {"control", "dir", []string{"dir", "dir" + string(filepath.Separator) + "file"}},
+               // ensure Walk doesn't wolk into the directory pointed to by the symlink
+               // (because it's supposed to use Lstat instead of Stat.
+               {"symlink_to_dir", "symlink", []string{"symlink"}},
+               {"overlay_to_symlink_to_dir", "overlay_symlink", []string{"overlay_symlink"}},
+       }
+
+       for _, tc := range testCases {
+               t.Run(tc.name, func(t *testing.T) {
+                       var got []string
+
+                       err := Walk(tc.dir, func(path string, info os.FileInfo, err error) error {
+                               got = append(got, path)
+                               if err != nil {
+                                       t.Errorf("walkfn: got non nil err argument: %v, want nil err argument", err)
+                               }
+                               return nil
+                       })
+                       if err != nil {
+                               t.Errorf("Walk: got error %q, want nil", err)
+                       }
+
+                       if !reflect.DeepEqual(got, tc.wantFiles) {
+                               t.Errorf("files examined by walk: got %v, want %v", got, tc.wantFiles)
+                       }
+               })
+       }
+
+}
+
+func TestLstat(t *testing.T) {
+       type file struct {
+               name  string
+               size  int64
+               mode  os.FileMode // mode & (os.ModeDir|0x700): only check 'user' permissions
+               isDir bool
+       }
+
+       testCases := []struct {
+               name    string
+               overlay string
+               path    string
+
+               want    file
+               wantErr bool
+       }{
+               {
+                       "regular_file",
+                       `{}
+-- file.txt --
+contents`,
+                       "file.txt",
+                       file{"file.txt", 9, 0600, false},
+                       false,
+               },
+               {
+                       "new_file_in_overlay",
+                       `{"Replace": {"file.txt": "dummy.txt"}}
+-- dummy.txt --
+contents`,
+                       "file.txt",
+                       file{"file.txt", 9, 0600, false},
+                       false,
+               },
+               {
+                       "file_replaced_in_overlay",
+                       `{"Replace": {"file.txt": "dummy.txt"}}
+-- file.txt --
+-- dummy.txt --
+contents`,
+                       "file.txt",
+                       file{"file.txt", 9, 0600, false},
+                       false,
+               },
+               {
+                       "file_cant_exist",
+                       `{"Replace": {"deleted": "dummy.txt"}}
+-- deleted/file.txt --
+-- dummy.txt --
+`,
+                       "deleted/file.txt",
+                       file{},
+                       true,
+               },
+               {
+                       "deleted",
+                       `{"Replace": {"deleted": ""}}
+-- deleted --
+`,
+                       "deleted",
+                       file{},
+                       true,
+               },
+               {
+                       "dir_on_disk",
+                       `{}
+-- dir/foo.txt --
+`,
+                       "dir",
+                       file{"dir", 0, 0700 | os.ModeDir, true},
+                       false,
+               },
+               {
+                       "dir_in_overlay",
+                       `{"Replace": {"dir/file.txt": "dummy.txt"}}
+-- dummy.txt --
+`,
+                       "dir",
+                       file{"dir", 0, 0500 | os.ModeDir, true},
+                       false,
+               },
+       }
+
+       for _, tc := range testCases {
+               t.Run(tc.name, func(t *testing.T) {
+                       initOverlay(t, tc.overlay)
+                       got, err := lstat(tc.path)
+                       if tc.wantErr {
+                               if err == nil {
+                                       t.Errorf("lstat(%q): got no error, want error", tc.path)
+                               }
+                               return
+                       }
+                       if err != nil {
+                               t.Fatalf("lstat(%q): got error %v, want no error", tc.path, err)
+                       }
+                       if got.Name() != tc.want.name {
+                               t.Errorf("lstat(%q).Name(): got %q, want %q", tc.path, got.Name(), tc.want.name)
+                       }
+                       if got.Mode()&(os.ModeDir|0700) != tc.want.mode {
+                               t.Errorf("lstat(%q).Mode()&(os.ModeDir|0700): got %v, want %v", tc.path, got.Mode()&(os.ModeDir|0700), tc.want.mode)
+                       }
+                       if got.IsDir() != tc.want.isDir {
+                               t.Errorf("lstat(%q).IsDir(): got %v, want %v", tc.path, got.IsDir(), tc.want.isDir)
+                       }
+                       if tc.want.isDir {
+                               return // don't check size for directories
+                       }
+                       if got.Size() != tc.want.size {
+                               t.Errorf("lstat(%q).Size(): got %v, want %v", tc.path, got.Size(), tc.want.size)
+                       }
+               })
+       }
+}
index a9bee0af4e0d9b6906ee145b4f0a116008d9c936..0f82026732006db9b06c73ba8166d2df18678966 100644 (file)
@@ -12,6 +12,7 @@ import (
        "strings"
 
        "cmd/go/internal/cfg"
+       "cmd/go/internal/fsys"
        "cmd/go/internal/imports"
        "cmd/go/internal/search"
 
@@ -53,7 +54,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 
        walkPkgs := func(root, importPathRoot string, prune pruning) {
                root = filepath.Clean(root)
-               err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
+               err := fsys.Walk(root, func(path string, fi os.FileInfo, err error) error {
                        if err != nil {
                                m.AddError(err)
                                return nil
index 868dbf5f9d50fb023e0e99807be4a284f73c9ed6..b1d2a9376b20abe4c3f0262e9e556c5e3e119a57 100644 (file)
@@ -7,6 +7,7 @@ package search
 import (
        "cmd/go/internal/base"
        "cmd/go/internal/cfg"
+       "cmd/go/internal/fsys"
        "fmt"
        "go/build"
        "os"
@@ -127,7 +128,7 @@ func (m *Match) MatchPackages() {
                if m.pattern == "cmd" {
                        root += "cmd" + string(filepath.Separator)
                }
-               err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
+               err := fsys.Walk(root, func(path string, fi os.FileInfo, err error) error {
                        if err != nil {
                                return err // Likely a permission error, which could interfere with matching.
                        }
@@ -263,8 +264,7 @@ func (m *Match) MatchDirs() {
                }
        }
 
-       err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-               // TODO(#39958): Handle walk for overlays.
+       err := fsys.Walk(dir, func(path string, fi os.FileInfo, err error) error {
                if err != nil {
                        return err // Likely a permission error, which could interfere with matching.
                }
@@ -273,7 +273,7 @@ func (m *Match) MatchDirs() {
                }
                top := false
                if path == dir {
-                       // filepath.Walk starts at dir and recurses. For the recursive case,
+                       // Walk starts at dir and recurses. For the recursive case,
                        // the path is the result of filepath.Join, which calls filepath.Clean.
                        // The initial case is not Cleaned, though, so we do this explicitly.
                        //
index 7d0e3c2c81bfdc59ab72fb332775b45ad1d1d686..1153975345a9feb286b88f4c9410194922882b24 100644 (file)
@@ -16,8 +16,17 @@ stdout '^\[h.go i.go\]$'
 ! go list ./dir3 # contains a file without a package statement
 go list -overlay overlay.json -f '{{.GoFiles}}' ./dir3 # overlay removes that file
 
+# Walking through an overlay
+go list -overlay overlay.json ./...
+cmp stdout want-list.txt
+
 # TODO(#39958): assembly files, C files, files that require cgo preprocessing
 
+-- want-list.txt --
+m
+m/dir
+m/dir2
+m/dir3
 -- go.mod --
 // TODO(#39958): Support and test overlays including go.mod itself (especially if mod=readonly)
 module m
@@ -34,21 +43,21 @@ package dir3
 -- overlay.json --
 {
     "Replace": {
-        "f.go": "overlay/f.go",
-        "dir/g.go": "overlay/dir_g.go",
-        "dir2/i.go": "overlay/dir2_i.go",
+        "f.go": "overlay/f_go",
+        "dir/g.go": "overlay/dir_g_go",
+        "dir2/i.go": "overlay/dir2_i_go",
         "dir3/bad.go": ""
     }
 }
--- overlay/f.go --
+-- overlay/f_go --
 package m
 
 func f() {
 }
--- overlay/dir_g.go --
+-- overlay/dir_g_go --
 package m
 
 func g() {
 }
--- overlay/dir2_i.go --
+-- overlay/dir2_i_go --
 package dir2