From: Bryan C. Mills Date: Thu, 6 Jun 2019 15:08:37 +0000 (-0400) Subject: cmd/go/internal/web: support file:// URLs X-Git-Tag: go1.13beta1~107 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=ec3ebf7bc31271b52c82b3fac193139b2788ed68;p=gostls13.git cmd/go/internal/web: support file:// URLs Fixes #27698 Fixes #32227 Change-Id: I2416408b3de2f9f1ae1af2911cc327a65d2c0170 Reviewed-on: https://go-review.googlesource.com/c/go/+/181037 Reviewed-by: Jay Conrod --- diff --git a/src/cmd/go/internal/get/vcs.go b/src/cmd/go/internal/get/vcs.go index c6516c8bb3..29d58e6350 100644 --- a/src/cmd/go/internal/get/vcs.go +++ b/src/cmd/go/internal/get/vcs.go @@ -856,6 +856,9 @@ func validateRepoRoot(repoRoot string) error { if url.Scheme == "" { return errors.New("no scheme") } + if url.Scheme == "file" { + return errors.New("file scheme disallowed") + } return nil } diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go index c1bc2776b9..50e26625a1 100644 --- a/src/cmd/go/internal/modfetch/proxy.go +++ b/src/cmd/go/internal/modfetch/proxy.go @@ -13,12 +13,9 @@ import ( "net/url" "os" pathpkg "path" - "path/filepath" - "runtime" "strings" "sync" "time" - "unicode" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -201,18 +198,6 @@ func (p *proxyRepo) getBytes(path string) ([]byte, error) { func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) { fullPath := pathpkg.Join(p.url.Path, path) - if p.url.Scheme == "file" { - rawPath, err := url.PathUnescape(fullPath) - if err != nil { - return nil, err - } - if runtime.GOOS == "windows" && len(rawPath) >= 4 && rawPath[0] == '/' && unicode.IsLetter(rune(rawPath[1])) && rawPath[2] == ':' { - // On Windows, file URLs look like "file:///C:/foo/bar". url.Path will - // start with a slash which must be removed. See golang.org/issue/6027. - rawPath = rawPath[1:] - } - return os.Open(filepath.FromSlash(rawPath)) - } target := *p.url target.Path = fullPath diff --git a/src/cmd/go/internal/web/file_test.go b/src/cmd/go/internal/web/file_test.go new file mode 100644 index 0000000000..e31ad71d4d --- /dev/null +++ b/src/cmd/go/internal/web/file_test.go @@ -0,0 +1,58 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +import ( + "errors" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestGetFileURL(t *testing.T) { + const content = "Hello, file!\n" + + f, err := ioutil.TempFile("", "web-TestGetFileURL") + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteString(content); err != nil { + t.Error(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + + u, err := urlFromFilePath(f.Name()) + if err != nil { + t.Fatal(err) + } + + b, err := GetBytes(u) + if err != nil { + t.Fatalf("GetBytes(%v) = _, %v", u, err) + } + if string(b) != content { + t.Fatalf("after writing %q to %s, GetBytes(%v) read %q", content, f.Name(), u, b) + } +} + +func TestGetNonexistentFile(t *testing.T) { + path, err := filepath.Abs("nonexistent") + if err != nil { + t.Fatal(err) + } + + u, err := urlFromFilePath(path) + if err != nil { + t.Fatal(err) + } + + b, err := GetBytes(u) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("GetBytes(%v) = %q, %v; want _, os.ErrNotExist", u, b, err) + } +} diff --git a/src/cmd/go/internal/web/http.go b/src/cmd/go/internal/web/http.go index 4e2b1c33dc..b790fe9916 100644 --- a/src/cmd/go/internal/web/http.go +++ b/src/cmd/go/internal/web/http.go @@ -53,6 +53,11 @@ var securityPreservingHTTPClient = &http.Client{ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) { start := time.Now() + + if url.Scheme == "file" { + return getFile(url) + } + if os.Getenv("TESTGOPROXY404") == "1" && url.Host == "proxy.golang.org" { res := &Response{ URL: Redacted(url), @@ -172,4 +177,41 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) { return r, nil } +func getFile(u *urlpkg.URL) (*Response, error) { + path, err := urlToFilePath(u) + if err != nil { + return nil, err + } + f, err := os.Open(path) + + if os.IsNotExist(err) { + return &Response{ + URL: Redacted(u), + Status: http.StatusText(http.StatusNotFound), + StatusCode: http.StatusNotFound, + Body: http.NoBody, + }, nil + } + + if os.IsPermission(err) { + return &Response{ + URL: Redacted(u), + Status: http.StatusText(http.StatusForbidden), + StatusCode: http.StatusForbidden, + Body: http.NoBody, + }, nil + } + + if err != nil { + return nil, err + } + + return &Response{ + URL: Redacted(u), + Status: http.StatusText(http.StatusOK), + StatusCode: http.StatusOK, + Body: f, + }, nil +} + func openBrowser(url string) bool { return browser.Open(url) } diff --git a/src/cmd/go/internal/web/url.go b/src/cmd/go/internal/web/url.go new file mode 100644 index 0000000000..146c51f0ae --- /dev/null +++ b/src/cmd/go/internal/web/url.go @@ -0,0 +1,95 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +import ( + "errors" + "net/url" + "path/filepath" + "strings" +) + +// TODO(golang.org/issue/32456): If accepted, move these functions into the +// net/url package. + +var errNotAbsolute = errors.New("path is not absolute") + +func urlToFilePath(u *url.URL) (string, error) { + if u.Scheme != "file" { + return "", errors.New("non-file URL") + } + + checkAbs := func(path string) (string, error) { + if !filepath.IsAbs(path) { + return "", errNotAbsolute + } + return path, nil + } + + if u.Path == "" { + if u.Host != "" || u.Opaque == "" { + return "", errors.New("file URL missing path") + } + return checkAbs(filepath.FromSlash(u.Opaque)) + } + + path, err := convertFileURLPath(u.Host, u.Path) + if err != nil { + return path, err + } + return checkAbs(path) +} + +func urlFromFilePath(path string) (*url.URL, error) { + if !filepath.IsAbs(path) { + return nil, errNotAbsolute + } + + // If path has a Windows volume name, convert the volume to a host and prefix + // per https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/. + if vol := filepath.VolumeName(path); vol != "" { + if strings.HasPrefix(vol, `\\`) { + path = filepath.ToSlash(path[2:]) + i := strings.IndexByte(path, '/') + + if i < 0 { + // A degenerate case. + // \\host.example.com (without a share name) + // becomes + // file://host.example.com/ + return &url.URL{ + Scheme: "file", + Host: path, + Path: "/", + }, nil + } + + // \\host.example.com\Share\path\to\file + // becomes + // file://host.example.com/Share/path/to/file + return &url.URL{ + Scheme: "file", + Host: path[:i], + Path: filepath.ToSlash(path[i:]), + }, nil + } + + // C:\path\to\file + // becomes + // file:///C:/path/to/file + return &url.URL{ + Scheme: "file", + Path: "/" + filepath.ToSlash(path), + }, nil + } + + // /path/to/file + // becomes + // file:///path/to/file + return &url.URL{ + Scheme: "file", + Path: filepath.ToSlash(path), + }, nil +} diff --git a/src/cmd/go/internal/web/url_other.go b/src/cmd/go/internal/web/url_other.go new file mode 100644 index 0000000000..bd243e591a --- /dev/null +++ b/src/cmd/go/internal/web/url_other.go @@ -0,0 +1,21 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//+build !windows + +package web + +import ( + "errors" + "path/filepath" +) + +func convertFileURLPath(host, path string) (string, error) { + switch host { + case "", "localhost": + default: + return "", errors.New("file URL specifies non-local host") + } + return filepath.FromSlash(path), nil +} diff --git a/src/cmd/go/internal/web/url_other_test.go b/src/cmd/go/internal/web/url_other_test.go new file mode 100644 index 0000000000..b4a74d94b9 --- /dev/null +++ b/src/cmd/go/internal/web/url_other_test.go @@ -0,0 +1,36 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//+build !windows + +package web + +var urlTests = []struct { + url string + filePath string + canonicalURL string // If empty, assume equal to url. + wantErr string +}{ + // Examples from RFC 8089: + { + url: `file:///path/to/file`, + filePath: `/path/to/file`, + }, + { + url: `file:/path/to/file`, + filePath: `/path/to/file`, + canonicalURL: `file:///path/to/file`, + }, + { + url: `file://localhost/path/to/file`, + filePath: `/path/to/file`, + canonicalURL: `file:///path/to/file`, + }, + + // We reject non-local files. + { + url: `file://host.example.com/path/to/file`, + wantErr: "file URL specifies non-local host", + }, +} diff --git a/src/cmd/go/internal/web/url_test.go b/src/cmd/go/internal/web/url_test.go new file mode 100644 index 0000000000..8f462f5325 --- /dev/null +++ b/src/cmd/go/internal/web/url_test.go @@ -0,0 +1,77 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +import ( + "net/url" + "testing" +) + +func TestURLToFilePath(t *testing.T) { + for _, tc := range urlTests { + if tc.url == "" { + continue + } + tc := tc + + t.Run(tc.url, func(t *testing.T) { + u, err := url.Parse(tc.url) + if err != nil { + t.Fatalf("url.Parse(%q): %v", tc.url, err) + } + + path, err := urlToFilePath(u) + if err != nil { + if err.Error() == tc.wantErr { + return + } + if tc.wantErr == "" { + t.Fatalf("urlToFilePath(%v): %v; want ", u, err) + } else { + t.Fatalf("urlToFilePath(%v): %v; want %s", u, err, tc.wantErr) + } + } + + if path != tc.filePath || tc.wantErr != "" { + t.Fatalf("urlToFilePath(%v) = %q, ; want %q, %s", u, path, tc.filePath, tc.wantErr) + } + }) + } +} + +func TestURLFromFilePath(t *testing.T) { + for _, tc := range urlTests { + if tc.filePath == "" { + continue + } + tc := tc + + t.Run(tc.filePath, func(t *testing.T) { + u, err := urlFromFilePath(tc.filePath) + if err != nil { + if err.Error() == tc.wantErr { + return + } + if tc.wantErr == "" { + t.Fatalf("urlFromFilePath(%v): %v; want ", tc.filePath, err) + } else { + t.Fatalf("urlFromFilePath(%v): %v; want %s", tc.filePath, err, tc.wantErr) + } + } + + if tc.wantErr != "" { + t.Fatalf("urlFromFilePath(%v) = ; want error: %s", tc.filePath, tc.wantErr) + } + + wantURL := tc.url + if tc.canonicalURL != "" { + wantURL = tc.canonicalURL + } + if u.String() != wantURL { + t.Errorf("urlFromFilePath(%v) = %v; want %s", tc.filePath, u, wantURL) + } + }) + } +} diff --git a/src/cmd/go/internal/web/url_windows.go b/src/cmd/go/internal/web/url_windows.go new file mode 100644 index 0000000000..2a65ec83f6 --- /dev/null +++ b/src/cmd/go/internal/web/url_windows.go @@ -0,0 +1,43 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +import ( + "errors" + "path/filepath" + "strings" +) + +func convertFileURLPath(host, path string) (string, error) { + if len(path) == 0 || path[0] != '/' { + return "", errNotAbsolute + } + + path = filepath.FromSlash(path) + + // We interpret Windows file URLs per the description in + // https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/. + + // The host part of a file URL (if any) is the UNC volume name, + // but RFC 8089 reserves the authority "localhost" for the local machine. + if host != "" && host != "localhost" { + // A common "legacy" format omits the leading slash before a drive letter, + // encoding the drive letter as the host instead of part of the path. + // (See https://blogs.msdn.microsoft.com/freeassociations/2005/05/19/the-bizarre-and-unhappy-story-of-file-urls/.) + // We do not support that format, but we should at least emit a more + // helpful error message for it. + if filepath.VolumeName(host) != "" { + return "", errors.New("file URL encodes volume in host field: too few slashes?") + } + return `\\` + host + path, nil + } + + // If host is empty, path must contain an initial slash followed by a + // drive letter and path. Remove the slash and verify that the path is valid. + if vol := filepath.VolumeName(path[1:]); vol == "" || strings.HasPrefix(vol, `\\`) { + return "", errors.New("file URL missing drive letter") + } + return path[1:], nil +} diff --git a/src/cmd/go/internal/web/url_windows_test.go b/src/cmd/go/internal/web/url_windows_test.go new file mode 100644 index 0000000000..06386a0389 --- /dev/null +++ b/src/cmd/go/internal/web/url_windows_test.go @@ -0,0 +1,94 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +var urlTests = []struct { + url string + filePath string + canonicalURL string // If empty, assume equal to url. + wantErr string +}{ + // Examples from https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/: + + { + url: `file://laptop/My%20Documents/FileSchemeURIs.doc`, + filePath: `\\laptop\My Documents\FileSchemeURIs.doc`, + }, + { + url: `file:///C:/Documents%20and%20Settings/davris/FileSchemeURIs.doc`, + filePath: `C:\Documents and Settings\davris\FileSchemeURIs.doc`, + }, + { + url: `file:///D:/Program%20Files/Viewer/startup.htm`, + filePath: `D:\Program Files\Viewer\startup.htm`, + }, + { + url: `file:///C:/Program%20Files/Music/Web%20Sys/main.html?REQUEST=RADIO`, + filePath: `C:\Program Files\Music\Web Sys\main.html`, + canonicalURL: `file:///C:/Program%20Files/Music/Web%20Sys/main.html`, + }, + { + url: `file://applib/products/a-b/abc_9/4148.920a/media/start.swf`, + filePath: `\\applib\products\a-b\abc_9\4148.920a\media\start.swf`, + }, + { + url: `file:////applib/products/a%2Db/abc%5F9/4148.920a/media/start.swf`, + wantErr: "file URL missing drive letter", + }, + { + url: `C:\Program Files\Music\Web Sys\main.html?REQUEST=RADIO`, + wantErr: "non-file URL", + }, + + // The example "file://D:\Program Files\Viewer\startup.htm" errors out in + // url.Parse, so we substitute a slash-based path for testing instead. + { + url: `file://D:/Program Files/Viewer/startup.htm`, + wantErr: "file URL encodes volume in host field: too few slashes?", + }, + + // The blog post discourages the use of non-ASCII characters because they + // depend on the user's current codepage. However, when we are working with Go + // strings we assume UTF-8 encoding, and our url package refuses to encode + // URLs to non-ASCII strings. + { + url: `file:///C:/exampleㄓ.txt`, + filePath: `C:\exampleㄓ.txt`, + canonicalURL: `file:///C:/example%E3%84%93.txt`, + }, + { + url: `file:///C:/example%E3%84%93.txt`, + filePath: `C:\exampleㄓ.txt`, + }, + + // Examples from RFC 8089: + + // We allow the drive-letter variation from section E.2, because it is + // simpler to support than not to. However, we do not generate the shorter + // form in the reverse direction. + { + url: `file:c:/path/to/file`, + filePath: `c:\path\to\file`, + canonicalURL: `file:///c:/path/to/file`, + }, + + // We encode the UNC share name as the authority following section E.3.1, + // because that is what the Microsoft blog post explicitly recommends. + { + url: `file://host.example.com/Share/path/to/file.txt`, + filePath: `\\host.example.com\Share\path\to\file.txt`, + }, + + // We decline the four- and five-slash variations from section E.3.2. + // The paths in these URLs would change meaning under path.Clean. + { + url: `file:////host.example.com/path/to/file`, + wantErr: "file URL missing drive letter", + }, + { + url: `file://///host.example.com/path/to/file`, + wantErr: "file URL missing drive letter", + }, +} diff --git a/src/cmd/go/testdata/script/mod_file_proxy.txt b/src/cmd/go/testdata/script/mod_file_proxy.txt index cf097f8c80..38d9fd25fb 100644 --- a/src/cmd/go/testdata/script/mod_file_proxy.txt +++ b/src/cmd/go/testdata/script/mod_file_proxy.txt @@ -11,11 +11,12 @@ go list grep v1.5.1 $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/list # Use download cache as file:/// proxy. -[windows] stop # TODO: file://$WORK puts backslashes in the URL env GOPATH=$WORK/gopath2 -env GOPROXY=file:///nonexist +[windows] env GOPROXY=file:///C:/nonexist +[!windows] env GOPROXY=file:///nonexist ! go list -env GOPROXY=file://$WORK/gopath1/pkg/mod/cache/download +[windows] env GOPROXY=file:///$WORK/gopath1/pkg/mod/cache/download +[!windows] env GOPROXY=file://$WORK/gopath1/pkg/mod/cache/download go list grep v1.5.1 $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/list diff --git a/src/cmd/go/testdata/script/mod_sumdb_file_path.txt b/src/cmd/go/testdata/script/mod_sumdb_file_path.txt new file mode 100644 index 0000000000..744632ec90 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_sumdb_file_path.txt @@ -0,0 +1,41 @@ +[!net] skip + +env GO111MODULE=on +env GOSUMDB= + +# With a file-based proxy with an empty checksum directory, +# downloading a new module should fail, even if a subsequent +# proxy contains a more complete mirror of the sum database. +[windows] env GOPROXY=file:///$WORK/sumproxy,https://proxy.golang.org +[!windows] env GOPROXY=file://$WORK/sumproxy,https://proxy.golang.org +! go get -d golang.org/x/text@v0.3.2 +stderr '^verifying golang.org/x/text.*: Not Found' + +# If the proxy does not claim to support the database, +# checksum verification should fall through to the next proxy, +# and downloading should succeed. +[windows] env GOPROXY=file:///$WORK/emptyproxy,https://proxy.golang.org +[!windows] env GOPROXY=file://$WORK/emptyproxy,https://proxy.golang.org +go get -d golang.org/x/text@v0.3.2 + +# Once the checksum is present in the go.sum file, +# an empty file-based sumdb can be used in conjunction with +# a fallback module mirror. +grep golang.org/x/text go.sum +go clean -modcache +[windows] env GOPROXY=file:///$WORK/sumproxy +[!windows] env GOPROXY=file://$WORK/sumproxy +! go get -d golang.org/x/text@v0.3.2 +[windows] env GOPROXY=file:///$WORK/sumproxy,https://proxy.golang.org +[!windows] env GOPROXY=file://$WORK/sumproxy,https://proxy.golang.org +go get -d golang.org/x/text@v0.3.2 + +-- go.mod -- +module example.com +go 1.13 +-- $WORK/emptyproxy/README.md -- +This proxy contains no modules. +-- $WORK/sumproxy/README.md -- +This proxy contains no modules. +-- $WORK/sumproxy/sumdb/sum.golang.org/supported -- +This proxy blocks checksum downloads from sum.golang.org.