// license that can be found in the LICENSE file.
// Package dirhash defines hashes over directory trees.
+// These hashes are recorded in go.sum files and in the Go checksum database,
+// to allow verifying that a newly-downloaded module has the expected content.
package dirhash
import (
"strings"
)
-var DefaultHash = Hash1
+// DefaultHash is the default hash function used in new go.sum entries.
+var DefaultHash Hash = Hash1
+// A Hash is a directory hash function.
+// It accepts a list of files along with a function that opens the content of each file.
+// It opens, reads, hashes, and closes each file and returns the overall directory hash.
type Hash func(files []string, open func(string) (io.ReadCloser, error)) (string, error)
+// Hash1 is the "h1:" directory hash function, using SHA-256.
+//
+// Hash1 is "h1:" followed by the base64-encoded SHA-256 hash of a summary
+// prepared as if by the Unix command:
+//
+// find . -type f | sort | sha256sum
+//
+// More precisely, the hashed summary contains a single line for each file in the list,
+// ordered by sort.Strings applied to the file names, where each line consists of
+// the hexadecimal SHA-256 hash of the file content,
+// two spaces (U+0020), the file name, and a newline (U+000A).
+//
+// File names with newlines (U+000A) are disallowed.
func Hash1(files []string, open func(string) (io.ReadCloser, error)) (string, error) {
h := sha256.New()
files = append([]string(nil), files...)
sort.Strings(files)
for _, file := range files {
if strings.Contains(file, "\n") {
- return "", errors.New("filenames with newlines are not supported")
+ return "", errors.New("dirhash: filenames with newlines are not supported")
}
r, err := open(file)
if err != nil {
return "h1:" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
}
+// HashDir returns the hash of the local file system directory dir,
+// replacing the directory name itself with prefix in the file names
+// used in the hash function.
func HashDir(dir, prefix string, hash Hash) (string, error) {
files, err := DirFiles(dir, prefix)
if err != nil {
return hash(files, osOpen)
}
+// DirFiles returns the list of files in the tree rooted at dir,
+// replacing the directory name dir with prefix in each name.
+// The resulting names always use forward slashes.
func DirFiles(dir, prefix string) ([]string, error) {
var files []string
dir = filepath.Clean(dir)
return files, nil
}
+// HashZip returns the hash of the file content in the named zip file.
+// Only the file names and their contents are included in the hash:
+// the exact zip file format encoding, compression method,
+// per-file modification times, and other metadata are ignored.
func HashZip(zipfile string, hash Hash) (string, error) {
z, err := zip.OpenReader(zipfile)
if err != nil {
if PkgMod == "" {
return "", fmt.Errorf("internal error: modfetch.PkgMod not set")
}
- enc, err := module.EncodePath(path)
+ enc, err := module.EscapePath(path)
if err != nil {
return "", err
}
if module.CanonicalVersion(m.Version) != m.Version {
return "", fmt.Errorf("non-canonical module version %q", m.Version)
}
- encVer, err := module.EncodeVersion(m.Version)
+ encVer, err := module.EscapeVersion(m.Version)
if err != nil {
return "", err
}
if PkgMod == "" {
return "", fmt.Errorf("internal error: modfetch.PkgMod not set")
}
- enc, err := module.EncodePath(m.Path)
+ enc, err := module.EscapePath(m.Path)
if err != nil {
return "", err
}
if module.CanonicalVersion(m.Version) != m.Version {
return "", fmt.Errorf("non-canonical module version %q", m.Version)
}
- encVer, err := module.EncodeVersion(m.Version)
+ encVer, err := module.EscapeVersion(m.Version)
if err != nil {
return "", err
}
if v == "" || v != module.CanonicalVersion(v) || IsPseudoVersion(v) {
continue
}
- if err := module.MatchPathMajor(v, r.pathMajor); err != nil {
+ if err := module.CheckPathMajor(v, r.pathMajor); err != nil {
if r.codeDir == "" && r.pathMajor == "" && semver.Major(v) > "v1" {
incompatible = append(incompatible, v)
}
}
}
- if err := module.MatchPathMajor(strings.TrimSuffix(info2.Version, "+incompatible"), r.pathMajor); err == nil {
+ if err := module.CheckPathMajor(strings.TrimSuffix(info2.Version, "+incompatible"), r.pathMajor); err == nil {
return nil, invalidf("+incompatible suffix not allowed: major version %s is compatible", semver.Major(info2.Version))
}
}
return checkGoMod()
}
- if err := module.MatchPathMajor(info2.Version, r.pathMajor); err != nil {
+ if err := module.CheckPathMajor(info2.Version, r.pathMajor); err != nil {
if canUseIncompatible() {
info2.Version += "+incompatible"
return checkGoMod()
tagIsCanonical = true
}
- if err := module.MatchPathMajor(v, r.pathMajor); err != nil {
+ if err := module.CheckPathMajor(v, r.pathMajor); err != nil {
if canUseIncompatible() {
return v + "+incompatible", tagIsCanonical
}
}
}()
- if err := module.MatchPathMajor(version, r.pathMajor); err != nil {
+ if err := module.CheckPathMajor(version, r.pathMajor); err != nil {
return err
}
return nil, fmt.Errorf("invalid proxy URL scheme (must be https, http, file): %s", web.Redacted(base))
}
- enc, err := module.EncodePath(path)
+ enc, err := module.EscapePath(path)
if err != nil {
return nil, err
}
}
func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
- encRev, err := module.EncodeVersion(rev)
+ encRev, err := module.EscapeVersion(rev)
if err != nil {
return nil, p.versionError(rev, err)
}
return nil, p.versionError(version, fmt.Errorf("internal error: version passed to GoMod is not canonical"))
}
- encVer, err := module.EncodeVersion(version)
+ encVer, err := module.EscapeVersion(version)
if err != nil {
return nil, p.versionError(version, err)
}
return p.versionError(version, fmt.Errorf("internal error: version passed to Zip is not canonical"))
}
- encVer, err := module.EncodeVersion(version)
+ encVer, err := module.EscapeVersion(version)
if err != nil {
return p.versionError(version, err)
}
"cmd/go/internal/module"
"cmd/go/internal/note"
"cmd/go/internal/str"
- "cmd/go/internal/sumweb"
+ "cmd/go/internal/sumdb"
"cmd/go/internal/web"
)
var (
dbOnce sync.Once
dbName string
- db *sumweb.Conn
+ db *sumdb.Client
dbErr error
)
-func dbDial() (dbName string, db *sumweb.Conn, err error) {
+func dbDial() (dbName string, db *sumdb.Client, err error) {
// $GOSUMDB can be "key" or "key url",
// and the key can be a full verifier key
// or a host on our list of known keys.
base = u
}
- return name, sumweb.NewConn(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
+ return name, sumdb.NewClient(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
}
type dbClient struct {
return err
}
if len(data) > 0 && !bytes.Equal(data, old) {
- return sumweb.ErrWriteConflict
+ return sumdb.ErrWriteConflict
}
if _, err := f.Seek(0, 0); err != nil {
return err
fmt.Fprintf(errs, "%s:%d: %v\n", f.Syntax.Name, line.Start.Line, err)
return
}
- if err := module.MatchPathMajor(v, pathMajor); err != nil {
+ if err := module.CheckPathMajor(v, pathMajor); err != nil {
fmt.Fprintf(errs, "%s:%d: %v\n", f.Syntax.Name, line.Start.Line, &Error{Verb: verb, ModPath: s, Err: err})
return
}
fmt.Fprintf(errs, "%s:%d: %v\n", f.Syntax.Name, line.Start.Line, err)
return
}
- if err := module.MatchPathMajor(v, pathMajor); err != nil {
+ if err := module.CheckPathMajor(v, pathMajor); err != nil {
fmt.Fprintf(errs, "%s:%d: %v\n", f.Syntax.Name, line.Start.Line, &Error{Verb: verb, ModPath: s, Err: err})
return
}
}
}
if vers != "" && module.CanonicalVersion(vers) == vers {
- if err := module.MatchPathMajor(vers, pathMajor); err == nil {
+ if err := module.CheckPathMajor(vers, pathMajor); err == nil {
return vers, nil
}
}
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// Package module defines the module.Version type
-// along with support code.
+// Package module defines the module.Version type along with support code.
+//
+// The module.Version type is a simple Path, Version pair:
+//
+// type Version struct {
+// Path string
+// Version string
+// }
+//
+// There are no restrictions imposed directly by use of this structure,
+// but additional checking functions, most notably Check, verify that
+// a particular path, version pair is valid.
+//
+// Escaped Paths
+//
+// Module paths appear as substrings of file system paths
+// (in the download cache) and of web server URLs in the proxy protocol.
+// In general we cannot rely on file systems to be case-sensitive,
+// nor can we rely on web servers, since they read from file systems.
+// That is, we cannot rely on the file system to keep rsc.io/QUOTE
+// and rsc.io/quote separate. Windows and macOS don't.
+// Instead, we must never require two different casings of a file path.
+// Because we want the download cache to match the proxy protocol,
+// and because we want the proxy protocol to be possible to serve
+// from a tree of static files (which might be stored on a case-insensitive
+// file system), the proxy protocol must never require two different casings
+// of a URL path either.
+//
+// One possibility would be to make the escaped form be the lowercase
+// hexadecimal encoding of the actual path bytes. This would avoid ever
+// needing different casings of a file path, but it would be fairly illegible
+// to most programmers when those paths appeared in the file system
+// (including in file paths in compiler errors and stack traces)
+// in web server logs, and so on. Instead, we want a safe escaped form that
+// leaves most paths unaltered.
+//
+// The safe escaped form is to replace every uppercase letter
+// with an exclamation mark followed by the letter's lowercase equivalent.
+//
+// For example,
+//
+// github.com/Azure/azure-sdk-for-go -> github.com/!azure/azure-sdk-for-go.
+// github.com/GoogleCloudPlatform/cloudsql-proxy -> github.com/!google!cloud!platform/cloudsql-proxy
+// github.com/Sirupsen/logrus -> github.com/!sirupsen/logrus.
+//
+// Import paths that avoid upper-case letters are left unchanged.
+// Note that because import paths are ASCII-only and avoid various
+// problematic punctuation (like : < and >), the escaped form is also ASCII-only
+// and avoids the same problematic punctuation.
+//
+// Import paths have never allowed exclamation marks, so there is no
+// need to define how to escape a literal !.
+//
+// Unicode Restrictions
+//
+// Today, paths are disallowed from using Unicode.
+//
+// Although paths are currently disallowed from using Unicode,
+// we would like at some point to allow Unicode letters as well, to assume that
+// file systems and URLs are Unicode-safe (storing UTF-8), and apply
+// the !-for-uppercase convention for escaping them in the file system.
+// But there are at least two subtle considerations.
+//
+// First, note that not all case-fold equivalent distinct runes
+// form an upper/lower pair.
+// For example, U+004B ('K'), U+006B ('k'), and U+212A ('K' for Kelvin)
+// are three distinct runes that case-fold to each other.
+// When we do add Unicode letters, we must not assume that upper/lower
+// are the only case-equivalent pairs.
+// Perhaps the Kelvin symbol would be disallowed entirely, for example.
+// Or perhaps it would escape as "!!k", or perhaps as "(212A)".
+//
+// Second, it would be nice to allow Unicode marks as well as letters,
+// but marks include combining marks, and then we must deal not
+// only with case folding but also normalization: both U+00E9 ('é')
+// and U+0065 U+0301 ('e' followed by combining acute accent)
+// look the same on the page and are treated by some file systems
+// as the same path. If we do allow Unicode marks in paths, there
+// must be some kind of normalization to allow only one canonical
+// encoding of any character used in an import path.
package module
// IMPORTANT NOTE
"cmd/go/internal/semver"
)
-// A Version is defined by a module path and version pair.
+// A Version (for clients, a module.Version) is defined by a module path and version pair.
+// These are stored in their plain (unescaped) form.
type Version struct {
+ // Path is a module path, like "golang.org/x/text" or "rsc.io/quote/v2".
Path string
// Version is usually a semantic version in canonical form.
Version string `json:",omitempty"`
}
+// String returns the module version syntax Path@Version.
+func (m Version) String() string {
+ return m.Path + "@" + m.Version
+}
+
// A ModuleError indicates an error specific to a module.
type ModuleError struct {
Path string
}
}
_, pathMajor, _ := SplitPathVersion(path)
- if err := MatchPathMajor(version, pathMajor); err != nil {
+ if err := CheckPathMajor(version, pathMajor); err != nil {
return &ModuleError{Path: path, Err: err}
}
return nil
// Paths can be ASCII letters, ASCII digits, and limited ASCII punctuation: + - . _ and ~.
// This matches what "go get" has historically recognized in import paths.
// TODO(rsc): We would like to allow Unicode letters, but that requires additional
-// care in the safe encoding (see note below).
+// care in the safe encoding (see "escaped paths" above).
func pathOK(r rune) bool {
if r < utf8.RuneSelf {
return r == '+' || r == '-' || r == '.' || r == '_' || r == '~' ||
// For now we allow all Unicode letters but otherwise limit to pathOK plus a few more punctuation characters.
// If we expand the set of allowed characters here, we have to
// work harder at detecting potential case-folding and normalization collisions.
-// See note about "safe encoding" below.
+// See note about "escaped paths" above.
func fileNameOK(r rune) bool {
if r < utf8.RuneSelf {
// Entire set of ASCII punctuation, from which we remove characters:
}
// CheckPath checks that a module path is valid.
+// A valid module path is a valid import path, as checked by CheckImportPath,
+// with two additional constraints.
+// First, the leading path element (up to the first slash, if any),
+// by convention a domain name, must contain only lower-case ASCII letters,
+// ASCII digits, dots (U+002E), and dashes (U+002D);
+// it must contain at least one dot and cannot start with a dash.
+// Second, for a final path element of the form /vN, where N looks numeric
+// (ASCII digits and dots) must not begin with a leading zero, must not be /v1,
+// and must not contain any dots. For paths beginning with "gopkg.in/",
+// this second requirement is replaced by a requirement that the path
+// follow the gopkg.in server's conventions.
func CheckPath(path string) error {
if err := checkPath(path, false); err != nil {
return fmt.Errorf("malformed module path %q: %v", path, err)
}
// CheckImportPath checks that an import path is valid.
+//
+// A valid import path consists of one or more valid path elements
+// separated by slashes (U+002F). (It must not begin with nor end in a slash.)
+//
+// A valid path element is a non-empty string made up of
+// ASCII letters, ASCII digits, and limited ASCII punctuation: + - . _ and ~.
+// It must not begin or end with a dot (U+002E), nor contain two dots in a row.
+//
+// The element prefix up to the first dot must not be a reserved file name
+// on Windows, regardless of case (CON, com1, NuL, and so on).
+//
+// CheckImportPath may be less restrictive in the future, but see the
+// top-level package documentation for additional information about
+// subtleties of Unicode.
func CheckImportPath(path string) error {
if err := checkPath(path, false); err != nil {
return fmt.Errorf("malformed import path %q: %v", path, err)
return nil
}
-// CheckFilePath checks whether a slash-separated file path is valid.
+// CheckFilePath checks that a slash-separated file path is valid.
+// The definition of a valid file path is the same as the definition
+// of a valid import path except that the set of allowed characters is larger:
+// all Unicode letters, ASCII digits, the ASCII space character (U+0020),
+// and the ASCII punctuation characters
+// “!#$%&()+,-.=@[]^_{}~”.
+// (The excluded punctuation characters, " * < > ? ` ' | / \ and :,
+// have special meanings in certain shells or operating systems.)
+//
+// CheckFilePath may be less restrictive in the future, but see the
+// top-level package documentation for additional information about
+// subtleties of Unicode.
func CheckFilePath(path string) error {
if err := checkPath(path, true); err != nil {
return fmt.Errorf("malformed file path %q: %v", path, err)
// and version is either empty or "/vN" for N >= 2.
// As a special case, gopkg.in paths are recognized directly;
// they require ".vN" instead of "/vN", and for all N, not just N >= 2.
+// SplitPathVersion returns with ok = false when presented with
+// a path whose last path element does not satisfy the constraints
+// applied by CheckPath, such as "example.com/pkg/v1" or "example.com/pkg/v1.2".
func SplitPathVersion(path string) (prefix, pathMajor string, ok bool) {
if strings.HasPrefix(path, "gopkg.in/") {
return splitGopkgIn(path)
return prefix, pathMajor, true
}
-// MatchPathMajor returns a non-nil error if the semantic version v
+// MatchPathMajor reports whether the semantic version v
+// matches the path major version pathMajor.
+//
+// MatchPathMajor returns true if and only if CheckPathMajor returns non-nil.
+func MatchPathMajor(v, pathMajor string) bool {
+ return CheckPathMajor(v, pathMajor) != nil
+}
+
+// CheckPathMajor returns a non-nil error if the semantic version v
// does not match the path major version pathMajor.
-func MatchPathMajor(v, pathMajor string) error {
+func CheckPathMajor(v, pathMajor string) error {
if strings.HasPrefix(pathMajor, ".v") && strings.HasSuffix(pathMajor, "-unstable") {
pathMajor = strings.TrimSuffix(pathMajor, "-unstable")
}
return cv
}
-// Sort sorts the list by Path, breaking ties by comparing Versions.
+// Sort sorts the list by Path, breaking ties by comparing Version fields.
+// The Version fields are interpreted as semantic versions (using semver.Compare)
+// optionally followed by a tie-breaking suffix introduced by a slash character,
+// like in "v0.0.1/go.mod".
func Sort(list []Version) {
sort.Slice(list, func(i, j int) bool {
mi := list[i]
})
}
-// Safe encodings
-//
-// Module paths appear as substrings of file system paths
-// (in the download cache) and of web server URLs in the proxy protocol.
-// In general we cannot rely on file systems to be case-sensitive,
-// nor can we rely on web servers, since they read from file systems.
-// That is, we cannot rely on the file system to keep rsc.io/QUOTE
-// and rsc.io/quote separate. Windows and macOS don't.
-// Instead, we must never require two different casings of a file path.
-// Because we want the download cache to match the proxy protocol,
-// and because we want the proxy protocol to be possible to serve
-// from a tree of static files (which might be stored on a case-insensitive
-// file system), the proxy protocol must never require two different casings
-// of a URL path either.
-//
-// One possibility would be to make the safe encoding be the lowercase
-// hexadecimal encoding of the actual path bytes. This would avoid ever
-// needing different casings of a file path, but it would be fairly illegible
-// to most programmers when those paths appeared in the file system
-// (including in file paths in compiler errors and stack traces)
-// in web server logs, and so on. Instead, we want a safe encoding that
-// leaves most paths unaltered.
-//
-// The safe encoding is this:
-// replace every uppercase letter with an exclamation mark
-// followed by the letter's lowercase equivalent.
-//
-// For example,
-// github.com/Azure/azure-sdk-for-go -> github.com/!azure/azure-sdk-for-go.
-// github.com/GoogleCloudPlatform/cloudsql-proxy -> github.com/!google!cloud!platform/cloudsql-proxy
-// github.com/Sirupsen/logrus -> github.com/!sirupsen/logrus.
-//
-// Import paths that avoid upper-case letters are left unchanged.
-// Note that because import paths are ASCII-only and avoid various
-// problematic punctuation (like : < and >), the safe encoding is also ASCII-only
-// and avoids the same problematic punctuation.
-//
-// Import paths have never allowed exclamation marks, so there is no
-// need to define how to encode a literal !.
-//
-// Although paths are disallowed from using Unicode (see pathOK above),
-// the eventual plan is to allow Unicode letters as well, to assume that
-// file systems and URLs are Unicode-safe (storing UTF-8), and apply
-// the !-for-uppercase convention. Note however that not all runes that
-// are different but case-fold equivalent are an upper/lower pair.
-// For example, U+004B ('K'), U+006B ('k'), and U+212A ('K' for Kelvin)
-// are considered to case-fold to each other. When we do add Unicode
-// letters, we must not assume that upper/lower are the only case-equivalent pairs.
-// Perhaps the Kelvin symbol would be disallowed entirely, for example.
-// Or perhaps it would encode as "!!k", or perhaps as "(212A)".
-//
-// Also, it would be nice to allow Unicode marks as well as letters,
-// but marks include combining marks, and then we must deal not
-// only with case folding but also normalization: both U+00E9 ('é')
-// and U+0065 U+0301 ('e' followed by combining acute accent)
-// look the same on the page and are treated by some file systems
-// as the same path. If we do allow Unicode marks in paths, there
-// must be some kind of normalization to allow only one canonical
-// encoding of any character used in an import path.
-
-// EncodePath returns the safe encoding of the given module path.
+// EscapePath returns the escaped form of the given module path.
// It fails if the module path is invalid.
-func EncodePath(path string) (encoding string, err error) {
+func EscapePath(path string) (escaped string, err error) {
if err := CheckPath(path); err != nil {
return "", err
}
- return encodeString(path)
+ return escapeString(path)
}
-// EncodeVersion returns the safe encoding of the given module version.
+// EscapeVersion returns the escaped form of the given module version.
// Versions are allowed to be in non-semver form but must be valid file names
// and not contain exclamation marks.
-func EncodeVersion(v string) (encoding string, err error) {
+func EscapeVersion(v string) (escaped string, err error) {
if err := checkElem(v, true); err != nil || strings.Contains(v, "!") {
return "", &InvalidVersionError{
Version: v,
Err: fmt.Errorf("disallowed version string"),
}
}
- return encodeString(v)
+ return escapeString(v)
}
-func encodeString(s string) (encoding string, err error) {
+func escapeString(s string) (escaped string, err error) {
haveUpper := false
for _, r := range s {
if r == '!' || r >= utf8.RuneSelf {
// This should be disallowed by CheckPath, but diagnose anyway.
- // The correctness of the encoding loop below depends on it.
- return "", fmt.Errorf("internal error: inconsistency in EncodePath")
+ // The correctness of the escaping loop below depends on it.
+ return "", fmt.Errorf("internal error: inconsistency in EscapePath")
}
if 'A' <= r && r <= 'Z' {
haveUpper = true
return string(buf), nil
}
-// DecodePath returns the module path of the given safe encoding.
-// It fails if the encoding is invalid or encodes an invalid path.
-func DecodePath(encoding string) (path string, err error) {
- path, ok := decodeString(encoding)
+// UnescapePath returns the module path for the given escaped path.
+// It fails if the escaped path is invalid or describes an invalid path.
+func UnescapePath(escaped string) (path string, err error) {
+ path, ok := unescapeString(escaped)
if !ok {
- return "", fmt.Errorf("invalid module path encoding %q", encoding)
+ return "", fmt.Errorf("invalid escaped module path %q", escaped)
}
if err := CheckPath(path); err != nil {
- return "", fmt.Errorf("invalid module path encoding %q: %v", encoding, err)
+ return "", fmt.Errorf("invalid escaped module path %q: %v", escaped, err)
}
return path, nil
}
-// DecodeVersion returns the version string for the given safe encoding.
-// It fails if the encoding is invalid or encodes an invalid version.
+// UnescapeVersion returns the version string for the given escaped version.
+// It fails if the escaped form is invalid or describes an invalid version.
// Versions are allowed to be in non-semver form but must be valid file names
// and not contain exclamation marks.
-func DecodeVersion(encoding string) (v string, err error) {
- v, ok := decodeString(encoding)
+func UnescapeVersion(escaped string) (v string, err error) {
+ v, ok := unescapeString(escaped)
if !ok {
- return "", fmt.Errorf("invalid version encoding %q", encoding)
+ return "", fmt.Errorf("invalid escaped version %q", escaped)
}
if err := checkElem(v, true); err != nil {
- return "", fmt.Errorf("disallowed version string %q", v)
+ return "", fmt.Errorf("invalid escaped version %q: %v", v, err)
}
return v, nil
}
-func decodeString(encoding string) (string, bool) {
+func unescapeString(escaped string) (string, bool) {
var buf []byte
bang := false
- for _, r := range encoding {
+ for _, r := range escaped {
if r >= utf8.RuneSelf {
return "", false
}
}
}
-var encodeTests = []struct {
+var escapeTests = []struct {
path string
- enc string // empty means same as path
+ esc string // empty means same as path
}{
{path: "ascii.com/abcdefghijklmnopqrstuvwxyz.-+/~_0123456789"},
- {path: "github.com/GoogleCloudPlatform/omega", enc: "github.com/!google!cloud!platform/omega"},
+ {path: "github.com/GoogleCloudPlatform/omega", esc: "github.com/!google!cloud!platform/omega"},
}
-func TestEncodePath(t *testing.T) {
+func TestEscapePath(t *testing.T) {
// Check invalid paths.
for _, tt := range checkPathTests {
if !tt.ok {
- _, err := EncodePath(tt.path)
+ _, err := EscapePath(tt.path)
if err == nil {
- t.Errorf("EncodePath(%q): succeeded, want error (invalid path)", tt.path)
+ t.Errorf("EscapePath(%q): succeeded, want error (invalid path)", tt.path)
}
}
}
// Check encodings.
- for _, tt := range encodeTests {
- enc, err := EncodePath(tt.path)
+ for _, tt := range escapeTests {
+ esc, err := EscapePath(tt.path)
if err != nil {
- t.Errorf("EncodePath(%q): unexpected error: %v", tt.path, err)
+ t.Errorf("EscapePath(%q): unexpected error: %v", tt.path, err)
continue
}
- want := tt.enc
+ want := tt.esc
if want == "" {
want = tt.path
}
- if enc != want {
- t.Errorf("EncodePath(%q) = %q, want %q", tt.path, enc, want)
+ if esc != want {
+ t.Errorf("EscapePath(%q) = %q, want %q", tt.path, esc, want)
}
}
}
-var badDecode = []string{
+var badUnescape = []string{
"github.com/GoogleCloudPlatform/omega",
"github.com/!google!cloud!platform!/omega",
"github.com/!0google!cloud!platform/omega",
"",
}
-func TestDecodePath(t *testing.T) {
+func TestUnescapePath(t *testing.T) {
// Check invalid decodings.
- for _, bad := range badDecode {
- _, err := DecodePath(bad)
+ for _, bad := range badUnescape {
+ _, err := UnescapePath(bad)
if err == nil {
- t.Errorf("DecodePath(%q): succeeded, want error (invalid decoding)", bad)
+ t.Errorf("UnescapePath(%q): succeeded, want error (invalid decoding)", bad)
}
}
// Check invalid paths (or maybe decodings).
for _, tt := range checkPathTests {
if !tt.ok {
- path, err := DecodePath(tt.path)
+ path, err := UnescapePath(tt.path)
if err == nil {
- t.Errorf("DecodePath(%q) = %q, want error (invalid path)", tt.path, path)
+ t.Errorf("UnescapePath(%q) = %q, want error (invalid path)", tt.path, path)
}
}
}
// Check encodings.
- for _, tt := range encodeTests {
- enc := tt.enc
- if enc == "" {
- enc = tt.path
+ for _, tt := range escapeTests {
+ esc := tt.esc
+ if esc == "" {
+ esc = tt.path
}
- path, err := DecodePath(enc)
+ path, err := UnescapePath(esc)
if err != nil {
- t.Errorf("DecodePath(%q): unexpected error: %v", enc, err)
+ t.Errorf("UnescapePath(%q): unexpected error: %v", esc, err)
continue
}
if path != tt.path {
- t.Errorf("DecodePath(%q) = %q, want %q", enc, path, tt.path)
+ t.Errorf("UnescapePath(%q) = %q, want %q", esc, path, tt.path)
}
}
}
Text: string(text),
}
- var buf bytes.Buffer
- buf.Write(text)
-
// Parse and verify signatures.
// Ignore duplicate signatures.
seen := make(map[nameHash]bool)
// Parallel cache.
// This file is copied from cmd/go/internal/par.
-package sumweb
+package sumdb
import (
"sync"
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package sumweb
+package sumdb
import (
"bytes"
"errors"
"fmt"
+ "path"
"strings"
"sync"
"sync/atomic"
+ "cmd/go/internal/module"
"cmd/go/internal/note"
- "cmd/go/internal/str"
"cmd/go/internal/tlog"
)
-// A Client provides the external operations
-// (file caching, HTTP fetches, and so on)
-// needed to implement the HTTP client Conn.
+// A ClientOps provides the external operations
+// (file caching, HTTP fetches, and so on) needed by the Client.
// The methods must be safe for concurrent use by multiple goroutines.
-type Client interface {
+type ClientOps interface {
// ReadRemote reads and returns the content served at the given path
- // on the remote database server. The path begins with "/lookup" or "/tile/".
+ // on the remote database server. The path begins with "/lookup" or "/tile/",
+ // and there is no need to parse the path in any way.
// It is the implementation's responsibility to turn that path into a full URL
// and make the HTTP request. ReadRemote should return an error for
// any non-200 HTTP response status.
// "key" returns a file containing the verifier key for the server.
//
// serverName + "/latest" returns a file containing the latest known
- // signed tree from the server. It is read and written (using WriteConfig).
+ // signed tree from the server.
// To signal that the client wishes to start with an "empty" signed tree,
// ReadConfig can return a successful empty result (0 bytes of data).
ReadConfig(file string) ([]byte, error)
// If the old []byte does not match the stored configuration,
// WriteConfig must return ErrWriteConflict.
// Otherwise, WriteConfig should atomically replace old with new.
+ // The "key" configuration file is never written using WriteConfig.
WriteConfig(file string, old, new []byte) error
// ReadCache reads and returns the content of the named cache file.
Log(msg string)
// SecurityError prints the given security error log message.
- // The Conn returns ErrSecurity from any operation that invokes SecurityError,
+ // The Client returns ErrSecurity from any operation that invokes SecurityError,
// but the return value is mainly for testing. In a real program,
// SecurityError should typically print the message and call log.Fatal or os.Exit.
SecurityError(msg string)
// ErrWriteConflict signals a write conflict during Client.WriteConfig.
var ErrWriteConflict = errors.New("write conflict")
-// ErrSecurity is returned by Conn operations that invoke Client.SecurityError.
+// ErrSecurity is returned by Client operations that invoke Client.SecurityError.
var ErrSecurity = errors.New("security error: misbehaving server")
-// A Conn is a client connection to a go.sum database.
+// A Client is a client connection to a checksum database.
// All the methods are safe for simultaneous use by multiple goroutines.
-type Conn struct {
- client Client // client-provided external world
+type Client struct {
+ ops ClientOps // access to operations in the external world
didLookup uint32
latestMsg []byte // encoded signed note for latest
tileSavedMu sync.Mutex
- tileSaved map[tlog.Tile]bool // which tiles have been saved using c.client.WriteCache already
+ tileSaved map[tlog.Tile]bool // which tiles have been saved using c.ops.WriteCache already
}
-// NewConn returns a new Conn using the given Client.
-func NewConn(client Client) *Conn {
- return &Conn{
- client: client,
+// NewClient returns a new Client using the given Client.
+func NewClient(ops ClientOps) *Client {
+ return &Client{
+ ops: ops,
}
}
-// init initializes the conn (if not already initialized)
+// init initiailzes the client (if not already initialized)
// and returns any initialization error.
-func (c *Conn) init() error {
+func (c *Client) init() error {
c.initOnce.Do(c.initWork)
return c.initErr
}
// initWork does the actual initialization work.
-func (c *Conn) initWork() {
+func (c *Client) initWork() {
defer func() {
if c.initErr != nil {
- c.initErr = fmt.Errorf("initializing sumweb.Conn: %v", c.initErr)
+ c.initErr = fmt.Errorf("initializing sumdb.Client: %v", c.initErr)
}
}()
}
c.tileSaved = make(map[tlog.Tile]bool)
- vkey, err := c.client.ReadConfig("key")
+ vkey, err := c.ops.ReadConfig("key")
if err != nil {
c.initErr = err
return
c.verifiers = note.VerifierList(verifier)
c.name = verifier.Name()
- data, err := c.client.ReadConfig(c.name + "/latest")
+ data, err := c.ops.ReadConfig(c.name + "/latest")
if err != nil {
c.initErr = err
return
}
}
-// SetTileHeight sets the tile height for the Conn.
+// SetTileHeight sets the tile height for the Client.
// Any call to SetTileHeight must happen before the first call to Lookup.
-// If SetTileHeight is not called, the Conn defaults to tile height 8.
-func (c *Conn) SetTileHeight(height int) {
+// If SetTileHeight is not called, the Client defaults to tile height 8.
+// SetTileHeight can be called at most once,
+// and if so it must be called before the first call to Lookup.
+func (c *Client) SetTileHeight(height int) {
if atomic.LoadUint32(&c.didLookup) != 0 {
panic("SetTileHeight used after Lookup")
}
+ if height <= 0 {
+ panic("invalid call to SetTileHeight")
+ }
if c.tileHeight != 0 {
panic("multiple calls to SetTileHeight")
}
c.tileHeight = height
}
-// SetGONOSUMDB sets the list of comma-separated GONOSUMDB patterns for the Conn.
+// SetGONOSUMDB sets the list of comma-separated GONOSUMDB patterns for the Client.
// For any module path matching one of the patterns,
// Lookup will return ErrGONOSUMDB.
-// Any call to SetGONOSUMDB must happen before the first call to Lookup.
-func (c *Conn) SetGONOSUMDB(list string) {
+// SetGONOSUMDB can be called at most once,
+// and if so it must be called before the first call to Lookup.
+func (c *Client) SetGONOSUMDB(list string) {
if atomic.LoadUint32(&c.didLookup) != 0 {
panic("SetGONOSUMDB used after Lookup")
}
// usually from the environment variable).
var ErrGONOSUMDB = errors.New("skipped (listed in GONOSUMDB)")
-func (c *Conn) skip(target string) bool {
- return str.GlobsMatchPath(c.nosumdb, target)
+func (c *Client) skip(target string) bool {
+ return globsMatchPath(c.nosumdb, target)
+}
+
+// globsMatchPath reports whether any path prefix of target
+// matches one of the glob patterns (as defined by path.Match)
+// in the comma-separated globs list.
+// It ignores any empty or malformed patterns in the list.
+func globsMatchPath(globs, target string) bool {
+ for globs != "" {
+ // Extract next non-empty glob in comma-separated list.
+ var glob string
+ if i := strings.Index(globs, ","); i >= 0 {
+ glob, globs = globs[:i], globs[i+1:]
+ } else {
+ glob, globs = globs, ""
+ }
+ if glob == "" {
+ continue
+ }
+
+ // A glob with N+1 path elements (N slashes) needs to be matched
+ // against the first N+1 path elements of target,
+ // which end just before the N+1'th slash.
+ n := strings.Count(glob, "/")
+ prefix := target
+ // Walk target, counting slashes, truncating at the N+1'th slash.
+ for i := 0; i < len(target); i++ {
+ if target[i] == '/' {
+ if n == 0 {
+ prefix = target[:i]
+ break
+ }
+ n--
+ }
+ }
+ if n > 0 {
+ // Not enough prefix elements.
+ continue
+ }
+ matched, _ := path.Match(glob, prefix)
+ if matched {
+ return true
+ }
+ }
+ return false
}
// Lookup returns the go.sum lines for the given module path and version.
// The version may end in a /go.mod suffix, in which case Lookup returns
// the go.sum lines for the module's go.mod-only hash.
-func (c *Conn) Lookup(path, vers string) (lines []string, err error) {
+func (c *Client) Lookup(path, vers string) (lines []string, err error) {
atomic.StoreUint32(&c.didLookup, 1)
if c.skip(path) {
}
// Prepare encoded cache filename / URL.
- epath, err := encodePath(path)
+ epath, err := module.EscapePath(path)
if err != nil {
return nil, err
}
- evers, err := encodeVersion(strings.TrimSuffix(vers, "/go.mod"))
+ evers, err := module.EscapeVersion(strings.TrimSuffix(vers, "/go.mod"))
if err != nil {
return nil, err
}
- file := c.name + "/lookup/" + epath + "@" + evers
remotePath := "/lookup/" + epath + "@" + evers
+ file := c.name + remotePath
// Fetch the data.
// The lookupCache avoids redundant ReadCache/GetURL operations
result := c.record.Do(file, func() interface{} {
// Try the on-disk cache, or else get from web.
writeCache := false
- data, err := c.client.ReadCache(file)
+ data, err := c.ops.ReadCache(file)
if err != nil {
- data, err = c.client.ReadRemote(remotePath)
+ data, err = c.ops.ReadRemote(remotePath)
if err != nil {
return cached{nil, err}
}
// Now that we've validated the record,
// save it to the on-disk cache (unless that's where it came from).
if writeCache {
- c.client.WriteCache(file, data)
+ c.ops.WriteCache(file, data)
}
return cached{data, nil}
}
// mergeLatest merges the tree head in msg
-// with the Conn's current latest tree head,
+// with the Client's current latest tree head,
// ensuring the result is a consistent timeline.
-// If the result is inconsistent, mergeLatest calls c.client.SecurityError
+// If the result is inconsistent, mergeLatest calls c.ops.SecurityError
// with a detailed security error message and then
-// (only if c.client.SecurityError does not exit the program) returns ErrSecurity.
-// If the Conn's current latest tree head moves forward,
+// (only if c.ops.SecurityError does not exit the program) returns ErrSecurity.
+// If the Client's current latest tree head moves forward,
// mergeLatest updates the underlying configuration file as well,
// taking care to merge any independent updates to that configuration.
-func (c *Conn) mergeLatest(msg []byte) error {
+func (c *Client) mergeLatest(msg []byte) error {
// Merge msg into our in-memory copy of the latest tree head.
when, err := c.mergeLatestMem(msg)
if err != nil {
// we need to merge any updates made there as well.
// Note that writeConfig is an atomic compare-and-swap.
for {
- msg, err := c.client.ReadConfig(c.name + "/latest")
+ msg, err := c.ops.ReadConfig(c.name + "/latest")
if err != nil {
return err
}
c.latestMu.Lock()
latestMsg := c.latestMsg
c.latestMu.Unlock()
- if err := c.client.WriteConfig(c.name+"/latest", msg, latestMsg); err != ErrWriteConflict {
+ if err := c.ops.WriteConfig(c.name+"/latest", msg, latestMsg); err != ErrWriteConflict {
// Success or a non-write-conflict error.
return err
}
// msgPast means msg was from before c.latest,
// msgNow means msg was exactly c.latest, and
// msgFuture means msg was from after c.latest, which has now been updated.
-func (c *Conn) mergeLatestMem(msg []byte) (when int, err error) {
+func (c *Client) mergeLatestMem(msg []byte) (when int, err error) {
if len(msg) == 0 {
// Accept empty msg as the unsigned, empty timeline.
c.latestMu.Lock()
// If an error occurs, such as malformed data or a network problem, checkTrees returns that error.
// If on the other hand checkTrees finds evidence of misbehavior, it prepares a detailed
// message and calls log.Fatal.
-func (c *Conn) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, newerNote []byte) error {
+func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, newerNote []byte) error {
thr := tlog.TileHashReader(newer, &c.tileReader)
h, err := tlog.TreeHash(older.N, thr)
if err != nil {
fmt.Fprintf(&buf, "\n\t%v", h)
}
}
- c.client.SecurityError(buf.String())
+ c.ops.SecurityError(buf.String())
return ErrSecurity
}
// checkRecord checks that record #id's hash matches data.
-func (c *Conn) checkRecord(id int64, data []byte) error {
+func (c *Client) checkRecord(id int64, data []byte) error {
c.latestMu.Lock()
latest := c.latest
c.latestMu.Unlock()
return fmt.Errorf("cannot authenticate record data in server response")
}
-// tileReader is a *Conn wrapper that implements tlog.TileReader.
+// tileReader is a *Client wrapper that implements tlog.TileReader.
// The separate type avoids exposing the ReadTiles and SaveTiles
-// methods on Conn itself.
+// methods on Client itself.
type tileReader struct {
- c *Conn
+ c *Client
}
func (r *tileReader) Height() int {
}
// tileCacheKey returns the cache key for the tile.
-func (c *Conn) tileCacheKey(tile tlog.Tile) string {
+func (c *Client) tileCacheKey(tile tlog.Tile) string {
return c.name + "/" + tile.Path()
}
// tileRemotePath returns the remote path for the tile.
-func (c *Conn) tileRemotePath(tile tlog.Tile) string {
+func (c *Client) tileRemotePath(tile tlog.Tile) string {
return "/" + tile.Path()
}
// readTile reads a single tile, either from the on-disk cache or the server.
-func (c *Conn) readTile(tile tlog.Tile) ([]byte, error) {
+func (c *Client) readTile(tile tlog.Tile) ([]byte, error) {
type cached struct {
data []byte
err error
result := c.tileCache.Do(tile, func() interface{} {
// Try the requested tile in on-disk cache.
- data, err := c.client.ReadCache(c.tileCacheKey(tile))
+ data, err := c.ops.ReadCache(c.tileCacheKey(tile))
if err == nil {
c.markTileSaved(tile)
return cached{data, nil}
// We only save authenticated tiles to the on-disk cache,
// so the recreated prefix is equally authenticated.
full := tile
- full.W = 1 << tile.H
+ full.W = 1 << uint(tile.H)
if tile != full {
- data, err := c.client.ReadCache(c.tileCacheKey(full))
+ data, err := c.ops.ReadCache(c.tileCacheKey(full))
if err == nil {
c.markTileSaved(tile) // don't save tile later; we already have full
return cached{data[:len(data)/full.W*tile.W], nil}
}
// Try requested tile from server.
- data, err = c.client.ReadRemote(c.tileRemotePath(tile))
+ data, err = c.ops.ReadRemote(c.tileRemotePath(tile))
if err == nil {
return cached{data, nil}
}
// the tile has been completed and only the complete one
// is available.
if tile != full {
- data, err := c.client.ReadRemote(c.tileRemotePath(full))
+ data, err := c.ops.ReadRemote(c.tileRemotePath(full))
if err == nil {
// Note: We could save the full tile in the on-disk cache here,
// but we don't know if it is valid yet, and we will only find out
// markTileSaved records that tile is already present in the on-disk cache,
// so that a future SaveTiles for that tile can be ignored.
-func (c *Conn) markTileSaved(tile tlog.Tile) {
+func (c *Client) markTileSaved(tile tlog.Tile) {
c.tileSavedMu.Lock()
c.tileSaved[tile] = true
c.tileSavedMu.Unlock()
// c.tileSaved[tile] is still true and we will not try to write it again.
// Next time we run maybe we'll redownload it again and be
// more successful.
- c.client.WriteCache(c.name+"/"+tile.Path(), data[i])
+ c.ops.WriteCache(c.name+"/"+tile.Path(), data[i])
}
}
}
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package sumweb
+package sumdb
import (
"bytes"
testSignerKey = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk"
)
-func TestConnLookup(t *testing.T) {
+func TestClientLookup(t *testing.T) {
tc := newTestClient(t)
tc.mustHaveLatest(1)
tc.mustHaveLatest(4)
}
-func TestConnBadTiles(t *testing.T) {
+func TestClientBadTiles(t *testing.T) {
tc := newTestClient(t)
flipBits := func() {
// Bad tiles in initial download.
tc.mustHaveLatest(1)
flipBits()
- _, err := tc.conn.Lookup("rsc.io/sampler", "v1.3.0")
- tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumweb.Conn: checking tree#1: downloaded inconsistent tile")
+ _, err := tc.client.Lookup("rsc.io/sampler", "v1.3.0")
+ tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile")
flipBits()
- tc.newConn()
+ tc.newClient()
tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
// Bad tiles after initial download.
flipBits()
- _, err = tc.conn.Lookup("rsc.io/Quote", "v1.5.2")
+ _, err = tc.client.Lookup("rsc.io/Quote", "v1.5.2")
tc.mustError(err, "rsc.io/Quote@v1.5.2: checking tree#3 against tree#4: downloaded inconsistent tile")
flipBits()
- tc.newConn()
+ tc.newClient()
tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=")
// Bad starting tree hash looks like bad tiles.
- tc.newConn()
+ tc.newClient()
text := tlog.FormatTree(tlog.Tree{N: 1, Hash: tlog.Hash{}})
data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer)
if err != nil {
tc.t.Fatal(err)
}
tc.config[testName+"/latest"] = data
- _, err = tc.conn.Lookup("rsc.io/sampler", "v1.3.0")
- tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumweb.Conn: checking tree#1: downloaded inconsistent tile")
+ _, err = tc.client.Lookup("rsc.io/sampler", "v1.3.0")
+ tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile")
}
-func TestConnFork(t *testing.T) {
+func TestClientFork(t *testing.T) {
tc := newTestClient(t)
tc2 := tc.fork()
key := "/lookup/rsc.io/pkg1@v1.5.2"
tc2.remote[key] = tc.remote[key]
- _, err := tc2.conn.Lookup("rsc.io/pkg1", "v1.5.2")
+ _, err := tc2.client.Lookup("rsc.io/pkg1", "v1.5.2")
tc2.mustError(err, ErrSecurity.Error())
/*
}
}
-func TestConnGONOSUMDB(t *testing.T) {
+func TestClientGONOSUMDB(t *testing.T) {
tc := newTestClient(t)
- tc.conn.SetGONOSUMDB("p,*/q")
- tc.conn.Lookup("rsc.io/sampler", "v1.3.0") // initialize before we turn off network
+ tc.client.SetGONOSUMDB("p,*/q")
+ tc.client.Lookup("rsc.io/sampler", "v1.3.0") // initialize before we turn off network
tc.getOK = false
ok := []string{
}
for _, path := range ok {
- _, err := tc.conn.Lookup(path, "v1.0.0")
+ _, err := tc.client.Lookup(path, "v1.0.0")
if err == ErrGONOSUMDB {
t.Errorf("Lookup(%q): ErrGONOSUMDB, wanted failed actual lookup", path)
}
}
for _, path := range skip {
- _, err := tc.conn.Lookup(path, "v1.0.0")
+ _, err := tc.client.Lookup(path, "v1.0.0")
if err != ErrGONOSUMDB {
t.Errorf("Lookup(%q): %v, wanted ErrGONOSUMDB", path, err)
}
// A testClient is a self-contained client-side testing environment.
type testClient struct {
t *testing.T // active test
- conn *Conn // conn being tested
+ client *Client // client being tested
tileHeight int // tile height to use (default 2)
getOK bool // should tc.GetURL succeed?
getTileOK bool // should tc.GetURL of tiles succeed?
// mu protects config, cache, log, security
// during concurrent use of the exported methods
- // by the conn itself (testClient is the Conn's Client,
+ // by the client itself (testClient is the Client's ClientOps,
// and the Client methods can both read and write these fields).
// Unexported methods invoked directly by the test
// (for example, addRecord) need not hold the mutex:
// for proper test execution those methods should only
- // be called when the Conn is idle and not using its Client.
+ // be called when the Client is idle and not using its ClientOps.
// Not holding the mutex in those methods ensures
// that if a mistake is made, go test -race will report it.
// (Holding the mutex would eliminate the race report but
t.Fatal(err)
}
- tc.newConn()
+ tc.newClient()
tc.addRecord("rsc.io/quote@v1.5.2", `rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=
rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
return tc
}
-// newConn resets the Conn associated with tc.
-// This clears any in-memory cache from the Conn
+// newClient resets the Client associated with tc.
+// This clears any in-memory cache from the Client
// but not tc's on-disk cache.
-func (tc *testClient) newConn() {
- tc.conn = NewConn(tc)
- tc.conn.SetTileHeight(tc.tileHeight)
+func (tc *testClient) newClient() {
+ tc.client = NewClient(tc)
+ tc.client.SetTileHeight(tc.tileHeight)
}
// mustLookup does a lookup for path@vers and checks that the lines that come back match want.
func (tc *testClient) mustLookup(path, vers, want string) {
tc.t.Helper()
- lines, err := tc.conn.Lookup(path, vers)
+ lines, err := tc.client.Lookup(path, vers)
if err != nil {
tc.t.Fatal(err)
}
cache: copyMap(tc.cache),
remote: copyMap(tc.remote),
}
- tc2.newConn()
+ tc2.newClient()
return tc2
}
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// Package sumweb implements the HTTP protocols for serving or accessing a go.sum database.
-package sumweb
+// Package sumdb implements the HTTP protocols for serving or accessing a module checksum database.
+package sumdb
import (
"context"
"os"
"strings"
+ "cmd/go/internal/module"
"cmd/go/internal/tlog"
)
-// A Server provides the external operations
-// (underlying database access and so on)
-// needed to implement the HTTP server Handler.
-type Server interface {
- // NewContext returns the context to use for the request r.
- NewContext(r *http.Request) (context.Context, error)
-
+// A ServerOps provides the external operations
+// (underlying database access and so on) needed by the Server.
+type ServerOps interface {
// Signed returns the signed hash of the latest tree.
Signed(ctx context.Context) ([]byte, error)
// ReadRecords returns the content for the n records id through id+n-1.
ReadRecords(ctx context.Context, id, n int64) ([][]byte, error)
- // Lookup looks up a record by its associated key ("module@version"),
+ // Lookup looks up a record for the given module,
// returning the record ID.
- Lookup(ctx context.Context, key string) (int64, error)
+ Lookup(ctx context.Context, m module.Version) (int64, error)
// ReadTileData reads the content of tile t.
// It is only invoked for hash tiles (t.L ≥ 0).
ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error)
}
-// A Handler is the go.sum database server handler,
-// which should be invoked to serve the paths listed in Paths.
-// The calling code is responsible for initializing Server.
-type Handler struct {
- Server Server
+// A Server is the checksum database HTTP server,
+// which implements http.Handler and should be invoked
+// to serve the paths listed in ServerPaths.
+type Server struct {
+ ops ServerOps
+}
+
+// NewServer returns a new Server using the given operations.
+func NewServer(ops ServerOps) *Server {
+ return &Server{ops: ops}
}
-// Paths are the URL paths for which Handler should be invoked.
+// ServerPaths are the URL paths the Server can (and should) serve.
//
// Typically a server will do:
//
-// handler := &sumweb.Handler{Server: srv}
-// for _, path := range sumweb.Paths {
-// http.HandleFunc(path, handler)
+// srv := sumdb.NewServer(ops)
+// for _, path := range sumdb.ServerPaths {
+// http.Handle(path, srv)
// }
//
-var Paths = []string{
+var ServerPaths = []string{
"/lookup/",
"/latest",
"/tile/",
var modVerRE = lazyregexp.New(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@]*)?(\+incompatible)?$`)
-func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- ctx, err := h.Server.NewContext(r)
- if err != nil {
- http.Error(w, err.Error(), 500)
- return
- }
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
switch {
default:
return
}
i := strings.Index(mod, "@")
- encPath, encVers := mod[:i], mod[i+1:]
- path, err := decodePath(encPath)
+ escPath, escVers := mod[:i], mod[i+1:]
+ path, err := module.UnescapePath(escPath)
if err != nil {
reportError(w, r, err)
return
}
- vers, err := decodeVersion(encVers)
+ vers, err := module.UnescapeVersion(escVers)
if err != nil {
reportError(w, r, err)
return
}
- id, err := h.Server.Lookup(ctx, path+"@"+vers)
+ id, err := s.ops.Lookup(ctx, module.Version{Path: path, Version: vers})
if err != nil {
reportError(w, r, err)
return
}
- records, err := h.Server.ReadRecords(ctx, id, 1)
+ records, err := s.ops.ReadRecords(ctx, id, 1)
if err != nil {
// This should never happen - the lookup says the record exists.
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
- signed, err := h.Server.Signed(ctx)
+ signed, err := s.ops.Signed(ctx)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
w.Write(signed)
case r.URL.Path == "/latest":
- data, err := h.Server.Signed(ctx)
+ data, err := s.ops.Signed(ctx)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
if t.L == -1 {
// Record data.
start := t.N << uint(t.H)
- records, err := h.Server.ReadRecords(ctx, start, int64(t.W))
+ records, err := s.ops.ReadRecords(ctx, start, int64(t.W))
if err != nil {
reportError(w, r, err)
return
return
}
- data, err := h.Server.ReadTileData(ctx, t)
+ data, err := s.ops.ReadTileData(ctx, t)
if err != nil {
reportError(w, r, err)
return
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package sumweb
+package sumdb
import (
"context"
"fmt"
- "net/http"
- "strings"
"sync"
+ "cmd/go/internal/module"
"cmd/go/internal/note"
"cmd/go/internal/tlog"
)
// NewTestServer constructs a new TestServer
// that will sign its tree with the given signer key
-// (see cmd/go/internal/note)
+// (see golang.org/x/mod/sumdb/note)
// and fetch new records as needed by calling gosum.
func NewTestServer(signer string, gosum func(path, vers string) ([]byte, error)) *TestServer {
return &TestServer{signer: signer, gosum: gosum}
return list, nil
}
-func (s *TestServer) NewContext(r *http.Request) (context.Context, error) {
- return nil, nil
-}
-
func (s *TestServer) Signed(ctx context.Context) ([]byte, error) {
s.mu.Lock()
defer s.mu.Unlock()
return list, nil
}
-func (s *TestServer) Lookup(ctx context.Context, key string) (int64, error) {
+func (s *TestServer) Lookup(ctx context.Context, m module.Version) (int64, error) {
+ key := m.String()
s.mu.Lock()
id, ok := s.lookup[key]
s.mu.Unlock()
}
// Look up module and compute go.sum lines.
- i := strings.Index(key, "@")
- if i < 0 {
- return 0, fmt.Errorf("invalid lookup key %q", key)
- }
- path, vers := key[:i], key[i+1:]
- data, err := s.gosum(path, vers)
+ data, err := s.gosum(m.Path, m.Version)
if err != nil {
return 0, err
}
+++ /dev/null
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// FS-safe encoding of module paths and versions.
-// Copied from cmd/go/internal/module and unexported.
-
-package sumweb
-
-import (
- "fmt"
- "unicode/utf8"
-)
-
-// Safe encodings
-//
-// Module paths appear as substrings of file system paths
-// (in the download cache) and of web server URLs in the proxy protocol.
-// In general we cannot rely on file systems to be case-sensitive,
-// nor can we rely on web servers, since they read from file systems.
-// That is, we cannot rely on the file system to keep rsc.io/QUOTE
-// and rsc.io/quote separate. Windows and macOS don't.
-// Instead, we must never require two different casings of a file path.
-// Because we want the download cache to match the proxy protocol,
-// and because we want the proxy protocol to be possible to serve
-// from a tree of static files (which might be stored on a case-insensitive
-// file system), the proxy protocol must never require two different casings
-// of a URL path either.
-//
-// One possibility would be to make the safe encoding be the lowercase
-// hexadecimal encoding of the actual path bytes. This would avoid ever
-// needing different casings of a file path, but it would be fairly illegible
-// to most programmers when those paths appeared in the file system
-// (including in file paths in compiler errors and stack traces)
-// in web server logs, and so on. Instead, we want a safe encoding that
-// leaves most paths unaltered.
-//
-// The safe encoding is this:
-// replace every uppercase letter with an exclamation mark
-// followed by the letter's lowercase equivalent.
-//
-// For example,
-// github.com/Azure/azure-sdk-for-go -> github.com/!azure/azure-sdk-for-go.
-// github.com/GoogleCloudPlatform/cloudsql-proxy -> github.com/!google!cloud!platform/cloudsql-proxy
-// github.com/Sirupsen/logrus -> github.com/!sirupsen/logrus.
-//
-// Import paths that avoid upper-case letters are left unchanged.
-// Note that because import paths are ASCII-only and avoid various
-// problematic punctuation (like : < and >), the safe encoding is also ASCII-only
-// and avoids the same problematic punctuation.
-//
-// Import paths have never allowed exclamation marks, so there is no
-// need to define how to encode a literal !.
-//
-// Although paths are disallowed from using Unicode (see pathOK above),
-// the eventual plan is to allow Unicode letters as well, to assume that
-// file systems and URLs are Unicode-safe (storing UTF-8), and apply
-// the !-for-uppercase convention. Note however that not all runes that
-// are different but case-fold equivalent are an upper/lower pair.
-// For example, U+004B ('K'), U+006B ('k'), and U+212A ('K' for Kelvin)
-// are considered to case-fold to each other. When we do add Unicode
-// letters, we must not assume that upper/lower are the only case-equivalent pairs.
-// Perhaps the Kelvin symbol would be disallowed entirely, for example.
-// Or perhaps it would encode as "!!k", or perhaps as "(212A)".
-//
-// Also, it would be nice to allow Unicode marks as well as letters,
-// but marks include combining marks, and then we must deal not
-// only with case folding but also normalization: both U+00E9 ('é')
-// and U+0065 U+0301 ('e' followed by combining acute accent)
-// look the same on the page and are treated by some file systems
-// as the same path. If we do allow Unicode marks in paths, there
-// must be some kind of normalization to allow only one canonical
-// encoding of any character used in an import path.
-
-// encodePath returns the safe encoding of the given module path.
-// It fails if the module path is invalid.
-func encodePath(path string) (encoding string, err error) {
- return encodeString(path)
-}
-
-// encodeVersion returns the safe encoding of the given module version.
-// Versions are allowed to be in non-semver form but must be valid file names
-// and not contain exclamation marks.
-func encodeVersion(v string) (encoding string, err error) {
- return encodeString(v)
-}
-
-func encodeString(s string) (encoding string, err error) {
- haveUpper := false
- for _, r := range s {
- if r == '!' || r >= utf8.RuneSelf {
- // This should be disallowed by CheckPath, but diagnose anyway.
- // The correctness of the encoding loop below depends on it.
- return "", fmt.Errorf("internal error: inconsistency in EncodePath")
- }
- if 'A' <= r && r <= 'Z' {
- haveUpper = true
- }
- }
-
- if !haveUpper {
- return s, nil
- }
-
- var buf []byte
- for _, r := range s {
- if 'A' <= r && r <= 'Z' {
- buf = append(buf, '!', byte(r+'a'-'A'))
- } else {
- buf = append(buf, byte(r))
- }
- }
- return string(buf), nil
-}
-
-// decodePath returns the module path of the given safe encoding.
-// It fails if the encoding is invalid or encodes an invalid path.
-func decodePath(encoding string) (path string, err error) {
- path, ok := decodeString(encoding)
- if !ok {
- return "", fmt.Errorf("invalid module path encoding %q", encoding)
- }
- return path, nil
-}
-
-// decodeVersion returns the version string for the given safe encoding.
-// It fails if the encoding is invalid or encodes an invalid version.
-// Versions are allowed to be in non-semver form but must be valid file names
-// and not contain exclamation marks.
-func decodeVersion(encoding string) (v string, err error) {
- v, ok := decodeString(encoding)
- if !ok {
- return "", fmt.Errorf("invalid version encoding %q", encoding)
- }
- return v, nil
-}
-
-func decodeString(encoding string) (string, bool) {
- var buf []byte
-
- bang := false
- for _, r := range encoding {
- if r >= utf8.RuneSelf {
- return "", false
- }
- if bang {
- bang = false
- if r < 'a' || 'z' < r {
- return "", false
- }
- buf = append(buf, byte(r+'A'-'a'))
- continue
- }
- if r == '!' {
- bang = true
- continue
- }
- if 'A' <= r && r <= 'Z' {
- return "", false
- }
- buf = append(buf, byte(r))
- }
- if bang {
- return "", false
- }
- return string(buf), true
-}
+++ /dev/null
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package sumweb
-
-import "testing"
-
-var encodeTests = []struct {
- path string
- enc string // empty means same as path
-}{
- {path: "ascii.com/abcdefghijklmnopqrstuvwxyz.-+/~_0123456789"},
- {path: "github.com/GoogleCloudPlatform/omega", enc: "github.com/!google!cloud!platform/omega"},
-}
-
-func TestEncodePath(t *testing.T) {
- // Check encodings.
- for _, tt := range encodeTests {
- enc, err := encodePath(tt.path)
- if err != nil {
- t.Errorf("encodePath(%q): unexpected error: %v", tt.path, err)
- continue
- }
- want := tt.enc
- if want == "" {
- want = tt.path
- }
- if enc != want {
- t.Errorf("encodePath(%q) = %q, want %q", tt.path, enc, want)
- }
- }
-}
-
-var badDecode = []string{
- "github.com/GoogleCloudPlatform/omega",
- "github.com/!google!cloud!platform!/omega",
- "github.com/!0google!cloud!platform/omega",
- "github.com/!_google!cloud!platform/omega",
- "github.com/!!google!cloud!platform/omega",
-}
-
-func TestDecodePath(t *testing.T) {
- // Check invalid decodings.
- for _, bad := range badDecode {
- _, err := decodePath(bad)
- if err == nil {
- t.Errorf("DecodePath(%q): succeeded, want error (invalid decoding)", bad)
- }
- }
-
- // Check encodings.
- for _, tt := range encodeTests {
- enc := tt.enc
- if enc == "" {
- enc = tt.path
- }
- path, err := decodePath(enc)
- if err != nil {
- t.Errorf("decodePath(%q): unexpected error: %v", enc, err)
- continue
- }
- if path != tt.path {
- t.Errorf("decodePath(%q) = %q, want %q", enc, path, tt.path)
- }
- }
-}
var errMalformedTree = errors.New("malformed tree note")
var treePrefix = []byte("go.sum database tree\n")
-// ParseTree parses a tree root description.
+// ParseTree parses a formatted tree root description.
func ParseTree(text []byte) (tree Tree, err error) {
// The message looks like:
//
// The special level L=-1 holds raw record data instead of hashes.
// In this case, the level encodes into a tile path as the path element
// "data" instead of "-1".
+//
+// See also https://golang.org/design/25530-sumdb#checksum-database
+// and https://research.swtch.com/tlog#tiling_a_log.
type Tile struct {
H int // height of tile (1 ≤ H ≤ 30)
L int // level in tiling (-1 ≤ L ≤ 63)
W int // width of tile (1 ≤ W ≤ 2**H; 2**H is complete tile)
}
-// TileForIndex returns the tile of height h ≥ 1
+// TileForIndex returns the tile of fixed height h ≥ 1
// and least width storing the given hash storage index.
+//
+// If h ≤ 0, TileForIndex panics.
func TileForIndex(h int, index int64) Tile {
- if h < 1 {
- panic("TileForIndex: invalid height")
+ if h <= 0 {
+ panic(fmt.Sprintf("TileForIndex: invalid height %d", h))
}
t, _, _ := tileForIndex(h, index)
return t
// that must be published when publishing from a tree of
// size newTreeSize to replace a tree of size oldTreeSize.
// (No tiles need to be published for a tree of size zero.)
+//
+// If h ≤ 0, TileForIndex panics.
func NewTiles(h int, oldTreeSize, newTreeSize int64) []Tile {
- if h < 1 {
+ if h <= 0 {
panic(fmt.Sprintf("NewTiles: invalid height %d", h))
}
H := uint(h)
// a data record for each tile (len(data) == len(tiles))
// and each data record must be the correct length
// (len(data[i]) == tiles[i].W*HashSize).
+ //
+ // An implementation of ReadTiles typically reads
+ // them from an on-disk cache or else from a remote
+ // tile server. Tile data downloaded from a server should
+ // be considered suspect and not saved into a persistent
+ // on-disk cache before returning from ReadTiles.
+ // When the client confirms the validity of the tile data,
+ // it will call SaveTiles to signal that they can be safely
+ // written to persistent storage.
+ // See also https://research.swtch.com/tlog#authenticating_tiles.
ReadTiles(tiles []Tile) (data [][]byte, err error)
// SaveTiles informs the TileReader that the tile data
// Package tlog implements a tamper-evident log
// used in the Go module go.sum database server.
//
-// This package is part of a DRAFT of what the go.sum database server will look like.
-// Do not assume the details here are final!
-//
// This package follows the design of Certificate Transparency (RFC 6962)
// and its proofs are compatible with that system.
// See TestCertificateTransparency.
"cmd/go/internal/module"
"cmd/go/internal/par"
"cmd/go/internal/semver"
- "cmd/go/internal/sumweb"
+ "cmd/go/internal/sumdb"
"cmd/go/internal/txtar"
)
// Prepopulate main sumdb.
for _, mod := range modList {
- sumdbHandler.Server.Lookup(nil, mod.Path+"@"+mod.Version)
+ sumdbOps.Lookup(nil, mod)
}
})
}
continue
}
encPath := strings.ReplaceAll(name[:i], "_", "/")
- path, err := module.DecodePath(encPath)
+ path, err := module.UnescapePath(encPath)
if err != nil {
if encPath != "example.com/invalidpath/v1" {
fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
continue
}
encVers := name[i+1:]
- vers, err := module.DecodeVersion(encVers)
+ vers, err := module.UnescapeVersion(encVers)
if err != nil {
fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
continue
testSumDBSignerKey = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk"
)
-var sumdbHandler = &sumweb.Handler{Server: sumweb.NewTestServer(testSumDBSignerKey, proxyGoSum)}
-var sumdbWrongHandler = &sumweb.Handler{Server: sumweb.NewTestServer(testSumDBSignerKey, proxyGoSumWrong)}
+var (
+ sumdbOps = sumdb.NewTestServer(testSumDBSignerKey, proxyGoSum)
+ sumdbServer = sumdb.NewServer(sumdbOps)
+
+ sumdbWrongOps = sumdb.NewTestServer(testSumDBSignerKey, proxyGoSumWrong)
+ sumdbWrongServer = sumdb.NewServer(sumdbWrongOps)
+)
// proxyHandler serves the Go module proxy protocol.
// See the proxy section of https://research.swtch.com/vgo-module.
// (Client thinks it is talking directly to a sumdb.)
if strings.HasPrefix(path, "sumdb-direct/") {
r.URL.Path = path[len("sumdb-direct"):]
- sumdbHandler.ServeHTTP(w, r)
+ sumdbServer.ServeHTTP(w, r)
return
}
// (Client thinks it is talking directly to a sumdb.)
if strings.HasPrefix(path, "sumdb-wrong/") {
r.URL.Path = path[len("sumdb-wrong"):]
- sumdbWrongHandler.ServeHTTP(w, r)
+ sumdbWrongServer.ServeHTTP(w, r)
return
}
// Request for $GOPROXY/sumdb/<name>/... goes to sumdb.
if sumdbPrefix := "sumdb/" + testSumDBName + "/"; strings.HasPrefix(path, sumdbPrefix) {
r.URL.Path = path[len(sumdbPrefix)-1:]
- sumdbHandler.ServeHTTP(w, r)
+ sumdbServer.ServeHTTP(w, r)
return
}
// latest version, including pseudo-versions.
if i := strings.LastIndex(path, "/@latest"); i >= 0 {
enc := path[:i]
- modPath, err := module.DecodePath(enc)
+ modPath, err := module.UnescapePath(enc)
if err != nil {
if !quiet {
fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
return
}
- encVers, err := module.EncodeVersion(latest)
+ encVers, err := module.EscapeVersion(latest)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
return
}
enc, file := path[:i], path[i+len("/@v/"):]
- path, err := module.DecodePath(enc)
+ path, err := module.UnescapePath(enc)
if err != nil {
if !quiet {
fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
return
}
encVers, ext := file[:i], file[i+1:]
- vers, err := module.DecodeVersion(encVers)
+ vers, err := module.UnescapeVersion(encVers)
if err != nil {
fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
http.NotFound(w, r)
var cmdGoDir, _ = os.Getwd()
func readArchive(path, vers string) (*txtar.Archive, error) {
- enc, err := module.EncodePath(path)
+ enc, err := module.EscapePath(path)
if err != nil {
return nil, err
}
- encVers, err := module.EncodeVersion(vers)
+ encVers, err := module.EscapeVersion(vers)
if err != nil {
return nil, err
}