From bf869c65d1a96da5db78a891430ea3acd7ddf1ab Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 13 Apr 2020 16:33:28 -0400 Subject: [PATCH] cmd/go: update vendored golang.org/x/mod This CL vendors go.mod parser changes for the retract directive. For #24031 Change-Id: Ief19b0eca4c7956eceadc893bb209da7e9ecf22c Reviewed-on: https://go-review.googlesource.com/c/go/+/228377 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills Reviewed-by: Michael Matloob --- src/cmd/go.mod | 2 +- src/cmd/go.sum | 5 +- .../vendor/golang.org/x/mod/modfile/read.go | 8 + .../vendor/golang.org/x/mod/modfile/rule.go | 243 ++++++++++++++++-- src/cmd/vendor/modules.txt | 2 +- 5 files changed, 235 insertions(+), 25 deletions(-) diff --git a/src/cmd/go.mod b/src/cmd/go.mod index 21670b9996..5c5c99e3cd 100644 --- a/src/cmd/go.mod +++ b/src/cmd/go.mod @@ -7,7 +7,7 @@ require ( github.com/ianlancetaylor/demangle v0.0.0-20200414190113-039b1ae3a340 // indirect golang.org/x/arch v0.0.0-20200511175325-f7c78586839d golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 + golang.org/x/mod v0.3.1-0.20200824162228-c0d644d00ab8 golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 // indirect golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 golang.org/x/xerrors v0.0.0-20200806184451-1a77d5e9f316 // indirect diff --git a/src/cmd/go.sum b/src/cmd/go.sum index 1b5ef515c2..69cebe1b23 100644 --- a/src/cmd/go.sum +++ b/src/cmd/go.sum @@ -14,8 +14,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 h1:R11LxkoUvECaAHdM5/ZOevSR7n+016EgTw8nbE1l+XM= -golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200824162228-c0d644d00ab8 h1:Qbq3laTJZip3mEOreFwHF81RGkkhIvmraRMINHNyWHE= +golang.org/x/mod v0.3.1-0.20200824162228-c0d644d00ab8/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -32,7 +32,6 @@ golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGg golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200806184451-1a77d5e9f316 h1:Jhw4VC65LaKnpq9FvcK+a8ZzrFm3D+UygvMMrhkOw70= golang.org/x/xerrors v0.0.0-20200806184451-1a77d5e9f316/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/src/cmd/vendor/golang.org/x/mod/modfile/read.go b/src/cmd/vendor/golang.org/x/mod/modfile/read.go index c1f2008ee4..2a961ca81c 100644 --- a/src/cmd/vendor/golang.org/x/mod/modfile/read.go +++ b/src/cmd/vendor/golang.org/x/mod/modfile/read.go @@ -477,9 +477,17 @@ func (in *input) startToken() { // endToken marks the end of an input token. // It records the actual token string in tok.text. +// A single trailing newline (LF or CRLF) will be removed from comment tokens. func (in *input) endToken(kind tokenKind) { in.token.kind = kind text := string(in.tokenStart[:len(in.tokenStart)-len(in.remaining)]) + if kind.isComment() { + if strings.HasSuffix(text, "\r\n") { + text = text[:len(text)-2] + } else { + text = strings.TrimSuffix(text, "\n") + } + } in.token.text = text in.token.endPos = in.pos } diff --git a/src/cmd/vendor/golang.org/x/mod/modfile/rule.go b/src/cmd/vendor/golang.org/x/mod/modfile/rule.go index 91ca6828df..83398dda5d 100644 --- a/src/cmd/vendor/golang.org/x/mod/modfile/rule.go +++ b/src/cmd/vendor/golang.org/x/mod/modfile/rule.go @@ -30,6 +30,7 @@ import ( "golang.org/x/mod/internal/lazyregexp" "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) // A File is the parsed, interpreted form of a go.mod file. @@ -39,6 +40,7 @@ type File struct { Require []*Require Exclude []*Exclude Replace []*Replace + Retract []*Retract Syntax *FileSyntax } @@ -75,6 +77,21 @@ type Replace struct { Syntax *Line } +// A Retract is a single retract statement. +type Retract struct { + VersionInterval + Rationale string + Syntax *Line +} + +// A VersionInterval represents a range of versions with upper and lower bounds. +// Intervals are closed: both bounds are included. When Low is equal to High, +// the interval may refer to a single version ('v1.2.3') or an interval +// ('[v1.2.3, v1.2.3]'); both have the same representation. +type VersionInterval struct { + Low, High string +} + func (f *File) AddModuleStmt(path string) error { if f.Syntax == nil { f.Syntax = new(FileSyntax) @@ -138,7 +155,7 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File for _, x := range fs.Stmt { switch x := x.(type) { case *Line: - f.add(&errs, x, x.Token[0], x.Token[1:], fix, strict) + f.add(&errs, nil, x, x.Token[0], x.Token[1:], fix, strict) case *LineBlock: if len(x.Token) > 1 { @@ -161,9 +178,9 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File }) } continue - case "module", "require", "exclude", "replace": + case "module", "require", "exclude", "replace", "retract": for _, l := range x.Line { - f.add(&errs, l, x.Token[0], l.Token, fix, strict) + f.add(&errs, x, l, x.Token[0], l.Token, fix, strict) } } } @@ -177,7 +194,7 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File var GoVersionRE = lazyregexp.New(`^([1-9][0-9]*)\.(0|[1-9][0-9]*)$`) -func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix VersionFixer, strict bool) { +func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, args []string, fix VersionFixer, strict bool) { // If strict is false, this module is a dependency. // We ignore all unknown directives as well as main-module-only // directives like replace and exclude. It will work better for @@ -186,7 +203,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix // and simply ignore those statements. if !strict { switch verb { - case "module", "require", "go": + case "go", "module", "retract", "require": // want these even for dependency go.mods default: return @@ -232,6 +249,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix f.Go = &Go{Syntax: line} f.Go.Version = args[0] + case "module": if f.Module != nil { errorf("repeated module statement") @@ -248,6 +266,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix return } f.Module.Mod = module.Version{Path: s} + case "require", "exclude": if len(args) != 2 { errorf("usage: %s module/path v1.2.3", verb) @@ -284,6 +303,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix Syntax: line, }) } + case "replace": arrow := 2 if len(args) >= 2 && args[1] == "=>" { @@ -347,6 +367,33 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix New: module.Version{Path: ns, Version: nv}, Syntax: line, }) + + case "retract": + rationale := parseRetractRationale(block, line) + vi, err := parseVersionInterval(verb, &args, fix) + if err != nil { + if strict { + wrapError(err) + return + } else { + // Only report errors parsing intervals in the main module. We may + // support additional syntax in the future, such as open and half-open + // intervals. Those can't be supported now, because they break the + // go.mod parser, even in lax mode. + return + } + } + if len(args) > 0 && strict { + // In the future, there may be additional information after the version. + errorf("unexpected token after version: %q", args[0]) + return + } + retract := &Retract{ + VersionInterval: vi, + Rationale: rationale, + Syntax: line, + } + f.Retract = append(f.Retract, retract) } } @@ -444,6 +491,53 @@ func AutoQuote(s string) string { return s } +func parseVersionInterval(verb string, args *[]string, fix VersionFixer) (VersionInterval, error) { + toks := *args + if len(toks) == 0 || toks[0] == "(" { + return VersionInterval{}, fmt.Errorf("expected '[' or version") + } + if toks[0] != "[" { + v, err := parseVersion(verb, "", &toks[0], fix) + if err != nil { + return VersionInterval{}, err + } + *args = toks[1:] + return VersionInterval{Low: v, High: v}, nil + } + toks = toks[1:] + + if len(toks) == 0 { + return VersionInterval{}, fmt.Errorf("expected version after '['") + } + low, err := parseVersion(verb, "", &toks[0], fix) + if err != nil { + return VersionInterval{}, err + } + toks = toks[1:] + + if len(toks) == 0 || toks[0] != "," { + return VersionInterval{}, fmt.Errorf("expected ',' after version") + } + toks = toks[1:] + + if len(toks) == 0 { + return VersionInterval{}, fmt.Errorf("expected version after ','") + } + high, err := parseVersion(verb, "", &toks[0], fix) + if err != nil { + return VersionInterval{}, err + } + toks = toks[1:] + + if len(toks) == 0 || toks[0] != "]" { + return VersionInterval{}, fmt.Errorf("expected ']' after version") + } + toks = toks[1:] + + *args = toks + return VersionInterval{Low: low, High: high}, nil +} + func parseString(s *string) (string, error) { t := *s if strings.HasPrefix(t, `"`) { @@ -461,6 +555,27 @@ func parseString(s *string) (string, error) { return t, nil } +// parseRetractRationale extracts the rationale for a retract directive from the +// surrounding comments. If the line does not have comments and is part of a +// block that does have comments, the block's comments are used. +func parseRetractRationale(block *LineBlock, line *Line) string { + comments := line.Comment() + if block != nil && len(comments.Before) == 0 && len(comments.Suffix) == 0 { + comments = block.Comment() + } + groups := [][]Comment{comments.Before, comments.Suffix} + var lines []string + for _, g := range groups { + for _, c := range g { + if !strings.HasPrefix(c.Token, "//") { + continue // blank line + } + lines = append(lines, strings.TrimSpace(strings.TrimPrefix(c.Token, "//"))) + } + } + return strings.Join(lines, "\n") +} + type ErrorList []Error func (e ErrorList) Error() string { @@ -494,6 +609,8 @@ func (e *Error) Error() string { var directive string if e.ModPath != "" { directive = fmt.Sprintf("%s %s: ", e.Verb, e.ModPath) + } else if e.Verb != "" { + directive = fmt.Sprintf("%s: ", e.Verb) } return pos + directive + e.Err.Error() @@ -585,6 +702,15 @@ func (f *File) Cleanup() { } f.Replace = f.Replace[:w] + w = 0 + for _, r := range f.Retract { + if r.Low != "" || r.High != "" { + f.Retract[w] = r + w++ + } + } + f.Retract = f.Retract[:w] + f.Syntax.Cleanup() } @@ -778,6 +904,34 @@ func (f *File) DropReplace(oldPath, oldVers string) error { return nil } +func (f *File) AddRetract(vi VersionInterval, rationale string) error { + r := &Retract{ + VersionInterval: vi, + } + if vi.Low == vi.High { + r.Syntax = f.Syntax.addLine(nil, "retract", AutoQuote(vi.Low)) + } else { + r.Syntax = f.Syntax.addLine(nil, "retract", "[", AutoQuote(vi.Low), ",", AutoQuote(vi.High), "]") + } + if rationale != "" { + for _, line := range strings.Split(rationale, "\n") { + com := Comment{Token: "// " + line} + r.Syntax.Comment().Before = append(r.Syntax.Comment().Before, com) + } + } + return nil +} + +func (f *File) DropRetract(vi VersionInterval) error { + for _, r := range f.Retract { + if r.VersionInterval == vi { + f.Syntax.removeLine(r.Syntax) + *r = Retract{} + } + } + return nil +} + func (f *File) SortBlocks() { f.removeDups() // otherwise sorting is unsafe @@ -786,28 +940,38 @@ func (f *File) SortBlocks() { if !ok { continue } - sort.Slice(block.Line, func(i, j int) bool { - li := block.Line[i] - lj := block.Line[j] - for k := 0; k < len(li.Token) && k < len(lj.Token); k++ { - if li.Token[k] != lj.Token[k] { - return li.Token[k] < lj.Token[k] - } - } - return len(li.Token) < len(lj.Token) + less := lineLess + if block.Token[0] == "retract" { + less = lineRetractLess + } + sort.SliceStable(block.Line, func(i, j int) bool { + return less(block.Line[i], block.Line[j]) }) } } +// removeDups removes duplicate exclude and replace directives. +// +// Earlier exclude directives take priority. +// +// Later replace directives take priority. +// +// require directives are not de-duplicated. That's left up to higher-level +// logic (MVS). +// +// retract directives are not de-duplicated since comments are +// meaningful, and versions may be retracted multiple times. func (f *File) removeDups() { - have := make(map[module.Version]bool) kill := make(map[*Line]bool) + + // Remove duplicate excludes. + haveExclude := make(map[module.Version]bool) for _, x := range f.Exclude { - if have[x.Mod] { + if haveExclude[x.Mod] { kill[x.Syntax] = true continue } - have[x.Mod] = true + haveExclude[x.Mod] = true } var excl []*Exclude for _, x := range f.Exclude { @@ -817,15 +981,16 @@ func (f *File) removeDups() { } f.Exclude = excl - have = make(map[module.Version]bool) + // Remove duplicate replacements. // Later replacements take priority over earlier ones. + haveReplace := make(map[module.Version]bool) for i := len(f.Replace) - 1; i >= 0; i-- { x := f.Replace[i] - if have[x.Old] { + if haveReplace[x.Old] { kill[x.Syntax] = true continue } - have[x.Old] = true + haveReplace[x.Old] = true } var repl []*Replace for _, x := range f.Replace { @@ -835,6 +1000,9 @@ func (f *File) removeDups() { } f.Replace = repl + // Duplicate require and retract directives are not removed. + + // Drop killed statements from the syntax tree. var stmts []Expr for _, stmt := range f.Syntax.Stmt { switch stmt := stmt.(type) { @@ -858,3 +1026,38 @@ func (f *File) removeDups() { } f.Syntax.Stmt = stmts } + +// lineLess returns whether li should be sorted before lj. It sorts +// lexicographically without assigning any special meaning to tokens. +func lineLess(li, lj *Line) bool { + for k := 0; k < len(li.Token) && k < len(lj.Token); k++ { + if li.Token[k] != lj.Token[k] { + return li.Token[k] < lj.Token[k] + } + } + return len(li.Token) < len(lj.Token) +} + +// lineRetractLess returns whether li should be sorted before lj for lines in +// a "retract" block. It treats each line as a version interval. Single versions +// are compared as if they were intervals with the same low and high version. +// Intervals are sorted in descending order, first by low version, then by +// high version, using semver.Compare. +func lineRetractLess(li, lj *Line) bool { + interval := func(l *Line) VersionInterval { + if len(l.Token) == 1 { + return VersionInterval{Low: l.Token[0], High: l.Token[0]} + } else if len(l.Token) == 5 && l.Token[0] == "[" && l.Token[2] == "," && l.Token[4] == "]" { + return VersionInterval{Low: l.Token[1], High: l.Token[3]} + } else { + // Line in unknown format. Treat as an invalid version. + return VersionInterval{} + } + } + vii := interval(li) + vij := interval(lj) + if cmp := semver.Compare(vii.Low, vij.Low); cmp != 0 { + return cmp > 0 + } + return semver.Compare(vii.High, vij.High) > 0 +} diff --git a/src/cmd/vendor/modules.txt b/src/cmd/vendor/modules.txt index 7272f04ff3..ab2f81a66b 100644 --- a/src/cmd/vendor/modules.txt +++ b/src/cmd/vendor/modules.txt @@ -29,7 +29,7 @@ golang.org/x/arch/x86/x86asm golang.org/x/crypto/ed25519 golang.org/x/crypto/ed25519/internal/edwards25519 golang.org/x/crypto/ssh/terminal -# golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 +# golang.org/x/mod v0.3.1-0.20200824162228-c0d644d00ab8 ## explicit golang.org/x/mod/internal/lazyregexp golang.org/x/mod/modfile -- 2.48.1