From c319fb07bcae09589f83f060a4fdfef93a521348 Mon Sep 17 00:00:00 2001 From: Julian Phillips Date: Mon, 20 Jun 2011 13:00:43 +1000 Subject: [PATCH] goinstall: Add support for arbitary code repositories Extend goinstall to support downloading from any hg/git/svn/bzr hosting site, not just the standard ones. The type of hosting is automatically checked by trying all the tools, so the import statement looks like: import "example.com/mything" Which will work for Mercurial (http), Subversion (http, svn), Git (http, git) and Bazaar (http, bzr) hosting. All the existing package imports will work through this new mechanism, but the existing hard-coded host support is left in place to ensure there is no change in behaviour. R=golang-dev, bradfitz, fvbommel, go.peter.90, n13m3y3r, adg, duperray.olivier CC=golang-dev https://golang.org/cl/4650043 --- src/cmd/goinstall/download.go | 231 ++++++++++++++++++++++++---------- 1 file changed, 168 insertions(+), 63 deletions(-) diff --git a/src/cmd/goinstall/download.go b/src/cmd/goinstall/download.go index 2edf85efdf..9bbbc91553 100644 --- a/src/cmd/goinstall/download.go +++ b/src/cmd/goinstall/download.go @@ -7,11 +7,15 @@ package main import ( + "exec" "http" "os" + "path" "path/filepath" "regexp" "strings" + "sync" + "time" ) const dashboardURL = "http://godashboard.appspot.com/package" @@ -31,74 +35,15 @@ func maybeReportToDashboard(path string) { } } -var vcsPatterns = map[string]*regexp.Regexp{ - "googlecode": regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/(svn|hg))(/[a-z0-9A-Z_.\-/]*)?$`), - "github": regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), - "bitbucket": regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), - "launchpad": regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`), -} - -// isRemote returns true if the provided package path -// matches one of the supported remote repositories. -func isRemote(pkg string) bool { - for _, r := range vcsPatterns { - if r.MatchString(pkg) { - return true - } - } - return false -} - -// download checks out or updates pkg from the remote server. -func download(pkg, srcDir string) os.Error { - if strings.Contains(pkg, "..") { - return os.ErrorString("invalid path (contains ..)") - } - if m := vcsPatterns["bitbucket"].FindStringSubmatch(pkg); m != nil { - if err := vcsCheckout(&hg, srcDir, m[1], "http://"+m[1], m[1]); err != nil { - return err - } - return nil - } - if m := vcsPatterns["googlecode"].FindStringSubmatch(pkg); m != nil { - var v *vcs - switch m[2] { - case "hg": - v = &hg - case "svn": - v = &svn - default: - // regexp only allows hg, svn to get through - panic("missing case in download: " + pkg) - } - if err := vcsCheckout(v, srcDir, m[1], "https://"+m[1], m[1]); err != nil { - return err - } - return nil - } - if m := vcsPatterns["github"].FindStringSubmatch(pkg); m != nil { - if strings.HasSuffix(m[1], ".git") { - return os.ErrorString("repository " + pkg + " should not have .git suffix") - } - if err := vcsCheckout(&git, srcDir, m[1], "http://"+m[1]+".git", m[1]); err != nil { - return err - } - return nil - } - if m := vcsPatterns["launchpad"].FindStringSubmatch(pkg); m != nil { - // Either lp.net/[/[/]] - // or lp.net/~//[/] - if err := vcsCheckout(&bzr, srcDir, m[1], "https://"+m[1], m[1]); err != nil { - return err - } - return nil - } - return os.ErrorString("unknown repository: " + pkg) +type host struct { + pattern *regexp.Regexp + protocol string } // a vcs represents a version control system // like Mercurial, Git, or Subversion. type vcs struct { + name string cmd string metadir string checkout string @@ -110,9 +55,23 @@ type vcs struct { log string logLimitFlag string logReleaseFlag string + check string + protocols []string + suffix string + findRepos bool + defaultHosts []host + + // Is this tool present? (set by findTools) + available bool +} + +type vcsMatch struct { + *vcs + prefix, repo string } var hg = vcs{ + name: "Mercurial", cmd: "hg", metadir: ".hg", checkout: "checkout", @@ -123,9 +82,17 @@ var hg = vcs{ log: "log", logLimitFlag: "-l1", logReleaseFlag: "-rrelease", + check: "identify", + protocols: []string{"http"}, + findRepos: true, + defaultHosts: []host{ + {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/hg)(/[a-z0-9A-Z_.\-/]*)?$`), "https"}, + {regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), "http"}, + }, } var git = vcs{ + name: "Git", cmd: "git", metadir: ".git", checkout: "checkout", @@ -136,9 +103,17 @@ var git = vcs{ log: "show-ref", logLimitFlag: "", logReleaseFlag: "release", + check: "peek-remote", + protocols: []string{"git", "http"}, + suffix: ".git", + findRepos: true, + defaultHosts: []host{ + {regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), "http"}, + }, } var svn = vcs{ + name: "Subversion", cmd: "svn", metadir: ".svn", checkout: "checkout", @@ -148,9 +123,16 @@ var svn = vcs{ log: "log", logLimitFlag: "-l1", logReleaseFlag: "release", + check: "info", + protocols: []string{"http", "svn"}, + findRepos: false, + defaultHosts: []host{ + {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/svn)(/[a-z0-9A-Z_.\-/]*)?$`), "https"}, + }, } var bzr = vcs{ + name: "Bazaar", cmd: "bzr", metadir: ".bzr", checkout: "update", @@ -162,6 +144,129 @@ var bzr = vcs{ log: "log", logLimitFlag: "-l1", logReleaseFlag: "-rrelease", + check: "info", + protocols: []string{"http", "bzr"}, + findRepos: true, + defaultHosts: []host{ + {regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`), "https"}, + }, +} + +var vcsList = []*vcs{&git, &hg, &bzr, &svn} + +func potentialPrefixes(pkg string) []string { + prefixes := []string{} + + parts := strings.Split(pkg, "/", -1) + elem := parts[0] + for _, part := range parts[1:] { + elem = path.Join(elem, part) + prefixes = append(prefixes, elem) + } + + return prefixes +} + +func tryCommand(c chan *vcsMatch, v *vcs, prefixes []string) { + for _, proto := range v.protocols { + for _, prefix := range prefixes { + repo := proto + "://" + prefix + v.suffix + if exec.Command(v.cmd, v.check, repo).Run() == nil { + c <- &vcsMatch{v, prefix, repo} + return + } + } + } +} + +var findToolsOnce sync.Once + +func findTools() { + for _, v := range vcsList { + v.available = exec.Command(v.cmd, "help").Run() == nil + } +} + +var logMissingToolsOnce sync.Once + +func logMissingTools() { + for _, v := range vcsList { + if !v.available { + logf("%s not found; %s packages will be ignored\n", v.cmd, v.name) + } + } +} + +func findVcs(pkg string) *vcsMatch { + c := make(chan *vcsMatch, len(vcsList)) + + findToolsOnce.Do(findTools) + + // we don't know how much of the name constitutes the repository prefix, so + // build a list of possibilities + prefixes := potentialPrefixes(pkg) + + for _, v := range vcsList { + if !v.available { + continue + } + if v.findRepos { + go tryCommand(c, v, prefixes) + } else { + go tryCommand(c, v, []string{pkg}) + } + } + + select { + case m := <-c: + return m + case <-time.After(20 * 1e9): + } + + logMissingToolsOnce.Do(logMissingTools) + + return nil +} + +// isRemote returns true if the first part of the package name looks like a +// hostname - i.e. contains at least one '.' and the last part is at least 2 +// characters. +func isRemote(pkg string) bool { + parts := strings.Split(pkg, "/", 2) + if len(parts) != 2 { + return false + } + parts = strings.Split(parts[0], ".", -1) + if len(parts) < 2 || len(parts[len(parts)-1]) < 2 { + return false + } + return true +} + +// download checks out or updates pkg from the remote server. +func download(pkg, srcDir string) os.Error { + if strings.Contains(pkg, "..") { + return os.ErrorString("invalid path (contains ..)") + } + var m *vcsMatch + for _, v := range vcsList { + for _, host := range v.defaultHosts { + if hm := host.pattern.FindStringSubmatch(pkg); hm != nil { + if v.suffix != "" && strings.HasSuffix(hm[1], v.suffix) { + return os.ErrorString("repository " + pkg + " should not have " + v.suffix + " suffix") + } + repo := host.protocol + "://" + hm[1] + v.suffix + m = &vcsMatch{v, hm[1], repo} + } + } + } + if m == nil { + m = findVcs(pkg) + } + if m == nil { + return os.ErrorString("cannot download: " + pkg) + } + return vcsCheckout(m.vcs, srcDir, m.prefix, m.repo, pkg) } // Try to detect if a "release" tag exists. If it does, update -- 2.50.0