"encoding/json"
"errors"
"fmt"
+ "io/ioutil"
"net/http"
"os"
"os/exec"
check string
protocols []string
suffix string
- defaultHosts []host
}
-var hg = vcs{
- name: "Mercurial",
- cmd: "hg",
- metadir: ".hg",
- checkout: "checkout",
- clone: "clone",
- update: "update",
- pull: "pull",
- tagList: "tags",
- tagListRe: regexp.MustCompile("([^ ]+)[^\n]+\n"),
- check: "identify",
- protocols: []string{"https", "http"},
- suffix: ".hg",
+func (v *vcs) String() string {
+ return v.name
}
-var git = vcs{
- name: "Git",
- cmd: "git",
- metadir: ".git",
- checkout: "checkout",
- clone: "clone",
- update: "pull",
- pull: "fetch",
- tagList: "tag",
- tagListRe: regexp.MustCompile("([^\n]+)\n"),
- check: "ls-remote",
- protocols: []string{"git", "https", "http"},
- suffix: ".git",
-}
+var vcsMap = map[string]*vcs{
+ "hg": &vcs{
+ name: "Mercurial",
+ cmd: "hg",
+ metadir: ".hg",
+ checkout: "checkout",
+ clone: "clone",
+ update: "update",
+ pull: "pull",
+ tagList: "tags",
+ tagListRe: regexp.MustCompile("([^ ]+)[^\n]+\n"),
+ check: "identify",
+ protocols: []string{"https", "http"},
+ suffix: ".hg",
+ },
-var svn = vcs{
- name: "Subversion",
- cmd: "svn",
- metadir: ".svn",
- checkout: "checkout",
- clone: "checkout",
- update: "update",
- check: "info",
- protocols: []string{"https", "http", "svn"},
- suffix: ".svn",
-}
+ "git": &vcs{
+ name: "Git",
+ cmd: "git",
+ metadir: ".git",
+ checkout: "checkout",
+ clone: "clone",
+ update: "pull",
+ pull: "fetch",
+ tagList: "tag",
+ tagListRe: regexp.MustCompile("([^\n]+)\n"),
+ check: "ls-remote",
+ protocols: []string{"git", "https", "http"},
+ suffix: ".git",
+ },
-var bzr = vcs{
- name: "Bazaar",
- cmd: "bzr",
- metadir: ".bzr",
- checkout: "update",
- clone: "branch",
- update: "update",
- updateRevFlag: "-r",
- pull: "pull",
- pullForceFlag: "--overwrite",
- tagList: "tags",
- tagListRe: regexp.MustCompile("([^ ]+)[^\n]+\n"),
- check: "info",
- protocols: []string{"https", "http", "bzr"},
- suffix: ".bzr",
+ "svn": &vcs{
+ name: "Subversion",
+ cmd: "svn",
+ metadir: ".svn",
+ checkout: "checkout",
+ clone: "checkout",
+ update: "update",
+ check: "info",
+ protocols: []string{"https", "http", "svn"},
+ suffix: ".svn",
+ },
+
+ "bzr": &vcs{
+ name: "Bazaar",
+ cmd: "bzr",
+ metadir: ".bzr",
+ checkout: "update",
+ clone: "branch",
+ update: "update",
+ updateRevFlag: "-r",
+ pull: "pull",
+ pullForceFlag: "--overwrite",
+ tagList: "tags",
+ tagListRe: regexp.MustCompile("([^ ]+)[^\n]+\n"),
+ check: "info",
+ protocols: []string{"https", "http", "bzr"},
+ suffix: ".bzr",
+ },
}
-var vcsList = []*vcs{&git, &hg, &bzr, &svn}
+type RemoteRepo interface {
+ // IsCheckedOut returns whether this repository is checked
+ // out inside the given srcDir (eg, $GOPATH/src).
+ IsCheckedOut(srcDir string) bool
+
+ // Repo returns the information about this repository: its url,
+ // the part of the import path that forms the repository root,
+ // and the version control system it uses. It may discover this
+ // information by using the supplied client to make HTTP requests.
+ Repo(_ *http.Client) (url, root string, vcs *vcs, err error)
+}
type host struct {
pattern *regexp.Regexp
- getVcs func(repo, path string) (*vcsMatch, error)
+ repo func(repo string) (RemoteRepo, error)
}
var knownHosts = []host{
{
- regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/(svn|git|hg))(/[a-z0-9A-Z_.\-/]*)?$`),
- googleVcs,
+ regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/(svn|git|hg))(/[a-z0-9A-Z_.\-/]+)?$`),
+ matchGoogleRepo,
+ },
+ {
+ regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+\.[a-z0-9\-]+)(/[a-z0-9A-Z_.\-/]+)?$`),
+ matchGoogleSubrepo,
},
{
- regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`),
- githubVcs,
+ regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$`),
+ matchGithubRepo,
},
{
- regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`),
- bitbucketVcs,
+ regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$`),
+ matchBitbucketRepo,
},
{
regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`),
- launchpadVcs,
+ matchLaunchpadRepo,
},
}
-type vcsMatch struct {
- *vcs
- prefix, repo string
+// baseRepo is the base implementation of RemoteRepo.
+type baseRepo struct {
+ url, root string
+ vcs *vcs
}
-func googleVcs(repo, path string) (*vcsMatch, error) {
- parts := strings.SplitN(repo, "/", 2)
- url := "https://" + repo
- switch parts[1] {
- case "svn":
- return &vcsMatch{&svn, repo, url}, nil
- case "git":
- return &vcsMatch{&git, repo, url}, nil
- case "hg":
- return &vcsMatch{&hg, repo, url}, nil
- }
- return nil, errors.New("unsupported googlecode vcs: " + parts[1])
+func (r *baseRepo) Repo(_ *http.Client) (url, root string, vcs *vcs, err error) {
+ return r.url, r.root, r.vcs, nil
}
-func githubVcs(repo, path string) (*vcsMatch, error) {
- if strings.HasSuffix(repo, ".git") {
- return nil, errors.New("path must not include .git suffix")
+// IsCheckedOut reports whether the repo root inside srcDir contains a
+// repository metadir. It updates the baseRepo's vcs field if necessary.
+func (r *baseRepo) IsCheckedOut(srcDir string) bool {
+ pkgPath := filepath.Join(srcDir, r.root)
+ if r.vcs == nil {
+ for _, vcs := range vcsMap {
+ if isDir(filepath.Join(pkgPath, vcs.metadir)) {
+ r.vcs = vcs
+ return true
+ }
+ }
+ return false
}
- return &vcsMatch{&git, repo, "http://" + repo + ".git"}, nil
+ return isDir(filepath.Join(pkgPath, r.vcs.metadir))
}
-func bitbucketVcs(repo, path string) (*vcsMatch, error) {
- const bitbucketApiUrl = "https://api.bitbucket.org/1.0/repositories/"
+// matchGoogleRepo handles matches of the form "repo.googlecode.com/vcs/path".
+func matchGoogleRepo(root string) (RemoteRepo, error) {
+ p := strings.SplitN(root, "/", 2)
+ if vcs := vcsMap[p[1]]; vcs != nil {
+ return &baseRepo{"https://" + root, root, vcs}, nil
+ }
+ return nil, errors.New("unsupported googlecode vcs: " + p[1])
+}
- if strings.HasSuffix(repo, ".git") {
+// matchGithubRepo handles matches for github.com repositories.
+func matchGithubRepo(root string) (RemoteRepo, error) {
+ if strings.HasSuffix(root, ".git") {
return nil, errors.New("path must not include .git suffix")
}
+ return &baseRepo{"http://" + root + ".git", root, vcsMap["git"]}, nil
+}
- parts := strings.SplitN(repo, "/", 2)
+// matchLaunchpadRepo handles matches for launchpad.net repositories.
+func matchLaunchpadRepo(root string) (RemoteRepo, error) {
+ return &baseRepo{"https://" + root, root, vcsMap["bzr"]}, nil
+}
- // Ask the bitbucket API what kind of repository this is.
- r, err := http.Get(bitbucketApiUrl + parts[1])
+// matchGoogleSubrepo matches repos like "code.google.com/p/repo.subrepo/path".
+// Note that it doesn't match primary Google Code repositories,
+// which should use the "foo.googlecode.com" form only. (for now)
+func matchGoogleSubrepo(id string) (RemoteRepo, error) {
+ root := "code.google.com/p/" + id
+ return &googleSubrepo{baseRepo{"https://" + root, root, nil}}, nil
+}
+
+// googleSubrepo implements a RemoteRepo that discovers a Google Code
+// repository's VCS type by scraping the code.google.com source checkout page.
+type googleSubrepo struct{ baseRepo }
+
+var googleSubrepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)
+
+func (r *googleSubrepo) Repo(client *http.Client) (url, root string, vcs *vcs, err error) {
+ if r.vcs != nil {
+ return r.url, r.root, r.vcs, nil
+ }
+
+ // Use the code.google.com source checkout page to find the VCS type.
+ const prefix = "code.google.com/p/"
+ p := strings.SplitN(r.root[len(prefix):], ".", 2)
+ u := fmt.Sprintf("https://%s%s/source/checkout?repo=%s", prefix, p[0], p[1])
+ resp, err := client.Get(u)
if err != nil {
- return nil, fmt.Errorf("error querying BitBucket API: %v", err)
+ return "", "", nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return "", "", nil, fmt.Errorf("fetching %s: %v", u, resp.Status)
+ }
+ b, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return "", "", nil, fmt.Errorf("fetching %s: %v", u, err)
+ }
+
+ // Scrape result for vcs details.
+ m := googleSubrepoRe.FindSubmatch(b)
+ if len(m) == 2 {
+ if v := vcsMap[string(m[1])]; v != nil {
+ r.vcs = v
+ return r.url, r.root, r.vcs, nil
+ }
}
- defer r.Body.Close()
- // Did we get a useful response?
- if r.StatusCode != 200 {
- return nil, fmt.Errorf("error querying BitBucket API: %v", r.Status)
+ return "", "", nil, errors.New("could not detect googlecode vcs")
+}
+
+// matchBitbucketRepo handles matches for all bitbucket.org repositories.
+func matchBitbucketRepo(root string) (RemoteRepo, error) {
+ if strings.HasSuffix(root, ".git") {
+ return nil, errors.New("path must not include .git suffix")
}
+ return &bitbucketRepo{baseRepo{root: root}}, nil
+}
+// bitbucketRepo implements a RemoteRepo that uses the BitBucket API to
+// discover the repository's VCS type.
+type bitbucketRepo struct{ baseRepo }
+
+func (r *bitbucketRepo) Repo(client *http.Client) (url, root string, vcs *vcs, err error) {
+ if r.vcs != nil && r.url != "" {
+ return r.url, r.root, r.vcs, nil
+ }
+
+ // Use the BitBucket API to find which kind of repository this is.
+ const apiUrl = "https://api.bitbucket.org/1.0/repositories/"
+ resp, err := client.Get(apiUrl + strings.SplitN(r.root, "/", 2)[1])
+ if err != nil {
+ return "", "", nil, fmt.Errorf("BitBucket API: %v", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return "", "", nil, fmt.Errorf("BitBucket API: %v", resp.Status)
+ }
var response struct {
Vcs string `json:"scm"`
}
- err = json.NewDecoder(r.Body).Decode(&response)
+ err = json.NewDecoder(resp.Body).Decode(&response)
if err != nil {
- return nil, fmt.Errorf("error querying BitBucket API: %v", err)
+ return "", "", nil, fmt.Errorf("BitBucket API: %v", err)
}
-
- // Now we should be able to construct a vcsMatch structure
switch response.Vcs {
case "git":
- return &vcsMatch{&git, repo, "http://" + repo + ".git"}, nil
+ r.url = "http://" + r.root + ".git"
case "hg":
- return &vcsMatch{&hg, repo, "http://" + repo}, nil
+ r.url = "http://" + r.root
+ default:
+ return "", "", nil, errors.New("unsupported bitbucket vcs: " + response.Vcs)
}
-
- return nil, errors.New("unsupported bitbucket vcs: " + response.Vcs)
-}
-
-func launchpadVcs(repo, path string) (*vcsMatch, error) {
- return &vcsMatch{&bzr, repo, "https://" + repo}, nil
+ if r.vcs = vcsMap[response.Vcs]; r.vcs == nil {
+ panic("vcs is nil when it should not be")
+ }
+ return r.url, r.root, r.vcs, nil
}
-// findPublicRepo checks whether pkg is located at one of
-// the supported code hosting sites and, if so, returns a match.
-func findPublicRepo(pkg string) (*vcsMatch, error) {
+// findPublicRepo checks whether importPath is a well-formed path for one of
+// the supported code hosting sites and, if so, returns a RemoteRepo.
+func findPublicRepo(importPath string) (RemoteRepo, error) {
for _, host := range knownHosts {
- if hm := host.pattern.FindStringSubmatch(pkg); hm != nil {
- return host.getVcs(hm[1], hm[2])
+ if hm := host.pattern.FindStringSubmatch(importPath); hm != nil {
+ return host.repo(hm[1])
}
}
return nil, nil
}
-// findAnyRepo looks for a vcs suffix in pkg (.git, etc) and returns a match.
-func findAnyRepo(pkg string) (*vcsMatch, error) {
- for _, v := range vcsList {
- i := strings.Index(pkg+"/", v.suffix+"/")
+// findAnyRepo matches import paths with a repo suffix (.git, etc).
+func findAnyRepo(importPath string) RemoteRepo {
+ for _, v := range vcsMap {
+ i := strings.Index(importPath+"/", v.suffix+"/")
if i < 0 {
continue
}
- if !strings.Contains(pkg[:i], "/") {
+ if !strings.Contains(importPath[:i], "/") {
continue // don't match vcs suffix in the host name
}
- if m := v.find(pkg[:i]); m != nil {
- return m, nil
+ return &anyRepo{
+ baseRepo{
+ root: importPath[:i] + v.suffix,
+ vcs: v,
+ },
+ importPath[:i],
}
- return nil, fmt.Errorf("couldn't find %s repository", v.name)
}
- return nil, nil
+ return nil
+}
+
+// anyRepo implements an discoverable remote repo with a suffix (.git, etc).
+type anyRepo struct {
+ baseRepo
+ rootWithoutSuffix string
+}
+
+func (r *anyRepo) Repo(_ *http.Client) (url, root string, vcs *vcs, err error) {
+ if r.url != "" {
+ return r.url, r.root, r.vcs, nil
+ }
+ url, err = r.vcs.findURL(r.rootWithoutSuffix)
+ if url == "" && err == nil {
+ err = fmt.Errorf("couldn't find %s repository", r.vcs.name)
+ }
+ if err != nil {
+ return "", "", nil, err
+ }
+ r.url = url
+ return r.url, r.root, r.vcs, nil
}
-func (v *vcs) find(pkg string) *vcsMatch {
+// findURL finds the URL for a given repo root by trying each combination of
+// protocol and suffix in series.
+func (v *vcs) findURL(root string) (string, error) {
for _, proto := range v.protocols {
for _, suffix := range []string{"", v.suffix} {
- repo := proto + "://" + pkg + suffix
- out, err := exec.Command(v.cmd, v.check, repo).CombinedOutput()
+ url := proto + "://" + root + suffix
+ out, err := exec.Command(v.cmd, v.check, url).CombinedOutput()
if err == nil {
- printf("find %s: found %s\n", pkg, repo)
- return &vcsMatch{v, pkg + v.suffix, repo}
+ printf("find %s: found %s\n", root, url)
+ return url, nil
}
- printf("find %s: %s %s %s: %v\n%s\n", pkg, v.cmd, v.check, repo, err, out)
+ printf("findURL(%s): %s %s %s: %v\n%s\n", root, v.cmd, v.check, url, err, out)
}
}
- return nil
-}
-
-// isRemote returns true if the first part of the package name looks like a
-// hostname - i.e. contains at least one '.' and the last part is at least 2
-// characters.
-func isRemote(pkg string) bool {
- parts := strings.SplitN(pkg, "/", 2)
- if len(parts) != 2 {
- return false
- }
- parts = strings.Split(parts[0], ".")
- if len(parts) < 2 || len(parts[len(parts)-1]) < 2 {
- return false
- }
- return true
+ return "", nil
}
-// download checks out or updates pkg from the remote server.
-func download(pkg, srcDir string) (public bool, err error) {
- if strings.Contains(pkg, "..") {
+// download checks out or updates the specified package from the remote server.
+func download(importPath, srcDir string) (public bool, err error) {
+ if strings.Contains(importPath, "..") {
err = errors.New("invalid path (contains ..)")
return
}
- m, err := findPublicRepo(pkg)
+
+ repo, err := findPublicRepo(importPath)
if err != nil {
- return
+ return false, err
}
- if m != nil {
+ if repo != nil {
public = true
} else {
- m, err = findAnyRepo(pkg)
- if err != nil {
- return
- }
+ repo = findAnyRepo(importPath)
}
- if m == nil {
- err = errors.New("cannot download: " + pkg)
+ if repo == nil {
+ err = errors.New("cannot download: " + importPath)
return
}
- err = m.checkoutRepo(srcDir, m.prefix, m.repo)
+ err = checkoutRepo(srcDir, repo)
return
}
+// checkoutRepo checks out repo into srcDir (if it's not checked out already)
+// and, if the -u flag is set, updates the repository.
+func checkoutRepo(srcDir string, repo RemoteRepo) error {
+ if !repo.IsCheckedOut(srcDir) {
+ // do checkout
+ url, root, vcs, err := repo.Repo(http.DefaultClient)
+ if err != nil {
+ return err
+ }
+ repoPath := filepath.Join(srcDir, root)
+ parent, _ := filepath.Split(repoPath)
+ if err = os.MkdirAll(parent, 0777); err != nil {
+ return err
+ }
+ if err = run(string(filepath.Separator), nil, vcs.cmd, vcs.clone, url, repoPath); err != nil {
+ return err
+ }
+ return vcs.updateRepo(repoPath)
+ }
+ if *update {
+ // do update
+ _, root, vcs, err := repo.Repo(http.DefaultClient)
+ if err != nil {
+ return err
+ }
+ repoPath := filepath.Join(srcDir, root)
+ // Retrieve new revisions from the remote branch, if the VCS
+ // supports this operation independently (e.g. svn doesn't)
+ if vcs.pull != "" {
+ if vcs.pullForceFlag != "" {
+ if err = run(repoPath, nil, vcs.cmd, vcs.pull, vcs.pullForceFlag); err != nil {
+ return err
+ }
+ } else if err = run(repoPath, nil, vcs.cmd, vcs.pull); err != nil {
+ return err
+ }
+ }
+ // Update to release or latest revision
+ return vcs.updateRepo(repoPath)
+ }
+ return nil
+}
+
// updateRepo gets a list of tags in the repository and
// checks out the tag closest to the current runtime.Version.
// If no matching tag is found, it just updates to tip.
-func (v *vcs) updateRepo(dst string) error {
+func (v *vcs) updateRepo(repoPath string) error {
if v.tagList == "" || v.tagListRe == nil {
// TODO(adg): fix for svn
- return run(dst, nil, v.cmd, v.update)
+ return run(repoPath, nil, v.cmd, v.update)
}
// Get tag list.
stderr := new(bytes.Buffer)
cmd := exec.Command(v.cmd, v.tagList)
- cmd.Dir = dst
+ cmd.Dir = repoPath
cmd.Stderr = stderr
b, err := cmd.Output()
if err != nil {
// Select tag.
if tag := selectTag(ver, tags); tag != "" {
printf("selecting revision %q\n", tag)
- return run(dst, nil, v.cmd, v.checkout, v.updateRevFlag+tag)
+ return run(repoPath, nil, v.cmd, v.checkout, v.updateRevFlag+tag)
}
// No matching tag found, make default selection.
printf("selecting tip\n")
- return run(dst, nil, v.cmd, v.update)
+ return run(repoPath, nil, v.cmd, v.update)
}
// selectTag returns the closest matching tag for a given version.
return match
}
-// checkoutRepo checks out repo into dst using vcs.
-// It tries to check out (or update, if the dst already
-// exists and -u was specified on the command line)
-// the repository at tag/branch "release". If there is no
-// such tag or branch, it falls back to the repository tip.
-func (vcs *vcs) checkoutRepo(srcDir, pkgprefix, repo string) error {
- dst := filepath.Join(srcDir, filepath.FromSlash(pkgprefix))
- dir, err := os.Stat(filepath.Join(dst, vcs.metadir))
- if err == nil && !dir.IsDirectory() {
- return errors.New("not a directory: " + dst)
- }
- if err != nil {
- parent, _ := filepath.Split(dst)
- if err = os.MkdirAll(parent, 0777); err != nil {
- return err
- }
- if err = run(string(filepath.Separator), nil, vcs.cmd, vcs.clone, repo, dst); err != nil {
- return err
- }
- return vcs.updateRepo(dst)
- }
- if *update {
- // Retrieve new revisions from the remote branch, if the VCS
- // supports this operation independently (e.g. svn doesn't)
- if vcs.pull != "" {
- if vcs.pullForceFlag != "" {
- if err = run(dst, nil, vcs.cmd, vcs.pull, vcs.pullForceFlag); err != nil {
- return err
- }
- } else if err = run(dst, nil, vcs.cmd, vcs.pull); err != nil {
- return err
- }
- }
- // Update to release or latest revision
- return vcs.updateRepo(dst)
- }
- return nil
+func isDir(dir string) bool {
+ fi, err := os.Stat(dir)
+ return err == nil && fi.IsDirectory()
}