--- /dev/null
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../src/Make.inc
+
+TARG=gobuilder
+GOFILES=\
+ exec.go\
+ hg.go\
+ http.go\
+ main.go\
+
+include ../../../src/Make.cmd
--- /dev/null
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+
+Go Builder is a continuous build client for the Go project.
+It integrates with the Go Dashboard AppEngine application.
+
+Go Builder is intended to run continuously as a background process.
+
+It periodically pulls updates from the Go Mercurial repository.
+
+When a newer revision is found, Go Builder creates a clone of the repository,
+runs all.bash, and reports build success or failure to the Go Dashboard.
+
+For a successful build, Go Builder will also run benchmarks
+(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.
+
+For release revision (a change description that matches "release.YYYY-MM-DD"),
+Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
+Go Google Code project's downloads section.
+
+Command-line options (and defaults):
+
+ -goarch="": $GOARCH
+ -goos="": $GOOS
+ The target architecture and operating system of this build client.
+
+ -goroot="": $GOROOT
+ A persistent Go checkout. Go Builder will periodically run 'hg pull -u'
+ from this location and use it as a source repository when cloning a
+ revision to be built.
+
+ -path="": Build Path
+ The base path in which building, testing, and archival will occur,
+ such as "/tmp/build". This can be considered volatile.
+
+ -keyfile="": Key File
+ The file containing the build key and Google Code credentials.
+ It is a text file of the format:
+
+ godashboard-key
+ googlecode-username
+ googlecode-password
+
+ If the Google Code credentials are not provided the archival step
+ will be skipped.
+
+ -host="godashboard.appspot.com": Go Dashboard Host
+ The location of the Go Dashboard application to which Go Builder will
+ report its results.
+
+ -pybin="/usr/bin/python": Python Binary
+ -hgbin="/usr/local/bin/hg": Mercurial Binary
+ These name the local Python and Mercurial binaries.
+ (Python is required only to run the Google Code uploader script, found
+ at $GOROOT/misc/dashboard/googlecode_upload.py.)
+
+*/
+package documentation
--- /dev/null
+package main
+
+import (
+ "bytes"
+ "exec"
+ "os"
+)
+
+// run is a simple wrapper for exec.Run/Close
+func run(envv []string, dir string, argv ...string) os.Error {
+ bin, err := exec.LookPath(argv[0])
+ if err != nil {
+ return err
+ }
+ p, err := exec.Run(bin, argv, envv, dir,
+ exec.DevNull, exec.DevNull, exec.PassThrough)
+ if err != nil {
+ return err
+ }
+ return p.Close()
+}
+
+// runLog runs a process and returns the combined stdout/stderr
+func runLog(envv []string, dir string, argv ...string) (o string, s int, err os.Error) {
+ s = -1
+ bin, err := exec.LookPath(argv[0])
+ if err != nil {
+ return
+ }
+ p, err := exec.Run(bin, argv, envv, dir,
+ exec.DevNull, exec.Pipe, exec.MergeWithStdout)
+ if err != nil {
+ return
+ }
+ b := new(bytes.Buffer)
+ _, err = b.ReadFrom(p.Stdout)
+ if err != nil {
+ return
+ }
+ w, err := p.Wait(0)
+ if err != nil {
+ return
+ }
+ return b.String(), w.WaitStatus.ExitStatus(), nil
+}
--- /dev/null
+package main
+
+import (
+ "os"
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+type Commit struct {
+ num int // mercurial revision number
+ node string // mercurial hash
+ parent string // hash of commit's parent
+ user string // author's name and email
+ date string // date of commit
+ desc string // description
+}
+
+// getCommit returns details about the Commit specified by the revision hash
+func getCommit(rev string) (c Commit, err os.Error) {
+ defer func() {
+ if err != nil {
+ err = os.NewError(fmt.Sprintf("getCommit: %s: %s",
+ rev, err))
+ }
+ }()
+ parts, err := getCommitParts(rev)
+ if err != nil {
+ return
+ }
+ num, err := strconv.Atoi(parts[0])
+ if err != nil {
+ return
+ }
+ parent := ""
+ if num > 0 {
+ prev := strconv.Itoa(num - 1)
+ if pparts, err := getCommitParts(prev); err == nil {
+ parent = pparts[1]
+ }
+ }
+ user := strings.Replace(parts[2], "<", "<", -1)
+ user = strings.Replace(user, ">", ">", -1)
+ return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
+}
+
+func getCommitParts(rev string) (parts []string, err os.Error) {
+ format := "{rev}>{node|escape}>{author|escape}>{date}>{desc}"
+ s, _, err := runLog(nil, goroot,
+ "hg", "log", "-r", rev, "-l", "1", "--template", format)
+ if err != nil {
+ return
+ }
+ return strings.Split(s, ">", -1), nil
+}
+
--- /dev/null
+package main
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/binary"
+ "fmt"
+ "http"
+ "os"
+ "regexp"
+)
+
+// getHighWater returns the current highwater revision hash for this builder
+func (b *Builder) getHighWater() (rev string, err os.Error) {
+ url := fmt.Sprintf("http://%s/hw-get?builder=%s",
+ *dashboardhost, b.name)
+ r, _, err := http.Get(url)
+ if err != nil {
+ return
+ }
+ buf := new(bytes.Buffer)
+ _, err = buf.ReadFrom(r.Body)
+ if err != nil {
+ return
+ }
+ r.Body.Close()
+ return buf.String(), nil
+}
+
+// recordResult sends build results to the dashboard
+func (b *Builder) recordResult(buildLog string, c Commit) os.Error {
+ return httpCommand("build", map[string]string{
+ "builder": b.name,
+ "key": b.key,
+ "node": c.node,
+ "parent": c.parent,
+ "user": c.user,
+ "date": c.date,
+ "desc": c.desc,
+ "log": buildLog,
+ })
+}
+
+// match lines like: "package.BechmarkFunc 100000 999 ns/op"
+var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op")
+
+// recordBenchmarks sends benchmark results to the dashboard
+func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
+ results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1)
+ var buf bytes.Buffer
+ b64 := base64.NewEncoder(base64.StdEncoding, &buf)
+ for _, r := range results {
+ for _, s := range r[1:] {
+ binary.Write(b64, binary.BigEndian, uint16(len(s)))
+ b64.Write([]byte(s))
+ }
+ }
+ b64.Close()
+ return httpCommand("benchmarks", map[string]string{
+ "builder": b.name,
+ "key": b.key,
+ "node": c.node,
+ "benchmarkdata": buf.String(),
+ })
+}
+
+func httpCommand(cmd string, args map[string]string) os.Error {
+ url := fmt.Sprintf("http://%v/%v", *dashboardhost, cmd)
+ _, err := http.PostForm(url, args)
+ return err
+}
--- /dev/null
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "path"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+)
+
+const (
+ codeProject = "go"
+ codePyScript = "misc/dashboard/googlecode_upload.py"
+ hgUrl = "https://go.googlecode.com/hg/"
+ waitInterval = 10e9 // time to wait before checking for new revs
+)
+
+var (
+ goroot = path.Join(os.Getenv("PWD"), "goroot")
+ releaseRegexp = regexp.MustCompile(`^release\.[0-9\-]+`)
+ dashboardhost = flag.String("dashboard", "godashboard.appspot.com",
+ "Godashboard Host")
+)
+
+type Builder struct {
+ name string
+ goos, goarch string
+ key string
+ codeUsername string
+ codePassword string
+}
+
+func main() {
+ flag.Parse()
+ builders := make(map[string]*Builder)
+ if len(flag.Args()) == 0{
+ log.Exit("No builders specified.")
+ }
+ for _, builder := range flag.Args() {
+ b, err := NewBuilder(builder)
+ if err != nil {
+ log.Exit(err)
+ }
+ builders[builder] = b
+ }
+ err := run(nil, "", "hg", "clone", hgUrl, goroot)
+ if err != nil {
+ log.Exit("Error cloning repository:", err)
+ }
+ // check for new commits and build them
+ for {
+ err := run(nil, goroot, "hg", "pull", "-u")
+ if err != nil {
+ log.Stderr("hg pull failed:", err)
+ time.Sleep(waitInterval)
+ continue
+ }
+ built := false
+ for _, b := range builders {
+ built = b.tryBuild() || built
+ }
+ // only wait if we didn't do anything
+ if !built {
+ time.Sleep(waitInterval)
+ }
+ }
+}
+
+func NewBuilder(builder string) (*Builder, os.Error) {
+ b := &Builder{name:builder}
+
+ // get goos/goarch from builder string
+ s := strings.Split(builder, "-", 3)
+ if len(s) == 2 {
+ b.goos, b.goarch = s[0], s[1]
+ } else {
+ return nil, os.NewError(fmt.Sprintf(
+ "unsupported builder form: %s", builder))
+ }
+
+ // read keys from keyfile
+ fn := path.Join(os.Getenv("HOME"), ".gobuildkey")
+ if isFile(fn + "-" + b.name) { // builder-specific file
+ fn += "-" + b.name
+ }
+ c, err := ioutil.ReadFile(fn)
+ if err != nil {
+ return nil, os.NewError(fmt.Sprintf("readKeys %s (%s): %s",
+ b.name, fn, err))
+ }
+ v := strings.Split(string(c), "\n", -1)
+ b.key = v[0]
+ if len(v) >= 3 {
+ b.codeUsername, b.codePassword = v[1], v[2]
+ }
+
+ return b, nil
+}
+
+// tryBuild checks for a new commit for this builder,
+// and builds it if one is found.
+// Its return value indicates whether a build happened or not.
+func (b *Builder) tryBuild() bool {
+ c, err := b.nextCommit()
+ if err != nil {
+ log.Stderr(err)
+ return false
+ }
+ if c == nil {
+ return false
+ }
+ log.Stderr("Building new revision: ", c.num)
+ err = b.build(*c)
+ if err != nil {
+ log.Stderr(err)
+ }
+ return true
+}
+
+// nextCommit returns the next unbuilt Commit for this builder
+func (b *Builder) nextCommit() (nextC *Commit, err os.Error) {
+ defer func() {
+ if err != nil {
+ err = os.NewError(fmt.Sprintf(
+ "%s nextCommit: %s", b.name, err))
+ }
+ }()
+ hw, err := b.getHighWater()
+ if err != nil {
+ return
+ }
+ c, err := getCommit(hw)
+ if err != nil {
+ return
+ }
+ next := c.num + 1
+ c, err = getCommit(strconv.Itoa(next))
+ if err == nil || c.num == next {
+ return &c, nil
+ }
+ return nil, nil
+}
+
+func (b *Builder) build(c Commit) (err os.Error) {
+ defer func() {
+ if err != nil {
+ err = os.NewError(fmt.Sprintf(
+ "%s buildRev commit: %s: %s",
+ b.name, c.num, err))
+ }
+ }()
+
+ // destroy old build candidate
+ err = run(nil, "", "rm", "-Rf", "go")
+ if err != nil {
+ return
+ }
+
+ // clone repo at revision num (new candidate)
+ err = run(nil, "",
+ "hg", "clone",
+ "-r", strconv.Itoa(c.num),
+ goroot, "go")
+ if err != nil {
+ return
+ }
+
+ // set up environment for build/bench execution
+ env := []string{
+ "GOOS=" + b.goos,
+ "GOARCH=" + b.goarch,
+ "GOROOT_FINAL=/usr/local/go",
+ "PATH=" + os.Getenv("PATH"),
+ }
+ srcDir := path.Join("", "go", "src")
+
+ // build the release candidate
+ buildLog, status, err := runLog(env, srcDir, "./all.bash")
+ if err != nil {
+ return
+ }
+ if status != 0 {
+ // record failure
+ return b.recordResult(buildLog, c)
+ }
+
+ // record success
+ if err = b.recordResult("", c); err != nil {
+ return
+ }
+
+ // run benchmarks and send to dashboard
+ pkgDir := path.Join(srcDir, "pkg")
+ benchLog, _, err := runLog(env, pkgDir, "gomake", "bench")
+ if err != nil {
+ log.Stderr("gomake bench:", err)
+ } else if err = b.recordBenchmarks(benchLog, c); err != nil {
+ log.Stderr("recordBenchmarks:", err)
+ }
+
+ // finish here if codeUsername and codePassword aren't set
+ if b.codeUsername == "" || b.codePassword == "" {
+ return
+ }
+
+ // if this is a release, create tgz and upload to google code
+ if release := releaseRegexp.FindString(c.desc); release != "" {
+ // clean out build state
+ err = run(env, srcDir, "sh", "clean.bash", "--nopkg")
+ if err != nil {
+ return
+ }
+ // upload binary release
+ err = b.codeUpload(release)
+ if err != nil {
+ return
+ }
+ }
+
+ return
+}
+
+func (b *Builder) codeUpload(release string) (err os.Error) {
+ defer func() {
+ if err != nil {
+ err = os.NewError(fmt.Sprintf(
+ "%s codeUpload release: %s: %s",
+ b.name, release, err))
+ }
+ }()
+ fn := fmt.Sprintf("%s.%s-%s.tar.gz", release, b.goos, b.goarch)
+ err = run(nil, "", "tar", "czf", fn, "go")
+ if err != nil {
+ return
+ }
+ return run(nil, "", "python",
+ path.Join(goroot, codePyScript),
+ "-s", release,
+ "-p", codeProject,
+ "-u", b.codeUsername,
+ "-w", b.codePassword,
+ "-l", fmt.Sprintf("%s,%s", b.goos, b.goarch),
+ fn)
+}
+
+func isDirectory(name string) bool {
+ s, err := os.Stat(name)
+ return err == nil && s.IsDirectory()
+}
+
+func isFile(name string) bool {
+ s, err := os.Stat(name)
+ return err == nil && (s.IsRegular() || s.IsSymlink())
+}