+++ /dev/null
-# Copyright 2009 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-include ../../../src/Make.inc
-
-TARG=gobuilder
-GOFILES=\
- exec.go\
- hg.go\
- http.go\
- main.go\
-
-include ../../../src/Make.cmd
+++ /dev/null
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-/*
-
-Go Builder is a continuous build client for the Go project.
-It integrates with the Go Dashboard AppEngine application.
-
-Go Builder is intended to run continuously as a background process.
-
-It periodically pulls updates from the Go Mercurial repository.
-
-When a newer revision is found, Go Builder creates a clone of the repository,
-runs all.bash, and reports build success or failure to the Go Dashboard.
-
-For a successful build, Go Builder will also run benchmarks
-(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.
-
-For release revision (a change description that matches "release.YYYY-MM-DD"),
-Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
-Go Google Code project's downloads section.
-
-Command-line options (and defaults):
-
- -goarch="": $GOARCH
- -goos="": $GOOS
- The target architecture and operating system of this build client.
-
- -goroot="": $GOROOT
- A persistent Go checkout. Go Builder will periodically run 'hg pull -u'
- from this location and use it as a source repository when cloning a
- revision to be built.
-
- -path="": Build Path
- The base path in which building, testing, and archival will occur,
- such as "/tmp/build". This can be considered volatile.
-
- -keyfile="": Key File
- The file containing the build key and Google Code credentials.
- It is a text file of the format:
-
- godashboard-key
- googlecode-username
- googlecode-password
-
- If the Google Code credentials are not provided the archival step
- will be skipped.
-
- -host="godashboard.appspot.com": Go Dashboard Host
- The location of the Go Dashboard application to which Go Builder will
- report its results.
-
- -pybin="/usr/bin/python": Python Binary
- -hgbin="/usr/local/bin/hg": Mercurial Binary
- These name the local Python and Mercurial binaries.
- (Python is required only to run the Google Code uploader script, found
- at $GOROOT/misc/dashboard/googlecode_upload.py.)
-
-*/
-package documentation
+++ /dev/null
-package main
-
-import (
- "bytes"
- "exec"
- "os"
-)
-
-// run is a simple wrapper for exec.Run/Close
-func run(envv []string, dir string, argv ...string) os.Error {
- bin, err := exec.LookPath(argv[0])
- if err != nil {
- return err
- }
- p, err := exec.Run(bin, argv, envv, dir,
- exec.DevNull, exec.DevNull, exec.PassThrough)
- if err != nil {
- return err
- }
- return p.Close()
-}
-
-// runLog runs a process and returns the combined stdout/stderr
-func runLog(envv []string, dir string, argv ...string) (o string, s int, err os.Error) {
- s = -1
- bin, err := exec.LookPath(argv[0])
- if err != nil {
- return
- }
- p, err := exec.Run(bin, argv, envv, dir,
- exec.DevNull, exec.Pipe, exec.MergeWithStdout)
- if err != nil {
- return
- }
- b := new(bytes.Buffer)
- _, err = b.ReadFrom(p.Stdout)
- if err != nil {
- return
- }
- w, err := p.Wait(0)
- if err != nil {
- return
- }
- return b.String(), w.WaitStatus.ExitStatus(), nil
-}
+++ /dev/null
-package main
-
-import (
- "os"
- "fmt"
- "strconv"
- "strings"
-)
-
-type Commit struct {
- num int // mercurial revision number
- node string // mercurial hash
- parent string // hash of commit's parent
- user string // author's name and email
- date string // date of commit
- desc string // description
-}
-
-// getCommit returns details about the Commit specified by the revision hash
-func getCommit(rev string) (c Commit, err os.Error) {
- defer func() {
- if err != nil {
- err = os.NewError(fmt.Sprintf("getCommit: %s: %s",
- rev, err))
- }
- }()
- parts, err := getCommitParts(rev)
- if err != nil {
- return
- }
- num, err := strconv.Atoi(parts[0])
- if err != nil {
- return
- }
- parent := ""
- if num > 0 {
- prev := strconv.Itoa(num - 1)
- if pparts, err := getCommitParts(prev); err == nil {
- parent = pparts[1]
- }
- }
- user := strings.Replace(parts[2], "<", "<", -1)
- user = strings.Replace(user, ">", ">", -1)
- return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
-}
-
-func getCommitParts(rev string) (parts []string, err os.Error) {
- format := "{rev}>{node|escape}>{author|escape}>{date}>{desc}"
- s, _, err := runLog(nil, goroot,
- "hg", "log", "-r", rev, "-l", "1", "--template", format)
- if err != nil {
- return
- }
- return strings.Split(s, ">", -1), nil
-}
-
+++ /dev/null
-package main
-
-import (
- "bytes"
- "encoding/base64"
- "encoding/binary"
- "fmt"
- "http"
- "os"
- "regexp"
-)
-
-// getHighWater returns the current highwater revision hash for this builder
-func (b *Builder) getHighWater() (rev string, err os.Error) {
- url := fmt.Sprintf("http://%s/hw-get?builder=%s",
- *dashboardhost, b.name)
- r, _, err := http.Get(url)
- if err != nil {
- return
- }
- buf := new(bytes.Buffer)
- _, err = buf.ReadFrom(r.Body)
- if err != nil {
- return
- }
- r.Body.Close()
- return buf.String(), nil
-}
-
-// recordResult sends build results to the dashboard
-func (b *Builder) recordResult(buildLog string, c Commit) os.Error {
- return httpCommand("build", map[string]string{
- "builder": b.name,
- "key": b.key,
- "node": c.node,
- "parent": c.parent,
- "user": c.user,
- "date": c.date,
- "desc": c.desc,
- "log": buildLog,
- })
-}
-
-// match lines like: "package.BechmarkFunc 100000 999 ns/op"
-var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op")
-
-// recordBenchmarks sends benchmark results to the dashboard
-func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
- results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1)
- var buf bytes.Buffer
- b64 := base64.NewEncoder(base64.StdEncoding, &buf)
- for _, r := range results {
- for _, s := range r[1:] {
- binary.Write(b64, binary.BigEndian, uint16(len(s)))
- b64.Write([]byte(s))
- }
- }
- b64.Close()
- return httpCommand("benchmarks", map[string]string{
- "builder": b.name,
- "key": b.key,
- "node": c.node,
- "benchmarkdata": buf.String(),
- })
-}
-
-func httpCommand(cmd string, args map[string]string) os.Error {
- url := fmt.Sprintf("http://%v/%v", *dashboardhost, cmd)
- _, err := http.PostForm(url, args)
- return err
-}
+++ /dev/null
-package main
-
-import (
- "flag"
- "fmt"
- "io/ioutil"
- "log"
- "os"
- "path"
- "regexp"
- "strconv"
- "strings"
- "time"
-)
-
-const (
- codeProject = "go"
- codePyScript = "misc/dashboard/googlecode_upload.py"
- hgUrl = "https://go.googlecode.com/hg/"
- waitInterval = 10e9 // time to wait before checking for new revs
-)
-
-var (
- goroot = path.Join(os.Getenv("PWD"), "goroot")
- releaseRegexp = regexp.MustCompile(`^release\.[0-9\-]+`)
- dashboardhost = flag.String("dashboard", "godashboard.appspot.com",
- "Godashboard Host")
-)
-
-type Builder struct {
- name string
- goos, goarch string
- key string
- codeUsername string
- codePassword string
-}
-
-func main() {
- flag.Parse()
- builders := make(map[string]*Builder)
- if len(flag.Args()) == 0{
- log.Exit("No builders specified.")
- }
- for _, builder := range flag.Args() {
- b, err := NewBuilder(builder)
- if err != nil {
- log.Exit(err)
- }
- builders[builder] = b
- }
- err := run(nil, "", "hg", "clone", hgUrl, goroot)
- if err != nil {
- log.Exit("Error cloning repository:", err)
- }
- // check for new commits and build them
- for {
- err := run(nil, goroot, "hg", "pull", "-u")
- if err != nil {
- log.Stderr("hg pull failed:", err)
- time.Sleep(waitInterval)
- continue
- }
- built := false
- for _, b := range builders {
- built = b.tryBuild() || built
- }
- // only wait if we didn't do anything
- if !built {
- time.Sleep(waitInterval)
- }
- }
-}
-
-func NewBuilder(builder string) (*Builder, os.Error) {
- b := &Builder{name:builder}
-
- // get goos/goarch from builder string
- s := strings.Split(builder, "-", 3)
- if len(s) == 2 {
- b.goos, b.goarch = s[0], s[1]
- } else {
- return nil, os.NewError(fmt.Sprintf(
- "unsupported builder form: %s", builder))
- }
-
- // read keys from keyfile
- fn := path.Join(os.Getenv("HOME"), ".gobuildkey")
- if isFile(fn + "-" + b.name) { // builder-specific file
- fn += "-" + b.name
- }
- c, err := ioutil.ReadFile(fn)
- if err != nil {
- return nil, os.NewError(fmt.Sprintf("readKeys %s (%s): %s",
- b.name, fn, err))
- }
- v := strings.Split(string(c), "\n", -1)
- b.key = v[0]
- if len(v) >= 3 {
- b.codeUsername, b.codePassword = v[1], v[2]
- }
-
- return b, nil
-}
-
-// tryBuild checks for a new commit for this builder,
-// and builds it if one is found.
-// Its return value indicates whether a build happened or not.
-func (b *Builder) tryBuild() bool {
- c, err := b.nextCommit()
- if err != nil {
- log.Stderr(err)
- return false
- }
- if c == nil {
- return false
- }
- log.Stderr("Building new revision: ", c.num)
- err = b.build(*c)
- if err != nil {
- log.Stderr(err)
- }
- return true
-}
-
-// nextCommit returns the next unbuilt Commit for this builder
-func (b *Builder) nextCommit() (nextC *Commit, err os.Error) {
- defer func() {
- if err != nil {
- err = os.NewError(fmt.Sprintf(
- "%s nextCommit: %s", b.name, err))
- }
- }()
- hw, err := b.getHighWater()
- if err != nil {
- return
- }
- c, err := getCommit(hw)
- if err != nil {
- return
- }
- next := c.num + 1
- c, err = getCommit(strconv.Itoa(next))
- if err == nil || c.num == next {
- return &c, nil
- }
- return nil, nil
-}
-
-func (b *Builder) build(c Commit) (err os.Error) {
- defer func() {
- if err != nil {
- err = os.NewError(fmt.Sprintf(
- "%s buildRev commit: %s: %s",
- b.name, c.num, err))
- }
- }()
-
- // destroy old build candidate
- err = run(nil, "", "rm", "-Rf", "go")
- if err != nil {
- return
- }
-
- // clone repo at revision num (new candidate)
- err = run(nil, "",
- "hg", "clone",
- "-r", strconv.Itoa(c.num),
- goroot, "go")
- if err != nil {
- return
- }
-
- // set up environment for build/bench execution
- env := []string{
- "GOOS=" + b.goos,
- "GOARCH=" + b.goarch,
- "GOROOT_FINAL=/usr/local/go",
- "PATH=" + os.Getenv("PATH"),
- }
- srcDir := path.Join("", "go", "src")
-
- // build the release candidate
- buildLog, status, err := runLog(env, srcDir, "./all.bash")
- if err != nil {
- return
- }
- if status != 0 {
- // record failure
- return b.recordResult(buildLog, c)
- }
-
- // record success
- if err = b.recordResult("", c); err != nil {
- return
- }
-
- // run benchmarks and send to dashboard
- pkgDir := path.Join(srcDir, "pkg")
- benchLog, _, err := runLog(env, pkgDir, "gomake", "bench")
- if err != nil {
- log.Stderr("gomake bench:", err)
- } else if err = b.recordBenchmarks(benchLog, c); err != nil {
- log.Stderr("recordBenchmarks:", err)
- }
-
- // finish here if codeUsername and codePassword aren't set
- if b.codeUsername == "" || b.codePassword == "" {
- return
- }
-
- // if this is a release, create tgz and upload to google code
- if release := releaseRegexp.FindString(c.desc); release != "" {
- // clean out build state
- err = run(env, srcDir, "sh", "clean.bash", "--nopkg")
- if err != nil {
- return
- }
- // upload binary release
- err = b.codeUpload(release)
- if err != nil {
- return
- }
- }
-
- return
-}
-
-func (b *Builder) codeUpload(release string) (err os.Error) {
- defer func() {
- if err != nil {
- err = os.NewError(fmt.Sprintf(
- "%s codeUpload release: %s: %s",
- b.name, release, err))
- }
- }()
- fn := fmt.Sprintf("%s.%s-%s.tar.gz", release, b.goos, b.goarch)
- err = run(nil, "", "tar", "czf", fn, "go")
- if err != nil {
- return
- }
- return run(nil, "", "python",
- path.Join(goroot, codePyScript),
- "-s", release,
- "-p", codeProject,
- "-u", b.codeUsername,
- "-w", b.codePassword,
- "-l", fmt.Sprintf("%s,%s", b.goos, b.goarch),
- fn)
-}
-
-func isDirectory(name string) bool {
- s, err := os.Stat(name)
- return err == nil && s.IsDirectory()
-}
-
-func isFile(name string) bool {
- s, err := os.Stat(name)
- return err == nil && (s.IsRegular() || s.IsSymlink())
-}