hg.go\
http.go\
main.go\
+ package.go\
include ../../../src/Make.cmd
-release: Build and deliver binary release archive
+ -rev=N: Build revision N and exit
+
+ -cmd="./all.bash": Build command (specify absolute or relative to go/src)
+
+ -v: Verbose logging
+
+ -external: External package builder mode (will not report Go build
+ state to dashboard, issue releases, or run benchmarks)
+
The key file should be located at $HOME/.gobuilder or, for a builder-specific
key, $HOME/.gobuilder-$BUILDER (eg, $HOME/.gobuilder-linux-amd64).
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
package main
import (
"bytes"
"exec"
"io"
+ "log"
"os"
"strings"
)
// run is a simple wrapper for exec.Run/Close
func run(envv []string, dir string, argv ...string) os.Error {
+ if *verbose {
+ log.Println("run", argv)
+ }
bin, err := pathLookup(argv[0])
if err != nil {
return err
// runLog runs a process and returns the combined stdout/stderr,
// as well as writing it to logfile (if specified).
func runLog(envv []string, logfile, dir string, argv ...string) (output string, exitStatus int, err os.Error) {
+ if *verbose {
+ log.Println("runLog", argv)
+ }
bin, err := pathLookup(argv[0])
if err != nil {
return
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
package main
import (
"fmt"
"os"
+ "regexp"
"strconv"
"strings"
)
}
return strings.Split(s, ">", 5), nil
}
+
+var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`)
+
+// getTag fetches a Commit by finding the first hg tag that matches re.
+func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) {
+ o, _, err := runLog(nil, "", goroot, "hg", "tags")
+ for _, l := range strings.Split(o, "\n", -1) {
+ tag = re.FindString(l)
+ if tag == "" {
+ continue
+ }
+ s := revisionRe.FindStringSubmatch(l)
+ if s == nil {
+ err = os.NewError("couldn't find revision number")
+ return
+ }
+ c, err = getCommit(s[1])
+ return
+ }
+ err = os.NewError("no matching tag found")
+ return
+}
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
package main
import (
"encoding/binary"
"fmt"
"http"
+ "json"
+ "log"
"os"
"regexp"
+ "strconv"
)
// getHighWater returns the current highwater revision hash for this builder
})
}
+// getPackages fetches a list of package paths from the dashboard
+func getPackages() (pkgs []string, err os.Error) {
+ r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard))
+ if err != nil {
+ return
+ }
+ defer r.Body.Close()
+ d := json.NewDecoder(r.Body)
+ var resp struct {
+ Packages []struct {
+ Path string
+ }
+ }
+ if err = d.Decode(&resp); err != nil {
+ return
+ }
+ for _, p := range resp.Packages {
+ pkgs = append(pkgs, p.Path)
+ }
+ return
+}
+
+// updatePackage sends package build results and info to the dashboard
+func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error {
+ args := map[string]string{
+ "builder": b.name,
+ "key": b.key,
+ "path": pkg,
+ "state": strconv.Btoa(state),
+ "log": buildLog,
+ "info": info,
+ "go_rev": strconv.Itoa(c.num),
+ }
+ return httpCommand("package", args)
+}
+
func httpCommand(cmd string, args map[string]string) os.Error {
+ if *verbose {
+ log.Println("httpCommand", cmd, args)
+ }
url := fmt.Sprintf("http://%v/%v", *dashboard, cmd)
_, err := http.PostForm(url, args)
return err
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
package main
import (
)
const (
- codeProject = "go"
- codePyScript = "misc/dashboard/googlecode_upload.py"
- hgUrl = "https://go.googlecode.com/hg/"
- waitInterval = 10e9 // time to wait before checking for new revs
- mkdirPerm = 0750
+ codeProject = "go"
+ codePyScript = "misc/dashboard/googlecode_upload.py"
+ hgUrl = "https://go.googlecode.com/hg/"
+ waitInterval = 10e9 // time to wait before checking for new revs
+ mkdirPerm = 0750
+ pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours
)
+// These variables are copied from the gobuilder's environment
+// to the envv of its subprocesses.
+var extraEnv = []string{
+ "GOHOSTOS",
+ "GOHOSTARCH",
+ "PATH",
+ "DISABLE_NET_TESTS",
+ "GOARM",
+}
+
type Builder struct {
name string
goos, goarch string
buildRelease = flag.Bool("release", false, "Build and upload binary release archives")
buildRevision = flag.String("rev", "", "Build specified revision and exit")
buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)")
+ external = flag.Bool("external", false, "Build external packages")
+ verbose = flag.Bool("v", false, "verbose")
)
var (
}
builders[i] = b
}
+
+ // set up work environment
if err := os.RemoveAll(*buildroot); err != nil {
log.Fatalf("Error removing build root (%s): %s", *buildroot, err)
}
if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil {
log.Fatal("Error cloning repository:", err)
}
+
// if specified, build revision and return
if *buildRevision != "" {
c, err := getCommit(*buildRevision)
}
return
}
+
+ // external package build mode
+ if *external {
+ if len(builders) != 1 {
+ log.Fatal("only one goos-goarch should be specified with -external")
+ }
+ builders[0].buildExternal()
+ }
+
+ // go continuous build mode (default)
// check for new commits and build them
for {
err := run(nil, goroot, "hg", "pull", "-u")
return b, nil
}
+// buildExternal downloads and builds external packages, and
+// reports their build status to the dashboard.
+// It will re-build all packages after pkgBuildInterval nanoseconds or
+// a new release tag is found.
+func (b *Builder) buildExternal() {
+ var prevTag string
+ var nextBuild int64
+ for {
+ time.Sleep(waitInterval)
+ err := run(nil, goroot, "hg", "pull", "-u")
+ if err != nil {
+ log.Println("hg pull failed:", err)
+ continue
+ }
+ c, tag, err := getTag(releaseRegexp)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ if *verbose {
+ log.Println("latest release:", tag)
+ }
+ // don't rebuild if there's no new release
+ // and it's been less than pkgBuildInterval
+ // nanoseconds since the last build.
+ if tag == prevTag && time.Nanoseconds() < nextBuild {
+ continue
+ }
+ // buildCommit will also build the packages
+ if err := b.buildCommit(c); err != nil {
+ log.Println(err)
+ continue
+ }
+ prevTag = tag
+ nextBuild = time.Nanoseconds() + pkgBuildInterval
+ }
+}
+
// build checks for a new commit for this builder
// and builds it if one is found.
// It returns true if a build was attempted.
return
}
- // set up environment for build/bench execution
- env := []string{
- "GOOS=" + b.goos,
- "GOARCH=" + b.goarch,
- "GOHOSTOS=" + os.Getenv("GOHOSTOS"),
- "GOHOSTARCH=" + os.Getenv("GOHOSTARCH"),
- "GOROOT_FINAL=/usr/local/go",
- "PATH=" + os.Getenv("PATH"),
- }
srcDir := path.Join(workpath, "go", "src")
// build
logfile := path.Join(workpath, "build.log")
- buildLog, status, err := runLog(env, logfile, srcDir, *buildCmd)
+ buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd)
if err != nil {
return fmt.Errorf("all.bash: %s", err)
}
+
+ // if we're in external mode, build all packages and return
+ if *external {
+ if status != 0 {
+ return os.NewError("go build failed")
+ }
+ return b.buildPackages(workpath, c)
+ }
+
if status != 0 {
// record failure
return b.recordResult(buildLog, c)
// if this is a release, create tgz and upload to google code
if release := releaseRegexp.FindString(c.desc); release != "" {
// clean out build state
- err = run(env, srcDir, "./clean.bash", "--nopkg")
+ err = run(b.envv(), srcDir, "./clean.bash", "--nopkg")
if err != nil {
return fmt.Errorf("clean.bash: %s", err)
}
return
}
+// envv returns an environment for build/bench execution
+func (b *Builder) envv() []string {
+ e := []string{
+ "GOOS=" + b.goos,
+ "GOARCH=" + b.goarch,
+ "GOROOT_FINAL=/usr/local/go",
+ }
+ for _, k := range extraEnv {
+ e = append(e, k+"="+os.Getenv(k))
+ }
+ return e
+}
+
func isDirectory(name string) bool {
s, err := os.Stat(name)
return err == nil && s.IsDirectory()
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "go/doc"
+ "go/parser"
+ "go/token"
+ "log"
+ "os"
+ "path"
+)
+
+func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
+ pkgs, err := getPackages()
+ if err != nil {
+ return err
+ }
+ for _, p := range pkgs {
+ goroot := path.Join(workpath, "go")
+ goinstall := path.Join(goroot, "bin", "goinstall")
+ envv := append(b.envv(), "GOROOT="+goroot)
+
+ // goinstall
+ buildLog, code, err := runLog(envv, "", goroot, goinstall, p)
+ if err != nil {
+ log.Printf("goinstall %v: %v", p, err)
+ continue
+ }
+ built := code != 0
+
+ // get doc comment from package source
+ info, err := getPackageComment(p, path.Join(goroot, "pkg", p))
+ if err != nil {
+ log.Printf("goinstall %v: %v", p, err)
+ }
+
+ // update dashboard with build state + info
+ err = b.updatePackage(p, built, buildLog, info, c)
+ if err != nil {
+ log.Printf("updatePackage %v: %v", p, err)
+ }
+ }
+ return nil
+}
+
+func getPackageComment(pkg, pkgpath string) (info string, err os.Error) {
+ fset := token.NewFileSet()
+ pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments)
+ if err != nil {
+ return
+ }
+ for name := range pkgs {
+ if name == "main" {
+ continue
+ }
+ if info != "" {
+ return "", os.NewError("multiple non-main package docs")
+ }
+ pdoc := doc.NewPackageDoc(pkgs[name], pkg)
+ info = pdoc.Doc
+ }
+ return
+}