"appengine/datastore"
"bytes"
"compress/gzip"
- "crypto/hmac"
"crypto/sha1"
"fmt"
- "http"
"io"
"io/ioutil"
- "json"
"os"
"strings"
)
-var defaultPackages = []*Package{
- &Package{Name: "Go"},
-}
-
-const (
- commitsPerPage = 30
- maxDatastoreStringLen = 500
-)
+const maxDatastoreStringLen = 500
// A Package describes a package that is listed on the dashboard.
type Package struct {
return t, nil
}
-// commitHandler retrieves commit data or records a new commit.
-//
-// For GET requests it returns a Commit value for the specified
-// packagePath and hash.
-//
-// For POST requests it reads a JSON-encoded Commit value from the request
-// body and creates a new Commit entity. It also updates the "tip" Tag for
-// each new commit at tip.
-//
-// This handler is used by a gobuilder process in -commit mode.
-func commitHandler(r *http.Request) (interface{}, os.Error) {
- c := appengine.NewContext(r)
- com := new(Commit)
-
- if r.Method == "GET" {
- com.PackagePath = r.FormValue("packagePath")
- com.Hash = r.FormValue("hash")
- if err := datastore.Get(c, com.Key(c), com); err != nil {
- return nil, fmt.Errorf("getting Commit: %v", err)
- }
- return com, nil
- }
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
-
- // POST request
- defer r.Body.Close()
- if err := json.NewDecoder(r.Body).Decode(com); err != nil {
- return nil, fmt.Errorf("decoding Body: %v", err)
- }
- if len(com.Desc) > maxDatastoreStringLen {
- com.Desc = com.Desc[:maxDatastoreStringLen]
- }
- if err := com.Valid(); err != nil {
- return nil, fmt.Errorf("validating Commit: %v", err)
- }
- tx := func(c appengine.Context) os.Error {
- return addCommit(c, com)
- }
- return nil, datastore.RunInTransaction(c, tx, nil)
-}
-
-// addCommit adds the Commit entity to the datastore and updates the tip Tag.
-// It must be run inside a datastore transaction.
-func addCommit(c appengine.Context, com *Commit) os.Error {
- var tc Commit // temp value so we don't clobber com
- err := datastore.Get(c, com.Key(c), &tc)
- if err != datastore.ErrNoSuchEntity {
- // if this commit is already in the datastore, do nothing
- if err == nil {
- return nil
- }
- return fmt.Errorf("getting Commit: %v", err)
- }
- // get the next commit number
- p, err := GetPackage(c, com.PackagePath)
- if err != nil {
- return fmt.Errorf("GetPackage: %v", err)
- }
- com.Num = p.NextNum
- p.NextNum++
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- return fmt.Errorf("putting Package: %v", err)
- }
- // if this isn't the first Commit test the parent commit exists
- if com.Num > 0 {
- n, err := datastore.NewQuery("Commit").
- Filter("Hash =", com.ParentHash).
- Ancestor(p.Key(c)).
- Count(c)
- if err != nil {
- return fmt.Errorf("testing for parent Commit: %v", err)
- }
- if n == 0 {
- return os.NewError("parent commit not found")
- }
- }
- // update the tip Tag if this is the Go repo
- if p.Path == "" {
- t := &Tag{Kind: "tip", Hash: com.Hash}
- if _, err = datastore.Put(c, t.Key(c), t); err != nil {
- return fmt.Errorf("putting Tag: %v", err)
- }
- }
- // put the Commit
- if _, err = datastore.Put(c, com.Key(c), com); err != nil {
- return fmt.Errorf("putting Commit: %v", err)
- }
- return nil
-}
-
-// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
-// request body and updates the Tag entity for the Kind of tag provided.
-//
-// This handler is used by a gobuilder process in -commit mode.
-func tagHandler(r *http.Request) (interface{}, os.Error) {
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
-
- t := new(Tag)
- defer r.Body.Close()
- if err := json.NewDecoder(r.Body).Decode(t); err != nil {
- return nil, err
- }
- if err := t.Valid(); err != nil {
- return nil, err
- }
- c := appengine.NewContext(r)
- _, err := datastore.Put(c, t.Key(c), t)
- return nil, err
-}
-
-// Todo is a todoHandler response.
-type Todo struct {
- Kind string // "build-go-commit" or "build-package"
- Data interface{}
-}
-
-// todoHandler returns the next action to be performed by a builder.
-// It expects "builder" and "kind" query parameters and returns a *Todo value.
-// Multiple "kind" parameters may be specified.
-func todoHandler(r *http.Request) (todo interface{}, err os.Error) {
- c := appengine.NewContext(r)
- builder := r.FormValue("builder")
- for _, kind := range r.Form["kind"] {
- var data interface{}
- switch kind {
- case "build-go-commit":
- data, err = buildTodo(c, builder, "", "")
- case "build-package":
- data, err = buildTodo(
- c, builder,
- r.FormValue("packagePath"),
- r.FormValue("goHash"),
- )
- }
- if data != nil || err != nil {
- return &Todo{Kind: kind, Data: data}, err
- }
- }
- return nil, nil
-}
-
-// buildTodo returns the next Commit to be built (or nil if none available).
-//
-// If packagePath and goHash are empty, it scans the first 20 Go Commits in
-// Num-descending order and returns the first one it finds that doesn't have a
-// Result for this builder.
-//
-// If provided with non-empty packagePath and goHash args, it scans the first
-// 20 Commits in Num-descending order for the specified packagePath and
-// returns the first that doesn't have a Result for this builder and goHash.
-func buildTodo(c appengine.Context, builder, packagePath, goHash string) (interface{}, os.Error) {
- p, err := GetPackage(c, packagePath)
- if err != nil {
- return nil, err
- }
-
- t := datastore.NewQuery("Commit").
- Ancestor(p.Key(c)).
- Limit(commitsPerPage).
- Order("-Num").
- Run(c)
- for {
- com := new(Commit)
- if _, err := t.Next(com); err != nil {
- if err == datastore.Done {
- err = nil
- }
- return nil, err
- }
- if com.Result(builder, goHash) == nil {
- return com, nil
- }
- }
- panic("unreachable")
-}
-
-// packagesHandler returns a list of the non-Go Packages monitored
-// by the dashboard.
-func packagesHandler(r *http.Request) (interface{}, os.Error) {
- return Packages(appengine.NewContext(r))
-}
-
// Packages returns all non-Go packages.
func Packages(c appengine.Context) ([]*Package, os.Error) {
var pkgs []*Package
}
return pkgs, nil
}
-
-// resultHandler records a build result.
-// It reads a JSON-encoded Result value from the request body,
-// creates a new Result entity, and updates the relevant Commit entity.
-// If the Log field is not empty, resultHandler creates a new Log entity
-// and updates the LogHash field before putting the Commit entity.
-func resultHandler(r *http.Request) (interface{}, os.Error) {
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
-
- c := appengine.NewContext(r)
- res := new(Result)
- defer r.Body.Close()
- if err := json.NewDecoder(r.Body).Decode(res); err != nil {
- return nil, fmt.Errorf("decoding Body: %v", err)
- }
- if err := res.Valid(); err != nil {
- return nil, fmt.Errorf("validating Result: %v", err)
- }
- // store the Log text if supplied
- if len(res.Log) > 0 {
- hash, err := PutLog(c, res.Log)
- if err != nil {
- return nil, fmt.Errorf("putting Log: %v", err)
- }
- res.LogHash = hash
- }
- tx := func(c appengine.Context) os.Error {
- // check Package exists
- if _, err := GetPackage(c, res.PackagePath); err != nil {
- return fmt.Errorf("GetPackage: %v", err)
- }
- // put Result
- if _, err := datastore.Put(c, res.Key(c), res); err != nil {
- return fmt.Errorf("putting Result: %v", err)
- }
- // add Result to Commit
- com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
- if err := com.AddResult(c, res); err != nil {
- return fmt.Errorf("AddResult: %v", err)
- }
- // Send build failure notifications, if necessary.
- // Note this must run after the call AddResult, which
- // populates the Commit's ResultData field.
- return notifyOnFailure(c, com, res.Builder)
- }
- return nil, datastore.RunInTransaction(c, tx, nil)
-}
-
-// logHandler displays log text for a given hash.
-// It handles paths like "/log/hash".
-func logHandler(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-type", "text/plain")
- c := appengine.NewContext(r)
- hash := r.URL.Path[len("/log/"):]
- key := datastore.NewKey(c, "Log", hash, 0, nil)
- l := new(Log)
- if err := datastore.Get(c, key, l); err != nil {
- logErr(w, r, err)
- return
- }
- b, err := l.Text()
- if err != nil {
- logErr(w, r, err)
- return
- }
- w.Write(b)
-}
-
-type dashHandler func(*http.Request) (interface{}, os.Error)
-
-type dashResponse struct {
- Response interface{}
- Error string
-}
-
-// errBadMethod is returned by a dashHandler when
-// the request has an unsuitable method.
-type errBadMethod string
-
-func (e errBadMethod) String() string {
- return "bad method: " + string(e)
-}
-
-// AuthHandler wraps a http.HandlerFunc with a handler that validates the
-// supplied key and builder query parameters.
-func AuthHandler(h dashHandler) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- c := appengine.NewContext(r)
-
- // Put the URL Query values into r.Form to avoid parsing the
- // request body when calling r.FormValue.
- r.Form = r.URL.Query()
-
- var err os.Error
- var resp interface{}
-
- // Validate key query parameter for POST requests only.
- key := r.FormValue("key")
- if r.Method == "POST" && key != secretKey && !appengine.IsDevAppServer() {
- h := hmac.NewMD5([]byte(secretKey))
- h.Write([]byte(r.FormValue("builder")))
- if key != fmt.Sprintf("%x", h.Sum()) {
- err = os.NewError("invalid key: " + key)
- }
- }
-
- // Call the original HandlerFunc and return the response.
- if err == nil {
- resp, err = h(r)
- }
-
- // Write JSON response.
- dashResp := &dashResponse{Response: resp}
- if err != nil {
- c.Errorf("%v", err)
- dashResp.Error = err.String()
- }
- w.Header().Set("Content-Type", "application/json")
- if err = json.NewEncoder(w).Encode(dashResp); err != nil {
- c.Criticalf("encoding response: %v", err)
- }
- }
-}
-
-func initHandler(w http.ResponseWriter, r *http.Request) {
- // TODO(adg): devise a better way of bootstrapping new packages
- c := appengine.NewContext(r)
- for _, p := range defaultPackages {
- if err := datastore.Get(c, p.Key(c), new(Package)); err == nil {
- continue
- } else if err != datastore.ErrNoSuchEntity {
- logErr(w, r, err)
- return
- }
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- logErr(w, r, err)
- return
- }
- }
- fmt.Fprint(w, "OK")
-}
-
-func init() {
- // admin handlers
- http.HandleFunc("/init", initHandler)
-
- // authenticated handlers
- http.HandleFunc("/commit", AuthHandler(commitHandler))
- http.HandleFunc("/packages", AuthHandler(packagesHandler))
- http.HandleFunc("/result", AuthHandler(resultHandler))
- http.HandleFunc("/tag", AuthHandler(tagHandler))
- http.HandleFunc("/todo", AuthHandler(todoHandler))
-
- // public handlers
- http.HandleFunc("/log/", logHandler)
-}
-
-func validHash(hash string) bool {
- // TODO(adg): correctly validate a hash
- return hash != ""
-}
-
-func logErr(w http.ResponseWriter, r *http.Request, err os.Error) {
- appengine.NewContext(r).Errorf("Error: %v", err)
- w.WriteHeader(http.StatusInternalServerError)
- fmt.Fprint(w, "Error: ", err)
-}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+ "appengine"
+ "appengine/datastore"
+ "crypto/hmac"
+ "fmt"
+ "http"
+ "json"
+ "os"
+)
+
+const commitsPerPage = 30
+
+// defaultPackages specifies the Package records to be created by initHandler.
+var defaultPackages = []*Package{
+ &Package{Name: "Go"},
+}
+
+// commitHandler retrieves commit data or records a new commit.
+//
+// For GET requests it returns a Commit value for the specified
+// packagePath and hash.
+//
+// For POST requests it reads a JSON-encoded Commit value from the request
+// body and creates a new Commit entity. It also updates the "tip" Tag for
+// each new commit at tip.
+//
+// This handler is used by a gobuilder process in -commit mode.
+func commitHandler(r *http.Request) (interface{}, os.Error) {
+ c := appengine.NewContext(r)
+ com := new(Commit)
+
+ if r.Method == "GET" {
+ com.PackagePath = r.FormValue("packagePath")
+ com.Hash = r.FormValue("hash")
+ if err := datastore.Get(c, com.Key(c), com); err != nil {
+ return nil, fmt.Errorf("getting Commit: %v", err)
+ }
+ return com, nil
+ }
+ if r.Method != "POST" {
+ return nil, errBadMethod(r.Method)
+ }
+
+ // POST request
+ defer r.Body.Close()
+ if err := json.NewDecoder(r.Body).Decode(com); err != nil {
+ return nil, fmt.Errorf("decoding Body: %v", err)
+ }
+ if len(com.Desc) > maxDatastoreStringLen {
+ com.Desc = com.Desc[:maxDatastoreStringLen]
+ }
+ if err := com.Valid(); err != nil {
+ return nil, fmt.Errorf("validating Commit: %v", err)
+ }
+ tx := func(c appengine.Context) os.Error {
+ return addCommit(c, com)
+ }
+ return nil, datastore.RunInTransaction(c, tx, nil)
+}
+
+// addCommit adds the Commit entity to the datastore and updates the tip Tag.
+// It must be run inside a datastore transaction.
+func addCommit(c appengine.Context, com *Commit) os.Error {
+ var tc Commit // temp value so we don't clobber com
+ err := datastore.Get(c, com.Key(c), &tc)
+ if err != datastore.ErrNoSuchEntity {
+ // if this commit is already in the datastore, do nothing
+ if err == nil {
+ return nil
+ }
+ return fmt.Errorf("getting Commit: %v", err)
+ }
+ // get the next commit number
+ p, err := GetPackage(c, com.PackagePath)
+ if err != nil {
+ return fmt.Errorf("GetPackage: %v", err)
+ }
+ com.Num = p.NextNum
+ p.NextNum++
+ if _, err := datastore.Put(c, p.Key(c), p); err != nil {
+ return fmt.Errorf("putting Package: %v", err)
+ }
+ // if this isn't the first Commit test the parent commit exists
+ if com.Num > 0 {
+ n, err := datastore.NewQuery("Commit").
+ Filter("Hash =", com.ParentHash).
+ Ancestor(p.Key(c)).
+ Count(c)
+ if err != nil {
+ return fmt.Errorf("testing for parent Commit: %v", err)
+ }
+ if n == 0 {
+ return os.NewError("parent commit not found")
+ }
+ }
+ // update the tip Tag if this is the Go repo
+ if p.Path == "" {
+ t := &Tag{Kind: "tip", Hash: com.Hash}
+ if _, err = datastore.Put(c, t.Key(c), t); err != nil {
+ return fmt.Errorf("putting Tag: %v", err)
+ }
+ }
+ // put the Commit
+ if _, err = datastore.Put(c, com.Key(c), com); err != nil {
+ return fmt.Errorf("putting Commit: %v", err)
+ }
+ return nil
+}
+
+// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
+// request body and updates the Tag entity for the Kind of tag provided.
+//
+// This handler is used by a gobuilder process in -commit mode.
+func tagHandler(r *http.Request) (interface{}, os.Error) {
+ if r.Method != "POST" {
+ return nil, errBadMethod(r.Method)
+ }
+
+ t := new(Tag)
+ defer r.Body.Close()
+ if err := json.NewDecoder(r.Body).Decode(t); err != nil {
+ return nil, err
+ }
+ if err := t.Valid(); err != nil {
+ return nil, err
+ }
+ c := appengine.NewContext(r)
+ _, err := datastore.Put(c, t.Key(c), t)
+ return nil, err
+}
+
+// Todo is a todoHandler response.
+type Todo struct {
+ Kind string // "build-go-commit" or "build-package"
+ Data interface{}
+}
+
+// todoHandler returns the next action to be performed by a builder.
+// It expects "builder" and "kind" query parameters and returns a *Todo value.
+// Multiple "kind" parameters may be specified.
+func todoHandler(r *http.Request) (todo interface{}, err os.Error) {
+ c := appengine.NewContext(r)
+ builder := r.FormValue("builder")
+ for _, kind := range r.Form["kind"] {
+ var data interface{}
+ switch kind {
+ case "build-go-commit":
+ data, err = buildTodo(c, builder, "", "")
+ case "build-package":
+ data, err = buildTodo(
+ c, builder,
+ r.FormValue("packagePath"),
+ r.FormValue("goHash"),
+ )
+ }
+ if data != nil || err != nil {
+ return &Todo{Kind: kind, Data: data}, err
+ }
+ }
+ return nil, nil
+}
+
+// buildTodo returns the next Commit to be built (or nil if none available).
+//
+// If packagePath and goHash are empty, it scans the first 20 Go Commits in
+// Num-descending order and returns the first one it finds that doesn't have a
+// Result for this builder.
+//
+// If provided with non-empty packagePath and goHash args, it scans the first
+// 20 Commits in Num-descending order for the specified packagePath and
+// returns the first that doesn't have a Result for this builder and goHash.
+func buildTodo(c appengine.Context, builder, packagePath, goHash string) (interface{}, os.Error) {
+ p, err := GetPackage(c, packagePath)
+ if err != nil {
+ return nil, err
+ }
+
+ t := datastore.NewQuery("Commit").
+ Ancestor(p.Key(c)).
+ Limit(commitsPerPage).
+ Order("-Num").
+ Run(c)
+ for {
+ com := new(Commit)
+ if _, err := t.Next(com); err != nil {
+ if err == datastore.Done {
+ err = nil
+ }
+ return nil, err
+ }
+ if com.Result(builder, goHash) == nil {
+ return com, nil
+ }
+ }
+ panic("unreachable")
+}
+
+// packagesHandler returns a list of the non-Go Packages monitored
+// by the dashboard.
+func packagesHandler(r *http.Request) (interface{}, os.Error) {
+ return Packages(appengine.NewContext(r))
+}
+
+// resultHandler records a build result.
+// It reads a JSON-encoded Result value from the request body,
+// creates a new Result entity, and updates the relevant Commit entity.
+// If the Log field is not empty, resultHandler creates a new Log entity
+// and updates the LogHash field before putting the Commit entity.
+func resultHandler(r *http.Request) (interface{}, os.Error) {
+ if r.Method != "POST" {
+ return nil, errBadMethod(r.Method)
+ }
+
+ c := appengine.NewContext(r)
+ res := new(Result)
+ defer r.Body.Close()
+ if err := json.NewDecoder(r.Body).Decode(res); err != nil {
+ return nil, fmt.Errorf("decoding Body: %v", err)
+ }
+ if err := res.Valid(); err != nil {
+ return nil, fmt.Errorf("validating Result: %v", err)
+ }
+ // store the Log text if supplied
+ if len(res.Log) > 0 {
+ hash, err := PutLog(c, res.Log)
+ if err != nil {
+ return nil, fmt.Errorf("putting Log: %v", err)
+ }
+ res.LogHash = hash
+ }
+ tx := func(c appengine.Context) os.Error {
+ // check Package exists
+ if _, err := GetPackage(c, res.PackagePath); err != nil {
+ return fmt.Errorf("GetPackage: %v", err)
+ }
+ // put Result
+ if _, err := datastore.Put(c, res.Key(c), res); err != nil {
+ return fmt.Errorf("putting Result: %v", err)
+ }
+ // add Result to Commit
+ com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
+ if err := com.AddResult(c, res); err != nil {
+ return fmt.Errorf("AddResult: %v", err)
+ }
+ // Send build failure notifications, if necessary.
+ // Note this must run after the call AddResult, which
+ // populates the Commit's ResultData field.
+ return notifyOnFailure(c, com, res.Builder)
+ }
+ return nil, datastore.RunInTransaction(c, tx, nil)
+}
+
+// logHandler displays log text for a given hash.
+// It handles paths like "/log/hash".
+func logHandler(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-type", "text/plain")
+ c := appengine.NewContext(r)
+ hash := r.URL.Path[len("/log/"):]
+ key := datastore.NewKey(c, "Log", hash, 0, nil)
+ l := new(Log)
+ if err := datastore.Get(c, key, l); err != nil {
+ logErr(w, r, err)
+ return
+ }
+ b, err := l.Text()
+ if err != nil {
+ logErr(w, r, err)
+ return
+ }
+ w.Write(b)
+}
+
+type dashHandler func(*http.Request) (interface{}, os.Error)
+
+type dashResponse struct {
+ Response interface{}
+ Error string
+}
+
+// errBadMethod is returned by a dashHandler when
+// the request has an unsuitable method.
+type errBadMethod string
+
+func (e errBadMethod) String() string {
+ return "bad method: " + string(e)
+}
+
+// AuthHandler wraps a http.HandlerFunc with a handler that validates the
+// supplied key and builder query parameters.
+func AuthHandler(h dashHandler) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ c := appengine.NewContext(r)
+
+ // Put the URL Query values into r.Form to avoid parsing the
+ // request body when calling r.FormValue.
+ r.Form = r.URL.Query()
+
+ var err os.Error
+ var resp interface{}
+
+ // Validate key query parameter for POST requests only.
+ key := r.FormValue("key")
+ if r.Method == "POST" && key != secretKey && !appengine.IsDevAppServer() {
+ h := hmac.NewMD5([]byte(secretKey))
+ h.Write([]byte(r.FormValue("builder")))
+ if key != fmt.Sprintf("%x", h.Sum()) {
+ err = os.NewError("invalid key: " + key)
+ }
+ }
+
+ // Call the original HandlerFunc and return the response.
+ if err == nil {
+ resp, err = h(r)
+ }
+
+ // Write JSON response.
+ dashResp := &dashResponse{Response: resp}
+ if err != nil {
+ c.Errorf("%v", err)
+ dashResp.Error = err.String()
+ }
+ w.Header().Set("Content-Type", "application/json")
+ if err = json.NewEncoder(w).Encode(dashResp); err != nil {
+ c.Criticalf("encoding response: %v", err)
+ }
+ }
+}
+
+func initHandler(w http.ResponseWriter, r *http.Request) {
+ // TODO(adg): devise a better way of bootstrapping new packages
+ c := appengine.NewContext(r)
+ for _, p := range defaultPackages {
+ if err := datastore.Get(c, p.Key(c), new(Package)); err == nil {
+ continue
+ } else if err != datastore.ErrNoSuchEntity {
+ logErr(w, r, err)
+ return
+ }
+ if _, err := datastore.Put(c, p.Key(c), p); err != nil {
+ logErr(w, r, err)
+ return
+ }
+ }
+ fmt.Fprint(w, "OK")
+}
+
+func init() {
+ // admin handlers
+ http.HandleFunc("/init", initHandler)
+
+ // authenticated handlers
+ http.HandleFunc("/commit", AuthHandler(commitHandler))
+ http.HandleFunc("/packages", AuthHandler(packagesHandler))
+ http.HandleFunc("/result", AuthHandler(resultHandler))
+ http.HandleFunc("/tag", AuthHandler(tagHandler))
+ http.HandleFunc("/todo", AuthHandler(todoHandler))
+
+ // public handlers
+ http.HandleFunc("/log/", logHandler)
+}
+
+func validHash(hash string) bool {
+ // TODO(adg): correctly validate a hash
+ return hash != ""
+}
+
+func logErr(w http.ResponseWriter, r *http.Request, err os.Error) {
+ appengine.NewContext(r).Errorf("Error: %v", err)
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, "Error: ", err)
+}