From: Andrew Gerrand Date: Thu, 15 Dec 2011 23:48:06 +0000 (+1100) Subject: misc/dashboard: user interface X-Git-Tag: weekly.2011-12-22~161 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=80103cd54fa1a6ae0cd75a8c545a365bf31f58cf;p=gostls13.git misc/dashboard: user interface R=rsc CC=golang-dev https://golang.org/cl/5461047 --- diff --git a/misc/dashboard/app/app.yaml b/misc/dashboard/app/app.yaml index 0fb6fec6ab..d16f1a2ff4 100644 --- a/misc/dashboard/app/app.yaml +++ b/misc/dashboard/app/app.yaml @@ -4,9 +4,11 @@ runtime: go api_version: 3 handlers: +- url: /static + static_dir: static - url: /log/.+ script: _go_app -- url: /(commit|packages|result|tag|todo) +- url: /(|commit|packages|result|tag|todo) script: _go_app - url: /(init|buildtest) script: _go_app diff --git a/misc/dashboard/app/build/build.go b/misc/dashboard/app/build/build.go index 98ace20abf..bf298c7bed 100644 --- a/misc/dashboard/app/build/build.go +++ b/misc/dashboard/app/build/build.go @@ -27,6 +27,10 @@ type Package struct { NextNum int // Num of the next head Commit } +func (p *Package) String() string { + return fmt.Sprintf("%s: %q", p.Path, p.Name) +} + func (p *Package) Key(c appengine.Context) *datastore.Key { key := p.Path if key == "" { @@ -35,6 +39,24 @@ func (p *Package) Key(c appengine.Context) *datastore.Key { return datastore.NewKey(c, "Package", key, 0, nil) } +// LastCommit returns the most recent Commit for this Package. +func (p *Package) LastCommit(c appengine.Context) (*Commit, os.Error) { + var commits []*Commit + _, err := datastore.NewQuery("Commit"). + Ancestor(p.Key(c)). + Order("-Time"). + Limit(1). + GetAll(c, &commits) + if err != nil { + return nil, err + } + if len(commits) != 1 { + return nil, datastore.ErrNoSuchEntity + } + return commits[0], nil +} + +// GetPackage fetches a Package by path from the datastore. func GetPackage(c appengine.Context, path string) (*Package, os.Error) { p := &Package{Path: path} err := datastore.Get(c, p.Key(c), p) @@ -59,11 +81,11 @@ type Commit struct { Desc string `datastore:",noindex"` Time datastore.Time - // Result is the Data string of each build Result for this Commit. + // ResultData is the Data string of each build Result for this Commit. // For non-Go commits, only the Results for the current Go tip, weekly, // and release Tags are stored here. This is purely de-normalized data. // The complete data set is stored in Result entities. - Result []string `datastore:",noindex"` + ResultData []string `datastore:",noindex"` } func (com *Commit) Key(c appengine.Context) *datastore.Key { @@ -91,28 +113,45 @@ func (com *Commit) AddResult(c appengine.Context, r *Result) os.Error { if err := datastore.Get(c, com.Key(c), com); err != nil { return err } - com.Result = append(com.Result, r.Data()) + com.ResultData = append(com.ResultData, r.Data()) _, err := datastore.Put(c, com.Key(c), com) return err } -func (com *Commit) HasResult(builder string) bool { - for _, r := range com.Result { - if strings.SplitN(r, "|", 2)[0] == builder { - return true +// Result returns the build Result for this Commit for the given builder/goHash. +func (c *Commit) Result(builder, goHash string) *Result { + for _, r := range c.ResultData { + p := strings.SplitN(r, "|", 4) + if len(p) != 4 || p[0] != builder || p[3] != goHash { + continue } + return partsToHash(c, p) } - return false + return nil } -func (com *Commit) HasGoHashResult(builder, goHash string) bool { - for _, r := range com.Result { +// Results returns the build Results for this Commit for the given goHash. +func (c *Commit) Results(goHash string) (results []*Result) { + for _, r := range c.ResultData { p := strings.SplitN(r, "|", 4) - if len(p) == 4 && p[0] == builder && p[3] == goHash { - return true + if len(p) != 4 || p[3] != goHash { + continue } + results = append(results, partsToHash(c, p)) + } + return +} + +// partsToHash converts a Commit and ResultData substrings to a Result. +func partsToHash(c *Commit, p []string) *Result { + return &Result{ + Builder: p[0], + Hash: c.Hash, + PackagePath: c.PackagePath, + GoHash: p[3], + OK: p[1] == "true", + LogHash: p[2], } - return false } // A Result describes a build result for a Commit on an OS/architecture. @@ -137,10 +176,6 @@ func (r *Result) Key(c appengine.Context) *datastore.Key { return datastore.NewKey(c, "Result", key, 0, p.Key(c)) } -func (r *Result) Data() string { - return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash) -} - func (r *Result) Valid() os.Error { if !validHash(r.Hash) { return os.NewError("invalid Hash") @@ -151,6 +186,12 @@ func (r *Result) Valid() os.Error { return nil } +// Data returns the Result in string format +// to be stored in Commit's ResultData field. +func (r *Result) Data() string { + return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash) +} + // A Log is a gzip-compressed log file stored under the SHA1 hash of the // uncompressed log text. type Log struct { @@ -179,12 +220,12 @@ type Tag struct { } func (t *Tag) Key(c appengine.Context) *datastore.Key { - p := &Package{Path: ""} + p := &Package{} return datastore.NewKey(c, "Tag", t.Kind, 0, p.Key(c)) } func (t *Tag) Valid() os.Error { - if t.Kind != "weekly" || t.Kind != "release" || t.Kind != "tip" { + if t.Kind != "weekly" && t.Kind != "release" && t.Kind != "tip" { return os.NewError("invalid Kind") } if !validHash(t.Hash) { @@ -193,6 +234,21 @@ func (t *Tag) Valid() os.Error { return nil } +// GetTag fetches a Tag by name from the datastore. +func GetTag(c appengine.Context, tag string) (*Tag, os.Error) { + t := &Tag{Kind: tag} + if err := datastore.Get(c, t.Key(c), t); err != nil { + if err == datastore.ErrNoSuchEntity { + return nil, os.NewError("tag not found: " + tag) + } + return nil, err + } + if err := t.Valid(); err != nil { + return nil, err + } + return t, nil +} + // commitHandler retrieves commit data or records a new commit. // // For GET requests it returns a Commit value for the specified @@ -332,13 +388,7 @@ func todoHandler(r *http.Request) (interface{}, os.Error) { } return nil, err } - var hasResult bool - if goHash != "" { - hasResult = com.HasGoHashResult(builder, goHash) - } else { - hasResult = com.HasResult(builder) - } - if !hasResult { + if com.Result(builder, goHash) == nil { return com.Hash, nil } } @@ -348,7 +398,11 @@ func todoHandler(r *http.Request) (interface{}, os.Error) { // packagesHandler returns a list of the non-Go Packages monitored // by the dashboard. func packagesHandler(r *http.Request) (interface{}, os.Error) { - c := appengine.NewContext(r) + return Packages(appengine.NewContext(r)) +} + +// Packages returns all non-Go packages. +func Packages(c appengine.Context) ([]*Package, os.Error) { var pkgs []*Package for t := datastore.NewQuery("Package").Run(c); ; { pkg := new(Package) @@ -407,6 +461,8 @@ func resultHandler(r *http.Request) (interface{}, os.Error) { return nil, datastore.RunInTransaction(c, tx, nil) } +// logHandler displays log text for a given hash. +// It handles paths like "/log/hash". func logHandler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) h := r.URL.Path[len("/log/"):] @@ -426,12 +482,6 @@ func logHandler(w http.ResponseWriter, r *http.Request) { } } -type errBadMethod string - -func (e errBadMethod) String() string { - return "bad method: " + string(e) -} - type dashHandler func(*http.Request) (interface{}, os.Error) type dashResponse struct { @@ -439,6 +489,14 @@ type dashResponse struct { Error string } +// errBadMethod is returned by a dashHandler when +// the request has an unsuitable method. +type errBadMethod string + +func (e errBadMethod) String() string { + return "bad method: " + string(e) +} + // AuthHandler wraps a http.HandlerFunc with a handler that validates the // supplied key and builder query parameters. func AuthHandler(h dashHandler) http.HandlerFunc { @@ -449,7 +507,8 @@ func AuthHandler(h dashHandler) http.HandlerFunc { // Validate key query parameter for POST requests only. key := r.FormValue("key") - if r.Method == "POST" && key != secretKey { + if r.Method == "POST" && key != secretKey && + !appengine.IsDevAppServer() { h := sha1.New() h.Write([]byte(r.FormValue("builder") + secretKey)) if key != fmt.Sprintf("%x", h.Sum()) { @@ -476,7 +535,7 @@ func AuthHandler(h dashHandler) http.HandlerFunc { func initHandler(w http.ResponseWriter, r *http.Request) { // TODO(adg): devise a better way of bootstrapping new packages var pkgs = []*Package{ - &Package{Name: "Go", Path: ""}, + &Package{Name: "Go"}, &Package{Name: "Test", Path: "code.google.com/p/go.test"}, } c := appengine.NewContext(r) diff --git a/misc/dashboard/app/build/test.go b/misc/dashboard/app/build/test.go index 09aa89eb4f..a6a2582e18 100644 --- a/misc/dashboard/app/build/test.go +++ b/misc/dashboard/app/build/test.go @@ -17,6 +17,7 @@ import ( "json" "os" "strings" + "time" "url" ) @@ -40,6 +41,19 @@ var testPackages = []*Package{ testPackage, } +var tCommitTime = time.Seconds() - 60*60*24*7 + +func tCommit(hash, parentHash string) *Commit { + tCommitTime += 60 * 60 * 12 // each commit should have a different time + return &Commit{ + Hash: hash, + ParentHash: parentHash, + Time: datastore.Time(tCommitTime * 1e6), + User: "adg", + Desc: "change description", + } +} + var testRequests = []struct { path string vals url.Values @@ -50,9 +64,9 @@ var testRequests = []struct { {"/packages", nil, nil, []*Package{testPackage}}, // Go repo - {"/commit", nil, &Commit{Hash: "0001", ParentHash: "0000"}, nil}, - {"/commit", nil, &Commit{Hash: "0002", ParentHash: "0001"}, nil}, - {"/commit", nil, &Commit{Hash: "0003", ParentHash: "0002"}, nil}, + {"/commit", nil, tCommit("0001", "0000"), nil}, + {"/commit", nil, tCommit("0002", "0001"), nil}, + {"/commit", nil, tCommit("0003", "0002"), nil}, {"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"}, {"/todo", url.Values{"builder": {"linux-amd64"}}, nil, "0003"}, {"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil}, @@ -67,8 +81,8 @@ var testRequests = []struct { {"/todo", url.Values{"builder": {"linux-amd64"}}, nil, "0002"}, // branches - {"/commit", nil, &Commit{Hash: "0004", ParentHash: "0003"}, nil}, - {"/commit", nil, &Commit{Hash: "0005", ParentHash: "0002"}, nil}, + {"/commit", nil, tCommit("0004", "0003"), nil}, + {"/commit", nil, tCommit("0005", "0002"), nil}, {"/todo", url.Values{"builder": {"linux-386"}}, nil, "0005"}, {"/result", nil, &Result{Builder: "linux-386", Hash: "0005", OK: true}, nil}, {"/todo", url.Values{"builder": {"linux-386"}}, nil, "0004"}, @@ -92,6 +106,7 @@ var testRequests = []struct { {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0001", OK: true}, nil}, {"/todo", url.Values{"builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, nil}, {"/todo", url.Values{"builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0002"}}, nil, "1003"}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0005", OK: false, Log: []byte("boo")}, nil}, } func testHandler(w http.ResponseWriter, r *http.Request) { diff --git a/misc/dashboard/app/build/ui.go b/misc/dashboard/app/build/ui.go new file mode 100644 index 0000000000..918b53e5bd --- /dev/null +++ b/misc/dashboard/app/build/ui.go @@ -0,0 +1,187 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO(adg): packages at weekly/release +// TODO(adg): some means to register new packages + +package build + +import ( + "appengine" + "appengine/datastore" + "exp/template/html" + "http" + "os" + "regexp" + "sort" + "strconv" + "strings" + "template" +) + +func init() { + http.HandleFunc("/", uiHandler) + html.Escape(uiTemplate) +} + +// uiHandler draws the build status page. +func uiHandler(w http.ResponseWriter, r *http.Request) { + // TODO(adg): put the HTML in memcache and invalidate on updates + c := appengine.NewContext(r) + + page, _ := strconv.Atoi(r.FormValue("page")) + if page < 0 { + page = 0 + } + + commits, err := goCommits(c, page) + if err != nil { + logErr(w, r, err) + return + } + builders := commitBuilders(commits) + + tipState, err := TagState(c, "tip") + if err != nil { + logErr(w, r, err) + return + } + + p := &Pagination{} + if len(commits) == commitsPerPage { + p.Next = page + 1 + } + if page > 0 { + p.Prev = page - 1 + p.HasPrev = true + } + data := &uiTemplateData{commits, builders, tipState, p} + if err := uiTemplate.Execute(w, data); err != nil { + logErr(w, r, err) + } +} + +type Pagination struct { + Next, Prev int + HasPrev bool +} + +// goCommits gets a slice of the latest Commits to the Go repository. +// If page > 0 it paginates by commitsPerPage. +func goCommits(c appengine.Context, page int) ([]*Commit, os.Error) { + q := datastore.NewQuery("Commit"). + Ancestor((&Package{}).Key(c)). + Order("-Time"). + Limit(commitsPerPage). + Offset(page * commitsPerPage) + var commits []*Commit + _, err := q.GetAll(c, &commits) + return commits, err +} + +// commitBuilders returns the names of the builders that provided +// Results for the provided commits. +func commitBuilders(commits []*Commit) []string { + builders := make(map[string]bool) + for _, commit := range commits { + for _, r := range commit.Results("") { + builders[r.Builder] = true + } + } + return keys(builders) +} + +func keys(m map[string]bool) (s []string) { + for k := range m { + s = append(s, k) + } + sort.Strings(s) + return +} + +// PackageState represents the state of a Package at a tag. +type PackageState struct { + *Package + *Commit + Results []*Result + OK bool +} + +// TagState fetches the results for all non-Go packages at the specified tag. +func TagState(c appengine.Context, name string) ([]*PackageState, os.Error) { + tag, err := GetTag(c, name) + if err != nil { + return nil, err + } + pkgs, err := Packages(c) + if err != nil { + return nil, err + } + var states []*PackageState + for _, pkg := range pkgs { + commit, err := pkg.LastCommit(c) + if err != nil { + c.Errorf("no Commit found: %v", pkg) + continue + } + results := commit.Results(tag.Hash) + ok := len(results) > 0 + for _, r := range results { + ok = ok && r.OK + } + states = append(states, &PackageState{ + pkg, commit, results, ok, + }) + } + return states, nil +} + +type uiTemplateData struct { + Commits []*Commit + Builders []string + TipState []*PackageState + Pagination *Pagination +} + +var uiTemplate = template.Must( + template.New("ui"). + Funcs(template.FuncMap{ + "builderTitle": builderTitle, + "shortHash": shortHash, + "repoURL": repoURL, + }). + ParseFile("build/ui.html"), +) + +// builderTitle formats "linux-amd64-foo" as "linux amd64 foo". +func builderTitle(s string) string { + return strings.Replace(s, "-", " ", -1) +} + +// shortHash returns a the short version of a hash. +func shortHash(hash string) string { + if len(hash) > 12 { + hash = hash[:12] + } + return hash +} + +// repoRe matches Google Code repositories and subrepositories (without paths). +var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+)(\.[a-z0-9\-]+)?$`) + +// repoURL returns the URL of a change at a Google Code repository or subrepo. +func repoURL(hash, packagePath string) (string, os.Error) { + if packagePath == "" { + return "https://code.google.com/p/go/source/detail?r=" + hash, nil + } + m := repoRe.FindStringSubmatch(packagePath) + if m == nil { + return "", os.NewError("unrecognized package: " + packagePath) + } + url := "https://code.google.com/p/" + m[1] + "/source/detail?r=" + hash + if len(m) > 2 { + url += "&repo=" + m[2][1:] + } + return url, nil +} diff --git a/misc/dashboard/app/build/ui.html b/misc/dashboard/app/build/ui.html new file mode 100644 index 0000000000..684ae1333f --- /dev/null +++ b/misc/dashboard/app/build/ui.html @@ -0,0 +1,138 @@ + + + + Go Build Dashboard + + + + +

Go Build Status

+ +

Go

+ + {{if $.Commits}} + + + + + {{range $.Builders}} + + {{end}} + + {{range $c := $.Commits}} + + + {{range $.Builders}} + + {{end}} + + + + + {{end}} +
 {{builderTitle .}}
{{shortHash .Hash}} + {{with $c.Result . ""}} + {{if .OK}} + ok + {{else}} + fail + {{end}} + {{else}} +   + {{end}} + {{.User}}{{.Time.Time}}{{.Desc}}
+ + {{with $.Pagination}} +
+ prev + next + top +
+ {{end}} + + {{else}} +

No commits to display. Hm.

+ {{end}} + +

Other packages

+ + + + + + + + {{range $state := $.TipState}} + + + + + + {{end}} +
StatePackage 
+ {{if .Results}} + + {{else}} +   + {{end}} + {{.Package.Name}} + {{range .Results}} +
+ {{$h := $state.Commit.Hash}} + {{shortHash $h}} + failed + on {{.Builder}}/{{shortHash .GoHash}} +
+ {{end}} +
+ + + diff --git a/misc/dashboard/app/static/status_alert.gif b/misc/dashboard/app/static/status_alert.gif new file mode 100644 index 0000000000..495d9d2e0c Binary files /dev/null and b/misc/dashboard/app/static/status_alert.gif differ diff --git a/misc/dashboard/app/static/status_good.gif b/misc/dashboard/app/static/status_good.gif new file mode 100644 index 0000000000..ef9c5a8f64 Binary files /dev/null and b/misc/dashboard/app/static/status_good.gif differ