# This is a utility script for implementing a Go build slave.
+import binascii
import httplib
import os
+import struct
import subprocess
import sys
import time
return doInit(args)
elif args[1] == 'hwget':
return doHWGet(args)
+ elif args[1] == 'hwset':
+ return doHWSet(args)
elif args[1] == 'next':
return doNext(args)
elif args[1] == 'record':
return doRecord(args)
+ elif args[1] == 'benchmarks':
+ return doBenchmarks(args)
else:
return usage(args[0])
Commands:
init <rev>: init the build bot with the given commit as the first in history
hwget <builder>: get the most recent revision built by the given builder
+ hwset <builder> <rev>: get the most recent revision built by the given builder
next <builder>: get the next revision number to by built by the given builder
record <builder> <rev> <ok|log file>: record a build result
+ benchmarks <builder> <rev> <log file>: record benchmark numbers
''' % name)
return 1
if reply.status == 200:
print reply.read()
elif reply.status == 500 and retries < 3:
+ time.sleep(3)
return doHWGet(args, retries = retries + 1)
else:
raise Failed('get-hw returned %d' % reply.status)
return 0
+def doHWSet(args):
+ if len(args) != 4:
+ return usage(args[0])
+ c = getCommit(args[3])
+ if c is None:
+ fatal('Cannot get commit %s' % args[3])
+
+ return command('hw-set', {'builder': args[2], 'hw': c.node})
+
def doNext(args):
if len(args) != 3:
return usage(args[0])
c = getCommit(rev)
next = getCommit(str(c.num + 1))
- if next is not None:
+ if next is not None and next.parent == c.node:
print c.num + 1
else:
print "<none>"
log = file(logfile, 'r').read()
return command('build', {'node': c.node, 'parent': c.parent, 'date': c.date, 'user': c.user, 'desc': c.desc, 'log': log, 'builder': builder})
-if __name__ == '__main__':
- sys.exit(main(sys.argv))
+def doBenchmarks(args):
+ if len(args) != 5:
+ return usage(args[0])
+ builder = args[2]
+ rev = args[3]
+ c = getCommit(rev)
+ if c is None:
+ print >>sys.stderr, "Bad revision:", rev
+ return 1
+
+ benchmarks = {}
+ for line in file(args[4], 'r').readlines():
+ if 'Benchmark' in line and 'ns/op' in line:
+ parts = line.split()
+ if parts[3] == 'ns/op':
+ benchmarks[parts[0]] = (parts[1], parts[2])
+
+ e = []
+ for (name, (a, b)) in benchmarks.items():
+ e.append(struct.pack('>H', len(name)))
+ e.append(name)
+ e.append(struct.pack('>H', len(a)))
+ e.append(a)
+ e.append(struct.pack('>H', len(b)))
+ e.append(b)
+ return command('benchmarks', {'node': c.node, 'builder': builder, 'benchmarkdata': binascii.b2a_base64(''.join(e))})
def encodeMultipartFormdata(fields, files):
"""fields is a sequence of (name, value) elements for regular form fields.
return command(cmd, args, retries = retries + 1)
if reply.status != 200:
raise Failed('Command "%s" returned %d' % (cmd, reply.status))
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
# license that can be found in the LICENSE file.
fatal() {
- echo $1
+ echo $0: $1 1>&2
exit 1
}
fi
if [ ! -f buildcontrol.py ] ; then
- fatal "Please include buildcontrol.py in this directory"
+ fatal 'Please include buildcontrol.py in this directory'
fi
if [ "x$BUILDER" == "x" ] ; then
- fatal "Please set \$BUILDER to the name of this builder"
+ fatal 'Please set $BUILDER to the name of this builder'
fi
if [ "x$BUILDHOST" == "x" ] ; then
- fatal "Please set \$BUILDHOST to the hostname of the gobuild server"
+ fatal 'Please set $BUILDHOST to the hostname of the gobuild server'
fi
if [ "x$GOARCH" == "x" -o "x$GOOS" == "x" ] ; then
- fatal "Please set $GOARCH and $GOOS"
+ fatal 'Please set $GOARCH and $GOOS'
fi
export PATH=$PATH:`pwd`/candidate/bin
else
echo "Recording success for $rev"
python ../../buildcontrol.py record $BUILDER $rev ok || fatal "Cannot record result"
+ echo "Running benchmarks"
+ cd pkg || fatal "failed to cd to pkg"
+ make bench > ../../benchmarks 2>&1
+ if [ $? -eq 0 ] ; then
+ python ../../../buildcontrol.py benchmarks $BUILDER $rev ../../benchmarks || fatal "Cannot record benchmarks"
+ fi
+ cd .. || fatal "failed to cd out of pkg"
fi
cd ../.. || fatal "Cannot cd up"
done
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.ext.webapp.util import run_wsgi_app
+import binascii
import datetime
import hashlib
import logging
import os
import re
+import struct
import key
-# The main class of state are commit objects. One of these exists for each of
+# The majority of our state are commit objects. One of these exists for each of
# the commits known to the build system. Their key names are of the form
# <commit number (%08x)> "-" <hg hash>. This means that a sorting by the key
# name is sufficient to order the commits.
# successful.
builds = db.StringListProperty()
+class Benchmark(db.Model):
+ name = db.StringProperty()
+
+class BenchmarkResult(db.Model):
+ num = db.IntegerProperty()
+ builder = db.StringProperty()
+ iterations = db.IntegerProperty()
+ nsperop = db.IntegerProperty()
+
# A Log contains the textual build log of a failed build. The key name is the
# hex digest of the SHA256 hash of the contents.
class Log(db.Model):
self.response.set_status(200)
self.response.out.write(hw.commit)
+class SetHighwater(webapp.RequestHandler):
+ def post(self):
+ if self.request.get('key') != key.accessKey:
+ self.response.set_status(403)
+ return
+
+ builder = self.request.get('builder')
+ newhw = self.request.get('hw')
+ q = Commit.all()
+ q.filter('node =', newhw)
+ c = q.get()
+ if c is None:
+ self.response.set_status(404)
+ return
+
+ hw = Highwater(key_name = 'hw-%s' % builder)
+ hw.commit = c.node
+ hw.put()
+
class LogHandler(webapp.RequestHandler):
def get(self):
self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
q.filter('node =', parent)
p = q.get()
if p is None:
+ logging.error('Cannot find parent %s of node %s' % (parent, node))
self.response.set_status(404)
return
parentnum, _ = p.key().name().split('-', 1)
self.response.set_status(200)
+class Benchmarks(webapp.RequestHandler):
+ def get(self):
+ q = Benchmark.all()
+ bs = q.fetch(10000)
+
+ self.response.set_status(200)
+ self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
+ self.response.out.write('{"benchmarks": [\n')
+
+ first = True
+ for b in bs:
+ if not first:
+ self.response.out.write(',"' + b.name + '"\n')
+ else:
+ self.response.out.write('"' + b.name + '"\n')
+ first = False
+ self.response.out.write(']}\n')
+
+ def post(self):
+ if self.request.get('key') != key.accessKey:
+ self.response.set_status(403)
+ return
+
+ builder = self.request.get('builder')
+ node = self.request.get('node')
+ if not validNode(node):
+ logging.error("Not valid node ('%s')", node)
+ self.response.set_status(500)
+ return
+
+ benchmarkdata = self.request.get('benchmarkdata')
+ benchmarkdata = binascii.a2b_base64(benchmarkdata)
+
+ def get_string(i):
+ l, = struct.unpack('>H', i[:2])
+ s = i[2:2+l]
+ if len(s) != l:
+ return None, None
+ return s, i[2+l:]
+
+ benchmarks = {}
+ while len(benchmarkdata) > 0:
+ name, benchmarkdata = get_string(benchmarkdata)
+ iterations_str, benchmarkdata = get_string(benchmarkdata)
+ time_str, benchmarkdata = get_string(benchmarkdata)
+ iterations = int(iterations_str)
+ time = int(time_str)
+
+ benchmarks[name] = (iterations, time)
+
+ q = Commit.all()
+ q.filter('node =', node)
+ n = q.get()
+ if n is None:
+ logging.error('Client asked for unknown commit while uploading benchmarks')
+ self.response.set_status(404)
+ return
+
+ for (benchmark, (iterations, time)) in benchmarks.items():
+ b = Benchmark.get_or_insert(benchmark.encode('base64'), name = benchmark)
+ r = BenchmarkResult(key_name = '%08x/builder' % n.num, parent = b, num = n.num, iterations = iterations, nsperop = time, builder = builder)
+ r.put()
+
+ self.response.set_status(200)
+
+class GetBenchmarks(webapp.RequestHandler):
+ def get(self):
+ self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
+ benchmark = self.request.path[12:].decode('hex').encode('base64')
+
+ b = Benchmark.get_by_key_name(benchmark)
+ if b is None:
+ self.response.set_status(404)
+ return
+
+ q = BenchmarkResult.all()
+ q.ancestor(b)
+ q.order('-__key__')
+ results = q.fetch(10000)
+
+ if len(results) == 0:
+ self.response.set_status(404)
+ return
+
+ max = -1
+ min = 2000000000
+ builders = set()
+ for r in results:
+ if max < r.num:
+ max = r.num
+ if min > r.num:
+ min = r.num
+ builders.add(r.builder)
+
+ res = {}
+ for b in builders:
+ res[b] = [[-1] * ((max - min) + 1), [-1] * ((max - min) + 1)]
+
+ for r in results:
+ res[r.builder][0][r.num - min] = r.iterations
+ res[r.builder][1][r.num - min] = r.nsperop
+
+ self.response.out.write(str(res))
+
class FixedOffset(datetime.tzinfo):
"""Fixed offset in minutes east from UTC."""
[('/', MainPage),
('/log/.*', LogHandler),
('/hw-get', GetHighwater),
+ ('/hw-set', SetHighwater),
('/init', Init),
('/build', Build),
+ ('/benchmarks', Benchmarks),
+ ('/benchmarks/.*', GetBenchmarks),
])
def main():