+++ /dev/null
-# Copyright 2010 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# This is the server part of the package dashboard.
-# It must be run by App Engine.
-
-from google.appengine.api import mail
-from google.appengine.api import memcache
-from google.appengine.api import taskqueue
-from google.appengine.api import urlfetch
-from google.appengine.api import users
-from google.appengine.ext import db
-from google.appengine.ext import webapp
-from google.appengine.ext.webapp import template
-from google.appengine.ext.webapp.util import run_wsgi_app
-import datetime
-import logging
-import os
-import re
-import sets
-import urllib2
-
-# local imports
-from auth import auth
-import toutf8
-import const
-
-template.register_template_library('toutf8')
-
-# Storage model for package info recorded on server.
-class Package(db.Model):
- path = db.StringProperty()
- web_url = db.StringProperty() # derived from path
- count = db.IntegerProperty() # grand total
- week_count = db.IntegerProperty() # rolling weekly count
- day_count = db.TextProperty(default='') # daily count
- last_install = db.DateTimeProperty()
-
- # data contributed by gobuilder
- info = db.StringProperty()
- ok = db.BooleanProperty()
- last_ok = db.DateTimeProperty()
-
- def get_day_count(self):
- counts = {}
- if not self.day_count:
- return counts
- for d in str(self.day_count).split('\n'):
- date, count = d.split(' ')
- counts[date] = int(count)
- return counts
-
- def set_day_count(self, count):
- days = []
- for day, count in count.items():
- days.append('%s %d' % (day, count))
- days.sort(reverse=True)
- days = days[:28]
- self.day_count = '\n'.join(days)
-
- def inc(self):
- count = self.get_day_count()
- today = str(datetime.date.today())
- count[today] = count.get(today, 0) + 1
- self.set_day_count(count)
- self.update_week_count(count)
- self.count += 1
-
- def update_week_count(self, count=None):
- if count is None:
- count = self.get_day_count()
- total = 0
- today = datetime.date.today()
- for i in range(7):
- day = str(today - datetime.timedelta(days=i))
- if day in count:
- total += count[day]
- self.week_count = total
-
-
-# PackageDaily kicks off the daily package maintenance cron job
-# and serves the associated task queue.
-class PackageDaily(webapp.RequestHandler):
-
- def get(self):
- # queue a task to update each package with a week_count > 0
- keys = Package.all(keys_only=True).filter('week_count >', 0)
- for key in keys:
- taskqueue.add(url='/package/daily', params={'key': key.name()})
-
- def post(self):
- # update a single package (in a task queue)
- def update(key):
- p = Package.get_by_key_name(key)
- if not p:
- return
- p.update_week_count()
- p.put()
- key = self.request.get('key')
- if not key:
- return
- db.run_in_transaction(update, key)
-
-
-class Project(db.Model):
- name = db.StringProperty(indexed=True)
- descr = db.StringProperty()
- web_url = db.StringProperty()
- package = db.ReferenceProperty(Package)
- category = db.StringProperty(indexed=True)
- tags = db.ListProperty(str)
- approved = db.BooleanProperty(indexed=True)
-
-
-re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$')
-re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg|git)(/[a-z0-9A-Z_.\-/]+)?$')
-re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$')
-re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$')
-
-def vc_to_web(path):
- if re_bitbucket.match(path):
- m = re_bitbucket.match(path)
- check_url = 'http://' + m.group(1) + '/?cmd=heads'
- web = 'http://' + m.group(1) + '/'
- elif re_github.match(path):
- m = re_github_web.match(path)
- check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/'
- web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/'
- elif re_googlecode.match(path):
- m = re_googlecode.match(path)
- check_url = 'http://'+path
- if not m.group(2): # append / after bare '/hg' or '/git'
- check_url += '/'
- web = 'http://code.google.com/p/' + path[:path.index('.')]
- elif re_launchpad.match(path):
- check_url = web = 'https://'+path
- else:
- return False, False
- return web, check_url
-
-re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)')
-re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)')
-re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)')
-re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?')
-re_striphttp = re.compile(r'https?://(www\.)?')
-
-def find_googlecode_vcs(path):
- # Perform http request to path/hg or path/git to check if they're
- # using mercurial or git. Otherwise, assume svn.
- for vcs in ['git', 'hg']:
- try:
- response = urlfetch.fetch('http://'+path+vcs, deadline=1)
- if response.status_code == 200:
- return vcs
- except: pass
- return 'svn'
-
-def web_to_vc(url):
- url = re_striphttp.sub('', url)
- m = re_bitbucket_web.match(url)
- if m:
- return 'bitbucket.org/'+m.group(1)+'/'+m.group(2)
- m = re_github_web.match(url)
- if m:
- return 'github.com/'+m.group(1)+'/'+m.group(2)
- m = re_googlecode_web.match(url)
- if m:
- path = m.group(1)+'.googlecode.com/'
- vcs = find_googlecode_vcs(path)
- return path + vcs
- m = re_launchpad_web.match(url)
- if m:
- return m.group(0)
- return False
-
-MaxPathLength = 100
-CacheTimeout = 3600
-
-class PackagePage(webapp.RequestHandler):
- def get(self):
- if self.request.get('fmt') == 'json':
- return self.json()
-
- html = memcache.get('view-package')
- if not html:
- tdata = {}
-
- q = Package.all().filter('week_count >', 0)
- q.order('-week_count')
- tdata['by_week_count'] = q.fetch(50)
-
- q = Package.all()
- q.order('-last_install')
- tdata['by_time'] = q.fetch(20)
-
- q = Package.all()
- q.order('-count')
- tdata['by_count'] = q.fetch(100)
-
- path = os.path.join(os.path.dirname(__file__), 'package.html')
- html = template.render(path, tdata)
- memcache.set('view-package', html, time=CacheTimeout)
-
- self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
- self.response.out.write(html)
-
- def json(self):
- json = memcache.get('view-package-json')
- if not json:
- q = Package.all()
- s = '{"packages": ['
- sep = ''
- for r in q:
- s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count)
- sep = ','
- s += '\n]}\n'
- json = s
- memcache.set('view-package-json', json, time=CacheTimeout)
- self.response.set_status(200)
- self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
- self.response.out.write(json)
-
- def can_get_url(self, url):
- try:
- urllib2.urlopen(urllib2.Request(url))
- return True
- except:
- return False
-
- def is_valid_package_path(self, path):
- return (re_bitbucket.match(path) or
- re_googlecode.match(path) or
- re_github.match(path) or
- re_launchpad.match(path))
-
- def record_pkg(self, path):
- # sanity check string
- if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path):
- return False
-
- # look in datastore
- key = 'pkg-' + path
- p = Package.get_by_key_name(key)
- if p is None:
- # not in datastore - verify URL before creating
- web, check_url = vc_to_web(path)
- if not web:
- logging.error('unrecognized path: %s', path)
- return False
- if not self.can_get_url(check_url):
- logging.error('cannot get %s', check_url)
- return False
- p = Package(key_name = key, path = path, count = 0, web_url = web)
-
- if auth(self.request):
- # builder updating package metadata
- p.info = self.request.get('info')
- p.ok = self.request.get('ok') == "true"
- if p.ok:
- p.last_ok = datetime.datetime.utcnow()
- else:
- # goinstall reporting an install
- p.inc()
- p.last_install = datetime.datetime.utcnow()
-
- # update package object
- p.put()
- return True
-
- def post(self):
- path = self.request.get('path')
- ok = db.run_in_transaction(self.record_pkg, path)
- if ok:
- self.response.set_status(200)
- self.response.out.write('ok')
- else:
- logging.error('invalid path in post: %s', path)
- self.response.set_status(500)
- self.response.out.write('not ok')
-
-class ProjectPage(webapp.RequestHandler):
-
- def get(self):
- admin = users.is_current_user_admin()
- if self.request.path == "/project/login":
- self.redirect(users.create_login_url("/project"))
- elif self.request.path == "/project/logout":
- self.redirect(users.create_logout_url("/project"))
- elif self.request.path == "/project/edit" and admin:
- self.edit()
- elif self.request.path == "/project/assoc" and admin:
- self.assoc()
- else:
- self.list()
-
- def assoc(self):
- projects = Project.all()
- for p in projects:
- if p.package:
- continue
- path = web_to_vc(p.web_url)
- if not path:
- continue
- pkg = Package.get_by_key_name("pkg-"+path)
- if not pkg:
- self.response.out.write('no: %s %s<br>' % (p.web_url, path))
- continue
- p.package = pkg
- p.put()
- self.response.out.write('yes: %s %s<br>' % (p.web_url, path))
-
- def post(self):
- if self.request.path == "/project/edit":
- self.edit(True)
- else:
- data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"]))
- if reduce(lambda x, y: x or not y, data.values(), False):
- data["submitMsg"] = "You must complete all the fields."
- self.list(data)
- return
- p = Project.get_by_key_name("proj-"+data["name"])
- if p is not None:
- data["submitMsg"] = "A project by this name already exists."
- self.list(data)
- return
- p = Project(key_name="proj-"+data["name"], **data)
- p.put()
-
- path = os.path.join(os.path.dirname(__file__), 'project-notify.txt')
- mail.send_mail(
- sender=const.mail_from,
- to=const.mail_submit_to,
- subject=const.mail_submit_subject,
- body=template.render(path, {'project': p}))
-
- self.list({"submitMsg": "Your project has been submitted."})
-
- def list(self, additional_data={}):
- cache_key = 'view-project-data'
- tag = self.request.get('tag', None)
- if tag:
- cache_key += '-'+tag
- data = memcache.get(cache_key)
- admin = users.is_current_user_admin()
- if admin or not data:
- projects = Project.all().order('category').order('name')
- if not admin:
- projects = projects.filter('approved =', True)
- projects = list(projects)
-
- tags = sets.Set()
- for p in projects:
- for t in p.tags:
- tags.add(t)
-
- if tag:
- projects = filter(lambda x: tag in x.tags, projects)
-
- data = {}
- data['tag'] = tag
- data['tags'] = tags
- data['projects'] = projects
- data['admin']= admin
- if not admin:
- memcache.set(cache_key, data, time=CacheTimeout)
-
- for k, v in additional_data.items():
- data[k] = v
-
- self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
- path = os.path.join(os.path.dirname(__file__), 'project.html')
- self.response.out.write(template.render(path, data))
-
- def edit(self, save=False):
- if save:
- name = self.request.get("orig_name")
- else:
- name = self.request.get("name")
-
- p = Project.get_by_key_name("proj-"+name)
- if not p:
- self.response.out.write("Couldn't find that Project.")
- return
-
- if save:
- if self.request.get("do") == "Delete":
- p.delete()
- else:
- pkg_name = self.request.get("package", None)
- if pkg_name:
- pkg = Package.get_by_key_name("pkg-"+pkg_name)
- if pkg:
- p.package = pkg.key()
- for f in ['name', 'descr', 'web_url', 'category']:
- setattr(p, f, self.request.get(f, None))
- p.approved = self.request.get("approved") == "1"
- p.tags = filter(lambda x: x, self.request.get("tags", "").split(","))
- p.put()
- memcache.delete('view-project-data')
- self.redirect('/project')
- return
-
- # get all project categories and tags
- cats, tags = sets.Set(), sets.Set()
- for r in Project.all():
- cats.add(r.category)
- for t in r.tags:
- tags.add(t)
-
- self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
- path = os.path.join(os.path.dirname(__file__), 'project-edit.html')
- self.response.out.write(template.render(path, {
- "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) }))
-
- def redirect(self, url):
- self.response.set_status(302)
- self.response.headers.add_header("Location", url)
-
-def main():
- app = webapp.WSGIApplication([
- ('/package', PackagePage),
- ('/package/daily', PackageDaily),
- ('/project.*', ProjectPage),
- ], debug=True)
- run_wsgi_app(app)
-
-if __name__ == '__main__':
- main()
--- /dev/null
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+from google.appengine.api import mail
+from google.appengine.api import memcache
+from google.appengine.api import users
+from google.appengine.ext import db
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import template
+from google.appengine.ext.webapp.util import run_wsgi_app
+import os
+import sets
+
+# local imports
+import toutf8
+import const
+
+template.register_template_library('toutf8')
+
+class Project(db.Model):
+ name = db.StringProperty(indexed=True)
+ descr = db.StringProperty()
+ web_url = db.StringProperty()
+ package = db.ReferenceProperty(Package)
+ category = db.StringProperty(indexed=True)
+ tags = db.ListProperty(str)
+ approved = db.BooleanProperty(indexed=True)
+
+CacheTimeout = 3600
+
+class ProjectPage(webapp.RequestHandler):
+
+ def get(self):
+ admin = users.is_current_user_admin()
+ if self.request.path == "/project/login":
+ self.redirect(users.create_login_url("/project"))
+ elif self.request.path == "/project/edit" and admin:
+ self.edit()
+ else:
+ self.list()
+
+ def post(self):
+ if self.request.path == "/project/edit":
+ self.edit(True)
+ else:
+ data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"]))
+ if reduce(lambda x, y: x or not y, data.values(), False):
+ data["submitMsg"] = "You must complete all the fields."
+ self.list(data)
+ return
+ p = Project.get_by_key_name("proj-"+data["name"])
+ if p is not None:
+ data["submitMsg"] = "A project by this name already exists."
+ self.list(data)
+ return
+ p = Project(key_name="proj-"+data["name"], **data)
+ p.put()
+
+ path = os.path.join(os.path.dirname(__file__), 'project-notify.txt')
+ mail.send_mail(
+ sender=const.mail_from,
+ to=const.mail_submit_to,
+ subject=const.mail_submit_subject,
+ body=template.render(path, {'project': p}))
+
+ self.list({"submitMsg": "Your project has been submitted."})
+
+ def list(self, additional_data={}):
+ cache_key = 'view-project-data'
+ tag = self.request.get('tag', None)
+ if tag:
+ cache_key += '-'+tag
+ data = memcache.get(cache_key)
+ admin = users.is_current_user_admin()
+ if admin or not data:
+ projects = Project.all().order('category').order('name')
+ if not admin:
+ projects = projects.filter('approved =', True)
+ projects = list(projects)
+
+ tags = sets.Set()
+ for p in projects:
+ for t in p.tags:
+ tags.add(t)
+
+ if tag:
+ projects = filter(lambda x: tag in x.tags, projects)
+
+ data = {}
+ data['tag'] = tag
+ data['tags'] = tags
+ data['projects'] = projects
+ data['admin']= admin
+ if not admin:
+ memcache.set(cache_key, data, time=CacheTimeout)
+
+ for k, v in additional_data.items():
+ data[k] = v
+
+ self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
+ path = os.path.join(os.path.dirname(__file__), 'project.html')
+ self.response.out.write(template.render(path, data))
+
+ def edit(self, save=False):
+ if save:
+ name = self.request.get("orig_name")
+ else:
+ name = self.request.get("name")
+
+ p = Project.get_by_key_name("proj-"+name)
+ if not p:
+ self.response.out.write("Couldn't find that Project.")
+ return
+
+ if save:
+ if self.request.get("do") == "Delete":
+ p.delete()
+ else:
+ pkg_name = self.request.get("package", None)
+ if pkg_name:
+ pkg = Package.get_by_key_name("pkg-"+pkg_name)
+ if pkg:
+ p.package = pkg.key()
+ for f in ['name', 'descr', 'web_url', 'category']:
+ setattr(p, f, self.request.get(f, None))
+ p.approved = self.request.get("approved") == "1"
+ p.tags = filter(lambda x: x, self.request.get("tags", "").split(","))
+ p.put()
+ memcache.delete('view-project-data')
+ self.redirect('/project')
+ return
+
+ # get all project categories and tags
+ cats, tags = sets.Set(), sets.Set()
+ for r in Project.all():
+ cats.add(r.category)
+ for t in r.tags:
+ tags.add(t)
+
+ self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
+ path = os.path.join(os.path.dirname(__file__), 'project-edit.html')
+ self.response.out.write(template.render(path, {
+ "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) }))
+
+ def redirect(self, url):
+ self.response.set_status(302)
+ self.response.headers.add_header("Location", url)
+
+def main():
+ app = webapp.WSGIApplication([
+ ('/.*', ProjectPage),
+ ], debug=True)
+ run_wsgi_app(app)
+
+if __name__ == '__main__':
+ main()