From 49ac7efd683152e4936f8013bb7a001470260034 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 17:18:55 -0500 Subject: Package signoff email report, initial revision Signed-off-by: Dan McGee --- packages/management/__init__.py | 0 packages/management/commands/__init__.py | 0 packages/management/commands/signoff_report.py | 110 +++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 packages/management/__init__.py create mode 100644 packages/management/commands/__init__.py create mode 100644 packages/management/commands/signoff_report.py (limited to 'packages/management') diff --git a/packages/management/__init__.py b/packages/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/management/commands/__init__.py b/packages/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py new file mode 100644 index 00000000..17e58f39 --- /dev/null +++ b/packages/management/commands/signoff_report.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +""" +signoff_report command + +Send an email summarizing the state of outstanding signoffs for the given +repository. + +Usage: ./manage.py signoff_report +""" + +from django.core.urlresolvers import reverse +from django.core.management.base import BaseCommand, CommandError +from django.contrib.auth.models import User +from django.contrib.sites.models import Site +from django.db.models import Count +from django.template import loader, Context + +from collections import namedtuple +from datetime import datetime, timedelta +import logging +from operator import attrgetter +import sys + +from main.models import Package, Repo +from packages.models import Signoff +from packages.utils import get_signoff_groups + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(BaseCommand): + args = " " + help = "Send a signoff report for the given repository." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v == 2: + logger.level = logging.DEBUG + + if len(args) != 2: + raise CommandError("email and repository must be provided") + + return generate_report(args[0], args[1]) + +def generate_report(email, repo_name): + repo = Repo.objects.get(name__iexact=repo_name) + # Collect all existing signoffs for these packages + signoff_groups = sorted(get_signoff_groups([repo]), + key=attrgetter('target_repo', 'arch', 'pkgbase')) + complete = [] + incomplete = [] + new = [] + old = [] + + new_hours = 24 + old_days = 14 + now = datetime.utcnow() + new_cutoff = now - timedelta(hours=new_hours) + old_cutoff = now - timedelta(days=old_days) + + for group in signoff_groups: + if group.approved(): + complete.append(group) + else: + incomplete.append(group) + if group.package.last_update > new_cutoff: + new.append(group) + if group.package.last_update < old_cutoff: + old.append(group) + + old.sort(key=attrgetter('last_update')) + + proto = 'https' + domain = Site.objects.get_current().domain + signoffs_url = '%s://%s%s' % (proto, domain, reverse('package-signoffs')) + + # and the fun bit + Leader = namedtuple('Leader', ['user', 'count']) + leaders = Signoff.objects.filter(created__gt=new_cutoff, + revoked__isnull=True).values_list('user').annotate( + signoff_count=Count('pk')).order_by('-signoff_count')[:5] + users = User.objects.in_bulk([l[0] for l in leaders]) + leaders = (Leader(users[l[0]], l[1]) for l in leaders) + + subject = 'Signoff report for [%s]' % repo.name.lower() + t = loader.get_template('packages/signoff_report.txt') + c = Context({ + 'repo': repo, + 'signoffs_url': signoffs_url, + 'incomplete': incomplete, + 'complete': complete, + 'new': new, + 'new_hours': new_hours, + 'old': old, + 'old_days': old_days, + 'leaders': leaders, + }) + from_addr = 'Arch Website Notification ' + #send_mail(subject, t.render(c), from_addr, email) + print t.render(c) + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 9a5410ba4b622b68306de53abfa28b5a49e30107 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 17:33:00 -0500 Subject: Make signoff_report command send email Signed-off-by: Dan McGee --- packages/management/commands/signoff_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/management') diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py index 17e58f39..02f3d985 100644 --- a/packages/management/commands/signoff_report.py +++ b/packages/management/commands/signoff_report.py @@ -8,6 +8,7 @@ repository. Usage: ./manage.py signoff_report """ +from django.core.mail import send_mail from django.core.urlresolvers import reverse from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User @@ -104,7 +105,6 @@ def generate_report(email, repo_name): 'leaders': leaders, }) from_addr = 'Arch Website Notification ' - #send_mail(subject, t.render(c), from_addr, email) - print t.render(c) + send_mail(subject, t.render(c), from_addr, [email]) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 5f2c3bf98baabf919681525e600639643aa2c119 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 20:39:59 -0500 Subject: Signoffs changes and improvements * Better signoff report with more detail * Show signoff specification in signoffs view * Honor disabled/bad flags and display in approval column * Various other small bugfixes and tweaks Signed-off-by: Dan McGee --- packages/management/commands/signoff_report.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'packages/management') diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py index 02f3d985..3431dada 100644 --- a/packages/management/commands/signoff_report.py +++ b/packages/management/commands/signoff_report.py @@ -56,6 +56,8 @@ def generate_report(email, repo_name): # Collect all existing signoffs for these packages signoff_groups = sorted(get_signoff_groups([repo]), key=attrgetter('target_repo', 'arch', 'pkgbase')) + disabled = [] + bad = [] complete = [] incomplete = [] new = [] @@ -68,10 +70,16 @@ def generate_report(email, repo_name): old_cutoff = now - timedelta(days=old_days) for group in signoff_groups: - if group.approved(): + spec = group.specification + if spec.known_bad: + bad.append(group) + elif not spec.enabled: + disabled.append(group) + elif group.approved(): complete.append(group) else: incomplete.append(group) + if group.package.last_update > new_cutoff: new.append(group) if group.package.last_update < old_cutoff: @@ -96,6 +104,9 @@ def generate_report(email, repo_name): c = Context({ 'repo': repo, 'signoffs_url': signoffs_url, + 'disabled': disabled, + 'bad': bad, + 'all': signoff_groups, 'incomplete': incomplete, 'complete': complete, 'new': new, -- cgit v1.2.3-2-g168b From e565fde00f56c7a01ff55a204a0a56d3ce4bf8b4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 4 Nov 2011 11:31:35 -0500 Subject: Signoff email: prune empty content Don't send the email at all if there are no packages even in the repository, and don't print empty sections. Signed-off-by: Dan McGee --- packages/management/commands/signoff_report.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'packages/management') diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py index 3431dada..3357bc1e 100644 --- a/packages/management/commands/signoff_report.py +++ b/packages/management/commands/signoff_report.py @@ -69,6 +69,10 @@ def generate_report(email, repo_name): new_cutoff = now - timedelta(hours=new_hours) old_cutoff = now - timedelta(days=old_days) + if len(signoff_groups) == 0: + # no need to send an email at all + return + for group in signoff_groups: spec = group.specification if spec.known_bad: -- cgit v1.2.3-2-g168b From 12408702eaf89ea338670ba808da9ef49e35c562 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Nov 2011 12:19:17 -0600 Subject: Allow population of signoff specs with SVN commit messages This pulls them from the latest SVN commit on trunk. We don't have a failproof method of getting the exact right commit, but this should be close if it is run on a regular basis via cron (aka hourly). Note that running locally, I needed the development version of South to get the migration included here to apply because of information_schema changes in the current version of MySQL. Signed-off-by: Dan McGee --- packages/management/commands/populate_signoffs.py | 89 +++++++++++++++++++++++ packages/management/commands/signoff_report.py | 2 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 packages/management/commands/populate_signoffs.py (limited to 'packages/management') diff --git a/packages/management/commands/populate_signoffs.py b/packages/management/commands/populate_signoffs.py new file mode 100644 index 00000000..5b5acbaf --- /dev/null +++ b/packages/management/commands/populate_signoffs.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" +populate_signoffs command + +Pull the latest commit message from SVN for a given package that is +signoff-eligible and does not have an existing comment attached. + +Usage: ./manage.py populate_signoffs +""" + +from datetime import datetime +import logging +import subprocess +import sys +from xml.etree.ElementTree import XML + +from django.conf import settings +from django.contrib.auth.models import User +from django.core.management.base import NoArgsCommand + +from ...models import SignoffSpecification +from ...utils import get_signoff_groups + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(NoArgsCommand): + help = "Pull the latest commit message from SVN for a given package that is signoff-eligible and does not have an existing comment attached" + + def handle_noargs(self, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v == 2: + logger.level = logging.DEBUG + + return add_signoff_comments() + +def svn_log(pkgbase, repo): + path = '%s%s/%s/trunk/' % (settings.SVN_BASE_URL, repo.svn_root, pkgbase) + cmd = ['svn', 'log', '--limit=1', '--xml', path] + log_data = subprocess.check_output(cmd) + # the XML format is very very simple, especially with only one revision + xml = XML(log_data) + revision = int(xml.find('logentry').get('revision')) + date = datetime.strptime(xml.findtext('logentry/date'), + '%Y-%m-%dT%H:%M:%S.%fZ') + return { + 'revision': revision, + 'date': date, + 'author': xml.findtext('logentry/author'), + 'message': xml.findtext('logentry/msg'), + } + +def create_specification(package, log): + trimmed_message = log['message'].strip() + spec = SignoffSpecification(pkgbase=package.pkgbase, + pkgver=package.pkgver, pkgrel=package.pkgrel, + epoch=package.epoch, arch=package.arch, repo=package.repo, + comments=trimmed_message) + try: + spec.user = User.objects.get(username=log['author']) + except User.DoesNotExist: + pass + + return spec + +def add_signoff_comments(): + logger.info("getting all signoff groups") + groups = get_signoff_groups() + logger.info("%d signoff groups found", len(groups)) + + for group in groups: + if not group.default_spec: + continue + + logger.debug("getting SVN log for %s (%s)", group.pkgbase, group.repo) + log = svn_log(group.pkgbase, group.repo) + logger.info("creating spec with SVN message for %s", group.pkgbase) + spec = create_specification(group.packages[0], log) + spec.save() + +# vim: set ts=4 sw=4 et: diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py index 3357bc1e..3b67f518 100644 --- a/packages/management/commands/signoff_report.py +++ b/packages/management/commands/signoff_report.py @@ -22,7 +22,7 @@ import logging from operator import attrgetter import sys -from main.models import Package, Repo +from main.models import Repo from packages.models import Signoff from packages.utils import get_signoff_groups -- cgit v1.2.3-2-g168b From 0344f8ad564644c50203985255fab1d053aed463 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 15 Nov 2011 15:04:33 -0600 Subject: Add ability to cache users by username on the UserFinder This is very useful in the signoff message population script where we are very likely to encounter the same users over and over. Signed-off-by: Dan McGee --- packages/management/commands/populate_signoffs.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'packages/management') diff --git a/packages/management/commands/populate_signoffs.py b/packages/management/commands/populate_signoffs.py index 5b5acbaf..ce5ec734 100644 --- a/packages/management/commands/populate_signoffs.py +++ b/packages/management/commands/populate_signoffs.py @@ -20,6 +20,7 @@ from django.core.management.base import NoArgsCommand from ...models import SignoffSpecification from ...utils import get_signoff_groups +from devel.utils import UserFinder logging.basicConfig( level=logging.INFO, @@ -58,17 +59,13 @@ def svn_log(pkgbase, repo): 'message': xml.findtext('logentry/msg'), } -def create_specification(package, log): +def create_specification(package, log, finder): trimmed_message = log['message'].strip() spec = SignoffSpecification(pkgbase=package.pkgbase, pkgver=package.pkgver, pkgrel=package.pkgrel, epoch=package.epoch, arch=package.arch, repo=package.repo, comments=trimmed_message) - try: - spec.user = User.objects.get(username=log['author']) - except User.DoesNotExist: - pass - + spec.user = finder.find_by_username(log['author']) return spec def add_signoff_comments(): @@ -76,6 +73,8 @@ def add_signoff_comments(): groups = get_signoff_groups() logger.info("%d signoff groups found", len(groups)) + finder = UserFinder() + for group in groups: if not group.default_spec: continue @@ -83,7 +82,7 @@ def add_signoff_comments(): logger.debug("getting SVN log for %s (%s)", group.pkgbase, group.repo) log = svn_log(group.pkgbase, group.repo) logger.info("creating spec with SVN message for %s", group.pkgbase) - spec = create_specification(group.packages[0], log) + spec = create_specification(group.packages[0], log, finder) spec.save() # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b